deepline 0.1.283 → 0.1.285

package/dist/bundling-sources/sdk/src/release.ts

@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: '0.1.283',
+  version: '0.1.285',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -107,6 +107,7 @@ import {
   TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
   classifyToolExecuteHttpFailure,
   createToolExecuteHttpFailureAttemptTracker,
+  parseToolExecuteRetryAfterMs,
   parseToolExecuteAuthScopeChangedError,
   ToolExecuteAuthScopeChangedError,
 } from './tool-execute-retry-policy';
@@ -303,9 +304,10 @@ const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
 const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
 const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 100;
 const MAP_FRAME_FLUSH_INTERVAL_MS = 250;
-// Batchable calls wait at most this long for same-key row continuations. The
-// window is fixed from the first item and never resets on later arrivals.
+// Tool scheduling lanes wait at most this long for same-lane row continuations.
+// The window is fixed from the first item and never resets on later arrivals.
 const TOOL_BATCH_COALESCE_WINDOW_MS = 5;
+const TOOL_SCALAR_COALESCE_WINDOW_MS = 5;
 const TOOL_RETRY_AFTER_FALLBACK_MS = 1_000;
 const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
 const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
@@ -335,6 +337,31 @@ const MAX_INLINE_CHILD_INVOCATIONS_PER_ROW = 512;
 // Bound the retained failure detail carried in parent aggregates so a fully
 // failing large map cannot grow the progress event without limit.
 const MAX_INLINE_CHILD_FAILURE_DETAIL = 25;
+
+class ToolExecuteResponseBodyTransportError extends Error {
+  readonly cause: unknown;
+
+  constructor(cause: unknown) {
+    super(
+      cause instanceof Error
+        ? cause.message
+        : 'Tool execute response body transport failed.',
+    );
+    this.name = 'ToolExecuteResponseBodyTransportError';
+    this.cause = cause;
+  }
+}
+
+class ToolExecuteInvalidJsonError extends Error {
+  readonly cause: unknown;
+
+  constructor(cause: unknown) {
+    super('Tool execute response body was not valid JSON.');
+    this.name = 'ToolExecuteInvalidJsonError';
+    this.cause = cause;
+  }
+}
+
 // A single batched runtime-step-receipt request (get/claim/complete/fail/seed)
 // carries at most this many keys. At map scale a run can hold 5k-10k receipt
 // keys; sending them all in one request made the server claim/complete them
@@ -1211,11 +1238,11 @@ export class PlayContextImpl {
   private rowStates = new Map<number, RowState>();
   private toolCallQueue: ToolCallRequest[] = [];
   /**
-   * Fixed, non-resetting coalescing deadlines for batchable ready work. A
+   * Fixed, non-resetting coalescing deadlines for ready scheduling lanes. A
    * deadline starts when the first request enters an empty batch bucket. New
    * arrivals never push it back, so sustained traffic cannot starve dispatch.
    */
-  private readonly toolBatchQueuedAtByKey = new Map<string, number>();
+  private readonly toolDispatchQueuedAtByLane = new Map<string, number>();
   private readonly toolDispatcherWakeWaiters = new Set<() => void>();
   private toolDispatcherFailure: unknown | null = null;
   private toolCallResolvers = new Map<
@@ -1396,14 +1423,14 @@ export class PlayContextImpl {
     this.checkpoint = options.checkpoint ?? emptyCheckpoint();
     this.durableMappedToolResultsBackedByReceipts = Boolean(
       (options.claimRuntimeStepReceipt || options.claimRuntimeStepReceipts) &&
-        (options.getRuntimeStepReceipt || options.getRuntimeStepReceipts) &&
-        (options.completeRuntimeStepReceipt ||
-          options.completeRuntimeStepReceipts),
+      (options.getRuntimeStepReceipt || options.getRuntimeStepReceipts) &&
+      (options.completeRuntimeStepReceipt ||
+        options.completeRuntimeStepReceipts),
     );
     this.durableDirectToolResultsBackedByReceipts = Boolean(
       options.claimRuntimeStepReceipt &&
-        options.getRuntimeStepReceipt &&
-        options.completeRuntimeStepReceipt,
+      options.getRuntimeStepReceipt &&
+      options.completeRuntimeStepReceipt,
     );
     if (this.durableDirectToolResultsBackedByReceipts) {
       // A resumed durable runner may receive a legacy checkpoint containing
@@ -6078,15 +6105,41 @@ export class PlayContextImpl {
     }
   }
 
-  private toolBatchCoalesceKey(request: ToolCallRequest): string | null {
+  private toolDispatchLane(request: ToolCallRequest): {
+    key: string;
+    readyCount: number;
+    maxGroupSize: number;
+    coalesceWindowMs: number;
+  } {
     const strategy =
       this.#options.getBatchOperationStrategy?.(request.toolId) ?? null;
-    if (!strategy) return null;
-    return [
-      request.toolId,
-      request.executionAuthScopeDigest ?? '',
-      String(strategy.toBucketKey(request.input)),
-    ].join('\u0000');
+    if (strategy) {
+      return {
+        key: [
+          'batch',
+          request.toolId,
+          request.executionAuthScopeDigest ?? '',
+          String(strategy.toBucketKey(request.input)),
+        ].join('\u0000'),
+        readyCount: strategy.maxBatchSize,
+        maxGroupSize: this.governor.policy.concurrency.toolCalls,
+        coalesceWindowMs:
+          this.#options.toolBatchCoalesceWindowMs ??
+          TOOL_BATCH_COALESCE_WINDOW_MS,
+      };
+    }
+    return {
+      key: [
+        'scalar',
+        request.toolId,
+        request.executionAuthScopeDigest ?? '',
+      ].join('\u0000'),
+      readyCount: this.governor.policy.pacing.workerToolBatchDefaultParallelism,
+      maxGroupSize: this.governor.policy.concurrency.toolCalls,
+      coalesceWindowMs:
+        this.#options.toolScalarCoalesceWindowMs ??
+        TOOL_SCALAR_COALESCE_WINDOW_MS,
+    };
   }
 
   private wakeToolDispatcher(): void {
@@ -6109,10 +6162,10 @@ export class PlayContextImpl {
       return;
     }
     this.toolCallQueue.push(request);
-    const batchKey = this.toolBatchCoalesceKey(request);
-    if (batchKey && !this.toolBatchQueuedAtByKey.has(batchKey)) {
-      this.toolBatchQueuedAtByKey.set(
-        batchKey,
+    const lane = this.toolDispatchLane(request);
+    if (!this.toolDispatchQueuedAtByLane.has(lane.key)) {
+      this.toolDispatchQueuedAtByLane.set(
+        lane.key,
         this.toolBatchDispatcherNowMs(),
       );
     }
@@ -6122,7 +6175,7 @@ export class PlayContextImpl {
   private rejectQueuedToolCalls(error: unknown): void {
     const queued = this.toolCallQueue;
     this.toolCallQueue = [];
-    this.toolBatchQueuedAtByKey.clear();
+    this.toolDispatchQueuedAtByLane.clear();
     for (const request of queued) {
       const resolver = this.toolCallResolvers.get(request.callId);
       if (!resolver) continue;
@@ -6131,39 +6184,45 @@ export class PlayContextImpl {
     }
   }
 
-  private takeDispatchableToolCalls(nowMs: number): {
+  private takeDispatchableToolCalls(
+    nowMs: number,
+    blockedLaneKeys: ReadonlySet<string>,
+  ): {
     requests: ToolCallRequest[];
+    laneKeys: Set<string>;
     nextDeadlineMs: number | null;
   } {
     if (this.toolCallQueue.length === 0) {
-      return { requests: [], nextDeadlineMs: null };
+      return { requests: [], laneKeys: new Set(), nextDeadlineMs: null };
     }
 
-    const batchCounts = new Map<string, number>();
-    const batchMaxSizes = new Map<string, number>();
+    const laneCounts = new Map<string, number>();
+    const lanes = new Map<
+      string,
+      {
+        readyCount: number;
+        maxGroupSize: number;
+        coalesceWindowMs: number;
+      }
+    >();
     for (const request of this.toolCallQueue) {
-      const batchKey = this.toolBatchCoalesceKey(request);
-      if (!batchKey) continue;
-      batchCounts.set(batchKey, (batchCounts.get(batchKey) ?? 0) + 1);
-      const strategy =
-        this.#options.getBatchOperationStrategy?.(request.toolId) ?? null;
-      if (strategy) batchMaxSizes.set(batchKey, strategy.maxBatchSize);
+      const lane = this.toolDispatchLane(request);
+      laneCounts.set(lane.key, (laneCounts.get(lane.key) ?? 0) + 1);
+      lanes.set(lane.key, lane);
     }
 
-    const readyBatchKeys = new Set<string>();
+    const readyLaneKeys = new Set<string>();
     let nextDeadlineMs: number | null = null;
-    for (const [batchKey, count] of batchCounts) {
-      const queuedAt = this.toolBatchQueuedAtByKey.get(batchKey) ?? nowMs;
-      if (!this.toolBatchQueuedAtByKey.has(batchKey)) {
-        this.toolBatchQueuedAtByKey.set(batchKey, queuedAt);
+    for (const [laneKey, count] of laneCounts) {
+      if (blockedLaneKeys.has(laneKey)) continue;
+      const queuedAt = this.toolDispatchQueuedAtByLane.get(laneKey) ?? nowMs;
+      if (!this.toolDispatchQueuedAtByLane.has(laneKey)) {
+        this.toolDispatchQueuedAtByLane.set(laneKey, queuedAt);
       }
-      const coalesceWindowMs =
-        this.#options.toolBatchCoalesceWindowMs ??
-        TOOL_BATCH_COALESCE_WINDOW_MS;
-      const deadline = queuedAt + coalesceWindowMs;
-      const maxBatchSize = batchMaxSizes.get(batchKey) ?? 1;
-      if (count >= maxBatchSize || deadline <= nowMs) {
-        readyBatchKeys.add(batchKey);
+      const lane = lanes.get(laneKey)!;
+      const deadline = queuedAt + lane.coalesceWindowMs;
+      if (count >= lane.readyCount || deadline <= nowMs) {
+        readyLaneKeys.add(laneKey);
       } else {
         nextDeadlineMs =
           nextDeadlineMs == null
@@ -6174,19 +6233,27 @@ export class PlayContextImpl {
 
     const requests: ToolCallRequest[] = [];
     const remaining: ToolCallRequest[] = [];
+    const selectedByLane = new Map<string, number>();
     for (const request of this.toolCallQueue) {
-      const batchKey = this.toolBatchCoalesceKey(request);
-      if (!batchKey || readyBatchKeys.has(batchKey)) {
+      const lane = this.toolDispatchLane(request);
+      const selectedCount = selectedByLane.get(lane.key) ?? 0;
+      if (
+        readyLaneKeys.has(lane.key) &&
+        requests.length < this.governor.policy.concurrency.toolCalls &&
+        selectedCount < lane.maxGroupSize
+      ) {
         requests.push(request);
+        selectedByLane.set(lane.key, selectedCount + 1);
       } else {
         remaining.push(request);
       }
     }
     this.toolCallQueue = remaining;
-    for (const batchKey of readyBatchKeys) {
-      this.toolBatchQueuedAtByKey.delete(batchKey);
+    const selectedLaneKeys = new Set(selectedByLane.keys());
+    for (const laneKey of selectedLaneKeys) {
+      this.toolDispatchQueuedAtByLane.delete(laneKey);
     }
-    return { requests, nextDeadlineMs };
+    return { requests, laneKeys: selectedLaneKeys, nextDeadlineMs };
   }
 
   private waitForToolDispatcherWake(deadlineMs: number | null): Promise<void> {
@@ -6216,10 +6283,25 @@ export class PlayContextImpl {
   }
 
   private async drainQueuedWork<T>(promises: Promise<T>[]): Promise<void> {
-    // One dispatcher owns ready work, fixed batch coalescing deadlines, and
-    // the in-flight registry. A row resumes after its own receipt persists; it
-    // never waits for unrelated sibling calls from the previous column.
+    // One dispatcher owns ready work, fixed lane coalescing deadlines, and the
+    // bounded in-flight registry. A row resumes after its own receipt persists;
+    // it never waits for unrelated sibling calls from the previous column.
     const inFlightToolExecutions = new Set<Promise<void>>();
+    const activeGroupsByLane = new Map<string, number>();
+    const maxInFlightGroups = Math.max(
+      1,
+      Math.floor(
+        this.#options.toolDispatcherMaxInFlightGroups ??
+          this.governor.policy.concurrency.toolDispatchGroups,
+      ),
+    );
+    const maxInFlightGroupsPerLane = Math.max(
+      1,
+      Math.floor(
+        this.#options.toolDispatcherMaxInFlightGroupsPerLane ??
+          this.governor.policy.concurrency.toolDispatchGroupsPerLane,
+      ),
+    );
     let rowsSettled = false;
     void Promise.allSettled(promises).then(() => {
       rowsSettled = true;
@@ -6238,9 +6320,22 @@ export class PlayContextImpl {
           throw this.toolDispatcherFailure;
         }
 
-        const dispatchable = this.takeDispatchableToolCalls(
-          this.toolBatchDispatcherNowMs(),
+        const blockedLaneKeys = new Set(
+          [...activeGroupsByLane]
+            .filter(([, count]) => count >= maxInFlightGroupsPerLane)
+            .map(([laneKey]) => laneKey),
         );
+        const dispatchable =
+          inFlightToolExecutions.size >= maxInFlightGroups
+            ? {
+                requests: [] as ToolCallRequest[],
+                laneKeys: new Set<string>(),
+                nextDeadlineMs: null,
+              }
+            : this.takeDispatchableToolCalls(
+                this.toolBatchDispatcherNowMs(),
+                blockedLaneKeys,
+              );
         if (dispatchable.requests.length > 0) {
           pass += 1;
           this.log(`  Batch pass ${pass}`);
@@ -6249,12 +6344,27 @@ export class PlayContextImpl {
               `queued=${this.toolCallQueue.length} ` +
               `in_flight_groups=${inFlightToolExecutions.size}`,
           );
+          for (const laneKey of dispatchable.laneKeys) {
+            activeGroupsByLane.set(
+              laneKey,
+              (activeGroupsByLane.get(laneKey) ?? 0) + 1,
+            );
+          }
           const tracked = this.executeBatchedToolCalls(dispatchable.requests)
             .catch((error) => {
               this.toolDispatcherFailure ??= error;
               this.rejectQueuedToolCalls(error);
             })
             .finally(() => {
+              for (const laneKey of dispatchable.laneKeys) {
+                const remainingGroups =
+                  (activeGroupsByLane.get(laneKey) ?? 1) - 1;
+                if (remainingGroups > 0) {
+                  activeGroupsByLane.set(laneKey, remainingGroups);
+                } else {
+                  activeGroupsByLane.delete(laneKey);
+                }
+              }
               inFlightToolExecutions.delete(tracked);
               this.wakeToolDispatcher();
             });
@@ -8547,7 +8657,9 @@ export class PlayContextImpl {
           };
 
           while (true) {
-            let response: Response;
+            let response: Response | null = null;
+            let responseData: Record<string, unknown> | null = null;
+            let responseErrorText: string | null = null;
             let providerCallStartedAt: number | null = null;
             let providerCallElapsedMs: number | null = null;
             const durableCallReceiptKey =
@@ -8700,12 +8812,31 @@ export class PlayContextImpl {
                         : {}),
                     }),
                   });
+                  if (response.ok) {
+                    try {
+                      responseData = (await response.json()) as Record<
+                        string,
+                        unknown
+                      >;
+                    } catch (error) {
+                      if (error instanceof SyntaxError) {
+                        throw new ToolExecuteInvalidJsonError(error);
+                      }
+                      throw new ToolExecuteResponseBodyTransportError(error);
+                    }
+                  } else {
+                    try {
+                      responseErrorText = await response.text();
+                    } catch (error) {
+                      throw new ToolExecuteResponseBodyTransportError(error);
+                    }
+                  }
                 } finally {
                   providerPermit.release();
                 }
                 if (runtimeReceiptReadTraceEnabled) {
                   this.log(
-                    `[perf] tool call id=${toolId} phase=integration_fetch_headers elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`,
+                    `[perf] tool call id=${toolId} phase=integration_fetch_body elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`,
                   );
                 }
                 providerCallElapsedMs = Date.now() - providerCallStartedAt;
@@ -8723,13 +8854,80 @@ export class PlayContextImpl {
               if (heartbeatFailure) {
                 throw heartbeatFailure;
               }
+              if (error instanceof ToolExecuteInvalidJsonError) {
+                throw new ToolHttpError(
+                  `Tool ${toolId} returned an invalid JSON response body.`,
+                  null,
+                  response?.status ?? 0,
+                  'repairable',
+                );
+              }
               const transportError = abortController?.signal.aborted
                 ? abortController.signal.reason instanceof Error
                   ? abortController.signal.reason
                   : new Error(
                       `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
                     )
-                : error;
+                : error instanceof ToolExecuteResponseBodyTransportError
+                  ? error.cause
+                  : error;
+              if (
+                error instanceof ToolExecuteResponseBodyTransportError &&
+                response?.status === 402
+              ) {
+                const diagnostic = describeTransportError(transportError);
+                throw new ToolHttpError(
+                  `Tool ${toolId} returned HTTP 402 but its response body could not be read; the run was halted because the payment or capacity denial could not be classified safely: ${diagnostic.message ?? 'unknown transport error'}`,
+                  {
+                    kind: 'billing_cap_exceeded',
+                    code: 'HTTP_402_BODY_UNREADABLE',
+                    error_category: 'billing',
+                    failure_origin: 'unknown',
+                  },
+                  402,
+                );
+              }
+              const responseBodyReplaySafe =
+                !(error instanceof ToolExecuteResponseBodyTransportError) ||
+                retrySafeTransientHttp ||
+                response?.status === 429;
+              if (!responseBodyReplaySafe) {
+                const diagnostic = describeTransportError(transportError);
+                this.log(
+                  `[runtime.transport_failure] ${JSON.stringify({
+                    tool_id: toolId,
+                    gateway_origin: transportGatewayOriginForDiagnostic(url),
+                    attempt: 1,
+                    max_attempts: 1,
+                    elapsed_ms:
+                      providerCallStartedAt === null
+                        ? 0
+                        : Date.now() - providerCallStartedAt,
+                    request_id: deeplineRequestId,
+                    aborted: abortController?.signal.aborted === true,
+                    response_headers_received: true,
+                    retry_safe: false,
+                    error: diagnostic,
+                  })}`,
+                );
+                throw new ToolHttpError(
+                  `Tool ${toolId} response body transport failed after response headers; the ambiguous call was not retried because this operation is not declared retry-safe: ${diagnostic.message ?? 'unknown transport error'}`,
+                  null,
+                  0,
+                  'repairable',
+                );
+              }
+              if (
+                error instanceof ToolExecuteResponseBodyTransportError &&
+                response?.status === 429
+              ) {
+                await this.reportToolBackpressure(
+                  toolId,
+                  parseToolExecuteRetryAfterMs(
+                    response.headers.get('retry-after'),
+                  ),
+                );
+              }
               await retryToolTransportFailure({
                 error: transportError,
                 elapsedMs:
@@ -8746,10 +8944,15 @@ export class PlayContextImpl {
               }
             }
 
+            if (!response) {
+              throw new Error(
+                `Tool ${toolId} transport completed without an HTTP response.`,
+              );
+            }
             span.setAttribute('plays.http_status_code', response.status);
 
             if (!response.ok) {
-              const text = await response.text();
+              const text = responseErrorText ?? '';
               if (
                 isDeeplineDeveloperTunnelOrigin502({
                   url,
@@ -8864,8 +9067,15 @@ export class PlayContextImpl {
               throw failure.error;
             }
 
-            const data = (await response.json()) as Record<string, unknown>;
-            const parsed = parseToolExecuteResponse(toolId, data);
+            if (!responseData) {
+              throw new ToolHttpError(
+                `Tool ${toolId} returned an empty successful response body.`,
+                null,
+                response.status,
+                'repairable',
+              );
+            }
+            const parsed = parseToolExecuteResponse(toolId, responseData);
             setSpanAttributes(span, {
               'plays.tool_result_kind':
                 parsed.result == null

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -690,11 +690,16 @@ export interface ContextOptions {
     nowMs: () => number;
     schedule: (delayMs: number, wake: () => void) => () => void;
   };
-  // How long batchable calls wait for same-key row continuations before
-  // dispatching, measured from the first queued item. Defaults to 5ms in
-  // production; tests override it to a large value to make coalescing
-  // deterministic instead of racing wall-clock milliseconds on slow CI.
+  // How long provider-native batch calls wait for same-key row continuations
+  // before dispatching, measured from the first queued item.
   toolBatchCoalesceWindowMs?: number;
+  // Scalar calls remain separate physical requests, but wait in one scheduling
+  // lane for this fixed window so pipelined replacement rows launch together.
+  toolScalarCoalesceWindowMs?: number;
+  /** Deterministic test override for the runner-local scheduling-group cap. */
+  toolDispatcherMaxInFlightGroups?: number;
+  /** Deterministic test override for the active-group cap per scheduling lane. */
+  toolDispatcherMaxInFlightGroupsPerLane?: number;
   queryCustomerDb?: CustomerDbQueryHandler;
   executeStructuredPlayDefinition?: (input: {
     definition: PlayStructuredDefinition;

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -268,6 +268,7 @@ export function createPlayExecutionGovernor(
   const budgetState = input.budgetState ?? new InMemoryBudgetStateBackend();
   const providerByTool = new Map<string, string>();
   const scopeByTool = new Map<string, [string, string]>();
+  const providerConcurrencySlotsByTool = new Map<string, Semaphore>();
 
   // When the rate-state backend owns pacing authoritatively (the Absurd/Node
   // app-runtime Postgres pacer runs the whole token bucket + AIMD in the row),
@@ -405,12 +406,38 @@ export function createPlayExecutionGovernor(
       // the declared rolling window under concurrent runners.
       const pacing = await resolveAdaptivePacing(toolId);
       const rateScope = await resolveScope(toolId, pacing.provider);
-      return await input.rateState.acquire({
-        bucketId: rateScope[0],
-        rateScopeToken: rateScope[1],
-        rules: pacing.rules,
-        signal: opts?.signal,
-      });
+      const declaredMaxConcurrency = Math.min(
+        ...pacing.rules.flatMap((rule) =>
+          rule.maxConcurrency != null ? [rule.maxConcurrency] : [],
+        ),
+      );
+      const localConcurrencySlot = Number.isFinite(declaredMaxConcurrency)
+        ? await (() => {
+            let slots = providerConcurrencySlotsByTool.get(toolId);
+            if (!slots) {
+              slots = new Semaphore(Math.max(1, declaredMaxConcurrency));
+              providerConcurrencySlotsByTool.set(toolId, slots);
+            }
+            return slots.acquire(opts?.signal);
+          })()
+        : null;
+      try {
+        const rateLease = await input.rateState.acquire({
+          bucketId: rateScope[0],
+          rateScopeToken: rateScope[1],
+          rules: pacing.rules,
+          signal: opts?.signal,
+        });
+        return {
+          release() {
+            rateLease.release();
+            localConcurrencySlot?.release();
+          },
+        };
+      } catch (error) {
+        localConcurrencySlot?.release();
+        throw error;
+      }
     },
 
     async suggestedParallelism(toolId, fallback) {

package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts

@@ -28,6 +28,10 @@ export interface ExecutionConcurrencyPolicy {
   readonly rowMax: number;
   /** Global backstop on concurrently in-flight tool calls across all providers. */
   readonly toolCalls: number;
+  /** Runner-local cap on concurrently active tool scheduling groups. */
+  readonly toolDispatchGroups: number;
+  /** Runner-local cap on active scheduling groups for one scalar/batch lane. */
+  readonly toolDispatchGroupsPerLane: number;
 }
 
 /**
@@ -88,6 +92,14 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
     // Global all-provider backstop. Per-provider pacing is the real limit; this
     // just stops a single run from opening an absurd number of sockets at once.
     toolCalls: 256,
+    // Scheduling groups contain bounded-dispatch workers and receipt buffers.
+    // Keep their promise topology finite even when pipelined replacement rows
+    // become ready one at a time behind a slower provider pacer.
+    toolDispatchGroups: 32,
+    // Preserve limited overlap so a fast row can enter its next column while a
+    // prior group's receipt persistence is still settling. Provider permits
+    // enforce declared maxConcurrency across these overlapping groups.
+    toolDispatchGroupsPerLane: 4,
   },
   budgets: {
     // Runaway guards, not workload limits. A 5,000-row map calling several tools

package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts

@@ -1,4 +1,4 @@
-import { isHardBillingToolHttpError } from './tool-http-errors';
+import { isHardBillingToolHttpError, ToolHttpError } from './tool-http-errors';
 import { isRuntimePersistenceCircuitOpenError } from './persistence-latch';
 import { isWorkspaceStorageNotReadyFailure } from './run-failure';
 
@@ -17,10 +17,7 @@ export function isAbortLikeError(error: unknown): boolean {
   if (!error) return false;
   if (error instanceof WorkflowAbortError) return true;
   if (error instanceof Error) {
-    if (error.name === 'WorkflowAbort' || error.name === 'AbortError') {
-      return true;
-    }
-    return /\b(cancell?ed|aborted|terminate[d]?)\b/i.test(error.message);
+    return error.name === 'WorkflowAbort' || error.name === 'AbortError';
   }
   return false;
 }
@@ -36,6 +33,23 @@ export function isAbortLikeError(error: unknown): boolean {
  */
 export function isRowIsolationExemptError(error: unknown): boolean {
   if (isAbortLikeError(error)) return true;
+  // A remote tool can return arbitrary provider/database wording. Its message
+  // must never be mistaken for a failure of the runner's own receipt or sheet
+  // persistence plane. Hard Deepline billing denials are the sole typed tool
+  // failure that intentionally remains run-fatal.
+  if (error instanceof ToolHttpError) {
+    return isHardBillingToolHttpError(error);
+  }
+  const nestedErrors =
+    error && typeof error === 'object'
+      ? (error as { errors?: unknown }).errors
+      : undefined;
+  if (
+    Array.isArray(nestedErrors) &&
+    nestedErrors.some(isRowIsolationExemptError)
+  ) {
+    return true;
+  }
   if (error instanceof Error && error.name === 'GovernorBudgetError') {
     return true;
   }
@@ -63,12 +77,13 @@ function isRuntimeReceiptPersistenceError(error: unknown): boolean {
 
 function isRuntimeStoragePersistenceError(error: unknown): boolean {
   if (!error || typeof error !== 'object') return false;
+  if (error instanceof ToolHttpError) return false;
   const nestedErrors = (error as { errors?: unknown }).errors;
-  if (
-    Array.isArray(nestedErrors) &&
-    nestedErrors.some(isRuntimeStoragePersistenceError)
-  ) {
-    return true;
+  if (Array.isArray(nestedErrors) && nestedErrors.length > 0) {
+    // AggregateError.message is caller-controlled summary text. Once typed
+    // children exist, classify those children and never fall through to a
+    // misleading container message.
+    return nestedErrors.some(isRuntimeStoragePersistenceError);
   }
   const message =
     error instanceof Error ? error.message : String((error as object) ?? '');

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -107,6 +107,10 @@ export type PlaySchedulerSubmitInput = {
   userEmail: string;
   userId?: string | null;
   source?: 'published' | 'ad_hoc' | 'draft';
+  /** Invocation origin, distinct from the artifact source above. */
+  triggerSource?: 'api' | 'webhook' | 'cron' | 'sql_listener';
+  /** Null lets a durable trigger wait for capacity; a number bounds queue delay. */
+  queueMaxDelaySeconds?: number | null;
   executionProfile?: string | null;
   /** runner backend to use for executing attempts */
   runtimeBackend: string;

package/dist/cli/index.js

@@ -718,7 +718,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.283",
+  version: "0.1.285",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/cli/index.mjs

@@ -703,7 +703,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.283",
+  version: "0.1.285",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.js

@@ -438,7 +438,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.283",
+  version: "0.1.285",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.mjs

@@ -367,7 +367,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.283",
+  version: "0.1.285",
   contracts: {
     api: {
       name: "sdk-http-api",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.283",
+  "version": "0.1.285",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {