deepline 0.1.201 → 0.1.203

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -53,6 +53,7 @@ import {
   CoordinatorRateStateBackend,
   type CoordinatorRatePort,
 } from '../../../shared_libs/play-runtime/governor/coordinator-rate-state-backend';
+import { BlockReservingBudgetStateBackend } from '../../../shared_libs/play-runtime/governor/block-reserving-budget-state-backend';
 import type { PacingRule } from '../../../shared_libs/play-runtime/governor/rate-state-backend';
 import {
   pacingPolicyForTool,
@@ -250,6 +251,7 @@ import { runtimeSheetSessionScope } from '../../../shared_libs/play-runtime/runt
 import {
   chooseWorkerMapDispatchPlan,
   estimateBatchedToolChunkSubrequests,
+  workflowSafeMapRowsPerChunk,
 } from './runtime/map-chunk-plan';
 import {
   applyCsvRenameProjection,
@@ -337,6 +339,8 @@ type RunRequest = {
   forceToolRefresh?: boolean;
   /** Monotonic coordinator redispatch epoch for receipt and sheet fencing. */
   runAttempt?: number;
+  /** Tenant/worktree scheduler schema for shared durable governor state. */
+  runtimeSchedulerSchema?: string | null;
   /** Optional inline CSV rows (for plays where ctx.csv was passed inline data). */
   inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
   /** Staged input files keyed by logical filename (used by ctx.csv). */
@@ -1823,7 +1827,7 @@ async function executeTool(
   },
   workflowStep?: WorkflowStep,
   onProviderBackpressure?: (retryAfterMs: number) => void,
-  onRetryAttempt?: () => void,
+  onRetryAttempt?: () => void | Promise<void>,
   transientHttpRetrySafe = false,
   abortSignal?: AbortSignal,
   runtimeDeadlineMs?: number,
@@ -1869,7 +1873,7 @@ async function executeToolWithLifecycle(
   workflowStep: WorkflowStep | undefined,
   callbacks: WorkerCtxCallbacks | undefined,
   onProviderBackpressure?: (retryAfterMs: number) => void,
-  onRetryAttempt?: () => void,
+  onRetryAttempt?: () => void | Promise<void>,
   transientHttpRetrySafe = false,
   abortSignal?: AbortSignal,
   runtimeDeadlineMs?: number,
@@ -2169,7 +2173,7 @@ async function callToolDirect(
       ) {
         throw lastError;
       }
-      onRetryAttempt?.();
+      await onRetryAttempt?.();
       const delayMs = TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS * transportAttempt;
       console.warn(
         `[deepline-run:${req.runId}] tool transport retry tool=${toolId} path=${path} attempt=${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} retryAfterMs=${delayMs} error=${redactSecretsFromLogString(message)}`,
@@ -2224,7 +2228,7 @@ async function callToolDirect(
     // Charge the retry budget per attempt, matching the cjs runner's
     // chargeBudget('retry') on every 429 / retryable-5xx retry.
     if (failure.chargeRetryBudget) {
-      onRetryAttempt?.();
+      await onRetryAttempt?.();
     }
     await sleepWorkerMs(failure.retryDelayMs);
   }
@@ -2954,7 +2958,7 @@ async function postRuntimeEgressFetch(
   req: RunRequest,
   payload: RuntimeEgressFetchPayload,
   onProviderBackpressure?: (retryAfterMs: number) => void,
-  onRetryAttempt?: () => void,
+  onRetryAttempt?: () => void | Promise<void>,
 ): Promise<WorkerFetchResponse> {
   let lastError: Error | null = null;
   for (
@@ -3008,7 +3012,7 @@ async function postRuntimeEgressFetch(
     if (attempt >= TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS) {
       throw lastError;
     }
-    onRetryAttempt?.();
+    await onRetryAttempt?.();
     await sleepWorkerMs(
       Math.min(5_000, Math.max(retryAfterMs, attempt * 1000)),
     );
@@ -4309,6 +4313,16 @@ function createGovernorForRun(req: RunRequest): {
       rootRunId: req.playCallGovernance?.rootRunId ?? req.runId,
     },
     rateState: new CoordinatorRateStateBackend(createCoordinatorRatePort(req)),
+    budgetState: new BlockReservingBudgetStateBackend(
+      async (input) =>
+        await postRuntimeApi(req.baseUrl, req.executorToken, {
+          action: 'governor_budget_charge',
+          rootRunId: input.rootRunId,
+          reservationId: input.reservationId,
+          charges: input.charges,
+          schedulerSchema: req.runtimeSchedulerSchema ?? null,
+        }),
+    ),
     resolvePacing,
     resume: resumeGovernanceFromRequest(req),
   });
@@ -4415,7 +4429,7 @@ function createMinimalWorkerCtx(
     workflowStep?: unknown;
     callbacks?: WorkerCtxCallbacks;
     onProviderBackpressure?: (retryAfterMs: number) => void;
-    onRetryAttempt?: () => void;
+    onRetryAttempt?: () => void | Promise<void>;
     transientHttpRetrySafe?: boolean;
     durableCallReceiptKey?: string | null;
     executionAuthScopeDigest?: string | null;
@@ -4604,7 +4618,13 @@ function createMinimalWorkerCtx(
       mapDispatchPlan.workEstimate.providerToolCallsPerRow > 0 ||
       mapDispatchPlan.workEstimate.childPlaySubmitsPerRow > 0 ||
       mapDispatchPlan.workEstimate.eventWaitsPerRow > 0;
-    const rowsPerChunk = mapDispatchPlan.rowsPerChunk;
+    const plannedRowsPerChunk = mapDispatchPlan.rowsPerChunk;
+    const rowsPerChunk = workflowSafeMapRowsPerChunk({
+      plannedRowsPerChunk,
+      rowAdmissionWindow: governor.policy.concurrency.rowDefault,
+      hasWorkflowStep: Boolean(workflowStep),
+      hasExternalSideEffects: mapHasExternalSideEffects,
+    });
     recordRunnerPerfTrace({
       req,
       phase: 'runner.map_dispatch_plan',
@@ -4612,6 +4632,7 @@ function createMinimalWorkerCtx(
       extra: {
         mapName: name,
         rowsPerChunk,
+        plannedRowsPerChunk,
         rowCountHint,
         estimatedExternalStepsPerRow:
           mapDispatchPlan.estimatedExternalStepsPerRow,
@@ -6949,7 +6970,7 @@ function createMinimalWorkerCtx(
           // across isolates. Charged inside the receipt boundary so a replay
           // (cache hit) never double-charges.
           const childRunId = `${req.runId}:child:${normalizedKey}`;
-          const childGovernance = governor.forkChild({
+          const childGovernance = await governor.forkChild({
             childPlayName: resolvedName,
             childRunId,
           });

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts

@@ -146,6 +146,31 @@ export type WorkerMapWorkEstimate = {
   eventWaitsPerRow: number;
 };
 
+/**
+ * Keep one externally-effectful Workflow chunk inside the same bounded row
+ * window used by runtime admission. The static planner may choose a larger
+ * chunk when provider calls coalesce, but a chunk is still the smallest point
+ * where the dispatcher can cross a durable sleep boundary. Capping only the
+ * native Workflow execution chunk prevents a slow 5k provider chunk from
+ * outliving one invocation and being retried before the dispatcher can yield.
+ */
+export function workflowSafeMapRowsPerChunk(input: {
+  plannedRowsPerChunk: number;
+  rowAdmissionWindow: number;
+  hasWorkflowStep: boolean;
+  hasExternalSideEffects: boolean;
+}): number {
+  const plannedRowsPerChunk = Math.max(
+    1,
+    Math.floor(input.plannedRowsPerChunk),
+  );
+  if (!input.hasWorkflowStep || !input.hasExternalSideEffects) {
+    return plannedRowsPerChunk;
+  }
+  const rowAdmissionWindow = Math.max(1, Math.floor(input.rowAdmissionWindow));
+  return Math.min(plannedRowsPerChunk, rowAdmissionWindow);
+}
+
 function fieldRoot(field: string | null | undefined): string {
   return String(field ?? '').split('.', 1)[0] ?? '';
 }

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts

@@ -3,6 +3,7 @@ import {
   executeChunkedRequests,
   type ChunkExecutionResult,
 } from '../../../../shared_libs/play-runtime/batch-runtime';
+import { dispatchBounded } from '../../../../shared_libs/play-runtime/bounded-dispatch';
 import type { AnyBatchOperationStrategy } from '../../../../shared_libs/play-runtime/batching-types';
 import {
   buildDurableToolCallAuthScopeDigest,
@@ -110,7 +111,7 @@ export type WorkerToolDispatchExecuteTool = (input: {
   workflowStep?: WorkflowStepLike;
   callbacks?: WorkerToolDispatchCallbacks;
   onProviderBackpressure?: (retryAfterMs: number) => void;
-  onRetryAttempt?: () => void;
+  onRetryAttempt?: () => void | Promise<void>;
   transientHttpRetrySafe?: boolean;
   durableCallReceiptKey?: string | null;
   executionAuthScopeDigest?: string | null;
@@ -2026,32 +2027,137 @@ export class WorkerToolBatchScheduler {
       claimedRequests.length < 2
     ) {
       const groupStartedAt = this.options.nowMs();
-      await Promise.all(
-        claimedRequests.map(async (claimed) => {
-          const { request } = claimed;
-          const toolContract = await this.options
-            .resolvePacing(toolId)
-            .catch(() => null);
-          const circuitOpenError = this.runtimePersistenceCircuitOpenError(1);
-          if (circuitOpenError) {
-            this.rejectRequests(
-              claimed,
-              await this.failureForRejectedToolRequest(
-                claimed,
-                circuitOpenError,
+      // Seed the dispatch width from the governor's provider-shaped parallelism
+      // instead of the flat policy.concurrency.toolCalls ceiling. A flat width
+      // launched every claimed row at once into the pacer, so unhinted providers
+      // 429'd on the first burst before AIMD could halve the rate. The shaped
+      // estimate is derived from the provider's RPS/maxConcurrency pacing rules
+      // (see suggestedParallelism in governor.ts), floored at 1 and capped by the
+      // global tool-call concurrency ceiling. Per-call pacing still gates each
+      // in-flight request, so this only trims the launch burst.
+      const toolCallConcurrencyCeiling =
+        this.options.governor.policy.concurrency.toolCalls;
+      const shapedToolParallelism =
+        await this.options.governor.suggestedParallelism(
+          toolId,
+          toolCallConcurrencyCeiling,
+        );
+      const dispatchWidth = Math.min(
+        toolCallConcurrencyCeiling,
+        Math.max(1, shapedToolParallelism),
+      );
+      await dispatchBounded(claimedRequests, dispatchWidth, async (claimed) => {
+        const { request } = claimed;
+        const toolContract = await this.options
+          .resolvePacing(toolId)
+          .catch(() => null);
+        const circuitOpenError = this.runtimePersistenceCircuitOpenError(1);
+        if (circuitOpenError) {
+          this.rejectRequests(
+            claimed,
+            await this.failureForRejectedToolRequest(claimed, circuitOpenError),
+          );
+          return;
+        }
+        // Each unbatched provider call takes its own tool slot: the Governor
+        // charges tool budget, holds a global tool-concurrency slot, and
+        // applies per-(org,provider) pacing before the call runs.
+        let slot: { release: () => void };
+        try {
+          slot = await this.options.governor.acquireToolSlot(toolId, {
+            signal: this.options.abortSignal,
+          });
+        } catch (error) {
+          this.rejectRequests(
+            claimed,
+            await this.failureForRejectedToolRequest(claimed, error),
+          );
+          return;
+        }
+        try {
+          let result: unknown;
+          try {
+            await this.markAdmittedToolRequestsRunning([claimed]);
+            this.options.budgetMeter?.count('provider');
+            this.options.budgetMeter?.count(
+              'egress',
+              Math.max(
+                0,
+                WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL - 1,
               ),
             );
-            return;
-          }
-          // Each unbatched provider call takes its own tool slot: the Governor
-          // charges tool budget, holds a global tool-concurrency slot, and
-          // applies per-(org,provider) pacing before the call runs.
-          let slot: { release: () => void };
-          try {
-            slot = await this.options.governor.acquireToolSlot(toolId, {
-              signal: this.options.abortSignal,
+            const durableReceiptKey = claimed.receiptKey
+              ? durableCallReceiptKeyForClaimed({
+                  req: this.options.req,
+                  claimed,
+                })
+              : null;
+            result = await executeWithWorkerReceiptHeartbeats({
+              playName: this.options.req.playName,
+              runId: this.options.req.runId,
+              runAttempt: this.options.req.runAttempt ?? 0,
+              ...(this.options.receiptStore
+                ? { receiptStore: this.options.receiptStore }
+                : {}),
+              budgetMeter: this.options.budgetMeter,
+              heartbeatIntervalMs:
+                this.options.runtimeReceiptHeartbeatIntervalMs,
+              targets: claimed.receiptKey
+                ? [
+                    {
+                      key: claimed.receiptKey,
+                      leaseId: claimed.receiptLeaseId,
+                    },
+                  ]
+                : [],
+              execute: () =>
+                this.options.executeTool({
+                  id: request.id,
+                  toolId,
+                  input: request.input,
+                  workflowStep: request.workflowStep,
+                  callbacks: this.options.callbacks,
+                  onProviderBackpressure: (retryAfterMs) =>
+                    this.reportBackpressure(toolId, retryAfterMs),
+                  onRetryAttempt: () =>
+                    this.options.governor.chargeBudget('retry'),
+                  transientHttpRetrySafe:
+                    toolContract?.retrySafeTransientHttp === true,
+                  ...(durableReceiptKey
+                    ? {
+                        durableCallReceiptKey: durableReceiptKey,
+                        executionAuthScopeDigest:
+                          request.executionAuthScopeDigest,
+                        providerIdempotencyKey:
+                          providerIdempotencyKeyForClaimed({
+                            req: this.options.req,
+                            claimed,
+                          }),
+                      }
+                    : {}),
+                  runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
+                    [claimed],
+                    {
+                      resolveOwnerTimeoutMs: (request) =>
+                        request.runtimeTimeoutMs,
+                    },
+                  ),
+                  directOptions: request.directOptions,
+                }),
             });
           } catch (error) {
+            if (error instanceof RuntimeLeaseLostError) {
+              this.rejectRequests(claimed, error);
+              return;
+            }
+            if (
+              await this.requeueAfterAuthScopeChanged(
+                [request, ...claimed.followers],
+                error,
+              )
+            ) {
+              return;
+            }
             this.rejectRequests(
               claimed,
               await this.failureForRejectedToolRequest(claimed, error),
@@ -2059,108 +2165,17 @@ export class WorkerToolBatchScheduler {
             return;
           }
           try {
-            let result: unknown;
-            try {
-              await this.markAdmittedToolRequestsRunning([claimed]);
-              this.options.budgetMeter?.count('provider');
-              this.options.budgetMeter?.count(
-                'egress',
-                Math.max(
-                  0,
-                  WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL - 1,
-                ),
-              );
-              const durableReceiptKey = claimed.receiptKey
-                ? durableCallReceiptKeyForClaimed({
-                    req: this.options.req,
-                    claimed,
-                  })
-                : null;
-              result = await executeWithWorkerReceiptHeartbeats({
-                playName: this.options.req.playName,
-                runId: this.options.req.runId,
-                runAttempt: this.options.req.runAttempt ?? 0,
-                ...(this.options.receiptStore
-                  ? { receiptStore: this.options.receiptStore }
-                  : {}),
-                budgetMeter: this.options.budgetMeter,
-                heartbeatIntervalMs:
-                  this.options.runtimeReceiptHeartbeatIntervalMs,
-                targets: claimed.receiptKey
-                  ? [
-                      {
-                        key: claimed.receiptKey,
-                        leaseId: claimed.receiptLeaseId,
-                      },
-                    ]
-                  : [],
-                execute: () =>
-                  this.options.executeTool({
-                    id: request.id,
-                    toolId,
-                    input: request.input,
-                    workflowStep: request.workflowStep,
-                    callbacks: this.options.callbacks,
-                    onProviderBackpressure: (retryAfterMs) =>
-                      this.reportBackpressure(toolId, retryAfterMs),
-                    onRetryAttempt: () =>
-                      this.options.governor.chargeBudget('retry'),
-                    transientHttpRetrySafe:
-                      toolContract?.retrySafeTransientHttp === true,
-                    ...(durableReceiptKey
-                      ? {
-                          durableCallReceiptKey: durableReceiptKey,
-                          executionAuthScopeDigest:
-                            request.executionAuthScopeDigest,
-                          providerIdempotencyKey:
-                            providerIdempotencyKeyForClaimed({
-                              req: this.options.req,
-                              claimed,
-                            }),
-                        }
-                      : {}),
-                    runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
-                      [claimed],
-                      {
-                        resolveOwnerTimeoutMs: (request) =>
-                          request.runtimeTimeoutMs,
-                      },
-                    ),
-                    directOptions: request.directOptions,
-                  }),
-              });
-            } catch (error) {
-              if (error instanceof RuntimeLeaseLostError) {
-                this.rejectRequests(claimed, error);
-                return;
-              }
-              if (
-                await this.requeueAfterAuthScopeChanged(
-                  [request, ...claimed.followers],
-                  error,
-                )
-              ) {
-                return;
-              }
-              this.rejectRequests(
-                claimed,
-                await this.failureForRejectedToolRequest(claimed, error),
-              );
-              return;
-            }
-            try {
-              this.settleRequests(
-                claimed,
-                await this.completeDurableToolRequest(claimed, result),
-              );
-            } catch (receiptError) {
-              this.rejectRequests(claimed, receiptError);
-            }
-          } finally {
-            slot.release();
+            this.settleRequests(
+              claimed,
+              await this.completeDurableToolRequest(claimed, result),
+            );
+          } catch (receiptError) {
+            this.rejectRequests(claimed, receiptError);
           }
-        }),
-      );
+        } finally {
+          slot.release();
+        }
+      });
       this.options.recordPerfTrace?.({
         phase: 'runner.tool.group',
         ms: this.options.nowMs() - groupStartedAt,

package/dist/bundling-sources/sdk/src/release.ts

@@ -106,10 +106,10 @@ export const SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: '0.1.201',
+  version: '0.1.203',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.201',
+    latest: '0.1.203',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -29,6 +29,8 @@ import type { PlayBundleArtifact } from '@shared_libs/plays/artifact-types';
 import type {
   PlayRunnerRateStateAcquireInput,
   PlayRunnerRateStateAcquireResult,
+  PlayRunnerBudgetChargeInput,
+  PlayRunnerBudgetChargeResult,
   PlayRunnerRateStatePenalizeInput,
   PlayRunnerRateStateReleaseInput,
 } from '@shared_libs/play-runtime/protocol';
@@ -176,6 +178,9 @@ type RuntimeApiRequest =
   | ({
       action: 'rate_state_penalize';
     } & PlayRunnerRateStatePenalizeInput)
+  | ({
+      action: 'governor_budget_charge';
+    } & PlayRunnerBudgetChargeInput)
   | {
       action: 'get_runtime_step_receipt';
       playName: string;
@@ -426,8 +431,8 @@ function summarizeAppRuntimeErrorBody(body: string): string {
   return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
 }
 
-function isTransientAppRuntimeFailureBody(body: string): boolean {
-  return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated/i.test(
+export function isTransientAppRuntimeFailureBody(body: string): boolean {
+  return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated|RUNTIME_SCHEDULER_SATURATED|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open/i.test(
     body,
   );
 }
@@ -751,6 +756,16 @@ export async function penalizeRateStateViaAppRuntime(
   });
 }
 
+export async function chargeGovernorBudgetViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: PlayRunnerBudgetChargeInput,
+): Promise<PlayRunnerBudgetChargeResult> {
+  return await postAppRuntimeApi<PlayRunnerBudgetChargeResult>(context, {
+    action: 'governor_budget_charge',
+    ...input,
+  });
+}
+
 export async function writeStagedFileFromAppRuntime(
   context: WorkerRuntimeApiContext,
   file: Pick<PlayStagedFileRef, 'storageKey'>,

package/dist/bundling-sources/shared_libs/play-runtime/bounded-dispatch.ts

@@ -0,0 +1,52 @@
+export type BoundedDispatchResult<TJob, TResult> = {
+  job: TJob;
+  index: number;
+  result: TResult | null;
+  error?: unknown;
+};
+
+/** Drain a finite queue with a fixed number of workers. Pacing and adaptive
+ * admission belong inside `execute`, at the RuntimeResourceGovernor seam. */
+export async function dispatchBoundedSettled<TJob, TResult>(
+  jobs: readonly TJob[],
+  concurrency: number,
+  execute: (job: TJob, index: number) => Promise<TResult>,
+): Promise<Array<BoundedDispatchResult<TJob, TResult>>> {
+  if (jobs.length === 0) return [];
+  const width = Math.min(
+    jobs.length,
+    Math.max(1, Math.floor(Number.isFinite(concurrency) ? concurrency : 1)),
+  );
+  const results = new Array<BoundedDispatchResult<TJob, TResult>>(jobs.length);
+  let cursor = 0;
+  const workers = Array.from({ length: width }, async () => {
+    while (true) {
+      const index = cursor;
+      cursor += 1;
+      if (index >= jobs.length) return;
+      const job = jobs[index]!;
+      try {
+        results[index] = {
+          job,
+          index,
+          result: await execute(job, index),
+        };
+      } catch (error) {
+        results[index] = { job, index, result: null, error };
+      }
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}
+
+export async function dispatchBounded<TJob, TResult>(
+  jobs: readonly TJob[],
+  concurrency: number,
+  execute: (job: TJob, index: number) => Promise<TResult>,
+): Promise<TResult[]> {
+  const settled = await dispatchBoundedSettled(jobs, concurrency, execute);
+  const failed = settled.find((entry) => entry.error !== undefined);
+  if (failed) throw failed.error;
+  return settled.map((entry) => entry.result as TResult);
+}