deepline 0.1.181 → 0.1.183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
  2. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
  3. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +633 -107
  4. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
  5. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1932 -1838
  6. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
  7. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
  8. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +124 -12
  9. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
  10. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
  11. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
  12. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
  13. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
  14. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
  15. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
  16. package/dist/bundling-sources/sdk/src/client.ts +48 -5
  17. package/dist/bundling-sources/sdk/src/http.ts +23 -8
  18. package/dist/bundling-sources/sdk/src/play.ts +200 -26
  19. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
  20. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  21. package/dist/bundling-sources/sdk/src/types.ts +2 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +132 -10
  23. package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
  25. package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
  26. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
  27. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1857 -314
  28. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +106 -6
  29. package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
  33. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
  34. package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
  35. package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
  36. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
  37. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
  38. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
  40. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
  42. package/dist/bundling-sources/shared_libs/play-runtime/query-result-dataset.ts +120 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
  44. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
  45. package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
  46. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
  47. package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
  48. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2037 -262
  50. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
  52. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
  53. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
  54. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
  56. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
  57. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
  58. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
  59. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +139 -17
  60. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
  61. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
  62. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
  63. package/dist/cli/index.js +85 -23
  64. package/dist/cli/index.mjs +85 -23
  65. package/dist/index.d.mts +4 -1
  66. package/dist/index.d.ts +4 -1
  67. package/dist/index.js +274 -35
  68. package/dist/index.mjs +274 -35
  69. package/dist/plays/bundle-play-file.mjs +2 -2
  70. package/package.json +1 -1
@@ -36,12 +36,6 @@ import {
36
36
  deterministicMapChunkStepName,
37
37
  type ExecutionPlan,
38
38
  } from '../../../shared_libs/play-runtime/execution-plan';
39
- import {
40
- compileRequestsWithStrategy,
41
- executeChunkedRequests,
42
- type ChunkExecutionResult,
43
- } from '../../../shared_libs/play-runtime/batch-runtime';
44
- import { getPlayRuntimeBatchStrategy } from '../../../shared_libs/play-runtime/play-runtime-batching-registry';
45
39
  import {
46
40
  STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
47
41
  STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
@@ -71,10 +65,10 @@ import {
71
65
  type WorkflowStepLike,
72
66
  } from './child-play-await';
73
67
  import { submitChildPlayThroughCoordinator } from './child-play-submit';
74
- import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
75
68
  import {
76
- parseToolExecuteResponse,
69
+ attachToolResultListDataset,
77
70
  createToolExecuteResult,
71
+ parseToolExecuteResponse,
78
72
  deserializeToolExecuteResult,
79
73
  isSerializedToolExecuteResult,
80
74
  isToolExecuteResult,
@@ -89,21 +83,35 @@ import {
89
83
  TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS,
90
84
  classifyToolExecuteHttpFailure,
91
85
  createToolExecuteHttpFailureAttemptTracker,
86
+ parseToolExecuteAuthScopeChangedError,
92
87
  } from '../../../shared_libs/play-runtime/tool-execute-retry-policy';
93
88
  import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
94
89
  import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
95
- import type { PlayRuntimeManifestMap } from '../../../shared_libs/plays/compiler-manifest';
90
+ import type {
91
+ PlayRuntimeManifest,
92
+ PlayRuntimeManifestMap,
93
+ } from '../../../shared_libs/plays/compiler-manifest';
96
94
  import {
97
95
  deriveToolRequestIdentity,
98
96
  derivePlayRowIdentity,
99
97
  derivePlayRowIdentityFromKey,
98
+ sha256Hex,
99
+ stableStringify,
100
100
  } from '../../../shared_libs/plays/row-identity';
101
+ import { createDeferredPlayDataset } from '../../../shared_libs/plays/dataset';
101
102
  import {
102
103
  buildDurableCtxCallCacheKey,
103
- buildDurableToolCallAuthScopeDigest,
104
- buildDurableToolCallCacheKey,
104
+ buildDurableRunPlaySemanticKey,
105
105
  } from '../../../shared_libs/play-runtime/durable-call-cache';
106
- import { buildScopedWorkReceiptKey } from '../../../shared_libs/play-runtime/work-receipts';
106
+ import {
107
+ resolveRuntimeToolReceiptWaitMaxAttempts,
108
+ resolveRuntimeToolReceiptWaitTimeoutMs,
109
+ } from '../../../shared_libs/play-runtime/durable-receipt-execution';
110
+ import {
111
+ QUERY_RESULT_DATASET_PAGE_SIZE,
112
+ isCustomerDbDatasetTool,
113
+ isQueryResultDatasetTool,
114
+ } from '../../../shared_libs/play-runtime/query-result-dataset';
107
115
  import { DEDUPE_DUPLICATE_KEY_SAMPLE_CAP } from '../../../shared_libs/play-runtime/map-row-identity';
108
116
  import {
109
117
  getTopLevelPipelineSubsteps,
@@ -142,26 +150,41 @@ import {
142
150
  } from './runtime/dataset-handles';
143
151
  import {
144
152
  runWorkerRuntimeReceiptBoundary,
145
- type WorkerRuntimeReceipt,
146
- type WorkerRuntimeReceiptClaim,
147
- type WorkerRuntimeReceiptStore,
153
+ RuntimeLeaseLostError,
154
+ type WorkerRuntimeReceiptExecutionContext,
148
155
  } from './runtime/receipts';
149
156
  import {
150
- RuntimeReceiptWaitTimeoutError,
151
- resolveRuntimeToolReceiptWaitMaxAttempts,
152
- resolveRuntimeToolReceiptWaitTimeoutMs,
153
- waitForCompletedRuntimeReceipt,
154
- } from '../../../shared_libs/play-runtime/durable-receipt-execution';
155
- import type { RuntimeStepReceipt } from '../../../shared_libs/play-runtime/ctx-types';
157
+ RuntimeReceiptPersistenceError,
158
+ WorkerToolBatchScheduler,
159
+ type WorkerPacingResolver,
160
+ type WorkerToolActionCacheVersionResolver,
161
+ } from './runtime/tool-dispatch';
162
+ import {
163
+ createWorkerWorkBudgetMeter,
164
+ type WorkerWorkBudgetMeter,
165
+ } from './runtime/work-budget';
166
+ import {
167
+ WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES,
168
+ WorkerRunWorkDispatcher,
169
+ type WorkerRunMapBatchesResult,
170
+ } from './runtime/run-work-dispatcher';
171
+ import { WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL } from './runtime/worker-platform-budget';
172
+ import {
173
+ PLAY_RUNTIME_CONTRACT,
174
+ PLAY_RUNTIME_CONTRACT_HEADER,
175
+ } from '../../../shared_libs/play-runtime/runtime-contract';
176
+ import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
177
+ import {
178
+ PLAY_RUNTIME_TEST_FAULT_HEADER,
179
+ type RuntimeTestPolicyOverrides,
180
+ } from '../../../shared_libs/play-runtime/test-runtime-seams';
181
+ import { createRuntimeToolAuthScopeDigestResolver } from '../../../shared_libs/play-runtime/auth-scope-resolver';
156
182
  import {
157
- canReclaimFailedWorkerToolReceipt,
158
- markWorkerToolReceiptResultCached,
159
- markWorkerToolReceiptResultExecution,
160
- planWorkerToolReceiptGroups,
161
- reclaimRunningReceiptGroupAfterWait,
162
- resolveWorkerToolReceiptGroupWaitMaxAttempts,
163
- resolveWorkerToolRuntimeTimeoutMs,
164
- } from './runtime/tool-receipts';
183
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
184
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
185
+ runtimeLeaseHeartbeatIntervalMs,
186
+ } from '../../../shared_libs/play-runtime/lease-policy';
187
+ import { RuntimeSheetRowsBlockedError } from '../../../shared_libs/play-runtime/runtime-sheet-errors';
165
188
  // The harness stub forwards leaf calls (validation, runtime-api HTTP) into
166
189
  // the long-lived Play Harness Worker via env.HARNESS. We import the
167
190
  // `setHarnessBinding` setter eagerly so it's available the moment
@@ -174,9 +197,12 @@ import {
174
197
  // re-bundle harness internals into per-play. Keep that in mind.
175
198
  import {
176
199
  harnessPersistCompletedSheetRows,
200
+ harnessHeartbeatSheetAttempt,
177
201
  harnessReadSheetDatasetRowKeys,
178
202
  harnessReadSheetDatasetRows,
179
203
  harnessReadStagedFileChunk,
204
+ harnessReleaseRuntimeReceipts,
205
+ harnessReleaseSheetAttempt,
180
206
  harnessStartSheetDataset,
181
207
  setHarnessBinding,
182
208
  } from '../../../sdk/src/plays/harness-stub';
@@ -207,7 +233,10 @@ import {
207
233
  stripMapRowOutcomeRuntimeFields,
208
234
  } from '../../../shared_libs/play-runtime/map-row-outcome';
209
235
  import { runtimeSheetSessionScope } from '../../../shared_libs/play-runtime/runtime-sheet-session';
210
- import { chooseWorkerMapRowsPerChunk } from './runtime/map-chunk-plan';
236
+ import {
237
+ chooseWorkerMapDispatchPlan,
238
+ estimateBatchedToolChunkSubrequests,
239
+ } from './runtime/map-chunk-plan';
211
240
  import {
212
241
  applyCsvRenameProjection,
213
242
  cloneCsvAliasedRow,
@@ -219,12 +248,7 @@ import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-f
219
248
  import {
220
249
  createRuntimePersistenceLatch,
221
250
  isRuntimePersistenceCircuitOpenError,
222
- partitionDispatchWaves,
223
- PROVIDER_DISPATCH_WAVE_SIZE,
224
- RuntimePersistenceCircuitOpenError,
225
- shouldShortenCompleteReceiptLadder,
226
251
  tripRuntimePersistenceLatch,
227
- type RuntimePersistenceLatch,
228
252
  } from '../../../shared_libs/play-runtime/persistence-latch';
229
253
  import {
230
254
  createReceiptSalvageBuffer,
@@ -290,8 +314,13 @@ type RunRequest = {
290
314
  playName: string;
291
315
  graphHash?: string | null;
292
316
  userEmail: string | null;
293
- force?: boolean | null;
294
317
  runtimeInput: Record<string, unknown>;
318
+ /** Start a fresh run graph and recompute runtime-sheet rows. */
319
+ force?: boolean;
320
+ /** Explicit durable tool receipt refresh that can re-execute completed provider calls. */
321
+ forceToolRefresh?: boolean;
322
+ /** Monotonic coordinator redispatch epoch for receipt and sheet fencing. */
323
+ runAttempt?: number;
295
324
  /** Optional inline CSV rows (for plays where ctx.csv was passed inline data). */
296
325
  inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
297
326
  /** Staged input files keyed by logical filename (used by ctx.csv). */
@@ -324,6 +353,10 @@ type RunRequest = {
324
353
  coordinatorUrl?: string | null;
325
354
  /** Request-scoped coordinator auth token for preview/dev direct control calls. */
326
355
  coordinatorInternalToken?: string | null;
356
+ /** Request-scoped, dev-only runtime fault injection header for black-box tests. */
357
+ runtimeTestFaultHeader?: string | null;
358
+ /** Request-scoped, dev-only policy overrides for black-box durability tests. */
359
+ testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
327
360
  /**
328
361
  * Total input rows known at submit time, when the dispatcher can count them.
329
362
  * `undefined` when the input streams from R2 (unknown row count).
@@ -339,9 +372,136 @@ type WorkerFileRef = {
339
372
  bytes?: number | null;
340
373
  };
341
374
 
375
+ function normalizeWorkerRunAttempt(value: unknown): number {
376
+ return typeof value === 'number' && Number.isFinite(value)
377
+ ? Math.max(0, Math.floor(value))
378
+ : 0;
379
+ }
380
+
381
+ function normalizePositiveInteger(value: unknown): number | undefined {
382
+ return typeof value === 'number' &&
383
+ Number.isFinite(value) &&
384
+ Number.isInteger(value) &&
385
+ value > 0
386
+ ? value
387
+ : undefined;
388
+ }
389
+
390
+ function normalizeRuntimeTestPolicyOverrides(
391
+ value: unknown,
392
+ ): RuntimeTestPolicyOverrides | null {
393
+ if (!isRecord(value)) return null;
394
+ const overrides: RuntimeTestPolicyOverrides = {};
395
+ const receiptLeaseTtlMs = normalizePositiveInteger(value.receiptLeaseTtlMs);
396
+ if (receiptLeaseTtlMs !== undefined) {
397
+ overrides.receiptLeaseTtlMs = receiptLeaseTtlMs;
398
+ }
399
+ const sheetAttemptLeaseMs = normalizePositiveInteger(
400
+ value.sheetAttemptLeaseMs,
401
+ );
402
+ if (sheetAttemptLeaseMs !== undefined) {
403
+ overrides.sheetAttemptLeaseMs = sheetAttemptLeaseMs;
404
+ }
405
+ const heartbeatIntervalMs = normalizePositiveInteger(
406
+ value.heartbeatIntervalMs,
407
+ );
408
+ if (heartbeatIntervalMs !== undefined) {
409
+ overrides.heartbeatIntervalMs = heartbeatIntervalMs;
410
+ }
411
+ const batchGraceMs = normalizePositiveInteger(value.batchGraceMs);
412
+ if (batchGraceMs !== undefined) {
413
+ overrides.batchGraceMs = batchGraceMs;
414
+ }
415
+ if (isRecord(value.workBudgetYieldLimits)) {
416
+ const limits: NonNullable<
417
+ RuntimeTestPolicyOverrides['workBudgetYieldLimits']
418
+ > = {};
419
+ for (const key of [
420
+ 'elapsed',
421
+ 'subrequest',
422
+ 'provider',
423
+ 'tool',
424
+ 'receipt',
425
+ 'sheet',
426
+ 'log',
427
+ 'egress',
428
+ ] as const) {
429
+ const limit = normalizePositiveInteger(value.workBudgetYieldLimits[key]);
430
+ if (limit !== undefined) {
431
+ limits[key] = limit;
432
+ }
433
+ }
434
+ if (Object.keys(limits).length > 0) {
435
+ overrides.workBudgetYieldLimits = limits;
436
+ }
437
+ }
438
+ return Object.keys(overrides).length > 0 ? overrides : null;
439
+ }
440
+
441
+ function runtimeReceiptLeaseTtlMs(req: RunRequest): number | undefined {
442
+ return req.testPolicyOverrides?.receiptLeaseTtlMs;
443
+ }
444
+
445
+ function runtimeSheetAttemptLeaseTtlMs(req: RunRequest): number | undefined {
446
+ return req.testPolicyOverrides?.sheetAttemptLeaseMs;
447
+ }
448
+
449
+ function runtimePolicyHeartbeatIntervalMs(
450
+ req: RunRequest,
451
+ fallbackLeaseTtlMs: number,
452
+ ): number {
453
+ return (
454
+ req.testPolicyOverrides?.heartbeatIntervalMs ??
455
+ runtimeLeaseHeartbeatIntervalMs(fallbackLeaseTtlMs)
456
+ );
457
+ }
458
+
342
459
  const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
343
460
  const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
461
+ const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
344
462
  const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
463
+ // A short durable sleep is enough to force a Cloudflare Workflows step
464
+ // checkpoint. The resume re-enters run() as a new Worker invocation with a
465
+ // fresh per-invocation subrequest budget (see the yield callback below), so
466
+ // there is no need to escalate the sleep to "encourage" re-invocation.
467
+ // A 1ms Workflow sleep is too small to reliably create a new platform
468
+ // invocation in preview; the dispatcher then resets its local budget while
469
+ // Cloudflare's per-invocation subrequest counter is still hot. Use a real
470
+ // boundary so budget-yielded maps resume with fresh platform headroom.
471
+ const WORK_DISPATCHER_YIELD_SLEEP_MS = 1_000;
472
+
473
+ type CustomerDbDatasetRequest = {
474
+ limit?: number;
475
+ offset?: number;
476
+ pageSize?: number;
477
+ totalRows?: number;
478
+ };
479
+
480
+ type WorkerToolDirectOptions = {
481
+ customerDbDataset?: CustomerDbDatasetRequest;
482
+ attachCustomerDbDataset?: boolean;
483
+ };
484
+
485
+ function rowsFromUnknown(value: unknown): Record<string, unknown>[] {
486
+ if (!Array.isArray(value)) return [];
487
+ return value.map((row) =>
488
+ row && typeof row === 'object' && !Array.isArray(row)
489
+ ? (row as Record<string, unknown>)
490
+ : { value: row },
491
+ );
492
+ }
493
+
494
+ function finiteNonNegativeInteger(value: unknown): number | null {
495
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
496
+ return null;
497
+ }
498
+ return Math.floor(value);
499
+ }
500
+
501
+ function finitePositiveInteger(value: unknown): number | null {
502
+ const integer = finiteNonNegativeInteger(value);
503
+ return integer !== null && integer > 0 ? integer : null;
504
+ }
345
505
 
346
506
  function getStringField(value: unknown, key: string): string | null {
347
507
  if (!isRecord(value)) return null;
@@ -376,7 +536,8 @@ type WorkerEnv = {
376
536
  /**
377
537
  * In-process Fetcher constructed by the coordinator and handed to the
378
538
  * per-graphHash play Worker. When present, runtime callbacks
379
- * (`/api/v2/plays/internal/runtime`, `/api/v2/plays/internal/*`,
539
+ * (`/api/v2/internal/play-runtime`,
540
+ * `/api/v2/internal/play-runtime/*`,
380
541
  * `/api/v2/plays/runtime-tools/*`) skip the public callback URL and route
381
542
  * directly through the coordinator's process to the configured app — saves
382
543
  * the *.workers.dev → CF edge → cloudflared → localhost chain on every
@@ -385,6 +546,7 @@ type WorkerEnv = {
385
546
  */
386
547
  RUNTIME_API?: {
387
548
  runtimeApiCall(input: {
549
+ contract?: number;
388
550
  executorToken: string;
389
551
  baseUrl?: string;
390
552
  path: string;
@@ -621,7 +783,7 @@ async function fetchRuntimeApi(
621
783
  ? Math.max(1, Math.ceil(options.timeoutMsOverride))
622
784
  : path === '/api/v2/plays/run'
623
785
  ? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
624
- : path === '/api/v2/plays/internal/egress-fetch'
786
+ : path === '/api/v2/internal/play-runtime/egress-fetch'
625
787
  ? RUNTIME_API_EGRESS_FETCH_TIMEOUT_MS
626
788
  : /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
627
789
  ? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
@@ -649,6 +811,10 @@ async function fetchRuntimeApi(
649
811
  );
650
812
  }, timeoutMs);
651
813
  });
814
+ const relayAbort = () => controller.abort(abortSignal?.reason);
815
+ if (abortSignal) {
816
+ abortSignal.addEventListener('abort', relayAbort, { once: true });
817
+ }
652
818
  try {
653
819
  const mergedInit: RequestInit = {
654
820
  ...init,
@@ -670,11 +836,10 @@ async function fetchRuntimeApi(
670
836
  timeoutMs,
671
837
  },
672
838
  );
673
- // RUNTIME_API service bindings do not consume RequestInit.signal once the RPC
674
- // is in flight. After dispatch, wait for the owner call to settle so the
675
- // durable receipt records the real provider result instead of failing early
676
- // and inviting duplicate provider work on retry.
677
839
  const response = await Promise.race([responsePromise, timeoutPromise]);
840
+ if (abortSignal?.aborted) {
841
+ throw abortError();
842
+ }
678
843
  if (await isRuntimeApiBindingNotFoundResponse(response)) {
679
844
  throw new Error(
680
845
  `[play-harness] RUNTIME_API service binding could not route ${path}; coordinator returned not found.`,
@@ -690,6 +855,9 @@ async function fetchRuntimeApi(
690
855
  }
691
856
  throw err;
692
857
  } finally {
858
+ if (abortSignal) {
859
+ abortSignal.removeEventListener('abort', relayAbort);
860
+ }
693
861
  if (timeout) clearTimeout(timeout);
694
862
  }
695
863
  }
@@ -706,8 +874,23 @@ async function callRuntimeApiRpcBinding(
706
874
  if (metadata) headers[EXECUTE_TOOL_METADATA_HEADER] = metadata;
707
875
  const contract = h.get(EXECUTE_RESPONSE_CONTRACT_HEADER);
708
876
  if (contract) headers[EXECUTE_RESPONSE_CONTRACT_HEADER] = contract;
877
+ const idempotencyKey = h.get('x-deepline-idempotency-key');
878
+ if (idempotencyKey) {
879
+ headers['x-deepline-idempotency-key'] = idempotencyKey;
880
+ }
881
+ const cacheControl = h.get('cache-control');
882
+ if (cacheControl) headers['cache-control'] = cacheControl;
883
+ const pragma = h.get('pragma');
884
+ if (pragma) headers.pragma = pragma;
885
+ const runtimeTestFaultHeader = h.get(PLAY_RUNTIME_TEST_FAULT_HEADER);
886
+ if (runtimeTestFaultHeader) {
887
+ headers[PLAY_RUNTIME_TEST_FAULT_HEADER] = runtimeTestFaultHeader;
888
+ }
889
+ const responseIntent = h.get(EXECUTE_RESPONSE_INTENT_HEADER);
890
+ if (responseIntent) headers[EXECUTE_RESPONSE_INTENT_HEADER] = responseIntent;
709
891
  const rawBody = typeof init.body === 'string' ? init.body : '';
710
892
  const result = await binding.runtimeApiCall({
893
+ contract: PLAY_RUNTIME_CONTRACT,
711
894
  executorToken: authorization.replace(/^Bearer\s+/i, '').trim(),
712
895
  baseUrl: input.baseUrl,
713
896
  path: input.path,
@@ -740,6 +923,7 @@ function runtimeApiHeaders(
740
923
  includeVercelBypass: boolean,
741
924
  ): Headers {
742
925
  const next = new Headers(headers);
926
+ next.set(PLAY_RUNTIME_CONTRACT_HEADER, String(PLAY_RUNTIME_CONTRACT));
743
927
  if (includeVercelBypass) {
744
928
  const bypassToken = cachedVercelProtectionBypassToken();
745
929
  if (bypassToken) {
@@ -932,9 +1116,172 @@ async function drainRunnerPerfTraces(req: RunRequest): Promise<void> {
932
1116
  ]);
933
1117
  }
934
1118
 
1119
+ let workerRequestSequence = 0;
1120
+
935
1121
  function makeRequestId(): string {
936
- // Workers crypto.randomUUID is available without nodejs_compat.
937
- return crypto.randomUUID();
1122
+ workerRequestSequence = (workerRequestSequence + 1) % Number.MAX_SAFE_INTEGER;
1123
+ return `worker-request-${workerRequestSequence}`;
1124
+ }
1125
+
1126
+ function makeRuntimeSheetAttempt(input: {
1127
+ runId: string;
1128
+ runAttempt?: number | null;
1129
+ mapName: string;
1130
+ chunkIndex: number | 'finalize';
1131
+ }): {
1132
+ attemptId: string;
1133
+ attemptOwnerRunId: string;
1134
+ attemptExpiresAt: string | null;
1135
+ attemptSeq: number;
1136
+ } {
1137
+ const runAttempt =
1138
+ typeof input.runAttempt === 'number' && Number.isFinite(input.runAttempt)
1139
+ ? Math.max(0, Math.floor(input.runAttempt))
1140
+ : 0;
1141
+ const attemptKey = sha256Hex(
1142
+ JSON.stringify({
1143
+ runId: input.runId,
1144
+ runAttempt,
1145
+ mapName: input.mapName,
1146
+ chunkIndex: input.chunkIndex,
1147
+ }),
1148
+ ).slice(0, 16);
1149
+ return {
1150
+ attemptId: `sheet:${input.mapName}:${input.chunkIndex}:attempt-${runAttempt}:${attemptKey}`,
1151
+ attemptOwnerRunId: input.runId,
1152
+ attemptExpiresAt: null,
1153
+ attemptSeq: runAttempt,
1154
+ };
1155
+ }
1156
+
1157
+ class RuntimeSheetAttemptLeaseLostError extends Error {
1158
+ readonly rowKeys: string[];
1159
+ readonly runId: string;
1160
+ readonly attemptId: string;
1161
+ readonly tableNamespace: string;
1162
+
1163
+ constructor(input: {
1164
+ rowKeys: string[];
1165
+ runId: string;
1166
+ attemptId: string;
1167
+ tableNamespace: string;
1168
+ cause?: unknown;
1169
+ }) {
1170
+ const rowSummary =
1171
+ input.rowKeys.length === 1
1172
+ ? input.rowKeys[0]!
1173
+ : `${input.rowKeys.length} rows`;
1174
+ super(
1175
+ `Runtime sheet attempt ${input.attemptId} lost ownership of ${rowSummary} in ${input.tableNamespace}.`,
1176
+ { cause: input.cause },
1177
+ );
1178
+ this.name = 'RuntimeSheetAttemptLeaseLostError';
1179
+ this.rowKeys = input.rowKeys;
1180
+ this.runId = input.runId;
1181
+ this.attemptId = input.attemptId;
1182
+ this.tableNamespace = input.tableNamespace;
1183
+ }
1184
+ }
1185
+
1186
+ async function executeWithRuntimeSheetAttemptHeartbeat<T>(input: {
1187
+ req: RunRequest;
1188
+ tableNamespace: string;
1189
+ sheetContract: PlaySheetContract;
1190
+ attemptId?: string | null;
1191
+ attemptOwnerRunId?: string | null;
1192
+ attemptSeq?: number | null;
1193
+ rowKeys: string[];
1194
+ execute: () => Promise<T> | T;
1195
+ }): Promise<T> {
1196
+ const attemptId = input.attemptId?.trim();
1197
+ const rowKeys = [
1198
+ ...new Set(input.rowKeys.map((key) => key.trim()).filter(Boolean)),
1199
+ ];
1200
+ if (!attemptId || rowKeys.length === 0) {
1201
+ return await input.execute();
1202
+ }
1203
+ const heartbeatIntervalMs = runtimePolicyHeartbeatIntervalMs(
1204
+ input.req,
1205
+ runtimeSheetAttemptLeaseTtlMs(input.req) ??
1206
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
1207
+ );
1208
+
1209
+ let stopped = false;
1210
+ let timeout: ReturnType<typeof setTimeout> | null = null;
1211
+ let rejectLeaseLost!: (error: RuntimeSheetAttemptLeaseLostError) => void;
1212
+ const leaseLost = new Promise<never>((_, reject) => {
1213
+ rejectLeaseLost = reject;
1214
+ });
1215
+
1216
+ const stop = () => {
1217
+ stopped = true;
1218
+ if (timeout !== null) {
1219
+ clearTimeout(timeout);
1220
+ timeout = null;
1221
+ }
1222
+ };
1223
+
1224
+ const heartbeatOnce = async () => {
1225
+ const result = await harnessHeartbeatSheetAttempt({
1226
+ ...runtimeSheetSessionScope(input.req),
1227
+ tableNamespace: input.tableNamespace,
1228
+ sheetContract: input.sheetContract,
1229
+ runId: input.req.runId,
1230
+ attemptId,
1231
+ attemptOwnerRunId: input.attemptOwnerRunId ?? input.req.runId,
1232
+ attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
1233
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
1234
+ keys: rowKeys,
1235
+ });
1236
+ const renewed = new Set(result.renewedKeys);
1237
+ const lostKeys = rowKeys.filter((key) => !renewed.has(key));
1238
+ if (lostKeys.length > 0) {
1239
+ throw new RuntimeSheetAttemptLeaseLostError({
1240
+ rowKeys: lostKeys,
1241
+ runId: input.req.runId,
1242
+ attemptId,
1243
+ tableNamespace: input.tableNamespace,
1244
+ });
1245
+ }
1246
+ };
1247
+
1248
+ const schedule = () => {
1249
+ timeout = setTimeout(() => {
1250
+ void (async () => {
1251
+ try {
1252
+ await heartbeatOnce();
1253
+ } catch (error) {
1254
+ stop();
1255
+ if (error instanceof RuntimeSheetAttemptLeaseLostError) {
1256
+ rejectLeaseLost(error);
1257
+ } else {
1258
+ rejectLeaseLost(
1259
+ new RuntimeSheetAttemptLeaseLostError({
1260
+ rowKeys,
1261
+ runId: input.req.runId,
1262
+ attemptId,
1263
+ tableNamespace: input.tableNamespace,
1264
+ cause: error,
1265
+ }),
1266
+ );
1267
+ }
1268
+ return;
1269
+ }
1270
+ if (!stopped) {
1271
+ schedule();
1272
+ }
1273
+ })();
1274
+ }, heartbeatIntervalMs);
1275
+ };
1276
+
1277
+ schedule();
1278
+ const execution = Promise.resolve().then(input.execute);
1279
+ execution.catch(() => {});
1280
+ try {
1281
+ return await Promise.race([execution, leaseLost]);
1282
+ } finally {
1283
+ stop();
1284
+ }
938
1285
  }
939
1286
 
940
1287
  /**
@@ -960,6 +1307,11 @@ async function postRuntimeApi<T>(
960
1307
  baseUrl: string,
961
1308
  executorToken: string,
962
1309
  body: unknown,
1310
+ options: {
1311
+ runtimeTestFaultHeader?: string | null;
1312
+ timeoutMsOverride?: number;
1313
+ timeoutErrorMessage?: string;
1314
+ } = {},
963
1315
  ): Promise<T> {
964
1316
  // Routes through the in-process RUNTIME_API service binding. Missing binding
965
1317
  // is an infra error in workers_edge, not a reason to fall back to public HTTP.
@@ -972,15 +1324,29 @@ async function postRuntimeApi<T>(
972
1324
  ) {
973
1325
  let res: Response;
974
1326
  try {
975
- res = await fetchRuntimeApi(baseUrl, '/api/v2/plays/internal/runtime', {
976
- method: 'POST',
977
- headers: {
978
- 'content-type': 'application/json',
979
- authorization: `Bearer ${executorToken}`,
980
- 'x-deepline-request-id': makeRequestId(),
1327
+ res = await fetchRuntimeApi(
1328
+ baseUrl,
1329
+ '/api/v2/internal/play-runtime',
1330
+ {
1331
+ method: 'POST',
1332
+ headers: {
1333
+ 'content-type': 'application/json',
1334
+ authorization: `Bearer ${executorToken}`,
1335
+ 'x-deepline-request-id': makeRequestId(),
1336
+ ...(options.runtimeTestFaultHeader?.trim()
1337
+ ? {
1338
+ [PLAY_RUNTIME_TEST_FAULT_HEADER]:
1339
+ options.runtimeTestFaultHeader.trim(),
1340
+ }
1341
+ : {}),
1342
+ },
1343
+ body: serializedBody,
981
1344
  },
982
- body: serializedBody,
983
- });
1345
+ {
1346
+ timeoutMsOverride: options.timeoutMsOverride,
1347
+ timeoutErrorMessage: options.timeoutErrorMessage,
1348
+ },
1349
+ );
984
1350
  } catch (error) {
985
1351
  lastError = error;
986
1352
  if (
@@ -1064,9 +1430,10 @@ async function postRuntimeApiBestEffort(
1064
1430
  baseUrl: string,
1065
1431
  executorToken: string,
1066
1432
  body: unknown,
1433
+ options: { runtimeTestFaultHeader?: string | null } = {},
1067
1434
  ): Promise<boolean> {
1068
1435
  try {
1069
- await postRuntimeApi(baseUrl, executorToken, body);
1436
+ await postRuntimeApi(baseUrl, executorToken, body, options);
1070
1437
  return true;
1071
1438
  } catch (error) {
1072
1439
  console.error(
@@ -1078,6 +1445,43 @@ async function postRuntimeApiBestEffort(
1078
1445
  }
1079
1446
  }
1080
1447
 
1448
+ async function readDurableChildRunTerminalState(input: {
1449
+ baseUrl: string;
1450
+ executorToken: string;
1451
+ parentRunId: string;
1452
+ childRunId: string;
1453
+ childPlayName: string;
1454
+ timeoutMs?: number;
1455
+ }): Promise<{ data?: unknown } | null> {
1456
+ const response = await postRuntimeApi<{
1457
+ state?: {
1458
+ data?: unknown;
1459
+ reason?: string;
1460
+ status?: string;
1461
+ runId?: string;
1462
+ playName?: string;
1463
+ parentRunId?: string | null;
1464
+ } | null;
1465
+ }>(
1466
+ input.baseUrl,
1467
+ input.executorToken,
1468
+ {
1469
+ action: 'read_child_run_terminal_snapshot',
1470
+ parentRunId: input.parentRunId,
1471
+ childRunId: input.childRunId,
1472
+ childPlayName: input.childPlayName,
1473
+ },
1474
+ {
1475
+ timeoutMsOverride: input.timeoutMs,
1476
+ timeoutErrorMessage: `[play-harness] child run terminal snapshot read timed out after ${Math.max(
1477
+ 1,
1478
+ Math.ceil(input.timeoutMs ?? RUNTIME_API_TIMEOUT_MS),
1479
+ )}ms. childRunId=${input.childRunId}`,
1480
+ },
1481
+ );
1482
+ return response.state ?? null;
1483
+ }
1484
+
1081
1485
  function workflowEventType(name: string): string {
1082
1486
  const normalized = name
1083
1487
  .trim()
@@ -1187,6 +1591,27 @@ async function hashChildPlayEventKey(input: unknown): Promise<string> {
1187
1591
  return (await hashJson(input)).slice(0, 32);
1188
1592
  }
1189
1593
 
1594
+ function workerChildRevisionFingerprint(input: {
1595
+ playName: string;
1596
+ manifest: PlayRuntimeManifest | null | undefined;
1597
+ }): string | null {
1598
+ const manifest = input.manifest;
1599
+ if (!manifest) return null;
1600
+ return sha256Hex(
1601
+ stableStringify({
1602
+ playId: manifest.playName?.trim() || input.playName,
1603
+ codeFormat: 'cjs_module',
1604
+ artifactHash: manifest.artifactHash ?? null,
1605
+ graphHash: manifest.graphHash ?? null,
1606
+ sourceHash: null,
1607
+ sourceCodeHash:
1608
+ typeof manifest.sourceCode === 'string'
1609
+ ? sha256Hex(manifest.sourceCode)
1610
+ : null,
1611
+ }),
1612
+ );
1613
+ }
1614
+
1190
1615
  async function childPlayEventKey(input: {
1191
1616
  key: string;
1192
1617
  workflowId: string;
@@ -1283,7 +1708,14 @@ async function signalParentPlayTerminal(input: {
1283
1708
 
1284
1709
  async function executeTool(
1285
1710
  req: RunRequest,
1286
- args: { id: string; toolId: string; input: Record<string, unknown> },
1711
+ args: {
1712
+ id: string;
1713
+ toolId: string;
1714
+ input: Record<string, unknown>;
1715
+ durableCallReceiptKey?: string | null;
1716
+ executionAuthScopeDigest?: string | null;
1717
+ providerIdempotencyKey?: string | null;
1718
+ },
1287
1719
  workflowStep?: WorkflowStep,
1288
1720
  onProviderBackpressure?: (retryAfterMs: number) => void,
1289
1721
  onRetryAttempt?: () => void,
@@ -1291,6 +1723,7 @@ async function executeTool(
1291
1723
  abortSignal?: AbortSignal,
1292
1724
  runtimeDeadlineMs?: number,
1293
1725
  runtimeTimeoutMs?: number,
1726
+ directOptions?: WorkerToolDirectOptions,
1294
1727
  ): Promise<ToolExecuteResult> {
1295
1728
  if (args.toolId === 'test_wait_for_event' && workflowStep) {
1296
1729
  const result = await waitForSyntheticIntegrationEvent(
@@ -1314,12 +1747,20 @@ async function executeTool(
1314
1747
  abortSignal,
1315
1748
  runtimeDeadlineMs,
1316
1749
  runtimeTimeoutMs,
1750
+ directOptions,
1317
1751
  );
1318
1752
  }
1319
1753
 
1320
1754
  async function executeToolWithLifecycle(
1321
1755
  req: RunRequest,
1322
- args: { id: string; toolId: string; input: Record<string, unknown> },
1756
+ args: {
1757
+ id: string;
1758
+ toolId: string;
1759
+ input: Record<string, unknown>;
1760
+ durableCallReceiptKey?: string | null;
1761
+ executionAuthScopeDigest?: string | null;
1762
+ providerIdempotencyKey?: string | null;
1763
+ },
1323
1764
  workflowStep: WorkflowStep | undefined,
1324
1765
  callbacks: WorkerCtxCallbacks | undefined,
1325
1766
  onProviderBackpressure?: (retryAfterMs: number) => void,
@@ -1328,6 +1769,7 @@ async function executeToolWithLifecycle(
1328
1769
  abortSignal?: AbortSignal,
1329
1770
  runtimeDeadlineMs?: number,
1330
1771
  runtimeTimeoutMs?: number,
1772
+ directOptions?: WorkerToolDirectOptions,
1331
1773
  ): Promise<ToolExecuteResult> {
1332
1774
  callbacks?.onToolCalled?.(args.toolId, nowMs());
1333
1775
  try {
@@ -1341,6 +1783,7 @@ async function executeToolWithLifecycle(
1341
1783
  abortSignal,
1342
1784
  runtimeDeadlineMs,
1343
1785
  runtimeTimeoutMs,
1786
+ directOptions,
1344
1787
  );
1345
1788
  } catch (error) {
1346
1789
  callbacks?.onToolFailed?.(args.toolId, nowMs());
@@ -1413,24 +1856,29 @@ async function waitForSyntheticIntegrationEvent(
1413
1856
  typeof input.timeout_ms === 'number' && Number.isFinite(input.timeout_ms)
1414
1857
  ? Math.max(1, Math.round(input.timeout_ms))
1415
1858
  : 30_000;
1416
- await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
1417
- action: 'append_run_events',
1418
- playId: req.runId,
1419
- events: [
1420
- {
1421
- type: 'log.appended',
1422
- runId: req.runId,
1423
- // 'system' (windowed text-dedupe channel), NOT 'worker': this line is
1424
- // emitted outside the harness log buffer, so it has no positional
1425
- // channelOffset and must not pollute the worker channel cursor.
1426
- source: 'system',
1427
- occurredAt: nowMs(),
1428
- lines: [
1429
- `Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
1430
- ],
1431
- } satisfies PlayRunLedgerEvent,
1432
- ],
1433
- });
1859
+ await postRuntimeApiBestEffort(
1860
+ req.baseUrl,
1861
+ req.executorToken,
1862
+ {
1863
+ action: 'append_run_events',
1864
+ playId: req.runId,
1865
+ events: [
1866
+ {
1867
+ type: 'log.appended',
1868
+ runId: req.runId,
1869
+ // 'system' (windowed text-dedupe channel), NOT 'worker': this line is
1870
+ // emitted outside the harness log buffer, so it has no positional
1871
+ // channelOffset and must not pollute the worker channel cursor.
1872
+ source: 'system',
1873
+ occurredAt: nowMs(),
1874
+ lines: [
1875
+ `Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
1876
+ ],
1877
+ } satisfies PlayRunLedgerEvent,
1878
+ ],
1879
+ },
1880
+ { runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null },
1881
+ );
1434
1882
  try {
1435
1883
  const event = (await (
1436
1884
  workflowStep.waitForEvent as unknown as (
@@ -1478,7 +1926,14 @@ async function waitForSyntheticIntegrationEvent(
1478
1926
 
1479
1927
  async function callToolDirect(
1480
1928
  req: RunRequest,
1481
- args: { id: string; toolId: string; input: Record<string, unknown> },
1929
+ args: {
1930
+ id: string;
1931
+ toolId: string;
1932
+ input: Record<string, unknown>;
1933
+ durableCallReceiptKey?: string | null;
1934
+ executionAuthScopeDigest?: string | null;
1935
+ providerIdempotencyKey?: string | null;
1936
+ },
1482
1937
  onProviderBackpressure?: (retryAfterMs: number) => void,
1483
1938
  // Invoked once per in-process retry attempt (429 / retryable 5xx / synthetic
1484
1939
  // transient) so the Governor charges chargeBudget('retry') per attempt — the
@@ -1490,9 +1945,20 @@ async function callToolDirect(
1490
1945
  abortSignal?: AbortSignal,
1491
1946
  runtimeDeadlineMs?: number,
1492
1947
  runtimeTimeoutMs?: number,
1948
+ directOptions?: WorkerToolDirectOptions,
1493
1949
  ): Promise<ToolExecuteResult> {
1494
1950
  const { id, toolId, input } = args;
1495
1951
  const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
1952
+ const durableCallReceiptKey = args.durableCallReceiptKey?.trim() || null;
1953
+ const executionAuthScopeDigest =
1954
+ durableCallReceiptKey && args.executionAuthScopeDigest
1955
+ ? args.executionAuthScopeDigest.trim() || null
1956
+ : null;
1957
+ const providerIdempotencyKey =
1958
+ args.providerIdempotencyKey?.trim() || durableCallReceiptKey;
1959
+ const deeplineRequestId = providerIdempotencyKey
1960
+ ? `ctx-tool-${sha256Hex(providerIdempotencyKey).slice(0, 32)}`
1961
+ : null;
1496
1962
  let lastError: Error | null = null;
1497
1963
  const httpFailureAttempts = createToolExecuteHttpFailureAttemptTracker();
1498
1964
  let requestAttempt = 0;
@@ -1515,13 +1981,66 @@ async function callToolDirect(
1515
1981
  headers: {
1516
1982
  'content-type': 'application/json',
1517
1983
  authorization: `Bearer ${req.executorToken}`,
1518
- 'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
1984
+ 'x-deepline-request-id':
1985
+ deeplineRequestId ??
1986
+ `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
1987
+ ...(providerIdempotencyKey
1988
+ ? { 'x-deepline-idempotency-key': providerIdempotencyKey }
1989
+ : {}),
1519
1990
  [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
1991
+ [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
1520
1992
  [EXECUTE_TOOL_METADATA_HEADER]: 'true',
1521
1993
  },
1522
1994
  body: JSON.stringify({
1523
1995
  payload: input,
1524
- metadata: { parent_run_id: req.runId },
1996
+ metadata: {
1997
+ parent_run_id: req.runId,
1998
+ ...(durableCallReceiptKey
1999
+ ? {
2000
+ durable_call_receipt_key: durableCallReceiptKey,
2001
+ ...(executionAuthScopeDigest
2002
+ ? {
2003
+ execution_auth_scope_digest: executionAuthScopeDigest,
2004
+ }
2005
+ : {}),
2006
+ }
2007
+ : {}),
2008
+ ...(directOptions?.customerDbDataset
2009
+ ? {
2010
+ query_result_dataset: {
2011
+ limit: directOptions.customerDbDataset.limit,
2012
+ offset: directOptions.customerDbDataset.offset,
2013
+ page_size: directOptions.customerDbDataset.pageSize,
2014
+ ...(directOptions.customerDbDataset.totalRows !==
2015
+ undefined
2016
+ ? {
2017
+ total_rows:
2018
+ directOptions.customerDbDataset.totalRows,
2019
+ }
2020
+ : {}),
2021
+ },
2022
+ ...(isCustomerDbDatasetTool(toolId)
2023
+ ? {
2024
+ customer_db_dataset: {
2025
+ limit: directOptions.customerDbDataset.limit,
2026
+ offset: directOptions.customerDbDataset.offset,
2027
+ page_size: directOptions.customerDbDataset.pageSize,
2028
+ ...(directOptions.customerDbDataset.totalRows !==
2029
+ undefined
2030
+ ? {
2031
+ total_rows:
2032
+ directOptions.customerDbDataset.totalRows,
2033
+ }
2034
+ : {}),
2035
+ },
2036
+ }
2037
+ : {}),
2038
+ }
2039
+ : {}),
2040
+ ...(providerIdempotencyKey
2041
+ ? { provider_idempotency_key: providerIdempotencyKey }
2042
+ : {}),
2043
+ },
1525
2044
  ...(req.integrationMode
1526
2045
  ? { integration_mode: req.integrationMode }
1527
2046
  : {}),
@@ -1561,10 +2080,20 @@ async function callToolDirect(
1561
2080
  parsed.result,
1562
2081
  parsed.metadata,
1563
2082
  parsed.status,
2083
+ directOptions?.attachCustomerDbDataset === false
2084
+ ? undefined
2085
+ : { req, input },
1564
2086
  );
1565
2087
  }
1566
2088
 
1567
2089
  const text = await res.text().catch(() => '');
2090
+ const authScopeChanged = parseToolExecuteAuthScopeChangedError({
2091
+ status: res.status,
2092
+ bodyText: text,
2093
+ });
2094
+ if (authScopeChanged) {
2095
+ throw authScopeChanged;
2096
+ }
1568
2097
  const httpFailureAttempt = httpFailureAttempts.next({
1569
2098
  toolId,
1570
2099
  status: res.status,
@@ -1598,60 +2127,146 @@ async function callToolDirect(
1598
2127
  throw lastError ?? new Error(`tool ${toolId} failed before execution.`);
1599
2128
  }
1600
2129
 
1601
- function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
1602
- if (toolId === 'test_rate_limit') {
1603
- // Batched members resolve metadata through this fallback because the lean
1604
- // worker does not bundle the catalog. It MUST mirror the same
1605
- // `email_status: emailStatus({...})` contract registered in
1606
- // src/lib/integrations/test/index.ts so batched results normalize to the
1607
- // same rich email_status OBJECT (status from statusMap, catch_all as a
1608
- // signal) that the single-execute real-metadata path produces. The legacy
1609
- // `transforms:['emailStatus']` + mx_security_gateway→'catch_all' string
1610
- // override coarsened email_status into a bare string and predates the
1611
- // emailStatus object contract (#1466).
1612
- return {
1613
- toolId,
1614
- extractors: {
1615
- email_status: {
1616
- paths: ['email_status'],
1617
- emailStatus: {
1618
- provider: 'test',
1619
- rawStatus: ['email_status'],
1620
- catchAll: ['mx_security_gateway'],
1621
- statusMap: {
1622
- valid: { status: 'valid', verdict: 'send', verified: true },
1623
- invalid: { status: 'invalid', verdict: 'drop', verified: false },
1624
- catch_all: {
1625
- status: 'catch_all',
1626
- verdict: 'verify_next',
1627
- verified: false,
1628
- },
1629
- unknown: { status: 'unknown', verdict: 'hold', verified: false },
1630
- },
1631
- },
1632
- },
1633
- },
1634
- targetGetters: {
1635
- email: ['email', 'value'],
1636
- email_status: ['email_status'],
1637
- },
1638
- };
1639
- }
1640
- return { toolId };
1641
- }
1642
-
1643
2130
  function wrapWorkerToolResult(
1644
2131
  toolId: string,
1645
2132
  result: unknown,
1646
2133
  metadata: ToolResultMetadataInput | null,
1647
2134
  status = result == null ? 'no_result' : 'completed',
2135
+ customerDbDatasetOptions?: {
2136
+ req: RunRequest;
2137
+ input: Record<string, unknown>;
2138
+ },
1648
2139
  ): ToolExecuteResult {
1649
- return createToolExecuteResult({
2140
+ const wrapped = createToolExecuteResult({
1650
2141
  status,
1651
2142
  result,
1652
2143
  metadata: metadata ?? { toolId },
1653
2144
  execution: toolExecutionMetadataForOutcome({ kind: 'live' }),
1654
2145
  });
2146
+ return attachWorkerCustomerDbDatasetResult(
2147
+ toolId,
2148
+ wrapped,
2149
+ customerDbDatasetOptions,
2150
+ );
2151
+ }
2152
+
2153
+ function attachWorkerCustomerDbDatasetResult(
2154
+ toolId: string,
2155
+ result: ToolExecuteResult,
2156
+ options?: {
2157
+ req: RunRequest;
2158
+ input: Record<string, unknown>;
2159
+ },
2160
+ ): ToolExecuteResult {
2161
+ if (!options || !isQueryResultDatasetTool(toolId)) return result;
2162
+ const raw = isRecord(result.toolOutput.raw) ? result.toolOutput.raw : null;
2163
+ const dataset = isRecord(raw?.dataset) ? raw.dataset : null;
2164
+ const rows = rowsFromUnknown(raw?.rows);
2165
+ const totalRows = finiteNonNegativeInteger(dataset?.total_rows);
2166
+ const sql =
2167
+ typeof options.input.sql === 'string'
2168
+ ? options.input.sql
2169
+ : typeof options.input.query === 'string'
2170
+ ? options.input.query
2171
+ : typeof raw?.sql === 'string'
2172
+ ? raw.sql
2173
+ : null;
2174
+ if (!dataset || totalRows === null || !sql) {
2175
+ return result;
2176
+ }
2177
+
2178
+ const datasetLimit =
2179
+ finitePositiveInteger(dataset.returned_limit) ?? totalRows;
2180
+ const previewRows = rows.slice(0, Math.min(rows.length, 25));
2181
+ const jobId = typeof result.job_id === 'string' ? result.job_id.trim() : '';
2182
+ const datasetId = `worker-tool-list:${sha256Hex(
2183
+ `${options.req.runId}:${jobId || Math.random()}`,
2184
+ )}`;
2185
+ const fetchPage = async (
2186
+ offset: number,
2187
+ limit: number,
2188
+ ): Promise<Record<string, unknown>[]> => {
2189
+ if (limit <= 0 || offset >= totalRows) return [];
2190
+ const page = await callToolDirect(
2191
+ options.req,
2192
+ {
2193
+ id: 'customer-db-dataset-page',
2194
+ toolId,
2195
+ input: options.input,
2196
+ },
2197
+ undefined,
2198
+ undefined,
2199
+ false,
2200
+ undefined,
2201
+ undefined,
2202
+ undefined,
2203
+ {
2204
+ attachCustomerDbDataset: false,
2205
+ customerDbDataset: {
2206
+ limit: datasetLimit,
2207
+ offset,
2208
+ pageSize: Math.min(limit, QUERY_RESULT_DATASET_PAGE_SIZE),
2209
+ totalRows,
2210
+ },
2211
+ },
2212
+ );
2213
+ const pageRaw = isRecord(page.toolOutput.raw) ? page.toolOutput.raw : null;
2214
+ return rowsFromUnknown(pageRaw?.rows);
2215
+ };
2216
+ const collectRows = async (
2217
+ limit: number | undefined,
2218
+ ): Promise<Record<string, unknown>[]> => {
2219
+ const target = Math.min(limit ?? totalRows, totalRows, datasetLimit);
2220
+ const collected: Record<string, unknown>[] = [];
2221
+ for (
2222
+ let offset = 0;
2223
+ offset < target;
2224
+ offset += QUERY_RESULT_DATASET_PAGE_SIZE
2225
+ ) {
2226
+ collected.push(
2227
+ ...(await fetchPage(
2228
+ offset,
2229
+ Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, target - offset),
2230
+ )),
2231
+ );
2232
+ }
2233
+ return collected.slice(0, target);
2234
+ };
2235
+ const playDataset = createDeferredPlayDataset({
2236
+ datasetKind: 'csv',
2237
+ datasetId,
2238
+ count: Math.min(totalRows, datasetLimit),
2239
+ previewRows,
2240
+ sourceLabel: 'toolResponse.raw.rows',
2241
+ tableNamespace: null,
2242
+ resolvers: {
2243
+ count: async () => Math.min(totalRows, datasetLimit),
2244
+ peek: async (limit) => collectRows(limit),
2245
+ materialize: async (limit) => collectRows(limit),
2246
+ iterate: () =>
2247
+ ({
2248
+ async *[Symbol.asyncIterator]() {
2249
+ const count = Math.min(totalRows, datasetLimit);
2250
+ for (
2251
+ let offset = 0;
2252
+ offset < count;
2253
+ offset += QUERY_RESULT_DATASET_PAGE_SIZE
2254
+ ) {
2255
+ yield* await fetchPage(
2256
+ offset,
2257
+ Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, count - offset),
2258
+ );
2259
+ }
2260
+ },
2261
+ }) as AsyncIterable<Record<string, unknown>>,
2262
+ },
2263
+ });
2264
+ return attachToolResultListDataset(result, {
2265
+ name: 'rows',
2266
+ path: 'toolResponse.raw.rows',
2267
+ dataset: playDataset,
2268
+ count: Math.min(totalRows, datasetLimit),
2269
+ });
1655
2270
  }
1656
2271
 
1657
2272
  type RecordedStepProgramOutput = {
@@ -1675,46 +2290,10 @@ type WorkerStepResolution = {
1675
2290
  status?: 'skipped';
1676
2291
  };
1677
2292
 
1678
- type WorkerToolBatchRequest = {
1679
- id: string;
1680
- cacheKey: string;
1681
- receiptKey: string | null;
1682
- force: boolean;
1683
- receiptWaitMaxAttempts: number;
1684
- runtimeTimeoutMs?: number;
1685
- toolId: string;
1686
- input: Record<string, unknown>;
1687
- workflowStep?: WorkflowStep;
1688
- resolve: (value: unknown) => void;
1689
- reject: (error: unknown) => void;
1690
- };
1691
-
1692
- type ClaimedWorkerToolBatchRequest = {
1693
- request: WorkerToolBatchRequest;
1694
- receiptKey: string | null;
1695
- followers: WorkerToolBatchRequest[];
1696
- };
1697
-
1698
- type PreparedWorkerToolBatchRequests = {
1699
- claimedRequests: ClaimedWorkerToolBatchRequest[];
1700
- deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[];
1701
- };
1702
-
1703
- function resolveClaimedWorkerToolRuntimeTimeoutMs(
1704
- claimedRequests: ClaimedWorkerToolBatchRequest[],
1705
- input: { runtimeDeadlineMs?: number },
1706
- ): number | undefined {
1707
- return resolveWorkerToolRuntimeTimeoutMs(claimedRequests, {
1708
- resolveOwnerTimeoutMs: (request) =>
1709
- resolveToolRuntimeApiTimeout({
1710
- requestInput: request.input,
1711
- timeoutMs: request.runtimeTimeoutMs,
1712
- runtimeDeadlineMs: input.runtimeDeadlineMs,
1713
- }).timeoutMs,
1714
- });
2293
+ function stepProgramColumnName(parentField: string, stepId: string): string {
2294
+ return sqlSafePlayColumnName(`${parentField}.${stepId}`);
1715
2295
  }
1716
2296
 
1717
- const WORKER_TOOL_BATCH_GRACE_MS = 250;
1718
2297
  const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
1719
2298
  const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
1720
2299
  const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
@@ -1727,1371 +2306,67 @@ const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
1727
2306
  */
1728
2307
  const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
1729
2308
 
1730
- class RuntimeReceiptPersistenceError extends Error {
1731
- readonly errors: unknown[];
1732
-
1733
- constructor(message: string, errors: unknown[] = []) {
1734
- super(message);
1735
- this.name = 'RuntimeReceiptPersistenceError';
1736
- this.errors = errors;
1737
- }
1738
- }
1739
-
1740
- function isRuntimeReceiptPersistenceError(error: unknown): boolean {
1741
- if (!error || typeof error !== 'object') return false;
1742
- if (error instanceof RuntimeReceiptPersistenceError) return true;
1743
- if (
1744
- error instanceof Error &&
1745
- error.name === 'RuntimeReceiptPersistenceError'
1746
- ) {
1747
- return true;
1748
- }
1749
- const nestedErrors = (error as { errors?: unknown }).errors;
1750
- return (
1751
- Array.isArray(nestedErrors) &&
1752
- nestedErrors.some(isRuntimeReceiptPersistenceError)
1753
- );
1754
- }
1755
-
1756
- function isRunFatalWorkerRowError(error: unknown): boolean {
1757
- return (
1758
- isCoordinatorProgressAuthorizationError(error) ||
1759
- isRowIsolationExemptError(error) ||
1760
- isRuntimeReceiptPersistenceError(error) ||
1761
- isRuntimePersistenceCircuitOpenError(error)
1762
- );
1763
- }
1764
-
1765
- /**
1766
- * completeReceipt salvage retry ladder. On a persistence failure after a billed
1767
- * call we retry the receipt completion past the server-side 5s connect window:
1768
- * 1 initial attempt + 3 retries at 1s / 3s / 9s (single isolate — no jitter).
1769
- * Bounded by the run deadline (`runtimeDeadlineMs`); we stop early unless there
1770
- * is at least one backoff's worth plus headroom of budget left.
1771
- */
1772
- const COMPLETE_RECEIPT_RETRY_BACKOFFS_MS = [1_000, 3_000, 9_000] as const;
1773
- const COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS = 10_000;
1774
-
1775
- // Fallback batch-chunk parallelism when a tool declares no provider rate hints.
1776
- // Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
1777
- // their previous batching behavior; declared providers tighten via the
1778
- // Governor's suggestedParallelism.
1779
- const WORKER_TOOL_BATCH_DEFAULT_PARALLELISM = 4;
1780
-
1781
2309
  /**
1782
- * In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
1783
- * throughput pacing (provider or Deepline limiter), not a tool defect, so it
1784
- * gets more patience than the generic 3-attempt budget: with retry-after-aware
1785
- * escalating delays (capped at 5s) this absorbs roughly 25s of sustained
1786
- * throttling before the call fails. Every retry still charges the Governor's
1787
- * retry budget, so a runaway storm stays bounded and loud.
2310
+ * Bounded-concurrent map-chunk dispatch fan-out for the workers_edge profile
2311
+ * (in-process play execution with no `workflowStep`). On workers_edge the whole
2312
+ * map shares ONE per-invocation subrequest window, so sequential chunk dispatch
2313
+ * buys no durability it is pure wall-clock cost. Running up to K chunks in
2314
+ * flight overlaps provider batches, sheet writes, and the per-chunk persist
2315
+ * barrier. K is capped at 3 to bound peak memory: each in-flight chunk holds up
2316
+ * to ~5000 rows plus their row buffers, so 3 concurrent chunks is the ceiling
2317
+ * we accept before row payloads risk pressuring the isolate. The native
2318
+ * Cloudflare Workflows path (workflowStep present) keeps K=1: there each chunk
2319
+ * is a durable step and sequential dispatch is what makes replay/resume sound.
1788
2320
  */
2321
+ const WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS = 3;
2322
+ const WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES =
2323
+ WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES;
2324
+ const WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES = 128;
1789
2325
 
1790
- function sleepWorkerMs(ms: number): Promise<void> {
1791
- return new Promise((resolve) => setTimeout(resolve, ms));
1792
- }
1793
-
1794
- function workerDurableToolCallCacheKey(input: {
1795
- req: RunRequest;
1796
- toolId: string;
1797
- requestInput: Record<string, unknown>;
1798
- providerActionVersion: string;
1799
- staleAfterSeconds?: number | null;
1800
- }): string {
1801
- return buildDurableToolCallCacheKey({
1802
- orgId: input.req.orgId,
1803
- playId: input.req.playName,
1804
- toolId: input.toolId,
1805
- requestInput: input.requestInput,
1806
- authScopeDigest: buildDurableToolCallAuthScopeDigest({
1807
- orgId: input.req.orgId,
1808
- userEmail: input.req.userEmail,
1809
- toolId: input.toolId,
1810
- }),
1811
- providerActionVersion: input.providerActionVersion,
1812
- staleAfterSeconds: input.staleAfterSeconds,
1813
- });
1814
- }
1815
-
1816
- function workerRuntimeReceiptKey(input: {
1817
- req: RunRequest;
1818
- key: string;
1819
- }): string {
1820
- const orgId = input.req.orgId?.trim() || 'org';
1821
- if (input.key.startsWith(`ctx:${orgId}:`)) {
1822
- return input.key;
1823
- }
1824
- return buildScopedWorkReceiptKey({
1825
- orgId: input.req.orgId,
1826
- playName: input.req.playName,
1827
- key: input.key,
1828
- });
1829
- }
1830
-
1831
- function stepProgramColumnName(parentField: string, stepId: string): string {
1832
- return sqlSafePlayColumnName(`${parentField}.${stepId}`);
1833
- }
1834
-
1835
- class WorkerToolBatchScheduler {
1836
- private queue: WorkerToolBatchRequest[] = [];
1837
- private scheduled = false;
1838
-
1839
- constructor(
1840
- private readonly req: RunRequest,
1841
- private readonly governor: PlayExecutionGovernor,
1842
- private readonly resolvePacing: WorkerPacingResolver,
1843
- private readonly resolveToolActionCacheVersion: WorkerToolActionCacheVersionResolver,
1844
- private readonly abortSignal?: AbortSignal,
1845
- private readonly onRequestsSettled?: (count: number) => void,
1846
- private readonly callbacks?: WorkerCtxCallbacks,
1847
- private readonly receiptStore?: WorkerRuntimeReceiptStore,
1848
- private readonly allowLocalRetryReceipts = false,
1849
- private readonly runtimeDeadlineMs?: number,
1850
- // Map-wide (or ctx-wide for the root scheduler) persistence-failure circuit
1851
- // breaker. Shared by every row worker so the first persistence failure
1852
- // halts NEW provider dispatch across the whole map.
1853
- private readonly latch: RuntimePersistenceLatch = createRuntimePersistenceLatch(),
1854
- // Run-scoped buffer for orphaned billed results whose receipt could not be
1855
- // completed even after the retry ladder.
1856
- private readonly salvage?: ReceiptSalvageBuffer,
1857
- ) {}
1858
-
1859
- /**
1860
- * Run `completeReceipt`/`completeReceipts` with the salvage retry ladder.
1861
- * Retries a receipt-completion failure past the server-side 5s connect window
1862
- * so a transient blip does not orphan a billed result. Bounded by the run
1863
- * deadline; skips retries when the latch has already burned a ladder in this
1864
- * map so many in-flight orphans do not each serialize a long loop.
1865
- */
1866
- private async completeReceiptWithRetryLadder<T>(
1867
- run: () => Promise<T>,
1868
- onGiveUp?: (attempts: number) => void,
1869
- ): Promise<T> {
1870
- let attempt = 0;
1871
- // A ladder that has already started retrying keeps its full budget even if
1872
- // a peer trips the latch mid-flight — only FRESH entrants collapse once the
1873
- // failure is confirmed. This is what guarantees the first ladder rides out
1874
- // a genuine transient blip.
1875
- let committedToFullLadder = false;
1876
- const giveUp = (error: unknown): never => {
1877
- this.latch.latchRetryBudgetExhausted = true;
1878
- onGiveUp?.(attempt);
1879
- throw error;
1880
- };
1881
- while (true) {
1882
- try {
1883
- return await run();
1884
- } catch (error) {
1885
- attempt += 1;
1886
- if (
1887
- shouldShortenCompleteReceiptLadder(this.latch, committedToFullLadder) ||
1888
- attempt > COMPLETE_RECEIPT_RETRY_BACKOFFS_MS.length ||
1889
- this.abortSignal?.aborted
1890
- ) {
1891
- return giveUp(error);
1892
- }
1893
- const backoffMs = COMPLETE_RECEIPT_RETRY_BACKOFFS_MS[attempt - 1]!;
1894
- // resolveRuntimeDeadlineRemainingMs THROWS WorkflowAbortError once the
1895
- // run deadline has passed — the correct loud outcome (there is no
1896
- // separate per-chunk step timeout). Route that throw through giveUp so
1897
- // the billed result is still salvaged and the ladder budget flag is
1898
- // set: a billed output must never lose its durable trace because the
1899
- // clock ran out mid-ladder.
1900
- let remainingMs: number;
1901
- try {
1902
- remainingMs = resolveRuntimeDeadlineRemainingMs(
1903
- this.runtimeDeadlineMs,
1904
- );
1905
- } catch (deadlineError) {
1906
- return giveUp(deadlineError);
1907
- }
1908
- if (remainingMs <= backoffMs + COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS) {
1909
- return giveUp(error);
1910
- }
1911
- // Past every give-up gate: this ladder is now committed to the full
1912
- // budget and must not be collapsed by a peer tripping the latch mid-sleep.
1913
- committedToFullLadder = true;
1914
- await sleepWorkerMs(backoffMs);
1915
- }
1916
- }
1917
- }
1918
-
1919
- /**
1920
- * Record a billed-but-unpersisted tool output into the run-scoped salvage
1921
- * buffer. Called only after the completeReceipt ladder is exhausted, with the
1922
- * exact serialized payload a receipt would have stored.
1923
- */
1924
- private recordReceiptSalvage(input: {
1925
- claimed: ClaimedWorkerToolBatchRequest;
1926
- output: unknown;
1927
- attempts: number;
1928
- }): void {
1929
- if (!this.salvage || !input.claimed.receiptKey) return;
1930
- this.salvage.push({
1931
- receiptKey: input.claimed.receiptKey,
1932
- toolId: input.claimed.request.toolId,
1933
- cacheKey: input.claimed.request.cacheKey,
1934
- output: input.output,
1935
- completeAttempts: input.attempts,
1936
- firstErrorAt: nowMs(),
1937
- });
1938
- }
1939
-
1940
- /**
1941
- * Report a provider 429 / Retry-After back into the Governor's shared pacer
1942
- * so future acquires for this (org, provider) bucket back off across all
1943
- * isolates. Provider comes from the same pacing resolver the Governor uses
1944
- * (the worker has no local catalog), so callers pass only the toolId.
1945
- */
1946
- private reportBackpressure(toolId: string, retryAfterMs: number): void {
1947
- if (retryAfterMs <= 0) return;
1948
- void (async () => {
1949
- const pacing = await this.resolvePacing(toolId).catch(() => null);
1950
- if (pacing?.provider) {
1951
- this.governor.reportProviderBackpressure({
1952
- provider: pacing.provider,
1953
- retryAfterMs,
1954
- });
1955
- }
1956
- })();
1957
- }
1958
-
1959
- async execute(
1960
- id: string,
1961
- toolId: string,
1962
- input: Record<string, unknown>,
1963
- workflowStep?: WorkflowStep,
1964
- options?: { force?: boolean; staleAfterSeconds?: number | null },
1965
- runtimeOptions?: { timeoutMs?: number; receiptWaitMs?: number },
1966
- ): Promise<unknown> {
1967
- const providerActionVersion =
1968
- await this.resolveToolActionCacheVersion(toolId);
1969
- return await new Promise((resolve, reject) => {
1970
- const queuedAt = nowMs();
1971
- const receiptKey = workerDurableToolCallCacheKey({
1972
- req: this.req,
1973
- toolId,
1974
- requestInput: input,
1975
- providerActionVersion,
1976
- staleAfterSeconds: options?.staleAfterSeconds,
1977
- });
1978
- this.queue.push({
1979
- id,
1980
- cacheKey: receiptKey,
1981
- receiptKey,
1982
- force: options?.force === true || !!this.req.force,
1983
- receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
1984
- typeof runtimeOptions?.receiptWaitMs === 'number'
1985
- ? { max_wait_ms: runtimeOptions.receiptWaitMs }
1986
- : input,
1987
- ),
1988
- ...(typeof runtimeOptions?.timeoutMs === 'number'
1989
- ? { runtimeTimeoutMs: runtimeOptions.timeoutMs }
1990
- : {}),
1991
- toolId,
1992
- input,
1993
- workflowStep,
1994
- resolve: (value) => {
1995
- recordRunnerPerfTrace({
1996
- req: this.req,
1997
- phase: 'runner.tool.request',
1998
- ms: nowMs() - queuedAt,
1999
- extra: { id, toolId },
2000
- });
2001
- resolve(value);
2002
- },
2003
- reject,
2004
- });
2005
- this.scheduleDrain();
2006
- });
2007
- }
2008
-
2009
- private scheduleDrain(): void {
2010
- if (this.scheduled) return;
2011
- this.scheduled = true;
2012
- queueMicrotask(async () => {
2013
- if (this.hasBatchableQueuedTool()) {
2014
- await new Promise((resolve) =>
2015
- setTimeout(resolve, WORKER_TOOL_BATCH_GRACE_MS),
2016
- );
2017
- }
2018
- void this.drain();
2019
- });
2020
- }
2021
-
2022
- private hasBatchableQueuedTool(): boolean {
2023
- return this.queue.some(
2024
- (request) =>
2025
- request.toolId !== 'test_wait_for_event' &&
2026
- getPlayRuntimeBatchStrategy(request.toolId) !== null,
2027
- );
2028
- }
2029
-
2030
- private async drain(): Promise<void> {
2031
- const requests = this.queue;
2032
- this.queue = [];
2033
- this.scheduled = false;
2034
- const drainStartedAt = nowMs();
2035
- await Promise.all(
2036
- [...groupWorkerToolRequestsByTool(requests).entries()].map(
2037
- async ([toolId, groupedRequests]) => {
2038
- await this.executeToolGroup(toolId, groupedRequests);
2039
- },
2040
- ),
2041
- );
2042
- recordRunnerPerfTrace({
2043
- req: this.req,
2044
- phase: 'runner.tool.drain',
2045
- ms: nowMs() - drainStartedAt,
2046
- extra: { requests: requests.length },
2047
- });
2048
- if (this.queue.length > 0) {
2049
- this.scheduleDrain();
2050
- }
2051
- }
2052
-
2053
- private async waitForDurableToolReceipt(input: {
2054
- receiptKey: string;
2055
- maxAttempts: number;
2056
- }): Promise<WorkerRuntimeReceipt> {
2057
- if (!this.receiptStore?.getReceipt) {
2058
- throw new Error(
2059
- 'Worker durable tool receipt wait requires receipt lookup.',
2060
- );
2061
- }
2062
- const receipt = await waitForCompletedRuntimeReceipt({
2063
- receiptKey: input.receiptKey,
2064
- maxAttempts: input.maxAttempts,
2065
- abortSignal: this.abortSignal,
2066
- store: {
2067
- getMany: async (receiptKeys) => {
2068
- const receipts = new Map<string, RuntimeStepReceipt>();
2069
- await Promise.all(
2070
- receiptKeys.map(async (key) => {
2071
- const receipt = await this.receiptStore!.getReceipt!({
2072
- playName: this.req.playName,
2073
- key,
2074
- });
2075
- if (!receipt) return;
2076
- receipts.set(key, {
2077
- key: receipt.key,
2078
- status: receipt.status,
2079
- output: receipt.output,
2080
- error: receipt.error ?? undefined,
2081
- runId: receipt.runId ?? null,
2082
- });
2083
- }),
2084
- );
2085
- return receipts;
2086
- },
2087
- },
2088
- });
2089
- return receipt;
2090
- }
2091
-
2092
- private settleRequests(
2093
- claimed: ClaimedWorkerToolBatchRequest,
2094
- result: unknown,
2095
- ): void {
2096
- claimed.request.resolve(result);
2097
- for (const follower of claimed.followers) {
2098
- follower.resolve(
2099
- claimed.receiptKey
2100
- ? markWorkerToolReceiptResultExecution(result, {
2101
- kind: 'in_flight',
2102
- receiptKey: claimed.receiptKey,
2103
- attachedToReceiptKey: claimed.receiptKey,
2104
- })
2105
- : result,
2106
- );
2107
- }
2108
- this.onRequestsSettled?.(1 + claimed.followers.length);
2109
- }
2110
-
2111
- private rejectRequests(
2112
- claimed: ClaimedWorkerToolBatchRequest,
2113
- error: unknown,
2114
- ): void {
2115
- claimed.request.reject(error);
2116
- for (const follower of claimed.followers) {
2117
- follower.reject(error);
2118
- }
2119
- this.onRequestsSettled?.(1 + claimed.followers.length);
2120
- }
2121
-
2122
- private rejectRawRequests(
2123
- requests: WorkerToolBatchRequest[],
2124
- error: unknown,
2125
- ): void {
2126
- for (const request of requests) {
2127
- request.reject(error);
2128
- }
2129
- this.onRequestsSettled?.(requests.length);
2130
- }
2131
-
2132
- private async resolveCompletedDurableToolReceiptGroup(input: {
2133
- group: WorkerToolBatchRequest[];
2134
- receiptKey: string;
2135
- receipt: WorkerRuntimeReceipt;
2136
- source: 'cache' | 'in_flight';
2137
- }): Promise<void> {
2138
- const [first] = input.group;
2139
- if (!first) return;
2140
- const value = deserializeDurableStepValue(input.receipt.output);
2141
- const result =
2142
- input.source === 'cache'
2143
- ? markWorkerToolReceiptResultCached(
2144
- value,
2145
- first.cacheKey,
2146
- input.receiptKey,
2147
- )
2148
- : markWorkerToolReceiptResultExecution(value, {
2149
- kind: 'in_flight',
2150
- receiptKey: input.receiptKey,
2151
- attachedToReceiptKey: input.receiptKey,
2152
- });
2153
- for (const request of input.group) {
2154
- request.resolve(result);
2155
- }
2156
- this.onRequestsSettled?.(input.group.length);
2157
- }
2158
-
2159
- private async reclaimRunningDurableToolReceiptGroupAfterWait(input: {
2160
- group: WorkerToolBatchRequest[];
2161
- receiptKey: string;
2162
- runningReceipt: WorkerRuntimeReceipt;
2163
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
2164
- const [request, ...followers] = input.group;
2165
- if (!request || !this.receiptStore) {
2166
- this.rejectRawRequests(
2167
- input.group,
2168
- new RuntimeReceiptWaitTimeoutError(input.receiptKey),
2169
- );
2170
- return [];
2171
- }
2172
- // A `running` receipt left by a DIFFERENT run whose last write is stale will
2173
- // never be completed by a live worker. Polling it burns one service-binding
2174
- // subrequest per tick, up to the receipt's wait budget (~1320) — many such
2175
- // orphans on a resume exhaust Cloudflare's per-invocation subrequest budget
2176
- // and kill the run. The orchestrator reclaims those immediately (skipping the
2177
- // poll) and re-executes, matching the "on resume it reclaims and re-executes"
2178
- // contract; live/fresh/same-run owners still poll before reclaiming. Kept as
2179
- // an injected helper so the skip-poll decision is unit-testable without
2180
- // loading this cloudflare:workers Worker entrypoint.
2181
- return await reclaimRunningReceiptGroupAfterWait<ClaimedWorkerToolBatchRequest>(
2182
- {
2183
- ownership: {
2184
- ownerRunId: input.runningReceipt.runId,
2185
- currentRunId: this.req.runId,
2186
- updatedAt: input.runningReceipt.updatedAt,
2187
- },
2188
- timeoutError: new RuntimeReceiptWaitTimeoutError(input.receiptKey),
2189
- poll: async () => {
2190
- try {
2191
- const receipt = await this.waitForDurableToolReceipt({
2192
- receiptKey: input.receiptKey,
2193
- maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
2194
- input.group,
2195
- (groupRequest) => groupRequest.receiptWaitMaxAttempts,
2196
- ),
2197
- });
2198
- await this.resolveCompletedDurableToolReceiptGroup({
2199
- group: input.group,
2200
- receiptKey: input.receiptKey,
2201
- receipt,
2202
- source: 'in_flight',
2203
- });
2204
- return { kind: 'completed' };
2205
- } catch (error) {
2206
- if (error instanceof RuntimeReceiptWaitTimeoutError) {
2207
- return { kind: 'timed_out', error };
2208
- }
2209
- return { kind: 'errored', error };
2210
- }
2211
- },
2212
- reclaim: (rejectionOnFallthrough) =>
2213
- this.forceReclaimRunningDurableToolReceiptGroup({
2214
- group: input.group,
2215
- request,
2216
- followers,
2217
- receiptKey: input.receiptKey,
2218
- rejectionOnFallthrough,
2219
- }),
2220
- reject: (error) => this.rejectRawRequests(input.group, error),
2221
- },
2222
- );
2223
- }
2224
-
2225
- /**
2226
- * Steal a `running` receipt (reclaimRunning) and route by claim disposition.
2227
- * Shared by the post-poll-timeout reclaim and the dead-owner fast path, so
2228
- * both dispose of a reclaimed/reused/failed receipt identically.
2229
- */
2230
- private async forceReclaimRunningDurableToolReceiptGroup(input: {
2231
- group: WorkerToolBatchRequest[];
2232
- request: WorkerToolBatchRequest;
2233
- followers: WorkerToolBatchRequest[];
2234
- receiptKey: string;
2235
- rejectionOnFallthrough: unknown;
2236
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
2237
- if (!this.receiptStore) {
2238
- this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
2239
- return [];
2240
- }
2241
- let claim: WorkerRuntimeReceiptClaim;
2242
- try {
2243
- claim = await this.receiptStore.claimReceipt({
2244
- playName: this.req.playName,
2245
- runId: this.req.runId,
2246
- key: input.receiptKey,
2247
- reclaimRunning: true,
2248
- });
2249
- } catch (error) {
2250
- this.rejectRawRequests(input.group, error);
2251
- return [];
2252
- }
2253
- if (claim.disposition === 'claimed') {
2254
- return [
2255
- {
2256
- request: input.request,
2257
- receiptKey: input.receiptKey,
2258
- followers: input.followers,
2259
- },
2260
- ];
2261
- }
2262
- if (claim.disposition === 'reused') {
2263
- await this.resolveCompletedDurableToolReceiptGroup({
2264
- group: input.group,
2265
- receiptKey: input.receiptKey,
2266
- receipt: claim.receipt,
2267
- source: 'cache',
2268
- });
2269
- return [];
2270
- }
2271
- if (claim.disposition === 'failed') {
2272
- return await this.reclaimFailedDurableToolReceiptGroup({
2273
- group: input.group,
2274
- receiptKey: input.receiptKey,
2275
- failedReceipt: claim.receipt,
2276
- });
2277
- }
2278
- this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
2279
- return [];
2280
- }
2281
-
2282
- private async reclaimFailedDurableToolReceiptGroup(input: {
2283
- group: WorkerToolBatchRequest[];
2284
- receiptKey: string;
2285
- failedReceipt: WorkerRuntimeReceipt;
2286
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
2287
- const failedError = new Error(
2288
- `Durable tool call ${input.receiptKey} failed: ${input.failedReceipt.error ?? 'unknown error'}`,
2289
- );
2290
- const [request, ...followers] = input.group;
2291
- if (!request || !this.receiptStore) {
2292
- this.rejectRawRequests(input.group, failedError);
2293
- return [];
2294
- }
2295
- if (
2296
- !canReclaimFailedWorkerToolReceipt({
2297
- error: input.failedReceipt.error,
2298
- })
2299
- ) {
2300
- this.rejectRawRequests(input.group, failedError);
2301
- return [];
2302
- }
2303
- let claim: WorkerRuntimeReceiptClaim;
2304
- try {
2305
- claim = await this.receiptStore.claimReceipt({
2306
- playName: this.req.playName,
2307
- runId: this.req.runId,
2308
- key: input.receiptKey,
2309
- });
2310
- } catch (error) {
2311
- this.rejectRawRequests(input.group, error);
2312
- return [];
2313
- }
2314
- if (claim.disposition === 'claimed') {
2315
- return [{ request, receiptKey: input.receiptKey, followers }];
2316
- }
2317
- if (claim.disposition === 'reused') {
2318
- await this.resolveCompletedDurableToolReceiptGroup({
2319
- group: input.group,
2320
- receiptKey: input.receiptKey,
2321
- receipt: claim.receipt,
2322
- source: 'cache',
2323
- });
2324
- return [];
2325
- }
2326
- if (claim.disposition === 'running') {
2327
- try {
2328
- const receipt = await this.waitForDurableToolReceipt({
2329
- receiptKey: input.receiptKey,
2330
- maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
2331
- input.group,
2332
- (groupRequest) => groupRequest.receiptWaitMaxAttempts,
2333
- ),
2334
- });
2335
- await this.resolveCompletedDurableToolReceiptGroup({
2336
- group: input.group,
2337
- receiptKey: input.receiptKey,
2338
- receipt,
2339
- source: 'in_flight',
2340
- });
2341
- } catch (error) {
2342
- this.rejectRawRequests(input.group, error);
2343
- }
2344
- return [];
2345
- }
2346
- this.rejectRawRequests(
2347
- input.group,
2348
- new Error(
2349
- `Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
2350
- ),
2351
- );
2352
- return [];
2353
- }
2354
-
2355
- private async failureForRejectedToolRequest(
2356
- claimed: ClaimedWorkerToolBatchRequest,
2357
- error: unknown,
2358
- ): Promise<unknown> {
2359
- try {
2360
- await this.failDurableToolRequest(claimed, error);
2361
- return error;
2362
- } catch (receiptError) {
2363
- tripRuntimePersistenceLatch(this.latch, receiptError);
2364
- return new RuntimeReceiptPersistenceError(
2365
- 'Tool call failed and durable receipt could not be marked failed',
2366
- [error, receiptError],
2367
- );
2368
- }
2369
- }
2370
-
2371
- private async claimDurableToolReceiptGroups(
2372
- groups: ReturnType<
2373
- typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
2374
- >['durableGroups'],
2375
- ): Promise<
2376
- Array<{
2377
- group: ReturnType<
2378
- typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
2379
- >['durableGroups'][number];
2380
- receiptKey: string;
2381
- claim: WorkerRuntimeReceiptClaim;
2382
- }>
2383
- > {
2384
- if (!this.receiptStore) return [];
2385
- const planned = groups.map((group) => ({
2386
- group,
2387
- receiptKey: workerRuntimeReceiptKey({
2388
- req: this.req,
2389
- key: group.claimableReceiptKey,
2390
- }),
2391
- }));
2392
- const claimOne = async (entry: (typeof planned)[number]) => ({
2393
- ...entry,
2394
- claim: await this.receiptStore!.claimReceipt({
2395
- playName: this.req.playName,
2396
- runId: this.req.runId,
2397
- key: entry.receiptKey,
2398
- ...(entry.group.forceDurableRefresh
2399
- ? { forceRefresh: true, reclaimRunning: true }
2400
- : {}),
2401
- }),
2402
- });
2403
- if (!this.receiptStore.claimReceipts) {
2404
- return await Promise.all(planned.map(claimOne));
2405
- }
2406
- const claimed: Array<{
2407
- group: (typeof planned)[number]['group'];
2408
- receiptKey: string;
2409
- claim: WorkerRuntimeReceiptClaim;
2410
- }> = [];
2411
- for (const forceDurableRefresh of [false, true]) {
2412
- const entries = planned.filter(
2413
- (entry) => entry.group.forceDurableRefresh === forceDurableRefresh,
2414
- );
2415
- if (entries.length === 0) continue;
2416
- const claims = await this.receiptStore.claimReceipts({
2417
- playName: this.req.playName,
2418
- runId: this.req.runId,
2419
- keys: entries.map((entry) => entry.receiptKey),
2420
- ...(forceDurableRefresh
2421
- ? { forceRefresh: true, reclaimRunning: true }
2422
- : {}),
2423
- });
2424
- entries.forEach((entry, index) => {
2425
- const claim = claims[index];
2426
- if (!claim) {
2427
- throw new Error(
2428
- `Runtime receipt batch claim did not return receipt ${entry.receiptKey}.`,
2429
- );
2430
- }
2431
- claimed.push({ ...entry, claim });
2432
- });
2433
- }
2434
- return claimed;
2435
- }
2436
-
2437
- private async prepareDurableToolRequests(
2438
- requests: WorkerToolBatchRequest[],
2439
- ): Promise<PreparedWorkerToolBatchRequests> {
2440
- if (!this.receiptStore) {
2441
- return {
2442
- claimedRequests: requests.map((request) => ({
2443
- request,
2444
- receiptKey: null,
2445
- followers: [],
2446
- })),
2447
- deferredClaimedRequests: [],
2448
- };
2449
- }
2450
-
2451
- const claimedRequests: ClaimedWorkerToolBatchRequest[] = [];
2452
- const deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[] =
2453
- [];
2454
- const receiptGroupPlan = planWorkerToolReceiptGroups(requests, {
2455
- allowLocalRetryReceipts: this.allowLocalRetryReceipts,
2456
- getReceiptInput: (request) => ({
2457
- durableReceiptKey: request.receiptKey,
2458
- localRetryCacheKey: request.cacheKey,
2459
- force: request.force,
2460
- }),
2461
- });
2462
- for (const request of receiptGroupPlan.localRequests) {
2463
- claimedRequests.push({
2464
- request,
2465
- receiptKey: null,
2466
- followers: [],
2467
- });
2468
- }
2469
-
2470
- try {
2471
- const claimEntries = await this.claimDurableToolReceiptGroups(
2472
- receiptGroupPlan.durableGroups,
2473
- );
2474
- for (const { group: groupState, receiptKey, claim } of claimEntries) {
2475
- const group = groupState.requests;
2476
- const [first] = group;
2477
- if (!first) continue;
2478
- if (claim.disposition === 'reused') {
2479
- const result = markWorkerToolReceiptResultCached(
2480
- deserializeDurableStepValue(claim.receipt.output),
2481
- first.cacheKey,
2482
- receiptKey,
2483
- );
2484
- for (const request of group) {
2485
- request.resolve(result);
2486
- }
2487
- this.onRequestsSettled?.(group.length);
2488
- continue;
2489
- }
2490
- if (claim.disposition === 'failed') {
2491
- claimedRequests.push(
2492
- ...(await this.reclaimFailedDurableToolReceiptGroup({
2493
- group,
2494
- receiptKey,
2495
- failedReceipt: claim.receipt,
2496
- })),
2497
- );
2498
- continue;
2499
- }
2500
- if (claim.disposition === 'running') {
2501
- deferredClaimedRequests.push(
2502
- this.reclaimRunningDurableToolReceiptGroupAfterWait({
2503
- group,
2504
- receiptKey,
2505
- runningReceipt: claim.receipt,
2506
- }),
2507
- );
2508
- continue;
2509
- }
2510
- const [request, ...followers] = group;
2511
- if (!request) continue;
2512
- claimedRequests.push({ request, receiptKey, followers });
2513
- }
2514
- } catch (error) {
2515
- // A receipt claim failure means the tenant Postgres is already
2516
- // unwritable: trip the breaker so remaining groups/rows stop dispatching.
2517
- tripRuntimePersistenceLatch(this.latch, error);
2518
- for (const group of receiptGroupPlan.durableGroups) {
2519
- this.rejectRawRequests(group.requests, error);
2520
- }
2521
- }
2522
- return { claimedRequests, deferredClaimedRequests };
2523
- }
2524
-
2525
- private async completeDurableToolRequest(
2526
- claimed: ClaimedWorkerToolBatchRequest,
2527
- result: unknown,
2528
- ): Promise<unknown> {
2529
- if (!this.receiptStore || !claimed.receiptKey) {
2530
- return result;
2531
- }
2532
- const ownerResult = markWorkerToolReceiptResultExecution(result, {
2533
- kind: 'live',
2534
- receiptKey: claimed.receiptKey,
2535
- });
2536
- const serializedOutput = serializeDurableStepValue(ownerResult);
2537
- const completed = await this.completeReceiptWithRetryLadder(
2538
- () =>
2539
- this.receiptStore!.completeReceipt({
2540
- playName: this.req.playName,
2541
- runId: this.req.runId,
2542
- key: claimed.receiptKey!,
2543
- output: serializedOutput,
2544
- }),
2545
- // Ladder exhausted: the call billed but the receipt could not land.
2546
- // Salvage the exact serialized output onto the run record before the
2547
- // caller turns this into a RuntimeReceiptPersistenceError.
2548
- (attempts) =>
2549
- this.recordReceiptSalvage({
2550
- claimed,
2551
- output: serializedOutput,
2552
- attempts,
2553
- }),
2554
- );
2555
- if (
2556
- completed &&
2557
- (completed.status === 'completed' || completed.status === 'skipped') &&
2558
- completed.output !== undefined
2559
- ) {
2560
- const recovered = deserializeDurableStepValue(completed.output);
2561
- return completed.runId && completed.runId !== this.req.runId
2562
- ? markWorkerToolReceiptResultCached(
2563
- recovered,
2564
- claimed.request.cacheKey,
2565
- claimed.receiptKey,
2566
- )
2567
- : recovered;
2568
- }
2569
- return ownerResult;
2570
- }
2571
-
2572
- private completedDurableToolRequestResult(input: {
2573
- claimed: ClaimedWorkerToolBatchRequest;
2574
- ownerResult: unknown;
2575
- completed: WorkerRuntimeReceipt | null | undefined;
2576
- }): unknown {
2577
- const { claimed, completed, ownerResult } = input;
2578
- if (
2579
- completed &&
2580
- (completed.status === 'completed' || completed.status === 'skipped') &&
2581
- completed.output !== undefined
2582
- ) {
2583
- const recovered = deserializeDurableStepValue(completed.output);
2584
- return completed.runId && completed.runId !== this.req.runId
2585
- ? markWorkerToolReceiptResultCached(
2586
- recovered,
2587
- claimed.request.cacheKey,
2588
- claimed.receiptKey ?? claimed.request.cacheKey,
2589
- )
2590
- : recovered;
2591
- }
2592
- return ownerResult;
2593
- }
2594
-
2595
- private async completeDurableToolRequests(
2596
- entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
2597
- ): Promise<unknown[]> {
2598
- const results: unknown[] = new Array(entries.length);
2599
- const durableEntries: Array<{
2600
- index: number;
2601
- claimed: ClaimedWorkerToolBatchRequest;
2602
- ownerResult: unknown;
2603
- }> = [];
2604
- for (let index = 0; index < entries.length; index += 1) {
2605
- const entry = entries[index]!;
2606
- if (!this.receiptStore || !entry.claimed.receiptKey) {
2607
- results[index] = entry.result;
2608
- continue;
2609
- }
2610
- const ownerResult = markWorkerToolReceiptResultExecution(entry.result, {
2611
- kind: 'live',
2612
- receiptKey: entry.claimed.receiptKey,
2613
- });
2614
- durableEntries.push({ index, claimed: entry.claimed, ownerResult });
2615
- }
2616
- if (durableEntries.length === 0) {
2617
- return results;
2618
- }
2619
- if (!this.receiptStore?.completeReceipts) {
2620
- await Promise.all(
2621
- durableEntries.map(async (entry) => {
2622
- results[entry.index] = await this.completeDurableToolRequest(
2623
- entry.claimed,
2624
- entries[entry.index]!.result,
2625
- );
2626
- }),
2627
- );
2628
- return results;
2629
- }
2630
- const serializedByEntry = durableEntries.map((entry) =>
2631
- serializeDurableStepValue(entry.ownerResult),
2632
- );
2633
- const completed = await this.completeReceiptWithRetryLadder(
2634
- () =>
2635
- this.receiptStore!.completeReceipts!({
2636
- playName: this.req.playName,
2637
- receipts: durableEntries.map((entry, entryIndex) => ({
2638
- runId: this.req.runId,
2639
- key: entry.claimed.receiptKey!,
2640
- output: serializedByEntry[entryIndex],
2641
- })),
2642
- }),
2643
- // Ladder exhausted: every entry in this batch billed but could not be
2644
- // marked completed. Salvage each serialized output before the caller
2645
- // turns this into a RuntimeReceiptPersistenceError.
2646
- (attempts) => {
2647
- durableEntries.forEach((entry, entryIndex) => {
2648
- this.recordReceiptSalvage({
2649
- claimed: entry.claimed,
2650
- output: serializedByEntry[entryIndex],
2651
- attempts,
2652
- });
2653
- });
2654
- },
2655
- );
2656
- durableEntries.forEach((entry, resultIndex) => {
2657
- results[entry.index] = this.completedDurableToolRequestResult({
2658
- claimed: entry.claimed,
2659
- ownerResult: entry.ownerResult,
2660
- completed: completed[resultIndex],
2661
- });
2662
- });
2663
- return results;
2664
- }
2665
-
2666
- private async failDurableToolRequest(
2667
- claimed: ClaimedWorkerToolBatchRequest,
2668
- error: unknown,
2669
- ): Promise<void> {
2670
- if (!this.receiptStore || !claimed.receiptKey) {
2671
- return;
2672
- }
2673
- await this.receiptStore.failReceipt({
2674
- playName: this.req.playName,
2675
- runId: this.req.runId,
2676
- key: claimed.receiptKey,
2677
- error: error instanceof Error ? error.message : String(error),
2678
- });
2679
- }
2680
-
2681
- private async failDurableToolRequests(
2682
- claimedRequests: ClaimedWorkerToolBatchRequest[],
2683
- error: unknown,
2684
- ): Promise<void> {
2685
- const durable = claimedRequests.filter(
2686
- (claimed) => this.receiptStore && claimed.receiptKey,
2687
- );
2688
- if (durable.length === 0) return;
2689
- if (!this.receiptStore?.failReceipts) {
2690
- await Promise.all(
2691
- durable.map((claimed) => this.failDurableToolRequest(claimed, error)),
2692
- );
2693
- return;
2694
- }
2695
- await this.receiptStore.failReceipts({
2696
- playName: this.req.playName,
2697
- receipts: durable.map((claimed) => ({
2698
- runId: this.req.runId,
2699
- key: claimed.receiptKey!,
2700
- error: error instanceof Error ? error.message : String(error),
2701
- })),
2702
- });
2703
- }
2704
-
2705
- private async executeToolGroup(
2706
- toolId: string,
2707
- requests: WorkerToolBatchRequest[],
2708
- ): Promise<void> {
2709
- // Circuit breaker, gated BEFORE the receipt claim: never-dispatched rows
2710
- // get NO receipt (not even `running`), so a re-run claims them fresh and
2711
- // executes them exactly once.
2712
- if (this.latch.tripped) {
2713
- this.latch.preventedCallCount += requests.length;
2714
- this.rejectRawRequests(
2715
- requests,
2716
- new RuntimePersistenceCircuitOpenError(this.latch),
2717
- );
2718
- return;
2719
- }
2720
- const { claimedRequests, deferredClaimedRequests } =
2721
- await this.prepareDurableToolRequests(requests);
2722
- await this.executeClaimedToolRequests(
2723
- toolId,
2724
- requests.length,
2725
- claimedRequests,
2726
- );
2727
- if (deferredClaimedRequests.length === 0) {
2728
- return;
2729
- }
2730
- const reclaimedRequests = (
2731
- await Promise.all(deferredClaimedRequests)
2732
- ).flat();
2733
- await this.executeClaimedToolRequests(
2734
- toolId,
2735
- requests.length,
2736
- reclaimedRequests,
2737
- );
2738
- }
2739
-
2740
- private async executeClaimedToolRequests(
2741
- toolId: string,
2742
- requestCount: number,
2743
- claimedRequests: ClaimedWorkerToolBatchRequest[],
2744
- ): Promise<void> {
2745
- if (claimedRequests.length === 0) {
2746
- return;
2747
- }
2748
- const strategy = getPlayRuntimeBatchStrategy(toolId);
2749
- if (
2750
- !strategy ||
2751
- toolId === 'test_wait_for_event' ||
2752
- claimedRequests.length < 2
2753
- ) {
2754
- const groupStartedAt = nowMs();
2755
- const dispatchOne = async (
2756
- claimed: ClaimedWorkerToolBatchRequest,
2757
- ): Promise<void> => {
2758
- const { request } = claimed;
2759
- // Circuit breaker: the latch may have tripped after this group was
2760
- // claimed but before this particular call started. Do NOT call the
2761
- // provider. This row has a `running` receipt (claimed, never
2762
- // executed); on resume it reclaims and re-executes — safe, it never
2763
- // billed.
2764
- if (this.latch.tripped) {
2765
- this.latch.preventedCallCount += 1 + claimed.followers.length;
2766
- this.rejectRequests(
2767
- claimed,
2768
- new RuntimePersistenceCircuitOpenError(this.latch),
2769
- );
2770
- return;
2771
- }
2772
- const toolContract = await this.resolvePacing(toolId).catch(
2773
- () => null,
2774
- );
2775
- // Each unbatched provider call takes its own tool slot: the Governor
2776
- // charges tool budget, holds a global tool-concurrency slot, and
2777
- // applies per-(org,provider) pacing before the call runs.
2778
- const slot = await this.governor.acquireToolSlot(toolId, {
2779
- signal: this.abortSignal,
2780
- });
2781
- try {
2782
- let result: unknown;
2783
- try {
2784
- result = await executeToolWithLifecycle(
2785
- this.req,
2786
- { id: request.id, toolId, input: request.input },
2787
- request.workflowStep,
2788
- this.callbacks,
2789
- (retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
2790
- () => this.governor.chargeBudget('retry'),
2791
- toolContract?.retrySafeTransientHttp === true,
2792
- this.abortSignal,
2793
- this.runtimeDeadlineMs,
2794
- resolveClaimedWorkerToolRuntimeTimeoutMs([claimed], {
2795
- runtimeDeadlineMs: this.runtimeDeadlineMs,
2796
- }),
2797
- );
2798
- } catch (error) {
2799
- this.rejectRequests(
2800
- claimed,
2801
- await this.failureForRejectedToolRequest(claimed, error),
2802
- );
2803
- return;
2804
- }
2805
- this.settleRequests(
2806
- claimed,
2807
- await this.completeDurableToolRequest(claimed, result),
2808
- );
2809
- } catch (receiptError) {
2810
- tripRuntimePersistenceLatch(this.latch, receiptError);
2811
- this.rejectRequests(
2812
- claimed,
2813
- new RuntimeReceiptPersistenceError(
2814
- 'Tool call succeeded but durable receipt could not be marked completed',
2815
- [receiptError],
2816
- ),
2817
- );
2818
- } finally {
2819
- slot.release();
2820
- }
2821
- };
2822
- // Bounded dispatch waves: an unbatched tool group can carry a whole chunk's
2823
- // rows (enrich compiles to one ~245-row chunk). Dispatching them all at
2824
- // once billed every call before the first receipt-completion failure could
2825
- // trip the latch (the 250/250 incident). Instead, dispatch one wave, AWAIT
2826
- // it so any completion failure trips the latch, then read the latch before
2827
- // the next wave — remaining waves are prevented (counted + rejected) exactly
2828
- // like the mid-wave gate above. The wave calls stay fully concurrent; the
2829
- // only added cost on the healthy path is one latch read per wave.
2830
- for (const wave of partitionDispatchWaves(
2831
- claimedRequests,
2832
- () => 1,
2833
- PROVIDER_DISPATCH_WAVE_SIZE,
2834
- )) {
2835
- if (this.latch.tripped) {
2836
- for (const claimed of wave) {
2837
- this.latch.preventedCallCount += 1 + claimed.followers.length;
2838
- this.rejectRequests(
2839
- claimed,
2840
- new RuntimePersistenceCircuitOpenError(this.latch),
2841
- );
2842
- }
2843
- continue;
2844
- }
2845
- await Promise.all(wave.map(dispatchOne));
2846
- }
2847
- recordRunnerPerfTrace({
2848
- req: this.req,
2849
- phase: 'runner.tool.group',
2850
- ms: nowMs() - groupStartedAt,
2851
- extra: {
2852
- toolId,
2853
- requests: requestCount,
2854
- executed: claimedRequests.length,
2855
- batched: false,
2856
- },
2857
- });
2858
- return;
2859
- }
2326
+ const workerMapResidentRowBytes = (row: unknown): number =>
2327
+ new TextEncoder().encode(JSON.stringify(row)).byteLength +
2328
+ WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES;
2860
2329
 
2861
- const batchStartedAt = nowMs();
2862
- await executeBatchedWorkerToolGroup({
2863
- req: this.req,
2864
- requests: claimedRequests,
2865
- strategy,
2866
- governor: this.governor,
2867
- suggestedParallelism: await this.governor.suggestedParallelism(
2868
- toolId,
2869
- WORKER_TOOL_BATCH_DEFAULT_PARALLELISM,
2870
- ),
2871
- abortSignal: this.abortSignal,
2872
- runtimeDeadlineMs: this.runtimeDeadlineMs,
2873
- reportBackpressure: (retryAfterMs) =>
2874
- this.reportBackpressure(toolId, retryAfterMs),
2875
- resolveToolContract: this.resolvePacing,
2876
- onRequestsSettled: this.onRequestsSettled,
2877
- callbacks: this.callbacks,
2878
- completeRequests: async (entries) =>
2879
- await this.completeDurableToolRequests(entries),
2880
- failRequests: async (claimedRequests, error) =>
2881
- await this.failDurableToolRequests(claimedRequests, error),
2882
- settleRequest: (claimed, result) => this.settleRequests(claimed, result),
2883
- rejectRequest: (claimed, error) => this.rejectRequests(claimed, error),
2884
- latch: this.latch,
2885
- });
2886
- recordRunnerPerfTrace({
2887
- req: this.req,
2888
- phase: 'runner.tool.group',
2889
- ms: nowMs() - batchStartedAt,
2890
- extra: {
2891
- toolId,
2892
- requests: requestCount,
2893
- executed: claimedRequests.length,
2894
- batched: true,
2895
- },
2896
- });
2330
+ function isRuntimeReceiptPersistenceError(error: unknown): boolean {
2331
+ if (!error || typeof error !== 'object') return false;
2332
+ if (error instanceof RuntimeReceiptPersistenceError) return true;
2333
+ if (
2334
+ error instanceof Error &&
2335
+ error.name === 'RuntimeReceiptPersistenceError'
2336
+ ) {
2337
+ return true;
2897
2338
  }
2339
+ const nestedErrors = (error as { errors?: unknown }).errors;
2340
+ return (
2341
+ Array.isArray(nestedErrors) &&
2342
+ nestedErrors.some(isRuntimeReceiptPersistenceError)
2343
+ );
2898
2344
  }
2899
2345
 
2900
- function groupWorkerToolRequestsByTool(
2901
- requests: WorkerToolBatchRequest[],
2902
- ): Map<string, WorkerToolBatchRequest[]> {
2903
- const byTool = new Map<string, WorkerToolBatchRequest[]>();
2904
- for (const request of requests) {
2905
- const existing = byTool.get(request.toolId);
2906
- if (existing) {
2907
- existing.push(request);
2908
- } else {
2909
- byTool.set(request.toolId, [request]);
2910
- }
2911
- }
2912
- return byTool;
2346
+ function isRunFatalWorkerRowError(error: unknown): boolean {
2347
+ return (
2348
+ isCoordinatorProgressAuthorizationError(error) ||
2349
+ isRowIsolationExemptError(error) ||
2350
+ isRuntimeReceiptPersistenceError(error) ||
2351
+ isRuntimePersistenceCircuitOpenError(error) ||
2352
+ error instanceof RuntimeSheetAttemptLeaseLostError ||
2353
+ error instanceof RuntimeLeaseLostError ||
2354
+ (error instanceof Error &&
2355
+ error.name === 'RuntimeSheetAttemptLeaseLostError')
2356
+ );
2913
2357
  }
2914
2358
 
2915
- async function executeBatchedWorkerToolGroup(input: {
2916
- req: RunRequest;
2917
- requests: ClaimedWorkerToolBatchRequest[];
2918
- strategy: AnyBatchOperationStrategy;
2919
- governor: PlayExecutionGovernor;
2920
- suggestedParallelism: number;
2921
- abortSignal?: AbortSignal;
2922
- runtimeDeadlineMs?: number;
2923
- reportBackpressure: (retryAfterMs: number) => void;
2924
- resolveToolContract: WorkerPacingResolver;
2925
- onRequestsSettled?: (count: number) => void;
2926
- callbacks?: WorkerCtxCallbacks;
2927
- completeRequests: (
2928
- entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
2929
- ) => Promise<unknown[]>;
2930
- failRequests: (
2931
- requests: ClaimedWorkerToolBatchRequest[],
2932
- error: unknown,
2933
- ) => Promise<void>;
2934
- settleRequest: (
2935
- request: ClaimedWorkerToolBatchRequest,
2936
- result: unknown,
2937
- ) => void;
2938
- rejectRequest: (
2939
- request: ClaimedWorkerToolBatchRequest,
2940
- error: unknown,
2941
- ) => void;
2942
- latch: RuntimePersistenceLatch;
2943
- }): Promise<void> {
2944
- const compiledBatches = compileRequestsWithStrategy({
2945
- requests: input.requests,
2946
- strategy: input.strategy,
2947
- getPayload: (request) => request.request.input,
2948
- });
2949
- recordRunnerPerfTrace({
2950
- req: input.req,
2951
- phase: 'runner.tool.batch.compile',
2952
- ms: 0,
2953
- extra: {
2954
- sourceOperation: input.strategy.sourceOperation,
2955
- batchOperation: input.strategy.batchOperation,
2956
- requests: input.requests.length,
2957
- batches: compiledBatches.length,
2958
- batchSizes: compiledBatches.map((batch) => batch.memberRequests.length),
2959
- },
2960
- });
2359
+ /**
2360
+ * In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
2361
+ * throughput pacing (provider or Deepline limiter), not a tool defect, so it
2362
+ * gets more patience than the generic 3-attempt budget: with retry-after-aware
2363
+ * escalating delays (capped at 5s) this absorbs roughly 25s of sustained
2364
+ * throttling before the call fails. Every retry still charges the Governor's
2365
+ * retry budget, so a runaway storm stays bounded and loud.
2366
+ */
2961
2367
 
2962
- await executeChunkedRequests({
2963
- requests: compiledBatches,
2964
- // Chunk parallelism is the Governor's per-tool suggestion (provider rate
2965
- // hints tightened to the policy ceiling), bounded by the batch count.
2966
- batchSize: Math.max(
2967
- 1,
2968
- Math.min(input.suggestedParallelism, compiledBatches.length || 1),
2969
- ),
2970
- // Bounded dispatch wave: cap the provider calls (counted by batch member
2971
- // size) a single chunk dispatches before the latch is read again, so a
2972
- // persistence failure prevents the rest of a large batch group instead of
2973
- // billing it all. A single over-budget batch still forms its own wave.
2974
- maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
2975
- weightOf: (batch) => batch.memberRequests.length,
2976
- execute: async (batch) => {
2977
- // Circuit breaker: once a persistence failure has occurred in this map,
2978
- // do NOT dispatch this batch's provider call. The chunk plumbing rejects
2979
- // its member requests with the latch error.
2980
- if (input.latch.tripped) {
2981
- input.latch.preventedCallCount += batch.memberRequests.length;
2982
- throw new RuntimePersistenceCircuitOpenError(input.latch);
2983
- }
2984
- const toolContract = await input
2985
- .resolveToolContract(batch.batchOperation)
2986
- .catch(() => null);
2987
- // One provider call per batch → one tool slot (budget + global
2988
- // concurrency + per-(org,provider) pacing) around the whole batch.
2989
- const slot = await input.governor.acquireToolSlot(batch.batchOperation, {
2990
- signal: input.abortSignal,
2991
- });
2992
- try {
2993
- input.callbacks?.onToolCalled?.(batch.batchOperation, nowMs());
2994
- return await executeTool(
2995
- input.req,
2996
- {
2997
- id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
2998
- toolId: batch.batchOperation,
2999
- input: batch.batchPayload,
3000
- },
3001
- undefined,
3002
- input.reportBackpressure,
3003
- () => input.governor.chargeBudget('retry'),
3004
- toolContract?.retrySafeTransientHttp === true,
3005
- input.abortSignal,
3006
- input.runtimeDeadlineMs,
3007
- resolveClaimedWorkerToolRuntimeTimeoutMs(batch.memberRequests, {
3008
- runtimeDeadlineMs: input.runtimeDeadlineMs,
3009
- }),
3010
- );
3011
- } catch (error) {
3012
- input.callbacks?.onToolFailed?.(batch.batchOperation, nowMs());
3013
- throw error;
3014
- } finally {
3015
- slot.release();
3016
- }
3017
- },
3018
- onChunkComplete: async (
3019
- chunkResults: Array<
3020
- ChunkExecutionResult<(typeof compiledBatches)[number], unknown>
3021
- >,
3022
- ) => {
3023
- for (const entry of chunkResults) {
3024
- if (entry.error !== undefined) {
3025
- // One batch's provider error stays scoped to that batch's member
3026
- // requests. Sibling batches in this chunk keep their results so a
3027
- // single provider hiccup cannot cascade into a whole-map failure.
3028
- let rejection: unknown = entry.error;
3029
- try {
3030
- await input.failRequests(entry.request.memberRequests, entry.error);
3031
- } catch (receiptError) {
3032
- tripRuntimePersistenceLatch(input.latch, receiptError);
3033
- rejection = new RuntimeReceiptPersistenceError(
3034
- 'Tool call failed and durable receipts could not be marked failed',
3035
- [entry.error, receiptError],
3036
- );
3037
- }
3038
- for (const claimed of entry.request.memberRequests) {
3039
- input.rejectRequest(claimed, rejection);
3040
- }
3041
- continue;
3042
- }
3043
- const batchResult = isToolExecuteResult(entry.result)
3044
- ? entry.result.toolOutput.raw
3045
- : entry.result;
3046
- const splitResults =
3047
- batchResult != null
3048
- ? entry.request.splitResults(batchResult)
3049
- : entry.request.memberRequests.map(() => null);
3050
- let completedResults: unknown[];
3051
- try {
3052
- completedResults = await input.completeRequests(
3053
- entry.request.memberRequests.map((claimed, index) => ({
3054
- claimed,
3055
- result: wrapWorkerToolResult(
3056
- claimed.request.toolId,
3057
- splitResults[index] ?? null,
3058
- toolMetadataFallback(claimed.request.toolId),
3059
- ),
3060
- })),
3061
- );
3062
- } catch (receiptError) {
3063
- tripRuntimePersistenceLatch(input.latch, receiptError);
3064
- const rejection = new RuntimeReceiptPersistenceError(
3065
- 'Tool call succeeded but durable receipts could not be marked completed',
3066
- [receiptError],
3067
- );
3068
- for (const claimed of entry.request.memberRequests) {
3069
- input.rejectRequest(claimed, rejection);
3070
- }
3071
- continue;
3072
- }
3073
- for (let index = 0; index < completedResults.length; index += 1) {
3074
- const claimed = entry.request.memberRequests[index]!;
3075
- const request = claimed.request;
3076
- input.settleRequest(claimed, completedResults[index]);
3077
- }
3078
- }
3079
- },
3080
- }).catch(async (error) => {
3081
- let rejection: unknown = error;
3082
- try {
3083
- await input.failRequests(input.requests, error);
3084
- } catch (receiptError) {
3085
- tripRuntimePersistenceLatch(input.latch, receiptError);
3086
- rejection = new RuntimeReceiptPersistenceError(
3087
- 'Tool call failed and durable receipts could not be marked failed',
3088
- [error, receiptError],
3089
- );
3090
- }
3091
- for (const claimed of input.requests) {
3092
- input.rejectRequest(claimed, rejection);
3093
- }
3094
- });
2368
+ function sleepWorkerMs(ms: number): Promise<void> {
2369
+ return new Promise((resolve) => setTimeout(resolve, ms));
3095
2370
  }
3096
2371
 
3097
2372
  function isCompletedWorkerFieldValue(value: unknown): boolean {
@@ -3102,7 +2377,8 @@ function isCompletedWorkerFieldValue(value: unknown): boolean {
3102
2377
  );
3103
2378
  }
3104
2379
 
3105
- type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
2380
+ type WorkerMapCompletedChunkSummary<T extends Record<string, unknown>> = {
2381
+ status: 'completed';
3106
2382
  chunkIndex: number;
3107
2383
  rangeStart: number;
3108
2384
  rangeEnd: number;
@@ -3126,6 +2402,23 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
3126
2402
  cachedRows?: T[];
3127
2403
  };
3128
2404
 
2405
+ type WorkerMapBlockedChunkSummary = {
2406
+ status: 'blocked';
2407
+ chunkIndex: number;
2408
+ rangeStart: number;
2409
+ rangeEnd: number;
2410
+ rowsRead: number;
2411
+ rowsBlocked: number;
2412
+ rowKeySamples: string[];
2413
+ blockedRowSamples: Record<string, unknown>[];
2414
+ outputDatasetId: string;
2415
+ message: string;
2416
+ };
2417
+
2418
+ type WorkerMapChunkSummary<T extends Record<string, unknown>> =
2419
+ | WorkerMapCompletedChunkSummary<T>
2420
+ | WorkerMapBlockedChunkSummary;
2421
+
3129
2422
  function serializeDurableStepValue<T>(value: T, depth = 0): T {
3130
2423
  if (depth > 20 || value == null) return value;
3131
2424
  if (isToolExecuteResult(value)) return serializeToolExecuteResult(value) as T;
@@ -3338,6 +2631,7 @@ async function executeWorkerStepProgram(
3338
2631
  ...(resolution.status ? { status: resolution.status } : {}),
3339
2632
  };
3340
2633
  };
2634
+
3341
2635
  const executeStep = async (): Promise<WorkerStepResolution> =>
3342
2636
  workflowStep
3343
2637
  ? await (
@@ -3562,7 +2856,7 @@ async function postRuntimeEgressFetch(
3562
2856
  ) {
3563
2857
  const response = await fetchRuntimeApi(
3564
2858
  req.baseUrl,
3565
- '/api/v2/plays/internal/egress-fetch',
2859
+ '/api/v2/internal/play-runtime/egress-fetch',
3566
2860
  {
3567
2861
  method: 'POST',
3568
2862
  headers: {
@@ -4138,6 +3432,81 @@ function resolveSheetContractFromReq(
4138
3432
  }
4139
3433
  }
4140
3434
 
3435
+ /**
3436
+ * Run-fatal teardown: release the runtime-sheet attempt leases and work-receipt
3437
+ * leases this run still holds, so an immediate rerun proceeds without waiting
3438
+ * out the 10-minute lease TTL. Every release is attempt/owner fenced downstream,
3439
+ * so it can never weaken a different run's live lease. Failures are logged loud
3440
+ * and swallowed here — the caller still throws the original run-fatal error.
3441
+ */
3442
+ async function releaseRuntimeLeasesOnTeardown(input: {
3443
+ req: RunRequest;
3444
+ leasedSheetNamespaces: Set<string>;
3445
+ emit: (event: RunnerEvent) => void;
3446
+ }): Promise<void> {
3447
+ const { req, leasedSheetNamespaces, emit } = input;
3448
+ const scope = runtimeSheetSessionScope(req);
3449
+ for (const tableNamespace of leasedSheetNamespaces) {
3450
+ const sheetContract = resolveSheetContractFromReq(req, tableNamespace);
3451
+ if (!sheetContract) continue;
3452
+ try {
3453
+ const released = await harnessReleaseSheetAttempt({
3454
+ ...scope,
3455
+ tableNamespace,
3456
+ sheetContract,
3457
+ runId: req.runId,
3458
+ attemptOwnerRunId: req.runId,
3459
+ attemptSeq: req.runAttempt ?? 0,
3460
+ });
3461
+ if (released.released > 0) {
3462
+ emit({
3463
+ type: 'log',
3464
+ level: 'info',
3465
+ message: `Released ${released.released} runtime sheet row lease(s) for ctx.dataset("${tableNamespace}") on run-fatal teardown.`,
3466
+ ts: nowMs(),
3467
+ });
3468
+ }
3469
+ } catch (releaseError) {
3470
+ emit({
3471
+ type: 'log',
3472
+ level: 'warn',
3473
+ message: `Runtime sheet attempt release failed for ctx.dataset("${tableNamespace}") on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
3474
+ releaseError instanceof Error
3475
+ ? releaseError.message
3476
+ : String(releaseError)
3477
+ }`,
3478
+ ts: nowMs(),
3479
+ });
3480
+ }
3481
+ }
3482
+ try {
3483
+ const released = await harnessReleaseRuntimeReceipts({
3484
+ ...scope,
3485
+ runId: req.runId,
3486
+ runAttempt: req.runAttempt ?? 0,
3487
+ });
3488
+ if (released.released > 0) {
3489
+ emit({
3490
+ type: 'log',
3491
+ level: 'info',
3492
+ message: `Released ${released.released} work-receipt lease(s) on run-fatal teardown.`,
3493
+ ts: nowMs(),
3494
+ });
3495
+ }
3496
+ } catch (releaseError) {
3497
+ emit({
3498
+ type: 'log',
3499
+ level: 'warn',
3500
+ message: `Work-receipt release failed on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
3501
+ releaseError instanceof Error
3502
+ ? releaseError.message
3503
+ : String(releaseError)
3504
+ }`,
3505
+ ts: nowMs(),
3506
+ });
3507
+ }
3508
+ }
3509
+
4141
3510
  function staticPipelineFromReq(req: RunRequest): PlayStaticPipeline | null {
4142
3511
  const snapshot = req.contractSnapshot as
4143
3512
  | { staticPipeline?: unknown }
@@ -4180,6 +3549,13 @@ async function persistCompletedMapRows(input: {
4180
3549
  rows: Record<string, unknown>[];
4181
3550
  outputFields: string[];
4182
3551
  extraOutputFields?: string[];
3552
+ attemptId?: string | null;
3553
+ attemptOwnerRunId?: string | null;
3554
+ attemptExpiresAt?: string | null;
3555
+ attemptSeq?: number | null;
3556
+ budgetMeter?: WorkerWorkBudgetMeter;
3557
+ force?: boolean;
3558
+ forceFailedRows?: boolean;
4183
3559
  }): Promise<{
4184
3560
  rows: number;
4185
3561
  written: number;
@@ -4225,23 +3601,28 @@ async function persistCompletedMapRows(input: {
4225
3601
  sheetContract,
4226
3602
  rows,
4227
3603
  outputFields,
3604
+ attemptId: input.attemptId ?? null,
3605
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3606
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3607
+ attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
3608
+ forceFailedRows: input.forceFailedRows === true,
4228
3609
  };
3610
+ input.budgetMeter?.count('sheet');
4229
3611
  const expectedKeys = [
4230
3612
  ...new Set(
4231
3613
  input.rows
4232
- .map((row) => resolveMapRowOutcomeKey(row))
4233
- .filter((key): key is string => Boolean(key)),
3614
+ .map((row) => resolveMapRowOutcomeKey(row) ?? row.key)
3615
+ .filter(Boolean),
4234
3616
  ),
4235
- ];
4236
- const expectedVisibleRows =
4237
- expectedKeys.length > 0 ? expectedKeys.length : rows.length;
3617
+ ] as string[];
3618
+ const expectedVisibleRows = expectedKeys.length || rows.length;
4238
3619
  const readVisibleRowCount = async () => {
4239
3620
  if (expectedKeys.length > 0) {
4240
3621
  const result = await harnessReadSheetDatasetRowKeys({
4241
3622
  ...sessionScope,
4242
3623
  tableNamespace,
4243
3624
  runId: req.runId,
4244
- rowMode: 'all',
3625
+ rowMode: 'terminalAnyRun',
4245
3626
  keys: expectedKeys,
4246
3627
  });
4247
3628
  return result.keys.length;
@@ -4257,6 +3638,24 @@ async function persistCompletedMapRows(input: {
4257
3638
  return result.rows.length;
4258
3639
  };
4259
3640
  const result = await harnessPersistCompletedSheetRows(persistRequest);
3641
+ const hasAttemptFence = Boolean(input.attemptId?.trim());
3642
+ const assertAttemptStillOwnsTerminalWrites = (
3643
+ written: number,
3644
+ visible: number | null,
3645
+ ) => {
3646
+ // Recovery finalization may include rows completed by an earlier attempt.
3647
+ // A complete terminal readback proves they were not lost.
3648
+ if (
3649
+ !hasAttemptFence ||
3650
+ written === rows.length ||
3651
+ (visible !== null && visible >= expectedVisibleRows)
3652
+ ) {
3653
+ return;
3654
+ }
3655
+ throw new Error(
3656
+ `Runtime sheet attempt lost ownership for ${tableNamespace}: wrote ${written}/${rows.length}; run ${req.runId}; attempt ${input.attemptId}.`,
3657
+ );
3658
+ };
4260
3659
  console.warn('[play-runner.persist_completed_map_rows.result]', {
4261
3660
  runId: req.runId,
4262
3661
  playName: req.playName,
@@ -4281,11 +3680,19 @@ async function persistCompletedMapRows(input: {
4281
3680
  sheetContract,
4282
3681
  rows: input.rows.map((row) => runtimeCsvStorageRow(row)),
4283
3682
  runId: req.runId,
3683
+ attemptId: input.attemptId ?? null,
3684
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3685
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3686
+ attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
3687
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
4284
3688
  inputOffset: 0,
4285
3689
  });
3690
+ input.budgetMeter?.count('sheet');
4286
3691
  const retry = await harnessPersistCompletedSheetRows(persistRequest);
3692
+ input.budgetMeter?.count('sheet');
4287
3693
  retryWritten = retry.rowsWritten;
4288
3694
  retryVisible = await readVisibleRowCount();
3695
+ assertAttemptStillOwnsTerminalWrites(retryWritten, retryVisible);
4289
3696
  if (retryVisible >= expectedVisibleRows) {
4290
3697
  return {
4291
3698
  rows: rows.length,
@@ -4299,6 +3706,7 @@ async function persistCompletedMapRows(input: {
4299
3706
  `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}/${expectedVisibleRows}; run ${req.runId}.`,
4300
3707
  );
4301
3708
  }
3709
+ assertAttemptStillOwnsTerminalWrites(result.rowsWritten, visibleRows);
4302
3710
  }
4303
3711
  if (result.rowsWritten !== rows.length && visibleRows < expectedVisibleRows) {
4304
3712
  throw new Error(
@@ -4320,8 +3728,14 @@ async function applyLiveMapRowUpdates(input: {
4320
3728
  updates: Array<Omit<PlayRowUpdate, 'rowId'> & { runId?: string }>;
4321
3729
  outputFields: string[];
4322
3730
  extraOutputFields?: string[];
3731
+ attemptId?: string | null;
3732
+ attemptOwnerRunId?: string | null;
3733
+ attemptExpiresAt?: string | null;
3734
+ attemptSeq?: number | null;
3735
+ budgetMeter?: WorkerWorkBudgetMeter;
4323
3736
  }): Promise<void> {
4324
3737
  if (input.updates.length === 0) return;
3738
+ input.budgetMeter?.count('sheet');
4325
3739
  const outputFields = [
4326
3740
  ...input.outputFields,
4327
3741
  ...(input.extraOutputFields ?? []).filter(
@@ -4344,7 +3758,16 @@ async function applyLiveMapRowUpdates(input: {
4344
3758
  contractSnapshot: input.req.contractSnapshot,
4345
3759
  runId: input.req.runId,
4346
3760
  userEmail: input.req.userEmail,
4347
- updates: input.updates,
3761
+ updates: input.updates.map((update) => ({
3762
+ ...update,
3763
+ attemptId: update.attemptId ?? input.attemptId ?? null,
3764
+ attemptOwnerRunId:
3765
+ update.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
3766
+ attemptExpiresAt:
3767
+ update.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
3768
+ attemptSeq:
3769
+ update.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
3770
+ })),
4348
3771
  },
4349
3772
  );
4350
3773
  }
@@ -4355,18 +3778,39 @@ async function prepareMapRows(input: {
4355
3778
  rows: Record<string, unknown>[];
4356
3779
  inputOffset: number;
4357
3780
  outputFields: string[];
3781
+ attemptId?: string | null;
3782
+ attemptOwnerRunId?: string | null;
3783
+ attemptExpiresAt?: string | null;
3784
+ attemptSeq?: number | null;
3785
+ budgetMeter?: WorkerWorkBudgetMeter;
4358
3786
  force?: boolean;
4359
3787
  }): Promise<{
4360
3788
  inserted: number;
4361
3789
  skipped: number;
4362
3790
  pendingRows: Record<string, unknown>[];
4363
3791
  completedRows: Record<string, unknown>[];
3792
+ blockedRows: Record<string, unknown>[];
3793
+ attemptId?: string | null;
3794
+ attemptOwnerRunId?: string | null;
3795
+ attemptExpiresAt?: string | null;
3796
+ attemptSeq?: number | null;
4364
3797
  }> {
4365
3798
  if (input.rows.length === 0) {
4366
- return { inserted: 0, skipped: 0, pendingRows: [], completedRows: [] };
3799
+ return {
3800
+ inserted: 0,
3801
+ skipped: 0,
3802
+ pendingRows: [],
3803
+ completedRows: [],
3804
+ blockedRows: [],
3805
+ attemptId: input.attemptId ?? null,
3806
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3807
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3808
+ attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
3809
+ };
4367
3810
  }
4368
3811
  const sessionScope = runtimeSheetSessionScope(input.req);
4369
3812
  const rows = input.rows.map((row) => runtimeCsvStorageRow(row));
3813
+ input.budgetMeter?.count('sheet');
4370
3814
  const result = await harnessStartSheetDataset({
4371
3815
  ...sessionScope,
4372
3816
  tableNamespace: input.tableNamespace,
@@ -4377,6 +3821,11 @@ async function prepareMapRows(input: {
4377
3821
  }),
4378
3822
  rows,
4379
3823
  inputOffset: input.inputOffset,
3824
+ attemptId: input.attemptId ?? null,
3825
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3826
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3827
+ attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
3828
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
4380
3829
  ...(input.force ? { force: true } : {}),
4381
3830
  });
4382
3831
  for (const timing of result.timings ?? []) {
@@ -4406,6 +3855,13 @@ async function prepareMapRows(input: {
4406
3855
  skipped: result.skipped,
4407
3856
  pendingRows: result.pendingRows,
4408
3857
  completedRows: result.completedRows,
3858
+ blockedRows: result.blockedRows ?? [],
3859
+ attemptId: result.attemptId ?? input.attemptId ?? null,
3860
+ attemptOwnerRunId:
3861
+ result.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
3862
+ attemptExpiresAt: result.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
3863
+ attemptSeq:
3864
+ result.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
4409
3865
  };
4410
3866
  }
4411
3867
 
@@ -4551,14 +4007,6 @@ function createCoordinatorRatePort(req: RunRequest): CoordinatorRatePort {
4551
4007
  * memoized per isolate. No hints → null (pacing is a no-op; the Governor's
4552
4008
  * global tool-concurrency slot still applies).
4553
4009
  */
4554
- type WorkerPacingResolver = (
4555
- toolId: string,
4556
- ) => Promise<
4557
- (ResolvedPacingPolicy & { retrySafeTransientHttp: boolean }) | null
4558
- >;
4559
-
4560
- type WorkerToolActionCacheVersionResolver = (toolId: string) => Promise<string>;
4561
-
4562
4010
  function createWorkerToolActionCacheVersionResolver(
4563
4011
  req: RunRequest,
4564
4012
  ): WorkerToolActionCacheVersionResolver {
@@ -4566,9 +4014,7 @@ function createWorkerToolActionCacheVersionResolver(
4566
4014
  return (toolId: string) => {
4567
4015
  const normalized = String(toolId || '').trim();
4568
4016
  if (!normalized) {
4569
- return Promise.reject(
4570
- new Error('Runtime tool metadata lookup requires a non-empty tool id.'),
4571
- );
4017
+ return Promise.reject(new Error('Tool metadata needs tool id.'));
4572
4018
  }
4573
4019
  const cached = cache.get(normalized);
4574
4020
  if (cached) return cached;
@@ -4605,6 +4051,44 @@ function createWorkerToolActionCacheVersionResolver(
4605
4051
  };
4606
4052
  }
4607
4053
 
4054
+ function createWorkerToolAuthScopeDigestResolver(req: RunRequest) {
4055
+ // Return type is inferred from createRuntimeToolAuthScopeDigestResolver
4056
+ // (RuntimeToolAuthScopeDigestResolver), whose `invalidate` member is
4057
+ // REQUIRED — the AUTH_SCOPE_CHANGED re-claim wiring below calls it directly,
4058
+ // and a resolver without eviction support must be a type error.
4059
+ return createRuntimeToolAuthScopeDigestResolver({
4060
+ missingToolIdMessage: 'Tool auth scope needs tool id.',
4061
+ fetchDigest: async (normalized) => {
4062
+ const runScopedPath =
4063
+ `/api/v2/plays/runtime-tools/${encodeURIComponent(normalized)}/auth-scope` +
4064
+ `?runId=${encodeURIComponent(req.runId)}`;
4065
+ const res = await fetchRuntimeApi(req.baseUrl, runScopedPath, {
4066
+ method: 'GET',
4067
+ headers: {
4068
+ authorization: `Bearer ${req.executorToken}`,
4069
+ 'cache-control': 'no-store',
4070
+ pragma: 'no-cache',
4071
+ },
4072
+ });
4073
+ if (!res.ok) {
4074
+ throw new Error(
4075
+ `Runtime tool auth scope lookup for ${normalized} failed (${res.status}): ${await res.text()}`,
4076
+ );
4077
+ }
4078
+ const body = (await res.json().catch(() => null)) as {
4079
+ digest?: unknown;
4080
+ } | null;
4081
+ const digest = typeof body?.digest === 'string' ? body.digest.trim() : '';
4082
+ if (!digest) {
4083
+ throw new Error(
4084
+ `Runtime tool auth scope lookup for ${normalized} returned no digest.`,
4085
+ );
4086
+ }
4087
+ return digest;
4088
+ },
4089
+ });
4090
+ }
4091
+
4608
4092
  function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
4609
4093
  const cache = new Map<
4610
4094
  string,
@@ -4746,13 +4230,22 @@ function createMinimalWorkerCtx(
4746
4230
  workflowStep?: WorkflowStep,
4747
4231
  abortSignal?: AbortSignal,
4748
4232
  callbacks?: WorkerCtxCallbacks,
4233
+ workBudgetMeter: WorkerWorkBudgetMeter = createWorkerWorkBudgetMeter({
4234
+ nowMs,
4235
+ }),
4749
4236
  runtimeDeadlineMs?: number,
4237
+ // Namespaces this run took a runtime-sheet attempt lease on. The run-fatal
4238
+ // teardown reads this set to release exactly those leases (never a namespace
4239
+ // whose table the run never created), so a rerun proceeds immediately.
4240
+ leasedSheetNamespaces?: Set<string>,
4750
4241
  receiptSalvage?: ReceiptSalvageBuffer,
4751
4242
  ): unknown {
4752
4243
  const { governor, resolvePacing: resolveToolPacing } =
4753
4244
  createGovernorForRun(req);
4754
4245
  const resolveToolActionCacheVersion =
4755
4246
  createWorkerToolActionCacheVersionResolver(req);
4247
+ const resolveToolAuthScopeDigest =
4248
+ createWorkerToolAuthScopeDigestResolver(req);
4756
4249
  // Play-call depth/count/per-parent budgets, child-play concurrency, and the
4757
4250
  // lineage snapshot are owned by the Governor (createGovernorForRun above).
4758
4251
  // The worker keeps only substrate mechanism here.
@@ -4763,7 +4256,7 @@ function createMinimalWorkerCtx(
4763
4256
  if (!auth) return {};
4764
4257
  const response = await fetchRuntimeApi(
4765
4258
  req.baseUrl,
4766
- '/api/v2/plays/internal/runtime',
4259
+ '/api/v2/internal/play-runtime',
4767
4260
  {
4768
4261
  method: 'POST',
4769
4262
  headers: {
@@ -4802,10 +4295,48 @@ function createMinimalWorkerCtx(
4802
4295
  orgId: req.orgId,
4803
4296
  preloadedDbSessions: req.preloadedDbSessions ?? null,
4804
4297
  userEmail: req.userEmail,
4298
+ runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null,
4299
+ runAttempt: req.runAttempt ?? 0,
4300
+ receiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
4805
4301
  });
4302
+ const executeDispatchedTool = async (input: {
4303
+ id: string;
4304
+ toolId: string;
4305
+ input: Record<string, unknown>;
4306
+ workflowStep?: unknown;
4307
+ callbacks?: WorkerCtxCallbacks;
4308
+ onProviderBackpressure?: (retryAfterMs: number) => void;
4309
+ onRetryAttempt?: () => void;
4310
+ transientHttpRetrySafe?: boolean;
4311
+ durableCallReceiptKey?: string | null;
4312
+ executionAuthScopeDigest?: string | null;
4313
+ providerIdempotencyKey?: string | null;
4314
+ runtimeTimeoutMs?: number;
4315
+ directOptions?: WorkerToolDirectOptions;
4316
+ }): Promise<ToolExecuteResult> =>
4317
+ await executeToolWithLifecycle(
4318
+ req,
4319
+ {
4320
+ id: input.id,
4321
+ toolId: input.toolId,
4322
+ input: input.input,
4323
+ durableCallReceiptKey: input.durableCallReceiptKey,
4324
+ executionAuthScopeDigest: input.executionAuthScopeDigest,
4325
+ providerIdempotencyKey: input.providerIdempotencyKey,
4326
+ },
4327
+ input.workflowStep as WorkflowStep | undefined,
4328
+ input.callbacks,
4329
+ input.onProviderBackpressure,
4330
+ input.onRetryAttempt,
4331
+ input.transientHttpRetrySafe === true,
4332
+ abortSignal,
4333
+ runtimeDeadlineMs,
4334
+ input.runtimeTimeoutMs,
4335
+ input.directOptions,
4336
+ );
4806
4337
  const executeWithRuntimeReceipt = async <T>(
4807
4338
  key: string,
4808
- execute: () => Promise<T> | T,
4339
+ execute: (context: WorkerRuntimeReceiptExecutionContext) => Promise<T> | T,
4809
4340
  options: {
4810
4341
  repairRunningReceiptForSameRun?: boolean;
4811
4342
  reclaimRunning?: boolean;
@@ -4819,7 +4350,8 @@ function createMinimalWorkerCtx(
4819
4350
  runId: req.runId,
4820
4351
  key,
4821
4352
  receiptStore,
4822
- execute: async () => serializeDurableStepValue(await execute()),
4353
+ execute: async (context) =>
4354
+ serializeDurableStepValue(await execute(context)),
4823
4355
  repairRunningReceiptForSameRun:
4824
4356
  options.repairRunningReceiptForSameRun ?? false,
4825
4357
  reclaimRunning: options.reclaimRunning ?? false,
@@ -4861,24 +4393,54 @@ function createMinimalWorkerCtx(
4861
4393
  staleAfterSeconds,
4862
4394
  });
4863
4395
  };
4864
- // Ctx-wide breaker for top-level ctx.tool calls (outside any map). Each map
4865
- // creates its own map-scoped latch (see runMap) so per-map dispatch stats and
4866
- // the runMap failure message stay map-local.
4396
+ const emitAuthScopeReclaimRunLog = (input: {
4397
+ toolId: string;
4398
+ requestIds: string[];
4399
+ }) => {
4400
+ // Durable run-log marker for the bounded AUTH_SCOPE_CHANGED re-claim
4401
+ // (mirrors the shared-runtime marker in shared_libs/play-runtime/context.ts
4402
+ // so run logs expose the credential transition on every execution path).
4403
+ emitEvent({
4404
+ type: 'log',
4405
+ level: 'info',
4406
+ message:
4407
+ `ctx.tools.execute(${input.toolId}): auth_scope_changed_reclaim ` +
4408
+ `(requests: ${input.requestIds.join(', ')}); credential identity ` +
4409
+ `changed mid-run, re-resolved the auth scope and re-claimed fresh ` +
4410
+ `receipts under the new scope.`,
4411
+ ts: nowMs(),
4412
+ });
4413
+ };
4867
4414
  const rootPersistenceLatch = createRuntimePersistenceLatch();
4868
- const rootToolBatchScheduler = new WorkerToolBatchScheduler(
4415
+ const rootToolBatchScheduler = new WorkerToolBatchScheduler({
4869
4416
  req,
4870
4417
  governor,
4871
- resolveToolPacing,
4418
+ resolvePacing: resolveToolPacing,
4872
4419
  resolveToolActionCacheVersion,
4420
+ resolveToolAuthScopeDigest,
4421
+ invalidateToolAuthScopeDigest: (toolId) =>
4422
+ resolveToolAuthScopeDigest.invalidate(toolId),
4423
+ onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
4424
+ executeTool: executeDispatchedTool,
4425
+ serializeDurableValue: serializeDurableStepValue,
4426
+ deserializeDurableValue: deserializeDurableStepValue,
4427
+ nowMs,
4428
+ batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
4429
+ recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
4873
4430
  abortSignal,
4874
- undefined,
4875
4431
  callbacks,
4876
4432
  receiptStore,
4877
- true,
4433
+ allowLocalRetryReceipts: true,
4434
+ runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
4435
+ runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
4436
+ req,
4437
+ runtimeReceiptLeaseTtlMs(req) ?? PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
4438
+ ),
4439
+ budgetMeter: workBudgetMeter,
4878
4440
  runtimeDeadlineMs,
4879
- rootPersistenceLatch,
4441
+ persistenceLatch: rootPersistenceLatch,
4880
4442
  receiptSalvage,
4881
- );
4443
+ });
4882
4444
  // Local ancestry chain that always ENDS with the currently-executing play
4883
4445
  // (req.playName). The /api/v2/plays/run lineage validator requires the
4884
4446
  // submitted ancestry's tail to equal the executor token's play name (i.e.
@@ -4913,23 +4475,33 @@ function createMinimalWorkerCtx(
4913
4475
  ): Promise<unknown> => {
4914
4476
  const mapStartedAt = nowMs();
4915
4477
  const mapNodeId = `map:${name}`;
4916
- // Map-wide persistence-failure circuit breaker. Shared across every chunk's
4917
- // scheduler (the scheduler is rebuilt per chunk) and the map's sheet-persist
4918
- // flush chain, so the first persistence failure halts NEW provider dispatch
4919
- // for the rest of the map and its prevented-call count surfaces in the
4920
- // runMap failure message below.
4921
- const persistenceLatch = createRuntimePersistenceLatch();
4922
4478
  const inputRows = rows;
4923
4479
  const rowCountHint = datasetRowCountHint(inputRows);
4924
4480
  const baseOffset = 0;
4925
4481
  const fieldEntries = Object.entries(fieldsDef);
4926
4482
  const plan = req.executionPlan;
4927
- const rowsPerChunk = chooseWorkerMapRowsPerChunk({
4483
+ const mapDispatchPlan = chooseWorkerMapDispatchPlan({
4928
4484
  mapName: name,
4929
4485
  rowCountHint,
4930
4486
  executionPlan: plan,
4931
4487
  staticPipeline: staticPipelineFromReq(req),
4932
4488
  });
4489
+ const rowsPerChunk = mapDispatchPlan.rowsPerChunk;
4490
+ recordRunnerPerfTrace({
4491
+ req,
4492
+ phase: 'runner.map_dispatch_plan',
4493
+ ms: 0,
4494
+ extra: {
4495
+ mapName: name,
4496
+ rowsPerChunk,
4497
+ rowCountHint,
4498
+ estimatedExternalStepsPerRow:
4499
+ mapDispatchPlan.estimatedExternalStepsPerRow,
4500
+ workEstimate: mapDispatchPlan.workEstimate,
4501
+ reason: mapDispatchPlan.reason,
4502
+ dynamicWorkerArtifactVersion: DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION,
4503
+ },
4504
+ });
4933
4505
  const outputFields = fieldEntries.map(([field]) => field);
4934
4506
  const updateMapProgress = (
4935
4507
  progress: LiveNodeProgressSnapshot,
@@ -4945,6 +4517,22 @@ function createMinimalWorkerCtx(
4945
4517
  forceFlush: options?.forceFlush ?? true,
4946
4518
  });
4947
4519
  };
4520
+ let queuedMapProgress: Promise<void> = Promise.resolve();
4521
+ let queuedMapProgressError: unknown = null;
4522
+ const enqueueMapProgressUpdate = (
4523
+ progress: LiveNodeProgressSnapshot,
4524
+ options?: { forceFlush?: boolean },
4525
+ ) => {
4526
+ queuedMapProgress = queuedMapProgress
4527
+ .then(() => Promise.resolve(updateMapProgress(progress, options)))
4528
+ .catch((error) => {
4529
+ queuedMapProgressError ??= error;
4530
+ });
4531
+ };
4532
+ const flushQueuedMapProgressUpdates = async () => {
4533
+ await queuedMapProgress;
4534
+ if (queuedMapProgressError) throw queuedMapProgressError;
4535
+ };
4948
4536
  const formatMapProgressMessage = (completed: number, total?: number) =>
4949
4537
  typeof total === 'number' && Number.isFinite(total) && total > 0
4950
4538
  ? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
@@ -4969,7 +4557,9 @@ function createMinimalWorkerCtx(
4969
4557
  return formatMapProgressMessage(completed, input.total);
4970
4558
  };
4971
4559
  const formatMapProcessingMessage = (rowsToExecute: number) =>
4972
- rowsToExecute > 0 ? `${rowsToExecute.toLocaleString()} rows` : null;
4560
+ rowsToExecute > 0
4561
+ ? `Processing ${rowsToExecute.toLocaleString()} rows`
4562
+ : null;
4973
4563
  const formatMapExecutionHeartbeatMessage = (input: {
4974
4564
  rowsToExecute: number;
4975
4565
  startedRows: number;
@@ -4983,7 +4573,7 @@ function createMinimalWorkerCtx(
4983
4573
  const waitingRows = Math.max(0, rowsToExecute - startedRows);
4984
4574
  const parts = [
4985
4575
  activeRows > 0 ? `${activeRows.toLocaleString()} active` : null,
4986
- waitingRows > 0 ? `${waitingRows.toLocaleString()} queued` : null,
4576
+ waitingRows > 0 ? `${waitingRows.toLocaleString()} waiting` : null,
4987
4577
  completedRows > 0 ? `${completedRows.toLocaleString()} done` : null,
4988
4578
  ].filter((part): part is string => Boolean(part));
4989
4579
  const base =
@@ -5147,6 +4737,12 @@ function createMinimalWorkerCtx(
5147
4737
  ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
5148
4738
  const chunkStartedAt = nowMs();
5149
4739
  assertNotAborted(abortSignal);
4740
+ const rowAttempt = makeRuntimeSheetAttempt({
4741
+ runId: req.runId,
4742
+ runAttempt: req.runAttempt,
4743
+ mapName: name,
4744
+ chunkIndex,
4745
+ });
5150
4746
  const keyStartedAt = nowMs();
5151
4747
  const chunkEntries = chunkRows.map((row, localIndex) => {
5152
4748
  const absoluteIndex = baseOffset + chunkStart + localIndex;
@@ -5174,8 +4770,21 @@ function createMinimalWorkerCtx(
5174
4770
  ...mapRowOutcomeRuntimeFields({ key: rowKey }),
5175
4771
  })),
5176
4772
  inputOffset: baseOffset + chunkStart,
4773
+ ...rowAttempt,
4774
+ budgetMeter: workBudgetMeter,
5177
4775
  force: !!req.force,
5178
4776
  });
4777
+ const activeRowAttempt = {
4778
+ attemptId: prepared.attemptId ?? rowAttempt.attemptId,
4779
+ attemptOwnerRunId:
4780
+ prepared.attemptOwnerRunId ?? rowAttempt.attemptOwnerRunId,
4781
+ attemptExpiresAt:
4782
+ prepared.attemptExpiresAt ?? rowAttempt.attemptExpiresAt,
4783
+ attemptSeq: prepared.attemptSeq ?? rowAttempt.attemptSeq,
4784
+ };
4785
+ // Record that this run holds attempt leases in `name`'s sheet so a
4786
+ // run-fatal teardown releases them (the table now exists).
4787
+ leasedSheetNamespaces?.add(name);
5179
4788
  recordRunnerPerfTrace({
5180
4789
  req,
5181
4790
  phase: 'runner.map_chunk.prepare_rows',
@@ -5188,14 +4797,15 @@ function createMinimalWorkerCtx(
5188
4797
  skipped: prepared.skipped,
5189
4798
  pendingRows: prepared.pendingRows.length,
5190
4799
  completedRows: prepared.completedRows.length,
4800
+ blockedRows: prepared.blockedRows.length,
5191
4801
  },
5192
4802
  });
5193
4803
  const progressTotalRows = rowCountHint ?? chunkRows.length;
5194
4804
  const preparedCompletedRows = Math.min(
5195
4805
  progressTotalRows,
5196
- totalRowsWritten + prepared.completedRows.length,
4806
+ totalRowsWritten,
5197
4807
  );
5198
- await updateMapProgress({
4808
+ enqueueMapProgressUpdate({
5199
4809
  completed: preparedCompletedRows,
5200
4810
  total: progressTotalRows,
5201
4811
  startedAt: mapStartedAt,
@@ -5208,6 +4818,9 @@ function createMinimalWorkerCtx(
5208
4818
  const pendingKeys = new Set<string>();
5209
4819
  const pendingRowsByKey = new Map<string, Record<string, unknown>>();
5210
4820
  const completedKeys = new Set<string>();
4821
+ const completedRowsByKey = new Map<string, Record<string, unknown>>();
4822
+ const blockedKeys = new Set<string>();
4823
+ const blockedRowsByKey = new Map<string, Record<string, unknown>>();
5211
4824
  const preparedKeys = new Set<string>();
5212
4825
  for (const row of prepared.pendingRows) {
5213
4826
  const key =
@@ -5225,14 +4838,95 @@ function createMinimalWorkerCtx(
5225
4838
  deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
5226
4839
  if (key) {
5227
4840
  completedKeys.add(key);
4841
+ completedRowsByKey.set(key, row);
5228
4842
  preparedKeys.add(key);
5229
4843
  }
5230
4844
  }
4845
+ for (const row of prepared.blockedRows) {
4846
+ const key =
4847
+ resolveMapRowOutcomeKey(row) ??
4848
+ deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
4849
+ if (key) {
4850
+ blockedKeys.add(key);
4851
+ blockedRowsByKey.set(key, row);
4852
+ preparedKeys.add(key);
4853
+ }
4854
+ }
4855
+ if (blockedKeys.size > 0) {
4856
+ const rowKeySamples = Array.from(blockedKeys).slice(
4857
+ 0,
4858
+ MAP_ROW_FAILURE_SAMPLE_LIMIT,
4859
+ );
4860
+ const message =
4861
+ `ctx.dataset("${name}") chunk ${chunkIndex} is blocked by ` +
4862
+ `${blockedKeys.size} active runtime sheet row lease(s) owned by another attempt. ` +
4863
+ `Retry after the owning attempt finishes or the row leases expire.` +
4864
+ (rowKeySamples.length > 0
4865
+ ? ` Blocked row keys: ${rowKeySamples.join(', ')}`
4866
+ : '');
4867
+ recordRunnerPerfTrace({
4868
+ req,
4869
+ phase: 'runner.map_chunk.blocked_rows',
4870
+ ms: nowMs() - prepareStartedAt,
4871
+ extra: {
4872
+ mapName: name,
4873
+ chunkIndex,
4874
+ rowsBlocked: blockedKeys.size,
4875
+ rowKeySamples,
4876
+ },
4877
+ });
4878
+ enqueueMapProgressUpdate({
4879
+ completed: preparedCompletedRows,
4880
+ total: progressTotalRows,
4881
+ waitingRows: blockedKeys.size,
4882
+ startedAt: mapStartedAt,
4883
+ message,
4884
+ });
4885
+ return {
4886
+ status: 'blocked',
4887
+ chunkIndex,
4888
+ rangeStart: baseOffset + chunkStart,
4889
+ rangeEnd: baseOffset + chunkStart + chunkRows.length,
4890
+ rowsRead: chunkRows.length,
4891
+ rowsBlocked: blockedKeys.size,
4892
+ rowKeySamples,
4893
+ blockedRowSamples: rowKeySamples.map((rowKey) => {
4894
+ const blockedRow = blockedRowsByKey.get(rowKey);
4895
+ const attemptId =
4896
+ typeof blockedRow?.__deeplineAttemptId === 'string'
4897
+ ? blockedRow.__deeplineAttemptId
4898
+ : typeof blockedRow?._attempt_id === 'string'
4899
+ ? blockedRow._attempt_id
4900
+ : activeRowAttempt.attemptId;
4901
+ return {
4902
+ __deeplineRowKey: rowKey,
4903
+ __deeplineAttemptId: attemptId,
4904
+ };
4905
+ }),
4906
+ outputDatasetId: `map:${name}`,
4907
+ message,
4908
+ };
4909
+ }
5231
4910
  const missingPreparedRows = chunkEntries.filter(
5232
4911
  ({ rowKey }) => !preparedKeys.has(rowKey),
5233
4912
  );
5234
- const rowsToExecuteEntries = chunkEntries.filter(
5235
- ({ rowKey }) => pendingKeys.has(rowKey) || !completedKeys.has(rowKey),
4913
+ if (missingPreparedRows.length > 0) {
4914
+ const rowKeySamples = missingPreparedRows
4915
+ .map(({ rowKey }) => rowKey)
4916
+ .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
4917
+ throw new Error(
4918
+ `ctx.dataset("${name}") runtime preparation omitted ${missingPreparedRows.length} row disposition(s). ` +
4919
+ `Every input row must be claimed, completed, or blocked before execution starts.` +
4920
+ (rowKeySamples.length > 0
4921
+ ? ` Missing row keys: ${rowKeySamples.join(', ')}`
4922
+ : ''),
4923
+ );
4924
+ }
4925
+ const rowsToExecuteEntries = chunkEntries.filter(({ rowKey }) =>
4926
+ pendingKeys.has(rowKey),
4927
+ );
4928
+ const completedChunkEntries = chunkEntries.filter(({ rowKey }) =>
4929
+ completedKeys.has(rowKey),
5236
4930
  );
5237
4931
  const uniqueRowsToExecuteEntries = [
5238
4932
  ...new Map(
@@ -5249,18 +4943,15 @@ function createMinimalWorkerCtx(
5249
4943
  rowsToExecute.length,
5250
4944
  );
5251
4945
  if (processingMessage) {
5252
- await updateMapProgress({
4946
+ enqueueMapProgressUpdate({
5253
4947
  completed: preparedCompletedRows,
5254
4948
  total: progressTotalRows,
5255
4949
  startedAt: mapStartedAt,
5256
4950
  message: processingMessage,
5257
4951
  });
5258
4952
  }
5259
- const rowsInserted = prepared.inserted + missingPreparedRows.length;
5260
- const rowsSkipped = Math.max(
5261
- 0,
5262
- prepared.skipped - missingPreparedRows.length,
5263
- );
4953
+ const rowsInserted = prepared.inserted;
4954
+ const rowsSkipped = prepared.skipped;
5264
4955
  let completedExecutedRows = 0;
5265
4956
  let failedExecutedRows = 0;
5266
4957
  let startedExecutedRows = 0;
@@ -5268,12 +4959,7 @@ function createMinimalWorkerCtx(
5268
4959
  let lastChunkProgressAt = 0;
5269
4960
  let lastExecutionHeartbeatAt = nowMs();
5270
4961
  const completedRowsForProgress = () =>
5271
- Math.min(
5272
- progressTotalRows,
5273
- totalRowsWritten +
5274
- prepared.completedRows.length +
5275
- completedExecutedRows,
5276
- );
4962
+ Math.min(progressTotalRows, totalRowsWritten + completedExecutedRows);
5277
4963
  const reportExecutionHeartbeat = (force = false) => {
5278
4964
  const now = nowMs();
5279
4965
  if (
@@ -5283,7 +4969,7 @@ function createMinimalWorkerCtx(
5283
4969
  return;
5284
4970
  }
5285
4971
  lastExecutionHeartbeatAt = now;
5286
- void updateMapProgress(
4972
+ enqueueMapProgressUpdate(
5287
4973
  {
5288
4974
  completed: completedRowsForProgress(),
5289
4975
  total: progressTotalRows,
@@ -5317,7 +5003,7 @@ function createMinimalWorkerCtx(
5317
5003
  return;
5318
5004
  }
5319
5005
  lastChunkProgressAt = now;
5320
- void updateMapProgress(
5006
+ enqueueMapProgressUpdate(
5321
5007
  {
5322
5008
  completed,
5323
5009
  total: progressTotalRows,
@@ -5348,23 +5034,44 @@ function createMinimalWorkerCtx(
5348
5034
  const failedRowEntries: Array<
5349
5035
  { row: T & Record<string, unknown>; error: string } | undefined
5350
5036
  > = new Array(rowsToExecute.length);
5037
+ const fatalReceiptPersistenceRowEntries: Array<
5038
+ { row: T & Record<string, unknown>; error: string } | undefined
5039
+ > = new Array(rowsToExecute.length);
5351
5040
  const executedCellMetaPatches: Array<
5352
5041
  Record<string, WorkerCellMetaPatchEntry> | undefined
5353
5042
  > = new Array(rowsToExecute.length);
5354
- const toolBatchScheduler = new WorkerToolBatchScheduler(
5043
+ const persistenceLatch = createRuntimePersistenceLatch();
5044
+ const toolBatchScheduler = new WorkerToolBatchScheduler({
5355
5045
  req,
5356
5046
  governor,
5357
- resolveToolPacing,
5047
+ resolvePacing: resolveToolPacing,
5358
5048
  resolveToolActionCacheVersion,
5049
+ resolveToolAuthScopeDigest,
5050
+ invalidateToolAuthScopeDigest: (toolId) =>
5051
+ resolveToolAuthScopeDigest.invalidate(toolId),
5052
+ onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
5053
+ executeTool: executeDispatchedTool,
5054
+ serializeDurableValue: serializeDurableStepValue,
5055
+ deserializeDurableValue: deserializeDurableStepValue,
5056
+ nowMs,
5057
+ batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
5058
+ recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
5359
5059
  abortSignal,
5360
- reportSettledToolRequests,
5060
+ onRequestsSettled: reportSettledToolRequests,
5361
5061
  callbacks,
5362
5062
  receiptStore,
5363
- false,
5063
+ allowLocalRetryReceipts: false,
5064
+ runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
5065
+ runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
5066
+ req,
5067
+ runtimeReceiptLeaseTtlMs(req) ??
5068
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
5069
+ ),
5070
+ budgetMeter: workBudgetMeter,
5364
5071
  runtimeDeadlineMs,
5365
5072
  persistenceLatch,
5366
5073
  receiptSalvage,
5367
- );
5074
+ });
5368
5075
  let stepCellsCompleted = 0;
5369
5076
  let stepCellsSkipped = 0;
5370
5077
  const generatedOutputFields = new Set<string>();
@@ -5443,6 +5150,8 @@ function createMinimalWorkerCtx(
5443
5150
  tableNamespace: name,
5444
5151
  outputFields,
5445
5152
  extraOutputFields: Array.from(generatedOutputFields),
5153
+ budgetMeter: workBudgetMeter,
5154
+ ...activeRowAttempt,
5446
5155
  rows: [
5447
5156
  ...rowsToPersist.map(({ row, executedIndex }) =>
5448
5157
  mapRowOutcomeRuntimeRow(
@@ -5506,9 +5215,6 @@ function createMinimalWorkerCtx(
5506
5215
  });
5507
5216
  persistFlushChain = task.catch((error) => {
5508
5217
  persistFailure ??= error;
5509
- // Output-sheet flush failed: trip the breaker so the scheduler stops
5510
- // dispatching new provider calls for the rest of this map.
5511
- tripRuntimePersistenceLatch(persistenceLatch, error);
5512
5218
  });
5513
5219
  return task;
5514
5220
  };
@@ -5550,6 +5256,8 @@ function createMinimalWorkerCtx(
5550
5256
  outputFields,
5551
5257
  extraOutputFields,
5552
5258
  updates,
5259
+ ...activeRowAttempt,
5260
+ budgetMeter: workBudgetMeter,
5553
5261
  });
5554
5262
  } catch (error) {
5555
5263
  liveUpdateFailures += 1;
@@ -5657,18 +5365,11 @@ function createMinimalWorkerCtx(
5657
5365
  request.input,
5658
5366
  workflowStep,
5659
5367
  {
5660
- force: request.force === true || !!req.force,
5368
+ force: request.force === true,
5369
+ runForceRefresh: req.forceToolRefresh === true,
5370
+ runForceFailedRefresh: req.force === true,
5661
5371
  staleAfterSeconds: request.staleAfterSeconds,
5662
5372
  },
5663
- // Parity with the play-level ctx.tools.execute (and the
5664
- // cjs runner's map path): the public ToolExecutionRequest
5665
- // runtime options must not be silently dropped for map
5666
- // rows — receiptWaitMs bounds how long a resumed run
5667
- // waits on a stuck `running` receipt before reclaiming.
5668
- {
5669
- timeoutMs: request.timeoutMs,
5670
- receiptWaitMs: request.receiptWaitMs,
5671
- },
5672
5373
  );
5673
5374
  },
5674
5375
  },
@@ -5777,8 +5478,28 @@ function createMinimalWorkerCtx(
5777
5478
  notePersistableRow(enriched);
5778
5479
  reportChunkProgress(false);
5779
5480
  } catch (rowError) {
5780
- // Abort/budget errors stay run-fatal and leave no partial
5781
- // state: rethrow immediately without recording the row.
5481
+ // Receipt persistence errors stay run-fatal, but keep the
5482
+ // row identity so teardown can make the recovery state loud.
5483
+ if (isRuntimeReceiptPersistenceError(rowError)) {
5484
+ const message = formatWorkerRowFailureMessage(rowError);
5485
+ if (activeField) {
5486
+ cellMetaPatch[activeField] = {
5487
+ status: 'failed',
5488
+ stage: activeField,
5489
+ runId: req.runId,
5490
+ error: message,
5491
+ };
5492
+ }
5493
+ executedCellMetaPatches[myIndex] =
5494
+ Object.keys(cellMetaPatch).length > 0
5495
+ ? cellMetaPatch
5496
+ : undefined;
5497
+ fatalReceiptPersistenceRowEntries[myIndex] = {
5498
+ row: enriched as T & Record<string, unknown>,
5499
+ error: message,
5500
+ };
5501
+ throw rowError;
5502
+ }
5782
5503
  if (isRunFatalWorkerRowError(rowError)) {
5783
5504
  throw rowError;
5784
5505
  }
@@ -5858,16 +5579,25 @@ function createMinimalWorkerCtx(
5858
5579
  return results;
5859
5580
  },
5860
5581
  );
5861
- while (!workerSettled) {
5862
- await Promise.race([
5863
- workerResultsPromise,
5864
- sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
5865
- ]);
5866
- if (!workerSettled) {
5867
- reportExecutionHeartbeat(false);
5868
- }
5869
- }
5870
- const workerResults = await workerResultsPromise;
5582
+ const workerResults = await executeWithRuntimeSheetAttemptHeartbeat({
5583
+ req,
5584
+ tableNamespace: name,
5585
+ sheetContract: requireSheetContract(req, name),
5586
+ ...activeRowAttempt,
5587
+ rowKeys: uniqueRowsToExecuteEntries.map((entry) => entry.rowKey),
5588
+ execute: async () => {
5589
+ while (!workerSettled) {
5590
+ await Promise.race([
5591
+ workerResultsPromise,
5592
+ sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
5593
+ ]);
5594
+ if (!workerSettled) {
5595
+ reportExecutionHeartbeat(false);
5596
+ }
5597
+ }
5598
+ return await workerResultsPromise;
5599
+ },
5600
+ });
5871
5601
  recordRunnerPerfTrace({
5872
5602
  req,
5873
5603
  phase: 'runner.map_chunk.execute_workers',
@@ -5896,26 +5626,89 @@ function createMinimalWorkerCtx(
5896
5626
  .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
5897
5627
  const fatalMapChunkSummary = async (
5898
5628
  error: unknown,
5899
- ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => ({
5900
- chunkIndex,
5901
- rangeStart: baseOffset + chunkStart,
5902
- rangeEnd: baseOffset + chunkStart,
5903
- rowsRead: chunkRows.length,
5904
- rowsWritten: 0,
5905
- rowsExecuted: rowsToExecute.length,
5906
- rowsCached: 0,
5907
- rowsDuplicateReused: duplicateInputReuseCount,
5908
- rowsInserted,
5909
- rowsSkipped,
5910
- rowsFailed: failedExecutedRows,
5911
- rowFailureSamples: buildRowFailureSamples(),
5912
- stepCellsCompleted,
5913
- stepCellsSkipped,
5914
- outputDatasetId: `map:${name}`,
5915
- hash: await hashJson([]),
5916
- fatalError: formatWorkerRowFailureMessage(error),
5917
- preview: [],
5918
- });
5629
+ ): Promise<
5630
+ WorkerMapCompletedChunkSummary<T & Record<string, unknown>>
5631
+ > => {
5632
+ const fatalError = formatWorkerRowFailureMessage(error);
5633
+ const breakerMessage = persistenceLatch.tripped
5634
+ ? ` Circuit breaker prevented ${persistenceLatch.preventedCallCount} queued provider call(s).`
5635
+ : '';
5636
+ return {
5637
+ status: 'completed',
5638
+ chunkIndex,
5639
+ rangeStart: baseOffset + chunkStart,
5640
+ rangeEnd: baseOffset + chunkStart,
5641
+ rowsRead: chunkRows.length,
5642
+ rowsWritten: 0,
5643
+ rowsExecuted: rowsToExecute.length,
5644
+ rowsCached: 0,
5645
+ rowsDuplicateReused: duplicateInputReuseCount,
5646
+ rowsInserted,
5647
+ rowsSkipped,
5648
+ rowsFailed: failedExecutedRows,
5649
+ rowFailureSamples: buildRowFailureSamples(),
5650
+ stepCellsCompleted,
5651
+ stepCellsSkipped,
5652
+ outputDatasetId: `map:${name}`,
5653
+ hash: await hashJson([]),
5654
+ fatalError: `${fatalError}${breakerMessage}`,
5655
+ preview: [],
5656
+ };
5657
+ };
5658
+ const markReceiptPersistenceFatalRowsFailed = async (error: unknown) => {
5659
+ const message = formatWorkerRowFailureMessage(error);
5660
+ const rows = uniqueRowsToExecuteEntries
5661
+ .map((entry, executedIndex) => {
5662
+ const failure = fatalReceiptPersistenceRowEntries[executedIndex];
5663
+ const row =
5664
+ failure?.row ??
5665
+ executedRows[executedIndex] ??
5666
+ (runtimeCsvExecutionRow(
5667
+ entry.row,
5668
+ pendingRowsByKey.get(entry.rowKey),
5669
+ ) as T & Record<string, unknown>);
5670
+ return mapRowOutcomeRuntimeRow(
5671
+ failedMapRowOutcome({
5672
+ key: entry.rowKey,
5673
+ inputIndex: entry.absoluteIndex,
5674
+ data: row,
5675
+ cellMetaPatch: executedCellMetaPatches[executedIndex],
5676
+ error: failure?.error ?? message,
5677
+ }),
5678
+ );
5679
+ })
5680
+ .filter((row): row is Record<string, unknown> => row !== null);
5681
+ if (rows.length === 0) return;
5682
+ try {
5683
+ const failureStats = await persistCompletedMapRows({
5684
+ req,
5685
+ tableNamespace: name,
5686
+ outputFields,
5687
+ extraOutputFields: Array.from(generatedOutputFields),
5688
+ budgetMeter: workBudgetMeter,
5689
+ ...activeRowAttempt,
5690
+ forceFailedRows: true,
5691
+ rows,
5692
+ });
5693
+ emitEvent({
5694
+ type: 'log',
5695
+ level: 'warn',
5696
+ message:
5697
+ `Runtime sheet marked ${failureStats.rows} row(s) failed for ctx.dataset("${name}") ` +
5698
+ `after a run-fatal receipt persistence error.`,
5699
+ ts: nowMs(),
5700
+ });
5701
+ } catch (cleanupError) {
5702
+ emitEvent({
5703
+ type: 'log',
5704
+ level: 'warn',
5705
+ message:
5706
+ `Runtime sheet failed-row cleanup after receipt persistence error failed for ctx.dataset("${name}"): ` +
5707
+ formatWorkerRowFailureMessage(cleanupError),
5708
+ ts: nowMs(),
5709
+ });
5710
+ }
5711
+ };
5919
5712
  const persistRowsStartedAt = nowMs();
5920
5713
  recordRunnerPerfTrace({
5921
5714
  req,
@@ -5944,6 +5737,7 @@ function createMinimalWorkerCtx(
5944
5737
  },
5945
5738
  });
5946
5739
  } catch (error) {
5740
+ tripRuntimePersistenceLatch(persistenceLatch, error);
5947
5741
  recordRunnerPerfTrace({
5948
5742
  req,
5949
5743
  phase: 'runner.map_chunk.persist_rows_error',
@@ -5962,31 +5756,24 @@ function createMinimalWorkerCtx(
5962
5756
  result.status === 'rejected',
5963
5757
  );
5964
5758
  if (rejectedWorker) {
5965
- if (
5966
- isRuntimeReceiptPersistenceError(rejectedWorker.reason) ||
5967
- isRuntimePersistenceCircuitOpenError(rejectedWorker.reason)
5968
- ) {
5969
- // Persistence-failure / circuit-breaker rejections are returned as a
5970
- // fatal summary (NOT rethrown into the durable step) so the chunk is
5971
- // not retried and provider calls are not re-billed.
5759
+ if (isRuntimeReceiptPersistenceError(rejectedWorker.reason)) {
5760
+ await markReceiptPersistenceFatalRowsFailed(rejectedWorker.reason);
5761
+ return await fatalMapChunkSummary(rejectedWorker.reason);
5762
+ }
5763
+ if (isRuntimePersistenceCircuitOpenError(rejectedWorker.reason)) {
5972
5764
  return await fatalMapChunkSummary(rejectedWorker.reason);
5973
5765
  }
5974
5766
  throw rejectedWorker.reason;
5975
5767
  }
5976
5768
  const resultByKey = new Map<string, T & Record<string, unknown>>();
5977
- for (const completedRow of prepared.completedRows) {
5978
- const key =
5979
- resolveMapRowOutcomeKey(completedRow) ??
5980
- deriveDefaultMapRowIdentity(
5981
- completedRow,
5982
- resolveRuntimeInputIndex(completedRow) ?? 0,
5983
- );
5984
- if (key) {
5985
- const cleanedRow = stripMapRowOutcomeRuntimeFields(
5986
- publicCsvOutputRow(completedRow),
5987
- );
5988
- resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
5989
- }
5769
+ for (const entry of completedChunkEntries) {
5770
+ const completedRow = completedRowsByKey.get(entry.rowKey);
5771
+ if (!completedRow || resultByKey.has(entry.rowKey)) continue;
5772
+ resultByKey.set(
5773
+ entry.rowKey,
5774
+ cloneCsvAliasedRow(entry.row, completedRow) as T &
5775
+ Record<string, unknown>,
5776
+ );
5990
5777
  }
5991
5778
  for (
5992
5779
  let executedIndex = 0;
@@ -6021,16 +5808,13 @@ function createMinimalWorkerCtx(
6021
5808
  } => entry !== null,
6022
5809
  );
6023
5810
  const out = outEntries.map((entry) => entry.row);
6024
- const executedSuccessCount = Math.max(
6025
- 0,
6026
- executedRows.length - failedExecutedRows,
6027
- );
6028
5811
  const rowFailureSamples = buildRowFailureSamples();
6029
5812
  const publicOut = out.map((row) => publicCsvOutputRow(row));
6030
5813
  const keyedOut = outEntries.map(({ key, inputIndex, row }) => ({
6031
5814
  ...row,
6032
5815
  ...mapRowOutcomeRuntimeFields({ key, inputIndex }),
6033
5816
  }));
5817
+ const completedRowsReused = completedChunkEntries.length;
6034
5818
  const hashStartedAt = nowMs();
6035
5819
  const hash = await hashJson(publicOut);
6036
5820
  const includeCachedRowsInChunkResult = !workflowStep;
@@ -6061,17 +5845,18 @@ function createMinimalWorkerCtx(
6061
5845
  rowsWritten: out.length,
6062
5846
  rowsExecuted: executedRows.length,
6063
5847
  rowsFailed: failedExecutedRows,
6064
- rowsCached: Math.max(0, out.length - executedSuccessCount),
5848
+ rowsCached: completedRowsReused,
6065
5849
  },
6066
5850
  });
6067
5851
  return {
5852
+ status: 'completed',
6068
5853
  chunkIndex,
6069
5854
  rangeStart: baseOffset + chunkStart,
6070
5855
  rangeEnd: baseOffset + chunkStart + out.length,
6071
5856
  rowsRead: chunkRows.length,
6072
5857
  rowsWritten: out.length,
6073
5858
  rowsExecuted: executedRows.length,
6074
- rowsCached: Math.max(0, out.length - executedSuccessCount),
5859
+ rowsCached: completedRowsReused,
6075
5860
  rowsDuplicateReused: duplicateInputReuseCount,
6076
5861
  rowsInserted,
6077
5862
  rowsSkipped,
@@ -6103,9 +5888,12 @@ function createMinimalWorkerCtx(
6103
5888
  let totalRowsInserted = 0;
6104
5889
  let totalRowsSkipped = 0;
6105
5890
  let totalRowsFailed = 0;
5891
+ let totalRowsBlocked = 0;
6106
5892
  let totalStepCellsCompleted = 0;
6107
5893
  let totalStepCellsSkipped = 0;
6108
5894
  const totalRowFailureSamples: Array<{ rowKey: string; error: string }> = [];
5895
+ const blockedRowKeySamples: string[] = [];
5896
+ const blockedRowSamples: Record<string, unknown>[] = [];
6109
5897
 
6110
5898
  const runChunkStep = async (
6111
5899
  chunkRows: T[],
@@ -6135,6 +5923,7 @@ function createMinimalWorkerCtx(
6135
5923
  const readPersistedRows = async (input: {
6136
5924
  limit: number;
6137
5925
  offset: number;
5926
+ rowMode?: 'output' | 'all';
6138
5927
  }) => {
6139
5928
  const result = await harnessReadSheetDatasetRows({
6140
5929
  baseUrl: req.baseUrl,
@@ -6143,6 +5932,7 @@ function createMinimalWorkerCtx(
6143
5932
  playName: req.playName,
6144
5933
  tableNamespace: name,
6145
5934
  runId: req.runId,
5935
+ rowMode: input.rowMode,
6146
5936
  limit: input.limit,
6147
5937
  offset: input.offset,
6148
5938
  userEmail: req.userEmail,
@@ -6151,19 +5941,61 @@ function createMinimalWorkerCtx(
6151
5941
  return result.rows as Array<T & Record<string, unknown>>;
6152
5942
  };
6153
5943
 
6154
- const finalize = async (totalRowsWritten: number) => {
5944
+ const finalize = async () => {
5945
+ let finalizedRowsWritten = totalRowsWritten;
5946
+ if (
5947
+ totalRowsFailed === 0 &&
5948
+ rowCountHint !== null &&
5949
+ finalizedRowsWritten > 0 &&
5950
+ finalizedRowsWritten < rowCountHint &&
5951
+ rowCountHint <= WORKER_DATASET_IN_MEMORY_ROWS
5952
+ ) {
5953
+ const persistedRows = await readPersistedRows({
5954
+ limit: rowCountHint,
5955
+ offset: 0,
5956
+ rowMode: 'all',
5957
+ });
5958
+ if (persistedRows.length > finalizedRowsWritten) {
5959
+ const inMemoryRowsWritten = finalizedRowsWritten;
5960
+ finalizedRowsWritten = persistedRows.length;
5961
+ totalRowsWritten = finalizedRowsWritten;
5962
+ cachedRows.length = 0;
5963
+ cachedRows.push(...persistedRows);
5964
+ canCacheRows = true;
5965
+ previewRows.length = 0;
5966
+ previewRows.push(
5967
+ ...persistedRows.slice(0, WORKER_DATASET_PREVIEW_ROWS),
5968
+ );
5969
+ emitEvent({
5970
+ type: 'log',
5971
+ level: 'warn',
5972
+ message:
5973
+ `Runtime sheet finalization reconciled ctx.dataset("${name}") ` +
5974
+ `from ${inMemoryRowsWritten}/${rowCountHint} in-memory row(s) ` +
5975
+ `to ${finalizedRowsWritten} visible persisted row(s).`,
5976
+ ts: nowMs(),
5977
+ });
5978
+ }
5979
+ if (persistedRows.length < rowCountHint) {
5980
+ throw new Error(
5981
+ `Runtime sheet finalization mismatch ctx.dataset("${name}"): ` +
5982
+ `expected ${rowCountHint}, saw ${persistedRows.length}; ` +
5983
+ `sum ${finalizedRowsWritten}; ${req.runId}`,
5984
+ );
5985
+ }
5986
+ }
6155
5987
  const failureSampleSummary =
6156
5988
  totalRowFailureSamples.length > 0
6157
5989
  ? ` First error: ${totalRowFailureSamples[0]!.error}`
6158
5990
  : '';
6159
5991
  const cacheSummary =
6160
5992
  totalRowsFailed > 0
6161
- ? `Map completed with partial failures: ${totalRowsWritten} succeeded, ` +
6162
- `${totalRowsFailed} failed (${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
5993
+ ? `Map completed with partial failures: ${finalizedRowsWritten} succeeded, ` +
5994
+ `${totalRowsFailed} failed (${totalRowsExecuted} executed) ` +
6163
5995
  `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}. ` +
6164
5996
  `Failed rows are persisted with their errors and re-execute on the next run.${failureSampleSummary}`
6165
- : `Map completed: ${totalRowsWritten} results ` +
6166
- `(${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
5997
+ : `Map completed: ${finalizedRowsWritten} results ` +
5998
+ `(${totalRowsExecuted} executed) ` +
6167
5999
  `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
6168
6000
  const completedAt = nowMs();
6169
6001
  const totalStepCells = totalStepCellsCompleted + totalStepCellsSkipped;
@@ -6171,17 +6003,21 @@ function createMinimalWorkerCtx(
6171
6003
  totalStepCells > 0
6172
6004
  ? ` cells=${totalStepCells}/${totalStepCellsCompleted}/${totalStepCellsSkipped}`
6173
6005
  : '';
6006
+ await flushQueuedMapProgressUpdates();
6174
6007
  callbacks?.onMapCompleted?.(mapNodeId, completedAt);
6175
6008
  await updateMapProgress({
6176
- completed: totalRowsWritten,
6177
- total: totalRowsWritten + totalRowsFailed,
6009
+ completed: finalizedRowsWritten,
6010
+ total: finalizedRowsWritten + totalRowsFailed,
6178
6011
  failed: totalRowsFailed,
6179
6012
  completedAt,
6180
6013
  updatedAt: completedAt,
6181
6014
  message:
6182
6015
  totalRowsFailed > 0
6183
- ? `${totalRowsWritten.toLocaleString()} succeeded, ${totalRowsFailed.toLocaleString()} failed`
6184
- : formatMapProgressMessage(totalRowsWritten, totalRowsWritten),
6016
+ ? `${finalizedRowsWritten.toLocaleString()} succeeded, ${totalRowsFailed.toLocaleString()} failed`
6017
+ : formatMapProgressMessage(
6018
+ finalizedRowsWritten,
6019
+ finalizedRowsWritten,
6020
+ ),
6185
6021
  });
6186
6022
  emitEvent({
6187
6023
  type: 'log',
@@ -6190,29 +6026,61 @@ function createMinimalWorkerCtx(
6190
6026
  ts: nowMs(),
6191
6027
  });
6192
6028
  if (
6193
- totalRowsWritten > 0 &&
6029
+ finalizedRowsWritten > 0 &&
6194
6030
  totalRowsFailed === 0 &&
6195
6031
  canCacheRows &&
6196
- cachedRows.length === totalRowsWritten
6032
+ cachedRows.length === finalizedRowsWritten
6197
6033
  ) {
6198
6034
  const persistedRows = await readPersistedRows({
6199
- limit: totalRowsWritten,
6035
+ limit: finalizedRowsWritten,
6200
6036
  offset: 0,
6201
6037
  });
6202
- if (persistedRows.length < totalRowsWritten) {
6038
+ if (persistedRows.length < finalizedRowsWritten) {
6039
+ const repairAttempt = makeRuntimeSheetAttempt({
6040
+ runId: req.runId,
6041
+ runAttempt: req.runAttempt,
6042
+ mapName: name,
6043
+ chunkIndex: 'finalize',
6044
+ });
6045
+ const repairRows = cachedRows.map((row) => runtimeCsvStorageRow(row));
6046
+ workBudgetMeter.count('sheet');
6047
+ const repairStart = await harnessStartSheetDataset({
6048
+ ...runtimeSheetSessionScope(req),
6049
+ tableNamespace: name,
6050
+ sheetContract: augmentSheetContractWithDatasetFields({
6051
+ contract: requireSheetContract(req, name),
6052
+ rows: repairRows,
6053
+ outputFields,
6054
+ }),
6055
+ rows: repairRows,
6056
+ runId: req.runId,
6057
+ inputOffset: 0,
6058
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
6059
+ ...repairAttempt,
6060
+ });
6061
+ const activeRepairAttempt = {
6062
+ attemptId: repairStart.attemptId ?? repairAttempt.attemptId,
6063
+ attemptOwnerRunId:
6064
+ repairStart.attemptOwnerRunId ?? repairAttempt.attemptOwnerRunId,
6065
+ attemptExpiresAt:
6066
+ repairStart.attemptExpiresAt ?? repairAttempt.attemptExpiresAt,
6067
+ attemptSeq: repairStart.attemptSeq ?? repairAttempt.attemptSeq,
6068
+ };
6203
6069
  const repair = await persistCompletedMapRows({
6204
6070
  req,
6205
6071
  tableNamespace: name,
6206
6072
  outputFields,
6207
6073
  extraOutputFields: [],
6208
6074
  rows: cachedRows,
6075
+ ...activeRepairAttempt,
6076
+ budgetMeter: workBudgetMeter,
6209
6077
  });
6210
6078
  emitEvent({
6211
6079
  type: 'log',
6212
6080
  level: 'warn',
6213
6081
  message:
6214
6082
  `Runtime sheet finalization repaired ctx.dataset("${name}") ` +
6215
- `from ${persistedRows.length}/${totalRowsWritten} visible row(s) ` +
6083
+ `from ${persistedRows.length}/${finalizedRowsWritten} visible row(s) ` +
6216
6084
  `(written=${repair.written}, visible=${repair.visible}` +
6217
6085
  (repair.retryWritten === null
6218
6086
  ? ''
@@ -6228,7 +6096,7 @@ function createMinimalWorkerCtx(
6228
6096
  return createPersistedDatasetHandle({
6229
6097
  playName: req.playName,
6230
6098
  name,
6231
- count: totalRowsWritten,
6099
+ count: finalizedRowsWritten,
6232
6100
  // In native Workflows, chunk summaries intentionally omit full row
6233
6101
  // payloads, so this preview only contains the bounded chunk samples.
6234
6102
  // Do not synchronously page rows back here: service-binding reads have
@@ -6242,7 +6110,7 @@ function createMinimalWorkerCtx(
6242
6110
  recordRunnerPerfTrace({ req, phase, ms, extra }),
6243
6111
  nowMs,
6244
6112
  workProgress: {
6245
- total: totalRowsWritten + totalRowsFailed,
6113
+ total: finalizedRowsWritten + totalRowsFailed,
6246
6114
  executed: totalRowsExecuted,
6247
6115
  reused: totalRowsCached,
6248
6116
  skipped: totalRowsCached,
@@ -6256,97 +6124,248 @@ function createMinimalWorkerCtx(
6256
6124
  };
6257
6125
 
6258
6126
  const failFastRowErrors = opts?.onRowError === 'fail';
6259
- let chunkIndex = 0;
6260
- let chunkStart = 0;
6261
- for await (const rawChunkRows of iterDatasetChunks(
6262
- inputRows,
6263
- rowsPerChunk,
6264
- )) {
6265
- assertNotAborted(abortSignal);
6266
- if (rawChunkRows.length === 0) continue;
6267
- // Drop duplicate explicit-key rows before anything downstream observes
6268
- // them. `chunkStart` keeps advancing by the original (pre-dedupe) chunk
6269
- // length so cross-chunk key indices and persisted input offsets stay
6270
- // aligned to the original input stream.
6271
- const chunkRows = dedupeExplicitRowKeys(rawChunkRows, chunkStart);
6272
- if (chunkRows.length === 0) {
6273
- chunkStart += rawChunkRows.length;
6274
- continue;
6275
- }
6276
- const chunkResult = await runChunkStep(chunkRows, chunkStart, chunkIndex);
6277
- if (chunkResult.fatalError) {
6278
- const circuitBreakerSentence =
6279
- persistenceLatch.tripped && persistenceLatch.preventedCallCount > 0
6280
- ? `Circuit breaker prevented ${persistenceLatch.preventedCallCount} queued provider call(s) ` +
6281
- `from dispatching after the first persistence failure. `
6282
- : '';
6283
- throw new Error(
6284
- `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
6285
- `outside the retryable chunk step. Provider calls already executed for ` +
6286
- `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
6287
- circuitBreakerSentence +
6288
- `Fix the runtime persistence cause and re-run to resume. ` +
6289
- `First error: ${chunkResult.fatalError}`,
6290
- );
6291
- }
6292
- totalRowsWritten += chunkResult.rowsWritten;
6293
- totalRowsExecuted += chunkResult.rowsExecuted;
6294
- totalRowsCached += chunkResult.rowsCached;
6295
- totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
6296
- totalRowsInserted += chunkResult.rowsInserted;
6297
- totalRowsSkipped += chunkResult.rowsSkipped;
6298
- totalRowsFailed += chunkResult.rowsFailed ?? 0;
6299
- totalStepCellsCompleted += chunkResult.stepCellsCompleted ?? 0;
6300
- totalStepCellsSkipped += chunkResult.stepCellsSkipped ?? 0;
6301
- for (const sample of chunkResult.rowFailureSamples ?? []) {
6302
- if (totalRowFailureSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6303
- break;
6127
+ // Bounded-concurrent chunk dispatch is only sound off the native Workflows
6128
+ // path: with a `workflowStep` each chunk is a durable step whose sequential
6129
+ // order makes replay/resume deterministic, so that path stays at K=1.
6130
+ // Chunk-level concurrency is otherwise safe under workers_edge because each
6131
+ // chunk owns a disjoint sheet row range with its own runtime-sheet attempt,
6132
+ // receipts complete inside each chunk's persist barrier, and child submits
6133
+ // use per-row durable receipt keys. It is disabled under onRowError:'fail'
6134
+ // on purpose: that mode relies on a first-failure short-circuit to skip the
6135
+ // remaining chunks and stop spending provider credits, and eager concurrent
6136
+ // launch would already have K-1 chunks in flight (credits spent) before the
6137
+ // short-circuit fires.
6138
+ const canRunChunksConcurrently = !workflowStep && !failFastRowErrors;
6139
+ const maxConcurrentMapChunks = canRunChunksConcurrently
6140
+ ? WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS
6141
+ : 1;
6142
+ const runWorkDispatcher = new WorkerRunWorkDispatcher({
6143
+ budgetMeter: workBudgetMeter,
6144
+ nowMs,
6145
+ yieldLimits: {
6146
+ elapsed: governor.policy.pacing.workerYieldElapsedMs,
6147
+ ...(req.testPolicyOverrides?.workBudgetYieldLimits ?? {}),
6148
+ },
6149
+ recordTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
6150
+ yieldBetweenChunks: async (yieldPoint) => {
6151
+ emitEvent({
6152
+ type: 'log',
6153
+ level: 'info',
6154
+ message:
6155
+ `Runtime dispatcher yielded ctx.dataset("${name}") before chunk ${yieldPoint.nextChunkIndex} ` +
6156
+ `(${yieldPoint.reason} budget).`,
6157
+ ts: nowMs(),
6158
+ });
6159
+ if (workflowStep) {
6160
+ // Cloudflare Workflows durability model: `step.sleep` writes a
6161
+ // durable checkpoint, and the resume re-enters `run()` as a NEW
6162
+ // Worker invocation. Cloudflare's subrequest ceiling is
6163
+ // per-invocation, and on resume the already-completed `step.do`
6164
+ // chunks are served from the durable step cache WITHOUT re-issuing
6165
+ // their subrequests (replay semantics) — so the post-yield chunk
6166
+ // runs against a fresh per-invocation subrequest budget.
6167
+ //
6168
+ // Isolate identity is the WRONG freshness proxy: Cloudflare routinely
6169
+ // reuses warm isolates across invocations, so a genuinely
6170
+ // fresh-budget resume commonly shares the pre-sleep isolate id (this
6171
+ // is exactly why the isolate-identity check reported `false` every
6172
+ // time and hard-failed multi-chunk maps). Crossing the durable
6173
+ // `step.sleep` boundary IS the fresh-budget signal.
6174
+ const sleepMs = WORK_DISPATCHER_YIELD_SLEEP_MS;
6175
+ await (
6176
+ workflowStep.sleep as unknown as (
6177
+ name: string,
6178
+ duration: number,
6179
+ ) => Promise<void>
6180
+ )(
6181
+ `work-dispatcher:${name}:${yieldPoint.nextChunkIndex}:${yieldPoint.attempt}`,
6182
+ sleepMs,
6183
+ );
6184
+ recordRunnerPerfTrace({
6185
+ req,
6186
+ phase: 'runner.work_dispatcher.yield_probe',
6187
+ ms: sleepMs,
6188
+ extra: {
6189
+ mapName: name,
6190
+ nextChunkIndex: yieldPoint.nextChunkIndex,
6191
+ reason: yieldPoint.reason,
6192
+ attempt: yieldPoint.attempt,
6193
+ freshBudget: true,
6194
+ invocationBoundary: 'workflow_step_sleep',
6195
+ },
6196
+ });
6197
+ return { freshBudget: true };
6304
6198
  }
6305
- totalRowFailureSamples.push(sample);
6306
- }
6307
- await updateMapProgress({
6308
- completed: totalRowsWritten,
6309
- total: rowCountHint ?? undefined,
6310
- ...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
6311
- message: formatMapProgressMessage(
6312
- totalRowsWritten,
6313
- rowCountHint ?? undefined,
6314
- ),
6315
- });
6316
- if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
6317
- previewRows.push(
6318
- ...chunkResult.preview.slice(
6199
+ // No durable Workflow step is available to force a new invocation here,
6200
+ // so we cannot obtain a fresh per-invocation platform budget. Report a
6201
+ // non-fresh yield: the dispatcher escalates and then fails loudly with
6202
+ // WorkerBudgetExhaustedError rather than silently overflowing the
6203
+ // subrequest ceiling mid-map.
6204
+ await sleepWorkerMs(0);
6205
+ return { freshBudget: false };
6206
+ },
6207
+ });
6208
+ let dispatchResult: WorkerRunMapBatchesResult;
6209
+ try {
6210
+ dispatchResult = await runWorkDispatcher.runMapBatches({
6211
+ mapName: name,
6212
+ chunks: iterDatasetChunks(inputRows, rowsPerChunk),
6213
+ maxConcurrentChunks: maxConcurrentMapChunks,
6214
+ maxResidentBatchBytes: WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES,
6215
+ prepareBatch: (rawChunkRows, chunkStart) => {
6216
+ assertNotAborted(abortSignal);
6217
+ if (rawChunkRows.length === 0) return [];
6218
+ // Drop duplicate explicit-key rows before anything downstream observes
6219
+ // them. `chunkStart` keeps advancing by the original (pre-dedupe) chunk
6220
+ // length so cross-chunk key indices and persisted input offsets stay
6221
+ // aligned to the original input stream.
6222
+ return dedupeExplicitRowKeys(rawChunkRows, chunkStart);
6223
+ },
6224
+ estimateRawBatchResidentBytes: (rawChunkRows) =>
6225
+ rawChunkRows.reduce(
6226
+ (total, row) => total + workerMapResidentRowBytes(row),
6227
+ 0,
6228
+ ),
6229
+ executeBatch: async ({ rows: chunkRows, chunkStart, chunkIndex }) => {
6230
+ assertNotAborted(abortSignal);
6231
+ return await runChunkStep(chunkRows, chunkStart, chunkIndex);
6232
+ },
6233
+ estimateBatchResidentBytes: ({ rows: chunkRows }) =>
6234
+ chunkRows.reduce(
6235
+ (total, row) => total + workerMapResidentRowBytes(row),
6319
6236
  0,
6320
- WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
6321
6237
  ),
6238
+ estimateBatchSubrequests: ({ rows: chunkRows }) => {
6239
+ const estimate = mapDispatchPlan.workEstimate;
6240
+ const unbatchedProviderSubrequests =
6241
+ estimate.unbatchedProviderToolCallsPerRow *
6242
+ WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL;
6243
+ // Static planning cannot know input-dependent batching buckets, so
6244
+ // this projects worst-case provider-batch fanout while still treating
6245
+ // durable receipt claim/complete/fail as bulk per drained tool group.
6246
+ const batchedProviderSubrequests =
6247
+ estimateBatchedToolChunkSubrequests({
6248
+ batchableToolCallsPerRow: Math.max(
6249
+ 0,
6250
+ estimate.providerToolCallsPerRow -
6251
+ estimate.unbatchedProviderToolCallsPerRow,
6252
+ ),
6253
+ coalescedBatchableToolCallsPerRow:
6254
+ estimate.coalescedBatchableProviderToolCallsPerRow,
6255
+ rows: chunkRows.length,
6256
+ assumedBatchSize: estimate.assumedBatchSize,
6257
+ });
6258
+ const controlSubrequests =
6259
+ estimate.childPlaySubmitsPerRow + estimate.eventWaitsPerRow;
6260
+ return (
6261
+ chunkRows.length *
6262
+ (unbatchedProviderSubrequests + controlSubrequests) +
6263
+ batchedProviderSubrequests +
6264
+ estimate.runtimeSheetReadsPerChunk +
6265
+ estimate.runtimeSheetWritesPerChunk
6266
+ );
6267
+ },
6268
+ onBatchComplete: async (chunkResult) => {
6269
+ if (chunkResult.status === 'blocked') {
6270
+ totalRowsBlocked += chunkResult.rowsBlocked;
6271
+ for (const key of chunkResult.rowKeySamples) {
6272
+ if (blockedRowKeySamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6273
+ break;
6274
+ }
6275
+ blockedRowKeySamples.push(key);
6276
+ }
6277
+ for (const row of chunkResult.blockedRowSamples) {
6278
+ if (blockedRowSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6279
+ break;
6280
+ }
6281
+ blockedRowSamples.push(row);
6282
+ }
6283
+ emitEvent({
6284
+ type: 'log',
6285
+ level: 'warn',
6286
+ message: chunkResult.message,
6287
+ ts: nowMs(),
6288
+ });
6289
+ return true;
6290
+ }
6291
+ if (chunkResult.fatalError) {
6292
+ throw new Error(
6293
+ `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
6294
+ `outside the retryable chunk step. Provider calls already executed for ` +
6295
+ `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
6296
+ `Fix the runtime persistence cause and re-run to resume. ` +
6297
+ `First error: ${chunkResult.fatalError}`,
6298
+ );
6299
+ }
6300
+ totalRowsWritten += chunkResult.rowsWritten;
6301
+ totalRowsExecuted += chunkResult.rowsExecuted;
6302
+ totalRowsCached += chunkResult.rowsCached;
6303
+ totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
6304
+ totalRowsInserted += chunkResult.rowsInserted;
6305
+ totalRowsSkipped += chunkResult.rowsSkipped;
6306
+ totalRowsFailed += chunkResult.rowsFailed ?? 0;
6307
+ totalStepCellsCompleted += chunkResult.stepCellsCompleted ?? 0;
6308
+ totalStepCellsSkipped += chunkResult.stepCellsSkipped ?? 0;
6309
+ for (const sample of chunkResult.rowFailureSamples ?? []) {
6310
+ if (totalRowFailureSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6311
+ break;
6312
+ }
6313
+ totalRowFailureSamples.push(sample);
6314
+ }
6315
+ enqueueMapProgressUpdate({
6316
+ completed: totalRowsWritten,
6317
+ total: rowCountHint ?? undefined,
6318
+ ...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
6319
+ message: formatMapProgressMessage(
6320
+ totalRowsWritten,
6321
+ rowCountHint ?? undefined,
6322
+ ),
6323
+ });
6324
+ if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
6325
+ previewRows.push(
6326
+ ...chunkResult.preview.slice(
6327
+ 0,
6328
+ WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
6329
+ ),
6330
+ );
6331
+ }
6332
+ if (canCacheRows) {
6333
+ const volatileRows = volatileWorkflowChunkRows.get(
6334
+ chunkResult.chunkIndex,
6335
+ );
6336
+ volatileWorkflowChunkRows.delete(chunkResult.chunkIndex);
6337
+ const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
6338
+ if (
6339
+ nextRows.length === chunkResult.rowsWritten &&
6340
+ cachedRows.length + nextRows.length <=
6341
+ WORKER_DATASET_IN_MEMORY_ROWS
6342
+ ) {
6343
+ cachedRows.push(...nextRows);
6344
+ } else {
6345
+ cachedRows.length = 0;
6346
+ volatileWorkflowChunkRows.clear();
6347
+ canCacheRows = false;
6348
+ }
6349
+ }
6350
+ // onRowError:'fail' short-circuit: once a chunk reports a row failure,
6351
+ // skip the remaining chunks entirely. The failing chunk itself completed
6352
+ // normally (no chunk-step retry storm) and persisted its rows, but
6353
+ // executing later chunks would keep spending provider credits on a run
6354
+ // the caller asked to fail fast. The post-loop fail-fast throw below
6355
+ // reports what committed before the stop.
6356
+ return failFastRowErrors && totalRowsFailed > 0;
6357
+ },
6358
+ });
6359
+ } catch (error) {
6360
+ try {
6361
+ await flushQueuedMapProgressUpdates();
6362
+ } catch (progressError) {
6363
+ throw new AggregateError(
6364
+ [error, progressError],
6365
+ `ctx.dataset("${name}") failed while flushing queued progress updates.`,
6322
6366
  );
6323
6367
  }
6324
- if (canCacheRows) {
6325
- const volatileRows = volatileWorkflowChunkRows.get(chunkIndex);
6326
- volatileWorkflowChunkRows.delete(chunkIndex);
6327
- const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
6328
- if (
6329
- nextRows.length === chunkResult.rowsWritten &&
6330
- cachedRows.length + nextRows.length <= WORKER_DATASET_IN_MEMORY_ROWS
6331
- ) {
6332
- cachedRows.push(...nextRows);
6333
- } else {
6334
- cachedRows.length = 0;
6335
- volatileWorkflowChunkRows.clear();
6336
- canCacheRows = false;
6337
- }
6338
- }
6339
- chunkStart += rawChunkRows.length;
6340
- chunkIndex += 1;
6341
- // onRowError:'fail' short-circuit: once a chunk reports a row failure,
6342
- // skip the remaining chunks entirely. The failing chunk itself completed
6343
- // normally (no chunk-step retry storm) and persisted its rows, but
6344
- // executing later chunks would keep spending provider credits on a run
6345
- // the caller asked to fail fast. The post-loop fail-fast throw below
6346
- // reports what committed before the stop.
6347
- if (failFastRowErrors && totalRowsFailed > 0) {
6348
- break;
6349
- }
6368
+ throw error;
6350
6369
  }
6351
6370
  if (totalDuplicateKeysDropped > 0) {
6352
6371
  const keySample = droppedDuplicateKeySamples.join(', ');
@@ -6359,6 +6378,13 @@ function createMinimalWorkerCtx(
6359
6378
  ts: nowMs(),
6360
6379
  });
6361
6380
  }
6381
+ if (totalRowsBlocked > 0) {
6382
+ await flushQueuedMapProgressUpdates();
6383
+ throw new RuntimeSheetRowsBlockedError({
6384
+ tableNamespace: name,
6385
+ blockedRows: blockedRowSamples,
6386
+ });
6387
+ }
6362
6388
  if (failFastRowErrors && totalRowsFailed > 0 && totalRowsWritten > 0) {
6363
6389
  // onRowError:'fail', PARTIAL failure (some rows committed): fail the run
6364
6390
  // without finalizing the dataset. The committed rows already persisted
@@ -6367,6 +6393,7 @@ function createMinimalWorkerCtx(
6367
6393
  // normally (no per-row throw inside the durable chunk step, so no
6368
6394
  // chunk-step retry storm); later chunks were skipped by the fail-fast
6369
6395
  // short-circuit in the chunk loop.
6396
+ await flushQueuedMapProgressUpdates();
6370
6397
  const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
6371
6398
  throw new Error(
6372
6399
  `ctx.dataset("${name}") failed for ${totalRowsFailed} executed row(s) under onRowError:'fail'. ` +
@@ -6384,7 +6411,7 @@ function createMinimalWorkerCtx(
6384
6411
  // step when no row otherwise succeeded) are summarized and registered as
6385
6412
  // a recovered dataset — the failed run then advertises a WORKING export
6386
6413
  // instead of a dead end (#15/#27). The run still fails (the throw below).
6387
- await finalize(totalRowsWritten);
6414
+ await finalize();
6388
6415
  const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
6389
6416
  throw new Error(
6390
6417
  `ctx.dataset("${name}") failed for all ${totalRowsFailed} executed rows. ` +
@@ -6392,7 +6419,7 @@ function createMinimalWorkerCtx(
6392
6419
  `(rows are persisted with per-row errors; fix the cause and re-run to resume)`,
6393
6420
  );
6394
6421
  }
6395
- const dataset = await finalize(totalRowsWritten);
6422
+ const dataset = await finalize();
6396
6423
  recordRunnerPerfTrace({
6397
6424
  req,
6398
6425
  phase: 'runner.map.total',
@@ -6401,7 +6428,7 @@ function createMinimalWorkerCtx(
6401
6428
  mapName: name,
6402
6429
  rowsWritten: totalRowsWritten,
6403
6430
  inputKind: rowCountHint === null ? 'streaming' : 'known_count',
6404
- chunks: chunkIndex,
6431
+ chunks: dispatchResult.chunksExecuted,
6405
6432
  },
6406
6433
  });
6407
6434
  return dataset;
@@ -6666,10 +6693,10 @@ function createMinimalWorkerCtx(
6666
6693
  request.input,
6667
6694
  workflowStep,
6668
6695
  {
6669
- force: request.force === true || !!req.force,
6696
+ force: request.force === true,
6697
+ runForceRefresh: req.forceToolRefresh === true,
6698
+ runForceFailedRefresh: req.force === true,
6670
6699
  staleAfterSeconds: request.staleAfterSeconds,
6671
- },
6672
- {
6673
6700
  timeoutMs: request.timeoutMs,
6674
6701
  receiptWaitMs: request.receiptWaitMs,
6675
6702
  },
@@ -6710,13 +6737,18 @@ function createMinimalWorkerCtx(
6710
6737
  if (!resolvedName) {
6711
6738
  throw new Error('ctx.runPlay(...) requires a resolvable play name.');
6712
6739
  }
6740
+ const childManifest = req.childPlayManifests?.[resolvedName];
6713
6741
  const receiptKey = buildDurableCtxCallCacheKey({
6714
6742
  orgId: req.orgId,
6715
6743
  playId: req.playName,
6716
6744
  kind: 'runPlay',
6717
6745
  id: normalizedKey,
6718
- semanticKey: await hashJson({
6746
+ semanticKey: buildDurableRunPlaySemanticKey({
6719
6747
  childPlayName: resolvedName,
6748
+ childRevisionFingerprint: workerChildRevisionFingerprint({
6749
+ playName: resolvedName,
6750
+ manifest: childManifest,
6751
+ }),
6720
6752
  input,
6721
6753
  }),
6722
6754
  staleAfterSeconds: options?.staleAfterSeconds,
@@ -6744,7 +6776,6 @@ function createMinimalWorkerCtx(
6744
6776
  message: `Starting child play ${resolvedName} (${normalizedKey})`,
6745
6777
  ts: nowMs(),
6746
6778
  });
6747
- const childManifest = req.childPlayManifests?.[resolvedName];
6748
6779
  if (!childManifest) {
6749
6780
  throw new Error(
6750
6781
  `ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
@@ -6782,7 +6813,7 @@ function createMinimalWorkerCtx(
6782
6813
  });
6783
6814
  let childPlaySlot: { release(): void } | null = null;
6784
6815
  try {
6785
- childPlaySlot = await governor.acquireChildPlaySlot({
6816
+ childPlaySlot = await governor.acquireChildSubmitSlot({
6786
6817
  signal: abortSignal,
6787
6818
  });
6788
6819
  const childSubmitStartedAt = nowMs();
@@ -6807,12 +6838,12 @@ function createMinimalWorkerCtx(
6807
6838
  options?.timeoutMs == null && !childNeedsWorkflowScheduler,
6808
6839
  body: {
6809
6840
  name: resolvedName,
6841
+ childIdempotencyKey: receiptKey,
6810
6842
  input: isRecord(input) ? input : {},
6811
6843
  orgId: req.orgId,
6812
6844
  callbackBaseUrl: req.callbackUrl,
6813
6845
  baseUrl: req.baseUrl,
6814
6846
  integrationMode: req.integrationMode ?? null,
6815
- forceToolRefresh: req.force === true,
6816
6847
  parentExecutorToken: req.executorToken,
6817
6848
  userEmail: req.userEmail ?? '',
6818
6849
  profile: 'workers_edge',
@@ -6924,6 +6955,8 @@ function createMinimalWorkerCtx(
6924
6955
  childNeedsWorkflowScheduler,
6925
6956
  },
6926
6957
  });
6958
+ childPlaySlot?.release();
6959
+ childPlaySlot = null;
6927
6960
  const startedStatus = String(started.status ?? '').toLowerCase();
6928
6961
  if (startedStatus === 'completed') {
6929
6962
  emitEvent({
@@ -6952,6 +6985,10 @@ function createMinimalWorkerCtx(
6952
6985
  }
6953
6986
  const childWaitStartedAt = nowMs();
6954
6987
  let waitResult: ChildPlayTerminalWaitResult;
6988
+ const readCachedChildTerminalState =
6989
+ cachedCoordinatorBinding?.readChildTerminalState?.bind(
6990
+ cachedCoordinatorBinding,
6991
+ );
6955
6992
  try {
6956
6993
  waitResult = await awaitChildTerminal({
6957
6994
  parentRunId: req.runId,
@@ -6968,20 +7005,31 @@ function createMinimalWorkerCtx(
6968
7005
  1_000,
6969
7006
  Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
6970
7007
  ),
6971
- coordinator: cachedCoordinatorBinding?.readChildTerminalState
6972
- ? {
6973
- readChildTerminalState: (
6974
- parentRunId,
6975
- eventKey,
6976
- timeoutMs,
6977
- ) =>
6978
- cachedCoordinatorBinding!.readChildTerminalState!(
6979
- parentRunId,
6980
- eventKey,
6981
- timeoutMs,
6982
- ),
6983
- }
6984
- : null,
7008
+ coordinator: {
7009
+ readRunTerminalState: (runId, timeoutMs) =>
7010
+ readDurableChildRunTerminalState({
7011
+ baseUrl: req.baseUrl,
7012
+ executorToken: req.executorToken,
7013
+ parentRunId: req.runId,
7014
+ childRunId: runId,
7015
+ childPlayName: resolvedName,
7016
+ timeoutMs,
7017
+ }),
7018
+ ...(readCachedChildTerminalState
7019
+ ? {
7020
+ readChildTerminalState: (
7021
+ parentRunId: string,
7022
+ eventKey: string,
7023
+ timeoutMs?: number,
7024
+ ) =>
7025
+ readCachedChildTerminalState(
7026
+ parentRunId,
7027
+ eventKey,
7028
+ timeoutMs,
7029
+ ),
7030
+ }
7031
+ : {}),
7032
+ },
6985
7033
  now: nowMs,
6986
7034
  hashJson,
6987
7035
  });
@@ -7164,6 +7212,7 @@ function createMinimalWorkerCtx(
7164
7212
  // Direct Worker fetch below is still limited to runtime origins and
7165
7213
  // IP literals; public IP literals also consume this slot because
7166
7214
  // they are customer egress.
7215
+ workBudgetMeter.count('egress');
7167
7216
  const egressResponse = await postRuntimeEgressFetch(
7168
7217
  req,
7169
7218
  {
@@ -7186,6 +7235,7 @@ function createMinimalWorkerCtx(
7186
7235
  }
7187
7236
  const fetchInit = { ...init, headers };
7188
7237
  delete fetchInit.auth;
7238
+ workBudgetMeter.count('egress');
7189
7239
  const response = await safeWorkerPublicFetch(url, fetchInit, {
7190
7240
  allowedOrigins,
7191
7241
  sensitiveHeaders: Object.keys(secretHeaderMarkers),
@@ -7256,6 +7306,7 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
7256
7306
  let req: RunRequest;
7257
7307
  try {
7258
7308
  req = (await request.json()) as RunRequest;
7309
+ req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
7259
7310
  } catch {
7260
7311
  return new Response('invalid JSON body', { status: 400 });
7261
7312
  }
@@ -7320,6 +7371,7 @@ async function handleRunInline(
7320
7371
  let req: RunRequest;
7321
7372
  try {
7322
7373
  req = (await request.json()) as RunRequest;
7374
+ req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
7323
7375
  } catch {
7324
7376
  return Response.json(
7325
7377
  {
@@ -7467,6 +7519,7 @@ async function executeRunRequest(
7467
7519
  ): Promise<WorkflowRunOutput> {
7468
7520
  installProcessExitTrap();
7469
7521
  const startedAt = nowMs();
7522
+ const workBudgetMeter = createWorkerWorkBudgetMeter({ nowMs });
7470
7523
  recordRunnerPerfTrace({
7471
7524
  req,
7472
7525
  phase: 'runner.execute_start',
@@ -7504,6 +7557,7 @@ async function executeRunRequest(
7504
7557
  const appendRunLogLine = (line: string) => {
7505
7558
  const trimmed = redactSecretsFromLogString(line.trim());
7506
7559
  if (!trimmed) return;
7560
+ workBudgetMeter.count('log');
7507
7561
  totalEmittedLogLines += 1;
7508
7562
  runLogBuffer = [...runLogBuffer, trimmed].slice(-RUN_LOG_BUFFER_LIMIT);
7509
7563
  pendingRunLogLines = [...pendingRunLogLines, trimmed];
@@ -7696,7 +7750,10 @@ async function executeRunRequest(
7696
7750
  const flushTerminalLedgerEvents = async (
7697
7751
  terminalEvent: PlayRunLedgerEvent,
7698
7752
  ): Promise<void> => {
7699
- if (!options?.persistResultDatasets) return;
7753
+ const shouldPersistTerminalLedger =
7754
+ options?.persistResultDatasets === true ||
7755
+ Boolean(req.playCallGovernance);
7756
+ if (!shouldPersistTerminalLedger) return;
7700
7757
  await ledgerFlushInFlight;
7701
7758
  const now = nowMs();
7702
7759
  dirtyProgressNodeIds = new Set([
@@ -7771,10 +7828,7 @@ async function executeRunRequest(
7771
7828
  stepLifecycle?.markPreDatasetStepsStarted(startedAt);
7772
7829
  flushLedgerEvents(false);
7773
7830
  const runtimeDeadlineMs = nowMs() + STANDARD_PLAY_RUNTIME_LIMIT_MS;
7774
- // Run-scoped buffer for provider results that billed but whose durable
7775
- // receipt could not be marked completed even after the retry ladder. On
7776
- // terminal failure the buffer is built into a size-capped salvage record and
7777
- // attached to the run.failed ledger event so billed data is never memory-only.
7831
+ const leasedSheetNamespaces = new Set<string>();
7778
7832
  const receiptSalvage = createReceiptSalvageBuffer();
7779
7833
  const ctx = createMinimalWorkerCtx(
7780
7834
  req,
@@ -7783,7 +7837,9 @@ async function executeRunRequest(
7783
7837
  workflowStep,
7784
7838
  abortSignal,
7785
7839
  workerCallbacks,
7840
+ workBudgetMeter,
7786
7841
  runtimeDeadlineMs,
7842
+ leasedSheetNamespaces,
7787
7843
  receiptSalvage,
7788
7844
  );
7789
7845
  // Hard wall-clock cap on active user-code runtime. CF Workflows does not
@@ -7792,15 +7848,18 @@ async function executeRunRequest(
7792
7848
  // token expires. Aborting the controller surfaces cooperatively through the
7793
7849
  // same assertNotAborted checks used for harness cancellation.
7794
7850
  let runtimeLimitExceeded = false;
7795
- const runtimeDeadlineTimer = setTimeout(
7796
- () => {
7797
- runtimeLimitExceeded = true;
7798
- if (!abortSignal.aborted) {
7799
- abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
7800
- }
7801
- },
7802
- Math.max(1, runtimeDeadlineMs - nowMs()),
7803
- );
7851
+ const runtimeDeadlineTimer =
7852
+ workflowStep === undefined || workflowStep === null
7853
+ ? setTimeout(
7854
+ () => {
7855
+ runtimeLimitExceeded = true;
7856
+ if (!abortSignal.aborted) {
7857
+ abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
7858
+ }
7859
+ },
7860
+ Math.max(1, runtimeDeadlineMs - nowMs()),
7861
+ )
7862
+ : null;
7804
7863
  try {
7805
7864
  const playStartedAt = nowMs();
7806
7865
  const result = await (
@@ -7891,7 +7950,6 @@ async function executeRunRequest(
7891
7950
  }
7892
7951
  if (options?.persistResultDatasets) {
7893
7952
  await persistProjectedResultDatasets();
7894
- const parentSignal = startParentTerminalSignal();
7895
7953
  // Capped runs settle compute billing BEFORE declaring run.completed: a
7896
7954
  // per-run cap denial (422 billing_cap_exceeded) must fail the run as
7897
7955
  // its ONLY terminal. Flushing completed first opens a race — watchers
@@ -7926,6 +7984,7 @@ async function executeRunRequest(
7926
7984
  ms: nowMs() - terminalUpdateStartedAt,
7927
7985
  });
7928
7986
 
7987
+ const parentSignal = startParentTerminalSignal();
7929
7988
  if (!capped) {
7930
7989
  const billingStartedAt = nowMs();
7931
7990
  const billingPromise = finalizeWorkerComputeBilling({
@@ -7953,6 +8012,20 @@ async function executeRunRequest(
7953
8012
  }
7954
8013
  }
7955
8014
  await parentSignal;
8015
+ } else if (req.playCallGovernance) {
8016
+ const childTerminalStartedAt = nowMs();
8017
+ await flushTerminalLedgerEvents({
8018
+ type: 'run.completed',
8019
+ runId: req.runId,
8020
+ source: 'worker',
8021
+ occurredAt: nowMs(),
8022
+ result: terminalResult,
8023
+ });
8024
+ recordRunnerPerfTrace({
8025
+ req,
8026
+ phase: 'runner.child_terminal_ledger_append',
8027
+ ms: nowMs() - childTerminalStartedAt,
8028
+ });
7956
8029
  }
7957
8030
  await startParentTerminalSignal();
7958
8031
  recordRunnerPerfTrace({
@@ -7979,16 +8052,8 @@ async function executeRunRequest(
7979
8052
  stepLifecycle?.markStartedFailed(nowMs());
7980
8053
  // A runtime-limit abort is a timeout failure, not a user cancellation, so
7981
8054
  // it should be reported as run.failed with the limit message rather than
7982
- // run.cancelled. The deadline can also surface as a WorkflowAbortError
7983
- // thrown synchronously from the completeReceipt retry ladder BEFORE the
7984
- // runtimeDeadlineTimer macrotask flips the flag — match the limit message
7985
- // directly so that race can never misclassify a timeout as a cancel (which
7986
- // would silently drop the salvage buffer of billed-but-unpersisted rows).
7987
- const runtimeLimitError =
7988
- runtimeLimitExceeded ||
7989
- (error instanceof Error &&
7990
- error.message === STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
7991
- const aborted = isAbortLikeError(error) && !runtimeLimitError;
8055
+ // run.cancelled.
8056
+ const aborted = isAbortLikeError(error) && !runtimeLimitExceeded;
7992
8057
  if (aborted) {
7993
8058
  // Flip the controller so any concurrent user code observes the abort
7994
8059
  // through ctx.signal. We mark the run cancelled instead of failed.
@@ -7997,19 +8062,28 @@ async function executeRunRequest(
7997
8062
  );
7998
8063
  }
7999
8064
  const failure = normalizePlayRunFailure(error);
8000
- // Billed-but-unpersisted salvage rides on the run.failed event's `result`.
8001
- // The run STILL FAILS salvage is a loud marker, never a success fallback.
8065
+ const message = failure.message;
8066
+ // Controlled run-fatal teardown: release the runtime-sheet row leases and
8067
+ // work-receipt leases this attempt still holds so an immediate rerun is not
8068
+ // blocked for the full lease TTL. Best-effort but LOUD — a release failure
8069
+ // is logged and never masks the original run-fatal error. TTL expiry stays
8070
+ // the recovery path for true crashes (isolate death) where this never runs.
8071
+ await releaseRuntimeLeasesOnTeardown({
8072
+ req,
8073
+ leasedSheetNamespaces,
8074
+ emit: wrappedEmit,
8075
+ });
8076
+ const errorBilling = extractErrorBilling(error);
8002
8077
  const salvage: ReceiptSalvage | null = aborted
8003
8078
  ? null
8004
8079
  : receiptSalvage.build();
8005
- const message =
8080
+ const terminalMessage =
8006
8081
  salvage && salvage.totalEntries > 0
8007
- ? `${failure.message} ${receiptSalvageFailureSuffix(salvage)}`
8008
- : failure.message;
8009
- const errorBilling = extractErrorBilling(error);
8010
- if (options?.persistResultDatasets) {
8082
+ ? `${message} ${receiptSalvageFailureSuffix(salvage)}`
8083
+ : message;
8084
+ if (options?.persistResultDatasets || req.playCallGovernance) {
8011
8085
  appendRunLogLine(
8012
- `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(message)}`,
8086
+ `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(terminalMessage)}`,
8013
8087
  );
8014
8088
  const terminalUpdateStartedAt = nowMs();
8015
8089
  await flushTerminalLedgerEvents({
@@ -8017,64 +8091,70 @@ async function executeRunRequest(
8017
8091
  runId: req.runId,
8018
8092
  source: 'worker',
8019
8093
  occurredAt: nowMs(),
8020
- error: message,
8094
+ error: terminalMessage,
8021
8095
  result: aborted
8022
8096
  ? undefined
8023
8097
  : {
8024
8098
  success: false,
8025
8099
  status: 'failed',
8026
- error: message,
8100
+ error: terminalMessage,
8101
+ ...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
8027
8102
  errors: [
8028
8103
  {
8029
8104
  code: failure.code,
8030
8105
  phase: failure.phase,
8031
- message,
8106
+ message: terminalMessage,
8032
8107
  retryable: failure.retryable,
8033
8108
  ...(errorBilling ? { billing: errorBilling } : {}),
8034
8109
  ...(failure.cause ? { cause: failure.cause } : {}),
8035
8110
  },
8036
8111
  ],
8037
- ...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
8038
8112
  },
8039
8113
  });
8040
8114
  recordRunnerPerfTrace({
8041
8115
  req,
8042
- phase: aborted
8043
- ? 'runner.terminal_ledger_append_cancelled'
8044
- : 'runner.terminal_ledger_append_failed',
8116
+ phase: req.playCallGovernance
8117
+ ? aborted
8118
+ ? 'runner.child_terminal_ledger_append_cancelled'
8119
+ : 'runner.child_terminal_ledger_append_failed'
8120
+ : aborted
8121
+ ? 'runner.terminal_ledger_append_cancelled'
8122
+ : 'runner.terminal_ledger_append_failed',
8045
8123
  ms: nowMs() - terminalUpdateStartedAt,
8046
8124
  extra: {
8047
8125
  errorCode: failure.code,
8048
8126
  errorPhase: failure.phase,
8049
8127
  },
8050
8128
  });
8051
- const billingStartedAt = nowMs();
8052
- await finalizeWorkerComputeBilling({
8053
- req,
8054
- success: false,
8055
- actionEstimate: 4,
8056
- })
8057
- .catch((finalizeError) => {
8058
- console.error(
8059
- `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
8060
- finalizeError instanceof Error
8061
- ? finalizeError.message
8062
- : String(finalizeError)
8063
- }`,
8064
- );
8129
+ if (options?.persistResultDatasets) {
8130
+ const billingStartedAt = nowMs();
8131
+ await finalizeWorkerComputeBilling({
8132
+ req,
8133
+ success: false,
8134
+ actionEstimate: 4,
8065
8135
  })
8066
- .finally(() => {
8067
- recordRunnerPerfTrace({
8068
- req,
8069
- phase: 'runner.compute_billing_finalize_failed',
8070
- ms: nowMs() - billingStartedAt,
8136
+ .catch((finalizeError) => {
8137
+ console.error(
8138
+ `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
8139
+ finalizeError instanceof Error
8140
+ ? finalizeError.message
8141
+ : String(finalizeError)
8142
+ }`,
8143
+ );
8144
+ })
8145
+ .finally(() => {
8146
+ recordRunnerPerfTrace({
8147
+ req,
8148
+ phase: 'runner.compute_billing_finalize_failed',
8149
+ ms: nowMs() - billingStartedAt,
8150
+ });
8071
8151
  });
8072
- });
8152
+ }
8073
8153
  }
8074
8154
  await signalParentPlayTerminal({
8075
8155
  req,
8076
8156
  status: aborted ? 'cancelled' : 'failed',
8077
- error: message,
8157
+ error: terminalMessage,
8078
8158
  }).catch(() => null);
8079
8159
  recordRunnerPerfTrace({
8080
8160
  req,
@@ -8088,7 +8168,9 @@ async function executeRunRequest(
8088
8168
  await drainRunnerPerfTraces(req);
8089
8169
  throw error;
8090
8170
  } finally {
8091
- clearTimeout(runtimeDeadlineTimer);
8171
+ if (runtimeDeadlineTimer) {
8172
+ clearTimeout(runtimeDeadlineTimer);
8173
+ }
8092
8174
  }
8093
8175
  }
8094
8176
 
@@ -8180,10 +8262,12 @@ function runRequestFromWorkflowParams(
8180
8262
  orgId: String(params.orgId ?? ''),
8181
8263
  playName: String(params.playName ?? ''),
8182
8264
  userEmail: typeof params.userEmail === 'string' ? params.userEmail : null,
8265
+ force: params.force === true,
8266
+ forceToolRefresh: params.forceToolRefresh === true,
8267
+ runAttempt: normalizeWorkerRunAttempt(params.runAttempt),
8183
8268
  runtimeInput: isRecord(params.input)
8184
8269
  ? (params.input as Record<string, unknown>)
8185
8270
  : {},
8186
- force: params.forceToolRefresh === true,
8187
8271
  inlineCsv: isInlineCsv(params.inlineCsv) ? params.inlineCsv : null,
8188
8272
  inputFiles:
8189
8273
  inputFile && inputStorageKey
@@ -8241,6 +8325,14 @@ function runRequestFromWorkflowParams(
8241
8325
  params.coordinatorInternalToken.trim()
8242
8326
  ? params.coordinatorInternalToken.trim()
8243
8327
  : null,
8328
+ runtimeTestFaultHeader:
8329
+ typeof params.runtimeTestFaultHeader === 'string' &&
8330
+ params.runtimeTestFaultHeader.trim()
8331
+ ? params.runtimeTestFaultHeader.trim()
8332
+ : null,
8333
+ testPolicyOverrides: normalizeRuntimeTestPolicyOverrides(
8334
+ params.testPolicyOverrides,
8335
+ ),
8244
8336
  totalRows:
8245
8337
  typeof params.totalRows === 'number' && Number.isFinite(params.totalRows)
8246
8338
  ? params.totalRows
@@ -8310,6 +8402,7 @@ async function persistResultDatasets(
8310
8402
  rows: chunk.map((row) => ({ ...row })),
8311
8403
  runId: req.runId,
8312
8404
  inputOffset,
8405
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
8313
8406
  userEmail: req.userEmail,
8314
8407
  preloadedDbSessions: req.preloadedDbSessions ?? null,
8315
8408
  });
@@ -8346,6 +8439,7 @@ async function persistResultDatasets(
8346
8439
  rows: dataset.rows,
8347
8440
  runId: req.runId,
8348
8441
  inputOffset: 0,
8442
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
8349
8443
  userEmail: req.userEmail,
8350
8444
  });
8351
8445
  }
@@ -8627,7 +8721,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
8627
8721
  try {
8628
8722
  const response = await fetchRuntimeApi(
8629
8723
  req.baseUrl,
8630
- '/api/v2/plays/internal/runtime',
8724
+ '/api/v2/internal/play-runtime',
8631
8725
  {
8632
8726
  method: 'POST',
8633
8727
  headers: {