deepline 0.1.182 → 0.1.183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
  2. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
  3. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +633 -107
  4. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
  5. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1747 -1890
  6. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
  7. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
  8. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
  9. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
  10. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
  11. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
  12. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
  13. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
  14. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
  15. package/dist/bundling-sources/sdk/src/client.ts +41 -0
  16. package/dist/bundling-sources/sdk/src/http.ts +23 -8
  17. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
  18. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  19. package/dist/bundling-sources/sdk/src/types.ts +2 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +132 -10
  21. package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
  23. package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1605 -281
  26. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +105 -6
  27. package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
  28. package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
  29. package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
  31. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
  32. package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
  33. package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
  35. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
  36. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
  37. package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
  40. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
  42. package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
  43. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
  44. package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2037 -262
  47. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
  48. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
  50. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
  51. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
  52. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
  53. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
  54. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
  56. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
  57. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
  58. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
  59. package/dist/cli/index.js +76 -17
  60. package/dist/cli/index.mjs +76 -17
  61. package/dist/index.d.mts +2 -0
  62. package/dist/index.d.ts +2 -0
  63. package/dist/index.js +63 -9
  64. package/dist/index.mjs +63 -9
  65. package/dist/plays/bundle-play-file.mjs +2 -2
  66. package/package.json +1 -1
@@ -36,12 +36,6 @@ import {
36
36
  deterministicMapChunkStepName,
37
37
  type ExecutionPlan,
38
38
  } from '../../../shared_libs/play-runtime/execution-plan';
39
- import {
40
- compileRequestsWithStrategy,
41
- executeChunkedRequests,
42
- type ChunkExecutionResult,
43
- } from '../../../shared_libs/play-runtime/batch-runtime';
44
- import { getPlayRuntimeBatchStrategy } from '../../../shared_libs/play-runtime/play-runtime-batching-registry';
45
39
  import {
46
40
  STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
47
41
  STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
@@ -71,11 +65,10 @@ import {
71
65
  type WorkflowStepLike,
72
66
  } from './child-play-await';
73
67
  import { submitChildPlayThroughCoordinator } from './child-play-submit';
74
- import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
75
68
  import {
76
69
  attachToolResultListDataset,
77
- parseToolExecuteResponse,
78
70
  createToolExecuteResult,
71
+ parseToolExecuteResponse,
79
72
  deserializeToolExecuteResult,
80
73
  isSerializedToolExecuteResult,
81
74
  isToolExecuteResult,
@@ -90,29 +83,35 @@ import {
90
83
  TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS,
91
84
  classifyToolExecuteHttpFailure,
92
85
  createToolExecuteHttpFailureAttemptTracker,
86
+ parseToolExecuteAuthScopeChangedError,
93
87
  } from '../../../shared_libs/play-runtime/tool-execute-retry-policy';
94
88
  import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
95
89
  import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
96
- import type { PlayRuntimeManifestMap } from '../../../shared_libs/plays/compiler-manifest';
90
+ import type {
91
+ PlayRuntimeManifest,
92
+ PlayRuntimeManifestMap,
93
+ } from '../../../shared_libs/plays/compiler-manifest';
97
94
  import {
98
95
  deriveToolRequestIdentity,
99
96
  derivePlayRowIdentity,
100
97
  derivePlayRowIdentityFromKey,
101
98
  sha256Hex,
99
+ stableStringify,
102
100
  } from '../../../shared_libs/plays/row-identity';
103
101
  import { createDeferredPlayDataset } from '../../../shared_libs/plays/dataset';
104
102
  import {
105
103
  buildDurableCtxCallCacheKey,
106
- buildDurableToolCallAuthScopeDigest,
107
- buildDurableToolCallCacheKey,
104
+ buildDurableRunPlaySemanticKey,
108
105
  } from '../../../shared_libs/play-runtime/durable-call-cache';
106
+ import {
107
+ resolveRuntimeToolReceiptWaitMaxAttempts,
108
+ resolveRuntimeToolReceiptWaitTimeoutMs,
109
+ } from '../../../shared_libs/play-runtime/durable-receipt-execution';
109
110
  import {
110
111
  QUERY_RESULT_DATASET_PAGE_SIZE,
111
112
  isCustomerDbDatasetTool,
112
- isQueryResultDatasetReadRequest,
113
113
  isQueryResultDatasetTool,
114
114
  } from '../../../shared_libs/play-runtime/query-result-dataset';
115
- import { buildScopedWorkReceiptKey } from '../../../shared_libs/play-runtime/work-receipts';
116
115
  import { DEDUPE_DUPLICATE_KEY_SAMPLE_CAP } from '../../../shared_libs/play-runtime/map-row-identity';
117
116
  import {
118
117
  getTopLevelPipelineSubsteps,
@@ -151,26 +150,41 @@ import {
151
150
  } from './runtime/dataset-handles';
152
151
  import {
153
152
  runWorkerRuntimeReceiptBoundary,
154
- type WorkerRuntimeReceipt,
155
- type WorkerRuntimeReceiptClaim,
156
- type WorkerRuntimeReceiptStore,
153
+ RuntimeLeaseLostError,
154
+ type WorkerRuntimeReceiptExecutionContext,
157
155
  } from './runtime/receipts';
158
156
  import {
159
- RuntimeReceiptWaitTimeoutError,
160
- resolveRuntimeToolReceiptWaitMaxAttempts,
161
- resolveRuntimeToolReceiptWaitTimeoutMs,
162
- waitForCompletedRuntimeReceipt,
163
- } from '../../../shared_libs/play-runtime/durable-receipt-execution';
164
- import type { RuntimeStepReceipt } from '../../../shared_libs/play-runtime/ctx-types';
157
+ RuntimeReceiptPersistenceError,
158
+ WorkerToolBatchScheduler,
159
+ type WorkerPacingResolver,
160
+ type WorkerToolActionCacheVersionResolver,
161
+ } from './runtime/tool-dispatch';
162
+ import {
163
+ createWorkerWorkBudgetMeter,
164
+ type WorkerWorkBudgetMeter,
165
+ } from './runtime/work-budget';
165
166
  import {
166
- canReclaimFailedWorkerToolReceipt,
167
- markWorkerToolReceiptResultCached,
168
- markWorkerToolReceiptResultExecution,
169
- planWorkerToolReceiptGroups,
170
- reclaimRunningReceiptGroupAfterWait,
171
- resolveWorkerToolReceiptGroupWaitMaxAttempts,
172
- resolveWorkerToolRuntimeTimeoutMs,
173
- } from './runtime/tool-receipts';
167
+ WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES,
168
+ WorkerRunWorkDispatcher,
169
+ type WorkerRunMapBatchesResult,
170
+ } from './runtime/run-work-dispatcher';
171
+ import { WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL } from './runtime/worker-platform-budget';
172
+ import {
173
+ PLAY_RUNTIME_CONTRACT,
174
+ PLAY_RUNTIME_CONTRACT_HEADER,
175
+ } from '../../../shared_libs/play-runtime/runtime-contract';
176
+ import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
177
+ import {
178
+ PLAY_RUNTIME_TEST_FAULT_HEADER,
179
+ type RuntimeTestPolicyOverrides,
180
+ } from '../../../shared_libs/play-runtime/test-runtime-seams';
181
+ import { createRuntimeToolAuthScopeDigestResolver } from '../../../shared_libs/play-runtime/auth-scope-resolver';
182
+ import {
183
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
184
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
185
+ runtimeLeaseHeartbeatIntervalMs,
186
+ } from '../../../shared_libs/play-runtime/lease-policy';
187
+ import { RuntimeSheetRowsBlockedError } from '../../../shared_libs/play-runtime/runtime-sheet-errors';
174
188
  // The harness stub forwards leaf calls (validation, runtime-api HTTP) into
175
189
  // the long-lived Play Harness Worker via env.HARNESS. We import the
176
190
  // `setHarnessBinding` setter eagerly so it's available the moment
@@ -183,9 +197,12 @@ import {
183
197
  // re-bundle harness internals into per-play. Keep that in mind.
184
198
  import {
185
199
  harnessPersistCompletedSheetRows,
200
+ harnessHeartbeatSheetAttempt,
186
201
  harnessReadSheetDatasetRowKeys,
187
202
  harnessReadSheetDatasetRows,
188
203
  harnessReadStagedFileChunk,
204
+ harnessReleaseRuntimeReceipts,
205
+ harnessReleaseSheetAttempt,
189
206
  harnessStartSheetDataset,
190
207
  setHarnessBinding,
191
208
  } from '../../../sdk/src/plays/harness-stub';
@@ -216,7 +233,10 @@ import {
216
233
  stripMapRowOutcomeRuntimeFields,
217
234
  } from '../../../shared_libs/play-runtime/map-row-outcome';
218
235
  import { runtimeSheetSessionScope } from '../../../shared_libs/play-runtime/runtime-sheet-session';
219
- import { chooseWorkerMapRowsPerChunk } from './runtime/map-chunk-plan';
236
+ import {
237
+ chooseWorkerMapDispatchPlan,
238
+ estimateBatchedToolChunkSubrequests,
239
+ } from './runtime/map-chunk-plan';
220
240
  import {
221
241
  applyCsvRenameProjection,
222
242
  cloneCsvAliasedRow,
@@ -228,12 +248,7 @@ import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-f
228
248
  import {
229
249
  createRuntimePersistenceLatch,
230
250
  isRuntimePersistenceCircuitOpenError,
231
- partitionDispatchWaves,
232
- PROVIDER_DISPATCH_WAVE_SIZE,
233
- RuntimePersistenceCircuitOpenError,
234
- shouldShortenCompleteReceiptLadder,
235
251
  tripRuntimePersistenceLatch,
236
- type RuntimePersistenceLatch,
237
252
  } from '../../../shared_libs/play-runtime/persistence-latch';
238
253
  import {
239
254
  createReceiptSalvageBuffer,
@@ -299,8 +314,13 @@ type RunRequest = {
299
314
  playName: string;
300
315
  graphHash?: string | null;
301
316
  userEmail: string | null;
302
- force?: boolean | null;
303
317
  runtimeInput: Record<string, unknown>;
318
+ /** Start a fresh run graph and recompute runtime-sheet rows. */
319
+ force?: boolean;
320
+ /** Explicit durable tool receipt refresh that can re-execute completed provider calls. */
321
+ forceToolRefresh?: boolean;
322
+ /** Monotonic coordinator redispatch epoch for receipt and sheet fencing. */
323
+ runAttempt?: number;
304
324
  /** Optional inline CSV rows (for plays where ctx.csv was passed inline data). */
305
325
  inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
306
326
  /** Staged input files keyed by logical filename (used by ctx.csv). */
@@ -333,6 +353,10 @@ type RunRequest = {
333
353
  coordinatorUrl?: string | null;
334
354
  /** Request-scoped coordinator auth token for preview/dev direct control calls. */
335
355
  coordinatorInternalToken?: string | null;
356
+ /** Request-scoped, dev-only runtime fault injection header for black-box tests. */
357
+ runtimeTestFaultHeader?: string | null;
358
+ /** Request-scoped, dev-only policy overrides for black-box durability tests. */
359
+ testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
336
360
  /**
337
361
  * Total input rows known at submit time, when the dispatcher can count them.
338
362
  * `undefined` when the input streams from R2 (unknown row count).
@@ -348,10 +372,104 @@ type WorkerFileRef = {
348
372
  bytes?: number | null;
349
373
  };
350
374
 
375
+ function normalizeWorkerRunAttempt(value: unknown): number {
376
+ return typeof value === 'number' && Number.isFinite(value)
377
+ ? Math.max(0, Math.floor(value))
378
+ : 0;
379
+ }
380
+
381
+ function normalizePositiveInteger(value: unknown): number | undefined {
382
+ return typeof value === 'number' &&
383
+ Number.isFinite(value) &&
384
+ Number.isInteger(value) &&
385
+ value > 0
386
+ ? value
387
+ : undefined;
388
+ }
389
+
390
+ function normalizeRuntimeTestPolicyOverrides(
391
+ value: unknown,
392
+ ): RuntimeTestPolicyOverrides | null {
393
+ if (!isRecord(value)) return null;
394
+ const overrides: RuntimeTestPolicyOverrides = {};
395
+ const receiptLeaseTtlMs = normalizePositiveInteger(value.receiptLeaseTtlMs);
396
+ if (receiptLeaseTtlMs !== undefined) {
397
+ overrides.receiptLeaseTtlMs = receiptLeaseTtlMs;
398
+ }
399
+ const sheetAttemptLeaseMs = normalizePositiveInteger(
400
+ value.sheetAttemptLeaseMs,
401
+ );
402
+ if (sheetAttemptLeaseMs !== undefined) {
403
+ overrides.sheetAttemptLeaseMs = sheetAttemptLeaseMs;
404
+ }
405
+ const heartbeatIntervalMs = normalizePositiveInteger(
406
+ value.heartbeatIntervalMs,
407
+ );
408
+ if (heartbeatIntervalMs !== undefined) {
409
+ overrides.heartbeatIntervalMs = heartbeatIntervalMs;
410
+ }
411
+ const batchGraceMs = normalizePositiveInteger(value.batchGraceMs);
412
+ if (batchGraceMs !== undefined) {
413
+ overrides.batchGraceMs = batchGraceMs;
414
+ }
415
+ if (isRecord(value.workBudgetYieldLimits)) {
416
+ const limits: NonNullable<
417
+ RuntimeTestPolicyOverrides['workBudgetYieldLimits']
418
+ > = {};
419
+ for (const key of [
420
+ 'elapsed',
421
+ 'subrequest',
422
+ 'provider',
423
+ 'tool',
424
+ 'receipt',
425
+ 'sheet',
426
+ 'log',
427
+ 'egress',
428
+ ] as const) {
429
+ const limit = normalizePositiveInteger(value.workBudgetYieldLimits[key]);
430
+ if (limit !== undefined) {
431
+ limits[key] = limit;
432
+ }
433
+ }
434
+ if (Object.keys(limits).length > 0) {
435
+ overrides.workBudgetYieldLimits = limits;
436
+ }
437
+ }
438
+ return Object.keys(overrides).length > 0 ? overrides : null;
439
+ }
440
+
441
+ function runtimeReceiptLeaseTtlMs(req: RunRequest): number | undefined {
442
+ return req.testPolicyOverrides?.receiptLeaseTtlMs;
443
+ }
444
+
445
+ function runtimeSheetAttemptLeaseTtlMs(req: RunRequest): number | undefined {
446
+ return req.testPolicyOverrides?.sheetAttemptLeaseMs;
447
+ }
448
+
449
+ function runtimePolicyHeartbeatIntervalMs(
450
+ req: RunRequest,
451
+ fallbackLeaseTtlMs: number,
452
+ ): number {
453
+ return (
454
+ req.testPolicyOverrides?.heartbeatIntervalMs ??
455
+ runtimeLeaseHeartbeatIntervalMs(fallbackLeaseTtlMs)
456
+ );
457
+ }
458
+
351
459
  const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
352
460
  const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
353
461
  const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
354
462
  const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
463
+ // A short durable sleep is enough to force a Cloudflare Workflows step
464
+ // checkpoint. The resume re-enters run() as a new Worker invocation with a
465
+ // fresh per-invocation subrequest budget (see the yield callback below), so
466
+ // there is no need to escalate the sleep to "encourage" re-invocation.
467
+ // A 1ms Workflow sleep is too small to reliably create a new platform
468
+ // invocation in preview; the dispatcher then resets its local budget while
469
+ // Cloudflare's per-invocation subrequest counter is still hot. Use a real
470
+ // boundary so budget-yielded maps resume with fresh platform headroom.
471
+ const WORK_DISPATCHER_YIELD_SLEEP_MS = 1_000;
472
+
355
473
  type CustomerDbDatasetRequest = {
356
474
  limit?: number;
357
475
  offset?: number;
@@ -418,7 +536,8 @@ type WorkerEnv = {
418
536
  /**
419
537
  * In-process Fetcher constructed by the coordinator and handed to the
420
538
  * per-graphHash play Worker. When present, runtime callbacks
421
- * (`/api/v2/plays/internal/runtime`, `/api/v2/plays/internal/*`,
539
+ * (`/api/v2/internal/play-runtime`,
540
+ * `/api/v2/internal/play-runtime/*`,
422
541
  * `/api/v2/plays/runtime-tools/*`) skip the public callback URL and route
423
542
  * directly through the coordinator's process to the configured app — saves
424
543
  * the *.workers.dev → CF edge → cloudflared → localhost chain on every
@@ -427,6 +546,7 @@ type WorkerEnv = {
427
546
  */
428
547
  RUNTIME_API?: {
429
548
  runtimeApiCall(input: {
549
+ contract?: number;
430
550
  executorToken: string;
431
551
  baseUrl?: string;
432
552
  path: string;
@@ -663,7 +783,7 @@ async function fetchRuntimeApi(
663
783
  ? Math.max(1, Math.ceil(options.timeoutMsOverride))
664
784
  : path === '/api/v2/plays/run'
665
785
  ? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
666
- : path === '/api/v2/plays/internal/egress-fetch'
786
+ : path === '/api/v2/internal/play-runtime/egress-fetch'
667
787
  ? RUNTIME_API_EGRESS_FETCH_TIMEOUT_MS
668
788
  : /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
669
789
  ? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
@@ -691,6 +811,10 @@ async function fetchRuntimeApi(
691
811
  );
692
812
  }, timeoutMs);
693
813
  });
814
+ const relayAbort = () => controller.abort(abortSignal?.reason);
815
+ if (abortSignal) {
816
+ abortSignal.addEventListener('abort', relayAbort, { once: true });
817
+ }
694
818
  try {
695
819
  const mergedInit: RequestInit = {
696
820
  ...init,
@@ -712,11 +836,10 @@ async function fetchRuntimeApi(
712
836
  timeoutMs,
713
837
  },
714
838
  );
715
- // RUNTIME_API service bindings do not consume RequestInit.signal once the RPC
716
- // is in flight. After dispatch, wait for the owner call to settle so the
717
- // durable receipt records the real provider result instead of failing early
718
- // and inviting duplicate provider work on retry.
719
839
  const response = await Promise.race([responsePromise, timeoutPromise]);
840
+ if (abortSignal?.aborted) {
841
+ throw abortError();
842
+ }
720
843
  if (await isRuntimeApiBindingNotFoundResponse(response)) {
721
844
  throw new Error(
722
845
  `[play-harness] RUNTIME_API service binding could not route ${path}; coordinator returned not found.`,
@@ -732,6 +855,9 @@ async function fetchRuntimeApi(
732
855
  }
733
856
  throw err;
734
857
  } finally {
858
+ if (abortSignal) {
859
+ abortSignal.removeEventListener('abort', relayAbort);
860
+ }
735
861
  if (timeout) clearTimeout(timeout);
736
862
  }
737
863
  }
@@ -748,10 +874,23 @@ async function callRuntimeApiRpcBinding(
748
874
  if (metadata) headers[EXECUTE_TOOL_METADATA_HEADER] = metadata;
749
875
  const contract = h.get(EXECUTE_RESPONSE_CONTRACT_HEADER);
750
876
  if (contract) headers[EXECUTE_RESPONSE_CONTRACT_HEADER] = contract;
877
+ const idempotencyKey = h.get('x-deepline-idempotency-key');
878
+ if (idempotencyKey) {
879
+ headers['x-deepline-idempotency-key'] = idempotencyKey;
880
+ }
881
+ const cacheControl = h.get('cache-control');
882
+ if (cacheControl) headers['cache-control'] = cacheControl;
883
+ const pragma = h.get('pragma');
884
+ if (pragma) headers.pragma = pragma;
885
+ const runtimeTestFaultHeader = h.get(PLAY_RUNTIME_TEST_FAULT_HEADER);
886
+ if (runtimeTestFaultHeader) {
887
+ headers[PLAY_RUNTIME_TEST_FAULT_HEADER] = runtimeTestFaultHeader;
888
+ }
751
889
  const responseIntent = h.get(EXECUTE_RESPONSE_INTENT_HEADER);
752
890
  if (responseIntent) headers[EXECUTE_RESPONSE_INTENT_HEADER] = responseIntent;
753
891
  const rawBody = typeof init.body === 'string' ? init.body : '';
754
892
  const result = await binding.runtimeApiCall({
893
+ contract: PLAY_RUNTIME_CONTRACT,
755
894
  executorToken: authorization.replace(/^Bearer\s+/i, '').trim(),
756
895
  baseUrl: input.baseUrl,
757
896
  path: input.path,
@@ -784,6 +923,7 @@ function runtimeApiHeaders(
784
923
  includeVercelBypass: boolean,
785
924
  ): Headers {
786
925
  const next = new Headers(headers);
926
+ next.set(PLAY_RUNTIME_CONTRACT_HEADER, String(PLAY_RUNTIME_CONTRACT));
787
927
  if (includeVercelBypass) {
788
928
  const bypassToken = cachedVercelProtectionBypassToken();
789
929
  if (bypassToken) {
@@ -976,9 +1116,172 @@ async function drainRunnerPerfTraces(req: RunRequest): Promise<void> {
976
1116
  ]);
977
1117
  }
978
1118
 
1119
+ let workerRequestSequence = 0;
1120
+
979
1121
  function makeRequestId(): string {
980
- // Workers crypto.randomUUID is available without nodejs_compat.
981
- return crypto.randomUUID();
1122
+ workerRequestSequence = (workerRequestSequence + 1) % Number.MAX_SAFE_INTEGER;
1123
+ return `worker-request-${workerRequestSequence}`;
1124
+ }
1125
+
1126
+ function makeRuntimeSheetAttempt(input: {
1127
+ runId: string;
1128
+ runAttempt?: number | null;
1129
+ mapName: string;
1130
+ chunkIndex: number | 'finalize';
1131
+ }): {
1132
+ attemptId: string;
1133
+ attemptOwnerRunId: string;
1134
+ attemptExpiresAt: string | null;
1135
+ attemptSeq: number;
1136
+ } {
1137
+ const runAttempt =
1138
+ typeof input.runAttempt === 'number' && Number.isFinite(input.runAttempt)
1139
+ ? Math.max(0, Math.floor(input.runAttempt))
1140
+ : 0;
1141
+ const attemptKey = sha256Hex(
1142
+ JSON.stringify({
1143
+ runId: input.runId,
1144
+ runAttempt,
1145
+ mapName: input.mapName,
1146
+ chunkIndex: input.chunkIndex,
1147
+ }),
1148
+ ).slice(0, 16);
1149
+ return {
1150
+ attemptId: `sheet:${input.mapName}:${input.chunkIndex}:attempt-${runAttempt}:${attemptKey}`,
1151
+ attemptOwnerRunId: input.runId,
1152
+ attemptExpiresAt: null,
1153
+ attemptSeq: runAttempt,
1154
+ };
1155
+ }
1156
+
1157
+ class RuntimeSheetAttemptLeaseLostError extends Error {
1158
+ readonly rowKeys: string[];
1159
+ readonly runId: string;
1160
+ readonly attemptId: string;
1161
+ readonly tableNamespace: string;
1162
+
1163
+ constructor(input: {
1164
+ rowKeys: string[];
1165
+ runId: string;
1166
+ attemptId: string;
1167
+ tableNamespace: string;
1168
+ cause?: unknown;
1169
+ }) {
1170
+ const rowSummary =
1171
+ input.rowKeys.length === 1
1172
+ ? input.rowKeys[0]!
1173
+ : `${input.rowKeys.length} rows`;
1174
+ super(
1175
+ `Runtime sheet attempt ${input.attemptId} lost ownership of ${rowSummary} in ${input.tableNamespace}.`,
1176
+ { cause: input.cause },
1177
+ );
1178
+ this.name = 'RuntimeSheetAttemptLeaseLostError';
1179
+ this.rowKeys = input.rowKeys;
1180
+ this.runId = input.runId;
1181
+ this.attemptId = input.attemptId;
1182
+ this.tableNamespace = input.tableNamespace;
1183
+ }
1184
+ }
1185
+
1186
+ async function executeWithRuntimeSheetAttemptHeartbeat<T>(input: {
1187
+ req: RunRequest;
1188
+ tableNamespace: string;
1189
+ sheetContract: PlaySheetContract;
1190
+ attemptId?: string | null;
1191
+ attemptOwnerRunId?: string | null;
1192
+ attemptSeq?: number | null;
1193
+ rowKeys: string[];
1194
+ execute: () => Promise<T> | T;
1195
+ }): Promise<T> {
1196
+ const attemptId = input.attemptId?.trim();
1197
+ const rowKeys = [
1198
+ ...new Set(input.rowKeys.map((key) => key.trim()).filter(Boolean)),
1199
+ ];
1200
+ if (!attemptId || rowKeys.length === 0) {
1201
+ return await input.execute();
1202
+ }
1203
+ const heartbeatIntervalMs = runtimePolicyHeartbeatIntervalMs(
1204
+ input.req,
1205
+ runtimeSheetAttemptLeaseTtlMs(input.req) ??
1206
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
1207
+ );
1208
+
1209
+ let stopped = false;
1210
+ let timeout: ReturnType<typeof setTimeout> | null = null;
1211
+ let rejectLeaseLost!: (error: RuntimeSheetAttemptLeaseLostError) => void;
1212
+ const leaseLost = new Promise<never>((_, reject) => {
1213
+ rejectLeaseLost = reject;
1214
+ });
1215
+
1216
+ const stop = () => {
1217
+ stopped = true;
1218
+ if (timeout !== null) {
1219
+ clearTimeout(timeout);
1220
+ timeout = null;
1221
+ }
1222
+ };
1223
+
1224
+ const heartbeatOnce = async () => {
1225
+ const result = await harnessHeartbeatSheetAttempt({
1226
+ ...runtimeSheetSessionScope(input.req),
1227
+ tableNamespace: input.tableNamespace,
1228
+ sheetContract: input.sheetContract,
1229
+ runId: input.req.runId,
1230
+ attemptId,
1231
+ attemptOwnerRunId: input.attemptOwnerRunId ?? input.req.runId,
1232
+ attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
1233
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
1234
+ keys: rowKeys,
1235
+ });
1236
+ const renewed = new Set(result.renewedKeys);
1237
+ const lostKeys = rowKeys.filter((key) => !renewed.has(key));
1238
+ if (lostKeys.length > 0) {
1239
+ throw new RuntimeSheetAttemptLeaseLostError({
1240
+ rowKeys: lostKeys,
1241
+ runId: input.req.runId,
1242
+ attemptId,
1243
+ tableNamespace: input.tableNamespace,
1244
+ });
1245
+ }
1246
+ };
1247
+
1248
+ const schedule = () => {
1249
+ timeout = setTimeout(() => {
1250
+ void (async () => {
1251
+ try {
1252
+ await heartbeatOnce();
1253
+ } catch (error) {
1254
+ stop();
1255
+ if (error instanceof RuntimeSheetAttemptLeaseLostError) {
1256
+ rejectLeaseLost(error);
1257
+ } else {
1258
+ rejectLeaseLost(
1259
+ new RuntimeSheetAttemptLeaseLostError({
1260
+ rowKeys,
1261
+ runId: input.req.runId,
1262
+ attemptId,
1263
+ tableNamespace: input.tableNamespace,
1264
+ cause: error,
1265
+ }),
1266
+ );
1267
+ }
1268
+ return;
1269
+ }
1270
+ if (!stopped) {
1271
+ schedule();
1272
+ }
1273
+ })();
1274
+ }, heartbeatIntervalMs);
1275
+ };
1276
+
1277
+ schedule();
1278
+ const execution = Promise.resolve().then(input.execute);
1279
+ execution.catch(() => {});
1280
+ try {
1281
+ return await Promise.race([execution, leaseLost]);
1282
+ } finally {
1283
+ stop();
1284
+ }
982
1285
  }
983
1286
 
984
1287
  /**
@@ -1004,6 +1307,11 @@ async function postRuntimeApi<T>(
1004
1307
  baseUrl: string,
1005
1308
  executorToken: string,
1006
1309
  body: unknown,
1310
+ options: {
1311
+ runtimeTestFaultHeader?: string | null;
1312
+ timeoutMsOverride?: number;
1313
+ timeoutErrorMessage?: string;
1314
+ } = {},
1007
1315
  ): Promise<T> {
1008
1316
  // Routes through the in-process RUNTIME_API service binding. Missing binding
1009
1317
  // is an infra error in workers_edge, not a reason to fall back to public HTTP.
@@ -1016,15 +1324,29 @@ async function postRuntimeApi<T>(
1016
1324
  ) {
1017
1325
  let res: Response;
1018
1326
  try {
1019
- res = await fetchRuntimeApi(baseUrl, '/api/v2/plays/internal/runtime', {
1020
- method: 'POST',
1021
- headers: {
1022
- 'content-type': 'application/json',
1023
- authorization: `Bearer ${executorToken}`,
1024
- 'x-deepline-request-id': makeRequestId(),
1327
+ res = await fetchRuntimeApi(
1328
+ baseUrl,
1329
+ '/api/v2/internal/play-runtime',
1330
+ {
1331
+ method: 'POST',
1332
+ headers: {
1333
+ 'content-type': 'application/json',
1334
+ authorization: `Bearer ${executorToken}`,
1335
+ 'x-deepline-request-id': makeRequestId(),
1336
+ ...(options.runtimeTestFaultHeader?.trim()
1337
+ ? {
1338
+ [PLAY_RUNTIME_TEST_FAULT_HEADER]:
1339
+ options.runtimeTestFaultHeader.trim(),
1340
+ }
1341
+ : {}),
1342
+ },
1343
+ body: serializedBody,
1025
1344
  },
1026
- body: serializedBody,
1027
- });
1345
+ {
1346
+ timeoutMsOverride: options.timeoutMsOverride,
1347
+ timeoutErrorMessage: options.timeoutErrorMessage,
1348
+ },
1349
+ );
1028
1350
  } catch (error) {
1029
1351
  lastError = error;
1030
1352
  if (
@@ -1108,9 +1430,10 @@ async function postRuntimeApiBestEffort(
1108
1430
  baseUrl: string,
1109
1431
  executorToken: string,
1110
1432
  body: unknown,
1433
+ options: { runtimeTestFaultHeader?: string | null } = {},
1111
1434
  ): Promise<boolean> {
1112
1435
  try {
1113
- await postRuntimeApi(baseUrl, executorToken, body);
1436
+ await postRuntimeApi(baseUrl, executorToken, body, options);
1114
1437
  return true;
1115
1438
  } catch (error) {
1116
1439
  console.error(
@@ -1122,6 +1445,43 @@ async function postRuntimeApiBestEffort(
1122
1445
  }
1123
1446
  }
1124
1447
 
1448
+ async function readDurableChildRunTerminalState(input: {
1449
+ baseUrl: string;
1450
+ executorToken: string;
1451
+ parentRunId: string;
1452
+ childRunId: string;
1453
+ childPlayName: string;
1454
+ timeoutMs?: number;
1455
+ }): Promise<{ data?: unknown } | null> {
1456
+ const response = await postRuntimeApi<{
1457
+ state?: {
1458
+ data?: unknown;
1459
+ reason?: string;
1460
+ status?: string;
1461
+ runId?: string;
1462
+ playName?: string;
1463
+ parentRunId?: string | null;
1464
+ } | null;
1465
+ }>(
1466
+ input.baseUrl,
1467
+ input.executorToken,
1468
+ {
1469
+ action: 'read_child_run_terminal_snapshot',
1470
+ parentRunId: input.parentRunId,
1471
+ childRunId: input.childRunId,
1472
+ childPlayName: input.childPlayName,
1473
+ },
1474
+ {
1475
+ timeoutMsOverride: input.timeoutMs,
1476
+ timeoutErrorMessage: `[play-harness] child run terminal snapshot read timed out after ${Math.max(
1477
+ 1,
1478
+ Math.ceil(input.timeoutMs ?? RUNTIME_API_TIMEOUT_MS),
1479
+ )}ms. childRunId=${input.childRunId}`,
1480
+ },
1481
+ );
1482
+ return response.state ?? null;
1483
+ }
1484
+
1125
1485
  function workflowEventType(name: string): string {
1126
1486
  const normalized = name
1127
1487
  .trim()
@@ -1231,6 +1591,27 @@ async function hashChildPlayEventKey(input: unknown): Promise<string> {
1231
1591
  return (await hashJson(input)).slice(0, 32);
1232
1592
  }
1233
1593
 
1594
+ function workerChildRevisionFingerprint(input: {
1595
+ playName: string;
1596
+ manifest: PlayRuntimeManifest | null | undefined;
1597
+ }): string | null {
1598
+ const manifest = input.manifest;
1599
+ if (!manifest) return null;
1600
+ return sha256Hex(
1601
+ stableStringify({
1602
+ playId: manifest.playName?.trim() || input.playName,
1603
+ codeFormat: 'cjs_module',
1604
+ artifactHash: manifest.artifactHash ?? null,
1605
+ graphHash: manifest.graphHash ?? null,
1606
+ sourceHash: null,
1607
+ sourceCodeHash:
1608
+ typeof manifest.sourceCode === 'string'
1609
+ ? sha256Hex(manifest.sourceCode)
1610
+ : null,
1611
+ }),
1612
+ );
1613
+ }
1614
+
1234
1615
  async function childPlayEventKey(input: {
1235
1616
  key: string;
1236
1617
  workflowId: string;
@@ -1327,7 +1708,14 @@ async function signalParentPlayTerminal(input: {
1327
1708
 
1328
1709
  async function executeTool(
1329
1710
  req: RunRequest,
1330
- args: { id: string; toolId: string; input: Record<string, unknown> },
1711
+ args: {
1712
+ id: string;
1713
+ toolId: string;
1714
+ input: Record<string, unknown>;
1715
+ durableCallReceiptKey?: string | null;
1716
+ executionAuthScopeDigest?: string | null;
1717
+ providerIdempotencyKey?: string | null;
1718
+ },
1331
1719
  workflowStep?: WorkflowStep,
1332
1720
  onProviderBackpressure?: (retryAfterMs: number) => void,
1333
1721
  onRetryAttempt?: () => void,
@@ -1365,7 +1753,14 @@ async function executeTool(
1365
1753
 
1366
1754
  async function executeToolWithLifecycle(
1367
1755
  req: RunRequest,
1368
- args: { id: string; toolId: string; input: Record<string, unknown> },
1756
+ args: {
1757
+ id: string;
1758
+ toolId: string;
1759
+ input: Record<string, unknown>;
1760
+ durableCallReceiptKey?: string | null;
1761
+ executionAuthScopeDigest?: string | null;
1762
+ providerIdempotencyKey?: string | null;
1763
+ },
1369
1764
  workflowStep: WorkflowStep | undefined,
1370
1765
  callbacks: WorkerCtxCallbacks | undefined,
1371
1766
  onProviderBackpressure?: (retryAfterMs: number) => void,
@@ -1461,24 +1856,29 @@ async function waitForSyntheticIntegrationEvent(
1461
1856
  typeof input.timeout_ms === 'number' && Number.isFinite(input.timeout_ms)
1462
1857
  ? Math.max(1, Math.round(input.timeout_ms))
1463
1858
  : 30_000;
1464
- await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
1465
- action: 'append_run_events',
1466
- playId: req.runId,
1467
- events: [
1468
- {
1469
- type: 'log.appended',
1470
- runId: req.runId,
1471
- // 'system' (windowed text-dedupe channel), NOT 'worker': this line is
1472
- // emitted outside the harness log buffer, so it has no positional
1473
- // channelOffset and must not pollute the worker channel cursor.
1474
- source: 'system',
1475
- occurredAt: nowMs(),
1476
- lines: [
1477
- `Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
1478
- ],
1479
- } satisfies PlayRunLedgerEvent,
1480
- ],
1481
- });
1859
+ await postRuntimeApiBestEffort(
1860
+ req.baseUrl,
1861
+ req.executorToken,
1862
+ {
1863
+ action: 'append_run_events',
1864
+ playId: req.runId,
1865
+ events: [
1866
+ {
1867
+ type: 'log.appended',
1868
+ runId: req.runId,
1869
+ // 'system' (windowed text-dedupe channel), NOT 'worker': this line is
1870
+ // emitted outside the harness log buffer, so it has no positional
1871
+ // channelOffset and must not pollute the worker channel cursor.
1872
+ source: 'system',
1873
+ occurredAt: nowMs(),
1874
+ lines: [
1875
+ `Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
1876
+ ],
1877
+ } satisfies PlayRunLedgerEvent,
1878
+ ],
1879
+ },
1880
+ { runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null },
1881
+ );
1482
1882
  try {
1483
1883
  const event = (await (
1484
1884
  workflowStep.waitForEvent as unknown as (
@@ -1526,7 +1926,14 @@ async function waitForSyntheticIntegrationEvent(
1526
1926
 
1527
1927
  async function callToolDirect(
1528
1928
  req: RunRequest,
1529
- args: { id: string; toolId: string; input: Record<string, unknown> },
1929
+ args: {
1930
+ id: string;
1931
+ toolId: string;
1932
+ input: Record<string, unknown>;
1933
+ durableCallReceiptKey?: string | null;
1934
+ executionAuthScopeDigest?: string | null;
1935
+ providerIdempotencyKey?: string | null;
1936
+ },
1530
1937
  onProviderBackpressure?: (retryAfterMs: number) => void,
1531
1938
  // Invoked once per in-process retry attempt (429 / retryable 5xx / synthetic
1532
1939
  // transient) so the Governor charges chargeBudget('retry') per attempt — the
@@ -1542,6 +1949,16 @@ async function callToolDirect(
1542
1949
  ): Promise<ToolExecuteResult> {
1543
1950
  const { id, toolId, input } = args;
1544
1951
  const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
1952
+ const durableCallReceiptKey = args.durableCallReceiptKey?.trim() || null;
1953
+ const executionAuthScopeDigest =
1954
+ durableCallReceiptKey && args.executionAuthScopeDigest
1955
+ ? args.executionAuthScopeDigest.trim() || null
1956
+ : null;
1957
+ const providerIdempotencyKey =
1958
+ args.providerIdempotencyKey?.trim() || durableCallReceiptKey;
1959
+ const deeplineRequestId = providerIdempotencyKey
1960
+ ? `ctx-tool-${sha256Hex(providerIdempotencyKey).slice(0, 32)}`
1961
+ : null;
1545
1962
  let lastError: Error | null = null;
1546
1963
  const httpFailureAttempts = createToolExecuteHttpFailureAttemptTracker();
1547
1964
  let requestAttempt = 0;
@@ -1564,7 +1981,12 @@ async function callToolDirect(
1564
1981
  headers: {
1565
1982
  'content-type': 'application/json',
1566
1983
  authorization: `Bearer ${req.executorToken}`,
1567
- 'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
1984
+ 'x-deepline-request-id':
1985
+ deeplineRequestId ??
1986
+ `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
1987
+ ...(providerIdempotencyKey
1988
+ ? { 'x-deepline-idempotency-key': providerIdempotencyKey }
1989
+ : {}),
1568
1990
  [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
1569
1991
  [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
1570
1992
  [EXECUTE_TOOL_METADATA_HEADER]: 'true',
@@ -1573,6 +1995,16 @@ async function callToolDirect(
1573
1995
  payload: input,
1574
1996
  metadata: {
1575
1997
  parent_run_id: req.runId,
1998
+ ...(durableCallReceiptKey
1999
+ ? {
2000
+ durable_call_receipt_key: durableCallReceiptKey,
2001
+ ...(executionAuthScopeDigest
2002
+ ? {
2003
+ execution_auth_scope_digest: executionAuthScopeDigest,
2004
+ }
2005
+ : {}),
2006
+ }
2007
+ : {}),
1576
2008
  ...(directOptions?.customerDbDataset
1577
2009
  ? {
1578
2010
  query_result_dataset: {
@@ -1605,6 +2037,9 @@ async function callToolDirect(
1605
2037
  : {}),
1606
2038
  }
1607
2039
  : {}),
2040
+ ...(providerIdempotencyKey
2041
+ ? { provider_idempotency_key: providerIdempotencyKey }
2042
+ : {}),
1608
2043
  },
1609
2044
  ...(req.integrationMode
1610
2045
  ? { integration_mode: req.integrationMode }
@@ -1652,6 +2087,13 @@ async function callToolDirect(
1652
2087
  }
1653
2088
 
1654
2089
  const text = await res.text().catch(() => '');
2090
+ const authScopeChanged = parseToolExecuteAuthScopeChangedError({
2091
+ status: res.status,
2092
+ bodyText: text,
2093
+ });
2094
+ if (authScopeChanged) {
2095
+ throw authScopeChanged;
2096
+ }
1655
2097
  const httpFailureAttempt = httpFailureAttempts.next({
1656
2098
  toolId,
1657
2099
  status: res.status,
@@ -1685,48 +2127,6 @@ async function callToolDirect(
1685
2127
  throw lastError ?? new Error(`tool ${toolId} failed before execution.`);
1686
2128
  }
1687
2129
 
1688
- function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
1689
- if (toolId === 'test_rate_limit') {
1690
- // Batched members resolve metadata through this fallback because the lean
1691
- // worker does not bundle the catalog. It MUST mirror the same
1692
- // `email_status: emailStatus({...})` contract registered in
1693
- // src/lib/integrations/test/index.ts so batched results normalize to the
1694
- // same rich email_status OBJECT (status from statusMap, catch_all as a
1695
- // signal) that the single-execute real-metadata path produces. The legacy
1696
- // `transforms:['emailStatus']` + mx_security_gateway→'catch_all' string
1697
- // override coarsened email_status into a bare string and predates the
1698
- // emailStatus object contract (#1466).
1699
- return {
1700
- toolId,
1701
- extractors: {
1702
- email_status: {
1703
- paths: ['email_status'],
1704
- emailStatus: {
1705
- provider: 'test',
1706
- rawStatus: ['email_status'],
1707
- catchAll: ['mx_security_gateway'],
1708
- statusMap: {
1709
- valid: { status: 'valid', verdict: 'send', verified: true },
1710
- invalid: { status: 'invalid', verdict: 'drop', verified: false },
1711
- catch_all: {
1712
- status: 'catch_all',
1713
- verdict: 'verify_next',
1714
- verified: false,
1715
- },
1716
- unknown: { status: 'unknown', verdict: 'hold', verified: false },
1717
- },
1718
- },
1719
- },
1720
- },
1721
- targetGetters: {
1722
- email: ['email', 'value'],
1723
- email_status: ['email_status'],
1724
- },
1725
- };
1726
- }
1727
- return { toolId };
1728
- }
1729
-
1730
2130
  function wrapWorkerToolResult(
1731
2131
  toolId: string,
1732
2132
  result: unknown,
@@ -1767,10 +2167,10 @@ function attachWorkerCustomerDbDatasetResult(
1767
2167
  typeof options.input.sql === 'string'
1768
2168
  ? options.input.sql
1769
2169
  : typeof options.input.query === 'string'
1770
- ? options.input.query
1771
- : typeof raw?.sql === 'string'
1772
- ? raw.sql
1773
- : null;
2170
+ ? options.input.query
2171
+ : typeof raw?.sql === 'string'
2172
+ ? raw.sql
2173
+ : null;
1774
2174
  if (!dataset || totalRows === null || !sql) {
1775
2175
  return result;
1776
2176
  }
@@ -1880,1455 +2280,93 @@ type RecordedStepProgramOutput = {
1880
2280
 
1881
2281
  type WorkerStepProgramRecorder = {
1882
2282
  parentField: string;
1883
- path: string[];
1884
- outputs: RecordedStepProgramOutput[];
1885
- onOutput?: (output: RecordedStepProgramOutput) => void | Promise<void>;
1886
- };
1887
-
1888
- type WorkerStepResolution = {
1889
- value: unknown;
1890
- status?: 'skipped';
1891
- };
1892
-
1893
- type WorkerToolBatchRequest = {
1894
- id: string;
1895
- cacheKey: string;
1896
- receiptKey: string | null;
1897
- force: boolean;
1898
- receiptWaitMaxAttempts: number;
1899
- runtimeTimeoutMs?: number;
1900
- toolId: string;
1901
- input: Record<string, unknown>;
1902
- workflowStep?: WorkflowStep;
1903
- directOptions?: WorkerToolDirectOptions;
1904
- resolve: (value: unknown) => void;
1905
- reject: (error: unknown) => void;
1906
- };
1907
-
1908
- type ClaimedWorkerToolBatchRequest = {
1909
- request: WorkerToolBatchRequest;
1910
- receiptKey: string | null;
1911
- followers: WorkerToolBatchRequest[];
1912
- };
1913
-
1914
- type PreparedWorkerToolBatchRequests = {
1915
- claimedRequests: ClaimedWorkerToolBatchRequest[];
1916
- deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[];
1917
- };
1918
-
1919
- function resolveClaimedWorkerToolRuntimeTimeoutMs(
1920
- claimedRequests: ClaimedWorkerToolBatchRequest[],
1921
- input: { runtimeDeadlineMs?: number },
1922
- ): number | undefined {
1923
- return resolveWorkerToolRuntimeTimeoutMs(claimedRequests, {
1924
- resolveOwnerTimeoutMs: (request) =>
1925
- resolveToolRuntimeApiTimeout({
1926
- requestInput: request.input,
1927
- timeoutMs: request.runtimeTimeoutMs,
1928
- runtimeDeadlineMs: input.runtimeDeadlineMs,
1929
- }).timeoutMs,
1930
- });
1931
- }
1932
-
1933
- const WORKER_TOOL_BATCH_GRACE_MS = 250;
1934
- const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
1935
- const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
1936
- const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
1937
- const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 1_000;
1938
- const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
1939
- /**
1940
- * Bounded number of per-row failure samples carried in chunk summaries and the
1941
- * map's terminal partial-failure log. Every failed row is persisted with its
1942
- * full error in the runtime sheet; the samples just keep run logs readable.
1943
- */
1944
- const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
1945
-
1946
- class RuntimeReceiptPersistenceError extends Error {
1947
- readonly errors: unknown[];
1948
-
1949
- constructor(message: string, errors: unknown[] = []) {
1950
- super(message);
1951
- this.name = 'RuntimeReceiptPersistenceError';
1952
- this.errors = errors;
1953
- }
1954
- }
1955
-
1956
- function isRuntimeReceiptPersistenceError(error: unknown): boolean {
1957
- if (!error || typeof error !== 'object') return false;
1958
- if (error instanceof RuntimeReceiptPersistenceError) return true;
1959
- if (
1960
- error instanceof Error &&
1961
- error.name === 'RuntimeReceiptPersistenceError'
1962
- ) {
1963
- return true;
1964
- }
1965
- const nestedErrors = (error as { errors?: unknown }).errors;
1966
- return (
1967
- Array.isArray(nestedErrors) &&
1968
- nestedErrors.some(isRuntimeReceiptPersistenceError)
1969
- );
1970
- }
1971
-
1972
- function isRunFatalWorkerRowError(error: unknown): boolean {
1973
- return (
1974
- isCoordinatorProgressAuthorizationError(error) ||
1975
- isRowIsolationExemptError(error) ||
1976
- isRuntimeReceiptPersistenceError(error) ||
1977
- isRuntimePersistenceCircuitOpenError(error)
1978
- );
1979
- }
1980
-
1981
- /**
1982
- * completeReceipt salvage retry ladder. On a persistence failure after a billed
1983
- * call we retry the receipt completion past the server-side 5s connect window:
1984
- * 1 initial attempt + 3 retries at 1s / 3s / 9s (single isolate — no jitter).
1985
- * Bounded by the run deadline (`runtimeDeadlineMs`); we stop early unless there
1986
- * is at least one backoff's worth plus headroom of budget left.
1987
- */
1988
- const COMPLETE_RECEIPT_RETRY_BACKOFFS_MS = [1_000, 3_000, 9_000] as const;
1989
- const COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS = 10_000;
1990
-
1991
- // Fallback batch-chunk parallelism when a tool declares no provider rate hints.
1992
- // Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
1993
- // their previous batching behavior; declared providers tighten via the
1994
- // Governor's suggestedParallelism.
1995
- const WORKER_TOOL_BATCH_DEFAULT_PARALLELISM = 4;
1996
-
1997
- /**
1998
- * In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
1999
- * throughput pacing (provider or Deepline limiter), not a tool defect, so it
2000
- * gets more patience than the generic 3-attempt budget: with retry-after-aware
2001
- * escalating delays (capped at 5s) this absorbs roughly 25s of sustained
2002
- * throttling before the call fails. Every retry still charges the Governor's
2003
- * retry budget, so a runaway storm stays bounded and loud.
2004
- */
2005
-
2006
- function sleepWorkerMs(ms: number): Promise<void> {
2007
- return new Promise((resolve) => setTimeout(resolve, ms));
2008
- }
2009
-
2010
- function workerDurableToolCallCacheKey(input: {
2011
- req: RunRequest;
2012
- toolId: string;
2013
- requestInput: Record<string, unknown>;
2014
- providerActionVersion: string;
2015
- staleAfterSeconds?: number | null;
2016
- }): string {
2017
- return buildDurableToolCallCacheKey({
2018
- orgId: input.req.orgId,
2019
- playId: input.req.playName,
2020
- toolId: input.toolId,
2021
- requestInput: input.requestInput,
2022
- authScopeDigest: buildDurableToolCallAuthScopeDigest({
2023
- orgId: input.req.orgId,
2024
- userEmail: input.req.userEmail,
2025
- toolId: input.toolId,
2026
- }),
2027
- providerActionVersion: input.providerActionVersion,
2028
- staleAfterSeconds: input.staleAfterSeconds,
2029
- });
2030
- }
2031
-
2032
- function workerRuntimeReceiptKey(input: {
2033
- req: RunRequest;
2034
- key: string;
2035
- }): string {
2036
- const orgId = input.req.orgId?.trim() || 'org';
2037
- if (input.key.startsWith(`ctx:${orgId}:`)) {
2038
- return input.key;
2039
- }
2040
- return buildScopedWorkReceiptKey({
2041
- orgId: input.req.orgId,
2042
- playName: input.req.playName,
2043
- key: input.key,
2044
- });
2045
- }
2046
-
2047
- function stepProgramColumnName(parentField: string, stepId: string): string {
2048
- return sqlSafePlayColumnName(`${parentField}.${stepId}`);
2049
- }
2050
-
2051
- class WorkerToolBatchScheduler {
2052
- private queue: WorkerToolBatchRequest[] = [];
2053
- private scheduled = false;
2054
-
2055
- constructor(
2056
- private readonly req: RunRequest,
2057
- private readonly governor: PlayExecutionGovernor,
2058
- private readonly resolvePacing: WorkerPacingResolver,
2059
- private readonly resolveToolActionCacheVersion: WorkerToolActionCacheVersionResolver,
2060
- private readonly abortSignal?: AbortSignal,
2061
- private readonly onRequestsSettled?: (count: number) => void,
2062
- private readonly callbacks?: WorkerCtxCallbacks,
2063
- private readonly receiptStore?: WorkerRuntimeReceiptStore,
2064
- private readonly allowLocalRetryReceipts = false,
2065
- private readonly runtimeDeadlineMs?: number,
2066
- // Map-wide (or ctx-wide for the root scheduler) persistence-failure circuit
2067
- // breaker. Shared by every row worker so the first persistence failure
2068
- // halts NEW provider dispatch across the whole map.
2069
- private readonly latch: RuntimePersistenceLatch = createRuntimePersistenceLatch(),
2070
- // Run-scoped buffer for orphaned billed results whose receipt could not be
2071
- // completed even after the retry ladder.
2072
- private readonly salvage?: ReceiptSalvageBuffer,
2073
- ) {}
2074
-
2075
- /**
2076
- * Run `completeReceipt`/`completeReceipts` with the salvage retry ladder.
2077
- * Retries a receipt-completion failure past the server-side 5s connect window
2078
- * so a transient blip does not orphan a billed result. Bounded by the run
2079
- * deadline; skips retries when the latch has already burned a ladder in this
2080
- * map so many in-flight orphans do not each serialize a long loop.
2081
- */
2082
- private async completeReceiptWithRetryLadder<T>(
2083
- run: () => Promise<T>,
2084
- onGiveUp?: (attempts: number) => void,
2085
- ): Promise<T> {
2086
- let attempt = 0;
2087
- // A ladder that has already started retrying keeps its full budget even if
2088
- // a peer trips the latch mid-flight — only FRESH entrants collapse once the
2089
- // failure is confirmed. This is what guarantees the first ladder rides out
2090
- // a genuine transient blip.
2091
- let committedToFullLadder = false;
2092
- const giveUp = (error: unknown): never => {
2093
- this.latch.latchRetryBudgetExhausted = true;
2094
- onGiveUp?.(attempt);
2095
- throw error;
2096
- };
2097
- while (true) {
2098
- try {
2099
- return await run();
2100
- } catch (error) {
2101
- attempt += 1;
2102
- if (
2103
- shouldShortenCompleteReceiptLadder(this.latch, committedToFullLadder) ||
2104
- attempt > COMPLETE_RECEIPT_RETRY_BACKOFFS_MS.length ||
2105
- this.abortSignal?.aborted
2106
- ) {
2107
- return giveUp(error);
2108
- }
2109
- const backoffMs = COMPLETE_RECEIPT_RETRY_BACKOFFS_MS[attempt - 1]!;
2110
- // resolveRuntimeDeadlineRemainingMs THROWS WorkflowAbortError once the
2111
- // run deadline has passed — the correct loud outcome (there is no
2112
- // separate per-chunk step timeout). Route that throw through giveUp so
2113
- // the billed result is still salvaged and the ladder budget flag is
2114
- // set: a billed output must never lose its durable trace because the
2115
- // clock ran out mid-ladder.
2116
- let remainingMs: number;
2117
- try {
2118
- remainingMs = resolveRuntimeDeadlineRemainingMs(
2119
- this.runtimeDeadlineMs,
2120
- );
2121
- } catch (deadlineError) {
2122
- return giveUp(deadlineError);
2123
- }
2124
- if (remainingMs <= backoffMs + COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS) {
2125
- return giveUp(error);
2126
- }
2127
- // Past every give-up gate: this ladder is now committed to the full
2128
- // budget and must not be collapsed by a peer tripping the latch mid-sleep.
2129
- committedToFullLadder = true;
2130
- await sleepWorkerMs(backoffMs);
2131
- }
2132
- }
2133
- }
2134
-
2135
- /**
2136
- * Record a billed-but-unpersisted tool output into the run-scoped salvage
2137
- * buffer. Called only after the completeReceipt ladder is exhausted, with the
2138
- * exact serialized payload a receipt would have stored.
2139
- */
2140
- private recordReceiptSalvage(input: {
2141
- claimed: ClaimedWorkerToolBatchRequest;
2142
- output: unknown;
2143
- attempts: number;
2144
- }): void {
2145
- if (!this.salvage || !input.claimed.receiptKey) return;
2146
- this.salvage.push({
2147
- receiptKey: input.claimed.receiptKey,
2148
- toolId: input.claimed.request.toolId,
2149
- cacheKey: input.claimed.request.cacheKey,
2150
- output: input.output,
2151
- completeAttempts: input.attempts,
2152
- firstErrorAt: nowMs(),
2153
- });
2154
- }
2155
-
2156
- /**
2157
- * Report a provider 429 / Retry-After back into the Governor's shared pacer
2158
- * so future acquires for this (org, provider) bucket back off across all
2159
- * isolates. Provider comes from the same pacing resolver the Governor uses
2160
- * (the worker has no local catalog), so callers pass only the toolId.
2161
- */
2162
- private reportBackpressure(toolId: string, retryAfterMs: number): void {
2163
- if (retryAfterMs <= 0) return;
2164
- void (async () => {
2165
- const pacing = await this.resolvePacing(toolId).catch(() => null);
2166
- if (pacing?.provider) {
2167
- this.governor.reportProviderBackpressure({
2168
- provider: pacing.provider,
2169
- retryAfterMs,
2170
- });
2171
- }
2172
- })();
2173
- }
2174
-
2175
- async execute(
2176
- id: string,
2177
- toolId: string,
2178
- input: Record<string, unknown>,
2179
- workflowStep?: WorkflowStep,
2180
- options?: { force?: boolean; staleAfterSeconds?: number | null },
2181
- runtimeOptions?: { timeoutMs?: number; receiptWaitMs?: number },
2182
- ): Promise<unknown> {
2183
- const providerActionVersion =
2184
- await this.resolveToolActionCacheVersion(toolId);
2185
- return await new Promise((resolve, reject) => {
2186
- const queuedAt = nowMs();
2187
- const receiptKey = workerDurableToolCallCacheKey({
2188
- req: this.req,
2189
- toolId,
2190
- requestInput: input,
2191
- providerActionVersion,
2192
- staleAfterSeconds: options?.staleAfterSeconds,
2193
- });
2194
- const queryResultDatasetRead = isQueryResultDatasetReadRequest(
2195
- toolId,
2196
- input,
2197
- );
2198
- const receiptEligible = !queryResultDatasetRead;
2199
- this.queue.push({
2200
- id,
2201
- cacheKey: receiptEligible ? receiptKey : '',
2202
- receiptKey: receiptEligible ? receiptKey : null,
2203
- force: options?.force === true || !!this.req.force,
2204
- receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
2205
- typeof runtimeOptions?.receiptWaitMs === 'number'
2206
- ? { max_wait_ms: runtimeOptions.receiptWaitMs }
2207
- : input,
2208
- ),
2209
- ...(typeof runtimeOptions?.timeoutMs === 'number'
2210
- ? { runtimeTimeoutMs: runtimeOptions.timeoutMs }
2211
- : {}),
2212
- toolId,
2213
- input,
2214
- workflowStep,
2215
- ...(queryResultDatasetRead
2216
- ? {
2217
- directOptions: {
2218
- customerDbDataset: {
2219
- offset: 0,
2220
- pageSize: QUERY_RESULT_DATASET_PAGE_SIZE,
2221
- },
2222
- },
2223
- }
2224
- : {}),
2225
- resolve: (value) => {
2226
- recordRunnerPerfTrace({
2227
- req: this.req,
2228
- phase: 'runner.tool.request',
2229
- ms: nowMs() - queuedAt,
2230
- extra: { id, toolId },
2231
- });
2232
- resolve(value);
2233
- },
2234
- reject,
2235
- });
2236
- this.scheduleDrain();
2237
- });
2238
- }
2239
-
2240
- private scheduleDrain(): void {
2241
- if (this.scheduled) return;
2242
- this.scheduled = true;
2243
- queueMicrotask(async () => {
2244
- if (this.hasBatchableQueuedTool()) {
2245
- await new Promise((resolve) =>
2246
- setTimeout(resolve, WORKER_TOOL_BATCH_GRACE_MS),
2247
- );
2248
- }
2249
- void this.drain();
2250
- });
2251
- }
2252
-
2253
- private hasBatchableQueuedTool(): boolean {
2254
- return this.queue.some(
2255
- (request) =>
2256
- request.toolId !== 'test_wait_for_event' &&
2257
- getPlayRuntimeBatchStrategy(request.toolId) !== null,
2258
- );
2259
- }
2260
-
2261
- private async drain(): Promise<void> {
2262
- const requests = this.queue;
2263
- this.queue = [];
2264
- this.scheduled = false;
2265
- const drainStartedAt = nowMs();
2266
- await Promise.all(
2267
- [...groupWorkerToolRequestsByTool(requests).entries()].map(
2268
- async ([toolId, groupedRequests]) => {
2269
- await this.executeToolGroup(toolId, groupedRequests);
2270
- },
2271
- ),
2272
- );
2273
- recordRunnerPerfTrace({
2274
- req: this.req,
2275
- phase: 'runner.tool.drain',
2276
- ms: nowMs() - drainStartedAt,
2277
- extra: { requests: requests.length },
2278
- });
2279
- if (this.queue.length > 0) {
2280
- this.scheduleDrain();
2281
- }
2282
- }
2283
-
2284
- private async waitForDurableToolReceipt(input: {
2285
- receiptKey: string;
2286
- maxAttempts: number;
2287
- }): Promise<WorkerRuntimeReceipt> {
2288
- if (!this.receiptStore?.getReceipt) {
2289
- throw new Error(
2290
- 'Worker durable tool receipt wait requires receipt lookup.',
2291
- );
2292
- }
2293
- const receipt = await waitForCompletedRuntimeReceipt({
2294
- receiptKey: input.receiptKey,
2295
- maxAttempts: input.maxAttempts,
2296
- abortSignal: this.abortSignal,
2297
- store: {
2298
- getMany: async (receiptKeys) => {
2299
- const receipts = new Map<string, RuntimeStepReceipt>();
2300
- await Promise.all(
2301
- receiptKeys.map(async (key) => {
2302
- const receipt = await this.receiptStore!.getReceipt!({
2303
- playName: this.req.playName,
2304
- key,
2305
- });
2306
- if (!receipt) return;
2307
- receipts.set(key, {
2308
- key: receipt.key,
2309
- status: receipt.status,
2310
- output: receipt.output,
2311
- error: receipt.error ?? undefined,
2312
- runId: receipt.runId ?? null,
2313
- });
2314
- }),
2315
- );
2316
- return receipts;
2317
- },
2318
- },
2319
- });
2320
- return receipt;
2321
- }
2322
-
2323
- private settleRequests(
2324
- claimed: ClaimedWorkerToolBatchRequest,
2325
- result: unknown,
2326
- ): void {
2327
- claimed.request.resolve(result);
2328
- for (const follower of claimed.followers) {
2329
- follower.resolve(
2330
- claimed.receiptKey
2331
- ? markWorkerToolReceiptResultExecution(result, {
2332
- kind: 'in_flight',
2333
- receiptKey: claimed.receiptKey,
2334
- attachedToReceiptKey: claimed.receiptKey,
2335
- })
2336
- : result,
2337
- );
2338
- }
2339
- this.onRequestsSettled?.(1 + claimed.followers.length);
2340
- }
2341
-
2342
- private rejectRequests(
2343
- claimed: ClaimedWorkerToolBatchRequest,
2344
- error: unknown,
2345
- ): void {
2346
- claimed.request.reject(error);
2347
- for (const follower of claimed.followers) {
2348
- follower.reject(error);
2349
- }
2350
- this.onRequestsSettled?.(1 + claimed.followers.length);
2351
- }
2352
-
2353
- private rejectRawRequests(
2354
- requests: WorkerToolBatchRequest[],
2355
- error: unknown,
2356
- ): void {
2357
- for (const request of requests) {
2358
- request.reject(error);
2359
- }
2360
- this.onRequestsSettled?.(requests.length);
2361
- }
2362
-
2363
- private async resolveCompletedDurableToolReceiptGroup(input: {
2364
- group: WorkerToolBatchRequest[];
2365
- receiptKey: string;
2366
- receipt: WorkerRuntimeReceipt;
2367
- source: 'cache' | 'in_flight';
2368
- }): Promise<void> {
2369
- const [first] = input.group;
2370
- if (!first) return;
2371
- const value = deserializeDurableStepValue(input.receipt.output);
2372
- const result =
2373
- input.source === 'cache'
2374
- ? markWorkerToolReceiptResultCached(
2375
- value,
2376
- first.cacheKey,
2377
- input.receiptKey,
2378
- )
2379
- : markWorkerToolReceiptResultExecution(value, {
2380
- kind: 'in_flight',
2381
- receiptKey: input.receiptKey,
2382
- attachedToReceiptKey: input.receiptKey,
2383
- });
2384
- for (const request of input.group) {
2385
- request.resolve(result);
2386
- }
2387
- this.onRequestsSettled?.(input.group.length);
2388
- }
2389
-
2390
- private async reclaimRunningDurableToolReceiptGroupAfterWait(input: {
2391
- group: WorkerToolBatchRequest[];
2392
- receiptKey: string;
2393
- runningReceipt: WorkerRuntimeReceipt;
2394
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
2395
- const [request, ...followers] = input.group;
2396
- if (!request || !this.receiptStore) {
2397
- this.rejectRawRequests(
2398
- input.group,
2399
- new RuntimeReceiptWaitTimeoutError(input.receiptKey),
2400
- );
2401
- return [];
2402
- }
2403
- // A `running` receipt left by a DIFFERENT run whose last write is stale will
2404
- // never be completed by a live worker. Polling it burns one service-binding
2405
- // subrequest per tick, up to the receipt's wait budget (~1320) — many such
2406
- // orphans on a resume exhaust Cloudflare's per-invocation subrequest budget
2407
- // and kill the run. The orchestrator reclaims those immediately (skipping the
2408
- // poll) and re-executes, matching the "on resume it reclaims and re-executes"
2409
- // contract; live/fresh/same-run owners still poll before reclaiming. Kept as
2410
- // an injected helper so the skip-poll decision is unit-testable without
2411
- // loading this cloudflare:workers Worker entrypoint.
2412
- return await reclaimRunningReceiptGroupAfterWait<ClaimedWorkerToolBatchRequest>(
2413
- {
2414
- ownership: {
2415
- ownerRunId: input.runningReceipt.runId,
2416
- currentRunId: this.req.runId,
2417
- updatedAt: input.runningReceipt.updatedAt,
2418
- },
2419
- timeoutError: new RuntimeReceiptWaitTimeoutError(input.receiptKey),
2420
- poll: async () => {
2421
- try {
2422
- const receipt = await this.waitForDurableToolReceipt({
2423
- receiptKey: input.receiptKey,
2424
- maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
2425
- input.group,
2426
- (groupRequest) => groupRequest.receiptWaitMaxAttempts,
2427
- ),
2428
- });
2429
- await this.resolveCompletedDurableToolReceiptGroup({
2430
- group: input.group,
2431
- receiptKey: input.receiptKey,
2432
- receipt,
2433
- source: 'in_flight',
2434
- });
2435
- return { kind: 'completed' };
2436
- } catch (error) {
2437
- if (error instanceof RuntimeReceiptWaitTimeoutError) {
2438
- return { kind: 'timed_out', error };
2439
- }
2440
- return { kind: 'errored', error };
2441
- }
2442
- },
2443
- reclaim: (rejectionOnFallthrough) =>
2444
- this.forceReclaimRunningDurableToolReceiptGroup({
2445
- group: input.group,
2446
- request,
2447
- followers,
2448
- receiptKey: input.receiptKey,
2449
- rejectionOnFallthrough,
2450
- }),
2451
- reject: (error) => this.rejectRawRequests(input.group, error),
2452
- },
2453
- );
2454
- }
2455
-
2456
- /**
2457
- * Steal a `running` receipt (reclaimRunning) and route by claim disposition.
2458
- * Shared by the post-poll-timeout reclaim and the dead-owner fast path, so
2459
- * both dispose of a reclaimed/reused/failed receipt identically.
2460
- */
2461
- private async forceReclaimRunningDurableToolReceiptGroup(input: {
2462
- group: WorkerToolBatchRequest[];
2463
- request: WorkerToolBatchRequest;
2464
- followers: WorkerToolBatchRequest[];
2465
- receiptKey: string;
2466
- rejectionOnFallthrough: unknown;
2467
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
2468
- if (!this.receiptStore) {
2469
- this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
2470
- return [];
2471
- }
2472
- let claim: WorkerRuntimeReceiptClaim;
2473
- try {
2474
- claim = await this.receiptStore.claimReceipt({
2475
- playName: this.req.playName,
2476
- runId: this.req.runId,
2477
- key: input.receiptKey,
2478
- reclaimRunning: true,
2479
- });
2480
- } catch (error) {
2481
- this.rejectRawRequests(input.group, error);
2482
- return [];
2483
- }
2484
- if (claim.disposition === 'claimed') {
2485
- return [
2486
- {
2487
- request: input.request,
2488
- receiptKey: input.receiptKey,
2489
- followers: input.followers,
2490
- },
2491
- ];
2492
- }
2493
- if (claim.disposition === 'reused') {
2494
- await this.resolveCompletedDurableToolReceiptGroup({
2495
- group: input.group,
2496
- receiptKey: input.receiptKey,
2497
- receipt: claim.receipt,
2498
- source: 'cache',
2499
- });
2500
- return [];
2501
- }
2502
- if (claim.disposition === 'failed') {
2503
- return await this.reclaimFailedDurableToolReceiptGroup({
2504
- group: input.group,
2505
- receiptKey: input.receiptKey,
2506
- failedReceipt: claim.receipt,
2507
- });
2508
- }
2509
- this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
2510
- return [];
2511
- }
2512
-
2513
- private async reclaimFailedDurableToolReceiptGroup(input: {
2514
- group: WorkerToolBatchRequest[];
2515
- receiptKey: string;
2516
- failedReceipt: WorkerRuntimeReceipt;
2517
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
2518
- const failedError = new Error(
2519
- `Durable tool call ${input.receiptKey} failed: ${input.failedReceipt.error ?? 'unknown error'}`,
2520
- );
2521
- const [request, ...followers] = input.group;
2522
- if (!request || !this.receiptStore) {
2523
- this.rejectRawRequests(input.group, failedError);
2524
- return [];
2525
- }
2526
- if (
2527
- !canReclaimFailedWorkerToolReceipt({
2528
- error: input.failedReceipt.error,
2529
- })
2530
- ) {
2531
- this.rejectRawRequests(input.group, failedError);
2532
- return [];
2533
- }
2534
- let claim: WorkerRuntimeReceiptClaim;
2535
- try {
2536
- claim = await this.receiptStore.claimReceipt({
2537
- playName: this.req.playName,
2538
- runId: this.req.runId,
2539
- key: input.receiptKey,
2540
- });
2541
- } catch (error) {
2542
- this.rejectRawRequests(input.group, error);
2543
- return [];
2544
- }
2545
- if (claim.disposition === 'claimed') {
2546
- return [{ request, receiptKey: input.receiptKey, followers }];
2547
- }
2548
- if (claim.disposition === 'reused') {
2549
- await this.resolveCompletedDurableToolReceiptGroup({
2550
- group: input.group,
2551
- receiptKey: input.receiptKey,
2552
- receipt: claim.receipt,
2553
- source: 'cache',
2554
- });
2555
- return [];
2556
- }
2557
- if (claim.disposition === 'running') {
2558
- try {
2559
- const receipt = await this.waitForDurableToolReceipt({
2560
- receiptKey: input.receiptKey,
2561
- maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
2562
- input.group,
2563
- (groupRequest) => groupRequest.receiptWaitMaxAttempts,
2564
- ),
2565
- });
2566
- await this.resolveCompletedDurableToolReceiptGroup({
2567
- group: input.group,
2568
- receiptKey: input.receiptKey,
2569
- receipt,
2570
- source: 'in_flight',
2571
- });
2572
- } catch (error) {
2573
- this.rejectRawRequests(input.group, error);
2574
- }
2575
- return [];
2576
- }
2577
- this.rejectRawRequests(
2578
- input.group,
2579
- new Error(
2580
- `Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
2581
- ),
2582
- );
2583
- return [];
2584
- }
2585
-
2586
- private async failureForRejectedToolRequest(
2587
- claimed: ClaimedWorkerToolBatchRequest,
2588
- error: unknown,
2589
- ): Promise<unknown> {
2590
- try {
2591
- await this.failDurableToolRequest(claimed, error);
2592
- return error;
2593
- } catch (receiptError) {
2594
- tripRuntimePersistenceLatch(this.latch, receiptError);
2595
- return new RuntimeReceiptPersistenceError(
2596
- 'Tool call failed and durable receipt could not be marked failed',
2597
- [error, receiptError],
2598
- );
2599
- }
2600
- }
2601
-
2602
- private async claimDurableToolReceiptGroups(
2603
- groups: ReturnType<
2604
- typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
2605
- >['durableGroups'],
2606
- ): Promise<
2607
- Array<{
2608
- group: ReturnType<
2609
- typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
2610
- >['durableGroups'][number];
2611
- receiptKey: string;
2612
- claim: WorkerRuntimeReceiptClaim;
2613
- }>
2614
- > {
2615
- if (!this.receiptStore) return [];
2616
- const planned = groups.map((group) => ({
2617
- group,
2618
- receiptKey: workerRuntimeReceiptKey({
2619
- req: this.req,
2620
- key: group.claimableReceiptKey,
2621
- }),
2622
- }));
2623
- const claimOne = async (entry: (typeof planned)[number]) => ({
2624
- ...entry,
2625
- claim: await this.receiptStore!.claimReceipt({
2626
- playName: this.req.playName,
2627
- runId: this.req.runId,
2628
- key: entry.receiptKey,
2629
- ...(entry.group.forceDurableRefresh
2630
- ? { forceRefresh: true, reclaimRunning: true }
2631
- : {}),
2632
- }),
2633
- });
2634
- if (!this.receiptStore.claimReceipts) {
2635
- return await Promise.all(planned.map(claimOne));
2636
- }
2637
- const claimed: Array<{
2638
- group: (typeof planned)[number]['group'];
2639
- receiptKey: string;
2640
- claim: WorkerRuntimeReceiptClaim;
2641
- }> = [];
2642
- for (const forceDurableRefresh of [false, true]) {
2643
- const entries = planned.filter(
2644
- (entry) => entry.group.forceDurableRefresh === forceDurableRefresh,
2645
- );
2646
- if (entries.length === 0) continue;
2647
- const claims = await this.receiptStore.claimReceipts({
2648
- playName: this.req.playName,
2649
- runId: this.req.runId,
2650
- keys: entries.map((entry) => entry.receiptKey),
2651
- ...(forceDurableRefresh
2652
- ? { forceRefresh: true, reclaimRunning: true }
2653
- : {}),
2654
- });
2655
- entries.forEach((entry, index) => {
2656
- const claim = claims[index];
2657
- if (!claim) {
2658
- throw new Error(
2659
- `Runtime receipt batch claim did not return receipt ${entry.receiptKey}.`,
2660
- );
2661
- }
2662
- claimed.push({ ...entry, claim });
2663
- });
2664
- }
2665
- return claimed;
2666
- }
2667
-
2668
- private async prepareDurableToolRequests(
2669
- requests: WorkerToolBatchRequest[],
2670
- ): Promise<PreparedWorkerToolBatchRequests> {
2671
- if (!this.receiptStore) {
2672
- return {
2673
- claimedRequests: requests.map((request) => ({
2674
- request,
2675
- receiptKey: null,
2676
- followers: [],
2677
- })),
2678
- deferredClaimedRequests: [],
2679
- };
2680
- }
2681
-
2682
- const claimedRequests: ClaimedWorkerToolBatchRequest[] = [];
2683
- const deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[] =
2684
- [];
2685
- const receiptGroupPlan = planWorkerToolReceiptGroups(requests, {
2686
- allowLocalRetryReceipts: this.allowLocalRetryReceipts,
2687
- getReceiptInput: (request) => ({
2688
- durableReceiptKey: request.receiptKey,
2689
- localRetryCacheKey: request.cacheKey,
2690
- force: request.force,
2691
- }),
2692
- });
2693
- for (const request of receiptGroupPlan.localRequests) {
2694
- claimedRequests.push({
2695
- request,
2696
- receiptKey: null,
2697
- followers: [],
2698
- });
2699
- }
2700
-
2701
- try {
2702
- const claimEntries = await this.claimDurableToolReceiptGroups(
2703
- receiptGroupPlan.durableGroups,
2704
- );
2705
- for (const { group: groupState, receiptKey, claim } of claimEntries) {
2706
- const group = groupState.requests;
2707
- const [first] = group;
2708
- if (!first) continue;
2709
- if (claim.disposition === 'reused') {
2710
- const result = markWorkerToolReceiptResultCached(
2711
- deserializeDurableStepValue(claim.receipt.output),
2712
- first.cacheKey,
2713
- receiptKey,
2714
- );
2715
- for (const request of group) {
2716
- request.resolve(result);
2717
- }
2718
- this.onRequestsSettled?.(group.length);
2719
- continue;
2720
- }
2721
- if (claim.disposition === 'failed') {
2722
- claimedRequests.push(
2723
- ...(await this.reclaimFailedDurableToolReceiptGroup({
2724
- group,
2725
- receiptKey,
2726
- failedReceipt: claim.receipt,
2727
- })),
2728
- );
2729
- continue;
2730
- }
2731
- if (claim.disposition === 'running') {
2732
- deferredClaimedRequests.push(
2733
- this.reclaimRunningDurableToolReceiptGroupAfterWait({
2734
- group,
2735
- receiptKey,
2736
- runningReceipt: claim.receipt,
2737
- }),
2738
- );
2739
- continue;
2740
- }
2741
- const [request, ...followers] = group;
2742
- if (!request) continue;
2743
- claimedRequests.push({ request, receiptKey, followers });
2744
- }
2745
- } catch (error) {
2746
- // A receipt claim failure means the tenant Postgres is already
2747
- // unwritable: trip the breaker so remaining groups/rows stop dispatching.
2748
- tripRuntimePersistenceLatch(this.latch, error);
2749
- for (const group of receiptGroupPlan.durableGroups) {
2750
- this.rejectRawRequests(group.requests, error);
2751
- }
2752
- }
2753
- return { claimedRequests, deferredClaimedRequests };
2754
- }
2755
-
2756
- private async completeDurableToolRequest(
2757
- claimed: ClaimedWorkerToolBatchRequest,
2758
- result: unknown,
2759
- ): Promise<unknown> {
2760
- if (!this.receiptStore || !claimed.receiptKey) {
2761
- return result;
2762
- }
2763
- const ownerResult = markWorkerToolReceiptResultExecution(result, {
2764
- kind: 'live',
2765
- receiptKey: claimed.receiptKey,
2766
- });
2767
- const serializedOutput = serializeDurableStepValue(ownerResult);
2768
- const completed = await this.completeReceiptWithRetryLadder(
2769
- () =>
2770
- this.receiptStore!.completeReceipt({
2771
- playName: this.req.playName,
2772
- runId: this.req.runId,
2773
- key: claimed.receiptKey!,
2774
- output: serializedOutput,
2775
- }),
2776
- // Ladder exhausted: the call billed but the receipt could not land.
2777
- // Salvage the exact serialized output onto the run record before the
2778
- // caller turns this into a RuntimeReceiptPersistenceError.
2779
- (attempts) =>
2780
- this.recordReceiptSalvage({
2781
- claimed,
2782
- output: serializedOutput,
2783
- attempts,
2784
- }),
2785
- );
2786
- if (
2787
- completed &&
2788
- (completed.status === 'completed' || completed.status === 'skipped') &&
2789
- completed.output !== undefined
2790
- ) {
2791
- const recovered = deserializeDurableStepValue(completed.output);
2792
- return completed.runId && completed.runId !== this.req.runId
2793
- ? markWorkerToolReceiptResultCached(
2794
- recovered,
2795
- claimed.request.cacheKey,
2796
- claimed.receiptKey,
2797
- )
2798
- : recovered;
2799
- }
2800
- return ownerResult;
2801
- }
2802
-
2803
- private completedDurableToolRequestResult(input: {
2804
- claimed: ClaimedWorkerToolBatchRequest;
2805
- ownerResult: unknown;
2806
- completed: WorkerRuntimeReceipt | null | undefined;
2807
- }): unknown {
2808
- const { claimed, completed, ownerResult } = input;
2809
- if (
2810
- completed &&
2811
- (completed.status === 'completed' || completed.status === 'skipped') &&
2812
- completed.output !== undefined
2813
- ) {
2814
- const recovered = deserializeDurableStepValue(completed.output);
2815
- return completed.runId && completed.runId !== this.req.runId
2816
- ? markWorkerToolReceiptResultCached(
2817
- recovered,
2818
- claimed.request.cacheKey,
2819
- claimed.receiptKey ?? claimed.request.cacheKey,
2820
- )
2821
- : recovered;
2822
- }
2823
- return ownerResult;
2824
- }
2825
-
2826
- private async completeDurableToolRequests(
2827
- entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
2828
- ): Promise<unknown[]> {
2829
- const results: unknown[] = new Array(entries.length);
2830
- const durableEntries: Array<{
2831
- index: number;
2832
- claimed: ClaimedWorkerToolBatchRequest;
2833
- ownerResult: unknown;
2834
- }> = [];
2835
- for (let index = 0; index < entries.length; index += 1) {
2836
- const entry = entries[index]!;
2837
- if (!this.receiptStore || !entry.claimed.receiptKey) {
2838
- results[index] = entry.result;
2839
- continue;
2840
- }
2841
- const ownerResult = markWorkerToolReceiptResultExecution(entry.result, {
2842
- kind: 'live',
2843
- receiptKey: entry.claimed.receiptKey,
2844
- });
2845
- durableEntries.push({ index, claimed: entry.claimed, ownerResult });
2846
- }
2847
- if (durableEntries.length === 0) {
2848
- return results;
2849
- }
2850
- if (!this.receiptStore?.completeReceipts) {
2851
- await Promise.all(
2852
- durableEntries.map(async (entry) => {
2853
- results[entry.index] = await this.completeDurableToolRequest(
2854
- entry.claimed,
2855
- entries[entry.index]!.result,
2856
- );
2857
- }),
2858
- );
2859
- return results;
2860
- }
2861
- const serializedByEntry = durableEntries.map((entry) =>
2862
- serializeDurableStepValue(entry.ownerResult),
2863
- );
2864
- const completed = await this.completeReceiptWithRetryLadder(
2865
- () =>
2866
- this.receiptStore!.completeReceipts!({
2867
- playName: this.req.playName,
2868
- receipts: durableEntries.map((entry, entryIndex) => ({
2869
- runId: this.req.runId,
2870
- key: entry.claimed.receiptKey!,
2871
- output: serializedByEntry[entryIndex],
2872
- })),
2873
- }),
2874
- // Ladder exhausted: every entry in this batch billed but could not be
2875
- // marked completed. Salvage each serialized output before the caller
2876
- // turns this into a RuntimeReceiptPersistenceError.
2877
- (attempts) => {
2878
- durableEntries.forEach((entry, entryIndex) => {
2879
- this.recordReceiptSalvage({
2880
- claimed: entry.claimed,
2881
- output: serializedByEntry[entryIndex],
2882
- attempts,
2883
- });
2884
- });
2885
- },
2886
- );
2887
- durableEntries.forEach((entry, resultIndex) => {
2888
- results[entry.index] = this.completedDurableToolRequestResult({
2889
- claimed: entry.claimed,
2890
- ownerResult: entry.ownerResult,
2891
- completed: completed[resultIndex],
2892
- });
2893
- });
2894
- return results;
2895
- }
2896
-
2897
- private async failDurableToolRequest(
2898
- claimed: ClaimedWorkerToolBatchRequest,
2899
- error: unknown,
2900
- ): Promise<void> {
2901
- if (!this.receiptStore || !claimed.receiptKey) {
2902
- return;
2903
- }
2904
- await this.receiptStore.failReceipt({
2905
- playName: this.req.playName,
2906
- runId: this.req.runId,
2907
- key: claimed.receiptKey,
2908
- error: error instanceof Error ? error.message : String(error),
2909
- });
2910
- }
2911
-
2912
- private async failDurableToolRequests(
2913
- claimedRequests: ClaimedWorkerToolBatchRequest[],
2914
- error: unknown,
2915
- ): Promise<void> {
2916
- const durable = claimedRequests.filter(
2917
- (claimed) => this.receiptStore && claimed.receiptKey,
2918
- );
2919
- if (durable.length === 0) return;
2920
- if (!this.receiptStore?.failReceipts) {
2921
- await Promise.all(
2922
- durable.map((claimed) => this.failDurableToolRequest(claimed, error)),
2923
- );
2924
- return;
2925
- }
2926
- await this.receiptStore.failReceipts({
2927
- playName: this.req.playName,
2928
- receipts: durable.map((claimed) => ({
2929
- runId: this.req.runId,
2930
- key: claimed.receiptKey!,
2931
- error: error instanceof Error ? error.message : String(error),
2932
- })),
2933
- });
2934
- }
2935
-
2936
- private async executeToolGroup(
2937
- toolId: string,
2938
- requests: WorkerToolBatchRequest[],
2939
- ): Promise<void> {
2940
- // Circuit breaker, gated BEFORE the receipt claim: never-dispatched rows
2941
- // get NO receipt (not even `running`), so a re-run claims them fresh and
2942
- // executes them exactly once.
2943
- if (this.latch.tripped) {
2944
- this.latch.preventedCallCount += requests.length;
2945
- this.rejectRawRequests(
2946
- requests,
2947
- new RuntimePersistenceCircuitOpenError(this.latch),
2948
- );
2949
- return;
2950
- }
2951
- const { claimedRequests, deferredClaimedRequests } =
2952
- await this.prepareDurableToolRequests(requests);
2953
- await this.executeClaimedToolRequests(
2954
- toolId,
2955
- requests.length,
2956
- claimedRequests,
2957
- );
2958
- if (deferredClaimedRequests.length === 0) {
2959
- return;
2960
- }
2961
- const reclaimedRequests = (
2962
- await Promise.all(deferredClaimedRequests)
2963
- ).flat();
2964
- await this.executeClaimedToolRequests(
2965
- toolId,
2966
- requests.length,
2967
- reclaimedRequests,
2968
- );
2969
- }
2970
-
2971
- private async executeClaimedToolRequests(
2972
- toolId: string,
2973
- requestCount: number,
2974
- claimedRequests: ClaimedWorkerToolBatchRequest[],
2975
- ): Promise<void> {
2976
- if (claimedRequests.length === 0) {
2977
- return;
2978
- }
2979
- const strategy = getPlayRuntimeBatchStrategy(toolId);
2980
- if (
2981
- !strategy ||
2982
- toolId === 'test_wait_for_event' ||
2983
- claimedRequests.length < 2
2984
- ) {
2985
- const groupStartedAt = nowMs();
2986
- const dispatchOne = async (
2987
- claimed: ClaimedWorkerToolBatchRequest,
2988
- ): Promise<void> => {
2989
- const { request } = claimed;
2990
- // Circuit breaker: the latch may have tripped after this group was
2991
- // claimed but before this particular call started. Do NOT call the
2992
- // provider. This row has a `running` receipt (claimed, never
2993
- // executed); on resume it reclaims and re-executes — safe, it never
2994
- // billed.
2995
- if (this.latch.tripped) {
2996
- this.latch.preventedCallCount += 1 + claimed.followers.length;
2997
- this.rejectRequests(
2998
- claimed,
2999
- new RuntimePersistenceCircuitOpenError(this.latch),
3000
- );
3001
- return;
3002
- }
3003
- const toolContract = await this.resolvePacing(toolId).catch(
3004
- () => null,
3005
- );
3006
- // Each unbatched provider call takes its own tool slot: the Governor
3007
- // charges tool budget, holds a global tool-concurrency slot, and
3008
- // applies per-(org,provider) pacing before the call runs.
3009
- const slot = await this.governor.acquireToolSlot(toolId, {
3010
- signal: this.abortSignal,
3011
- });
3012
- try {
3013
- let result: unknown;
3014
- try {
3015
- result = await executeToolWithLifecycle(
3016
- this.req,
3017
- { id: request.id, toolId, input: request.input },
3018
- request.workflowStep,
3019
- this.callbacks,
3020
- (retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
3021
- () => this.governor.chargeBudget('retry'),
3022
- toolContract?.retrySafeTransientHttp === true,
3023
- this.abortSignal,
3024
- this.runtimeDeadlineMs,
3025
- resolveClaimedWorkerToolRuntimeTimeoutMs([claimed], {
3026
- runtimeDeadlineMs: this.runtimeDeadlineMs,
3027
- }),
3028
- request.directOptions,
3029
- );
3030
- } catch (error) {
3031
- this.rejectRequests(
3032
- claimed,
3033
- await this.failureForRejectedToolRequest(claimed, error),
3034
- );
3035
- return;
3036
- }
3037
- this.settleRequests(
3038
- claimed,
3039
- await this.completeDurableToolRequest(claimed, result),
3040
- );
3041
- } catch (receiptError) {
3042
- tripRuntimePersistenceLatch(this.latch, receiptError);
3043
- this.rejectRequests(
3044
- claimed,
3045
- new RuntimeReceiptPersistenceError(
3046
- 'Tool call succeeded but durable receipt could not be marked completed',
3047
- [receiptError],
3048
- ),
3049
- );
3050
- } finally {
3051
- slot.release();
3052
- }
3053
- };
3054
- // Bounded dispatch waves: an unbatched tool group can carry a whole chunk's
3055
- // rows (enrich compiles to one ~245-row chunk). Dispatching them all at
3056
- // once billed every call before the first receipt-completion failure could
3057
- // trip the latch (the 250/250 incident). Instead, dispatch one wave, AWAIT
3058
- // it so any completion failure trips the latch, then read the latch before
3059
- // the next wave — remaining waves are prevented (counted + rejected) exactly
3060
- // like the mid-wave gate above. The wave calls stay fully concurrent; the
3061
- // only added cost on the healthy path is one latch read per wave.
3062
- for (const wave of partitionDispatchWaves(
3063
- claimedRequests,
3064
- () => 1,
3065
- PROVIDER_DISPATCH_WAVE_SIZE,
3066
- )) {
3067
- if (this.latch.tripped) {
3068
- for (const claimed of wave) {
3069
- this.latch.preventedCallCount += 1 + claimed.followers.length;
3070
- this.rejectRequests(
3071
- claimed,
3072
- new RuntimePersistenceCircuitOpenError(this.latch),
3073
- );
3074
- }
3075
- continue;
3076
- }
3077
- await Promise.all(wave.map(dispatchOne));
3078
- }
3079
- recordRunnerPerfTrace({
3080
- req: this.req,
3081
- phase: 'runner.tool.group',
3082
- ms: nowMs() - groupStartedAt,
3083
- extra: {
3084
- toolId,
3085
- requests: requestCount,
3086
- executed: claimedRequests.length,
3087
- batched: false,
3088
- },
3089
- });
3090
- return;
3091
- }
2283
+ path: string[];
2284
+ outputs: RecordedStepProgramOutput[];
2285
+ onOutput?: (output: RecordedStepProgramOutput) => void | Promise<void>;
2286
+ };
3092
2287
 
3093
- const batchStartedAt = nowMs();
3094
- await executeBatchedWorkerToolGroup({
3095
- req: this.req,
3096
- requests: claimedRequests,
3097
- strategy,
3098
- governor: this.governor,
3099
- suggestedParallelism: await this.governor.suggestedParallelism(
3100
- toolId,
3101
- WORKER_TOOL_BATCH_DEFAULT_PARALLELISM,
3102
- ),
3103
- abortSignal: this.abortSignal,
3104
- runtimeDeadlineMs: this.runtimeDeadlineMs,
3105
- reportBackpressure: (retryAfterMs) =>
3106
- this.reportBackpressure(toolId, retryAfterMs),
3107
- resolveToolContract: this.resolvePacing,
3108
- onRequestsSettled: this.onRequestsSettled,
3109
- callbacks: this.callbacks,
3110
- completeRequests: async (entries) =>
3111
- await this.completeDurableToolRequests(entries),
3112
- failRequests: async (claimedRequests, error) =>
3113
- await this.failDurableToolRequests(claimedRequests, error),
3114
- settleRequest: (claimed, result) => this.settleRequests(claimed, result),
3115
- rejectRequest: (claimed, error) => this.rejectRequests(claimed, error),
3116
- latch: this.latch,
3117
- });
3118
- recordRunnerPerfTrace({
3119
- req: this.req,
3120
- phase: 'runner.tool.group',
3121
- ms: nowMs() - batchStartedAt,
3122
- extra: {
3123
- toolId,
3124
- requests: requestCount,
3125
- executed: claimedRequests.length,
3126
- batched: true,
3127
- },
3128
- });
3129
- }
2288
+ type WorkerStepResolution = {
2289
+ value: unknown;
2290
+ status?: 'skipped';
2291
+ };
2292
+
2293
+ function stepProgramColumnName(parentField: string, stepId: string): string {
2294
+ return sqlSafePlayColumnName(`${parentField}.${stepId}`);
3130
2295
  }
3131
2296
 
3132
- function groupWorkerToolRequestsByTool(
3133
- requests: WorkerToolBatchRequest[],
3134
- ): Map<string, WorkerToolBatchRequest[]> {
3135
- const byTool = new Map<string, WorkerToolBatchRequest[]>();
3136
- for (const request of requests) {
3137
- const existing = byTool.get(request.toolId);
3138
- if (existing) {
3139
- existing.push(request);
3140
- } else {
3141
- byTool.set(request.toolId, [request]);
3142
- }
2297
+ const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
2298
+ const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
2299
+ const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
2300
+ const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 1_000;
2301
+ const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
2302
+ /**
2303
+ * Bounded number of per-row failure samples carried in chunk summaries and the
2304
+ * map's terminal partial-failure log. Every failed row is persisted with its
2305
+ * full error in the runtime sheet; the samples just keep run logs readable.
2306
+ */
2307
+ const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
2308
+
2309
+ /**
2310
+ * Bounded-concurrent map-chunk dispatch fan-out for the workers_edge profile
2311
+ * (in-process play execution with no `workflowStep`). On workers_edge the whole
2312
+ * map shares ONE per-invocation subrequest window, so sequential chunk dispatch
2313
+ * buys no durability — it is pure wall-clock cost. Running up to K chunks in
2314
+ * flight overlaps provider batches, sheet writes, and the per-chunk persist
2315
+ * barrier. K is capped at 3 to bound peak memory: each in-flight chunk holds up
2316
+ * to ~5000 rows plus their row buffers, so 3 concurrent chunks is the ceiling
2317
+ * we accept before row payloads risk pressuring the isolate. The native
2318
+ * Cloudflare Workflows path (workflowStep present) keeps K=1: there each chunk
2319
+ * is a durable step and sequential dispatch is what makes replay/resume sound.
2320
+ */
2321
+ const WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS = 3;
2322
+ const WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES =
2323
+ WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES;
2324
+ const WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES = 128;
2325
+
2326
+ const workerMapResidentRowBytes = (row: unknown): number =>
2327
+ new TextEncoder().encode(JSON.stringify(row)).byteLength +
2328
+ WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES;
2329
+
2330
+ function isRuntimeReceiptPersistenceError(error: unknown): boolean {
2331
+ if (!error || typeof error !== 'object') return false;
2332
+ if (error instanceof RuntimeReceiptPersistenceError) return true;
2333
+ if (
2334
+ error instanceof Error &&
2335
+ error.name === 'RuntimeReceiptPersistenceError'
2336
+ ) {
2337
+ return true;
3143
2338
  }
3144
- return byTool;
2339
+ const nestedErrors = (error as { errors?: unknown }).errors;
2340
+ return (
2341
+ Array.isArray(nestedErrors) &&
2342
+ nestedErrors.some(isRuntimeReceiptPersistenceError)
2343
+ );
3145
2344
  }
3146
2345
 
3147
- async function executeBatchedWorkerToolGroup(input: {
3148
- req: RunRequest;
3149
- requests: ClaimedWorkerToolBatchRequest[];
3150
- strategy: AnyBatchOperationStrategy;
3151
- governor: PlayExecutionGovernor;
3152
- suggestedParallelism: number;
3153
- abortSignal?: AbortSignal;
3154
- runtimeDeadlineMs?: number;
3155
- reportBackpressure: (retryAfterMs: number) => void;
3156
- resolveToolContract: WorkerPacingResolver;
3157
- onRequestsSettled?: (count: number) => void;
3158
- callbacks?: WorkerCtxCallbacks;
3159
- completeRequests: (
3160
- entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
3161
- ) => Promise<unknown[]>;
3162
- failRequests: (
3163
- requests: ClaimedWorkerToolBatchRequest[],
3164
- error: unknown,
3165
- ) => Promise<void>;
3166
- settleRequest: (
3167
- request: ClaimedWorkerToolBatchRequest,
3168
- result: unknown,
3169
- ) => void;
3170
- rejectRequest: (
3171
- request: ClaimedWorkerToolBatchRequest,
3172
- error: unknown,
3173
- ) => void;
3174
- latch: RuntimePersistenceLatch;
3175
- }): Promise<void> {
3176
- const compiledBatches = compileRequestsWithStrategy({
3177
- requests: input.requests,
3178
- strategy: input.strategy,
3179
- getPayload: (request) => request.request.input,
3180
- });
3181
- recordRunnerPerfTrace({
3182
- req: input.req,
3183
- phase: 'runner.tool.batch.compile',
3184
- ms: 0,
3185
- extra: {
3186
- sourceOperation: input.strategy.sourceOperation,
3187
- batchOperation: input.strategy.batchOperation,
3188
- requests: input.requests.length,
3189
- batches: compiledBatches.length,
3190
- batchSizes: compiledBatches.map((batch) => batch.memberRequests.length),
3191
- },
3192
- });
2346
+ function isRunFatalWorkerRowError(error: unknown): boolean {
2347
+ return (
2348
+ isCoordinatorProgressAuthorizationError(error) ||
2349
+ isRowIsolationExemptError(error) ||
2350
+ isRuntimeReceiptPersistenceError(error) ||
2351
+ isRuntimePersistenceCircuitOpenError(error) ||
2352
+ error instanceof RuntimeSheetAttemptLeaseLostError ||
2353
+ error instanceof RuntimeLeaseLostError ||
2354
+ (error instanceof Error &&
2355
+ error.name === 'RuntimeSheetAttemptLeaseLostError')
2356
+ );
2357
+ }
3193
2358
 
3194
- await executeChunkedRequests({
3195
- requests: compiledBatches,
3196
- // Chunk parallelism is the Governor's per-tool suggestion (provider rate
3197
- // hints tightened to the policy ceiling), bounded by the batch count.
3198
- batchSize: Math.max(
3199
- 1,
3200
- Math.min(input.suggestedParallelism, compiledBatches.length || 1),
3201
- ),
3202
- // Bounded dispatch wave: cap the provider calls (counted by batch member
3203
- // size) a single chunk dispatches before the latch is read again, so a
3204
- // persistence failure prevents the rest of a large batch group instead of
3205
- // billing it all. A single over-budget batch still forms its own wave.
3206
- maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
3207
- weightOf: (batch) => batch.memberRequests.length,
3208
- execute: async (batch) => {
3209
- // Circuit breaker: once a persistence failure has occurred in this map,
3210
- // do NOT dispatch this batch's provider call. The chunk plumbing rejects
3211
- // its member requests with the latch error.
3212
- if (input.latch.tripped) {
3213
- input.latch.preventedCallCount += batch.memberRequests.length;
3214
- throw new RuntimePersistenceCircuitOpenError(input.latch);
3215
- }
3216
- const toolContract = await input
3217
- .resolveToolContract(batch.batchOperation)
3218
- .catch(() => null);
3219
- // One provider call per batch → one tool slot (budget + global
3220
- // concurrency + per-(org,provider) pacing) around the whole batch.
3221
- const slot = await input.governor.acquireToolSlot(batch.batchOperation, {
3222
- signal: input.abortSignal,
3223
- });
3224
- try {
3225
- input.callbacks?.onToolCalled?.(batch.batchOperation, nowMs());
3226
- return await executeTool(
3227
- input.req,
3228
- {
3229
- id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
3230
- toolId: batch.batchOperation,
3231
- input: batch.batchPayload,
3232
- },
3233
- undefined,
3234
- input.reportBackpressure,
3235
- () => input.governor.chargeBudget('retry'),
3236
- toolContract?.retrySafeTransientHttp === true,
3237
- input.abortSignal,
3238
- input.runtimeDeadlineMs,
3239
- resolveClaimedWorkerToolRuntimeTimeoutMs(batch.memberRequests, {
3240
- runtimeDeadlineMs: input.runtimeDeadlineMs,
3241
- }),
3242
- );
3243
- } catch (error) {
3244
- input.callbacks?.onToolFailed?.(batch.batchOperation, nowMs());
3245
- throw error;
3246
- } finally {
3247
- slot.release();
3248
- }
3249
- },
3250
- onChunkComplete: async (
3251
- chunkResults: Array<
3252
- ChunkExecutionResult<(typeof compiledBatches)[number], unknown>
3253
- >,
3254
- ) => {
3255
- for (const entry of chunkResults) {
3256
- if (entry.error !== undefined) {
3257
- // One batch's provider error stays scoped to that batch's member
3258
- // requests. Sibling batches in this chunk keep their results so a
3259
- // single provider hiccup cannot cascade into a whole-map failure.
3260
- let rejection: unknown = entry.error;
3261
- try {
3262
- await input.failRequests(entry.request.memberRequests, entry.error);
3263
- } catch (receiptError) {
3264
- tripRuntimePersistenceLatch(input.latch, receiptError);
3265
- rejection = new RuntimeReceiptPersistenceError(
3266
- 'Tool call failed and durable receipts could not be marked failed',
3267
- [entry.error, receiptError],
3268
- );
3269
- }
3270
- for (const claimed of entry.request.memberRequests) {
3271
- input.rejectRequest(claimed, rejection);
3272
- }
3273
- continue;
3274
- }
3275
- const batchResult = isToolExecuteResult(entry.result)
3276
- ? entry.result.toolOutput.raw
3277
- : entry.result;
3278
- const splitResults =
3279
- batchResult != null
3280
- ? entry.request.splitResults(batchResult)
3281
- : entry.request.memberRequests.map(() => null);
3282
- let completedResults: unknown[];
3283
- try {
3284
- completedResults = await input.completeRequests(
3285
- entry.request.memberRequests.map((claimed, index) => ({
3286
- claimed,
3287
- result: wrapWorkerToolResult(
3288
- claimed.request.toolId,
3289
- splitResults[index] ?? null,
3290
- toolMetadataFallback(claimed.request.toolId),
3291
- splitResults[index] == null ? 'no_result' : 'completed',
3292
- {
3293
- req: input.req,
3294
- input: claimed.request.input,
3295
- },
3296
- ),
3297
- })),
3298
- );
3299
- } catch (receiptError) {
3300
- tripRuntimePersistenceLatch(input.latch, receiptError);
3301
- const rejection = new RuntimeReceiptPersistenceError(
3302
- 'Tool call succeeded but durable receipts could not be marked completed',
3303
- [receiptError],
3304
- );
3305
- for (const claimed of entry.request.memberRequests) {
3306
- input.rejectRequest(claimed, rejection);
3307
- }
3308
- continue;
3309
- }
3310
- for (let index = 0; index < completedResults.length; index += 1) {
3311
- const claimed = entry.request.memberRequests[index]!;
3312
- const request = claimed.request;
3313
- input.settleRequest(claimed, completedResults[index]);
3314
- }
3315
- }
3316
- },
3317
- }).catch(async (error) => {
3318
- let rejection: unknown = error;
3319
- try {
3320
- await input.failRequests(input.requests, error);
3321
- } catch (receiptError) {
3322
- tripRuntimePersistenceLatch(input.latch, receiptError);
3323
- rejection = new RuntimeReceiptPersistenceError(
3324
- 'Tool call failed and durable receipts could not be marked failed',
3325
- [error, receiptError],
3326
- );
3327
- }
3328
- for (const claimed of input.requests) {
3329
- input.rejectRequest(claimed, rejection);
3330
- }
3331
- });
2359
+ /**
2360
+ * In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
2361
+ * throughput pacing (provider or Deepline limiter), not a tool defect, so it
2362
+ * gets more patience than the generic 3-attempt budget: with retry-after-aware
2363
+ * escalating delays (capped at 5s) this absorbs roughly 25s of sustained
2364
+ * throttling before the call fails. Every retry still charges the Governor's
2365
+ * retry budget, so a runaway storm stays bounded and loud.
2366
+ */
2367
+
2368
+ function sleepWorkerMs(ms: number): Promise<void> {
2369
+ return new Promise((resolve) => setTimeout(resolve, ms));
3332
2370
  }
3333
2371
 
3334
2372
  function isCompletedWorkerFieldValue(value: unknown): boolean {
@@ -3339,7 +2377,8 @@ function isCompletedWorkerFieldValue(value: unknown): boolean {
3339
2377
  );
3340
2378
  }
3341
2379
 
3342
- type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
2380
+ type WorkerMapCompletedChunkSummary<T extends Record<string, unknown>> = {
2381
+ status: 'completed';
3343
2382
  chunkIndex: number;
3344
2383
  rangeStart: number;
3345
2384
  rangeEnd: number;
@@ -3363,6 +2402,23 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
3363
2402
  cachedRows?: T[];
3364
2403
  };
3365
2404
 
2405
+ type WorkerMapBlockedChunkSummary = {
2406
+ status: 'blocked';
2407
+ chunkIndex: number;
2408
+ rangeStart: number;
2409
+ rangeEnd: number;
2410
+ rowsRead: number;
2411
+ rowsBlocked: number;
2412
+ rowKeySamples: string[];
2413
+ blockedRowSamples: Record<string, unknown>[];
2414
+ outputDatasetId: string;
2415
+ message: string;
2416
+ };
2417
+
2418
+ type WorkerMapChunkSummary<T extends Record<string, unknown>> =
2419
+ | WorkerMapCompletedChunkSummary<T>
2420
+ | WorkerMapBlockedChunkSummary;
2421
+
3366
2422
  function serializeDurableStepValue<T>(value: T, depth = 0): T {
3367
2423
  if (depth > 20 || value == null) return value;
3368
2424
  if (isToolExecuteResult(value)) return serializeToolExecuteResult(value) as T;
@@ -3575,6 +2631,7 @@ async function executeWorkerStepProgram(
3575
2631
  ...(resolution.status ? { status: resolution.status } : {}),
3576
2632
  };
3577
2633
  };
2634
+
3578
2635
  const executeStep = async (): Promise<WorkerStepResolution> =>
3579
2636
  workflowStep
3580
2637
  ? await (
@@ -3799,7 +2856,7 @@ async function postRuntimeEgressFetch(
3799
2856
  ) {
3800
2857
  const response = await fetchRuntimeApi(
3801
2858
  req.baseUrl,
3802
- '/api/v2/plays/internal/egress-fetch',
2859
+ '/api/v2/internal/play-runtime/egress-fetch',
3803
2860
  {
3804
2861
  method: 'POST',
3805
2862
  headers: {
@@ -4375,6 +3432,81 @@ function resolveSheetContractFromReq(
4375
3432
  }
4376
3433
  }
4377
3434
 
3435
+ /**
3436
+ * Run-fatal teardown: release the runtime-sheet attempt leases and work-receipt
3437
+ * leases this run still holds, so an immediate rerun proceeds without waiting
3438
+ * out the 10-minute lease TTL. Every release is attempt/owner fenced downstream,
3439
+ * so it can never weaken a different run's live lease. Failures are logged loud
3440
+ * and swallowed here — the caller still throws the original run-fatal error.
3441
+ */
3442
+ async function releaseRuntimeLeasesOnTeardown(input: {
3443
+ req: RunRequest;
3444
+ leasedSheetNamespaces: Set<string>;
3445
+ emit: (event: RunnerEvent) => void;
3446
+ }): Promise<void> {
3447
+ const { req, leasedSheetNamespaces, emit } = input;
3448
+ const scope = runtimeSheetSessionScope(req);
3449
+ for (const tableNamespace of leasedSheetNamespaces) {
3450
+ const sheetContract = resolveSheetContractFromReq(req, tableNamespace);
3451
+ if (!sheetContract) continue;
3452
+ try {
3453
+ const released = await harnessReleaseSheetAttempt({
3454
+ ...scope,
3455
+ tableNamespace,
3456
+ sheetContract,
3457
+ runId: req.runId,
3458
+ attemptOwnerRunId: req.runId,
3459
+ attemptSeq: req.runAttempt ?? 0,
3460
+ });
3461
+ if (released.released > 0) {
3462
+ emit({
3463
+ type: 'log',
3464
+ level: 'info',
3465
+ message: `Released ${released.released} runtime sheet row lease(s) for ctx.dataset("${tableNamespace}") on run-fatal teardown.`,
3466
+ ts: nowMs(),
3467
+ });
3468
+ }
3469
+ } catch (releaseError) {
3470
+ emit({
3471
+ type: 'log',
3472
+ level: 'warn',
3473
+ message: `Runtime sheet attempt release failed for ctx.dataset("${tableNamespace}") on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
3474
+ releaseError instanceof Error
3475
+ ? releaseError.message
3476
+ : String(releaseError)
3477
+ }`,
3478
+ ts: nowMs(),
3479
+ });
3480
+ }
3481
+ }
3482
+ try {
3483
+ const released = await harnessReleaseRuntimeReceipts({
3484
+ ...scope,
3485
+ runId: req.runId,
3486
+ runAttempt: req.runAttempt ?? 0,
3487
+ });
3488
+ if (released.released > 0) {
3489
+ emit({
3490
+ type: 'log',
3491
+ level: 'info',
3492
+ message: `Released ${released.released} work-receipt lease(s) on run-fatal teardown.`,
3493
+ ts: nowMs(),
3494
+ });
3495
+ }
3496
+ } catch (releaseError) {
3497
+ emit({
3498
+ type: 'log',
3499
+ level: 'warn',
3500
+ message: `Work-receipt release failed on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
3501
+ releaseError instanceof Error
3502
+ ? releaseError.message
3503
+ : String(releaseError)
3504
+ }`,
3505
+ ts: nowMs(),
3506
+ });
3507
+ }
3508
+ }
3509
+
4378
3510
  function staticPipelineFromReq(req: RunRequest): PlayStaticPipeline | null {
4379
3511
  const snapshot = req.contractSnapshot as
4380
3512
  | { staticPipeline?: unknown }
@@ -4417,6 +3549,13 @@ async function persistCompletedMapRows(input: {
4417
3549
  rows: Record<string, unknown>[];
4418
3550
  outputFields: string[];
4419
3551
  extraOutputFields?: string[];
3552
+ attemptId?: string | null;
3553
+ attemptOwnerRunId?: string | null;
3554
+ attemptExpiresAt?: string | null;
3555
+ attemptSeq?: number | null;
3556
+ budgetMeter?: WorkerWorkBudgetMeter;
3557
+ force?: boolean;
3558
+ forceFailedRows?: boolean;
4420
3559
  }): Promise<{
4421
3560
  rows: number;
4422
3561
  written: number;
@@ -4462,23 +3601,28 @@ async function persistCompletedMapRows(input: {
4462
3601
  sheetContract,
4463
3602
  rows,
4464
3603
  outputFields,
3604
+ attemptId: input.attemptId ?? null,
3605
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3606
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3607
+ attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
3608
+ forceFailedRows: input.forceFailedRows === true,
4465
3609
  };
3610
+ input.budgetMeter?.count('sheet');
4466
3611
  const expectedKeys = [
4467
3612
  ...new Set(
4468
3613
  input.rows
4469
- .map((row) => resolveMapRowOutcomeKey(row))
4470
- .filter((key): key is string => Boolean(key)),
3614
+ .map((row) => resolveMapRowOutcomeKey(row) ?? row.key)
3615
+ .filter(Boolean),
4471
3616
  ),
4472
- ];
4473
- const expectedVisibleRows =
4474
- expectedKeys.length > 0 ? expectedKeys.length : rows.length;
3617
+ ] as string[];
3618
+ const expectedVisibleRows = expectedKeys.length || rows.length;
4475
3619
  const readVisibleRowCount = async () => {
4476
3620
  if (expectedKeys.length > 0) {
4477
3621
  const result = await harnessReadSheetDatasetRowKeys({
4478
3622
  ...sessionScope,
4479
3623
  tableNamespace,
4480
3624
  runId: req.runId,
4481
- rowMode: 'all',
3625
+ rowMode: 'terminalAnyRun',
4482
3626
  keys: expectedKeys,
4483
3627
  });
4484
3628
  return result.keys.length;
@@ -4494,6 +3638,24 @@ async function persistCompletedMapRows(input: {
4494
3638
  return result.rows.length;
4495
3639
  };
4496
3640
  const result = await harnessPersistCompletedSheetRows(persistRequest);
3641
+ const hasAttemptFence = Boolean(input.attemptId?.trim());
3642
+ const assertAttemptStillOwnsTerminalWrites = (
3643
+ written: number,
3644
+ visible: number | null,
3645
+ ) => {
3646
+ // Recovery finalization may include rows completed by an earlier attempt.
3647
+ // A complete terminal readback proves they were not lost.
3648
+ if (
3649
+ !hasAttemptFence ||
3650
+ written === rows.length ||
3651
+ (visible !== null && visible >= expectedVisibleRows)
3652
+ ) {
3653
+ return;
3654
+ }
3655
+ throw new Error(
3656
+ `Runtime sheet attempt lost ownership for ${tableNamespace}: wrote ${written}/${rows.length}; run ${req.runId}; attempt ${input.attemptId}.`,
3657
+ );
3658
+ };
4497
3659
  console.warn('[play-runner.persist_completed_map_rows.result]', {
4498
3660
  runId: req.runId,
4499
3661
  playName: req.playName,
@@ -4518,11 +3680,19 @@ async function persistCompletedMapRows(input: {
4518
3680
  sheetContract,
4519
3681
  rows: input.rows.map((row) => runtimeCsvStorageRow(row)),
4520
3682
  runId: req.runId,
3683
+ attemptId: input.attemptId ?? null,
3684
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3685
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3686
+ attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
3687
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
4521
3688
  inputOffset: 0,
4522
3689
  });
3690
+ input.budgetMeter?.count('sheet');
4523
3691
  const retry = await harnessPersistCompletedSheetRows(persistRequest);
3692
+ input.budgetMeter?.count('sheet');
4524
3693
  retryWritten = retry.rowsWritten;
4525
3694
  retryVisible = await readVisibleRowCount();
3695
+ assertAttemptStillOwnsTerminalWrites(retryWritten, retryVisible);
4526
3696
  if (retryVisible >= expectedVisibleRows) {
4527
3697
  return {
4528
3698
  rows: rows.length,
@@ -4536,6 +3706,7 @@ async function persistCompletedMapRows(input: {
4536
3706
  `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}/${expectedVisibleRows}; run ${req.runId}.`,
4537
3707
  );
4538
3708
  }
3709
+ assertAttemptStillOwnsTerminalWrites(result.rowsWritten, visibleRows);
4539
3710
  }
4540
3711
  if (result.rowsWritten !== rows.length && visibleRows < expectedVisibleRows) {
4541
3712
  throw new Error(
@@ -4557,8 +3728,14 @@ async function applyLiveMapRowUpdates(input: {
4557
3728
  updates: Array<Omit<PlayRowUpdate, 'rowId'> & { runId?: string }>;
4558
3729
  outputFields: string[];
4559
3730
  extraOutputFields?: string[];
3731
+ attemptId?: string | null;
3732
+ attemptOwnerRunId?: string | null;
3733
+ attemptExpiresAt?: string | null;
3734
+ attemptSeq?: number | null;
3735
+ budgetMeter?: WorkerWorkBudgetMeter;
4560
3736
  }): Promise<void> {
4561
3737
  if (input.updates.length === 0) return;
3738
+ input.budgetMeter?.count('sheet');
4562
3739
  const outputFields = [
4563
3740
  ...input.outputFields,
4564
3741
  ...(input.extraOutputFields ?? []).filter(
@@ -4581,7 +3758,16 @@ async function applyLiveMapRowUpdates(input: {
4581
3758
  contractSnapshot: input.req.contractSnapshot,
4582
3759
  runId: input.req.runId,
4583
3760
  userEmail: input.req.userEmail,
4584
- updates: input.updates,
3761
+ updates: input.updates.map((update) => ({
3762
+ ...update,
3763
+ attemptId: update.attemptId ?? input.attemptId ?? null,
3764
+ attemptOwnerRunId:
3765
+ update.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
3766
+ attemptExpiresAt:
3767
+ update.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
3768
+ attemptSeq:
3769
+ update.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
3770
+ })),
4585
3771
  },
4586
3772
  );
4587
3773
  }
@@ -4592,18 +3778,39 @@ async function prepareMapRows(input: {
4592
3778
  rows: Record<string, unknown>[];
4593
3779
  inputOffset: number;
4594
3780
  outputFields: string[];
3781
+ attemptId?: string | null;
3782
+ attemptOwnerRunId?: string | null;
3783
+ attemptExpiresAt?: string | null;
3784
+ attemptSeq?: number | null;
3785
+ budgetMeter?: WorkerWorkBudgetMeter;
4595
3786
  force?: boolean;
4596
3787
  }): Promise<{
4597
3788
  inserted: number;
4598
3789
  skipped: number;
4599
3790
  pendingRows: Record<string, unknown>[];
4600
3791
  completedRows: Record<string, unknown>[];
3792
+ blockedRows: Record<string, unknown>[];
3793
+ attemptId?: string | null;
3794
+ attemptOwnerRunId?: string | null;
3795
+ attemptExpiresAt?: string | null;
3796
+ attemptSeq?: number | null;
4601
3797
  }> {
4602
3798
  if (input.rows.length === 0) {
4603
- return { inserted: 0, skipped: 0, pendingRows: [], completedRows: [] };
3799
+ return {
3800
+ inserted: 0,
3801
+ skipped: 0,
3802
+ pendingRows: [],
3803
+ completedRows: [],
3804
+ blockedRows: [],
3805
+ attemptId: input.attemptId ?? null,
3806
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3807
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3808
+ attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
3809
+ };
4604
3810
  }
4605
3811
  const sessionScope = runtimeSheetSessionScope(input.req);
4606
3812
  const rows = input.rows.map((row) => runtimeCsvStorageRow(row));
3813
+ input.budgetMeter?.count('sheet');
4607
3814
  const result = await harnessStartSheetDataset({
4608
3815
  ...sessionScope,
4609
3816
  tableNamespace: input.tableNamespace,
@@ -4614,6 +3821,11 @@ async function prepareMapRows(input: {
4614
3821
  }),
4615
3822
  rows,
4616
3823
  inputOffset: input.inputOffset,
3824
+ attemptId: input.attemptId ?? null,
3825
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3826
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3827
+ attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
3828
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
4617
3829
  ...(input.force ? { force: true } : {}),
4618
3830
  });
4619
3831
  for (const timing of result.timings ?? []) {
@@ -4643,6 +3855,13 @@ async function prepareMapRows(input: {
4643
3855
  skipped: result.skipped,
4644
3856
  pendingRows: result.pendingRows,
4645
3857
  completedRows: result.completedRows,
3858
+ blockedRows: result.blockedRows ?? [],
3859
+ attemptId: result.attemptId ?? input.attemptId ?? null,
3860
+ attemptOwnerRunId:
3861
+ result.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
3862
+ attemptExpiresAt: result.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
3863
+ attemptSeq:
3864
+ result.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
4646
3865
  };
4647
3866
  }
4648
3867
 
@@ -4788,14 +4007,6 @@ function createCoordinatorRatePort(req: RunRequest): CoordinatorRatePort {
4788
4007
  * memoized per isolate. No hints → null (pacing is a no-op; the Governor's
4789
4008
  * global tool-concurrency slot still applies).
4790
4009
  */
4791
- type WorkerPacingResolver = (
4792
- toolId: string,
4793
- ) => Promise<
4794
- (ResolvedPacingPolicy & { retrySafeTransientHttp: boolean }) | null
4795
- >;
4796
-
4797
- type WorkerToolActionCacheVersionResolver = (toolId: string) => Promise<string>;
4798
-
4799
4010
  function createWorkerToolActionCacheVersionResolver(
4800
4011
  req: RunRequest,
4801
4012
  ): WorkerToolActionCacheVersionResolver {
@@ -4803,9 +4014,7 @@ function createWorkerToolActionCacheVersionResolver(
4803
4014
  return (toolId: string) => {
4804
4015
  const normalized = String(toolId || '').trim();
4805
4016
  if (!normalized) {
4806
- return Promise.reject(
4807
- new Error('Runtime tool metadata lookup requires a non-empty tool id.'),
4808
- );
4017
+ return Promise.reject(new Error('Tool metadata needs tool id.'));
4809
4018
  }
4810
4019
  const cached = cache.get(normalized);
4811
4020
  if (cached) return cached;
@@ -4842,6 +4051,44 @@ function createWorkerToolActionCacheVersionResolver(
4842
4051
  };
4843
4052
  }
4844
4053
 
4054
+ function createWorkerToolAuthScopeDigestResolver(req: RunRequest) {
4055
+ // Return type is inferred from createRuntimeToolAuthScopeDigestResolver
4056
+ // (RuntimeToolAuthScopeDigestResolver), whose `invalidate` member is
4057
+ // REQUIRED — the AUTH_SCOPE_CHANGED re-claim wiring below calls it directly,
4058
+ // and a resolver without eviction support must be a type error.
4059
+ return createRuntimeToolAuthScopeDigestResolver({
4060
+ missingToolIdMessage: 'Tool auth scope needs tool id.',
4061
+ fetchDigest: async (normalized) => {
4062
+ const runScopedPath =
4063
+ `/api/v2/plays/runtime-tools/${encodeURIComponent(normalized)}/auth-scope` +
4064
+ `?runId=${encodeURIComponent(req.runId)}`;
4065
+ const res = await fetchRuntimeApi(req.baseUrl, runScopedPath, {
4066
+ method: 'GET',
4067
+ headers: {
4068
+ authorization: `Bearer ${req.executorToken}`,
4069
+ 'cache-control': 'no-store',
4070
+ pragma: 'no-cache',
4071
+ },
4072
+ });
4073
+ if (!res.ok) {
4074
+ throw new Error(
4075
+ `Runtime tool auth scope lookup for ${normalized} failed (${res.status}): ${await res.text()}`,
4076
+ );
4077
+ }
4078
+ const body = (await res.json().catch(() => null)) as {
4079
+ digest?: unknown;
4080
+ } | null;
4081
+ const digest = typeof body?.digest === 'string' ? body.digest.trim() : '';
4082
+ if (!digest) {
4083
+ throw new Error(
4084
+ `Runtime tool auth scope lookup for ${normalized} returned no digest.`,
4085
+ );
4086
+ }
4087
+ return digest;
4088
+ },
4089
+ });
4090
+ }
4091
+
4845
4092
  function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
4846
4093
  const cache = new Map<
4847
4094
  string,
@@ -4983,13 +4230,22 @@ function createMinimalWorkerCtx(
4983
4230
  workflowStep?: WorkflowStep,
4984
4231
  abortSignal?: AbortSignal,
4985
4232
  callbacks?: WorkerCtxCallbacks,
4233
+ workBudgetMeter: WorkerWorkBudgetMeter = createWorkerWorkBudgetMeter({
4234
+ nowMs,
4235
+ }),
4986
4236
  runtimeDeadlineMs?: number,
4237
+ // Namespaces this run took a runtime-sheet attempt lease on. The run-fatal
4238
+ // teardown reads this set to release exactly those leases (never a namespace
4239
+ // whose table the run never created), so a rerun proceeds immediately.
4240
+ leasedSheetNamespaces?: Set<string>,
4987
4241
  receiptSalvage?: ReceiptSalvageBuffer,
4988
4242
  ): unknown {
4989
4243
  const { governor, resolvePacing: resolveToolPacing } =
4990
4244
  createGovernorForRun(req);
4991
4245
  const resolveToolActionCacheVersion =
4992
4246
  createWorkerToolActionCacheVersionResolver(req);
4247
+ const resolveToolAuthScopeDigest =
4248
+ createWorkerToolAuthScopeDigestResolver(req);
4993
4249
  // Play-call depth/count/per-parent budgets, child-play concurrency, and the
4994
4250
  // lineage snapshot are owned by the Governor (createGovernorForRun above).
4995
4251
  // The worker keeps only substrate mechanism here.
@@ -5000,7 +4256,7 @@ function createMinimalWorkerCtx(
5000
4256
  if (!auth) return {};
5001
4257
  const response = await fetchRuntimeApi(
5002
4258
  req.baseUrl,
5003
- '/api/v2/plays/internal/runtime',
4259
+ '/api/v2/internal/play-runtime',
5004
4260
  {
5005
4261
  method: 'POST',
5006
4262
  headers: {
@@ -5039,10 +4295,48 @@ function createMinimalWorkerCtx(
5039
4295
  orgId: req.orgId,
5040
4296
  preloadedDbSessions: req.preloadedDbSessions ?? null,
5041
4297
  userEmail: req.userEmail,
4298
+ runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null,
4299
+ runAttempt: req.runAttempt ?? 0,
4300
+ receiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
5042
4301
  });
4302
+ const executeDispatchedTool = async (input: {
4303
+ id: string;
4304
+ toolId: string;
4305
+ input: Record<string, unknown>;
4306
+ workflowStep?: unknown;
4307
+ callbacks?: WorkerCtxCallbacks;
4308
+ onProviderBackpressure?: (retryAfterMs: number) => void;
4309
+ onRetryAttempt?: () => void;
4310
+ transientHttpRetrySafe?: boolean;
4311
+ durableCallReceiptKey?: string | null;
4312
+ executionAuthScopeDigest?: string | null;
4313
+ providerIdempotencyKey?: string | null;
4314
+ runtimeTimeoutMs?: number;
4315
+ directOptions?: WorkerToolDirectOptions;
4316
+ }): Promise<ToolExecuteResult> =>
4317
+ await executeToolWithLifecycle(
4318
+ req,
4319
+ {
4320
+ id: input.id,
4321
+ toolId: input.toolId,
4322
+ input: input.input,
4323
+ durableCallReceiptKey: input.durableCallReceiptKey,
4324
+ executionAuthScopeDigest: input.executionAuthScopeDigest,
4325
+ providerIdempotencyKey: input.providerIdempotencyKey,
4326
+ },
4327
+ input.workflowStep as WorkflowStep | undefined,
4328
+ input.callbacks,
4329
+ input.onProviderBackpressure,
4330
+ input.onRetryAttempt,
4331
+ input.transientHttpRetrySafe === true,
4332
+ abortSignal,
4333
+ runtimeDeadlineMs,
4334
+ input.runtimeTimeoutMs,
4335
+ input.directOptions,
4336
+ );
5043
4337
  const executeWithRuntimeReceipt = async <T>(
5044
4338
  key: string,
5045
- execute: () => Promise<T> | T,
4339
+ execute: (context: WorkerRuntimeReceiptExecutionContext) => Promise<T> | T,
5046
4340
  options: {
5047
4341
  repairRunningReceiptForSameRun?: boolean;
5048
4342
  reclaimRunning?: boolean;
@@ -5056,7 +4350,8 @@ function createMinimalWorkerCtx(
5056
4350
  runId: req.runId,
5057
4351
  key,
5058
4352
  receiptStore,
5059
- execute: async () => serializeDurableStepValue(await execute()),
4353
+ execute: async (context) =>
4354
+ serializeDurableStepValue(await execute(context)),
5060
4355
  repairRunningReceiptForSameRun:
5061
4356
  options.repairRunningReceiptForSameRun ?? false,
5062
4357
  reclaimRunning: options.reclaimRunning ?? false,
@@ -5098,24 +4393,54 @@ function createMinimalWorkerCtx(
5098
4393
  staleAfterSeconds,
5099
4394
  });
5100
4395
  };
5101
- // Ctx-wide breaker for top-level ctx.tool calls (outside any map). Each map
5102
- // creates its own map-scoped latch (see runMap) so per-map dispatch stats and
5103
- // the runMap failure message stay map-local.
4396
+ const emitAuthScopeReclaimRunLog = (input: {
4397
+ toolId: string;
4398
+ requestIds: string[];
4399
+ }) => {
4400
+ // Durable run-log marker for the bounded AUTH_SCOPE_CHANGED re-claim
4401
+ // (mirrors the shared-runtime marker in shared_libs/play-runtime/context.ts
4402
+ // so run logs expose the credential transition on every execution path).
4403
+ emitEvent({
4404
+ type: 'log',
4405
+ level: 'info',
4406
+ message:
4407
+ `ctx.tools.execute(${input.toolId}): auth_scope_changed_reclaim ` +
4408
+ `(requests: ${input.requestIds.join(', ')}); credential identity ` +
4409
+ `changed mid-run, re-resolved the auth scope and re-claimed fresh ` +
4410
+ `receipts under the new scope.`,
4411
+ ts: nowMs(),
4412
+ });
4413
+ };
5104
4414
  const rootPersistenceLatch = createRuntimePersistenceLatch();
5105
- const rootToolBatchScheduler = new WorkerToolBatchScheduler(
4415
+ const rootToolBatchScheduler = new WorkerToolBatchScheduler({
5106
4416
  req,
5107
4417
  governor,
5108
- resolveToolPacing,
4418
+ resolvePacing: resolveToolPacing,
5109
4419
  resolveToolActionCacheVersion,
4420
+ resolveToolAuthScopeDigest,
4421
+ invalidateToolAuthScopeDigest: (toolId) =>
4422
+ resolveToolAuthScopeDigest.invalidate(toolId),
4423
+ onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
4424
+ executeTool: executeDispatchedTool,
4425
+ serializeDurableValue: serializeDurableStepValue,
4426
+ deserializeDurableValue: deserializeDurableStepValue,
4427
+ nowMs,
4428
+ batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
4429
+ recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
5110
4430
  abortSignal,
5111
- undefined,
5112
4431
  callbacks,
5113
4432
  receiptStore,
5114
- true,
4433
+ allowLocalRetryReceipts: true,
4434
+ runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
4435
+ runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
4436
+ req,
4437
+ runtimeReceiptLeaseTtlMs(req) ?? PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
4438
+ ),
4439
+ budgetMeter: workBudgetMeter,
5115
4440
  runtimeDeadlineMs,
5116
- rootPersistenceLatch,
4441
+ persistenceLatch: rootPersistenceLatch,
5117
4442
  receiptSalvage,
5118
- );
4443
+ });
5119
4444
  // Local ancestry chain that always ENDS with the currently-executing play
5120
4445
  // (req.playName). The /api/v2/plays/run lineage validator requires the
5121
4446
  // submitted ancestry's tail to equal the executor token's play name (i.e.
@@ -5150,23 +4475,33 @@ function createMinimalWorkerCtx(
5150
4475
  ): Promise<unknown> => {
5151
4476
  const mapStartedAt = nowMs();
5152
4477
  const mapNodeId = `map:${name}`;
5153
- // Map-wide persistence-failure circuit breaker. Shared across every chunk's
5154
- // scheduler (the scheduler is rebuilt per chunk) and the map's sheet-persist
5155
- // flush chain, so the first persistence failure halts NEW provider dispatch
5156
- // for the rest of the map and its prevented-call count surfaces in the
5157
- // runMap failure message below.
5158
- const persistenceLatch = createRuntimePersistenceLatch();
5159
4478
  const inputRows = rows;
5160
4479
  const rowCountHint = datasetRowCountHint(inputRows);
5161
4480
  const baseOffset = 0;
5162
4481
  const fieldEntries = Object.entries(fieldsDef);
5163
4482
  const plan = req.executionPlan;
5164
- const rowsPerChunk = chooseWorkerMapRowsPerChunk({
4483
+ const mapDispatchPlan = chooseWorkerMapDispatchPlan({
5165
4484
  mapName: name,
5166
4485
  rowCountHint,
5167
4486
  executionPlan: plan,
5168
4487
  staticPipeline: staticPipelineFromReq(req),
5169
4488
  });
4489
+ const rowsPerChunk = mapDispatchPlan.rowsPerChunk;
4490
+ recordRunnerPerfTrace({
4491
+ req,
4492
+ phase: 'runner.map_dispatch_plan',
4493
+ ms: 0,
4494
+ extra: {
4495
+ mapName: name,
4496
+ rowsPerChunk,
4497
+ rowCountHint,
4498
+ estimatedExternalStepsPerRow:
4499
+ mapDispatchPlan.estimatedExternalStepsPerRow,
4500
+ workEstimate: mapDispatchPlan.workEstimate,
4501
+ reason: mapDispatchPlan.reason,
4502
+ dynamicWorkerArtifactVersion: DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION,
4503
+ },
4504
+ });
5170
4505
  const outputFields = fieldEntries.map(([field]) => field);
5171
4506
  const updateMapProgress = (
5172
4507
  progress: LiveNodeProgressSnapshot,
@@ -5182,6 +4517,22 @@ function createMinimalWorkerCtx(
5182
4517
  forceFlush: options?.forceFlush ?? true,
5183
4518
  });
5184
4519
  };
4520
+ let queuedMapProgress: Promise<void> = Promise.resolve();
4521
+ let queuedMapProgressError: unknown = null;
4522
+ const enqueueMapProgressUpdate = (
4523
+ progress: LiveNodeProgressSnapshot,
4524
+ options?: { forceFlush?: boolean },
4525
+ ) => {
4526
+ queuedMapProgress = queuedMapProgress
4527
+ .then(() => Promise.resolve(updateMapProgress(progress, options)))
4528
+ .catch((error) => {
4529
+ queuedMapProgressError ??= error;
4530
+ });
4531
+ };
4532
+ const flushQueuedMapProgressUpdates = async () => {
4533
+ await queuedMapProgress;
4534
+ if (queuedMapProgressError) throw queuedMapProgressError;
4535
+ };
5185
4536
  const formatMapProgressMessage = (completed: number, total?: number) =>
5186
4537
  typeof total === 'number' && Number.isFinite(total) && total > 0
5187
4538
  ? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
@@ -5206,7 +4557,9 @@ function createMinimalWorkerCtx(
5206
4557
  return formatMapProgressMessage(completed, input.total);
5207
4558
  };
5208
4559
  const formatMapProcessingMessage = (rowsToExecute: number) =>
5209
- rowsToExecute > 0 ? `${rowsToExecute.toLocaleString()} rows` : null;
4560
+ rowsToExecute > 0
4561
+ ? `Processing ${rowsToExecute.toLocaleString()} rows`
4562
+ : null;
5210
4563
  const formatMapExecutionHeartbeatMessage = (input: {
5211
4564
  rowsToExecute: number;
5212
4565
  startedRows: number;
@@ -5220,7 +4573,7 @@ function createMinimalWorkerCtx(
5220
4573
  const waitingRows = Math.max(0, rowsToExecute - startedRows);
5221
4574
  const parts = [
5222
4575
  activeRows > 0 ? `${activeRows.toLocaleString()} active` : null,
5223
- waitingRows > 0 ? `${waitingRows.toLocaleString()} queued` : null,
4576
+ waitingRows > 0 ? `${waitingRows.toLocaleString()} waiting` : null,
5224
4577
  completedRows > 0 ? `${completedRows.toLocaleString()} done` : null,
5225
4578
  ].filter((part): part is string => Boolean(part));
5226
4579
  const base =
@@ -5384,6 +4737,12 @@ function createMinimalWorkerCtx(
5384
4737
  ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
5385
4738
  const chunkStartedAt = nowMs();
5386
4739
  assertNotAborted(abortSignal);
4740
+ const rowAttempt = makeRuntimeSheetAttempt({
4741
+ runId: req.runId,
4742
+ runAttempt: req.runAttempt,
4743
+ mapName: name,
4744
+ chunkIndex,
4745
+ });
5387
4746
  const keyStartedAt = nowMs();
5388
4747
  const chunkEntries = chunkRows.map((row, localIndex) => {
5389
4748
  const absoluteIndex = baseOffset + chunkStart + localIndex;
@@ -5411,8 +4770,21 @@ function createMinimalWorkerCtx(
5411
4770
  ...mapRowOutcomeRuntimeFields({ key: rowKey }),
5412
4771
  })),
5413
4772
  inputOffset: baseOffset + chunkStart,
4773
+ ...rowAttempt,
4774
+ budgetMeter: workBudgetMeter,
5414
4775
  force: !!req.force,
5415
4776
  });
4777
+ const activeRowAttempt = {
4778
+ attemptId: prepared.attemptId ?? rowAttempt.attemptId,
4779
+ attemptOwnerRunId:
4780
+ prepared.attemptOwnerRunId ?? rowAttempt.attemptOwnerRunId,
4781
+ attemptExpiresAt:
4782
+ prepared.attemptExpiresAt ?? rowAttempt.attemptExpiresAt,
4783
+ attemptSeq: prepared.attemptSeq ?? rowAttempt.attemptSeq,
4784
+ };
4785
+ // Record that this run holds attempt leases in `name`'s sheet so a
4786
+ // run-fatal teardown releases them (the table now exists).
4787
+ leasedSheetNamespaces?.add(name);
5416
4788
  recordRunnerPerfTrace({
5417
4789
  req,
5418
4790
  phase: 'runner.map_chunk.prepare_rows',
@@ -5425,14 +4797,15 @@ function createMinimalWorkerCtx(
5425
4797
  skipped: prepared.skipped,
5426
4798
  pendingRows: prepared.pendingRows.length,
5427
4799
  completedRows: prepared.completedRows.length,
4800
+ blockedRows: prepared.blockedRows.length,
5428
4801
  },
5429
4802
  });
5430
4803
  const progressTotalRows = rowCountHint ?? chunkRows.length;
5431
4804
  const preparedCompletedRows = Math.min(
5432
4805
  progressTotalRows,
5433
- totalRowsWritten + prepared.completedRows.length,
4806
+ totalRowsWritten,
5434
4807
  );
5435
- await updateMapProgress({
4808
+ enqueueMapProgressUpdate({
5436
4809
  completed: preparedCompletedRows,
5437
4810
  total: progressTotalRows,
5438
4811
  startedAt: mapStartedAt,
@@ -5445,6 +4818,9 @@ function createMinimalWorkerCtx(
5445
4818
  const pendingKeys = new Set<string>();
5446
4819
  const pendingRowsByKey = new Map<string, Record<string, unknown>>();
5447
4820
  const completedKeys = new Set<string>();
4821
+ const completedRowsByKey = new Map<string, Record<string, unknown>>();
4822
+ const blockedKeys = new Set<string>();
4823
+ const blockedRowsByKey = new Map<string, Record<string, unknown>>();
5448
4824
  const preparedKeys = new Set<string>();
5449
4825
  for (const row of prepared.pendingRows) {
5450
4826
  const key =
@@ -5462,14 +4838,95 @@ function createMinimalWorkerCtx(
5462
4838
  deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
5463
4839
  if (key) {
5464
4840
  completedKeys.add(key);
4841
+ completedRowsByKey.set(key, row);
4842
+ preparedKeys.add(key);
4843
+ }
4844
+ }
4845
+ for (const row of prepared.blockedRows) {
4846
+ const key =
4847
+ resolveMapRowOutcomeKey(row) ??
4848
+ deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
4849
+ if (key) {
4850
+ blockedKeys.add(key);
4851
+ blockedRowsByKey.set(key, row);
5465
4852
  preparedKeys.add(key);
5466
4853
  }
5467
4854
  }
4855
+ if (blockedKeys.size > 0) {
4856
+ const rowKeySamples = Array.from(blockedKeys).slice(
4857
+ 0,
4858
+ MAP_ROW_FAILURE_SAMPLE_LIMIT,
4859
+ );
4860
+ const message =
4861
+ `ctx.dataset("${name}") chunk ${chunkIndex} is blocked by ` +
4862
+ `${blockedKeys.size} active runtime sheet row lease(s) owned by another attempt. ` +
4863
+ `Retry after the owning attempt finishes or the row leases expire.` +
4864
+ (rowKeySamples.length > 0
4865
+ ? ` Blocked row keys: ${rowKeySamples.join(', ')}`
4866
+ : '');
4867
+ recordRunnerPerfTrace({
4868
+ req,
4869
+ phase: 'runner.map_chunk.blocked_rows',
4870
+ ms: nowMs() - prepareStartedAt,
4871
+ extra: {
4872
+ mapName: name,
4873
+ chunkIndex,
4874
+ rowsBlocked: blockedKeys.size,
4875
+ rowKeySamples,
4876
+ },
4877
+ });
4878
+ enqueueMapProgressUpdate({
4879
+ completed: preparedCompletedRows,
4880
+ total: progressTotalRows,
4881
+ waitingRows: blockedKeys.size,
4882
+ startedAt: mapStartedAt,
4883
+ message,
4884
+ });
4885
+ return {
4886
+ status: 'blocked',
4887
+ chunkIndex,
4888
+ rangeStart: baseOffset + chunkStart,
4889
+ rangeEnd: baseOffset + chunkStart + chunkRows.length,
4890
+ rowsRead: chunkRows.length,
4891
+ rowsBlocked: blockedKeys.size,
4892
+ rowKeySamples,
4893
+ blockedRowSamples: rowKeySamples.map((rowKey) => {
4894
+ const blockedRow = blockedRowsByKey.get(rowKey);
4895
+ const attemptId =
4896
+ typeof blockedRow?.__deeplineAttemptId === 'string'
4897
+ ? blockedRow.__deeplineAttemptId
4898
+ : typeof blockedRow?._attempt_id === 'string'
4899
+ ? blockedRow._attempt_id
4900
+ : activeRowAttempt.attemptId;
4901
+ return {
4902
+ __deeplineRowKey: rowKey,
4903
+ __deeplineAttemptId: attemptId,
4904
+ };
4905
+ }),
4906
+ outputDatasetId: `map:${name}`,
4907
+ message,
4908
+ };
4909
+ }
5468
4910
  const missingPreparedRows = chunkEntries.filter(
5469
4911
  ({ rowKey }) => !preparedKeys.has(rowKey),
5470
4912
  );
5471
- const rowsToExecuteEntries = chunkEntries.filter(
5472
- ({ rowKey }) => pendingKeys.has(rowKey) || !completedKeys.has(rowKey),
4913
+ if (missingPreparedRows.length > 0) {
4914
+ const rowKeySamples = missingPreparedRows
4915
+ .map(({ rowKey }) => rowKey)
4916
+ .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
4917
+ throw new Error(
4918
+ `ctx.dataset("${name}") runtime preparation omitted ${missingPreparedRows.length} row disposition(s). ` +
4919
+ `Every input row must be claimed, completed, or blocked before execution starts.` +
4920
+ (rowKeySamples.length > 0
4921
+ ? ` Missing row keys: ${rowKeySamples.join(', ')}`
4922
+ : ''),
4923
+ );
4924
+ }
4925
+ const rowsToExecuteEntries = chunkEntries.filter(({ rowKey }) =>
4926
+ pendingKeys.has(rowKey),
4927
+ );
4928
+ const completedChunkEntries = chunkEntries.filter(({ rowKey }) =>
4929
+ completedKeys.has(rowKey),
5473
4930
  );
5474
4931
  const uniqueRowsToExecuteEntries = [
5475
4932
  ...new Map(
@@ -5486,18 +4943,15 @@ function createMinimalWorkerCtx(
5486
4943
  rowsToExecute.length,
5487
4944
  );
5488
4945
  if (processingMessage) {
5489
- await updateMapProgress({
4946
+ enqueueMapProgressUpdate({
5490
4947
  completed: preparedCompletedRows,
5491
4948
  total: progressTotalRows,
5492
4949
  startedAt: mapStartedAt,
5493
4950
  message: processingMessage,
5494
4951
  });
5495
4952
  }
5496
- const rowsInserted = prepared.inserted + missingPreparedRows.length;
5497
- const rowsSkipped = Math.max(
5498
- 0,
5499
- prepared.skipped - missingPreparedRows.length,
5500
- );
4953
+ const rowsInserted = prepared.inserted;
4954
+ const rowsSkipped = prepared.skipped;
5501
4955
  let completedExecutedRows = 0;
5502
4956
  let failedExecutedRows = 0;
5503
4957
  let startedExecutedRows = 0;
@@ -5505,12 +4959,7 @@ function createMinimalWorkerCtx(
5505
4959
  let lastChunkProgressAt = 0;
5506
4960
  let lastExecutionHeartbeatAt = nowMs();
5507
4961
  const completedRowsForProgress = () =>
5508
- Math.min(
5509
- progressTotalRows,
5510
- totalRowsWritten +
5511
- prepared.completedRows.length +
5512
- completedExecutedRows,
5513
- );
4962
+ Math.min(progressTotalRows, totalRowsWritten + completedExecutedRows);
5514
4963
  const reportExecutionHeartbeat = (force = false) => {
5515
4964
  const now = nowMs();
5516
4965
  if (
@@ -5520,7 +4969,7 @@ function createMinimalWorkerCtx(
5520
4969
  return;
5521
4970
  }
5522
4971
  lastExecutionHeartbeatAt = now;
5523
- void updateMapProgress(
4972
+ enqueueMapProgressUpdate(
5524
4973
  {
5525
4974
  completed: completedRowsForProgress(),
5526
4975
  total: progressTotalRows,
@@ -5554,7 +5003,7 @@ function createMinimalWorkerCtx(
5554
5003
  return;
5555
5004
  }
5556
5005
  lastChunkProgressAt = now;
5557
- void updateMapProgress(
5006
+ enqueueMapProgressUpdate(
5558
5007
  {
5559
5008
  completed,
5560
5009
  total: progressTotalRows,
@@ -5585,23 +5034,44 @@ function createMinimalWorkerCtx(
5585
5034
  const failedRowEntries: Array<
5586
5035
  { row: T & Record<string, unknown>; error: string } | undefined
5587
5036
  > = new Array(rowsToExecute.length);
5037
+ const fatalReceiptPersistenceRowEntries: Array<
5038
+ { row: T & Record<string, unknown>; error: string } | undefined
5039
+ > = new Array(rowsToExecute.length);
5588
5040
  const executedCellMetaPatches: Array<
5589
5041
  Record<string, WorkerCellMetaPatchEntry> | undefined
5590
5042
  > = new Array(rowsToExecute.length);
5591
- const toolBatchScheduler = new WorkerToolBatchScheduler(
5043
+ const persistenceLatch = createRuntimePersistenceLatch();
5044
+ const toolBatchScheduler = new WorkerToolBatchScheduler({
5592
5045
  req,
5593
5046
  governor,
5594
- resolveToolPacing,
5047
+ resolvePacing: resolveToolPacing,
5595
5048
  resolveToolActionCacheVersion,
5049
+ resolveToolAuthScopeDigest,
5050
+ invalidateToolAuthScopeDigest: (toolId) =>
5051
+ resolveToolAuthScopeDigest.invalidate(toolId),
5052
+ onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
5053
+ executeTool: executeDispatchedTool,
5054
+ serializeDurableValue: serializeDurableStepValue,
5055
+ deserializeDurableValue: deserializeDurableStepValue,
5056
+ nowMs,
5057
+ batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
5058
+ recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
5596
5059
  abortSignal,
5597
- reportSettledToolRequests,
5060
+ onRequestsSettled: reportSettledToolRequests,
5598
5061
  callbacks,
5599
5062
  receiptStore,
5600
- false,
5063
+ allowLocalRetryReceipts: false,
5064
+ runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
5065
+ runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
5066
+ req,
5067
+ runtimeReceiptLeaseTtlMs(req) ??
5068
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
5069
+ ),
5070
+ budgetMeter: workBudgetMeter,
5601
5071
  runtimeDeadlineMs,
5602
5072
  persistenceLatch,
5603
5073
  receiptSalvage,
5604
- );
5074
+ });
5605
5075
  let stepCellsCompleted = 0;
5606
5076
  let stepCellsSkipped = 0;
5607
5077
  const generatedOutputFields = new Set<string>();
@@ -5680,6 +5150,8 @@ function createMinimalWorkerCtx(
5680
5150
  tableNamespace: name,
5681
5151
  outputFields,
5682
5152
  extraOutputFields: Array.from(generatedOutputFields),
5153
+ budgetMeter: workBudgetMeter,
5154
+ ...activeRowAttempt,
5683
5155
  rows: [
5684
5156
  ...rowsToPersist.map(({ row, executedIndex }) =>
5685
5157
  mapRowOutcomeRuntimeRow(
@@ -5743,9 +5215,6 @@ function createMinimalWorkerCtx(
5743
5215
  });
5744
5216
  persistFlushChain = task.catch((error) => {
5745
5217
  persistFailure ??= error;
5746
- // Output-sheet flush failed: trip the breaker so the scheduler stops
5747
- // dispatching new provider calls for the rest of this map.
5748
- tripRuntimePersistenceLatch(persistenceLatch, error);
5749
5218
  });
5750
5219
  return task;
5751
5220
  };
@@ -5787,6 +5256,8 @@ function createMinimalWorkerCtx(
5787
5256
  outputFields,
5788
5257
  extraOutputFields,
5789
5258
  updates,
5259
+ ...activeRowAttempt,
5260
+ budgetMeter: workBudgetMeter,
5790
5261
  });
5791
5262
  } catch (error) {
5792
5263
  liveUpdateFailures += 1;
@@ -5894,18 +5365,11 @@ function createMinimalWorkerCtx(
5894
5365
  request.input,
5895
5366
  workflowStep,
5896
5367
  {
5897
- force: request.force === true || !!req.force,
5368
+ force: request.force === true,
5369
+ runForceRefresh: req.forceToolRefresh === true,
5370
+ runForceFailedRefresh: req.force === true,
5898
5371
  staleAfterSeconds: request.staleAfterSeconds,
5899
5372
  },
5900
- // Parity with the play-level ctx.tools.execute (and the
5901
- // cjs runner's map path): the public ToolExecutionRequest
5902
- // runtime options must not be silently dropped for map
5903
- // rows — receiptWaitMs bounds how long a resumed run
5904
- // waits on a stuck `running` receipt before reclaiming.
5905
- {
5906
- timeoutMs: request.timeoutMs,
5907
- receiptWaitMs: request.receiptWaitMs,
5908
- },
5909
5373
  );
5910
5374
  },
5911
5375
  },
@@ -6014,8 +5478,28 @@ function createMinimalWorkerCtx(
6014
5478
  notePersistableRow(enriched);
6015
5479
  reportChunkProgress(false);
6016
5480
  } catch (rowError) {
6017
- // Abort/budget errors stay run-fatal and leave no partial
6018
- // state: rethrow immediately without recording the row.
5481
+ // Receipt persistence errors stay run-fatal, but keep the
5482
+ // row identity so teardown can make the recovery state loud.
5483
+ if (isRuntimeReceiptPersistenceError(rowError)) {
5484
+ const message = formatWorkerRowFailureMessage(rowError);
5485
+ if (activeField) {
5486
+ cellMetaPatch[activeField] = {
5487
+ status: 'failed',
5488
+ stage: activeField,
5489
+ runId: req.runId,
5490
+ error: message,
5491
+ };
5492
+ }
5493
+ executedCellMetaPatches[myIndex] =
5494
+ Object.keys(cellMetaPatch).length > 0
5495
+ ? cellMetaPatch
5496
+ : undefined;
5497
+ fatalReceiptPersistenceRowEntries[myIndex] = {
5498
+ row: enriched as T & Record<string, unknown>,
5499
+ error: message,
5500
+ };
5501
+ throw rowError;
5502
+ }
6019
5503
  if (isRunFatalWorkerRowError(rowError)) {
6020
5504
  throw rowError;
6021
5505
  }
@@ -6095,16 +5579,25 @@ function createMinimalWorkerCtx(
6095
5579
  return results;
6096
5580
  },
6097
5581
  );
6098
- while (!workerSettled) {
6099
- await Promise.race([
6100
- workerResultsPromise,
6101
- sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
6102
- ]);
6103
- if (!workerSettled) {
6104
- reportExecutionHeartbeat(false);
6105
- }
6106
- }
6107
- const workerResults = await workerResultsPromise;
5582
+ const workerResults = await executeWithRuntimeSheetAttemptHeartbeat({
5583
+ req,
5584
+ tableNamespace: name,
5585
+ sheetContract: requireSheetContract(req, name),
5586
+ ...activeRowAttempt,
5587
+ rowKeys: uniqueRowsToExecuteEntries.map((entry) => entry.rowKey),
5588
+ execute: async () => {
5589
+ while (!workerSettled) {
5590
+ await Promise.race([
5591
+ workerResultsPromise,
5592
+ sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
5593
+ ]);
5594
+ if (!workerSettled) {
5595
+ reportExecutionHeartbeat(false);
5596
+ }
5597
+ }
5598
+ return await workerResultsPromise;
5599
+ },
5600
+ });
6108
5601
  recordRunnerPerfTrace({
6109
5602
  req,
6110
5603
  phase: 'runner.map_chunk.execute_workers',
@@ -6133,26 +5626,89 @@ function createMinimalWorkerCtx(
6133
5626
  .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
6134
5627
  const fatalMapChunkSummary = async (
6135
5628
  error: unknown,
6136
- ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => ({
6137
- chunkIndex,
6138
- rangeStart: baseOffset + chunkStart,
6139
- rangeEnd: baseOffset + chunkStart,
6140
- rowsRead: chunkRows.length,
6141
- rowsWritten: 0,
6142
- rowsExecuted: rowsToExecute.length,
6143
- rowsCached: 0,
6144
- rowsDuplicateReused: duplicateInputReuseCount,
6145
- rowsInserted,
6146
- rowsSkipped,
6147
- rowsFailed: failedExecutedRows,
6148
- rowFailureSamples: buildRowFailureSamples(),
6149
- stepCellsCompleted,
6150
- stepCellsSkipped,
6151
- outputDatasetId: `map:${name}`,
6152
- hash: await hashJson([]),
6153
- fatalError: formatWorkerRowFailureMessage(error),
6154
- preview: [],
6155
- });
5629
+ ): Promise<
5630
+ WorkerMapCompletedChunkSummary<T & Record<string, unknown>>
5631
+ > => {
5632
+ const fatalError = formatWorkerRowFailureMessage(error);
5633
+ const breakerMessage = persistenceLatch.tripped
5634
+ ? ` Circuit breaker prevented ${persistenceLatch.preventedCallCount} queued provider call(s).`
5635
+ : '';
5636
+ return {
5637
+ status: 'completed',
5638
+ chunkIndex,
5639
+ rangeStart: baseOffset + chunkStart,
5640
+ rangeEnd: baseOffset + chunkStart,
5641
+ rowsRead: chunkRows.length,
5642
+ rowsWritten: 0,
5643
+ rowsExecuted: rowsToExecute.length,
5644
+ rowsCached: 0,
5645
+ rowsDuplicateReused: duplicateInputReuseCount,
5646
+ rowsInserted,
5647
+ rowsSkipped,
5648
+ rowsFailed: failedExecutedRows,
5649
+ rowFailureSamples: buildRowFailureSamples(),
5650
+ stepCellsCompleted,
5651
+ stepCellsSkipped,
5652
+ outputDatasetId: `map:${name}`,
5653
+ hash: await hashJson([]),
5654
+ fatalError: `${fatalError}${breakerMessage}`,
5655
+ preview: [],
5656
+ };
5657
+ };
5658
+ const markReceiptPersistenceFatalRowsFailed = async (error: unknown) => {
5659
+ const message = formatWorkerRowFailureMessage(error);
5660
+ const rows = uniqueRowsToExecuteEntries
5661
+ .map((entry, executedIndex) => {
5662
+ const failure = fatalReceiptPersistenceRowEntries[executedIndex];
5663
+ const row =
5664
+ failure?.row ??
5665
+ executedRows[executedIndex] ??
5666
+ (runtimeCsvExecutionRow(
5667
+ entry.row,
5668
+ pendingRowsByKey.get(entry.rowKey),
5669
+ ) as T & Record<string, unknown>);
5670
+ return mapRowOutcomeRuntimeRow(
5671
+ failedMapRowOutcome({
5672
+ key: entry.rowKey,
5673
+ inputIndex: entry.absoluteIndex,
5674
+ data: row,
5675
+ cellMetaPatch: executedCellMetaPatches[executedIndex],
5676
+ error: failure?.error ?? message,
5677
+ }),
5678
+ );
5679
+ })
5680
+ .filter((row): row is Record<string, unknown> => row !== null);
5681
+ if (rows.length === 0) return;
5682
+ try {
5683
+ const failureStats = await persistCompletedMapRows({
5684
+ req,
5685
+ tableNamespace: name,
5686
+ outputFields,
5687
+ extraOutputFields: Array.from(generatedOutputFields),
5688
+ budgetMeter: workBudgetMeter,
5689
+ ...activeRowAttempt,
5690
+ forceFailedRows: true,
5691
+ rows,
5692
+ });
5693
+ emitEvent({
5694
+ type: 'log',
5695
+ level: 'warn',
5696
+ message:
5697
+ `Runtime sheet marked ${failureStats.rows} row(s) failed for ctx.dataset("${name}") ` +
5698
+ `after a run-fatal receipt persistence error.`,
5699
+ ts: nowMs(),
5700
+ });
5701
+ } catch (cleanupError) {
5702
+ emitEvent({
5703
+ type: 'log',
5704
+ level: 'warn',
5705
+ message:
5706
+ `Runtime sheet failed-row cleanup after receipt persistence error failed for ctx.dataset("${name}"): ` +
5707
+ formatWorkerRowFailureMessage(cleanupError),
5708
+ ts: nowMs(),
5709
+ });
5710
+ }
5711
+ };
6156
5712
  const persistRowsStartedAt = nowMs();
6157
5713
  recordRunnerPerfTrace({
6158
5714
  req,
@@ -6181,6 +5737,7 @@ function createMinimalWorkerCtx(
6181
5737
  },
6182
5738
  });
6183
5739
  } catch (error) {
5740
+ tripRuntimePersistenceLatch(persistenceLatch, error);
6184
5741
  recordRunnerPerfTrace({
6185
5742
  req,
6186
5743
  phase: 'runner.map_chunk.persist_rows_error',
@@ -6199,31 +5756,24 @@ function createMinimalWorkerCtx(
6199
5756
  result.status === 'rejected',
6200
5757
  );
6201
5758
  if (rejectedWorker) {
6202
- if (
6203
- isRuntimeReceiptPersistenceError(rejectedWorker.reason) ||
6204
- isRuntimePersistenceCircuitOpenError(rejectedWorker.reason)
6205
- ) {
6206
- // Persistence-failure / circuit-breaker rejections are returned as a
6207
- // fatal summary (NOT rethrown into the durable step) so the chunk is
6208
- // not retried and provider calls are not re-billed.
5759
+ if (isRuntimeReceiptPersistenceError(rejectedWorker.reason)) {
5760
+ await markReceiptPersistenceFatalRowsFailed(rejectedWorker.reason);
5761
+ return await fatalMapChunkSummary(rejectedWorker.reason);
5762
+ }
5763
+ if (isRuntimePersistenceCircuitOpenError(rejectedWorker.reason)) {
6209
5764
  return await fatalMapChunkSummary(rejectedWorker.reason);
6210
5765
  }
6211
5766
  throw rejectedWorker.reason;
6212
5767
  }
6213
5768
  const resultByKey = new Map<string, T & Record<string, unknown>>();
6214
- for (const completedRow of prepared.completedRows) {
6215
- const key =
6216
- resolveMapRowOutcomeKey(completedRow) ??
6217
- deriveDefaultMapRowIdentity(
6218
- completedRow,
6219
- resolveRuntimeInputIndex(completedRow) ?? 0,
6220
- );
6221
- if (key) {
6222
- const cleanedRow = stripMapRowOutcomeRuntimeFields(
6223
- publicCsvOutputRow(completedRow),
6224
- );
6225
- resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
6226
- }
5769
+ for (const entry of completedChunkEntries) {
5770
+ const completedRow = completedRowsByKey.get(entry.rowKey);
5771
+ if (!completedRow || resultByKey.has(entry.rowKey)) continue;
5772
+ resultByKey.set(
5773
+ entry.rowKey,
5774
+ cloneCsvAliasedRow(entry.row, completedRow) as T &
5775
+ Record<string, unknown>,
5776
+ );
6227
5777
  }
6228
5778
  for (
6229
5779
  let executedIndex = 0;
@@ -6258,16 +5808,13 @@ function createMinimalWorkerCtx(
6258
5808
  } => entry !== null,
6259
5809
  );
6260
5810
  const out = outEntries.map((entry) => entry.row);
6261
- const executedSuccessCount = Math.max(
6262
- 0,
6263
- executedRows.length - failedExecutedRows,
6264
- );
6265
5811
  const rowFailureSamples = buildRowFailureSamples();
6266
5812
  const publicOut = out.map((row) => publicCsvOutputRow(row));
6267
5813
  const keyedOut = outEntries.map(({ key, inputIndex, row }) => ({
6268
5814
  ...row,
6269
5815
  ...mapRowOutcomeRuntimeFields({ key, inputIndex }),
6270
5816
  }));
5817
+ const completedRowsReused = completedChunkEntries.length;
6271
5818
  const hashStartedAt = nowMs();
6272
5819
  const hash = await hashJson(publicOut);
6273
5820
  const includeCachedRowsInChunkResult = !workflowStep;
@@ -6298,17 +5845,18 @@ function createMinimalWorkerCtx(
6298
5845
  rowsWritten: out.length,
6299
5846
  rowsExecuted: executedRows.length,
6300
5847
  rowsFailed: failedExecutedRows,
6301
- rowsCached: Math.max(0, out.length - executedSuccessCount),
5848
+ rowsCached: completedRowsReused,
6302
5849
  },
6303
5850
  });
6304
5851
  return {
5852
+ status: 'completed',
6305
5853
  chunkIndex,
6306
5854
  rangeStart: baseOffset + chunkStart,
6307
5855
  rangeEnd: baseOffset + chunkStart + out.length,
6308
5856
  rowsRead: chunkRows.length,
6309
5857
  rowsWritten: out.length,
6310
5858
  rowsExecuted: executedRows.length,
6311
- rowsCached: Math.max(0, out.length - executedSuccessCount),
5859
+ rowsCached: completedRowsReused,
6312
5860
  rowsDuplicateReused: duplicateInputReuseCount,
6313
5861
  rowsInserted,
6314
5862
  rowsSkipped,
@@ -6340,9 +5888,12 @@ function createMinimalWorkerCtx(
6340
5888
  let totalRowsInserted = 0;
6341
5889
  let totalRowsSkipped = 0;
6342
5890
  let totalRowsFailed = 0;
5891
+ let totalRowsBlocked = 0;
6343
5892
  let totalStepCellsCompleted = 0;
6344
5893
  let totalStepCellsSkipped = 0;
6345
5894
  const totalRowFailureSamples: Array<{ rowKey: string; error: string }> = [];
5895
+ const blockedRowKeySamples: string[] = [];
5896
+ const blockedRowSamples: Record<string, unknown>[] = [];
6346
5897
 
6347
5898
  const runChunkStep = async (
6348
5899
  chunkRows: T[],
@@ -6372,6 +5923,7 @@ function createMinimalWorkerCtx(
6372
5923
  const readPersistedRows = async (input: {
6373
5924
  limit: number;
6374
5925
  offset: number;
5926
+ rowMode?: 'output' | 'all';
6375
5927
  }) => {
6376
5928
  const result = await harnessReadSheetDatasetRows({
6377
5929
  baseUrl: req.baseUrl,
@@ -6380,6 +5932,7 @@ function createMinimalWorkerCtx(
6380
5932
  playName: req.playName,
6381
5933
  tableNamespace: name,
6382
5934
  runId: req.runId,
5935
+ rowMode: input.rowMode,
6383
5936
  limit: input.limit,
6384
5937
  offset: input.offset,
6385
5938
  userEmail: req.userEmail,
@@ -6388,19 +5941,61 @@ function createMinimalWorkerCtx(
6388
5941
  return result.rows as Array<T & Record<string, unknown>>;
6389
5942
  };
6390
5943
 
6391
- const finalize = async (totalRowsWritten: number) => {
5944
+ const finalize = async () => {
5945
+ let finalizedRowsWritten = totalRowsWritten;
5946
+ if (
5947
+ totalRowsFailed === 0 &&
5948
+ rowCountHint !== null &&
5949
+ finalizedRowsWritten > 0 &&
5950
+ finalizedRowsWritten < rowCountHint &&
5951
+ rowCountHint <= WORKER_DATASET_IN_MEMORY_ROWS
5952
+ ) {
5953
+ const persistedRows = await readPersistedRows({
5954
+ limit: rowCountHint,
5955
+ offset: 0,
5956
+ rowMode: 'all',
5957
+ });
5958
+ if (persistedRows.length > finalizedRowsWritten) {
5959
+ const inMemoryRowsWritten = finalizedRowsWritten;
5960
+ finalizedRowsWritten = persistedRows.length;
5961
+ totalRowsWritten = finalizedRowsWritten;
5962
+ cachedRows.length = 0;
5963
+ cachedRows.push(...persistedRows);
5964
+ canCacheRows = true;
5965
+ previewRows.length = 0;
5966
+ previewRows.push(
5967
+ ...persistedRows.slice(0, WORKER_DATASET_PREVIEW_ROWS),
5968
+ );
5969
+ emitEvent({
5970
+ type: 'log',
5971
+ level: 'warn',
5972
+ message:
5973
+ `Runtime sheet finalization reconciled ctx.dataset("${name}") ` +
5974
+ `from ${inMemoryRowsWritten}/${rowCountHint} in-memory row(s) ` +
5975
+ `to ${finalizedRowsWritten} visible persisted row(s).`,
5976
+ ts: nowMs(),
5977
+ });
5978
+ }
5979
+ if (persistedRows.length < rowCountHint) {
5980
+ throw new Error(
5981
+ `Runtime sheet finalization mismatch ctx.dataset("${name}"): ` +
5982
+ `expected ${rowCountHint}, saw ${persistedRows.length}; ` +
5983
+ `sum ${finalizedRowsWritten}; ${req.runId}`,
5984
+ );
5985
+ }
5986
+ }
6392
5987
  const failureSampleSummary =
6393
5988
  totalRowFailureSamples.length > 0
6394
5989
  ? ` First error: ${totalRowFailureSamples[0]!.error}`
6395
5990
  : '';
6396
5991
  const cacheSummary =
6397
5992
  totalRowsFailed > 0
6398
- ? `Map completed with partial failures: ${totalRowsWritten} succeeded, ` +
6399
- `${totalRowsFailed} failed (${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
5993
+ ? `Map completed with partial failures: ${finalizedRowsWritten} succeeded, ` +
5994
+ `${totalRowsFailed} failed (${totalRowsExecuted} executed) ` +
6400
5995
  `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}. ` +
6401
5996
  `Failed rows are persisted with their errors and re-execute on the next run.${failureSampleSummary}`
6402
- : `Map completed: ${totalRowsWritten} results ` +
6403
- `(${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
5997
+ : `Map completed: ${finalizedRowsWritten} results ` +
5998
+ `(${totalRowsExecuted} executed) ` +
6404
5999
  `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
6405
6000
  const completedAt = nowMs();
6406
6001
  const totalStepCells = totalStepCellsCompleted + totalStepCellsSkipped;
@@ -6408,17 +6003,21 @@ function createMinimalWorkerCtx(
6408
6003
  totalStepCells > 0
6409
6004
  ? ` cells=${totalStepCells}/${totalStepCellsCompleted}/${totalStepCellsSkipped}`
6410
6005
  : '';
6006
+ await flushQueuedMapProgressUpdates();
6411
6007
  callbacks?.onMapCompleted?.(mapNodeId, completedAt);
6412
6008
  await updateMapProgress({
6413
- completed: totalRowsWritten,
6414
- total: totalRowsWritten + totalRowsFailed,
6009
+ completed: finalizedRowsWritten,
6010
+ total: finalizedRowsWritten + totalRowsFailed,
6415
6011
  failed: totalRowsFailed,
6416
6012
  completedAt,
6417
6013
  updatedAt: completedAt,
6418
6014
  message:
6419
6015
  totalRowsFailed > 0
6420
- ? `${totalRowsWritten.toLocaleString()} succeeded, ${totalRowsFailed.toLocaleString()} failed`
6421
- : formatMapProgressMessage(totalRowsWritten, totalRowsWritten),
6016
+ ? `${finalizedRowsWritten.toLocaleString()} succeeded, ${totalRowsFailed.toLocaleString()} failed`
6017
+ : formatMapProgressMessage(
6018
+ finalizedRowsWritten,
6019
+ finalizedRowsWritten,
6020
+ ),
6422
6021
  });
6423
6022
  emitEvent({
6424
6023
  type: 'log',
@@ -6427,29 +6026,61 @@ function createMinimalWorkerCtx(
6427
6026
  ts: nowMs(),
6428
6027
  });
6429
6028
  if (
6430
- totalRowsWritten > 0 &&
6029
+ finalizedRowsWritten > 0 &&
6431
6030
  totalRowsFailed === 0 &&
6432
6031
  canCacheRows &&
6433
- cachedRows.length === totalRowsWritten
6032
+ cachedRows.length === finalizedRowsWritten
6434
6033
  ) {
6435
6034
  const persistedRows = await readPersistedRows({
6436
- limit: totalRowsWritten,
6035
+ limit: finalizedRowsWritten,
6437
6036
  offset: 0,
6438
6037
  });
6439
- if (persistedRows.length < totalRowsWritten) {
6038
+ if (persistedRows.length < finalizedRowsWritten) {
6039
+ const repairAttempt = makeRuntimeSheetAttempt({
6040
+ runId: req.runId,
6041
+ runAttempt: req.runAttempt,
6042
+ mapName: name,
6043
+ chunkIndex: 'finalize',
6044
+ });
6045
+ const repairRows = cachedRows.map((row) => runtimeCsvStorageRow(row));
6046
+ workBudgetMeter.count('sheet');
6047
+ const repairStart = await harnessStartSheetDataset({
6048
+ ...runtimeSheetSessionScope(req),
6049
+ tableNamespace: name,
6050
+ sheetContract: augmentSheetContractWithDatasetFields({
6051
+ contract: requireSheetContract(req, name),
6052
+ rows: repairRows,
6053
+ outputFields,
6054
+ }),
6055
+ rows: repairRows,
6056
+ runId: req.runId,
6057
+ inputOffset: 0,
6058
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
6059
+ ...repairAttempt,
6060
+ });
6061
+ const activeRepairAttempt = {
6062
+ attemptId: repairStart.attemptId ?? repairAttempt.attemptId,
6063
+ attemptOwnerRunId:
6064
+ repairStart.attemptOwnerRunId ?? repairAttempt.attemptOwnerRunId,
6065
+ attemptExpiresAt:
6066
+ repairStart.attemptExpiresAt ?? repairAttempt.attemptExpiresAt,
6067
+ attemptSeq: repairStart.attemptSeq ?? repairAttempt.attemptSeq,
6068
+ };
6440
6069
  const repair = await persistCompletedMapRows({
6441
6070
  req,
6442
6071
  tableNamespace: name,
6443
6072
  outputFields,
6444
6073
  extraOutputFields: [],
6445
6074
  rows: cachedRows,
6075
+ ...activeRepairAttempt,
6076
+ budgetMeter: workBudgetMeter,
6446
6077
  });
6447
6078
  emitEvent({
6448
6079
  type: 'log',
6449
6080
  level: 'warn',
6450
6081
  message:
6451
6082
  `Runtime sheet finalization repaired ctx.dataset("${name}") ` +
6452
- `from ${persistedRows.length}/${totalRowsWritten} visible row(s) ` +
6083
+ `from ${persistedRows.length}/${finalizedRowsWritten} visible row(s) ` +
6453
6084
  `(written=${repair.written}, visible=${repair.visible}` +
6454
6085
  (repair.retryWritten === null
6455
6086
  ? ''
@@ -6465,7 +6096,7 @@ function createMinimalWorkerCtx(
6465
6096
  return createPersistedDatasetHandle({
6466
6097
  playName: req.playName,
6467
6098
  name,
6468
- count: totalRowsWritten,
6099
+ count: finalizedRowsWritten,
6469
6100
  // In native Workflows, chunk summaries intentionally omit full row
6470
6101
  // payloads, so this preview only contains the bounded chunk samples.
6471
6102
  // Do not synchronously page rows back here: service-binding reads have
@@ -6479,7 +6110,7 @@ function createMinimalWorkerCtx(
6479
6110
  recordRunnerPerfTrace({ req, phase, ms, extra }),
6480
6111
  nowMs,
6481
6112
  workProgress: {
6482
- total: totalRowsWritten + totalRowsFailed,
6113
+ total: finalizedRowsWritten + totalRowsFailed,
6483
6114
  executed: totalRowsExecuted,
6484
6115
  reused: totalRowsCached,
6485
6116
  skipped: totalRowsCached,
@@ -6493,97 +6124,248 @@ function createMinimalWorkerCtx(
6493
6124
  };
6494
6125
 
6495
6126
  const failFastRowErrors = opts?.onRowError === 'fail';
6496
- let chunkIndex = 0;
6497
- let chunkStart = 0;
6498
- for await (const rawChunkRows of iterDatasetChunks(
6499
- inputRows,
6500
- rowsPerChunk,
6501
- )) {
6502
- assertNotAborted(abortSignal);
6503
- if (rawChunkRows.length === 0) continue;
6504
- // Drop duplicate explicit-key rows before anything downstream observes
6505
- // them. `chunkStart` keeps advancing by the original (pre-dedupe) chunk
6506
- // length so cross-chunk key indices and persisted input offsets stay
6507
- // aligned to the original input stream.
6508
- const chunkRows = dedupeExplicitRowKeys(rawChunkRows, chunkStart);
6509
- if (chunkRows.length === 0) {
6510
- chunkStart += rawChunkRows.length;
6511
- continue;
6512
- }
6513
- const chunkResult = await runChunkStep(chunkRows, chunkStart, chunkIndex);
6514
- if (chunkResult.fatalError) {
6515
- const circuitBreakerSentence =
6516
- persistenceLatch.tripped && persistenceLatch.preventedCallCount > 0
6517
- ? `Circuit breaker prevented ${persistenceLatch.preventedCallCount} queued provider call(s) ` +
6518
- `from dispatching after the first persistence failure. `
6519
- : '';
6520
- throw new Error(
6521
- `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
6522
- `outside the retryable chunk step. Provider calls already executed for ` +
6523
- `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
6524
- circuitBreakerSentence +
6525
- `Fix the runtime persistence cause and re-run to resume. ` +
6526
- `First error: ${chunkResult.fatalError}`,
6527
- );
6528
- }
6529
- totalRowsWritten += chunkResult.rowsWritten;
6530
- totalRowsExecuted += chunkResult.rowsExecuted;
6531
- totalRowsCached += chunkResult.rowsCached;
6532
- totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
6533
- totalRowsInserted += chunkResult.rowsInserted;
6534
- totalRowsSkipped += chunkResult.rowsSkipped;
6535
- totalRowsFailed += chunkResult.rowsFailed ?? 0;
6536
- totalStepCellsCompleted += chunkResult.stepCellsCompleted ?? 0;
6537
- totalStepCellsSkipped += chunkResult.stepCellsSkipped ?? 0;
6538
- for (const sample of chunkResult.rowFailureSamples ?? []) {
6539
- if (totalRowFailureSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6540
- break;
6127
+ // Bounded-concurrent chunk dispatch is only sound off the native Workflows
6128
+ // path: with a `workflowStep` each chunk is a durable step whose sequential
6129
+ // order makes replay/resume deterministic, so that path stays at K=1.
6130
+ // Chunk-level concurrency is otherwise safe under workers_edge because each
6131
+ // chunk owns a disjoint sheet row range with its own runtime-sheet attempt,
6132
+ // receipts complete inside each chunk's persist barrier, and child submits
6133
+ // use per-row durable receipt keys. It is disabled under onRowError:'fail'
6134
+ // on purpose: that mode relies on a first-failure short-circuit to skip the
6135
+ // remaining chunks and stop spending provider credits, and eager concurrent
6136
+ // launch would already have K-1 chunks in flight (credits spent) before the
6137
+ // short-circuit fires.
6138
+ const canRunChunksConcurrently = !workflowStep && !failFastRowErrors;
6139
+ const maxConcurrentMapChunks = canRunChunksConcurrently
6140
+ ? WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS
6141
+ : 1;
6142
+ const runWorkDispatcher = new WorkerRunWorkDispatcher({
6143
+ budgetMeter: workBudgetMeter,
6144
+ nowMs,
6145
+ yieldLimits: {
6146
+ elapsed: governor.policy.pacing.workerYieldElapsedMs,
6147
+ ...(req.testPolicyOverrides?.workBudgetYieldLimits ?? {}),
6148
+ },
6149
+ recordTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
6150
+ yieldBetweenChunks: async (yieldPoint) => {
6151
+ emitEvent({
6152
+ type: 'log',
6153
+ level: 'info',
6154
+ message:
6155
+ `Runtime dispatcher yielded ctx.dataset("${name}") before chunk ${yieldPoint.nextChunkIndex} ` +
6156
+ `(${yieldPoint.reason} budget).`,
6157
+ ts: nowMs(),
6158
+ });
6159
+ if (workflowStep) {
6160
+ // Cloudflare Workflows durability model: `step.sleep` writes a
6161
+ // durable checkpoint, and the resume re-enters `run()` as a NEW
6162
+ // Worker invocation. Cloudflare's subrequest ceiling is
6163
+ // per-invocation, and on resume the already-completed `step.do`
6164
+ // chunks are served from the durable step cache WITHOUT re-issuing
6165
+ // their subrequests (replay semantics) — so the post-yield chunk
6166
+ // runs against a fresh per-invocation subrequest budget.
6167
+ //
6168
+ // Isolate identity is the WRONG freshness proxy: Cloudflare routinely
6169
+ // reuses warm isolates across invocations, so a genuinely
6170
+ // fresh-budget resume commonly shares the pre-sleep isolate id (this
6171
+ // is exactly why the isolate-identity check reported `false` every
6172
+ // time and hard-failed multi-chunk maps). Crossing the durable
6173
+ // `step.sleep` boundary IS the fresh-budget signal.
6174
+ const sleepMs = WORK_DISPATCHER_YIELD_SLEEP_MS;
6175
+ await (
6176
+ workflowStep.sleep as unknown as (
6177
+ name: string,
6178
+ duration: number,
6179
+ ) => Promise<void>
6180
+ )(
6181
+ `work-dispatcher:${name}:${yieldPoint.nextChunkIndex}:${yieldPoint.attempt}`,
6182
+ sleepMs,
6183
+ );
6184
+ recordRunnerPerfTrace({
6185
+ req,
6186
+ phase: 'runner.work_dispatcher.yield_probe',
6187
+ ms: sleepMs,
6188
+ extra: {
6189
+ mapName: name,
6190
+ nextChunkIndex: yieldPoint.nextChunkIndex,
6191
+ reason: yieldPoint.reason,
6192
+ attempt: yieldPoint.attempt,
6193
+ freshBudget: true,
6194
+ invocationBoundary: 'workflow_step_sleep',
6195
+ },
6196
+ });
6197
+ return { freshBudget: true };
6541
6198
  }
6542
- totalRowFailureSamples.push(sample);
6543
- }
6544
- await updateMapProgress({
6545
- completed: totalRowsWritten,
6546
- total: rowCountHint ?? undefined,
6547
- ...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
6548
- message: formatMapProgressMessage(
6549
- totalRowsWritten,
6550
- rowCountHint ?? undefined,
6551
- ),
6552
- });
6553
- if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
6554
- previewRows.push(
6555
- ...chunkResult.preview.slice(
6199
+ // No durable Workflow step is available to force a new invocation here,
6200
+ // so we cannot obtain a fresh per-invocation platform budget. Report a
6201
+ // non-fresh yield: the dispatcher escalates and then fails loudly with
6202
+ // WorkerBudgetExhaustedError rather than silently overflowing the
6203
+ // subrequest ceiling mid-map.
6204
+ await sleepWorkerMs(0);
6205
+ return { freshBudget: false };
6206
+ },
6207
+ });
6208
+ let dispatchResult: WorkerRunMapBatchesResult;
6209
+ try {
6210
+ dispatchResult = await runWorkDispatcher.runMapBatches({
6211
+ mapName: name,
6212
+ chunks: iterDatasetChunks(inputRows, rowsPerChunk),
6213
+ maxConcurrentChunks: maxConcurrentMapChunks,
6214
+ maxResidentBatchBytes: WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES,
6215
+ prepareBatch: (rawChunkRows, chunkStart) => {
6216
+ assertNotAborted(abortSignal);
6217
+ if (rawChunkRows.length === 0) return [];
6218
+ // Drop duplicate explicit-key rows before anything downstream observes
6219
+ // them. `chunkStart` keeps advancing by the original (pre-dedupe) chunk
6220
+ // length so cross-chunk key indices and persisted input offsets stay
6221
+ // aligned to the original input stream.
6222
+ return dedupeExplicitRowKeys(rawChunkRows, chunkStart);
6223
+ },
6224
+ estimateRawBatchResidentBytes: (rawChunkRows) =>
6225
+ rawChunkRows.reduce(
6226
+ (total, row) => total + workerMapResidentRowBytes(row),
6227
+ 0,
6228
+ ),
6229
+ executeBatch: async ({ rows: chunkRows, chunkStart, chunkIndex }) => {
6230
+ assertNotAborted(abortSignal);
6231
+ return await runChunkStep(chunkRows, chunkStart, chunkIndex);
6232
+ },
6233
+ estimateBatchResidentBytes: ({ rows: chunkRows }) =>
6234
+ chunkRows.reduce(
6235
+ (total, row) => total + workerMapResidentRowBytes(row),
6556
6236
  0,
6557
- WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
6558
6237
  ),
6238
+ estimateBatchSubrequests: ({ rows: chunkRows }) => {
6239
+ const estimate = mapDispatchPlan.workEstimate;
6240
+ const unbatchedProviderSubrequests =
6241
+ estimate.unbatchedProviderToolCallsPerRow *
6242
+ WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL;
6243
+ // Static planning cannot know input-dependent batching buckets, so
6244
+ // this projects worst-case provider-batch fanout while still treating
6245
+ // durable receipt claim/complete/fail as bulk per drained tool group.
6246
+ const batchedProviderSubrequests =
6247
+ estimateBatchedToolChunkSubrequests({
6248
+ batchableToolCallsPerRow: Math.max(
6249
+ 0,
6250
+ estimate.providerToolCallsPerRow -
6251
+ estimate.unbatchedProviderToolCallsPerRow,
6252
+ ),
6253
+ coalescedBatchableToolCallsPerRow:
6254
+ estimate.coalescedBatchableProviderToolCallsPerRow,
6255
+ rows: chunkRows.length,
6256
+ assumedBatchSize: estimate.assumedBatchSize,
6257
+ });
6258
+ const controlSubrequests =
6259
+ estimate.childPlaySubmitsPerRow + estimate.eventWaitsPerRow;
6260
+ return (
6261
+ chunkRows.length *
6262
+ (unbatchedProviderSubrequests + controlSubrequests) +
6263
+ batchedProviderSubrequests +
6264
+ estimate.runtimeSheetReadsPerChunk +
6265
+ estimate.runtimeSheetWritesPerChunk
6266
+ );
6267
+ },
6268
+ onBatchComplete: async (chunkResult) => {
6269
+ if (chunkResult.status === 'blocked') {
6270
+ totalRowsBlocked += chunkResult.rowsBlocked;
6271
+ for (const key of chunkResult.rowKeySamples) {
6272
+ if (blockedRowKeySamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6273
+ break;
6274
+ }
6275
+ blockedRowKeySamples.push(key);
6276
+ }
6277
+ for (const row of chunkResult.blockedRowSamples) {
6278
+ if (blockedRowSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6279
+ break;
6280
+ }
6281
+ blockedRowSamples.push(row);
6282
+ }
6283
+ emitEvent({
6284
+ type: 'log',
6285
+ level: 'warn',
6286
+ message: chunkResult.message,
6287
+ ts: nowMs(),
6288
+ });
6289
+ return true;
6290
+ }
6291
+ if (chunkResult.fatalError) {
6292
+ throw new Error(
6293
+ `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
6294
+ `outside the retryable chunk step. Provider calls already executed for ` +
6295
+ `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
6296
+ `Fix the runtime persistence cause and re-run to resume. ` +
6297
+ `First error: ${chunkResult.fatalError}`,
6298
+ );
6299
+ }
6300
+ totalRowsWritten += chunkResult.rowsWritten;
6301
+ totalRowsExecuted += chunkResult.rowsExecuted;
6302
+ totalRowsCached += chunkResult.rowsCached;
6303
+ totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
6304
+ totalRowsInserted += chunkResult.rowsInserted;
6305
+ totalRowsSkipped += chunkResult.rowsSkipped;
6306
+ totalRowsFailed += chunkResult.rowsFailed ?? 0;
6307
+ totalStepCellsCompleted += chunkResult.stepCellsCompleted ?? 0;
6308
+ totalStepCellsSkipped += chunkResult.stepCellsSkipped ?? 0;
6309
+ for (const sample of chunkResult.rowFailureSamples ?? []) {
6310
+ if (totalRowFailureSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6311
+ break;
6312
+ }
6313
+ totalRowFailureSamples.push(sample);
6314
+ }
6315
+ enqueueMapProgressUpdate({
6316
+ completed: totalRowsWritten,
6317
+ total: rowCountHint ?? undefined,
6318
+ ...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
6319
+ message: formatMapProgressMessage(
6320
+ totalRowsWritten,
6321
+ rowCountHint ?? undefined,
6322
+ ),
6323
+ });
6324
+ if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
6325
+ previewRows.push(
6326
+ ...chunkResult.preview.slice(
6327
+ 0,
6328
+ WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
6329
+ ),
6330
+ );
6331
+ }
6332
+ if (canCacheRows) {
6333
+ const volatileRows = volatileWorkflowChunkRows.get(
6334
+ chunkResult.chunkIndex,
6335
+ );
6336
+ volatileWorkflowChunkRows.delete(chunkResult.chunkIndex);
6337
+ const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
6338
+ if (
6339
+ nextRows.length === chunkResult.rowsWritten &&
6340
+ cachedRows.length + nextRows.length <=
6341
+ WORKER_DATASET_IN_MEMORY_ROWS
6342
+ ) {
6343
+ cachedRows.push(...nextRows);
6344
+ } else {
6345
+ cachedRows.length = 0;
6346
+ volatileWorkflowChunkRows.clear();
6347
+ canCacheRows = false;
6348
+ }
6349
+ }
6350
+ // onRowError:'fail' short-circuit: once a chunk reports a row failure,
6351
+ // skip the remaining chunks entirely. The failing chunk itself completed
6352
+ // normally (no chunk-step retry storm) and persisted its rows, but
6353
+ // executing later chunks would keep spending provider credits on a run
6354
+ // the caller asked to fail fast. The post-loop fail-fast throw below
6355
+ // reports what committed before the stop.
6356
+ return failFastRowErrors && totalRowsFailed > 0;
6357
+ },
6358
+ });
6359
+ } catch (error) {
6360
+ try {
6361
+ await flushQueuedMapProgressUpdates();
6362
+ } catch (progressError) {
6363
+ throw new AggregateError(
6364
+ [error, progressError],
6365
+ `ctx.dataset("${name}") failed while flushing queued progress updates.`,
6559
6366
  );
6560
6367
  }
6561
- if (canCacheRows) {
6562
- const volatileRows = volatileWorkflowChunkRows.get(chunkIndex);
6563
- volatileWorkflowChunkRows.delete(chunkIndex);
6564
- const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
6565
- if (
6566
- nextRows.length === chunkResult.rowsWritten &&
6567
- cachedRows.length + nextRows.length <= WORKER_DATASET_IN_MEMORY_ROWS
6568
- ) {
6569
- cachedRows.push(...nextRows);
6570
- } else {
6571
- cachedRows.length = 0;
6572
- volatileWorkflowChunkRows.clear();
6573
- canCacheRows = false;
6574
- }
6575
- }
6576
- chunkStart += rawChunkRows.length;
6577
- chunkIndex += 1;
6578
- // onRowError:'fail' short-circuit: once a chunk reports a row failure,
6579
- // skip the remaining chunks entirely. The failing chunk itself completed
6580
- // normally (no chunk-step retry storm) and persisted its rows, but
6581
- // executing later chunks would keep spending provider credits on a run
6582
- // the caller asked to fail fast. The post-loop fail-fast throw below
6583
- // reports what committed before the stop.
6584
- if (failFastRowErrors && totalRowsFailed > 0) {
6585
- break;
6586
- }
6368
+ throw error;
6587
6369
  }
6588
6370
  if (totalDuplicateKeysDropped > 0) {
6589
6371
  const keySample = droppedDuplicateKeySamples.join(', ');
@@ -6596,6 +6378,13 @@ function createMinimalWorkerCtx(
6596
6378
  ts: nowMs(),
6597
6379
  });
6598
6380
  }
6381
+ if (totalRowsBlocked > 0) {
6382
+ await flushQueuedMapProgressUpdates();
6383
+ throw new RuntimeSheetRowsBlockedError({
6384
+ tableNamespace: name,
6385
+ blockedRows: blockedRowSamples,
6386
+ });
6387
+ }
6599
6388
  if (failFastRowErrors && totalRowsFailed > 0 && totalRowsWritten > 0) {
6600
6389
  // onRowError:'fail', PARTIAL failure (some rows committed): fail the run
6601
6390
  // without finalizing the dataset. The committed rows already persisted
@@ -6604,6 +6393,7 @@ function createMinimalWorkerCtx(
6604
6393
  // normally (no per-row throw inside the durable chunk step, so no
6605
6394
  // chunk-step retry storm); later chunks were skipped by the fail-fast
6606
6395
  // short-circuit in the chunk loop.
6396
+ await flushQueuedMapProgressUpdates();
6607
6397
  const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
6608
6398
  throw new Error(
6609
6399
  `ctx.dataset("${name}") failed for ${totalRowsFailed} executed row(s) under onRowError:'fail'. ` +
@@ -6621,7 +6411,7 @@ function createMinimalWorkerCtx(
6621
6411
  // step when no row otherwise succeeded) are summarized and registered as
6622
6412
  // a recovered dataset — the failed run then advertises a WORKING export
6623
6413
  // instead of a dead end (#15/#27). The run still fails (the throw below).
6624
- await finalize(totalRowsWritten);
6414
+ await finalize();
6625
6415
  const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
6626
6416
  throw new Error(
6627
6417
  `ctx.dataset("${name}") failed for all ${totalRowsFailed} executed rows. ` +
@@ -6629,7 +6419,7 @@ function createMinimalWorkerCtx(
6629
6419
  `(rows are persisted with per-row errors; fix the cause and re-run to resume)`,
6630
6420
  );
6631
6421
  }
6632
- const dataset = await finalize(totalRowsWritten);
6422
+ const dataset = await finalize();
6633
6423
  recordRunnerPerfTrace({
6634
6424
  req,
6635
6425
  phase: 'runner.map.total',
@@ -6638,7 +6428,7 @@ function createMinimalWorkerCtx(
6638
6428
  mapName: name,
6639
6429
  rowsWritten: totalRowsWritten,
6640
6430
  inputKind: rowCountHint === null ? 'streaming' : 'known_count',
6641
- chunks: chunkIndex,
6431
+ chunks: dispatchResult.chunksExecuted,
6642
6432
  },
6643
6433
  });
6644
6434
  return dataset;
@@ -6903,10 +6693,10 @@ function createMinimalWorkerCtx(
6903
6693
  request.input,
6904
6694
  workflowStep,
6905
6695
  {
6906
- force: request.force === true || !!req.force,
6696
+ force: request.force === true,
6697
+ runForceRefresh: req.forceToolRefresh === true,
6698
+ runForceFailedRefresh: req.force === true,
6907
6699
  staleAfterSeconds: request.staleAfterSeconds,
6908
- },
6909
- {
6910
6700
  timeoutMs: request.timeoutMs,
6911
6701
  receiptWaitMs: request.receiptWaitMs,
6912
6702
  },
@@ -6947,13 +6737,18 @@ function createMinimalWorkerCtx(
6947
6737
  if (!resolvedName) {
6948
6738
  throw new Error('ctx.runPlay(...) requires a resolvable play name.');
6949
6739
  }
6740
+ const childManifest = req.childPlayManifests?.[resolvedName];
6950
6741
  const receiptKey = buildDurableCtxCallCacheKey({
6951
6742
  orgId: req.orgId,
6952
6743
  playId: req.playName,
6953
6744
  kind: 'runPlay',
6954
6745
  id: normalizedKey,
6955
- semanticKey: await hashJson({
6746
+ semanticKey: buildDurableRunPlaySemanticKey({
6956
6747
  childPlayName: resolvedName,
6748
+ childRevisionFingerprint: workerChildRevisionFingerprint({
6749
+ playName: resolvedName,
6750
+ manifest: childManifest,
6751
+ }),
6957
6752
  input,
6958
6753
  }),
6959
6754
  staleAfterSeconds: options?.staleAfterSeconds,
@@ -6981,7 +6776,6 @@ function createMinimalWorkerCtx(
6981
6776
  message: `Starting child play ${resolvedName} (${normalizedKey})`,
6982
6777
  ts: nowMs(),
6983
6778
  });
6984
- const childManifest = req.childPlayManifests?.[resolvedName];
6985
6779
  if (!childManifest) {
6986
6780
  throw new Error(
6987
6781
  `ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
@@ -7019,7 +6813,7 @@ function createMinimalWorkerCtx(
7019
6813
  });
7020
6814
  let childPlaySlot: { release(): void } | null = null;
7021
6815
  try {
7022
- childPlaySlot = await governor.acquireChildPlaySlot({
6816
+ childPlaySlot = await governor.acquireChildSubmitSlot({
7023
6817
  signal: abortSignal,
7024
6818
  });
7025
6819
  const childSubmitStartedAt = nowMs();
@@ -7044,12 +6838,12 @@ function createMinimalWorkerCtx(
7044
6838
  options?.timeoutMs == null && !childNeedsWorkflowScheduler,
7045
6839
  body: {
7046
6840
  name: resolvedName,
6841
+ childIdempotencyKey: receiptKey,
7047
6842
  input: isRecord(input) ? input : {},
7048
6843
  orgId: req.orgId,
7049
6844
  callbackBaseUrl: req.callbackUrl,
7050
6845
  baseUrl: req.baseUrl,
7051
6846
  integrationMode: req.integrationMode ?? null,
7052
- forceToolRefresh: req.force === true,
7053
6847
  parentExecutorToken: req.executorToken,
7054
6848
  userEmail: req.userEmail ?? '',
7055
6849
  profile: 'workers_edge',
@@ -7161,6 +6955,8 @@ function createMinimalWorkerCtx(
7161
6955
  childNeedsWorkflowScheduler,
7162
6956
  },
7163
6957
  });
6958
+ childPlaySlot?.release();
6959
+ childPlaySlot = null;
7164
6960
  const startedStatus = String(started.status ?? '').toLowerCase();
7165
6961
  if (startedStatus === 'completed') {
7166
6962
  emitEvent({
@@ -7189,6 +6985,10 @@ function createMinimalWorkerCtx(
7189
6985
  }
7190
6986
  const childWaitStartedAt = nowMs();
7191
6987
  let waitResult: ChildPlayTerminalWaitResult;
6988
+ const readCachedChildTerminalState =
6989
+ cachedCoordinatorBinding?.readChildTerminalState?.bind(
6990
+ cachedCoordinatorBinding,
6991
+ );
7192
6992
  try {
7193
6993
  waitResult = await awaitChildTerminal({
7194
6994
  parentRunId: req.runId,
@@ -7205,20 +7005,31 @@ function createMinimalWorkerCtx(
7205
7005
  1_000,
7206
7006
  Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
7207
7007
  ),
7208
- coordinator: cachedCoordinatorBinding?.readChildTerminalState
7209
- ? {
7210
- readChildTerminalState: (
7211
- parentRunId,
7212
- eventKey,
7213
- timeoutMs,
7214
- ) =>
7215
- cachedCoordinatorBinding!.readChildTerminalState!(
7216
- parentRunId,
7217
- eventKey,
7218
- timeoutMs,
7219
- ),
7220
- }
7221
- : null,
7008
+ coordinator: {
7009
+ readRunTerminalState: (runId, timeoutMs) =>
7010
+ readDurableChildRunTerminalState({
7011
+ baseUrl: req.baseUrl,
7012
+ executorToken: req.executorToken,
7013
+ parentRunId: req.runId,
7014
+ childRunId: runId,
7015
+ childPlayName: resolvedName,
7016
+ timeoutMs,
7017
+ }),
7018
+ ...(readCachedChildTerminalState
7019
+ ? {
7020
+ readChildTerminalState: (
7021
+ parentRunId: string,
7022
+ eventKey: string,
7023
+ timeoutMs?: number,
7024
+ ) =>
7025
+ readCachedChildTerminalState(
7026
+ parentRunId,
7027
+ eventKey,
7028
+ timeoutMs,
7029
+ ),
7030
+ }
7031
+ : {}),
7032
+ },
7222
7033
  now: nowMs,
7223
7034
  hashJson,
7224
7035
  });
@@ -7401,6 +7212,7 @@ function createMinimalWorkerCtx(
7401
7212
  // Direct Worker fetch below is still limited to runtime origins and
7402
7213
  // IP literals; public IP literals also consume this slot because
7403
7214
  // they are customer egress.
7215
+ workBudgetMeter.count('egress');
7404
7216
  const egressResponse = await postRuntimeEgressFetch(
7405
7217
  req,
7406
7218
  {
@@ -7423,6 +7235,7 @@ function createMinimalWorkerCtx(
7423
7235
  }
7424
7236
  const fetchInit = { ...init, headers };
7425
7237
  delete fetchInit.auth;
7238
+ workBudgetMeter.count('egress');
7426
7239
  const response = await safeWorkerPublicFetch(url, fetchInit, {
7427
7240
  allowedOrigins,
7428
7241
  sensitiveHeaders: Object.keys(secretHeaderMarkers),
@@ -7493,6 +7306,7 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
7493
7306
  let req: RunRequest;
7494
7307
  try {
7495
7308
  req = (await request.json()) as RunRequest;
7309
+ req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
7496
7310
  } catch {
7497
7311
  return new Response('invalid JSON body', { status: 400 });
7498
7312
  }
@@ -7557,6 +7371,7 @@ async function handleRunInline(
7557
7371
  let req: RunRequest;
7558
7372
  try {
7559
7373
  req = (await request.json()) as RunRequest;
7374
+ req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
7560
7375
  } catch {
7561
7376
  return Response.json(
7562
7377
  {
@@ -7704,6 +7519,7 @@ async function executeRunRequest(
7704
7519
  ): Promise<WorkflowRunOutput> {
7705
7520
  installProcessExitTrap();
7706
7521
  const startedAt = nowMs();
7522
+ const workBudgetMeter = createWorkerWorkBudgetMeter({ nowMs });
7707
7523
  recordRunnerPerfTrace({
7708
7524
  req,
7709
7525
  phase: 'runner.execute_start',
@@ -7741,6 +7557,7 @@ async function executeRunRequest(
7741
7557
  const appendRunLogLine = (line: string) => {
7742
7558
  const trimmed = redactSecretsFromLogString(line.trim());
7743
7559
  if (!trimmed) return;
7560
+ workBudgetMeter.count('log');
7744
7561
  totalEmittedLogLines += 1;
7745
7562
  runLogBuffer = [...runLogBuffer, trimmed].slice(-RUN_LOG_BUFFER_LIMIT);
7746
7563
  pendingRunLogLines = [...pendingRunLogLines, trimmed];
@@ -7933,7 +7750,10 @@ async function executeRunRequest(
7933
7750
  const flushTerminalLedgerEvents = async (
7934
7751
  terminalEvent: PlayRunLedgerEvent,
7935
7752
  ): Promise<void> => {
7936
- if (!options?.persistResultDatasets) return;
7753
+ const shouldPersistTerminalLedger =
7754
+ options?.persistResultDatasets === true ||
7755
+ Boolean(req.playCallGovernance);
7756
+ if (!shouldPersistTerminalLedger) return;
7937
7757
  await ledgerFlushInFlight;
7938
7758
  const now = nowMs();
7939
7759
  dirtyProgressNodeIds = new Set([
@@ -8008,10 +7828,7 @@ async function executeRunRequest(
8008
7828
  stepLifecycle?.markPreDatasetStepsStarted(startedAt);
8009
7829
  flushLedgerEvents(false);
8010
7830
  const runtimeDeadlineMs = nowMs() + STANDARD_PLAY_RUNTIME_LIMIT_MS;
8011
- // Run-scoped buffer for provider results that billed but whose durable
8012
- // receipt could not be marked completed even after the retry ladder. On
8013
- // terminal failure the buffer is built into a size-capped salvage record and
8014
- // attached to the run.failed ledger event so billed data is never memory-only.
7831
+ const leasedSheetNamespaces = new Set<string>();
8015
7832
  const receiptSalvage = createReceiptSalvageBuffer();
8016
7833
  const ctx = createMinimalWorkerCtx(
8017
7834
  req,
@@ -8020,7 +7837,9 @@ async function executeRunRequest(
8020
7837
  workflowStep,
8021
7838
  abortSignal,
8022
7839
  workerCallbacks,
7840
+ workBudgetMeter,
8023
7841
  runtimeDeadlineMs,
7842
+ leasedSheetNamespaces,
8024
7843
  receiptSalvage,
8025
7844
  );
8026
7845
  // Hard wall-clock cap on active user-code runtime. CF Workflows does not
@@ -8029,15 +7848,18 @@ async function executeRunRequest(
8029
7848
  // token expires. Aborting the controller surfaces cooperatively through the
8030
7849
  // same assertNotAborted checks used for harness cancellation.
8031
7850
  let runtimeLimitExceeded = false;
8032
- const runtimeDeadlineTimer = setTimeout(
8033
- () => {
8034
- runtimeLimitExceeded = true;
8035
- if (!abortSignal.aborted) {
8036
- abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
8037
- }
8038
- },
8039
- Math.max(1, runtimeDeadlineMs - nowMs()),
8040
- );
7851
+ const runtimeDeadlineTimer =
7852
+ workflowStep === undefined || workflowStep === null
7853
+ ? setTimeout(
7854
+ () => {
7855
+ runtimeLimitExceeded = true;
7856
+ if (!abortSignal.aborted) {
7857
+ abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
7858
+ }
7859
+ },
7860
+ Math.max(1, runtimeDeadlineMs - nowMs()),
7861
+ )
7862
+ : null;
8041
7863
  try {
8042
7864
  const playStartedAt = nowMs();
8043
7865
  const result = await (
@@ -8128,7 +7950,6 @@ async function executeRunRequest(
8128
7950
  }
8129
7951
  if (options?.persistResultDatasets) {
8130
7952
  await persistProjectedResultDatasets();
8131
- const parentSignal = startParentTerminalSignal();
8132
7953
  // Capped runs settle compute billing BEFORE declaring run.completed: a
8133
7954
  // per-run cap denial (422 billing_cap_exceeded) must fail the run as
8134
7955
  // its ONLY terminal. Flushing completed first opens a race — watchers
@@ -8163,6 +7984,7 @@ async function executeRunRequest(
8163
7984
  ms: nowMs() - terminalUpdateStartedAt,
8164
7985
  });
8165
7986
 
7987
+ const parentSignal = startParentTerminalSignal();
8166
7988
  if (!capped) {
8167
7989
  const billingStartedAt = nowMs();
8168
7990
  const billingPromise = finalizeWorkerComputeBilling({
@@ -8190,6 +8012,20 @@ async function executeRunRequest(
8190
8012
  }
8191
8013
  }
8192
8014
  await parentSignal;
8015
+ } else if (req.playCallGovernance) {
8016
+ const childTerminalStartedAt = nowMs();
8017
+ await flushTerminalLedgerEvents({
8018
+ type: 'run.completed',
8019
+ runId: req.runId,
8020
+ source: 'worker',
8021
+ occurredAt: nowMs(),
8022
+ result: terminalResult,
8023
+ });
8024
+ recordRunnerPerfTrace({
8025
+ req,
8026
+ phase: 'runner.child_terminal_ledger_append',
8027
+ ms: nowMs() - childTerminalStartedAt,
8028
+ });
8193
8029
  }
8194
8030
  await startParentTerminalSignal();
8195
8031
  recordRunnerPerfTrace({
@@ -8216,16 +8052,8 @@ async function executeRunRequest(
8216
8052
  stepLifecycle?.markStartedFailed(nowMs());
8217
8053
  // A runtime-limit abort is a timeout failure, not a user cancellation, so
8218
8054
  // it should be reported as run.failed with the limit message rather than
8219
- // run.cancelled. The deadline can also surface as a WorkflowAbortError
8220
- // thrown synchronously from the completeReceipt retry ladder BEFORE the
8221
- // runtimeDeadlineTimer macrotask flips the flag — match the limit message
8222
- // directly so that race can never misclassify a timeout as a cancel (which
8223
- // would silently drop the salvage buffer of billed-but-unpersisted rows).
8224
- const runtimeLimitError =
8225
- runtimeLimitExceeded ||
8226
- (error instanceof Error &&
8227
- error.message === STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
8228
- const aborted = isAbortLikeError(error) && !runtimeLimitError;
8055
+ // run.cancelled.
8056
+ const aborted = isAbortLikeError(error) && !runtimeLimitExceeded;
8229
8057
  if (aborted) {
8230
8058
  // Flip the controller so any concurrent user code observes the abort
8231
8059
  // through ctx.signal. We mark the run cancelled instead of failed.
@@ -8234,19 +8062,28 @@ async function executeRunRequest(
8234
8062
  );
8235
8063
  }
8236
8064
  const failure = normalizePlayRunFailure(error);
8237
- // Billed-but-unpersisted salvage rides on the run.failed event's `result`.
8238
- // The run STILL FAILS salvage is a loud marker, never a success fallback.
8065
+ const message = failure.message;
8066
+ // Controlled run-fatal teardown: release the runtime-sheet row leases and
8067
+ // work-receipt leases this attempt still holds so an immediate rerun is not
8068
+ // blocked for the full lease TTL. Best-effort but LOUD — a release failure
8069
+ // is logged and never masks the original run-fatal error. TTL expiry stays
8070
+ // the recovery path for true crashes (isolate death) where this never runs.
8071
+ await releaseRuntimeLeasesOnTeardown({
8072
+ req,
8073
+ leasedSheetNamespaces,
8074
+ emit: wrappedEmit,
8075
+ });
8076
+ const errorBilling = extractErrorBilling(error);
8239
8077
  const salvage: ReceiptSalvage | null = aborted
8240
8078
  ? null
8241
8079
  : receiptSalvage.build();
8242
- const message =
8080
+ const terminalMessage =
8243
8081
  salvage && salvage.totalEntries > 0
8244
- ? `${failure.message} ${receiptSalvageFailureSuffix(salvage)}`
8245
- : failure.message;
8246
- const errorBilling = extractErrorBilling(error);
8247
- if (options?.persistResultDatasets) {
8082
+ ? `${message} ${receiptSalvageFailureSuffix(salvage)}`
8083
+ : message;
8084
+ if (options?.persistResultDatasets || req.playCallGovernance) {
8248
8085
  appendRunLogLine(
8249
- `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(message)}`,
8086
+ `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(terminalMessage)}`,
8250
8087
  );
8251
8088
  const terminalUpdateStartedAt = nowMs();
8252
8089
  await flushTerminalLedgerEvents({
@@ -8254,64 +8091,70 @@ async function executeRunRequest(
8254
8091
  runId: req.runId,
8255
8092
  source: 'worker',
8256
8093
  occurredAt: nowMs(),
8257
- error: message,
8094
+ error: terminalMessage,
8258
8095
  result: aborted
8259
8096
  ? undefined
8260
8097
  : {
8261
8098
  success: false,
8262
8099
  status: 'failed',
8263
- error: message,
8100
+ error: terminalMessage,
8101
+ ...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
8264
8102
  errors: [
8265
8103
  {
8266
8104
  code: failure.code,
8267
8105
  phase: failure.phase,
8268
- message,
8106
+ message: terminalMessage,
8269
8107
  retryable: failure.retryable,
8270
8108
  ...(errorBilling ? { billing: errorBilling } : {}),
8271
8109
  ...(failure.cause ? { cause: failure.cause } : {}),
8272
8110
  },
8273
8111
  ],
8274
- ...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
8275
8112
  },
8276
8113
  });
8277
8114
  recordRunnerPerfTrace({
8278
8115
  req,
8279
- phase: aborted
8280
- ? 'runner.terminal_ledger_append_cancelled'
8281
- : 'runner.terminal_ledger_append_failed',
8116
+ phase: req.playCallGovernance
8117
+ ? aborted
8118
+ ? 'runner.child_terminal_ledger_append_cancelled'
8119
+ : 'runner.child_terminal_ledger_append_failed'
8120
+ : aborted
8121
+ ? 'runner.terminal_ledger_append_cancelled'
8122
+ : 'runner.terminal_ledger_append_failed',
8282
8123
  ms: nowMs() - terminalUpdateStartedAt,
8283
8124
  extra: {
8284
8125
  errorCode: failure.code,
8285
8126
  errorPhase: failure.phase,
8286
8127
  },
8287
8128
  });
8288
- const billingStartedAt = nowMs();
8289
- await finalizeWorkerComputeBilling({
8290
- req,
8291
- success: false,
8292
- actionEstimate: 4,
8293
- })
8294
- .catch((finalizeError) => {
8295
- console.error(
8296
- `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
8297
- finalizeError instanceof Error
8298
- ? finalizeError.message
8299
- : String(finalizeError)
8300
- }`,
8301
- );
8129
+ if (options?.persistResultDatasets) {
8130
+ const billingStartedAt = nowMs();
8131
+ await finalizeWorkerComputeBilling({
8132
+ req,
8133
+ success: false,
8134
+ actionEstimate: 4,
8302
8135
  })
8303
- .finally(() => {
8304
- recordRunnerPerfTrace({
8305
- req,
8306
- phase: 'runner.compute_billing_finalize_failed',
8307
- ms: nowMs() - billingStartedAt,
8136
+ .catch((finalizeError) => {
8137
+ console.error(
8138
+ `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
8139
+ finalizeError instanceof Error
8140
+ ? finalizeError.message
8141
+ : String(finalizeError)
8142
+ }`,
8143
+ );
8144
+ })
8145
+ .finally(() => {
8146
+ recordRunnerPerfTrace({
8147
+ req,
8148
+ phase: 'runner.compute_billing_finalize_failed',
8149
+ ms: nowMs() - billingStartedAt,
8150
+ });
8308
8151
  });
8309
- });
8152
+ }
8310
8153
  }
8311
8154
  await signalParentPlayTerminal({
8312
8155
  req,
8313
8156
  status: aborted ? 'cancelled' : 'failed',
8314
- error: message,
8157
+ error: terminalMessage,
8315
8158
  }).catch(() => null);
8316
8159
  recordRunnerPerfTrace({
8317
8160
  req,
@@ -8325,7 +8168,9 @@ async function executeRunRequest(
8325
8168
  await drainRunnerPerfTraces(req);
8326
8169
  throw error;
8327
8170
  } finally {
8328
- clearTimeout(runtimeDeadlineTimer);
8171
+ if (runtimeDeadlineTimer) {
8172
+ clearTimeout(runtimeDeadlineTimer);
8173
+ }
8329
8174
  }
8330
8175
  }
8331
8176
 
@@ -8417,10 +8262,12 @@ function runRequestFromWorkflowParams(
8417
8262
  orgId: String(params.orgId ?? ''),
8418
8263
  playName: String(params.playName ?? ''),
8419
8264
  userEmail: typeof params.userEmail === 'string' ? params.userEmail : null,
8265
+ force: params.force === true,
8266
+ forceToolRefresh: params.forceToolRefresh === true,
8267
+ runAttempt: normalizeWorkerRunAttempt(params.runAttempt),
8420
8268
  runtimeInput: isRecord(params.input)
8421
8269
  ? (params.input as Record<string, unknown>)
8422
8270
  : {},
8423
- force: params.forceToolRefresh === true,
8424
8271
  inlineCsv: isInlineCsv(params.inlineCsv) ? params.inlineCsv : null,
8425
8272
  inputFiles:
8426
8273
  inputFile && inputStorageKey
@@ -8478,6 +8325,14 @@ function runRequestFromWorkflowParams(
8478
8325
  params.coordinatorInternalToken.trim()
8479
8326
  ? params.coordinatorInternalToken.trim()
8480
8327
  : null,
8328
+ runtimeTestFaultHeader:
8329
+ typeof params.runtimeTestFaultHeader === 'string' &&
8330
+ params.runtimeTestFaultHeader.trim()
8331
+ ? params.runtimeTestFaultHeader.trim()
8332
+ : null,
8333
+ testPolicyOverrides: normalizeRuntimeTestPolicyOverrides(
8334
+ params.testPolicyOverrides,
8335
+ ),
8481
8336
  totalRows:
8482
8337
  typeof params.totalRows === 'number' && Number.isFinite(params.totalRows)
8483
8338
  ? params.totalRows
@@ -8547,6 +8402,7 @@ async function persistResultDatasets(
8547
8402
  rows: chunk.map((row) => ({ ...row })),
8548
8403
  runId: req.runId,
8549
8404
  inputOffset,
8405
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
8550
8406
  userEmail: req.userEmail,
8551
8407
  preloadedDbSessions: req.preloadedDbSessions ?? null,
8552
8408
  });
@@ -8583,6 +8439,7 @@ async function persistResultDatasets(
8583
8439
  rows: dataset.rows,
8584
8440
  runId: req.runId,
8585
8441
  inputOffset: 0,
8442
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
8586
8443
  userEmail: req.userEmail,
8587
8444
  });
8588
8445
  }
@@ -8864,7 +8721,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
8864
8721
  try {
8865
8722
  const response = await fetchRuntimeApi(
8866
8723
  req.baseUrl,
8867
- '/api/v2/plays/internal/runtime',
8724
+ '/api/v2/internal/play-runtime',
8868
8725
  {
8869
8726
  method: 'POST',
8870
8727
  headers: {