deepline 0.1.182 → 0.1.184

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
  2. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
  3. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +639 -109
  4. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
  5. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1752 -1899
  6. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
  7. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
  8. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
  9. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
  10. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
  11. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
  12. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
  13. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
  14. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
  15. package/dist/bundling-sources/sdk/src/client.ts +41 -0
  16. package/dist/bundling-sources/sdk/src/http.ts +23 -8
  17. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
  18. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  19. package/dist/bundling-sources/sdk/src/types.ts +2 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +133 -10
  21. package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
  23. package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1612 -287
  26. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +105 -6
  27. package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
  28. package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
  29. package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
  31. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
  32. package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
  33. package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
  35. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
  36. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
  37. package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
  40. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
  42. package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
  43. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
  44. package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +21 -0
  47. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2038 -262
  48. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
  50. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
  52. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
  53. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
  54. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
  56. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
  57. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
  58. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
  59. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +28 -11
  60. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +38 -6
  61. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
  62. package/dist/cli/index.js +76 -17
  63. package/dist/cli/index.mjs +76 -17
  64. package/dist/index.d.mts +2 -0
  65. package/dist/index.d.ts +2 -0
  66. package/dist/index.js +63 -9
  67. package/dist/index.mjs +63 -9
  68. package/dist/plays/bundle-play-file.mjs +20 -6
  69. package/package.json +1 -1
@@ -36,12 +36,6 @@ import {
36
36
  deterministicMapChunkStepName,
37
37
  type ExecutionPlan,
38
38
  } from '../../../shared_libs/play-runtime/execution-plan';
39
- import {
40
- compileRequestsWithStrategy,
41
- executeChunkedRequests,
42
- type ChunkExecutionResult,
43
- } from '../../../shared_libs/play-runtime/batch-runtime';
44
- import { getPlayRuntimeBatchStrategy } from '../../../shared_libs/play-runtime/play-runtime-batching-registry';
45
39
  import {
46
40
  STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
47
41
  STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
@@ -71,11 +65,10 @@ import {
71
65
  type WorkflowStepLike,
72
66
  } from './child-play-await';
73
67
  import { submitChildPlayThroughCoordinator } from './child-play-submit';
74
- import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
75
68
  import {
76
69
  attachToolResultListDataset,
77
- parseToolExecuteResponse,
78
70
  createToolExecuteResult,
71
+ parseToolExecuteResponse,
79
72
  deserializeToolExecuteResult,
80
73
  isSerializedToolExecuteResult,
81
74
  isToolExecuteResult,
@@ -90,29 +83,35 @@ import {
90
83
  TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS,
91
84
  classifyToolExecuteHttpFailure,
92
85
  createToolExecuteHttpFailureAttemptTracker,
86
+ parseToolExecuteAuthScopeChangedError,
93
87
  } from '../../../shared_libs/play-runtime/tool-execute-retry-policy';
94
88
  import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
95
89
  import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
96
- import type { PlayRuntimeManifestMap } from '../../../shared_libs/plays/compiler-manifest';
90
+ import type {
91
+ PlayRuntimeManifest,
92
+ PlayRuntimeManifestMap,
93
+ } from '../../../shared_libs/plays/compiler-manifest';
97
94
  import {
98
95
  deriveToolRequestIdentity,
99
96
  derivePlayRowIdentity,
100
97
  derivePlayRowIdentityFromKey,
101
98
  sha256Hex,
99
+ stableStringify,
102
100
  } from '../../../shared_libs/plays/row-identity';
103
101
  import { createDeferredPlayDataset } from '../../../shared_libs/plays/dataset';
104
102
  import {
105
103
  buildDurableCtxCallCacheKey,
106
- buildDurableToolCallAuthScopeDigest,
107
- buildDurableToolCallCacheKey,
104
+ buildDurableRunPlaySemanticKey,
108
105
  } from '../../../shared_libs/play-runtime/durable-call-cache';
106
+ import {
107
+ resolveRuntimeToolReceiptWaitMaxAttempts,
108
+ resolveRuntimeToolReceiptWaitTimeoutMs,
109
+ } from '../../../shared_libs/play-runtime/durable-receipt-execution';
109
110
  import {
110
111
  QUERY_RESULT_DATASET_PAGE_SIZE,
111
112
  isCustomerDbDatasetTool,
112
- isQueryResultDatasetReadRequest,
113
113
  isQueryResultDatasetTool,
114
114
  } from '../../../shared_libs/play-runtime/query-result-dataset';
115
- import { buildScopedWorkReceiptKey } from '../../../shared_libs/play-runtime/work-receipts';
116
115
  import { DEDUPE_DUPLICATE_KEY_SAMPLE_CAP } from '../../../shared_libs/play-runtime/map-row-identity';
117
116
  import {
118
117
  getTopLevelPipelineSubsteps,
@@ -151,26 +150,45 @@ import {
151
150
  } from './runtime/dataset-handles';
152
151
  import {
153
152
  runWorkerRuntimeReceiptBoundary,
154
- type WorkerRuntimeReceipt,
155
- type WorkerRuntimeReceiptClaim,
156
- type WorkerRuntimeReceiptStore,
153
+ RuntimeLeaseLostError,
154
+ type WorkerRuntimeReceiptExecutionContext,
157
155
  } from './runtime/receipts';
158
156
  import {
159
- RuntimeReceiptWaitTimeoutError,
160
- resolveRuntimeToolReceiptWaitMaxAttempts,
161
- resolveRuntimeToolReceiptWaitTimeoutMs,
162
- waitForCompletedRuntimeReceipt,
163
- } from '../../../shared_libs/play-runtime/durable-receipt-execution';
164
- import type { RuntimeStepReceipt } from '../../../shared_libs/play-runtime/ctx-types';
157
+ RuntimeReceiptPersistenceError,
158
+ WorkerToolBatchScheduler,
159
+ type WorkerPacingResolver,
160
+ type WorkerToolActionCacheVersionResolver,
161
+ } from './runtime/tool-dispatch';
162
+ import {
163
+ createWorkerWorkBudgetMeter,
164
+ type WorkerWorkBudgetMeter,
165
+ } from './runtime/work-budget';
166
+ import {
167
+ WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES,
168
+ WorkerRunWorkDispatcher,
169
+ type WorkerRunMapBatchesResult,
170
+ } from './runtime/run-work-dispatcher';
171
+ import { WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL } from './runtime/worker-platform-budget';
172
+ import {
173
+ PLAY_RUNTIME_CONTRACT,
174
+ PLAY_RUNTIME_CONTRACT_HEADER,
175
+ } from '../../../shared_libs/play-runtime/runtime-contract';
176
+ import {
177
+ PLAY_RUNTIME_API_COMPAT_PATH,
178
+ PLAY_RUNTIME_EGRESS_FETCH_COMPAT_PATH,
179
+ } from '../../../shared_libs/play-runtime/runtime-api-paths';
180
+ import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
181
+ import {
182
+ PLAY_RUNTIME_TEST_FAULT_HEADER,
183
+ type RuntimeTestPolicyOverrides,
184
+ } from '../../../shared_libs/play-runtime/test-runtime-seams';
185
+ import { createRuntimeToolAuthScopeDigestResolver } from '../../../shared_libs/play-runtime/auth-scope-resolver';
165
186
  import {
166
- canReclaimFailedWorkerToolReceipt,
167
- markWorkerToolReceiptResultCached,
168
- markWorkerToolReceiptResultExecution,
169
- planWorkerToolReceiptGroups,
170
- reclaimRunningReceiptGroupAfterWait,
171
- resolveWorkerToolReceiptGroupWaitMaxAttempts,
172
- resolveWorkerToolRuntimeTimeoutMs,
173
- } from './runtime/tool-receipts';
187
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
188
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
189
+ runtimeLeaseHeartbeatIntervalMs,
190
+ } from '../../../shared_libs/play-runtime/lease-policy';
191
+ import { RuntimeSheetRowsBlockedError } from '../../../shared_libs/play-runtime/runtime-sheet-errors';
174
192
  // The harness stub forwards leaf calls (validation, runtime-api HTTP) into
175
193
  // the long-lived Play Harness Worker via env.HARNESS. We import the
176
194
  // `setHarnessBinding` setter eagerly so it's available the moment
@@ -183,9 +201,12 @@ import {
183
201
  // re-bundle harness internals into per-play. Keep that in mind.
184
202
  import {
185
203
  harnessPersistCompletedSheetRows,
204
+ harnessHeartbeatSheetAttempt,
186
205
  harnessReadSheetDatasetRowKeys,
187
206
  harnessReadSheetDatasetRows,
188
207
  harnessReadStagedFileChunk,
208
+ harnessReleaseRuntimeReceipts,
209
+ harnessReleaseSheetAttempt,
189
210
  harnessStartSheetDataset,
190
211
  setHarnessBinding,
191
212
  } from '../../../sdk/src/plays/harness-stub';
@@ -216,7 +237,10 @@ import {
216
237
  stripMapRowOutcomeRuntimeFields,
217
238
  } from '../../../shared_libs/play-runtime/map-row-outcome';
218
239
  import { runtimeSheetSessionScope } from '../../../shared_libs/play-runtime/runtime-sheet-session';
219
- import { chooseWorkerMapRowsPerChunk } from './runtime/map-chunk-plan';
240
+ import {
241
+ chooseWorkerMapDispatchPlan,
242
+ estimateBatchedToolChunkSubrequests,
243
+ } from './runtime/map-chunk-plan';
220
244
  import {
221
245
  applyCsvRenameProjection,
222
246
  cloneCsvAliasedRow,
@@ -228,12 +252,7 @@ import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-f
228
252
  import {
229
253
  createRuntimePersistenceLatch,
230
254
  isRuntimePersistenceCircuitOpenError,
231
- partitionDispatchWaves,
232
- PROVIDER_DISPATCH_WAVE_SIZE,
233
- RuntimePersistenceCircuitOpenError,
234
- shouldShortenCompleteReceiptLadder,
235
255
  tripRuntimePersistenceLatch,
236
- type RuntimePersistenceLatch,
237
256
  } from '../../../shared_libs/play-runtime/persistence-latch';
238
257
  import {
239
258
  createReceiptSalvageBuffer,
@@ -299,8 +318,13 @@ type RunRequest = {
299
318
  playName: string;
300
319
  graphHash?: string | null;
301
320
  userEmail: string | null;
302
- force?: boolean | null;
303
321
  runtimeInput: Record<string, unknown>;
322
+ /** Start a fresh run graph and recompute runtime-sheet rows. */
323
+ force?: boolean;
324
+ /** Explicit durable tool receipt refresh that can re-execute completed provider calls. */
325
+ forceToolRefresh?: boolean;
326
+ /** Monotonic coordinator redispatch epoch for receipt and sheet fencing. */
327
+ runAttempt?: number;
304
328
  /** Optional inline CSV rows (for plays where ctx.csv was passed inline data). */
305
329
  inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
306
330
  /** Staged input files keyed by logical filename (used by ctx.csv). */
@@ -333,6 +357,10 @@ type RunRequest = {
333
357
  coordinatorUrl?: string | null;
334
358
  /** Request-scoped coordinator auth token for preview/dev direct control calls. */
335
359
  coordinatorInternalToken?: string | null;
360
+ /** Request-scoped, dev-only runtime fault injection header for black-box tests. */
361
+ runtimeTestFaultHeader?: string | null;
362
+ /** Request-scoped, dev-only policy overrides for black-box durability tests. */
363
+ testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
336
364
  /**
337
365
  * Total input rows known at submit time, when the dispatcher can count them.
338
366
  * `undefined` when the input streams from R2 (unknown row count).
@@ -348,10 +376,104 @@ type WorkerFileRef = {
348
376
  bytes?: number | null;
349
377
  };
350
378
 
379
+ function normalizeWorkerRunAttempt(value: unknown): number {
380
+ return typeof value === 'number' && Number.isFinite(value)
381
+ ? Math.max(0, Math.floor(value))
382
+ : 0;
383
+ }
384
+
385
+ function normalizePositiveInteger(value: unknown): number | undefined {
386
+ return typeof value === 'number' &&
387
+ Number.isFinite(value) &&
388
+ Number.isInteger(value) &&
389
+ value > 0
390
+ ? value
391
+ : undefined;
392
+ }
393
+
394
+ function normalizeRuntimeTestPolicyOverrides(
395
+ value: unknown,
396
+ ): RuntimeTestPolicyOverrides | null {
397
+ if (!isRecord(value)) return null;
398
+ const overrides: RuntimeTestPolicyOverrides = {};
399
+ const receiptLeaseTtlMs = normalizePositiveInteger(value.receiptLeaseTtlMs);
400
+ if (receiptLeaseTtlMs !== undefined) {
401
+ overrides.receiptLeaseTtlMs = receiptLeaseTtlMs;
402
+ }
403
+ const sheetAttemptLeaseMs = normalizePositiveInteger(
404
+ value.sheetAttemptLeaseMs,
405
+ );
406
+ if (sheetAttemptLeaseMs !== undefined) {
407
+ overrides.sheetAttemptLeaseMs = sheetAttemptLeaseMs;
408
+ }
409
+ const heartbeatIntervalMs = normalizePositiveInteger(
410
+ value.heartbeatIntervalMs,
411
+ );
412
+ if (heartbeatIntervalMs !== undefined) {
413
+ overrides.heartbeatIntervalMs = heartbeatIntervalMs;
414
+ }
415
+ const batchGraceMs = normalizePositiveInteger(value.batchGraceMs);
416
+ if (batchGraceMs !== undefined) {
417
+ overrides.batchGraceMs = batchGraceMs;
418
+ }
419
+ if (isRecord(value.workBudgetYieldLimits)) {
420
+ const limits: NonNullable<
421
+ RuntimeTestPolicyOverrides['workBudgetYieldLimits']
422
+ > = {};
423
+ for (const key of [
424
+ 'elapsed',
425
+ 'subrequest',
426
+ 'provider',
427
+ 'tool',
428
+ 'receipt',
429
+ 'sheet',
430
+ 'log',
431
+ 'egress',
432
+ ] as const) {
433
+ const limit = normalizePositiveInteger(value.workBudgetYieldLimits[key]);
434
+ if (limit !== undefined) {
435
+ limits[key] = limit;
436
+ }
437
+ }
438
+ if (Object.keys(limits).length > 0) {
439
+ overrides.workBudgetYieldLimits = limits;
440
+ }
441
+ }
442
+ return Object.keys(overrides).length > 0 ? overrides : null;
443
+ }
444
+
445
+ function runtimeReceiptLeaseTtlMs(req: RunRequest): number | undefined {
446
+ return req.testPolicyOverrides?.receiptLeaseTtlMs;
447
+ }
448
+
449
+ function runtimeSheetAttemptLeaseTtlMs(req: RunRequest): number | undefined {
450
+ return req.testPolicyOverrides?.sheetAttemptLeaseMs;
451
+ }
452
+
453
+ function runtimePolicyHeartbeatIntervalMs(
454
+ req: RunRequest,
455
+ fallbackLeaseTtlMs: number,
456
+ ): number {
457
+ return (
458
+ req.testPolicyOverrides?.heartbeatIntervalMs ??
459
+ runtimeLeaseHeartbeatIntervalMs(fallbackLeaseTtlMs)
460
+ );
461
+ }
462
+
351
463
  const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
352
464
  const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
353
465
  const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
354
466
  const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
467
+ // A short durable sleep is enough to force a Cloudflare Workflows step
468
+ // checkpoint. The resume re-enters run() as a new Worker invocation with a
469
+ // fresh per-invocation subrequest budget (see the yield callback below), so
470
+ // there is no need to escalate the sleep to "encourage" re-invocation.
471
+ // A 1ms Workflow sleep is too small to reliably create a new platform
472
+ // invocation in preview; the dispatcher then resets its local budget while
473
+ // Cloudflare's per-invocation subrequest counter is still hot. Use a real
474
+ // boundary so budget-yielded maps resume with fresh platform headroom.
475
+ const WORK_DISPATCHER_YIELD_SLEEP_MS = 1_000;
476
+
355
477
  type CustomerDbDatasetRequest = {
356
478
  limit?: number;
357
479
  offset?: number;
@@ -418,7 +540,8 @@ type WorkerEnv = {
418
540
  /**
419
541
  * In-process Fetcher constructed by the coordinator and handed to the
420
542
  * per-graphHash play Worker. When present, runtime callbacks
421
- * (`/api/v2/plays/internal/runtime`, `/api/v2/plays/internal/*`,
543
+ * (`/api/v2/internal/play-runtime`,
544
+ * `/api/v2/internal/play-runtime/*`,
422
545
  * `/api/v2/plays/runtime-tools/*`) skip the public callback URL and route
423
546
  * directly through the coordinator's process to the configured app — saves
424
547
  * the *.workers.dev → CF edge → cloudflared → localhost chain on every
@@ -427,6 +550,7 @@ type WorkerEnv = {
427
550
  */
428
551
  RUNTIME_API?: {
429
552
  runtimeApiCall(input: {
553
+ contract?: number;
430
554
  executorToken: string;
431
555
  baseUrl?: string;
432
556
  path: string;
@@ -663,7 +787,7 @@ async function fetchRuntimeApi(
663
787
  ? Math.max(1, Math.ceil(options.timeoutMsOverride))
664
788
  : path === '/api/v2/plays/run'
665
789
  ? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
666
- : path === '/api/v2/plays/internal/egress-fetch'
790
+ : path === PLAY_RUNTIME_EGRESS_FETCH_COMPAT_PATH
667
791
  ? RUNTIME_API_EGRESS_FETCH_TIMEOUT_MS
668
792
  : /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
669
793
  ? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
@@ -691,6 +815,10 @@ async function fetchRuntimeApi(
691
815
  );
692
816
  }, timeoutMs);
693
817
  });
818
+ const relayAbort = () => controller.abort(abortSignal?.reason);
819
+ if (abortSignal) {
820
+ abortSignal.addEventListener('abort', relayAbort, { once: true });
821
+ }
694
822
  try {
695
823
  const mergedInit: RequestInit = {
696
824
  ...init,
@@ -712,11 +840,10 @@ async function fetchRuntimeApi(
712
840
  timeoutMs,
713
841
  },
714
842
  );
715
- // RUNTIME_API service bindings do not consume RequestInit.signal once the RPC
716
- // is in flight. After dispatch, wait for the owner call to settle so the
717
- // durable receipt records the real provider result instead of failing early
718
- // and inviting duplicate provider work on retry.
719
843
  const response = await Promise.race([responsePromise, timeoutPromise]);
844
+ if (abortSignal?.aborted) {
845
+ throw abortError();
846
+ }
720
847
  if (await isRuntimeApiBindingNotFoundResponse(response)) {
721
848
  throw new Error(
722
849
  `[play-harness] RUNTIME_API service binding could not route ${path}; coordinator returned not found.`,
@@ -732,6 +859,9 @@ async function fetchRuntimeApi(
732
859
  }
733
860
  throw err;
734
861
  } finally {
862
+ if (abortSignal) {
863
+ abortSignal.removeEventListener('abort', relayAbort);
864
+ }
735
865
  if (timeout) clearTimeout(timeout);
736
866
  }
737
867
  }
@@ -748,10 +878,23 @@ async function callRuntimeApiRpcBinding(
748
878
  if (metadata) headers[EXECUTE_TOOL_METADATA_HEADER] = metadata;
749
879
  const contract = h.get(EXECUTE_RESPONSE_CONTRACT_HEADER);
750
880
  if (contract) headers[EXECUTE_RESPONSE_CONTRACT_HEADER] = contract;
881
+ const idempotencyKey = h.get('x-deepline-idempotency-key');
882
+ if (idempotencyKey) {
883
+ headers['x-deepline-idempotency-key'] = idempotencyKey;
884
+ }
885
+ const cacheControl = h.get('cache-control');
886
+ if (cacheControl) headers['cache-control'] = cacheControl;
887
+ const pragma = h.get('pragma');
888
+ if (pragma) headers.pragma = pragma;
889
+ const runtimeTestFaultHeader = h.get(PLAY_RUNTIME_TEST_FAULT_HEADER);
890
+ if (runtimeTestFaultHeader) {
891
+ headers[PLAY_RUNTIME_TEST_FAULT_HEADER] = runtimeTestFaultHeader;
892
+ }
751
893
  const responseIntent = h.get(EXECUTE_RESPONSE_INTENT_HEADER);
752
894
  if (responseIntent) headers[EXECUTE_RESPONSE_INTENT_HEADER] = responseIntent;
753
895
  const rawBody = typeof init.body === 'string' ? init.body : '';
754
896
  const result = await binding.runtimeApiCall({
897
+ contract: PLAY_RUNTIME_CONTRACT,
755
898
  executorToken: authorization.replace(/^Bearer\s+/i, '').trim(),
756
899
  baseUrl: input.baseUrl,
757
900
  path: input.path,
@@ -784,6 +927,7 @@ function runtimeApiHeaders(
784
927
  includeVercelBypass: boolean,
785
928
  ): Headers {
786
929
  const next = new Headers(headers);
930
+ next.set(PLAY_RUNTIME_CONTRACT_HEADER, String(PLAY_RUNTIME_CONTRACT));
787
931
  if (includeVercelBypass) {
788
932
  const bypassToken = cachedVercelProtectionBypassToken();
789
933
  if (bypassToken) {
@@ -976,9 +1120,172 @@ async function drainRunnerPerfTraces(req: RunRequest): Promise<void> {
976
1120
  ]);
977
1121
  }
978
1122
 
1123
+ let workerRequestSequence = 0;
1124
+
979
1125
  function makeRequestId(): string {
980
- // Workers crypto.randomUUID is available without nodejs_compat.
981
- return crypto.randomUUID();
1126
+ workerRequestSequence = (workerRequestSequence + 1) % Number.MAX_SAFE_INTEGER;
1127
+ return `worker-request-${workerRequestSequence}`;
1128
+ }
1129
+
1130
+ function makeRuntimeSheetAttempt(input: {
1131
+ runId: string;
1132
+ runAttempt?: number | null;
1133
+ mapName: string;
1134
+ chunkIndex: number | 'finalize';
1135
+ }): {
1136
+ attemptId: string;
1137
+ attemptOwnerRunId: string;
1138
+ attemptExpiresAt: string | null;
1139
+ attemptSeq: number;
1140
+ } {
1141
+ const runAttempt =
1142
+ typeof input.runAttempt === 'number' && Number.isFinite(input.runAttempt)
1143
+ ? Math.max(0, Math.floor(input.runAttempt))
1144
+ : 0;
1145
+ const attemptKey = sha256Hex(
1146
+ JSON.stringify({
1147
+ runId: input.runId,
1148
+ runAttempt,
1149
+ mapName: input.mapName,
1150
+ chunkIndex: input.chunkIndex,
1151
+ }),
1152
+ ).slice(0, 16);
1153
+ return {
1154
+ attemptId: `sheet:${input.mapName}:${input.chunkIndex}:attempt-${runAttempt}:${attemptKey}`,
1155
+ attemptOwnerRunId: input.runId,
1156
+ attemptExpiresAt: null,
1157
+ attemptSeq: runAttempt,
1158
+ };
1159
+ }
1160
+
1161
+ class RuntimeSheetAttemptLeaseLostError extends Error {
1162
+ readonly rowKeys: string[];
1163
+ readonly runId: string;
1164
+ readonly attemptId: string;
1165
+ readonly tableNamespace: string;
1166
+
1167
+ constructor(input: {
1168
+ rowKeys: string[];
1169
+ runId: string;
1170
+ attemptId: string;
1171
+ tableNamespace: string;
1172
+ cause?: unknown;
1173
+ }) {
1174
+ const rowSummary =
1175
+ input.rowKeys.length === 1
1176
+ ? input.rowKeys[0]!
1177
+ : `${input.rowKeys.length} rows`;
1178
+ super(
1179
+ `Runtime sheet attempt ${input.attemptId} lost ownership of ${rowSummary} in ${input.tableNamespace}.`,
1180
+ { cause: input.cause },
1181
+ );
1182
+ this.name = 'RuntimeSheetAttemptLeaseLostError';
1183
+ this.rowKeys = input.rowKeys;
1184
+ this.runId = input.runId;
1185
+ this.attemptId = input.attemptId;
1186
+ this.tableNamespace = input.tableNamespace;
1187
+ }
1188
+ }
1189
+
1190
+ async function executeWithRuntimeSheetAttemptHeartbeat<T>(input: {
1191
+ req: RunRequest;
1192
+ tableNamespace: string;
1193
+ sheetContract: PlaySheetContract;
1194
+ attemptId?: string | null;
1195
+ attemptOwnerRunId?: string | null;
1196
+ attemptSeq?: number | null;
1197
+ rowKeys: string[];
1198
+ execute: () => Promise<T> | T;
1199
+ }): Promise<T> {
1200
+ const attemptId = input.attemptId?.trim();
1201
+ const rowKeys = [
1202
+ ...new Set(input.rowKeys.map((key) => key.trim()).filter(Boolean)),
1203
+ ];
1204
+ if (!attemptId || rowKeys.length === 0) {
1205
+ return await input.execute();
1206
+ }
1207
+ const heartbeatIntervalMs = runtimePolicyHeartbeatIntervalMs(
1208
+ input.req,
1209
+ runtimeSheetAttemptLeaseTtlMs(input.req) ??
1210
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
1211
+ );
1212
+
1213
+ let stopped = false;
1214
+ let timeout: ReturnType<typeof setTimeout> | null = null;
1215
+ let rejectLeaseLost!: (error: RuntimeSheetAttemptLeaseLostError) => void;
1216
+ const leaseLost = new Promise<never>((_, reject) => {
1217
+ rejectLeaseLost = reject;
1218
+ });
1219
+
1220
+ const stop = () => {
1221
+ stopped = true;
1222
+ if (timeout !== null) {
1223
+ clearTimeout(timeout);
1224
+ timeout = null;
1225
+ }
1226
+ };
1227
+
1228
+ const heartbeatOnce = async () => {
1229
+ const result = await harnessHeartbeatSheetAttempt({
1230
+ ...runtimeSheetSessionScope(input.req),
1231
+ tableNamespace: input.tableNamespace,
1232
+ sheetContract: input.sheetContract,
1233
+ runId: input.req.runId,
1234
+ attemptId,
1235
+ attemptOwnerRunId: input.attemptOwnerRunId ?? input.req.runId,
1236
+ attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
1237
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
1238
+ keys: rowKeys,
1239
+ });
1240
+ const renewed = new Set(result.renewedKeys);
1241
+ const lostKeys = rowKeys.filter((key) => !renewed.has(key));
1242
+ if (lostKeys.length > 0) {
1243
+ throw new RuntimeSheetAttemptLeaseLostError({
1244
+ rowKeys: lostKeys,
1245
+ runId: input.req.runId,
1246
+ attemptId,
1247
+ tableNamespace: input.tableNamespace,
1248
+ });
1249
+ }
1250
+ };
1251
+
1252
+ const schedule = () => {
1253
+ timeout = setTimeout(() => {
1254
+ void (async () => {
1255
+ try {
1256
+ await heartbeatOnce();
1257
+ } catch (error) {
1258
+ stop();
1259
+ if (error instanceof RuntimeSheetAttemptLeaseLostError) {
1260
+ rejectLeaseLost(error);
1261
+ } else {
1262
+ rejectLeaseLost(
1263
+ new RuntimeSheetAttemptLeaseLostError({
1264
+ rowKeys,
1265
+ runId: input.req.runId,
1266
+ attemptId,
1267
+ tableNamespace: input.tableNamespace,
1268
+ cause: error,
1269
+ }),
1270
+ );
1271
+ }
1272
+ return;
1273
+ }
1274
+ if (!stopped) {
1275
+ schedule();
1276
+ }
1277
+ })();
1278
+ }, heartbeatIntervalMs);
1279
+ };
1280
+
1281
+ schedule();
1282
+ const execution = Promise.resolve().then(input.execute);
1283
+ execution.catch(() => {});
1284
+ try {
1285
+ return await Promise.race([execution, leaseLost]);
1286
+ } finally {
1287
+ stop();
1288
+ }
982
1289
  }
983
1290
 
984
1291
  /**
@@ -1004,6 +1311,11 @@ async function postRuntimeApi<T>(
1004
1311
  baseUrl: string,
1005
1312
  executorToken: string,
1006
1313
  body: unknown,
1314
+ options: {
1315
+ runtimeTestFaultHeader?: string | null;
1316
+ timeoutMsOverride?: number;
1317
+ timeoutErrorMessage?: string;
1318
+ } = {},
1007
1319
  ): Promise<T> {
1008
1320
  // Routes through the in-process RUNTIME_API service binding. Missing binding
1009
1321
  // is an infra error in workers_edge, not a reason to fall back to public HTTP.
@@ -1016,15 +1328,29 @@ async function postRuntimeApi<T>(
1016
1328
  ) {
1017
1329
  let res: Response;
1018
1330
  try {
1019
- res = await fetchRuntimeApi(baseUrl, '/api/v2/plays/internal/runtime', {
1020
- method: 'POST',
1021
- headers: {
1022
- 'content-type': 'application/json',
1023
- authorization: `Bearer ${executorToken}`,
1024
- 'x-deepline-request-id': makeRequestId(),
1331
+ res = await fetchRuntimeApi(
1332
+ baseUrl,
1333
+ PLAY_RUNTIME_API_COMPAT_PATH,
1334
+ {
1335
+ method: 'POST',
1336
+ headers: {
1337
+ 'content-type': 'application/json',
1338
+ authorization: `Bearer ${executorToken}`,
1339
+ 'x-deepline-request-id': makeRequestId(),
1340
+ ...(options.runtimeTestFaultHeader?.trim()
1341
+ ? {
1342
+ [PLAY_RUNTIME_TEST_FAULT_HEADER]:
1343
+ options.runtimeTestFaultHeader.trim(),
1344
+ }
1345
+ : {}),
1346
+ },
1347
+ body: serializedBody,
1025
1348
  },
1026
- body: serializedBody,
1027
- });
1349
+ {
1350
+ timeoutMsOverride: options.timeoutMsOverride,
1351
+ timeoutErrorMessage: options.timeoutErrorMessage,
1352
+ },
1353
+ );
1028
1354
  } catch (error) {
1029
1355
  lastError = error;
1030
1356
  if (
@@ -1108,9 +1434,10 @@ async function postRuntimeApiBestEffort(
1108
1434
  baseUrl: string,
1109
1435
  executorToken: string,
1110
1436
  body: unknown,
1437
+ options: { runtimeTestFaultHeader?: string | null } = {},
1111
1438
  ): Promise<boolean> {
1112
1439
  try {
1113
- await postRuntimeApi(baseUrl, executorToken, body);
1440
+ await postRuntimeApi(baseUrl, executorToken, body, options);
1114
1441
  return true;
1115
1442
  } catch (error) {
1116
1443
  console.error(
@@ -1122,6 +1449,43 @@ async function postRuntimeApiBestEffort(
1122
1449
  }
1123
1450
  }
1124
1451
 
1452
+ async function readDurableChildRunTerminalState(input: {
1453
+ baseUrl: string;
1454
+ executorToken: string;
1455
+ parentRunId: string;
1456
+ childRunId: string;
1457
+ childPlayName: string;
1458
+ timeoutMs?: number;
1459
+ }): Promise<{ data?: unknown } | null> {
1460
+ const response = await postRuntimeApi<{
1461
+ state?: {
1462
+ data?: unknown;
1463
+ reason?: string;
1464
+ status?: string;
1465
+ runId?: string;
1466
+ playName?: string;
1467
+ parentRunId?: string | null;
1468
+ } | null;
1469
+ }>(
1470
+ input.baseUrl,
1471
+ input.executorToken,
1472
+ {
1473
+ action: 'read_child_run_terminal_snapshot',
1474
+ parentRunId: input.parentRunId,
1475
+ childRunId: input.childRunId,
1476
+ childPlayName: input.childPlayName,
1477
+ },
1478
+ {
1479
+ timeoutMsOverride: input.timeoutMs,
1480
+ timeoutErrorMessage: `[play-harness] child run terminal snapshot read timed out after ${Math.max(
1481
+ 1,
1482
+ Math.ceil(input.timeoutMs ?? RUNTIME_API_TIMEOUT_MS),
1483
+ )}ms. childRunId=${input.childRunId}`,
1484
+ },
1485
+ );
1486
+ return response.state ?? null;
1487
+ }
1488
+
1125
1489
  function workflowEventType(name: string): string {
1126
1490
  const normalized = name
1127
1491
  .trim()
@@ -1231,6 +1595,27 @@ async function hashChildPlayEventKey(input: unknown): Promise<string> {
1231
1595
  return (await hashJson(input)).slice(0, 32);
1232
1596
  }
1233
1597
 
1598
+ function workerChildRevisionFingerprint(input: {
1599
+ playName: string;
1600
+ manifest: PlayRuntimeManifest | null | undefined;
1601
+ }): string | null {
1602
+ const manifest = input.manifest;
1603
+ if (!manifest) return null;
1604
+ return sha256Hex(
1605
+ stableStringify({
1606
+ playId: manifest.playName?.trim() || input.playName,
1607
+ codeFormat: 'cjs_module',
1608
+ artifactHash: manifest.artifactHash ?? null,
1609
+ graphHash: manifest.graphHash ?? null,
1610
+ sourceHash: null,
1611
+ sourceCodeHash:
1612
+ typeof manifest.sourceCode === 'string'
1613
+ ? sha256Hex(manifest.sourceCode)
1614
+ : null,
1615
+ }),
1616
+ );
1617
+ }
1618
+
1234
1619
  async function childPlayEventKey(input: {
1235
1620
  key: string;
1236
1621
  workflowId: string;
@@ -1327,7 +1712,14 @@ async function signalParentPlayTerminal(input: {
1327
1712
 
1328
1713
  async function executeTool(
1329
1714
  req: RunRequest,
1330
- args: { id: string; toolId: string; input: Record<string, unknown> },
1715
+ args: {
1716
+ id: string;
1717
+ toolId: string;
1718
+ input: Record<string, unknown>;
1719
+ durableCallReceiptKey?: string | null;
1720
+ executionAuthScopeDigest?: string | null;
1721
+ providerIdempotencyKey?: string | null;
1722
+ },
1331
1723
  workflowStep?: WorkflowStep,
1332
1724
  onProviderBackpressure?: (retryAfterMs: number) => void,
1333
1725
  onRetryAttempt?: () => void,
@@ -1365,7 +1757,14 @@ async function executeTool(
1365
1757
 
1366
1758
  async function executeToolWithLifecycle(
1367
1759
  req: RunRequest,
1368
- args: { id: string; toolId: string; input: Record<string, unknown> },
1760
+ args: {
1761
+ id: string;
1762
+ toolId: string;
1763
+ input: Record<string, unknown>;
1764
+ durableCallReceiptKey?: string | null;
1765
+ executionAuthScopeDigest?: string | null;
1766
+ providerIdempotencyKey?: string | null;
1767
+ },
1369
1768
  workflowStep: WorkflowStep | undefined,
1370
1769
  callbacks: WorkerCtxCallbacks | undefined,
1371
1770
  onProviderBackpressure?: (retryAfterMs: number) => void,
@@ -1461,24 +1860,29 @@ async function waitForSyntheticIntegrationEvent(
1461
1860
  typeof input.timeout_ms === 'number' && Number.isFinite(input.timeout_ms)
1462
1861
  ? Math.max(1, Math.round(input.timeout_ms))
1463
1862
  : 30_000;
1464
- await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
1465
- action: 'append_run_events',
1466
- playId: req.runId,
1467
- events: [
1468
- {
1469
- type: 'log.appended',
1470
- runId: req.runId,
1471
- // 'system' (windowed text-dedupe channel), NOT 'worker': this line is
1472
- // emitted outside the harness log buffer, so it has no positional
1473
- // channelOffset and must not pollute the worker channel cursor.
1474
- source: 'system',
1475
- occurredAt: nowMs(),
1476
- lines: [
1477
- `Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
1478
- ],
1479
- } satisfies PlayRunLedgerEvent,
1480
- ],
1481
- });
1863
+ await postRuntimeApiBestEffort(
1864
+ req.baseUrl,
1865
+ req.executorToken,
1866
+ {
1867
+ action: 'append_run_events',
1868
+ playId: req.runId,
1869
+ events: [
1870
+ {
1871
+ type: 'log.appended',
1872
+ runId: req.runId,
1873
+ // 'system' (windowed text-dedupe channel), NOT 'worker': this line is
1874
+ // emitted outside the harness log buffer, so it has no positional
1875
+ // channelOffset and must not pollute the worker channel cursor.
1876
+ source: 'system',
1877
+ occurredAt: nowMs(),
1878
+ lines: [
1879
+ `Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
1880
+ ],
1881
+ } satisfies PlayRunLedgerEvent,
1882
+ ],
1883
+ },
1884
+ { runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null },
1885
+ );
1482
1886
  try {
1483
1887
  const event = (await (
1484
1888
  workflowStep.waitForEvent as unknown as (
@@ -1526,7 +1930,14 @@ async function waitForSyntheticIntegrationEvent(
1526
1930
 
1527
1931
  async function callToolDirect(
1528
1932
  req: RunRequest,
1529
- args: { id: string; toolId: string; input: Record<string, unknown> },
1933
+ args: {
1934
+ id: string;
1935
+ toolId: string;
1936
+ input: Record<string, unknown>;
1937
+ durableCallReceiptKey?: string | null;
1938
+ executionAuthScopeDigest?: string | null;
1939
+ providerIdempotencyKey?: string | null;
1940
+ },
1530
1941
  onProviderBackpressure?: (retryAfterMs: number) => void,
1531
1942
  // Invoked once per in-process retry attempt (429 / retryable 5xx / synthetic
1532
1943
  // transient) so the Governor charges chargeBudget('retry') per attempt — the
@@ -1542,6 +1953,16 @@ async function callToolDirect(
1542
1953
  ): Promise<ToolExecuteResult> {
1543
1954
  const { id, toolId, input } = args;
1544
1955
  const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
1956
+ const durableCallReceiptKey = args.durableCallReceiptKey?.trim() || null;
1957
+ const executionAuthScopeDigest =
1958
+ durableCallReceiptKey && args.executionAuthScopeDigest
1959
+ ? args.executionAuthScopeDigest.trim() || null
1960
+ : null;
1961
+ const providerIdempotencyKey =
1962
+ args.providerIdempotencyKey?.trim() || durableCallReceiptKey;
1963
+ const deeplineRequestId = providerIdempotencyKey
1964
+ ? `ctx-tool-${sha256Hex(providerIdempotencyKey).slice(0, 32)}`
1965
+ : null;
1545
1966
  let lastError: Error | null = null;
1546
1967
  const httpFailureAttempts = createToolExecuteHttpFailureAttemptTracker();
1547
1968
  let requestAttempt = 0;
@@ -1564,7 +1985,12 @@ async function callToolDirect(
1564
1985
  headers: {
1565
1986
  'content-type': 'application/json',
1566
1987
  authorization: `Bearer ${req.executorToken}`,
1567
- 'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
1988
+ 'x-deepline-request-id':
1989
+ deeplineRequestId ??
1990
+ `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
1991
+ ...(providerIdempotencyKey
1992
+ ? { 'x-deepline-idempotency-key': providerIdempotencyKey }
1993
+ : {}),
1568
1994
  [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
1569
1995
  [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
1570
1996
  [EXECUTE_TOOL_METADATA_HEADER]: 'true',
@@ -1573,6 +1999,16 @@ async function callToolDirect(
1573
1999
  payload: input,
1574
2000
  metadata: {
1575
2001
  parent_run_id: req.runId,
2002
+ ...(durableCallReceiptKey
2003
+ ? {
2004
+ durable_call_receipt_key: durableCallReceiptKey,
2005
+ ...(executionAuthScopeDigest
2006
+ ? {
2007
+ execution_auth_scope_digest: executionAuthScopeDigest,
2008
+ }
2009
+ : {}),
2010
+ }
2011
+ : {}),
1576
2012
  ...(directOptions?.customerDbDataset
1577
2013
  ? {
1578
2014
  query_result_dataset: {
@@ -1605,6 +2041,9 @@ async function callToolDirect(
1605
2041
  : {}),
1606
2042
  }
1607
2043
  : {}),
2044
+ ...(providerIdempotencyKey
2045
+ ? { provider_idempotency_key: providerIdempotencyKey }
2046
+ : {}),
1608
2047
  },
1609
2048
  ...(req.integrationMode
1610
2049
  ? { integration_mode: req.integrationMode }
@@ -1652,6 +2091,13 @@ async function callToolDirect(
1652
2091
  }
1653
2092
 
1654
2093
  const text = await res.text().catch(() => '');
2094
+ const authScopeChanged = parseToolExecuteAuthScopeChangedError({
2095
+ status: res.status,
2096
+ bodyText: text,
2097
+ });
2098
+ if (authScopeChanged) {
2099
+ throw authScopeChanged;
2100
+ }
1655
2101
  const httpFailureAttempt = httpFailureAttempts.next({
1656
2102
  toolId,
1657
2103
  status: res.status,
@@ -1685,48 +2131,6 @@ async function callToolDirect(
1685
2131
  throw lastError ?? new Error(`tool ${toolId} failed before execution.`);
1686
2132
  }
1687
2133
 
1688
- function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
1689
- if (toolId === 'test_rate_limit') {
1690
- // Batched members resolve metadata through this fallback because the lean
1691
- // worker does not bundle the catalog. It MUST mirror the same
1692
- // `email_status: emailStatus({...})` contract registered in
1693
- // src/lib/integrations/test/index.ts so batched results normalize to the
1694
- // same rich email_status OBJECT (status from statusMap, catch_all as a
1695
- // signal) that the single-execute real-metadata path produces. The legacy
1696
- // `transforms:['emailStatus']` + mx_security_gateway→'catch_all' string
1697
- // override coarsened email_status into a bare string and predates the
1698
- // emailStatus object contract (#1466).
1699
- return {
1700
- toolId,
1701
- extractors: {
1702
- email_status: {
1703
- paths: ['email_status'],
1704
- emailStatus: {
1705
- provider: 'test',
1706
- rawStatus: ['email_status'],
1707
- catchAll: ['mx_security_gateway'],
1708
- statusMap: {
1709
- valid: { status: 'valid', verdict: 'send', verified: true },
1710
- invalid: { status: 'invalid', verdict: 'drop', verified: false },
1711
- catch_all: {
1712
- status: 'catch_all',
1713
- verdict: 'verify_next',
1714
- verified: false,
1715
- },
1716
- unknown: { status: 'unknown', verdict: 'hold', verified: false },
1717
- },
1718
- },
1719
- },
1720
- },
1721
- targetGetters: {
1722
- email: ['email', 'value'],
1723
- email_status: ['email_status'],
1724
- },
1725
- };
1726
- }
1727
- return { toolId };
1728
- }
1729
-
1730
2134
  function wrapWorkerToolResult(
1731
2135
  toolId: string,
1732
2136
  result: unknown,
@@ -1767,10 +2171,10 @@ function attachWorkerCustomerDbDatasetResult(
1767
2171
  typeof options.input.sql === 'string'
1768
2172
  ? options.input.sql
1769
2173
  : typeof options.input.query === 'string'
1770
- ? options.input.query
1771
- : typeof raw?.sql === 'string'
1772
- ? raw.sql
1773
- : null;
2174
+ ? options.input.query
2175
+ : typeof raw?.sql === 'string'
2176
+ ? raw.sql
2177
+ : null;
1774
2178
  if (!dataset || totalRows === null || !sql) {
1775
2179
  return result;
1776
2180
  }
@@ -1882,1453 +2286,91 @@ type WorkerStepProgramRecorder = {
1882
2286
  parentField: string;
1883
2287
  path: string[];
1884
2288
  outputs: RecordedStepProgramOutput[];
1885
- onOutput?: (output: RecordedStepProgramOutput) => void | Promise<void>;
1886
- };
1887
-
1888
- type WorkerStepResolution = {
1889
- value: unknown;
1890
- status?: 'skipped';
1891
- };
1892
-
1893
- type WorkerToolBatchRequest = {
1894
- id: string;
1895
- cacheKey: string;
1896
- receiptKey: string | null;
1897
- force: boolean;
1898
- receiptWaitMaxAttempts: number;
1899
- runtimeTimeoutMs?: number;
1900
- toolId: string;
1901
- input: Record<string, unknown>;
1902
- workflowStep?: WorkflowStep;
1903
- directOptions?: WorkerToolDirectOptions;
1904
- resolve: (value: unknown) => void;
1905
- reject: (error: unknown) => void;
1906
- };
1907
-
1908
- type ClaimedWorkerToolBatchRequest = {
1909
- request: WorkerToolBatchRequest;
1910
- receiptKey: string | null;
1911
- followers: WorkerToolBatchRequest[];
1912
- };
1913
-
1914
- type PreparedWorkerToolBatchRequests = {
1915
- claimedRequests: ClaimedWorkerToolBatchRequest[];
1916
- deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[];
1917
- };
1918
-
1919
- function resolveClaimedWorkerToolRuntimeTimeoutMs(
1920
- claimedRequests: ClaimedWorkerToolBatchRequest[],
1921
- input: { runtimeDeadlineMs?: number },
1922
- ): number | undefined {
1923
- return resolveWorkerToolRuntimeTimeoutMs(claimedRequests, {
1924
- resolveOwnerTimeoutMs: (request) =>
1925
- resolveToolRuntimeApiTimeout({
1926
- requestInput: request.input,
1927
- timeoutMs: request.runtimeTimeoutMs,
1928
- runtimeDeadlineMs: input.runtimeDeadlineMs,
1929
- }).timeoutMs,
1930
- });
1931
- }
1932
-
1933
- const WORKER_TOOL_BATCH_GRACE_MS = 250;
1934
- const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
1935
- const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
1936
- const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
1937
- const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 1_000;
1938
- const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
1939
- /**
1940
- * Bounded number of per-row failure samples carried in chunk summaries and the
1941
- * map's terminal partial-failure log. Every failed row is persisted with its
1942
- * full error in the runtime sheet; the samples just keep run logs readable.
1943
- */
1944
- const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
1945
-
1946
- class RuntimeReceiptPersistenceError extends Error {
1947
- readonly errors: unknown[];
1948
-
1949
- constructor(message: string, errors: unknown[] = []) {
1950
- super(message);
1951
- this.name = 'RuntimeReceiptPersistenceError';
1952
- this.errors = errors;
1953
- }
1954
- }
1955
-
1956
- function isRuntimeReceiptPersistenceError(error: unknown): boolean {
1957
- if (!error || typeof error !== 'object') return false;
1958
- if (error instanceof RuntimeReceiptPersistenceError) return true;
1959
- if (
1960
- error instanceof Error &&
1961
- error.name === 'RuntimeReceiptPersistenceError'
1962
- ) {
1963
- return true;
1964
- }
1965
- const nestedErrors = (error as { errors?: unknown }).errors;
1966
- return (
1967
- Array.isArray(nestedErrors) &&
1968
- nestedErrors.some(isRuntimeReceiptPersistenceError)
1969
- );
1970
- }
1971
-
1972
- function isRunFatalWorkerRowError(error: unknown): boolean {
1973
- return (
1974
- isCoordinatorProgressAuthorizationError(error) ||
1975
- isRowIsolationExemptError(error) ||
1976
- isRuntimeReceiptPersistenceError(error) ||
1977
- isRuntimePersistenceCircuitOpenError(error)
1978
- );
1979
- }
1980
-
1981
- /**
1982
- * completeReceipt salvage retry ladder. On a persistence failure after a billed
1983
- * call we retry the receipt completion past the server-side 5s connect window:
1984
- * 1 initial attempt + 3 retries at 1s / 3s / 9s (single isolate — no jitter).
1985
- * Bounded by the run deadline (`runtimeDeadlineMs`); we stop early unless there
1986
- * is at least one backoff's worth plus headroom of budget left.
1987
- */
1988
- const COMPLETE_RECEIPT_RETRY_BACKOFFS_MS = [1_000, 3_000, 9_000] as const;
1989
- const COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS = 10_000;
1990
-
1991
- // Fallback batch-chunk parallelism when a tool declares no provider rate hints.
1992
- // Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
1993
- // their previous batching behavior; declared providers tighten via the
1994
- // Governor's suggestedParallelism.
1995
- const WORKER_TOOL_BATCH_DEFAULT_PARALLELISM = 4;
1996
-
1997
- /**
1998
- * In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
1999
- * throughput pacing (provider or Deepline limiter), not a tool defect, so it
2000
- * gets more patience than the generic 3-attempt budget: with retry-after-aware
2001
- * escalating delays (capped at 5s) this absorbs roughly 25s of sustained
2002
- * throttling before the call fails. Every retry still charges the Governor's
2003
- * retry budget, so a runaway storm stays bounded and loud.
2004
- */
2005
-
2006
- function sleepWorkerMs(ms: number): Promise<void> {
2007
- return new Promise((resolve) => setTimeout(resolve, ms));
2008
- }
2009
-
2010
- function workerDurableToolCallCacheKey(input: {
2011
- req: RunRequest;
2012
- toolId: string;
2013
- requestInput: Record<string, unknown>;
2014
- providerActionVersion: string;
2015
- staleAfterSeconds?: number | null;
2016
- }): string {
2017
- return buildDurableToolCallCacheKey({
2018
- orgId: input.req.orgId,
2019
- playId: input.req.playName,
2020
- toolId: input.toolId,
2021
- requestInput: input.requestInput,
2022
- authScopeDigest: buildDurableToolCallAuthScopeDigest({
2023
- orgId: input.req.orgId,
2024
- userEmail: input.req.userEmail,
2025
- toolId: input.toolId,
2026
- }),
2027
- providerActionVersion: input.providerActionVersion,
2028
- staleAfterSeconds: input.staleAfterSeconds,
2029
- });
2030
- }
2031
-
2032
- function workerRuntimeReceiptKey(input: {
2033
- req: RunRequest;
2034
- key: string;
2035
- }): string {
2036
- const orgId = input.req.orgId?.trim() || 'org';
2037
- if (input.key.startsWith(`ctx:${orgId}:`)) {
2038
- return input.key;
2039
- }
2040
- return buildScopedWorkReceiptKey({
2041
- orgId: input.req.orgId,
2042
- playName: input.req.playName,
2043
- key: input.key,
2044
- });
2045
- }
2046
-
2047
- function stepProgramColumnName(parentField: string, stepId: string): string {
2048
- return sqlSafePlayColumnName(`${parentField}.${stepId}`);
2049
- }
2050
-
2051
- class WorkerToolBatchScheduler {
2052
- private queue: WorkerToolBatchRequest[] = [];
2053
- private scheduled = false;
2054
-
2055
- constructor(
2056
- private readonly req: RunRequest,
2057
- private readonly governor: PlayExecutionGovernor,
2058
- private readonly resolvePacing: WorkerPacingResolver,
2059
- private readonly resolveToolActionCacheVersion: WorkerToolActionCacheVersionResolver,
2060
- private readonly abortSignal?: AbortSignal,
2061
- private readonly onRequestsSettled?: (count: number) => void,
2062
- private readonly callbacks?: WorkerCtxCallbacks,
2063
- private readonly receiptStore?: WorkerRuntimeReceiptStore,
2064
- private readonly allowLocalRetryReceipts = false,
2065
- private readonly runtimeDeadlineMs?: number,
2066
- // Map-wide (or ctx-wide for the root scheduler) persistence-failure circuit
2067
- // breaker. Shared by every row worker so the first persistence failure
2068
- // halts NEW provider dispatch across the whole map.
2069
- private readonly latch: RuntimePersistenceLatch = createRuntimePersistenceLatch(),
2070
- // Run-scoped buffer for orphaned billed results whose receipt could not be
2071
- // completed even after the retry ladder.
2072
- private readonly salvage?: ReceiptSalvageBuffer,
2073
- ) {}
2074
-
2075
- /**
2076
- * Run `completeReceipt`/`completeReceipts` with the salvage retry ladder.
2077
- * Retries a receipt-completion failure past the server-side 5s connect window
2078
- * so a transient blip does not orphan a billed result. Bounded by the run
2079
- * deadline; skips retries when the latch has already burned a ladder in this
2080
- * map so many in-flight orphans do not each serialize a long loop.
2081
- */
2082
- private async completeReceiptWithRetryLadder<T>(
2083
- run: () => Promise<T>,
2084
- onGiveUp?: (attempts: number) => void,
2085
- ): Promise<T> {
2086
- let attempt = 0;
2087
- // A ladder that has already started retrying keeps its full budget even if
2088
- // a peer trips the latch mid-flight — only FRESH entrants collapse once the
2089
- // failure is confirmed. This is what guarantees the first ladder rides out
2090
- // a genuine transient blip.
2091
- let committedToFullLadder = false;
2092
- const giveUp = (error: unknown): never => {
2093
- this.latch.latchRetryBudgetExhausted = true;
2094
- onGiveUp?.(attempt);
2095
- throw error;
2096
- };
2097
- while (true) {
2098
- try {
2099
- return await run();
2100
- } catch (error) {
2101
- attempt += 1;
2102
- if (
2103
- shouldShortenCompleteReceiptLadder(this.latch, committedToFullLadder) ||
2104
- attempt > COMPLETE_RECEIPT_RETRY_BACKOFFS_MS.length ||
2105
- this.abortSignal?.aborted
2106
- ) {
2107
- return giveUp(error);
2108
- }
2109
- const backoffMs = COMPLETE_RECEIPT_RETRY_BACKOFFS_MS[attempt - 1]!;
2110
- // resolveRuntimeDeadlineRemainingMs THROWS WorkflowAbortError once the
2111
- // run deadline has passed — the correct loud outcome (there is no
2112
- // separate per-chunk step timeout). Route that throw through giveUp so
2113
- // the billed result is still salvaged and the ladder budget flag is
2114
- // set: a billed output must never lose its durable trace because the
2115
- // clock ran out mid-ladder.
2116
- let remainingMs: number;
2117
- try {
2118
- remainingMs = resolveRuntimeDeadlineRemainingMs(
2119
- this.runtimeDeadlineMs,
2120
- );
2121
- } catch (deadlineError) {
2122
- return giveUp(deadlineError);
2123
- }
2124
- if (remainingMs <= backoffMs + COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS) {
2125
- return giveUp(error);
2126
- }
2127
- // Past every give-up gate: this ladder is now committed to the full
2128
- // budget and must not be collapsed by a peer tripping the latch mid-sleep.
2129
- committedToFullLadder = true;
2130
- await sleepWorkerMs(backoffMs);
2131
- }
2132
- }
2133
- }
2134
-
2135
- /**
2136
- * Record a billed-but-unpersisted tool output into the run-scoped salvage
2137
- * buffer. Called only after the completeReceipt ladder is exhausted, with the
2138
- * exact serialized payload a receipt would have stored.
2139
- */
2140
- private recordReceiptSalvage(input: {
2141
- claimed: ClaimedWorkerToolBatchRequest;
2142
- output: unknown;
2143
- attempts: number;
2144
- }): void {
2145
- if (!this.salvage || !input.claimed.receiptKey) return;
2146
- this.salvage.push({
2147
- receiptKey: input.claimed.receiptKey,
2148
- toolId: input.claimed.request.toolId,
2149
- cacheKey: input.claimed.request.cacheKey,
2150
- output: input.output,
2151
- completeAttempts: input.attempts,
2152
- firstErrorAt: nowMs(),
2153
- });
2154
- }
2155
-
2156
- /**
2157
- * Report a provider 429 / Retry-After back into the Governor's shared pacer
2158
- * so future acquires for this (org, provider) bucket back off across all
2159
- * isolates. Provider comes from the same pacing resolver the Governor uses
2160
- * (the worker has no local catalog), so callers pass only the toolId.
2161
- */
2162
- private reportBackpressure(toolId: string, retryAfterMs: number): void {
2163
- if (retryAfterMs <= 0) return;
2164
- void (async () => {
2165
- const pacing = await this.resolvePacing(toolId).catch(() => null);
2166
- if (pacing?.provider) {
2167
- this.governor.reportProviderBackpressure({
2168
- provider: pacing.provider,
2169
- retryAfterMs,
2170
- });
2171
- }
2172
- })();
2173
- }
2174
-
2175
- async execute(
2176
- id: string,
2177
- toolId: string,
2178
- input: Record<string, unknown>,
2179
- workflowStep?: WorkflowStep,
2180
- options?: { force?: boolean; staleAfterSeconds?: number | null },
2181
- runtimeOptions?: { timeoutMs?: number; receiptWaitMs?: number },
2182
- ): Promise<unknown> {
2183
- const providerActionVersion =
2184
- await this.resolveToolActionCacheVersion(toolId);
2185
- return await new Promise((resolve, reject) => {
2186
- const queuedAt = nowMs();
2187
- const receiptKey = workerDurableToolCallCacheKey({
2188
- req: this.req,
2189
- toolId,
2190
- requestInput: input,
2191
- providerActionVersion,
2192
- staleAfterSeconds: options?.staleAfterSeconds,
2193
- });
2194
- const queryResultDatasetRead = isQueryResultDatasetReadRequest(
2195
- toolId,
2196
- input,
2197
- );
2198
- const receiptEligible = !queryResultDatasetRead;
2199
- this.queue.push({
2200
- id,
2201
- cacheKey: receiptEligible ? receiptKey : '',
2202
- receiptKey: receiptEligible ? receiptKey : null,
2203
- force: options?.force === true || !!this.req.force,
2204
- receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
2205
- typeof runtimeOptions?.receiptWaitMs === 'number'
2206
- ? { max_wait_ms: runtimeOptions.receiptWaitMs }
2207
- : input,
2208
- ),
2209
- ...(typeof runtimeOptions?.timeoutMs === 'number'
2210
- ? { runtimeTimeoutMs: runtimeOptions.timeoutMs }
2211
- : {}),
2212
- toolId,
2213
- input,
2214
- workflowStep,
2215
- ...(queryResultDatasetRead
2216
- ? {
2217
- directOptions: {
2218
- customerDbDataset: {
2219
- offset: 0,
2220
- pageSize: QUERY_RESULT_DATASET_PAGE_SIZE,
2221
- },
2222
- },
2223
- }
2224
- : {}),
2225
- resolve: (value) => {
2226
- recordRunnerPerfTrace({
2227
- req: this.req,
2228
- phase: 'runner.tool.request',
2229
- ms: nowMs() - queuedAt,
2230
- extra: { id, toolId },
2231
- });
2232
- resolve(value);
2233
- },
2234
- reject,
2235
- });
2236
- this.scheduleDrain();
2237
- });
2238
- }
2239
-
2240
- private scheduleDrain(): void {
2241
- if (this.scheduled) return;
2242
- this.scheduled = true;
2243
- queueMicrotask(async () => {
2244
- if (this.hasBatchableQueuedTool()) {
2245
- await new Promise((resolve) =>
2246
- setTimeout(resolve, WORKER_TOOL_BATCH_GRACE_MS),
2247
- );
2248
- }
2249
- void this.drain();
2250
- });
2251
- }
2252
-
2253
- private hasBatchableQueuedTool(): boolean {
2254
- return this.queue.some(
2255
- (request) =>
2256
- request.toolId !== 'test_wait_for_event' &&
2257
- getPlayRuntimeBatchStrategy(request.toolId) !== null,
2258
- );
2259
- }
2260
-
2261
- private async drain(): Promise<void> {
2262
- const requests = this.queue;
2263
- this.queue = [];
2264
- this.scheduled = false;
2265
- const drainStartedAt = nowMs();
2266
- await Promise.all(
2267
- [...groupWorkerToolRequestsByTool(requests).entries()].map(
2268
- async ([toolId, groupedRequests]) => {
2269
- await this.executeToolGroup(toolId, groupedRequests);
2270
- },
2271
- ),
2272
- );
2273
- recordRunnerPerfTrace({
2274
- req: this.req,
2275
- phase: 'runner.tool.drain',
2276
- ms: nowMs() - drainStartedAt,
2277
- extra: { requests: requests.length },
2278
- });
2279
- if (this.queue.length > 0) {
2280
- this.scheduleDrain();
2281
- }
2282
- }
2283
-
2284
- private async waitForDurableToolReceipt(input: {
2285
- receiptKey: string;
2286
- maxAttempts: number;
2287
- }): Promise<WorkerRuntimeReceipt> {
2288
- if (!this.receiptStore?.getReceipt) {
2289
- throw new Error(
2290
- 'Worker durable tool receipt wait requires receipt lookup.',
2291
- );
2292
- }
2293
- const receipt = await waitForCompletedRuntimeReceipt({
2294
- receiptKey: input.receiptKey,
2295
- maxAttempts: input.maxAttempts,
2296
- abortSignal: this.abortSignal,
2297
- store: {
2298
- getMany: async (receiptKeys) => {
2299
- const receipts = new Map<string, RuntimeStepReceipt>();
2300
- await Promise.all(
2301
- receiptKeys.map(async (key) => {
2302
- const receipt = await this.receiptStore!.getReceipt!({
2303
- playName: this.req.playName,
2304
- key,
2305
- });
2306
- if (!receipt) return;
2307
- receipts.set(key, {
2308
- key: receipt.key,
2309
- status: receipt.status,
2310
- output: receipt.output,
2311
- error: receipt.error ?? undefined,
2312
- runId: receipt.runId ?? null,
2313
- });
2314
- }),
2315
- );
2316
- return receipts;
2317
- },
2318
- },
2319
- });
2320
- return receipt;
2321
- }
2322
-
2323
- private settleRequests(
2324
- claimed: ClaimedWorkerToolBatchRequest,
2325
- result: unknown,
2326
- ): void {
2327
- claimed.request.resolve(result);
2328
- for (const follower of claimed.followers) {
2329
- follower.resolve(
2330
- claimed.receiptKey
2331
- ? markWorkerToolReceiptResultExecution(result, {
2332
- kind: 'in_flight',
2333
- receiptKey: claimed.receiptKey,
2334
- attachedToReceiptKey: claimed.receiptKey,
2335
- })
2336
- : result,
2337
- );
2338
- }
2339
- this.onRequestsSettled?.(1 + claimed.followers.length);
2340
- }
2341
-
2342
- private rejectRequests(
2343
- claimed: ClaimedWorkerToolBatchRequest,
2344
- error: unknown,
2345
- ): void {
2346
- claimed.request.reject(error);
2347
- for (const follower of claimed.followers) {
2348
- follower.reject(error);
2349
- }
2350
- this.onRequestsSettled?.(1 + claimed.followers.length);
2351
- }
2352
-
2353
- private rejectRawRequests(
2354
- requests: WorkerToolBatchRequest[],
2355
- error: unknown,
2356
- ): void {
2357
- for (const request of requests) {
2358
- request.reject(error);
2359
- }
2360
- this.onRequestsSettled?.(requests.length);
2361
- }
2362
-
2363
- private async resolveCompletedDurableToolReceiptGroup(input: {
2364
- group: WorkerToolBatchRequest[];
2365
- receiptKey: string;
2366
- receipt: WorkerRuntimeReceipt;
2367
- source: 'cache' | 'in_flight';
2368
- }): Promise<void> {
2369
- const [first] = input.group;
2370
- if (!first) return;
2371
- const value = deserializeDurableStepValue(input.receipt.output);
2372
- const result =
2373
- input.source === 'cache'
2374
- ? markWorkerToolReceiptResultCached(
2375
- value,
2376
- first.cacheKey,
2377
- input.receiptKey,
2378
- )
2379
- : markWorkerToolReceiptResultExecution(value, {
2380
- kind: 'in_flight',
2381
- receiptKey: input.receiptKey,
2382
- attachedToReceiptKey: input.receiptKey,
2383
- });
2384
- for (const request of input.group) {
2385
- request.resolve(result);
2386
- }
2387
- this.onRequestsSettled?.(input.group.length);
2388
- }
2389
-
2390
- private async reclaimRunningDurableToolReceiptGroupAfterWait(input: {
2391
- group: WorkerToolBatchRequest[];
2392
- receiptKey: string;
2393
- runningReceipt: WorkerRuntimeReceipt;
2394
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
2395
- const [request, ...followers] = input.group;
2396
- if (!request || !this.receiptStore) {
2397
- this.rejectRawRequests(
2398
- input.group,
2399
- new RuntimeReceiptWaitTimeoutError(input.receiptKey),
2400
- );
2401
- return [];
2402
- }
2403
- // A `running` receipt left by a DIFFERENT run whose last write is stale will
2404
- // never be completed by a live worker. Polling it burns one service-binding
2405
- // subrequest per tick, up to the receipt's wait budget (~1320) — many such
2406
- // orphans on a resume exhaust Cloudflare's per-invocation subrequest budget
2407
- // and kill the run. The orchestrator reclaims those immediately (skipping the
2408
- // poll) and re-executes, matching the "on resume it reclaims and re-executes"
2409
- // contract; live/fresh/same-run owners still poll before reclaiming. Kept as
2410
- // an injected helper so the skip-poll decision is unit-testable without
2411
- // loading this cloudflare:workers Worker entrypoint.
2412
- return await reclaimRunningReceiptGroupAfterWait<ClaimedWorkerToolBatchRequest>(
2413
- {
2414
- ownership: {
2415
- ownerRunId: input.runningReceipt.runId,
2416
- currentRunId: this.req.runId,
2417
- updatedAt: input.runningReceipt.updatedAt,
2418
- },
2419
- timeoutError: new RuntimeReceiptWaitTimeoutError(input.receiptKey),
2420
- poll: async () => {
2421
- try {
2422
- const receipt = await this.waitForDurableToolReceipt({
2423
- receiptKey: input.receiptKey,
2424
- maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
2425
- input.group,
2426
- (groupRequest) => groupRequest.receiptWaitMaxAttempts,
2427
- ),
2428
- });
2429
- await this.resolveCompletedDurableToolReceiptGroup({
2430
- group: input.group,
2431
- receiptKey: input.receiptKey,
2432
- receipt,
2433
- source: 'in_flight',
2434
- });
2435
- return { kind: 'completed' };
2436
- } catch (error) {
2437
- if (error instanceof RuntimeReceiptWaitTimeoutError) {
2438
- return { kind: 'timed_out', error };
2439
- }
2440
- return { kind: 'errored', error };
2441
- }
2442
- },
2443
- reclaim: (rejectionOnFallthrough) =>
2444
- this.forceReclaimRunningDurableToolReceiptGroup({
2445
- group: input.group,
2446
- request,
2447
- followers,
2448
- receiptKey: input.receiptKey,
2449
- rejectionOnFallthrough,
2450
- }),
2451
- reject: (error) => this.rejectRawRequests(input.group, error),
2452
- },
2453
- );
2454
- }
2455
-
2456
- /**
2457
- * Steal a `running` receipt (reclaimRunning) and route by claim disposition.
2458
- * Shared by the post-poll-timeout reclaim and the dead-owner fast path, so
2459
- * both dispose of a reclaimed/reused/failed receipt identically.
2460
- */
2461
- private async forceReclaimRunningDurableToolReceiptGroup(input: {
2462
- group: WorkerToolBatchRequest[];
2463
- request: WorkerToolBatchRequest;
2464
- followers: WorkerToolBatchRequest[];
2465
- receiptKey: string;
2466
- rejectionOnFallthrough: unknown;
2467
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
2468
- if (!this.receiptStore) {
2469
- this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
2470
- return [];
2471
- }
2472
- let claim: WorkerRuntimeReceiptClaim;
2473
- try {
2474
- claim = await this.receiptStore.claimReceipt({
2475
- playName: this.req.playName,
2476
- runId: this.req.runId,
2477
- key: input.receiptKey,
2478
- reclaimRunning: true,
2479
- });
2480
- } catch (error) {
2481
- this.rejectRawRequests(input.group, error);
2482
- return [];
2483
- }
2484
- if (claim.disposition === 'claimed') {
2485
- return [
2486
- {
2487
- request: input.request,
2488
- receiptKey: input.receiptKey,
2489
- followers: input.followers,
2490
- },
2491
- ];
2492
- }
2493
- if (claim.disposition === 'reused') {
2494
- await this.resolveCompletedDurableToolReceiptGroup({
2495
- group: input.group,
2496
- receiptKey: input.receiptKey,
2497
- receipt: claim.receipt,
2498
- source: 'cache',
2499
- });
2500
- return [];
2501
- }
2502
- if (claim.disposition === 'failed') {
2503
- return await this.reclaimFailedDurableToolReceiptGroup({
2504
- group: input.group,
2505
- receiptKey: input.receiptKey,
2506
- failedReceipt: claim.receipt,
2507
- });
2508
- }
2509
- this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
2510
- return [];
2511
- }
2512
-
2513
- private async reclaimFailedDurableToolReceiptGroup(input: {
2514
- group: WorkerToolBatchRequest[];
2515
- receiptKey: string;
2516
- failedReceipt: WorkerRuntimeReceipt;
2517
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
2518
- const failedError = new Error(
2519
- `Durable tool call ${input.receiptKey} failed: ${input.failedReceipt.error ?? 'unknown error'}`,
2520
- );
2521
- const [request, ...followers] = input.group;
2522
- if (!request || !this.receiptStore) {
2523
- this.rejectRawRequests(input.group, failedError);
2524
- return [];
2525
- }
2526
- if (
2527
- !canReclaimFailedWorkerToolReceipt({
2528
- error: input.failedReceipt.error,
2529
- })
2530
- ) {
2531
- this.rejectRawRequests(input.group, failedError);
2532
- return [];
2533
- }
2534
- let claim: WorkerRuntimeReceiptClaim;
2535
- try {
2536
- claim = await this.receiptStore.claimReceipt({
2537
- playName: this.req.playName,
2538
- runId: this.req.runId,
2539
- key: input.receiptKey,
2540
- });
2541
- } catch (error) {
2542
- this.rejectRawRequests(input.group, error);
2543
- return [];
2544
- }
2545
- if (claim.disposition === 'claimed') {
2546
- return [{ request, receiptKey: input.receiptKey, followers }];
2547
- }
2548
- if (claim.disposition === 'reused') {
2549
- await this.resolveCompletedDurableToolReceiptGroup({
2550
- group: input.group,
2551
- receiptKey: input.receiptKey,
2552
- receipt: claim.receipt,
2553
- source: 'cache',
2554
- });
2555
- return [];
2556
- }
2557
- if (claim.disposition === 'running') {
2558
- try {
2559
- const receipt = await this.waitForDurableToolReceipt({
2560
- receiptKey: input.receiptKey,
2561
- maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
2562
- input.group,
2563
- (groupRequest) => groupRequest.receiptWaitMaxAttempts,
2564
- ),
2565
- });
2566
- await this.resolveCompletedDurableToolReceiptGroup({
2567
- group: input.group,
2568
- receiptKey: input.receiptKey,
2569
- receipt,
2570
- source: 'in_flight',
2571
- });
2572
- } catch (error) {
2573
- this.rejectRawRequests(input.group, error);
2574
- }
2575
- return [];
2576
- }
2577
- this.rejectRawRequests(
2578
- input.group,
2579
- new Error(
2580
- `Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
2581
- ),
2582
- );
2583
- return [];
2584
- }
2585
-
2586
- private async failureForRejectedToolRequest(
2587
- claimed: ClaimedWorkerToolBatchRequest,
2588
- error: unknown,
2589
- ): Promise<unknown> {
2590
- try {
2591
- await this.failDurableToolRequest(claimed, error);
2592
- return error;
2593
- } catch (receiptError) {
2594
- tripRuntimePersistenceLatch(this.latch, receiptError);
2595
- return new RuntimeReceiptPersistenceError(
2596
- 'Tool call failed and durable receipt could not be marked failed',
2597
- [error, receiptError],
2598
- );
2599
- }
2600
- }
2601
-
2602
- private async claimDurableToolReceiptGroups(
2603
- groups: ReturnType<
2604
- typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
2605
- >['durableGroups'],
2606
- ): Promise<
2607
- Array<{
2608
- group: ReturnType<
2609
- typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
2610
- >['durableGroups'][number];
2611
- receiptKey: string;
2612
- claim: WorkerRuntimeReceiptClaim;
2613
- }>
2614
- > {
2615
- if (!this.receiptStore) return [];
2616
- const planned = groups.map((group) => ({
2617
- group,
2618
- receiptKey: workerRuntimeReceiptKey({
2619
- req: this.req,
2620
- key: group.claimableReceiptKey,
2621
- }),
2622
- }));
2623
- const claimOne = async (entry: (typeof planned)[number]) => ({
2624
- ...entry,
2625
- claim: await this.receiptStore!.claimReceipt({
2626
- playName: this.req.playName,
2627
- runId: this.req.runId,
2628
- key: entry.receiptKey,
2629
- ...(entry.group.forceDurableRefresh
2630
- ? { forceRefresh: true, reclaimRunning: true }
2631
- : {}),
2632
- }),
2633
- });
2634
- if (!this.receiptStore.claimReceipts) {
2635
- return await Promise.all(planned.map(claimOne));
2636
- }
2637
- const claimed: Array<{
2638
- group: (typeof planned)[number]['group'];
2639
- receiptKey: string;
2640
- claim: WorkerRuntimeReceiptClaim;
2641
- }> = [];
2642
- for (const forceDurableRefresh of [false, true]) {
2643
- const entries = planned.filter(
2644
- (entry) => entry.group.forceDurableRefresh === forceDurableRefresh,
2645
- );
2646
- if (entries.length === 0) continue;
2647
- const claims = await this.receiptStore.claimReceipts({
2648
- playName: this.req.playName,
2649
- runId: this.req.runId,
2650
- keys: entries.map((entry) => entry.receiptKey),
2651
- ...(forceDurableRefresh
2652
- ? { forceRefresh: true, reclaimRunning: true }
2653
- : {}),
2654
- });
2655
- entries.forEach((entry, index) => {
2656
- const claim = claims[index];
2657
- if (!claim) {
2658
- throw new Error(
2659
- `Runtime receipt batch claim did not return receipt ${entry.receiptKey}.`,
2660
- );
2661
- }
2662
- claimed.push({ ...entry, claim });
2663
- });
2664
- }
2665
- return claimed;
2666
- }
2667
-
2668
- private async prepareDurableToolRequests(
2669
- requests: WorkerToolBatchRequest[],
2670
- ): Promise<PreparedWorkerToolBatchRequests> {
2671
- if (!this.receiptStore) {
2672
- return {
2673
- claimedRequests: requests.map((request) => ({
2674
- request,
2675
- receiptKey: null,
2676
- followers: [],
2677
- })),
2678
- deferredClaimedRequests: [],
2679
- };
2680
- }
2681
-
2682
- const claimedRequests: ClaimedWorkerToolBatchRequest[] = [];
2683
- const deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[] =
2684
- [];
2685
- const receiptGroupPlan = planWorkerToolReceiptGroups(requests, {
2686
- allowLocalRetryReceipts: this.allowLocalRetryReceipts,
2687
- getReceiptInput: (request) => ({
2688
- durableReceiptKey: request.receiptKey,
2689
- localRetryCacheKey: request.cacheKey,
2690
- force: request.force,
2691
- }),
2692
- });
2693
- for (const request of receiptGroupPlan.localRequests) {
2694
- claimedRequests.push({
2695
- request,
2696
- receiptKey: null,
2697
- followers: [],
2698
- });
2699
- }
2700
-
2701
- try {
2702
- const claimEntries = await this.claimDurableToolReceiptGroups(
2703
- receiptGroupPlan.durableGroups,
2704
- );
2705
- for (const { group: groupState, receiptKey, claim } of claimEntries) {
2706
- const group = groupState.requests;
2707
- const [first] = group;
2708
- if (!first) continue;
2709
- if (claim.disposition === 'reused') {
2710
- const result = markWorkerToolReceiptResultCached(
2711
- deserializeDurableStepValue(claim.receipt.output),
2712
- first.cacheKey,
2713
- receiptKey,
2714
- );
2715
- for (const request of group) {
2716
- request.resolve(result);
2717
- }
2718
- this.onRequestsSettled?.(group.length);
2719
- continue;
2720
- }
2721
- if (claim.disposition === 'failed') {
2722
- claimedRequests.push(
2723
- ...(await this.reclaimFailedDurableToolReceiptGroup({
2724
- group,
2725
- receiptKey,
2726
- failedReceipt: claim.receipt,
2727
- })),
2728
- );
2729
- continue;
2730
- }
2731
- if (claim.disposition === 'running') {
2732
- deferredClaimedRequests.push(
2733
- this.reclaimRunningDurableToolReceiptGroupAfterWait({
2734
- group,
2735
- receiptKey,
2736
- runningReceipt: claim.receipt,
2737
- }),
2738
- );
2739
- continue;
2740
- }
2741
- const [request, ...followers] = group;
2742
- if (!request) continue;
2743
- claimedRequests.push({ request, receiptKey, followers });
2744
- }
2745
- } catch (error) {
2746
- // A receipt claim failure means the tenant Postgres is already
2747
- // unwritable: trip the breaker so remaining groups/rows stop dispatching.
2748
- tripRuntimePersistenceLatch(this.latch, error);
2749
- for (const group of receiptGroupPlan.durableGroups) {
2750
- this.rejectRawRequests(group.requests, error);
2751
- }
2752
- }
2753
- return { claimedRequests, deferredClaimedRequests };
2754
- }
2755
-
2756
- private async completeDurableToolRequest(
2757
- claimed: ClaimedWorkerToolBatchRequest,
2758
- result: unknown,
2759
- ): Promise<unknown> {
2760
- if (!this.receiptStore || !claimed.receiptKey) {
2761
- return result;
2762
- }
2763
- const ownerResult = markWorkerToolReceiptResultExecution(result, {
2764
- kind: 'live',
2765
- receiptKey: claimed.receiptKey,
2766
- });
2767
- const serializedOutput = serializeDurableStepValue(ownerResult);
2768
- const completed = await this.completeReceiptWithRetryLadder(
2769
- () =>
2770
- this.receiptStore!.completeReceipt({
2771
- playName: this.req.playName,
2772
- runId: this.req.runId,
2773
- key: claimed.receiptKey!,
2774
- output: serializedOutput,
2775
- }),
2776
- // Ladder exhausted: the call billed but the receipt could not land.
2777
- // Salvage the exact serialized output onto the run record before the
2778
- // caller turns this into a RuntimeReceiptPersistenceError.
2779
- (attempts) =>
2780
- this.recordReceiptSalvage({
2781
- claimed,
2782
- output: serializedOutput,
2783
- attempts,
2784
- }),
2785
- );
2786
- if (
2787
- completed &&
2788
- (completed.status === 'completed' || completed.status === 'skipped') &&
2789
- completed.output !== undefined
2790
- ) {
2791
- const recovered = deserializeDurableStepValue(completed.output);
2792
- return completed.runId && completed.runId !== this.req.runId
2793
- ? markWorkerToolReceiptResultCached(
2794
- recovered,
2795
- claimed.request.cacheKey,
2796
- claimed.receiptKey,
2797
- )
2798
- : recovered;
2799
- }
2800
- return ownerResult;
2801
- }
2802
-
2803
- private completedDurableToolRequestResult(input: {
2804
- claimed: ClaimedWorkerToolBatchRequest;
2805
- ownerResult: unknown;
2806
- completed: WorkerRuntimeReceipt | null | undefined;
2807
- }): unknown {
2808
- const { claimed, completed, ownerResult } = input;
2809
- if (
2810
- completed &&
2811
- (completed.status === 'completed' || completed.status === 'skipped') &&
2812
- completed.output !== undefined
2813
- ) {
2814
- const recovered = deserializeDurableStepValue(completed.output);
2815
- return completed.runId && completed.runId !== this.req.runId
2816
- ? markWorkerToolReceiptResultCached(
2817
- recovered,
2818
- claimed.request.cacheKey,
2819
- claimed.receiptKey ?? claimed.request.cacheKey,
2820
- )
2821
- : recovered;
2822
- }
2823
- return ownerResult;
2824
- }
2825
-
2826
- private async completeDurableToolRequests(
2827
- entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
2828
- ): Promise<unknown[]> {
2829
- const results: unknown[] = new Array(entries.length);
2830
- const durableEntries: Array<{
2831
- index: number;
2832
- claimed: ClaimedWorkerToolBatchRequest;
2833
- ownerResult: unknown;
2834
- }> = [];
2835
- for (let index = 0; index < entries.length; index += 1) {
2836
- const entry = entries[index]!;
2837
- if (!this.receiptStore || !entry.claimed.receiptKey) {
2838
- results[index] = entry.result;
2839
- continue;
2840
- }
2841
- const ownerResult = markWorkerToolReceiptResultExecution(entry.result, {
2842
- kind: 'live',
2843
- receiptKey: entry.claimed.receiptKey,
2844
- });
2845
- durableEntries.push({ index, claimed: entry.claimed, ownerResult });
2846
- }
2847
- if (durableEntries.length === 0) {
2848
- return results;
2849
- }
2850
- if (!this.receiptStore?.completeReceipts) {
2851
- await Promise.all(
2852
- durableEntries.map(async (entry) => {
2853
- results[entry.index] = await this.completeDurableToolRequest(
2854
- entry.claimed,
2855
- entries[entry.index]!.result,
2856
- );
2857
- }),
2858
- );
2859
- return results;
2860
- }
2861
- const serializedByEntry = durableEntries.map((entry) =>
2862
- serializeDurableStepValue(entry.ownerResult),
2863
- );
2864
- const completed = await this.completeReceiptWithRetryLadder(
2865
- () =>
2866
- this.receiptStore!.completeReceipts!({
2867
- playName: this.req.playName,
2868
- receipts: durableEntries.map((entry, entryIndex) => ({
2869
- runId: this.req.runId,
2870
- key: entry.claimed.receiptKey!,
2871
- output: serializedByEntry[entryIndex],
2872
- })),
2873
- }),
2874
- // Ladder exhausted: every entry in this batch billed but could not be
2875
- // marked completed. Salvage each serialized output before the caller
2876
- // turns this into a RuntimeReceiptPersistenceError.
2877
- (attempts) => {
2878
- durableEntries.forEach((entry, entryIndex) => {
2879
- this.recordReceiptSalvage({
2880
- claimed: entry.claimed,
2881
- output: serializedByEntry[entryIndex],
2882
- attempts,
2883
- });
2884
- });
2885
- },
2886
- );
2887
- durableEntries.forEach((entry, resultIndex) => {
2888
- results[entry.index] = this.completedDurableToolRequestResult({
2889
- claimed: entry.claimed,
2890
- ownerResult: entry.ownerResult,
2891
- completed: completed[resultIndex],
2892
- });
2893
- });
2894
- return results;
2895
- }
2896
-
2897
- private async failDurableToolRequest(
2898
- claimed: ClaimedWorkerToolBatchRequest,
2899
- error: unknown,
2900
- ): Promise<void> {
2901
- if (!this.receiptStore || !claimed.receiptKey) {
2902
- return;
2903
- }
2904
- await this.receiptStore.failReceipt({
2905
- playName: this.req.playName,
2906
- runId: this.req.runId,
2907
- key: claimed.receiptKey,
2908
- error: error instanceof Error ? error.message : String(error),
2909
- });
2910
- }
2911
-
2912
- private async failDurableToolRequests(
2913
- claimedRequests: ClaimedWorkerToolBatchRequest[],
2914
- error: unknown,
2915
- ): Promise<void> {
2916
- const durable = claimedRequests.filter(
2917
- (claimed) => this.receiptStore && claimed.receiptKey,
2918
- );
2919
- if (durable.length === 0) return;
2920
- if (!this.receiptStore?.failReceipts) {
2921
- await Promise.all(
2922
- durable.map((claimed) => this.failDurableToolRequest(claimed, error)),
2923
- );
2924
- return;
2925
- }
2926
- await this.receiptStore.failReceipts({
2927
- playName: this.req.playName,
2928
- receipts: durable.map((claimed) => ({
2929
- runId: this.req.runId,
2930
- key: claimed.receiptKey!,
2931
- error: error instanceof Error ? error.message : String(error),
2932
- })),
2933
- });
2934
- }
2935
-
2936
- private async executeToolGroup(
2937
- toolId: string,
2938
- requests: WorkerToolBatchRequest[],
2939
- ): Promise<void> {
2940
- // Circuit breaker, gated BEFORE the receipt claim: never-dispatched rows
2941
- // get NO receipt (not even `running`), so a re-run claims them fresh and
2942
- // executes them exactly once.
2943
- if (this.latch.tripped) {
2944
- this.latch.preventedCallCount += requests.length;
2945
- this.rejectRawRequests(
2946
- requests,
2947
- new RuntimePersistenceCircuitOpenError(this.latch),
2948
- );
2949
- return;
2950
- }
2951
- const { claimedRequests, deferredClaimedRequests } =
2952
- await this.prepareDurableToolRequests(requests);
2953
- await this.executeClaimedToolRequests(
2954
- toolId,
2955
- requests.length,
2956
- claimedRequests,
2957
- );
2958
- if (deferredClaimedRequests.length === 0) {
2959
- return;
2960
- }
2961
- const reclaimedRequests = (
2962
- await Promise.all(deferredClaimedRequests)
2963
- ).flat();
2964
- await this.executeClaimedToolRequests(
2965
- toolId,
2966
- requests.length,
2967
- reclaimedRequests,
2968
- );
2969
- }
2970
-
2971
- private async executeClaimedToolRequests(
2972
- toolId: string,
2973
- requestCount: number,
2974
- claimedRequests: ClaimedWorkerToolBatchRequest[],
2975
- ): Promise<void> {
2976
- if (claimedRequests.length === 0) {
2977
- return;
2978
- }
2979
- const strategy = getPlayRuntimeBatchStrategy(toolId);
2980
- if (
2981
- !strategy ||
2982
- toolId === 'test_wait_for_event' ||
2983
- claimedRequests.length < 2
2984
- ) {
2985
- const groupStartedAt = nowMs();
2986
- const dispatchOne = async (
2987
- claimed: ClaimedWorkerToolBatchRequest,
2988
- ): Promise<void> => {
2989
- const { request } = claimed;
2990
- // Circuit breaker: the latch may have tripped after this group was
2991
- // claimed but before this particular call started. Do NOT call the
2992
- // provider. This row has a `running` receipt (claimed, never
2993
- // executed); on resume it reclaims and re-executes — safe, it never
2994
- // billed.
2995
- if (this.latch.tripped) {
2996
- this.latch.preventedCallCount += 1 + claimed.followers.length;
2997
- this.rejectRequests(
2998
- claimed,
2999
- new RuntimePersistenceCircuitOpenError(this.latch),
3000
- );
3001
- return;
3002
- }
3003
- const toolContract = await this.resolvePacing(toolId).catch(
3004
- () => null,
3005
- );
3006
- // Each unbatched provider call takes its own tool slot: the Governor
3007
- // charges tool budget, holds a global tool-concurrency slot, and
3008
- // applies per-(org,provider) pacing before the call runs.
3009
- const slot = await this.governor.acquireToolSlot(toolId, {
3010
- signal: this.abortSignal,
3011
- });
3012
- try {
3013
- let result: unknown;
3014
- try {
3015
- result = await executeToolWithLifecycle(
3016
- this.req,
3017
- { id: request.id, toolId, input: request.input },
3018
- request.workflowStep,
3019
- this.callbacks,
3020
- (retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
3021
- () => this.governor.chargeBudget('retry'),
3022
- toolContract?.retrySafeTransientHttp === true,
3023
- this.abortSignal,
3024
- this.runtimeDeadlineMs,
3025
- resolveClaimedWorkerToolRuntimeTimeoutMs([claimed], {
3026
- runtimeDeadlineMs: this.runtimeDeadlineMs,
3027
- }),
3028
- request.directOptions,
3029
- );
3030
- } catch (error) {
3031
- this.rejectRequests(
3032
- claimed,
3033
- await this.failureForRejectedToolRequest(claimed, error),
3034
- );
3035
- return;
3036
- }
3037
- this.settleRequests(
3038
- claimed,
3039
- await this.completeDurableToolRequest(claimed, result),
3040
- );
3041
- } catch (receiptError) {
3042
- tripRuntimePersistenceLatch(this.latch, receiptError);
3043
- this.rejectRequests(
3044
- claimed,
3045
- new RuntimeReceiptPersistenceError(
3046
- 'Tool call succeeded but durable receipt could not be marked completed',
3047
- [receiptError],
3048
- ),
3049
- );
3050
- } finally {
3051
- slot.release();
3052
- }
3053
- };
3054
- // Bounded dispatch waves: an unbatched tool group can carry a whole chunk's
3055
- // rows (enrich compiles to one ~245-row chunk). Dispatching them all at
3056
- // once billed every call before the first receipt-completion failure could
3057
- // trip the latch (the 250/250 incident). Instead, dispatch one wave, AWAIT
3058
- // it so any completion failure trips the latch, then read the latch before
3059
- // the next wave — remaining waves are prevented (counted + rejected) exactly
3060
- // like the mid-wave gate above. The wave calls stay fully concurrent; the
3061
- // only added cost on the healthy path is one latch read per wave.
3062
- for (const wave of partitionDispatchWaves(
3063
- claimedRequests,
3064
- () => 1,
3065
- PROVIDER_DISPATCH_WAVE_SIZE,
3066
- )) {
3067
- if (this.latch.tripped) {
3068
- for (const claimed of wave) {
3069
- this.latch.preventedCallCount += 1 + claimed.followers.length;
3070
- this.rejectRequests(
3071
- claimed,
3072
- new RuntimePersistenceCircuitOpenError(this.latch),
3073
- );
3074
- }
3075
- continue;
3076
- }
3077
- await Promise.all(wave.map(dispatchOne));
3078
- }
3079
- recordRunnerPerfTrace({
3080
- req: this.req,
3081
- phase: 'runner.tool.group',
3082
- ms: nowMs() - groupStartedAt,
3083
- extra: {
3084
- toolId,
3085
- requests: requestCount,
3086
- executed: claimedRequests.length,
3087
- batched: false,
3088
- },
3089
- });
3090
- return;
3091
- }
2289
+ onOutput?: (output: RecordedStepProgramOutput) => void | Promise<void>;
2290
+ };
3092
2291
 
3093
- const batchStartedAt = nowMs();
3094
- await executeBatchedWorkerToolGroup({
3095
- req: this.req,
3096
- requests: claimedRequests,
3097
- strategy,
3098
- governor: this.governor,
3099
- suggestedParallelism: await this.governor.suggestedParallelism(
3100
- toolId,
3101
- WORKER_TOOL_BATCH_DEFAULT_PARALLELISM,
3102
- ),
3103
- abortSignal: this.abortSignal,
3104
- runtimeDeadlineMs: this.runtimeDeadlineMs,
3105
- reportBackpressure: (retryAfterMs) =>
3106
- this.reportBackpressure(toolId, retryAfterMs),
3107
- resolveToolContract: this.resolvePacing,
3108
- onRequestsSettled: this.onRequestsSettled,
3109
- callbacks: this.callbacks,
3110
- completeRequests: async (entries) =>
3111
- await this.completeDurableToolRequests(entries),
3112
- failRequests: async (claimedRequests, error) =>
3113
- await this.failDurableToolRequests(claimedRequests, error),
3114
- settleRequest: (claimed, result) => this.settleRequests(claimed, result),
3115
- rejectRequest: (claimed, error) => this.rejectRequests(claimed, error),
3116
- latch: this.latch,
3117
- });
3118
- recordRunnerPerfTrace({
3119
- req: this.req,
3120
- phase: 'runner.tool.group',
3121
- ms: nowMs() - batchStartedAt,
3122
- extra: {
3123
- toolId,
3124
- requests: requestCount,
3125
- executed: claimedRequests.length,
3126
- batched: true,
3127
- },
3128
- });
3129
- }
2292
+ type WorkerStepResolution = {
2293
+ value: unknown;
2294
+ status?: 'skipped';
2295
+ };
2296
+
2297
+ function stepProgramColumnName(parentField: string, stepId: string): string {
2298
+ return sqlSafePlayColumnName(`${parentField}.${stepId}`);
3130
2299
  }
3131
2300
 
3132
- function groupWorkerToolRequestsByTool(
3133
- requests: WorkerToolBatchRequest[],
3134
- ): Map<string, WorkerToolBatchRequest[]> {
3135
- const byTool = new Map<string, WorkerToolBatchRequest[]>();
3136
- for (const request of requests) {
3137
- const existing = byTool.get(request.toolId);
3138
- if (existing) {
3139
- existing.push(request);
3140
- } else {
3141
- byTool.set(request.toolId, [request]);
3142
- }
2301
+ const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
2302
+ const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
2303
+ const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
2304
+ const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 1_000;
2305
+ const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
2306
+ /**
2307
+ * Bounded number of per-row failure samples carried in chunk summaries and the
2308
+ * map's terminal partial-failure log. Every failed row is persisted with its
2309
+ * full error in the runtime sheet; the samples just keep run logs readable.
2310
+ */
2311
+ const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
2312
+
2313
+ /**
2314
+ * Bounded-concurrent map-chunk dispatch fan-out for the workers_edge profile
2315
+ * (in-process play execution with no `workflowStep`). On workers_edge the whole
2316
+ * map shares ONE per-invocation subrequest window, so sequential chunk dispatch
2317
+ * buys no durability — it is pure wall-clock cost. Running up to K chunks in
2318
+ * flight overlaps provider batches, sheet writes, and the per-chunk persist
2319
+ * barrier. K is capped at 3 to bound peak memory: each in-flight chunk holds up
2320
+ * to ~5000 rows plus their row buffers, so 3 concurrent chunks is the ceiling
2321
+ * we accept before row payloads risk pressuring the isolate. The native
2322
+ * Cloudflare Workflows path (workflowStep present) keeps K=1: there each chunk
2323
+ * is a durable step and sequential dispatch is what makes replay/resume sound.
2324
+ */
2325
+ const WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS = 3;
2326
+ const WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES =
2327
+ WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES;
2328
+ const WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES = 128;
2329
+
2330
+ const workerMapResidentRowBytes = (row: unknown): number =>
2331
+ new TextEncoder().encode(JSON.stringify(row)).byteLength +
2332
+ WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES;
2333
+
2334
+ function isRuntimeReceiptPersistenceError(error: unknown): boolean {
2335
+ if (!error || typeof error !== 'object') return false;
2336
+ if (error instanceof RuntimeReceiptPersistenceError) return true;
2337
+ if (
2338
+ error instanceof Error &&
2339
+ error.name === 'RuntimeReceiptPersistenceError'
2340
+ ) {
2341
+ return true;
3143
2342
  }
3144
- return byTool;
2343
+ const nestedErrors = (error as { errors?: unknown }).errors;
2344
+ return (
2345
+ Array.isArray(nestedErrors) &&
2346
+ nestedErrors.some(isRuntimeReceiptPersistenceError)
2347
+ );
3145
2348
  }
3146
2349
 
3147
- async function executeBatchedWorkerToolGroup(input: {
3148
- req: RunRequest;
3149
- requests: ClaimedWorkerToolBatchRequest[];
3150
- strategy: AnyBatchOperationStrategy;
3151
- governor: PlayExecutionGovernor;
3152
- suggestedParallelism: number;
3153
- abortSignal?: AbortSignal;
3154
- runtimeDeadlineMs?: number;
3155
- reportBackpressure: (retryAfterMs: number) => void;
3156
- resolveToolContract: WorkerPacingResolver;
3157
- onRequestsSettled?: (count: number) => void;
3158
- callbacks?: WorkerCtxCallbacks;
3159
- completeRequests: (
3160
- entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
3161
- ) => Promise<unknown[]>;
3162
- failRequests: (
3163
- requests: ClaimedWorkerToolBatchRequest[],
3164
- error: unknown,
3165
- ) => Promise<void>;
3166
- settleRequest: (
3167
- request: ClaimedWorkerToolBatchRequest,
3168
- result: unknown,
3169
- ) => void;
3170
- rejectRequest: (
3171
- request: ClaimedWorkerToolBatchRequest,
3172
- error: unknown,
3173
- ) => void;
3174
- latch: RuntimePersistenceLatch;
3175
- }): Promise<void> {
3176
- const compiledBatches = compileRequestsWithStrategy({
3177
- requests: input.requests,
3178
- strategy: input.strategy,
3179
- getPayload: (request) => request.request.input,
3180
- });
3181
- recordRunnerPerfTrace({
3182
- req: input.req,
3183
- phase: 'runner.tool.batch.compile',
3184
- ms: 0,
3185
- extra: {
3186
- sourceOperation: input.strategy.sourceOperation,
3187
- batchOperation: input.strategy.batchOperation,
3188
- requests: input.requests.length,
3189
- batches: compiledBatches.length,
3190
- batchSizes: compiledBatches.map((batch) => batch.memberRequests.length),
3191
- },
3192
- });
2350
+ function isRunFatalWorkerRowError(error: unknown): boolean {
2351
+ return (
2352
+ isCoordinatorProgressAuthorizationError(error) ||
2353
+ isRowIsolationExemptError(error) ||
2354
+ isRuntimeReceiptPersistenceError(error) ||
2355
+ isRuntimePersistenceCircuitOpenError(error) ||
2356
+ error instanceof RuntimeSheetAttemptLeaseLostError ||
2357
+ error instanceof RuntimeLeaseLostError ||
2358
+ (error instanceof Error &&
2359
+ error.name === 'RuntimeSheetAttemptLeaseLostError')
2360
+ );
2361
+ }
3193
2362
 
3194
- await executeChunkedRequests({
3195
- requests: compiledBatches,
3196
- // Chunk parallelism is the Governor's per-tool suggestion (provider rate
3197
- // hints tightened to the policy ceiling), bounded by the batch count.
3198
- batchSize: Math.max(
3199
- 1,
3200
- Math.min(input.suggestedParallelism, compiledBatches.length || 1),
3201
- ),
3202
- // Bounded dispatch wave: cap the provider calls (counted by batch member
3203
- // size) a single chunk dispatches before the latch is read again, so a
3204
- // persistence failure prevents the rest of a large batch group instead of
3205
- // billing it all. A single over-budget batch still forms its own wave.
3206
- maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
3207
- weightOf: (batch) => batch.memberRequests.length,
3208
- execute: async (batch) => {
3209
- // Circuit breaker: once a persistence failure has occurred in this map,
3210
- // do NOT dispatch this batch's provider call. The chunk plumbing rejects
3211
- // its member requests with the latch error.
3212
- if (input.latch.tripped) {
3213
- input.latch.preventedCallCount += batch.memberRequests.length;
3214
- throw new RuntimePersistenceCircuitOpenError(input.latch);
3215
- }
3216
- const toolContract = await input
3217
- .resolveToolContract(batch.batchOperation)
3218
- .catch(() => null);
3219
- // One provider call per batch → one tool slot (budget + global
3220
- // concurrency + per-(org,provider) pacing) around the whole batch.
3221
- const slot = await input.governor.acquireToolSlot(batch.batchOperation, {
3222
- signal: input.abortSignal,
3223
- });
3224
- try {
3225
- input.callbacks?.onToolCalled?.(batch.batchOperation, nowMs());
3226
- return await executeTool(
3227
- input.req,
3228
- {
3229
- id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
3230
- toolId: batch.batchOperation,
3231
- input: batch.batchPayload,
3232
- },
3233
- undefined,
3234
- input.reportBackpressure,
3235
- () => input.governor.chargeBudget('retry'),
3236
- toolContract?.retrySafeTransientHttp === true,
3237
- input.abortSignal,
3238
- input.runtimeDeadlineMs,
3239
- resolveClaimedWorkerToolRuntimeTimeoutMs(batch.memberRequests, {
3240
- runtimeDeadlineMs: input.runtimeDeadlineMs,
3241
- }),
3242
- );
3243
- } catch (error) {
3244
- input.callbacks?.onToolFailed?.(batch.batchOperation, nowMs());
3245
- throw error;
3246
- } finally {
3247
- slot.release();
3248
- }
3249
- },
3250
- onChunkComplete: async (
3251
- chunkResults: Array<
3252
- ChunkExecutionResult<(typeof compiledBatches)[number], unknown>
3253
- >,
3254
- ) => {
3255
- for (const entry of chunkResults) {
3256
- if (entry.error !== undefined) {
3257
- // One batch's provider error stays scoped to that batch's member
3258
- // requests. Sibling batches in this chunk keep their results so a
3259
- // single provider hiccup cannot cascade into a whole-map failure.
3260
- let rejection: unknown = entry.error;
3261
- try {
3262
- await input.failRequests(entry.request.memberRequests, entry.error);
3263
- } catch (receiptError) {
3264
- tripRuntimePersistenceLatch(input.latch, receiptError);
3265
- rejection = new RuntimeReceiptPersistenceError(
3266
- 'Tool call failed and durable receipts could not be marked failed',
3267
- [entry.error, receiptError],
3268
- );
3269
- }
3270
- for (const claimed of entry.request.memberRequests) {
3271
- input.rejectRequest(claimed, rejection);
3272
- }
3273
- continue;
3274
- }
3275
- const batchResult = isToolExecuteResult(entry.result)
3276
- ? entry.result.toolOutput.raw
3277
- : entry.result;
3278
- const splitResults =
3279
- batchResult != null
3280
- ? entry.request.splitResults(batchResult)
3281
- : entry.request.memberRequests.map(() => null);
3282
- let completedResults: unknown[];
3283
- try {
3284
- completedResults = await input.completeRequests(
3285
- entry.request.memberRequests.map((claimed, index) => ({
3286
- claimed,
3287
- result: wrapWorkerToolResult(
3288
- claimed.request.toolId,
3289
- splitResults[index] ?? null,
3290
- toolMetadataFallback(claimed.request.toolId),
3291
- splitResults[index] == null ? 'no_result' : 'completed',
3292
- {
3293
- req: input.req,
3294
- input: claimed.request.input,
3295
- },
3296
- ),
3297
- })),
3298
- );
3299
- } catch (receiptError) {
3300
- tripRuntimePersistenceLatch(input.latch, receiptError);
3301
- const rejection = new RuntimeReceiptPersistenceError(
3302
- 'Tool call succeeded but durable receipts could not be marked completed',
3303
- [receiptError],
3304
- );
3305
- for (const claimed of entry.request.memberRequests) {
3306
- input.rejectRequest(claimed, rejection);
3307
- }
3308
- continue;
3309
- }
3310
- for (let index = 0; index < completedResults.length; index += 1) {
3311
- const claimed = entry.request.memberRequests[index]!;
3312
- const request = claimed.request;
3313
- input.settleRequest(claimed, completedResults[index]);
3314
- }
3315
- }
3316
- },
3317
- }).catch(async (error) => {
3318
- let rejection: unknown = error;
3319
- try {
3320
- await input.failRequests(input.requests, error);
3321
- } catch (receiptError) {
3322
- tripRuntimePersistenceLatch(input.latch, receiptError);
3323
- rejection = new RuntimeReceiptPersistenceError(
3324
- 'Tool call failed and durable receipts could not be marked failed',
3325
- [error, receiptError],
3326
- );
3327
- }
3328
- for (const claimed of input.requests) {
3329
- input.rejectRequest(claimed, rejection);
3330
- }
3331
- });
2363
+ /**
2364
+ * In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
2365
+ * throughput pacing (provider or Deepline limiter), not a tool defect, so it
2366
+ * gets more patience than the generic 3-attempt budget: with retry-after-aware
2367
+ * escalating delays (capped at 5s) this absorbs roughly 25s of sustained
2368
+ * throttling before the call fails. Every retry still charges the Governor's
2369
+ * retry budget, so a runaway storm stays bounded and loud.
2370
+ */
2371
+
2372
+ function sleepWorkerMs(ms: number): Promise<void> {
2373
+ return new Promise((resolve) => setTimeout(resolve, ms));
3332
2374
  }
3333
2375
 
3334
2376
  function isCompletedWorkerFieldValue(value: unknown): boolean {
@@ -3339,7 +2381,8 @@ function isCompletedWorkerFieldValue(value: unknown): boolean {
3339
2381
  );
3340
2382
  }
3341
2383
 
3342
- type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
2384
+ type WorkerMapCompletedChunkSummary<T extends Record<string, unknown>> = {
2385
+ status: 'completed';
3343
2386
  chunkIndex: number;
3344
2387
  rangeStart: number;
3345
2388
  rangeEnd: number;
@@ -3363,6 +2406,22 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
3363
2406
  cachedRows?: T[];
3364
2407
  };
3365
2408
 
2409
+ type WorkerMapBlockedChunkSummary = {
2410
+ status: 'blocked';
2411
+ chunkIndex: number;
2412
+ rangeStart: number;
2413
+ rangeEnd: number;
2414
+ rowsRead: number;
2415
+ rowsBlocked: number;
2416
+ rowKeySamples: string[];
2417
+ blockedRowSamples: Record<string, unknown>[];
2418
+ outputDatasetId: string;
2419
+ message: string;
2420
+ };
2421
+
2422
+ type WorkerMapChunkSummary<T extends Record<string, unknown>> =
2423
+ WorkerMapCompletedChunkSummary<T> | WorkerMapBlockedChunkSummary;
2424
+
3366
2425
  function serializeDurableStepValue<T>(value: T, depth = 0): T {
3367
2426
  if (depth > 20 || value == null) return value;
3368
2427
  if (isToolExecuteResult(value)) return serializeToolExecuteResult(value) as T;
@@ -3420,9 +2479,7 @@ type WorkerConditionalStepResolver = {
3420
2479
  type WorkerStepProgramStep = {
3421
2480
  name: string;
3422
2481
  resolver:
3423
- | WorkerStepResolver
3424
- | WorkerConditionalStepResolver
3425
- | WorkerStepProgram;
2482
+ WorkerStepResolver | WorkerConditionalStepResolver | WorkerStepProgram;
3426
2483
  };
3427
2484
 
3428
2485
  type WorkerStepProgram = {
@@ -3575,6 +2632,7 @@ async function executeWorkerStepProgram(
3575
2632
  ...(resolution.status ? { status: resolution.status } : {}),
3576
2633
  };
3577
2634
  };
2635
+
3578
2636
  const executeStep = async (): Promise<WorkerStepResolution> =>
3579
2637
  workflowStep
3580
2638
  ? await (
@@ -3799,7 +2857,7 @@ async function postRuntimeEgressFetch(
3799
2857
  ) {
3800
2858
  const response = await fetchRuntimeApi(
3801
2859
  req.baseUrl,
3802
- '/api/v2/plays/internal/egress-fetch',
2860
+ PLAY_RUNTIME_EGRESS_FETCH_COMPAT_PATH,
3803
2861
  {
3804
2862
  method: 'POST',
3805
2863
  headers: {
@@ -4358,9 +3416,7 @@ function resolveSheetContractFromReq(
4358
3416
  // `staticPipeline` field is what `resolveSheetContractForTableNamespace`
4359
3417
  // expects. Reach in safely.
4360
3418
  const snapshot = req.contractSnapshot as
4361
- | { staticPipeline?: unknown }
4362
- | undefined
4363
- | null;
3419
+ { staticPipeline?: unknown } | undefined | null;
4364
3420
  const pipeline = snapshot?.staticPipeline;
4365
3421
  if (!pipeline || typeof pipeline !== 'object' || Array.isArray(pipeline)) {
4366
3422
  return null;
@@ -4375,11 +3431,84 @@ function resolveSheetContractFromReq(
4375
3431
  }
4376
3432
  }
4377
3433
 
3434
+ /**
3435
+ * Run-fatal teardown: release the runtime-sheet attempt leases and work-receipt
3436
+ * leases this run still holds, so an immediate rerun proceeds without waiting
3437
+ * out the 10-minute lease TTL. Every release is attempt/owner fenced downstream,
3438
+ * so it can never weaken a different run's live lease. Failures are logged loud
3439
+ * and swallowed here — the caller still throws the original run-fatal error.
3440
+ */
3441
+ async function releaseRuntimeLeasesOnTeardown(input: {
3442
+ req: RunRequest;
3443
+ leasedSheetNamespaces: Set<string>;
3444
+ emit: (event: RunnerEvent) => void;
3445
+ }): Promise<void> {
3446
+ const { req, leasedSheetNamespaces, emit } = input;
3447
+ const scope = runtimeSheetSessionScope(req);
3448
+ for (const tableNamespace of leasedSheetNamespaces) {
3449
+ const sheetContract = resolveSheetContractFromReq(req, tableNamespace);
3450
+ if (!sheetContract) continue;
3451
+ try {
3452
+ const released = await harnessReleaseSheetAttempt({
3453
+ ...scope,
3454
+ tableNamespace,
3455
+ sheetContract,
3456
+ runId: req.runId,
3457
+ attemptOwnerRunId: req.runId,
3458
+ attemptSeq: req.runAttempt ?? 0,
3459
+ });
3460
+ if (released.released > 0) {
3461
+ emit({
3462
+ type: 'log',
3463
+ level: 'info',
3464
+ message: `Released ${released.released} runtime sheet row lease(s) for ctx.dataset("${tableNamespace}") on run-fatal teardown.`,
3465
+ ts: nowMs(),
3466
+ });
3467
+ }
3468
+ } catch (releaseError) {
3469
+ emit({
3470
+ type: 'log',
3471
+ level: 'warn',
3472
+ message: `Runtime sheet attempt release failed for ctx.dataset("${tableNamespace}") on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
3473
+ releaseError instanceof Error
3474
+ ? releaseError.message
3475
+ : String(releaseError)
3476
+ }`,
3477
+ ts: nowMs(),
3478
+ });
3479
+ }
3480
+ }
3481
+ try {
3482
+ const released = await harnessReleaseRuntimeReceipts({
3483
+ ...scope,
3484
+ runId: req.runId,
3485
+ runAttempt: req.runAttempt ?? 0,
3486
+ });
3487
+ if (released.released > 0) {
3488
+ emit({
3489
+ type: 'log',
3490
+ level: 'info',
3491
+ message: `Released ${released.released} work-receipt lease(s) on run-fatal teardown.`,
3492
+ ts: nowMs(),
3493
+ });
3494
+ }
3495
+ } catch (releaseError) {
3496
+ emit({
3497
+ type: 'log',
3498
+ level: 'warn',
3499
+ message: `Work-receipt release failed on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
3500
+ releaseError instanceof Error
3501
+ ? releaseError.message
3502
+ : String(releaseError)
3503
+ }`,
3504
+ ts: nowMs(),
3505
+ });
3506
+ }
3507
+ }
3508
+
4378
3509
  function staticPipelineFromReq(req: RunRequest): PlayStaticPipeline | null {
4379
3510
  const snapshot = req.contractSnapshot as
4380
- | { staticPipeline?: unknown }
4381
- | undefined
4382
- | null;
3511
+ { staticPipeline?: unknown } | undefined | null;
4383
3512
  const pipeline = snapshot?.staticPipeline;
4384
3513
  if (!pipeline || typeof pipeline !== 'object' || Array.isArray(pipeline)) {
4385
3514
  return null;
@@ -4417,6 +3546,13 @@ async function persistCompletedMapRows(input: {
4417
3546
  rows: Record<string, unknown>[];
4418
3547
  outputFields: string[];
4419
3548
  extraOutputFields?: string[];
3549
+ attemptId?: string | null;
3550
+ attemptOwnerRunId?: string | null;
3551
+ attemptExpiresAt?: string | null;
3552
+ attemptSeq?: number | null;
3553
+ budgetMeter?: WorkerWorkBudgetMeter;
3554
+ force?: boolean;
3555
+ forceFailedRows?: boolean;
4420
3556
  }): Promise<{
4421
3557
  rows: number;
4422
3558
  written: number;
@@ -4462,23 +3598,28 @@ async function persistCompletedMapRows(input: {
4462
3598
  sheetContract,
4463
3599
  rows,
4464
3600
  outputFields,
3601
+ attemptId: input.attemptId ?? null,
3602
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3603
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3604
+ attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
3605
+ forceFailedRows: input.forceFailedRows === true,
4465
3606
  };
3607
+ input.budgetMeter?.count('sheet');
4466
3608
  const expectedKeys = [
4467
3609
  ...new Set(
4468
3610
  input.rows
4469
- .map((row) => resolveMapRowOutcomeKey(row))
4470
- .filter((key): key is string => Boolean(key)),
3611
+ .map((row) => resolveMapRowOutcomeKey(row) ?? row.key)
3612
+ .filter(Boolean),
4471
3613
  ),
4472
- ];
4473
- const expectedVisibleRows =
4474
- expectedKeys.length > 0 ? expectedKeys.length : rows.length;
3614
+ ] as string[];
3615
+ const expectedVisibleRows = expectedKeys.length || rows.length;
4475
3616
  const readVisibleRowCount = async () => {
4476
3617
  if (expectedKeys.length > 0) {
4477
3618
  const result = await harnessReadSheetDatasetRowKeys({
4478
3619
  ...sessionScope,
4479
3620
  tableNamespace,
4480
3621
  runId: req.runId,
4481
- rowMode: 'all',
3622
+ rowMode: 'terminalAnyRun',
4482
3623
  keys: expectedKeys,
4483
3624
  });
4484
3625
  return result.keys.length;
@@ -4494,6 +3635,24 @@ async function persistCompletedMapRows(input: {
4494
3635
  return result.rows.length;
4495
3636
  };
4496
3637
  const result = await harnessPersistCompletedSheetRows(persistRequest);
3638
+ const hasAttemptFence = Boolean(input.attemptId?.trim());
3639
+ const assertAttemptStillOwnsTerminalWrites = (
3640
+ written: number,
3641
+ visible: number | null,
3642
+ ) => {
3643
+ // Recovery finalization may include rows completed by an earlier attempt.
3644
+ // A complete terminal readback proves they were not lost.
3645
+ if (
3646
+ !hasAttemptFence ||
3647
+ written === rows.length ||
3648
+ (visible !== null && visible >= expectedVisibleRows)
3649
+ ) {
3650
+ return;
3651
+ }
3652
+ throw new Error(
3653
+ `Runtime sheet attempt lost ownership for ${tableNamespace}: wrote ${written}/${rows.length}; run ${req.runId}; attempt ${input.attemptId}.`,
3654
+ );
3655
+ };
4497
3656
  console.warn('[play-runner.persist_completed_map_rows.result]', {
4498
3657
  runId: req.runId,
4499
3658
  playName: req.playName,
@@ -4518,11 +3677,19 @@ async function persistCompletedMapRows(input: {
4518
3677
  sheetContract,
4519
3678
  rows: input.rows.map((row) => runtimeCsvStorageRow(row)),
4520
3679
  runId: req.runId,
3680
+ attemptId: input.attemptId ?? null,
3681
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3682
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3683
+ attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
3684
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
4521
3685
  inputOffset: 0,
4522
3686
  });
3687
+ input.budgetMeter?.count('sheet');
4523
3688
  const retry = await harnessPersistCompletedSheetRows(persistRequest);
3689
+ input.budgetMeter?.count('sheet');
4524
3690
  retryWritten = retry.rowsWritten;
4525
3691
  retryVisible = await readVisibleRowCount();
3692
+ assertAttemptStillOwnsTerminalWrites(retryWritten, retryVisible);
4526
3693
  if (retryVisible >= expectedVisibleRows) {
4527
3694
  return {
4528
3695
  rows: rows.length,
@@ -4536,6 +3703,7 @@ async function persistCompletedMapRows(input: {
4536
3703
  `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}/${expectedVisibleRows}; run ${req.runId}.`,
4537
3704
  );
4538
3705
  }
3706
+ assertAttemptStillOwnsTerminalWrites(result.rowsWritten, visibleRows);
4539
3707
  }
4540
3708
  if (result.rowsWritten !== rows.length && visibleRows < expectedVisibleRows) {
4541
3709
  throw new Error(
@@ -4557,8 +3725,14 @@ async function applyLiveMapRowUpdates(input: {
4557
3725
  updates: Array<Omit<PlayRowUpdate, 'rowId'> & { runId?: string }>;
4558
3726
  outputFields: string[];
4559
3727
  extraOutputFields?: string[];
3728
+ attemptId?: string | null;
3729
+ attemptOwnerRunId?: string | null;
3730
+ attemptExpiresAt?: string | null;
3731
+ attemptSeq?: number | null;
3732
+ budgetMeter?: WorkerWorkBudgetMeter;
4560
3733
  }): Promise<void> {
4561
3734
  if (input.updates.length === 0) return;
3735
+ input.budgetMeter?.count('sheet');
4562
3736
  const outputFields = [
4563
3737
  ...input.outputFields,
4564
3738
  ...(input.extraOutputFields ?? []).filter(
@@ -4581,7 +3755,16 @@ async function applyLiveMapRowUpdates(input: {
4581
3755
  contractSnapshot: input.req.contractSnapshot,
4582
3756
  runId: input.req.runId,
4583
3757
  userEmail: input.req.userEmail,
4584
- updates: input.updates,
3758
+ updates: input.updates.map((update) => ({
3759
+ ...update,
3760
+ attemptId: update.attemptId ?? input.attemptId ?? null,
3761
+ attemptOwnerRunId:
3762
+ update.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
3763
+ attemptExpiresAt:
3764
+ update.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
3765
+ attemptSeq:
3766
+ update.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
3767
+ })),
4585
3768
  },
4586
3769
  );
4587
3770
  }
@@ -4592,18 +3775,39 @@ async function prepareMapRows(input: {
4592
3775
  rows: Record<string, unknown>[];
4593
3776
  inputOffset: number;
4594
3777
  outputFields: string[];
3778
+ attemptId?: string | null;
3779
+ attemptOwnerRunId?: string | null;
3780
+ attemptExpiresAt?: string | null;
3781
+ attemptSeq?: number | null;
3782
+ budgetMeter?: WorkerWorkBudgetMeter;
4595
3783
  force?: boolean;
4596
3784
  }): Promise<{
4597
3785
  inserted: number;
4598
3786
  skipped: number;
4599
3787
  pendingRows: Record<string, unknown>[];
4600
3788
  completedRows: Record<string, unknown>[];
3789
+ blockedRows: Record<string, unknown>[];
3790
+ attemptId?: string | null;
3791
+ attemptOwnerRunId?: string | null;
3792
+ attemptExpiresAt?: string | null;
3793
+ attemptSeq?: number | null;
4601
3794
  }> {
4602
3795
  if (input.rows.length === 0) {
4603
- return { inserted: 0, skipped: 0, pendingRows: [], completedRows: [] };
3796
+ return {
3797
+ inserted: 0,
3798
+ skipped: 0,
3799
+ pendingRows: [],
3800
+ completedRows: [],
3801
+ blockedRows: [],
3802
+ attemptId: input.attemptId ?? null,
3803
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3804
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3805
+ attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
3806
+ };
4604
3807
  }
4605
3808
  const sessionScope = runtimeSheetSessionScope(input.req);
4606
3809
  const rows = input.rows.map((row) => runtimeCsvStorageRow(row));
3810
+ input.budgetMeter?.count('sheet');
4607
3811
  const result = await harnessStartSheetDataset({
4608
3812
  ...sessionScope,
4609
3813
  tableNamespace: input.tableNamespace,
@@ -4614,6 +3818,11 @@ async function prepareMapRows(input: {
4614
3818
  }),
4615
3819
  rows,
4616
3820
  inputOffset: input.inputOffset,
3821
+ attemptId: input.attemptId ?? null,
3822
+ attemptOwnerRunId: input.attemptOwnerRunId ?? null,
3823
+ attemptExpiresAt: input.attemptExpiresAt ?? null,
3824
+ attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
3825
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
4617
3826
  ...(input.force ? { force: true } : {}),
4618
3827
  });
4619
3828
  for (const timing of result.timings ?? []) {
@@ -4643,6 +3852,13 @@ async function prepareMapRows(input: {
4643
3852
  skipped: result.skipped,
4644
3853
  pendingRows: result.pendingRows,
4645
3854
  completedRows: result.completedRows,
3855
+ blockedRows: result.blockedRows ?? [],
3856
+ attemptId: result.attemptId ?? input.attemptId ?? null,
3857
+ attemptOwnerRunId:
3858
+ result.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
3859
+ attemptExpiresAt: result.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
3860
+ attemptSeq:
3861
+ result.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
4646
3862
  };
4647
3863
  }
4648
3864
 
@@ -4788,14 +4004,6 @@ function createCoordinatorRatePort(req: RunRequest): CoordinatorRatePort {
4788
4004
  * memoized per isolate. No hints → null (pacing is a no-op; the Governor's
4789
4005
  * global tool-concurrency slot still applies).
4790
4006
  */
4791
- type WorkerPacingResolver = (
4792
- toolId: string,
4793
- ) => Promise<
4794
- (ResolvedPacingPolicy & { retrySafeTransientHttp: boolean }) | null
4795
- >;
4796
-
4797
- type WorkerToolActionCacheVersionResolver = (toolId: string) => Promise<string>;
4798
-
4799
4007
  function createWorkerToolActionCacheVersionResolver(
4800
4008
  req: RunRequest,
4801
4009
  ): WorkerToolActionCacheVersionResolver {
@@ -4803,9 +4011,7 @@ function createWorkerToolActionCacheVersionResolver(
4803
4011
  return (toolId: string) => {
4804
4012
  const normalized = String(toolId || '').trim();
4805
4013
  if (!normalized) {
4806
- return Promise.reject(
4807
- new Error('Runtime tool metadata lookup requires a non-empty tool id.'),
4808
- );
4014
+ return Promise.reject(new Error('Tool metadata needs tool id.'));
4809
4015
  }
4810
4016
  const cached = cache.get(normalized);
4811
4017
  if (cached) return cached;
@@ -4842,6 +4048,44 @@ function createWorkerToolActionCacheVersionResolver(
4842
4048
  };
4843
4049
  }
4844
4050
 
4051
+ function createWorkerToolAuthScopeDigestResolver(req: RunRequest) {
4052
+ // Return type is inferred from createRuntimeToolAuthScopeDigestResolver
4053
+ // (RuntimeToolAuthScopeDigestResolver), whose `invalidate` member is
4054
+ // REQUIRED — the AUTH_SCOPE_CHANGED re-claim wiring below calls it directly,
4055
+ // and a resolver without eviction support must be a type error.
4056
+ return createRuntimeToolAuthScopeDigestResolver({
4057
+ missingToolIdMessage: 'Tool auth scope needs tool id.',
4058
+ fetchDigest: async (normalized) => {
4059
+ const runScopedPath =
4060
+ `/api/v2/plays/runtime-tools/${encodeURIComponent(normalized)}/auth-scope` +
4061
+ `?runId=${encodeURIComponent(req.runId)}`;
4062
+ const res = await fetchRuntimeApi(req.baseUrl, runScopedPath, {
4063
+ method: 'GET',
4064
+ headers: {
4065
+ authorization: `Bearer ${req.executorToken}`,
4066
+ 'cache-control': 'no-store',
4067
+ pragma: 'no-cache',
4068
+ },
4069
+ });
4070
+ if (!res.ok) {
4071
+ throw new Error(
4072
+ `Runtime tool auth scope lookup for ${normalized} failed (${res.status}): ${await res.text()}`,
4073
+ );
4074
+ }
4075
+ const body = (await res.json().catch(() => null)) as {
4076
+ digest?: unknown;
4077
+ } | null;
4078
+ const digest = typeof body?.digest === 'string' ? body.digest.trim() : '';
4079
+ if (!digest) {
4080
+ throw new Error(
4081
+ `Runtime tool auth scope lookup for ${normalized} returned no digest.`,
4082
+ );
4083
+ }
4084
+ return digest;
4085
+ },
4086
+ });
4087
+ }
4088
+
4845
4089
  function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
4846
4090
  const cache = new Map<
4847
4091
  string,
@@ -4983,13 +4227,22 @@ function createMinimalWorkerCtx(
4983
4227
  workflowStep?: WorkflowStep,
4984
4228
  abortSignal?: AbortSignal,
4985
4229
  callbacks?: WorkerCtxCallbacks,
4230
+ workBudgetMeter: WorkerWorkBudgetMeter = createWorkerWorkBudgetMeter({
4231
+ nowMs,
4232
+ }),
4986
4233
  runtimeDeadlineMs?: number,
4234
+ // Namespaces this run took a runtime-sheet attempt lease on. The run-fatal
4235
+ // teardown reads this set to release exactly those leases (never a namespace
4236
+ // whose table the run never created), so a rerun proceeds immediately.
4237
+ leasedSheetNamespaces?: Set<string>,
4987
4238
  receiptSalvage?: ReceiptSalvageBuffer,
4988
4239
  ): unknown {
4989
4240
  const { governor, resolvePacing: resolveToolPacing } =
4990
4241
  createGovernorForRun(req);
4991
4242
  const resolveToolActionCacheVersion =
4992
4243
  createWorkerToolActionCacheVersionResolver(req);
4244
+ const resolveToolAuthScopeDigest =
4245
+ createWorkerToolAuthScopeDigestResolver(req);
4993
4246
  // Play-call depth/count/per-parent budgets, child-play concurrency, and the
4994
4247
  // lineage snapshot are owned by the Governor (createGovernorForRun above).
4995
4248
  // The worker keeps only substrate mechanism here.
@@ -5000,7 +4253,7 @@ function createMinimalWorkerCtx(
5000
4253
  if (!auth) return {};
5001
4254
  const response = await fetchRuntimeApi(
5002
4255
  req.baseUrl,
5003
- '/api/v2/plays/internal/runtime',
4256
+ PLAY_RUNTIME_API_COMPAT_PATH,
5004
4257
  {
5005
4258
  method: 'POST',
5006
4259
  headers: {
@@ -5039,10 +4292,48 @@ function createMinimalWorkerCtx(
5039
4292
  orgId: req.orgId,
5040
4293
  preloadedDbSessions: req.preloadedDbSessions ?? null,
5041
4294
  userEmail: req.userEmail,
4295
+ runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null,
4296
+ runAttempt: req.runAttempt ?? 0,
4297
+ receiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
5042
4298
  });
4299
+ const executeDispatchedTool = async (input: {
4300
+ id: string;
4301
+ toolId: string;
4302
+ input: Record<string, unknown>;
4303
+ workflowStep?: unknown;
4304
+ callbacks?: WorkerCtxCallbacks;
4305
+ onProviderBackpressure?: (retryAfterMs: number) => void;
4306
+ onRetryAttempt?: () => void;
4307
+ transientHttpRetrySafe?: boolean;
4308
+ durableCallReceiptKey?: string | null;
4309
+ executionAuthScopeDigest?: string | null;
4310
+ providerIdempotencyKey?: string | null;
4311
+ runtimeTimeoutMs?: number;
4312
+ directOptions?: WorkerToolDirectOptions;
4313
+ }): Promise<ToolExecuteResult> =>
4314
+ await executeToolWithLifecycle(
4315
+ req,
4316
+ {
4317
+ id: input.id,
4318
+ toolId: input.toolId,
4319
+ input: input.input,
4320
+ durableCallReceiptKey: input.durableCallReceiptKey,
4321
+ executionAuthScopeDigest: input.executionAuthScopeDigest,
4322
+ providerIdempotencyKey: input.providerIdempotencyKey,
4323
+ },
4324
+ input.workflowStep as WorkflowStep | undefined,
4325
+ input.callbacks,
4326
+ input.onProviderBackpressure,
4327
+ input.onRetryAttempt,
4328
+ input.transientHttpRetrySafe === true,
4329
+ abortSignal,
4330
+ runtimeDeadlineMs,
4331
+ input.runtimeTimeoutMs,
4332
+ input.directOptions,
4333
+ );
5043
4334
  const executeWithRuntimeReceipt = async <T>(
5044
4335
  key: string,
5045
- execute: () => Promise<T> | T,
4336
+ execute: (context: WorkerRuntimeReceiptExecutionContext) => Promise<T> | T,
5046
4337
  options: {
5047
4338
  repairRunningReceiptForSameRun?: boolean;
5048
4339
  reclaimRunning?: boolean;
@@ -5056,7 +4347,8 @@ function createMinimalWorkerCtx(
5056
4347
  runId: req.runId,
5057
4348
  key,
5058
4349
  receiptStore,
5059
- execute: async () => serializeDurableStepValue(await execute()),
4350
+ execute: async (context) =>
4351
+ serializeDurableStepValue(await execute(context)),
5060
4352
  repairRunningReceiptForSameRun:
5061
4353
  options.repairRunningReceiptForSameRun ?? false,
5062
4354
  reclaimRunning: options.reclaimRunning ?? false,
@@ -5098,24 +4390,54 @@ function createMinimalWorkerCtx(
5098
4390
  staleAfterSeconds,
5099
4391
  });
5100
4392
  };
5101
- // Ctx-wide breaker for top-level ctx.tool calls (outside any map). Each map
5102
- // creates its own map-scoped latch (see runMap) so per-map dispatch stats and
5103
- // the runMap failure message stay map-local.
4393
+ const emitAuthScopeReclaimRunLog = (input: {
4394
+ toolId: string;
4395
+ requestIds: string[];
4396
+ }) => {
4397
+ // Durable run-log marker for the bounded AUTH_SCOPE_CHANGED re-claim
4398
+ // (mirrors the shared-runtime marker in shared_libs/play-runtime/context.ts
4399
+ // so run logs expose the credential transition on every execution path).
4400
+ emitEvent({
4401
+ type: 'log',
4402
+ level: 'info',
4403
+ message:
4404
+ `ctx.tools.execute(${input.toolId}): auth_scope_changed_reclaim ` +
4405
+ `(requests: ${input.requestIds.join(', ')}); credential identity ` +
4406
+ `changed mid-run, re-resolved the auth scope and re-claimed fresh ` +
4407
+ `receipts under the new scope.`,
4408
+ ts: nowMs(),
4409
+ });
4410
+ };
5104
4411
  const rootPersistenceLatch = createRuntimePersistenceLatch();
5105
- const rootToolBatchScheduler = new WorkerToolBatchScheduler(
4412
+ const rootToolBatchScheduler = new WorkerToolBatchScheduler({
5106
4413
  req,
5107
4414
  governor,
5108
- resolveToolPacing,
4415
+ resolvePacing: resolveToolPacing,
5109
4416
  resolveToolActionCacheVersion,
4417
+ resolveToolAuthScopeDigest,
4418
+ invalidateToolAuthScopeDigest: (toolId) =>
4419
+ resolveToolAuthScopeDigest.invalidate(toolId),
4420
+ onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
4421
+ executeTool: executeDispatchedTool,
4422
+ serializeDurableValue: serializeDurableStepValue,
4423
+ deserializeDurableValue: deserializeDurableStepValue,
4424
+ nowMs,
4425
+ batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
4426
+ recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
5110
4427
  abortSignal,
5111
- undefined,
5112
4428
  callbacks,
5113
4429
  receiptStore,
5114
- true,
4430
+ allowLocalRetryReceipts: true,
4431
+ runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
4432
+ runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
4433
+ req,
4434
+ runtimeReceiptLeaseTtlMs(req) ?? PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
4435
+ ),
4436
+ budgetMeter: workBudgetMeter,
5115
4437
  runtimeDeadlineMs,
5116
- rootPersistenceLatch,
4438
+ persistenceLatch: rootPersistenceLatch,
5117
4439
  receiptSalvage,
5118
- );
4440
+ });
5119
4441
  // Local ancestry chain that always ENDS with the currently-executing play
5120
4442
  // (req.playName). The /api/v2/plays/run lineage validator requires the
5121
4443
  // submitted ancestry's tail to equal the executor token's play name (i.e.
@@ -5150,23 +4472,33 @@ function createMinimalWorkerCtx(
5150
4472
  ): Promise<unknown> => {
5151
4473
  const mapStartedAt = nowMs();
5152
4474
  const mapNodeId = `map:${name}`;
5153
- // Map-wide persistence-failure circuit breaker. Shared across every chunk's
5154
- // scheduler (the scheduler is rebuilt per chunk) and the map's sheet-persist
5155
- // flush chain, so the first persistence failure halts NEW provider dispatch
5156
- // for the rest of the map and its prevented-call count surfaces in the
5157
- // runMap failure message below.
5158
- const persistenceLatch = createRuntimePersistenceLatch();
5159
4475
  const inputRows = rows;
5160
4476
  const rowCountHint = datasetRowCountHint(inputRows);
5161
4477
  const baseOffset = 0;
5162
4478
  const fieldEntries = Object.entries(fieldsDef);
5163
4479
  const plan = req.executionPlan;
5164
- const rowsPerChunk = chooseWorkerMapRowsPerChunk({
4480
+ const mapDispatchPlan = chooseWorkerMapDispatchPlan({
5165
4481
  mapName: name,
5166
4482
  rowCountHint,
5167
4483
  executionPlan: plan,
5168
4484
  staticPipeline: staticPipelineFromReq(req),
5169
4485
  });
4486
+ const rowsPerChunk = mapDispatchPlan.rowsPerChunk;
4487
+ recordRunnerPerfTrace({
4488
+ req,
4489
+ phase: 'runner.map_dispatch_plan',
4490
+ ms: 0,
4491
+ extra: {
4492
+ mapName: name,
4493
+ rowsPerChunk,
4494
+ rowCountHint,
4495
+ estimatedExternalStepsPerRow:
4496
+ mapDispatchPlan.estimatedExternalStepsPerRow,
4497
+ workEstimate: mapDispatchPlan.workEstimate,
4498
+ reason: mapDispatchPlan.reason,
4499
+ dynamicWorkerArtifactVersion: DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION,
4500
+ },
4501
+ });
5170
4502
  const outputFields = fieldEntries.map(([field]) => field);
5171
4503
  const updateMapProgress = (
5172
4504
  progress: LiveNodeProgressSnapshot,
@@ -5182,6 +4514,22 @@ function createMinimalWorkerCtx(
5182
4514
  forceFlush: options?.forceFlush ?? true,
5183
4515
  });
5184
4516
  };
4517
+ let queuedMapProgress: Promise<void> = Promise.resolve();
4518
+ let queuedMapProgressError: unknown = null;
4519
+ const enqueueMapProgressUpdate = (
4520
+ progress: LiveNodeProgressSnapshot,
4521
+ options?: { forceFlush?: boolean },
4522
+ ) => {
4523
+ queuedMapProgress = queuedMapProgress
4524
+ .then(() => Promise.resolve(updateMapProgress(progress, options)))
4525
+ .catch((error) => {
4526
+ queuedMapProgressError ??= error;
4527
+ });
4528
+ };
4529
+ const flushQueuedMapProgressUpdates = async () => {
4530
+ await queuedMapProgress;
4531
+ if (queuedMapProgressError) throw queuedMapProgressError;
4532
+ };
5185
4533
  const formatMapProgressMessage = (completed: number, total?: number) =>
5186
4534
  typeof total === 'number' && Number.isFinite(total) && total > 0
5187
4535
  ? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
@@ -5206,7 +4554,9 @@ function createMinimalWorkerCtx(
5206
4554
  return formatMapProgressMessage(completed, input.total);
5207
4555
  };
5208
4556
  const formatMapProcessingMessage = (rowsToExecute: number) =>
5209
- rowsToExecute > 0 ? `${rowsToExecute.toLocaleString()} rows` : null;
4557
+ rowsToExecute > 0
4558
+ ? `Processing ${rowsToExecute.toLocaleString()} rows`
4559
+ : null;
5210
4560
  const formatMapExecutionHeartbeatMessage = (input: {
5211
4561
  rowsToExecute: number;
5212
4562
  startedRows: number;
@@ -5220,7 +4570,7 @@ function createMinimalWorkerCtx(
5220
4570
  const waitingRows = Math.max(0, rowsToExecute - startedRows);
5221
4571
  const parts = [
5222
4572
  activeRows > 0 ? `${activeRows.toLocaleString()} active` : null,
5223
- waitingRows > 0 ? `${waitingRows.toLocaleString()} queued` : null,
4573
+ waitingRows > 0 ? `${waitingRows.toLocaleString()} waiting` : null,
5224
4574
  completedRows > 0 ? `${completedRows.toLocaleString()} done` : null,
5225
4575
  ].filter((part): part is string => Boolean(part));
5226
4576
  const base =
@@ -5384,6 +4734,12 @@ function createMinimalWorkerCtx(
5384
4734
  ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
5385
4735
  const chunkStartedAt = nowMs();
5386
4736
  assertNotAborted(abortSignal);
4737
+ const rowAttempt = makeRuntimeSheetAttempt({
4738
+ runId: req.runId,
4739
+ runAttempt: req.runAttempt,
4740
+ mapName: name,
4741
+ chunkIndex,
4742
+ });
5387
4743
  const keyStartedAt = nowMs();
5388
4744
  const chunkEntries = chunkRows.map((row, localIndex) => {
5389
4745
  const absoluteIndex = baseOffset + chunkStart + localIndex;
@@ -5411,8 +4767,21 @@ function createMinimalWorkerCtx(
5411
4767
  ...mapRowOutcomeRuntimeFields({ key: rowKey }),
5412
4768
  })),
5413
4769
  inputOffset: baseOffset + chunkStart,
4770
+ ...rowAttempt,
4771
+ budgetMeter: workBudgetMeter,
5414
4772
  force: !!req.force,
5415
4773
  });
4774
+ const activeRowAttempt = {
4775
+ attemptId: prepared.attemptId ?? rowAttempt.attemptId,
4776
+ attemptOwnerRunId:
4777
+ prepared.attemptOwnerRunId ?? rowAttempt.attemptOwnerRunId,
4778
+ attemptExpiresAt:
4779
+ prepared.attemptExpiresAt ?? rowAttempt.attemptExpiresAt,
4780
+ attemptSeq: prepared.attemptSeq ?? rowAttempt.attemptSeq,
4781
+ };
4782
+ // Record that this run holds attempt leases in `name`'s sheet so a
4783
+ // run-fatal teardown releases them (the table now exists).
4784
+ leasedSheetNamespaces?.add(name);
5416
4785
  recordRunnerPerfTrace({
5417
4786
  req,
5418
4787
  phase: 'runner.map_chunk.prepare_rows',
@@ -5425,14 +4794,15 @@ function createMinimalWorkerCtx(
5425
4794
  skipped: prepared.skipped,
5426
4795
  pendingRows: prepared.pendingRows.length,
5427
4796
  completedRows: prepared.completedRows.length,
4797
+ blockedRows: prepared.blockedRows.length,
5428
4798
  },
5429
4799
  });
5430
4800
  const progressTotalRows = rowCountHint ?? chunkRows.length;
5431
4801
  const preparedCompletedRows = Math.min(
5432
4802
  progressTotalRows,
5433
- totalRowsWritten + prepared.completedRows.length,
4803
+ totalRowsWritten,
5434
4804
  );
5435
- await updateMapProgress({
4805
+ enqueueMapProgressUpdate({
5436
4806
  completed: preparedCompletedRows,
5437
4807
  total: progressTotalRows,
5438
4808
  startedAt: mapStartedAt,
@@ -5445,6 +4815,9 @@ function createMinimalWorkerCtx(
5445
4815
  const pendingKeys = new Set<string>();
5446
4816
  const pendingRowsByKey = new Map<string, Record<string, unknown>>();
5447
4817
  const completedKeys = new Set<string>();
4818
+ const completedRowsByKey = new Map<string, Record<string, unknown>>();
4819
+ const blockedKeys = new Set<string>();
4820
+ const blockedRowsByKey = new Map<string, Record<string, unknown>>();
5448
4821
  const preparedKeys = new Set<string>();
5449
4822
  for (const row of prepared.pendingRows) {
5450
4823
  const key =
@@ -5462,14 +4835,95 @@ function createMinimalWorkerCtx(
5462
4835
  deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
5463
4836
  if (key) {
5464
4837
  completedKeys.add(key);
4838
+ completedRowsByKey.set(key, row);
4839
+ preparedKeys.add(key);
4840
+ }
4841
+ }
4842
+ for (const row of prepared.blockedRows) {
4843
+ const key =
4844
+ resolveMapRowOutcomeKey(row) ??
4845
+ deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
4846
+ if (key) {
4847
+ blockedKeys.add(key);
4848
+ blockedRowsByKey.set(key, row);
5465
4849
  preparedKeys.add(key);
5466
4850
  }
5467
4851
  }
4852
+ if (blockedKeys.size > 0) {
4853
+ const rowKeySamples = Array.from(blockedKeys).slice(
4854
+ 0,
4855
+ MAP_ROW_FAILURE_SAMPLE_LIMIT,
4856
+ );
4857
+ const message =
4858
+ `ctx.dataset("${name}") chunk ${chunkIndex} is blocked by ` +
4859
+ `${blockedKeys.size} active runtime sheet row lease(s) owned by another attempt. ` +
4860
+ `Retry after the owning attempt finishes or the row leases expire.` +
4861
+ (rowKeySamples.length > 0
4862
+ ? ` Blocked row keys: ${rowKeySamples.join(', ')}`
4863
+ : '');
4864
+ recordRunnerPerfTrace({
4865
+ req,
4866
+ phase: 'runner.map_chunk.blocked_rows',
4867
+ ms: nowMs() - prepareStartedAt,
4868
+ extra: {
4869
+ mapName: name,
4870
+ chunkIndex,
4871
+ rowsBlocked: blockedKeys.size,
4872
+ rowKeySamples,
4873
+ },
4874
+ });
4875
+ enqueueMapProgressUpdate({
4876
+ completed: preparedCompletedRows,
4877
+ total: progressTotalRows,
4878
+ waitingRows: blockedKeys.size,
4879
+ startedAt: mapStartedAt,
4880
+ message,
4881
+ });
4882
+ return {
4883
+ status: 'blocked',
4884
+ chunkIndex,
4885
+ rangeStart: baseOffset + chunkStart,
4886
+ rangeEnd: baseOffset + chunkStart + chunkRows.length,
4887
+ rowsRead: chunkRows.length,
4888
+ rowsBlocked: blockedKeys.size,
4889
+ rowKeySamples,
4890
+ blockedRowSamples: rowKeySamples.map((rowKey) => {
4891
+ const blockedRow = blockedRowsByKey.get(rowKey);
4892
+ const attemptId =
4893
+ typeof blockedRow?.__deeplineAttemptId === 'string'
4894
+ ? blockedRow.__deeplineAttemptId
4895
+ : typeof blockedRow?._attempt_id === 'string'
4896
+ ? blockedRow._attempt_id
4897
+ : activeRowAttempt.attemptId;
4898
+ return {
4899
+ __deeplineRowKey: rowKey,
4900
+ __deeplineAttemptId: attemptId,
4901
+ };
4902
+ }),
4903
+ outputDatasetId: `map:${name}`,
4904
+ message,
4905
+ };
4906
+ }
5468
4907
  const missingPreparedRows = chunkEntries.filter(
5469
4908
  ({ rowKey }) => !preparedKeys.has(rowKey),
5470
4909
  );
5471
- const rowsToExecuteEntries = chunkEntries.filter(
5472
- ({ rowKey }) => pendingKeys.has(rowKey) || !completedKeys.has(rowKey),
4910
+ if (missingPreparedRows.length > 0) {
4911
+ const rowKeySamples = missingPreparedRows
4912
+ .map(({ rowKey }) => rowKey)
4913
+ .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
4914
+ throw new Error(
4915
+ `ctx.dataset("${name}") runtime preparation omitted ${missingPreparedRows.length} row disposition(s). ` +
4916
+ `Every input row must be claimed, completed, or blocked before execution starts.` +
4917
+ (rowKeySamples.length > 0
4918
+ ? ` Missing row keys: ${rowKeySamples.join(', ')}`
4919
+ : ''),
4920
+ );
4921
+ }
4922
+ const rowsToExecuteEntries = chunkEntries.filter(({ rowKey }) =>
4923
+ pendingKeys.has(rowKey),
4924
+ );
4925
+ const completedChunkEntries = chunkEntries.filter(({ rowKey }) =>
4926
+ completedKeys.has(rowKey),
5473
4927
  );
5474
4928
  const uniqueRowsToExecuteEntries = [
5475
4929
  ...new Map(
@@ -5486,18 +4940,15 @@ function createMinimalWorkerCtx(
5486
4940
  rowsToExecute.length,
5487
4941
  );
5488
4942
  if (processingMessage) {
5489
- await updateMapProgress({
4943
+ enqueueMapProgressUpdate({
5490
4944
  completed: preparedCompletedRows,
5491
4945
  total: progressTotalRows,
5492
4946
  startedAt: mapStartedAt,
5493
4947
  message: processingMessage,
5494
4948
  });
5495
4949
  }
5496
- const rowsInserted = prepared.inserted + missingPreparedRows.length;
5497
- const rowsSkipped = Math.max(
5498
- 0,
5499
- prepared.skipped - missingPreparedRows.length,
5500
- );
4950
+ const rowsInserted = prepared.inserted;
4951
+ const rowsSkipped = prepared.skipped;
5501
4952
  let completedExecutedRows = 0;
5502
4953
  let failedExecutedRows = 0;
5503
4954
  let startedExecutedRows = 0;
@@ -5505,12 +4956,7 @@ function createMinimalWorkerCtx(
5505
4956
  let lastChunkProgressAt = 0;
5506
4957
  let lastExecutionHeartbeatAt = nowMs();
5507
4958
  const completedRowsForProgress = () =>
5508
- Math.min(
5509
- progressTotalRows,
5510
- totalRowsWritten +
5511
- prepared.completedRows.length +
5512
- completedExecutedRows,
5513
- );
4959
+ Math.min(progressTotalRows, totalRowsWritten + completedExecutedRows);
5514
4960
  const reportExecutionHeartbeat = (force = false) => {
5515
4961
  const now = nowMs();
5516
4962
  if (
@@ -5520,7 +4966,7 @@ function createMinimalWorkerCtx(
5520
4966
  return;
5521
4967
  }
5522
4968
  lastExecutionHeartbeatAt = now;
5523
- void updateMapProgress(
4969
+ enqueueMapProgressUpdate(
5524
4970
  {
5525
4971
  completed: completedRowsForProgress(),
5526
4972
  total: progressTotalRows,
@@ -5554,7 +5000,7 @@ function createMinimalWorkerCtx(
5554
5000
  return;
5555
5001
  }
5556
5002
  lastChunkProgressAt = now;
5557
- void updateMapProgress(
5003
+ enqueueMapProgressUpdate(
5558
5004
  {
5559
5005
  completed,
5560
5006
  total: progressTotalRows,
@@ -5585,23 +5031,44 @@ function createMinimalWorkerCtx(
5585
5031
  const failedRowEntries: Array<
5586
5032
  { row: T & Record<string, unknown>; error: string } | undefined
5587
5033
  > = new Array(rowsToExecute.length);
5034
+ const fatalReceiptPersistenceRowEntries: Array<
5035
+ { row: T & Record<string, unknown>; error: string } | undefined
5036
+ > = new Array(rowsToExecute.length);
5588
5037
  const executedCellMetaPatches: Array<
5589
5038
  Record<string, WorkerCellMetaPatchEntry> | undefined
5590
5039
  > = new Array(rowsToExecute.length);
5591
- const toolBatchScheduler = new WorkerToolBatchScheduler(
5040
+ const persistenceLatch = createRuntimePersistenceLatch();
5041
+ const toolBatchScheduler = new WorkerToolBatchScheduler({
5592
5042
  req,
5593
5043
  governor,
5594
- resolveToolPacing,
5044
+ resolvePacing: resolveToolPacing,
5595
5045
  resolveToolActionCacheVersion,
5046
+ resolveToolAuthScopeDigest,
5047
+ invalidateToolAuthScopeDigest: (toolId) =>
5048
+ resolveToolAuthScopeDigest.invalidate(toolId),
5049
+ onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
5050
+ executeTool: executeDispatchedTool,
5051
+ serializeDurableValue: serializeDurableStepValue,
5052
+ deserializeDurableValue: deserializeDurableStepValue,
5053
+ nowMs,
5054
+ batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
5055
+ recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
5596
5056
  abortSignal,
5597
- reportSettledToolRequests,
5057
+ onRequestsSettled: reportSettledToolRequests,
5598
5058
  callbacks,
5599
5059
  receiptStore,
5600
- false,
5060
+ allowLocalRetryReceipts: false,
5061
+ runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
5062
+ runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
5063
+ req,
5064
+ runtimeReceiptLeaseTtlMs(req) ??
5065
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
5066
+ ),
5067
+ budgetMeter: workBudgetMeter,
5601
5068
  runtimeDeadlineMs,
5602
5069
  persistenceLatch,
5603
5070
  receiptSalvage,
5604
- );
5071
+ });
5605
5072
  let stepCellsCompleted = 0;
5606
5073
  let stepCellsSkipped = 0;
5607
5074
  const generatedOutputFields = new Set<string>();
@@ -5680,6 +5147,8 @@ function createMinimalWorkerCtx(
5680
5147
  tableNamespace: name,
5681
5148
  outputFields,
5682
5149
  extraOutputFields: Array.from(generatedOutputFields),
5150
+ budgetMeter: workBudgetMeter,
5151
+ ...activeRowAttempt,
5683
5152
  rows: [
5684
5153
  ...rowsToPersist.map(({ row, executedIndex }) =>
5685
5154
  mapRowOutcomeRuntimeRow(
@@ -5743,9 +5212,6 @@ function createMinimalWorkerCtx(
5743
5212
  });
5744
5213
  persistFlushChain = task.catch((error) => {
5745
5214
  persistFailure ??= error;
5746
- // Output-sheet flush failed: trip the breaker so the scheduler stops
5747
- // dispatching new provider calls for the rest of this map.
5748
- tripRuntimePersistenceLatch(persistenceLatch, error);
5749
5215
  });
5750
5216
  return task;
5751
5217
  };
@@ -5787,6 +5253,8 @@ function createMinimalWorkerCtx(
5787
5253
  outputFields,
5788
5254
  extraOutputFields,
5789
5255
  updates,
5256
+ ...activeRowAttempt,
5257
+ budgetMeter: workBudgetMeter,
5790
5258
  });
5791
5259
  } catch (error) {
5792
5260
  liveUpdateFailures += 1;
@@ -5894,18 +5362,11 @@ function createMinimalWorkerCtx(
5894
5362
  request.input,
5895
5363
  workflowStep,
5896
5364
  {
5897
- force: request.force === true || !!req.force,
5365
+ force: request.force === true,
5366
+ runForceRefresh: req.forceToolRefresh === true,
5367
+ runForceFailedRefresh: req.force === true,
5898
5368
  staleAfterSeconds: request.staleAfterSeconds,
5899
5369
  },
5900
- // Parity with the play-level ctx.tools.execute (and the
5901
- // cjs runner's map path): the public ToolExecutionRequest
5902
- // runtime options must not be silently dropped for map
5903
- // rows — receiptWaitMs bounds how long a resumed run
5904
- // waits on a stuck `running` receipt before reclaiming.
5905
- {
5906
- timeoutMs: request.timeoutMs,
5907
- receiptWaitMs: request.receiptWaitMs,
5908
- },
5909
5370
  );
5910
5371
  },
5911
5372
  },
@@ -6014,8 +5475,28 @@ function createMinimalWorkerCtx(
6014
5475
  notePersistableRow(enriched);
6015
5476
  reportChunkProgress(false);
6016
5477
  } catch (rowError) {
6017
- // Abort/budget errors stay run-fatal and leave no partial
6018
- // state: rethrow immediately without recording the row.
5478
+ // Receipt persistence errors stay run-fatal, but keep the
5479
+ // row identity so teardown can make the recovery state loud.
5480
+ if (isRuntimeReceiptPersistenceError(rowError)) {
5481
+ const message = formatWorkerRowFailureMessage(rowError);
5482
+ if (activeField) {
5483
+ cellMetaPatch[activeField] = {
5484
+ status: 'failed',
5485
+ stage: activeField,
5486
+ runId: req.runId,
5487
+ error: message,
5488
+ };
5489
+ }
5490
+ executedCellMetaPatches[myIndex] =
5491
+ Object.keys(cellMetaPatch).length > 0
5492
+ ? cellMetaPatch
5493
+ : undefined;
5494
+ fatalReceiptPersistenceRowEntries[myIndex] = {
5495
+ row: enriched as T & Record<string, unknown>,
5496
+ error: message,
5497
+ };
5498
+ throw rowError;
5499
+ }
6019
5500
  if (isRunFatalWorkerRowError(rowError)) {
6020
5501
  throw rowError;
6021
5502
  }
@@ -6095,16 +5576,25 @@ function createMinimalWorkerCtx(
6095
5576
  return results;
6096
5577
  },
6097
5578
  );
6098
- while (!workerSettled) {
6099
- await Promise.race([
6100
- workerResultsPromise,
6101
- sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
6102
- ]);
6103
- if (!workerSettled) {
6104
- reportExecutionHeartbeat(false);
6105
- }
6106
- }
6107
- const workerResults = await workerResultsPromise;
5579
+ const workerResults = await executeWithRuntimeSheetAttemptHeartbeat({
5580
+ req,
5581
+ tableNamespace: name,
5582
+ sheetContract: requireSheetContract(req, name),
5583
+ ...activeRowAttempt,
5584
+ rowKeys: uniqueRowsToExecuteEntries.map((entry) => entry.rowKey),
5585
+ execute: async () => {
5586
+ while (!workerSettled) {
5587
+ await Promise.race([
5588
+ workerResultsPromise,
5589
+ sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
5590
+ ]);
5591
+ if (!workerSettled) {
5592
+ reportExecutionHeartbeat(false);
5593
+ }
5594
+ }
5595
+ return await workerResultsPromise;
5596
+ },
5597
+ });
6108
5598
  recordRunnerPerfTrace({
6109
5599
  req,
6110
5600
  phase: 'runner.map_chunk.execute_workers',
@@ -6133,26 +5623,89 @@ function createMinimalWorkerCtx(
6133
5623
  .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
6134
5624
  const fatalMapChunkSummary = async (
6135
5625
  error: unknown,
6136
- ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => ({
6137
- chunkIndex,
6138
- rangeStart: baseOffset + chunkStart,
6139
- rangeEnd: baseOffset + chunkStart,
6140
- rowsRead: chunkRows.length,
6141
- rowsWritten: 0,
6142
- rowsExecuted: rowsToExecute.length,
6143
- rowsCached: 0,
6144
- rowsDuplicateReused: duplicateInputReuseCount,
6145
- rowsInserted,
6146
- rowsSkipped,
6147
- rowsFailed: failedExecutedRows,
6148
- rowFailureSamples: buildRowFailureSamples(),
6149
- stepCellsCompleted,
6150
- stepCellsSkipped,
6151
- outputDatasetId: `map:${name}`,
6152
- hash: await hashJson([]),
6153
- fatalError: formatWorkerRowFailureMessage(error),
6154
- preview: [],
6155
- });
5626
+ ): Promise<
5627
+ WorkerMapCompletedChunkSummary<T & Record<string, unknown>>
5628
+ > => {
5629
+ const fatalError = formatWorkerRowFailureMessage(error);
5630
+ const breakerMessage = persistenceLatch.tripped
5631
+ ? ` Circuit breaker prevented ${persistenceLatch.preventedCallCount} queued provider call(s).`
5632
+ : '';
5633
+ return {
5634
+ status: 'completed',
5635
+ chunkIndex,
5636
+ rangeStart: baseOffset + chunkStart,
5637
+ rangeEnd: baseOffset + chunkStart,
5638
+ rowsRead: chunkRows.length,
5639
+ rowsWritten: 0,
5640
+ rowsExecuted: rowsToExecute.length,
5641
+ rowsCached: 0,
5642
+ rowsDuplicateReused: duplicateInputReuseCount,
5643
+ rowsInserted,
5644
+ rowsSkipped,
5645
+ rowsFailed: failedExecutedRows,
5646
+ rowFailureSamples: buildRowFailureSamples(),
5647
+ stepCellsCompleted,
5648
+ stepCellsSkipped,
5649
+ outputDatasetId: `map:${name}`,
5650
+ hash: await hashJson([]),
5651
+ fatalError: `${fatalError}${breakerMessage}`,
5652
+ preview: [],
5653
+ };
5654
+ };
5655
+ const markReceiptPersistenceFatalRowsFailed = async (error: unknown) => {
5656
+ const message = formatWorkerRowFailureMessage(error);
5657
+ const rows = uniqueRowsToExecuteEntries
5658
+ .map((entry, executedIndex) => {
5659
+ const failure = fatalReceiptPersistenceRowEntries[executedIndex];
5660
+ const row =
5661
+ failure?.row ??
5662
+ executedRows[executedIndex] ??
5663
+ (runtimeCsvExecutionRow(
5664
+ entry.row,
5665
+ pendingRowsByKey.get(entry.rowKey),
5666
+ ) as T & Record<string, unknown>);
5667
+ return mapRowOutcomeRuntimeRow(
5668
+ failedMapRowOutcome({
5669
+ key: entry.rowKey,
5670
+ inputIndex: entry.absoluteIndex,
5671
+ data: row,
5672
+ cellMetaPatch: executedCellMetaPatches[executedIndex],
5673
+ error: failure?.error ?? message,
5674
+ }),
5675
+ );
5676
+ })
5677
+ .filter((row): row is Record<string, unknown> => row !== null);
5678
+ if (rows.length === 0) return;
5679
+ try {
5680
+ const failureStats = await persistCompletedMapRows({
5681
+ req,
5682
+ tableNamespace: name,
5683
+ outputFields,
5684
+ extraOutputFields: Array.from(generatedOutputFields),
5685
+ budgetMeter: workBudgetMeter,
5686
+ ...activeRowAttempt,
5687
+ forceFailedRows: true,
5688
+ rows,
5689
+ });
5690
+ emitEvent({
5691
+ type: 'log',
5692
+ level: 'warn',
5693
+ message:
5694
+ `Runtime sheet marked ${failureStats.rows} row(s) failed for ctx.dataset("${name}") ` +
5695
+ `after a run-fatal receipt persistence error.`,
5696
+ ts: nowMs(),
5697
+ });
5698
+ } catch (cleanupError) {
5699
+ emitEvent({
5700
+ type: 'log',
5701
+ level: 'warn',
5702
+ message:
5703
+ `Runtime sheet failed-row cleanup after receipt persistence error failed for ctx.dataset("${name}"): ` +
5704
+ formatWorkerRowFailureMessage(cleanupError),
5705
+ ts: nowMs(),
5706
+ });
5707
+ }
5708
+ };
6156
5709
  const persistRowsStartedAt = nowMs();
6157
5710
  recordRunnerPerfTrace({
6158
5711
  req,
@@ -6181,6 +5734,7 @@ function createMinimalWorkerCtx(
6181
5734
  },
6182
5735
  });
6183
5736
  } catch (error) {
5737
+ tripRuntimePersistenceLatch(persistenceLatch, error);
6184
5738
  recordRunnerPerfTrace({
6185
5739
  req,
6186
5740
  phase: 'runner.map_chunk.persist_rows_error',
@@ -6199,31 +5753,24 @@ function createMinimalWorkerCtx(
6199
5753
  result.status === 'rejected',
6200
5754
  );
6201
5755
  if (rejectedWorker) {
6202
- if (
6203
- isRuntimeReceiptPersistenceError(rejectedWorker.reason) ||
6204
- isRuntimePersistenceCircuitOpenError(rejectedWorker.reason)
6205
- ) {
6206
- // Persistence-failure / circuit-breaker rejections are returned as a
6207
- // fatal summary (NOT rethrown into the durable step) so the chunk is
6208
- // not retried and provider calls are not re-billed.
5756
+ if (isRuntimeReceiptPersistenceError(rejectedWorker.reason)) {
5757
+ await markReceiptPersistenceFatalRowsFailed(rejectedWorker.reason);
5758
+ return await fatalMapChunkSummary(rejectedWorker.reason);
5759
+ }
5760
+ if (isRuntimePersistenceCircuitOpenError(rejectedWorker.reason)) {
6209
5761
  return await fatalMapChunkSummary(rejectedWorker.reason);
6210
5762
  }
6211
5763
  throw rejectedWorker.reason;
6212
5764
  }
6213
5765
  const resultByKey = new Map<string, T & Record<string, unknown>>();
6214
- for (const completedRow of prepared.completedRows) {
6215
- const key =
6216
- resolveMapRowOutcomeKey(completedRow) ??
6217
- deriveDefaultMapRowIdentity(
6218
- completedRow,
6219
- resolveRuntimeInputIndex(completedRow) ?? 0,
6220
- );
6221
- if (key) {
6222
- const cleanedRow = stripMapRowOutcomeRuntimeFields(
6223
- publicCsvOutputRow(completedRow),
6224
- );
6225
- resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
6226
- }
5766
+ for (const entry of completedChunkEntries) {
5767
+ const completedRow = completedRowsByKey.get(entry.rowKey);
5768
+ if (!completedRow || resultByKey.has(entry.rowKey)) continue;
5769
+ resultByKey.set(
5770
+ entry.rowKey,
5771
+ cloneCsvAliasedRow(entry.row, completedRow) as T &
5772
+ Record<string, unknown>,
5773
+ );
6227
5774
  }
6228
5775
  for (
6229
5776
  let executedIndex = 0;
@@ -6258,16 +5805,13 @@ function createMinimalWorkerCtx(
6258
5805
  } => entry !== null,
6259
5806
  );
6260
5807
  const out = outEntries.map((entry) => entry.row);
6261
- const executedSuccessCount = Math.max(
6262
- 0,
6263
- executedRows.length - failedExecutedRows,
6264
- );
6265
5808
  const rowFailureSamples = buildRowFailureSamples();
6266
5809
  const publicOut = out.map((row) => publicCsvOutputRow(row));
6267
5810
  const keyedOut = outEntries.map(({ key, inputIndex, row }) => ({
6268
5811
  ...row,
6269
5812
  ...mapRowOutcomeRuntimeFields({ key, inputIndex }),
6270
5813
  }));
5814
+ const completedRowsReused = completedChunkEntries.length;
6271
5815
  const hashStartedAt = nowMs();
6272
5816
  const hash = await hashJson(publicOut);
6273
5817
  const includeCachedRowsInChunkResult = !workflowStep;
@@ -6298,17 +5842,18 @@ function createMinimalWorkerCtx(
6298
5842
  rowsWritten: out.length,
6299
5843
  rowsExecuted: executedRows.length,
6300
5844
  rowsFailed: failedExecutedRows,
6301
- rowsCached: Math.max(0, out.length - executedSuccessCount),
5845
+ rowsCached: completedRowsReused,
6302
5846
  },
6303
5847
  });
6304
5848
  return {
5849
+ status: 'completed',
6305
5850
  chunkIndex,
6306
5851
  rangeStart: baseOffset + chunkStart,
6307
5852
  rangeEnd: baseOffset + chunkStart + out.length,
6308
5853
  rowsRead: chunkRows.length,
6309
5854
  rowsWritten: out.length,
6310
5855
  rowsExecuted: executedRows.length,
6311
- rowsCached: Math.max(0, out.length - executedSuccessCount),
5856
+ rowsCached: completedRowsReused,
6312
5857
  rowsDuplicateReused: duplicateInputReuseCount,
6313
5858
  rowsInserted,
6314
5859
  rowsSkipped,
@@ -6340,9 +5885,12 @@ function createMinimalWorkerCtx(
6340
5885
  let totalRowsInserted = 0;
6341
5886
  let totalRowsSkipped = 0;
6342
5887
  let totalRowsFailed = 0;
5888
+ let totalRowsBlocked = 0;
6343
5889
  let totalStepCellsCompleted = 0;
6344
5890
  let totalStepCellsSkipped = 0;
6345
5891
  const totalRowFailureSamples: Array<{ rowKey: string; error: string }> = [];
5892
+ const blockedRowKeySamples: string[] = [];
5893
+ const blockedRowSamples: Record<string, unknown>[] = [];
6346
5894
 
6347
5895
  const runChunkStep = async (
6348
5896
  chunkRows: T[],
@@ -6372,6 +5920,7 @@ function createMinimalWorkerCtx(
6372
5920
  const readPersistedRows = async (input: {
6373
5921
  limit: number;
6374
5922
  offset: number;
5923
+ rowMode?: 'output' | 'all';
6375
5924
  }) => {
6376
5925
  const result = await harnessReadSheetDatasetRows({
6377
5926
  baseUrl: req.baseUrl,
@@ -6380,6 +5929,7 @@ function createMinimalWorkerCtx(
6380
5929
  playName: req.playName,
6381
5930
  tableNamespace: name,
6382
5931
  runId: req.runId,
5932
+ rowMode: input.rowMode,
6383
5933
  limit: input.limit,
6384
5934
  offset: input.offset,
6385
5935
  userEmail: req.userEmail,
@@ -6388,19 +5938,61 @@ function createMinimalWorkerCtx(
6388
5938
  return result.rows as Array<T & Record<string, unknown>>;
6389
5939
  };
6390
5940
 
6391
- const finalize = async (totalRowsWritten: number) => {
5941
+ const finalize = async () => {
5942
+ let finalizedRowsWritten = totalRowsWritten;
5943
+ if (
5944
+ totalRowsFailed === 0 &&
5945
+ rowCountHint !== null &&
5946
+ finalizedRowsWritten > 0 &&
5947
+ finalizedRowsWritten < rowCountHint &&
5948
+ rowCountHint <= WORKER_DATASET_IN_MEMORY_ROWS
5949
+ ) {
5950
+ const persistedRows = await readPersistedRows({
5951
+ limit: rowCountHint,
5952
+ offset: 0,
5953
+ rowMode: 'all',
5954
+ });
5955
+ if (persistedRows.length > finalizedRowsWritten) {
5956
+ const inMemoryRowsWritten = finalizedRowsWritten;
5957
+ finalizedRowsWritten = persistedRows.length;
5958
+ totalRowsWritten = finalizedRowsWritten;
5959
+ cachedRows.length = 0;
5960
+ cachedRows.push(...persistedRows);
5961
+ canCacheRows = true;
5962
+ previewRows.length = 0;
5963
+ previewRows.push(
5964
+ ...persistedRows.slice(0, WORKER_DATASET_PREVIEW_ROWS),
5965
+ );
5966
+ emitEvent({
5967
+ type: 'log',
5968
+ level: 'warn',
5969
+ message:
5970
+ `Runtime sheet finalization reconciled ctx.dataset("${name}") ` +
5971
+ `from ${inMemoryRowsWritten}/${rowCountHint} in-memory row(s) ` +
5972
+ `to ${finalizedRowsWritten} visible persisted row(s).`,
5973
+ ts: nowMs(),
5974
+ });
5975
+ }
5976
+ if (persistedRows.length < rowCountHint) {
5977
+ throw new Error(
5978
+ `Runtime sheet finalization mismatch ctx.dataset("${name}"): ` +
5979
+ `expected ${rowCountHint}, saw ${persistedRows.length}; ` +
5980
+ `sum ${finalizedRowsWritten}; ${req.runId}`,
5981
+ );
5982
+ }
5983
+ }
6392
5984
  const failureSampleSummary =
6393
5985
  totalRowFailureSamples.length > 0
6394
5986
  ? ` First error: ${totalRowFailureSamples[0]!.error}`
6395
5987
  : '';
6396
5988
  const cacheSummary =
6397
5989
  totalRowsFailed > 0
6398
- ? `Map completed with partial failures: ${totalRowsWritten} succeeded, ` +
6399
- `${totalRowsFailed} failed (${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
5990
+ ? `Map completed with partial failures: ${finalizedRowsWritten} succeeded, ` +
5991
+ `${totalRowsFailed} failed (${totalRowsExecuted} executed) ` +
6400
5992
  `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}. ` +
6401
5993
  `Failed rows are persisted with their errors and re-execute on the next run.${failureSampleSummary}`
6402
- : `Map completed: ${totalRowsWritten} results ` +
6403
- `(${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
5994
+ : `Map completed: ${finalizedRowsWritten} results ` +
5995
+ `(${totalRowsExecuted} executed) ` +
6404
5996
  `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
6405
5997
  const completedAt = nowMs();
6406
5998
  const totalStepCells = totalStepCellsCompleted + totalStepCellsSkipped;
@@ -6408,17 +6000,21 @@ function createMinimalWorkerCtx(
6408
6000
  totalStepCells > 0
6409
6001
  ? ` cells=${totalStepCells}/${totalStepCellsCompleted}/${totalStepCellsSkipped}`
6410
6002
  : '';
6003
+ await flushQueuedMapProgressUpdates();
6411
6004
  callbacks?.onMapCompleted?.(mapNodeId, completedAt);
6412
6005
  await updateMapProgress({
6413
- completed: totalRowsWritten,
6414
- total: totalRowsWritten + totalRowsFailed,
6006
+ completed: finalizedRowsWritten,
6007
+ total: finalizedRowsWritten + totalRowsFailed,
6415
6008
  failed: totalRowsFailed,
6416
6009
  completedAt,
6417
6010
  updatedAt: completedAt,
6418
6011
  message:
6419
6012
  totalRowsFailed > 0
6420
- ? `${totalRowsWritten.toLocaleString()} succeeded, ${totalRowsFailed.toLocaleString()} failed`
6421
- : formatMapProgressMessage(totalRowsWritten, totalRowsWritten),
6013
+ ? `${finalizedRowsWritten.toLocaleString()} succeeded, ${totalRowsFailed.toLocaleString()} failed`
6014
+ : formatMapProgressMessage(
6015
+ finalizedRowsWritten,
6016
+ finalizedRowsWritten,
6017
+ ),
6422
6018
  });
6423
6019
  emitEvent({
6424
6020
  type: 'log',
@@ -6427,29 +6023,61 @@ function createMinimalWorkerCtx(
6427
6023
  ts: nowMs(),
6428
6024
  });
6429
6025
  if (
6430
- totalRowsWritten > 0 &&
6026
+ finalizedRowsWritten > 0 &&
6431
6027
  totalRowsFailed === 0 &&
6432
6028
  canCacheRows &&
6433
- cachedRows.length === totalRowsWritten
6029
+ cachedRows.length === finalizedRowsWritten
6434
6030
  ) {
6435
6031
  const persistedRows = await readPersistedRows({
6436
- limit: totalRowsWritten,
6032
+ limit: finalizedRowsWritten,
6437
6033
  offset: 0,
6438
6034
  });
6439
- if (persistedRows.length < totalRowsWritten) {
6035
+ if (persistedRows.length < finalizedRowsWritten) {
6036
+ const repairAttempt = makeRuntimeSheetAttempt({
6037
+ runId: req.runId,
6038
+ runAttempt: req.runAttempt,
6039
+ mapName: name,
6040
+ chunkIndex: 'finalize',
6041
+ });
6042
+ const repairRows = cachedRows.map((row) => runtimeCsvStorageRow(row));
6043
+ workBudgetMeter.count('sheet');
6044
+ const repairStart = await harnessStartSheetDataset({
6045
+ ...runtimeSheetSessionScope(req),
6046
+ tableNamespace: name,
6047
+ sheetContract: augmentSheetContractWithDatasetFields({
6048
+ contract: requireSheetContract(req, name),
6049
+ rows: repairRows,
6050
+ outputFields,
6051
+ }),
6052
+ rows: repairRows,
6053
+ runId: req.runId,
6054
+ inputOffset: 0,
6055
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
6056
+ ...repairAttempt,
6057
+ });
6058
+ const activeRepairAttempt = {
6059
+ attemptId: repairStart.attemptId ?? repairAttempt.attemptId,
6060
+ attemptOwnerRunId:
6061
+ repairStart.attemptOwnerRunId ?? repairAttempt.attemptOwnerRunId,
6062
+ attemptExpiresAt:
6063
+ repairStart.attemptExpiresAt ?? repairAttempt.attemptExpiresAt,
6064
+ attemptSeq: repairStart.attemptSeq ?? repairAttempt.attemptSeq,
6065
+ };
6440
6066
  const repair = await persistCompletedMapRows({
6441
6067
  req,
6442
6068
  tableNamespace: name,
6443
6069
  outputFields,
6444
6070
  extraOutputFields: [],
6445
6071
  rows: cachedRows,
6072
+ ...activeRepairAttempt,
6073
+ budgetMeter: workBudgetMeter,
6446
6074
  });
6447
6075
  emitEvent({
6448
6076
  type: 'log',
6449
6077
  level: 'warn',
6450
6078
  message:
6451
6079
  `Runtime sheet finalization repaired ctx.dataset("${name}") ` +
6452
- `from ${persistedRows.length}/${totalRowsWritten} visible row(s) ` +
6080
+ `from ${persistedRows.length}/${finalizedRowsWritten} visible row(s) ` +
6453
6081
  `(written=${repair.written}, visible=${repair.visible}` +
6454
6082
  (repair.retryWritten === null
6455
6083
  ? ''
@@ -6465,7 +6093,7 @@ function createMinimalWorkerCtx(
6465
6093
  return createPersistedDatasetHandle({
6466
6094
  playName: req.playName,
6467
6095
  name,
6468
- count: totalRowsWritten,
6096
+ count: finalizedRowsWritten,
6469
6097
  // In native Workflows, chunk summaries intentionally omit full row
6470
6098
  // payloads, so this preview only contains the bounded chunk samples.
6471
6099
  // Do not synchronously page rows back here: service-binding reads have
@@ -6479,7 +6107,7 @@ function createMinimalWorkerCtx(
6479
6107
  recordRunnerPerfTrace({ req, phase, ms, extra }),
6480
6108
  nowMs,
6481
6109
  workProgress: {
6482
- total: totalRowsWritten + totalRowsFailed,
6110
+ total: finalizedRowsWritten + totalRowsFailed,
6483
6111
  executed: totalRowsExecuted,
6484
6112
  reused: totalRowsCached,
6485
6113
  skipped: totalRowsCached,
@@ -6493,97 +6121,248 @@ function createMinimalWorkerCtx(
6493
6121
  };
6494
6122
 
6495
6123
  const failFastRowErrors = opts?.onRowError === 'fail';
6496
- let chunkIndex = 0;
6497
- let chunkStart = 0;
6498
- for await (const rawChunkRows of iterDatasetChunks(
6499
- inputRows,
6500
- rowsPerChunk,
6501
- )) {
6502
- assertNotAborted(abortSignal);
6503
- if (rawChunkRows.length === 0) continue;
6504
- // Drop duplicate explicit-key rows before anything downstream observes
6505
- // them. `chunkStart` keeps advancing by the original (pre-dedupe) chunk
6506
- // length so cross-chunk key indices and persisted input offsets stay
6507
- // aligned to the original input stream.
6508
- const chunkRows = dedupeExplicitRowKeys(rawChunkRows, chunkStart);
6509
- if (chunkRows.length === 0) {
6510
- chunkStart += rawChunkRows.length;
6511
- continue;
6512
- }
6513
- const chunkResult = await runChunkStep(chunkRows, chunkStart, chunkIndex);
6514
- if (chunkResult.fatalError) {
6515
- const circuitBreakerSentence =
6516
- persistenceLatch.tripped && persistenceLatch.preventedCallCount > 0
6517
- ? `Circuit breaker prevented ${persistenceLatch.preventedCallCount} queued provider call(s) ` +
6518
- `from dispatching after the first persistence failure. `
6519
- : '';
6520
- throw new Error(
6521
- `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
6522
- `outside the retryable chunk step. Provider calls already executed for ` +
6523
- `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
6524
- circuitBreakerSentence +
6525
- `Fix the runtime persistence cause and re-run to resume. ` +
6526
- `First error: ${chunkResult.fatalError}`,
6527
- );
6528
- }
6529
- totalRowsWritten += chunkResult.rowsWritten;
6530
- totalRowsExecuted += chunkResult.rowsExecuted;
6531
- totalRowsCached += chunkResult.rowsCached;
6532
- totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
6533
- totalRowsInserted += chunkResult.rowsInserted;
6534
- totalRowsSkipped += chunkResult.rowsSkipped;
6535
- totalRowsFailed += chunkResult.rowsFailed ?? 0;
6536
- totalStepCellsCompleted += chunkResult.stepCellsCompleted ?? 0;
6537
- totalStepCellsSkipped += chunkResult.stepCellsSkipped ?? 0;
6538
- for (const sample of chunkResult.rowFailureSamples ?? []) {
6539
- if (totalRowFailureSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6540
- break;
6124
+ // Bounded-concurrent chunk dispatch is only sound off the native Workflows
6125
+ // path: with a `workflowStep` each chunk is a durable step whose sequential
6126
+ // order makes replay/resume deterministic, so that path stays at K=1.
6127
+ // Chunk-level concurrency is otherwise safe under workers_edge because each
6128
+ // chunk owns a disjoint sheet row range with its own runtime-sheet attempt,
6129
+ // receipts complete inside each chunk's persist barrier, and child submits
6130
+ // use per-row durable receipt keys. It is disabled under onRowError:'fail'
6131
+ // on purpose: that mode relies on a first-failure short-circuit to skip the
6132
+ // remaining chunks and stop spending provider credits, and eager concurrent
6133
+ // launch would already have K-1 chunks in flight (credits spent) before the
6134
+ // short-circuit fires.
6135
+ const canRunChunksConcurrently = !workflowStep && !failFastRowErrors;
6136
+ const maxConcurrentMapChunks = canRunChunksConcurrently
6137
+ ? WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS
6138
+ : 1;
6139
+ const runWorkDispatcher = new WorkerRunWorkDispatcher({
6140
+ budgetMeter: workBudgetMeter,
6141
+ nowMs,
6142
+ yieldLimits: {
6143
+ elapsed: governor.policy.pacing.workerYieldElapsedMs,
6144
+ ...(req.testPolicyOverrides?.workBudgetYieldLimits ?? {}),
6145
+ },
6146
+ recordTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
6147
+ yieldBetweenChunks: async (yieldPoint) => {
6148
+ emitEvent({
6149
+ type: 'log',
6150
+ level: 'info',
6151
+ message:
6152
+ `Runtime dispatcher yielded ctx.dataset("${name}") before chunk ${yieldPoint.nextChunkIndex} ` +
6153
+ `(${yieldPoint.reason} budget).`,
6154
+ ts: nowMs(),
6155
+ });
6156
+ if (workflowStep) {
6157
+ // Cloudflare Workflows durability model: `step.sleep` writes a
6158
+ // durable checkpoint, and the resume re-enters `run()` as a NEW
6159
+ // Worker invocation. Cloudflare's subrequest ceiling is
6160
+ // per-invocation, and on resume the already-completed `step.do`
6161
+ // chunks are served from the durable step cache WITHOUT re-issuing
6162
+ // their subrequests (replay semantics) — so the post-yield chunk
6163
+ // runs against a fresh per-invocation subrequest budget.
6164
+ //
6165
+ // Isolate identity is the WRONG freshness proxy: Cloudflare routinely
6166
+ // reuses warm isolates across invocations, so a genuinely
6167
+ // fresh-budget resume commonly shares the pre-sleep isolate id (this
6168
+ // is exactly why the isolate-identity check reported `false` every
6169
+ // time and hard-failed multi-chunk maps). Crossing the durable
6170
+ // `step.sleep` boundary IS the fresh-budget signal.
6171
+ const sleepMs = WORK_DISPATCHER_YIELD_SLEEP_MS;
6172
+ await (
6173
+ workflowStep.sleep as unknown as (
6174
+ name: string,
6175
+ duration: number,
6176
+ ) => Promise<void>
6177
+ )(
6178
+ `work-dispatcher:${name}:${yieldPoint.nextChunkIndex}:${yieldPoint.attempt}`,
6179
+ sleepMs,
6180
+ );
6181
+ recordRunnerPerfTrace({
6182
+ req,
6183
+ phase: 'runner.work_dispatcher.yield_probe',
6184
+ ms: sleepMs,
6185
+ extra: {
6186
+ mapName: name,
6187
+ nextChunkIndex: yieldPoint.nextChunkIndex,
6188
+ reason: yieldPoint.reason,
6189
+ attempt: yieldPoint.attempt,
6190
+ freshBudget: true,
6191
+ invocationBoundary: 'workflow_step_sleep',
6192
+ },
6193
+ });
6194
+ return { freshBudget: true };
6541
6195
  }
6542
- totalRowFailureSamples.push(sample);
6543
- }
6544
- await updateMapProgress({
6545
- completed: totalRowsWritten,
6546
- total: rowCountHint ?? undefined,
6547
- ...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
6548
- message: formatMapProgressMessage(
6549
- totalRowsWritten,
6550
- rowCountHint ?? undefined,
6551
- ),
6552
- });
6553
- if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
6554
- previewRows.push(
6555
- ...chunkResult.preview.slice(
6196
+ // No durable Workflow step is available to force a new invocation here,
6197
+ // so we cannot obtain a fresh per-invocation platform budget. Report a
6198
+ // non-fresh yield: the dispatcher escalates and then fails loudly with
6199
+ // WorkerBudgetExhaustedError rather than silently overflowing the
6200
+ // subrequest ceiling mid-map.
6201
+ await sleepWorkerMs(0);
6202
+ return { freshBudget: false };
6203
+ },
6204
+ });
6205
+ let dispatchResult: WorkerRunMapBatchesResult;
6206
+ try {
6207
+ dispatchResult = await runWorkDispatcher.runMapBatches({
6208
+ mapName: name,
6209
+ chunks: iterDatasetChunks(inputRows, rowsPerChunk),
6210
+ maxConcurrentChunks: maxConcurrentMapChunks,
6211
+ maxResidentBatchBytes: WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES,
6212
+ prepareBatch: (rawChunkRows, chunkStart) => {
6213
+ assertNotAborted(abortSignal);
6214
+ if (rawChunkRows.length === 0) return [];
6215
+ // Drop duplicate explicit-key rows before anything downstream observes
6216
+ // them. `chunkStart` keeps advancing by the original (pre-dedupe) chunk
6217
+ // length so cross-chunk key indices and persisted input offsets stay
6218
+ // aligned to the original input stream.
6219
+ return dedupeExplicitRowKeys(rawChunkRows, chunkStart);
6220
+ },
6221
+ estimateRawBatchResidentBytes: (rawChunkRows) =>
6222
+ rawChunkRows.reduce(
6223
+ (total, row) => total + workerMapResidentRowBytes(row),
6224
+ 0,
6225
+ ),
6226
+ executeBatch: async ({ rows: chunkRows, chunkStart, chunkIndex }) => {
6227
+ assertNotAborted(abortSignal);
6228
+ return await runChunkStep(chunkRows, chunkStart, chunkIndex);
6229
+ },
6230
+ estimateBatchResidentBytes: ({ rows: chunkRows }) =>
6231
+ chunkRows.reduce(
6232
+ (total, row) => total + workerMapResidentRowBytes(row),
6556
6233
  0,
6557
- WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
6558
6234
  ),
6235
+ estimateBatchSubrequests: ({ rows: chunkRows }) => {
6236
+ const estimate = mapDispatchPlan.workEstimate;
6237
+ const unbatchedProviderSubrequests =
6238
+ estimate.unbatchedProviderToolCallsPerRow *
6239
+ WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL;
6240
+ // Static planning cannot know input-dependent batching buckets, so
6241
+ // this projects worst-case provider-batch fanout while still treating
6242
+ // durable receipt claim/complete/fail as bulk per drained tool group.
6243
+ const batchedProviderSubrequests =
6244
+ estimateBatchedToolChunkSubrequests({
6245
+ batchableToolCallsPerRow: Math.max(
6246
+ 0,
6247
+ estimate.providerToolCallsPerRow -
6248
+ estimate.unbatchedProviderToolCallsPerRow,
6249
+ ),
6250
+ coalescedBatchableToolCallsPerRow:
6251
+ estimate.coalescedBatchableProviderToolCallsPerRow,
6252
+ rows: chunkRows.length,
6253
+ assumedBatchSize: estimate.assumedBatchSize,
6254
+ });
6255
+ const controlSubrequests =
6256
+ estimate.childPlaySubmitsPerRow + estimate.eventWaitsPerRow;
6257
+ return (
6258
+ chunkRows.length *
6259
+ (unbatchedProviderSubrequests + controlSubrequests) +
6260
+ batchedProviderSubrequests +
6261
+ estimate.runtimeSheetReadsPerChunk +
6262
+ estimate.runtimeSheetWritesPerChunk
6263
+ );
6264
+ },
6265
+ onBatchComplete: async (chunkResult) => {
6266
+ if (chunkResult.status === 'blocked') {
6267
+ totalRowsBlocked += chunkResult.rowsBlocked;
6268
+ for (const key of chunkResult.rowKeySamples) {
6269
+ if (blockedRowKeySamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6270
+ break;
6271
+ }
6272
+ blockedRowKeySamples.push(key);
6273
+ }
6274
+ for (const row of chunkResult.blockedRowSamples) {
6275
+ if (blockedRowSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6276
+ break;
6277
+ }
6278
+ blockedRowSamples.push(row);
6279
+ }
6280
+ emitEvent({
6281
+ type: 'log',
6282
+ level: 'warn',
6283
+ message: chunkResult.message,
6284
+ ts: nowMs(),
6285
+ });
6286
+ return true;
6287
+ }
6288
+ if (chunkResult.fatalError) {
6289
+ throw new Error(
6290
+ `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
6291
+ `outside the retryable chunk step. Provider calls already executed for ` +
6292
+ `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
6293
+ `Fix the runtime persistence cause and re-run to resume. ` +
6294
+ `First error: ${chunkResult.fatalError}`,
6295
+ );
6296
+ }
6297
+ totalRowsWritten += chunkResult.rowsWritten;
6298
+ totalRowsExecuted += chunkResult.rowsExecuted;
6299
+ totalRowsCached += chunkResult.rowsCached;
6300
+ totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
6301
+ totalRowsInserted += chunkResult.rowsInserted;
6302
+ totalRowsSkipped += chunkResult.rowsSkipped;
6303
+ totalRowsFailed += chunkResult.rowsFailed ?? 0;
6304
+ totalStepCellsCompleted += chunkResult.stepCellsCompleted ?? 0;
6305
+ totalStepCellsSkipped += chunkResult.stepCellsSkipped ?? 0;
6306
+ for (const sample of chunkResult.rowFailureSamples ?? []) {
6307
+ if (totalRowFailureSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
6308
+ break;
6309
+ }
6310
+ totalRowFailureSamples.push(sample);
6311
+ }
6312
+ enqueueMapProgressUpdate({
6313
+ completed: totalRowsWritten,
6314
+ total: rowCountHint ?? undefined,
6315
+ ...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
6316
+ message: formatMapProgressMessage(
6317
+ totalRowsWritten,
6318
+ rowCountHint ?? undefined,
6319
+ ),
6320
+ });
6321
+ if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
6322
+ previewRows.push(
6323
+ ...chunkResult.preview.slice(
6324
+ 0,
6325
+ WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
6326
+ ),
6327
+ );
6328
+ }
6329
+ if (canCacheRows) {
6330
+ const volatileRows = volatileWorkflowChunkRows.get(
6331
+ chunkResult.chunkIndex,
6332
+ );
6333
+ volatileWorkflowChunkRows.delete(chunkResult.chunkIndex);
6334
+ const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
6335
+ if (
6336
+ nextRows.length === chunkResult.rowsWritten &&
6337
+ cachedRows.length + nextRows.length <=
6338
+ WORKER_DATASET_IN_MEMORY_ROWS
6339
+ ) {
6340
+ cachedRows.push(...nextRows);
6341
+ } else {
6342
+ cachedRows.length = 0;
6343
+ volatileWorkflowChunkRows.clear();
6344
+ canCacheRows = false;
6345
+ }
6346
+ }
6347
+ // onRowError:'fail' short-circuit: once a chunk reports a row failure,
6348
+ // skip the remaining chunks entirely. The failing chunk itself completed
6349
+ // normally (no chunk-step retry storm) and persisted its rows, but
6350
+ // executing later chunks would keep spending provider credits on a run
6351
+ // the caller asked to fail fast. The post-loop fail-fast throw below
6352
+ // reports what committed before the stop.
6353
+ return failFastRowErrors && totalRowsFailed > 0;
6354
+ },
6355
+ });
6356
+ } catch (error) {
6357
+ try {
6358
+ await flushQueuedMapProgressUpdates();
6359
+ } catch (progressError) {
6360
+ throw new AggregateError(
6361
+ [error, progressError],
6362
+ `ctx.dataset("${name}") failed while flushing queued progress updates.`,
6559
6363
  );
6560
6364
  }
6561
- if (canCacheRows) {
6562
- const volatileRows = volatileWorkflowChunkRows.get(chunkIndex);
6563
- volatileWorkflowChunkRows.delete(chunkIndex);
6564
- const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
6565
- if (
6566
- nextRows.length === chunkResult.rowsWritten &&
6567
- cachedRows.length + nextRows.length <= WORKER_DATASET_IN_MEMORY_ROWS
6568
- ) {
6569
- cachedRows.push(...nextRows);
6570
- } else {
6571
- cachedRows.length = 0;
6572
- volatileWorkflowChunkRows.clear();
6573
- canCacheRows = false;
6574
- }
6575
- }
6576
- chunkStart += rawChunkRows.length;
6577
- chunkIndex += 1;
6578
- // onRowError:'fail' short-circuit: once a chunk reports a row failure,
6579
- // skip the remaining chunks entirely. The failing chunk itself completed
6580
- // normally (no chunk-step retry storm) and persisted its rows, but
6581
- // executing later chunks would keep spending provider credits on a run
6582
- // the caller asked to fail fast. The post-loop fail-fast throw below
6583
- // reports what committed before the stop.
6584
- if (failFastRowErrors && totalRowsFailed > 0) {
6585
- break;
6586
- }
6365
+ throw error;
6587
6366
  }
6588
6367
  if (totalDuplicateKeysDropped > 0) {
6589
6368
  const keySample = droppedDuplicateKeySamples.join(', ');
@@ -6596,6 +6375,13 @@ function createMinimalWorkerCtx(
6596
6375
  ts: nowMs(),
6597
6376
  });
6598
6377
  }
6378
+ if (totalRowsBlocked > 0) {
6379
+ await flushQueuedMapProgressUpdates();
6380
+ throw new RuntimeSheetRowsBlockedError({
6381
+ tableNamespace: name,
6382
+ blockedRows: blockedRowSamples,
6383
+ });
6384
+ }
6599
6385
  if (failFastRowErrors && totalRowsFailed > 0 && totalRowsWritten > 0) {
6600
6386
  // onRowError:'fail', PARTIAL failure (some rows committed): fail the run
6601
6387
  // without finalizing the dataset. The committed rows already persisted
@@ -6604,6 +6390,7 @@ function createMinimalWorkerCtx(
6604
6390
  // normally (no per-row throw inside the durable chunk step, so no
6605
6391
  // chunk-step retry storm); later chunks were skipped by the fail-fast
6606
6392
  // short-circuit in the chunk loop.
6393
+ await flushQueuedMapProgressUpdates();
6607
6394
  const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
6608
6395
  throw new Error(
6609
6396
  `ctx.dataset("${name}") failed for ${totalRowsFailed} executed row(s) under onRowError:'fail'. ` +
@@ -6621,7 +6408,7 @@ function createMinimalWorkerCtx(
6621
6408
  // step when no row otherwise succeeded) are summarized and registered as
6622
6409
  // a recovered dataset — the failed run then advertises a WORKING export
6623
6410
  // instead of a dead end (#15/#27). The run still fails (the throw below).
6624
- await finalize(totalRowsWritten);
6411
+ await finalize();
6625
6412
  const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
6626
6413
  throw new Error(
6627
6414
  `ctx.dataset("${name}") failed for all ${totalRowsFailed} executed rows. ` +
@@ -6629,7 +6416,7 @@ function createMinimalWorkerCtx(
6629
6416
  `(rows are persisted with per-row errors; fix the cause and re-run to resume)`,
6630
6417
  );
6631
6418
  }
6632
- const dataset = await finalize(totalRowsWritten);
6419
+ const dataset = await finalize();
6633
6420
  recordRunnerPerfTrace({
6634
6421
  req,
6635
6422
  phase: 'runner.map.total',
@@ -6638,7 +6425,7 @@ function createMinimalWorkerCtx(
6638
6425
  mapName: name,
6639
6426
  rowsWritten: totalRowsWritten,
6640
6427
  inputKind: rowCountHint === null ? 'streaming' : 'known_count',
6641
- chunks: chunkIndex,
6428
+ chunks: dispatchResult.chunksExecuted,
6642
6429
  },
6643
6430
  });
6644
6431
  return dataset;
@@ -6903,10 +6690,10 @@ function createMinimalWorkerCtx(
6903
6690
  request.input,
6904
6691
  workflowStep,
6905
6692
  {
6906
- force: request.force === true || !!req.force,
6693
+ force: request.force === true,
6694
+ runForceRefresh: req.forceToolRefresh === true,
6695
+ runForceFailedRefresh: req.force === true,
6907
6696
  staleAfterSeconds: request.staleAfterSeconds,
6908
- },
6909
- {
6910
6697
  timeoutMs: request.timeoutMs,
6911
6698
  receiptWaitMs: request.receiptWaitMs,
6912
6699
  },
@@ -6947,13 +6734,18 @@ function createMinimalWorkerCtx(
6947
6734
  if (!resolvedName) {
6948
6735
  throw new Error('ctx.runPlay(...) requires a resolvable play name.');
6949
6736
  }
6737
+ const childManifest = req.childPlayManifests?.[resolvedName];
6950
6738
  const receiptKey = buildDurableCtxCallCacheKey({
6951
6739
  orgId: req.orgId,
6952
6740
  playId: req.playName,
6953
6741
  kind: 'runPlay',
6954
6742
  id: normalizedKey,
6955
- semanticKey: await hashJson({
6743
+ semanticKey: buildDurableRunPlaySemanticKey({
6956
6744
  childPlayName: resolvedName,
6745
+ childRevisionFingerprint: workerChildRevisionFingerprint({
6746
+ playName: resolvedName,
6747
+ manifest: childManifest,
6748
+ }),
6957
6749
  input,
6958
6750
  }),
6959
6751
  staleAfterSeconds: options?.staleAfterSeconds,
@@ -6981,7 +6773,6 @@ function createMinimalWorkerCtx(
6981
6773
  message: `Starting child play ${resolvedName} (${normalizedKey})`,
6982
6774
  ts: nowMs(),
6983
6775
  });
6984
- const childManifest = req.childPlayManifests?.[resolvedName];
6985
6776
  if (!childManifest) {
6986
6777
  throw new Error(
6987
6778
  `ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
@@ -7019,7 +6810,7 @@ function createMinimalWorkerCtx(
7019
6810
  });
7020
6811
  let childPlaySlot: { release(): void } | null = null;
7021
6812
  try {
7022
- childPlaySlot = await governor.acquireChildPlaySlot({
6813
+ childPlaySlot = await governor.acquireChildSubmitSlot({
7023
6814
  signal: abortSignal,
7024
6815
  });
7025
6816
  const childSubmitStartedAt = nowMs();
@@ -7044,12 +6835,12 @@ function createMinimalWorkerCtx(
7044
6835
  options?.timeoutMs == null && !childNeedsWorkflowScheduler,
7045
6836
  body: {
7046
6837
  name: resolvedName,
6838
+ childIdempotencyKey: receiptKey,
7047
6839
  input: isRecord(input) ? input : {},
7048
6840
  orgId: req.orgId,
7049
6841
  callbackBaseUrl: req.callbackUrl,
7050
6842
  baseUrl: req.baseUrl,
7051
6843
  integrationMode: req.integrationMode ?? null,
7052
- forceToolRefresh: req.force === true,
7053
6844
  parentExecutorToken: req.executorToken,
7054
6845
  userEmail: req.userEmail ?? '',
7055
6846
  profile: 'workers_edge',
@@ -7161,6 +6952,8 @@ function createMinimalWorkerCtx(
7161
6952
  childNeedsWorkflowScheduler,
7162
6953
  },
7163
6954
  });
6955
+ childPlaySlot?.release();
6956
+ childPlaySlot = null;
7164
6957
  const startedStatus = String(started.status ?? '').toLowerCase();
7165
6958
  if (startedStatus === 'completed') {
7166
6959
  emitEvent({
@@ -7189,6 +6982,10 @@ function createMinimalWorkerCtx(
7189
6982
  }
7190
6983
  const childWaitStartedAt = nowMs();
7191
6984
  let waitResult: ChildPlayTerminalWaitResult;
6985
+ const readCachedChildTerminalState =
6986
+ cachedCoordinatorBinding?.readChildTerminalState?.bind(
6987
+ cachedCoordinatorBinding,
6988
+ );
7192
6989
  try {
7193
6990
  waitResult = await awaitChildTerminal({
7194
6991
  parentRunId: req.runId,
@@ -7196,8 +6993,7 @@ function createMinimalWorkerCtx(
7196
6993
  // the small structural shape ChildPlayAwait needs; bridge it the
7197
6994
  // same way the inline implementation did.
7198
6995
  workflowStep: workflowStep as unknown as
7199
- | WorkflowStepLike
7200
- | undefined,
6996
+ WorkflowStepLike | undefined,
7201
6997
  workflowId,
7202
6998
  playName: resolvedName,
7203
6999
  key: normalizedKey,
@@ -7205,20 +7001,31 @@ function createMinimalWorkerCtx(
7205
7001
  1_000,
7206
7002
  Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
7207
7003
  ),
7208
- coordinator: cachedCoordinatorBinding?.readChildTerminalState
7209
- ? {
7210
- readChildTerminalState: (
7211
- parentRunId,
7212
- eventKey,
7213
- timeoutMs,
7214
- ) =>
7215
- cachedCoordinatorBinding!.readChildTerminalState!(
7216
- parentRunId,
7217
- eventKey,
7218
- timeoutMs,
7219
- ),
7220
- }
7221
- : null,
7004
+ coordinator: {
7005
+ readRunTerminalState: (runId, timeoutMs) =>
7006
+ readDurableChildRunTerminalState({
7007
+ baseUrl: req.baseUrl,
7008
+ executorToken: req.executorToken,
7009
+ parentRunId: req.runId,
7010
+ childRunId: runId,
7011
+ childPlayName: resolvedName,
7012
+ timeoutMs,
7013
+ }),
7014
+ ...(readCachedChildTerminalState
7015
+ ? {
7016
+ readChildTerminalState: (
7017
+ parentRunId: string,
7018
+ eventKey: string,
7019
+ timeoutMs?: number,
7020
+ ) =>
7021
+ readCachedChildTerminalState(
7022
+ parentRunId,
7023
+ eventKey,
7024
+ timeoutMs,
7025
+ ),
7026
+ }
7027
+ : {}),
7028
+ },
7222
7029
  now: nowMs,
7223
7030
  hashJson,
7224
7031
  });
@@ -7401,6 +7208,7 @@ function createMinimalWorkerCtx(
7401
7208
  // Direct Worker fetch below is still limited to runtime origins and
7402
7209
  // IP literals; public IP literals also consume this slot because
7403
7210
  // they are customer egress.
7211
+ workBudgetMeter.count('egress');
7404
7212
  const egressResponse = await postRuntimeEgressFetch(
7405
7213
  req,
7406
7214
  {
@@ -7423,6 +7231,7 @@ function createMinimalWorkerCtx(
7423
7231
  }
7424
7232
  const fetchInit = { ...init, headers };
7425
7233
  delete fetchInit.auth;
7234
+ workBudgetMeter.count('egress');
7426
7235
  const response = await safeWorkerPublicFetch(url, fetchInit, {
7427
7236
  allowedOrigins,
7428
7237
  sensitiveHeaders: Object.keys(secretHeaderMarkers),
@@ -7493,6 +7302,7 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
7493
7302
  let req: RunRequest;
7494
7303
  try {
7495
7304
  req = (await request.json()) as RunRequest;
7305
+ req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
7496
7306
  } catch {
7497
7307
  return new Response('invalid JSON body', { status: 400 });
7498
7308
  }
@@ -7557,6 +7367,7 @@ async function handleRunInline(
7557
7367
  let req: RunRequest;
7558
7368
  try {
7559
7369
  req = (await request.json()) as RunRequest;
7370
+ req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
7560
7371
  } catch {
7561
7372
  return Response.json(
7562
7373
  {
@@ -7704,6 +7515,7 @@ async function executeRunRequest(
7704
7515
  ): Promise<WorkflowRunOutput> {
7705
7516
  installProcessExitTrap();
7706
7517
  const startedAt = nowMs();
7518
+ const workBudgetMeter = createWorkerWorkBudgetMeter({ nowMs });
7707
7519
  recordRunnerPerfTrace({
7708
7520
  req,
7709
7521
  phase: 'runner.execute_start',
@@ -7741,6 +7553,7 @@ async function executeRunRequest(
7741
7553
  const appendRunLogLine = (line: string) => {
7742
7554
  const trimmed = redactSecretsFromLogString(line.trim());
7743
7555
  if (!trimmed) return;
7556
+ workBudgetMeter.count('log');
7744
7557
  totalEmittedLogLines += 1;
7745
7558
  runLogBuffer = [...runLogBuffer, trimmed].slice(-RUN_LOG_BUFFER_LIMIT);
7746
7559
  pendingRunLogLines = [...pendingRunLogLines, trimmed];
@@ -7933,7 +7746,10 @@ async function executeRunRequest(
7933
7746
  const flushTerminalLedgerEvents = async (
7934
7747
  terminalEvent: PlayRunLedgerEvent,
7935
7748
  ): Promise<void> => {
7936
- if (!options?.persistResultDatasets) return;
7749
+ const shouldPersistTerminalLedger =
7750
+ options?.persistResultDatasets === true ||
7751
+ Boolean(req.playCallGovernance);
7752
+ if (!shouldPersistTerminalLedger) return;
7937
7753
  await ledgerFlushInFlight;
7938
7754
  const now = nowMs();
7939
7755
  dirtyProgressNodeIds = new Set([
@@ -8008,10 +7824,7 @@ async function executeRunRequest(
8008
7824
  stepLifecycle?.markPreDatasetStepsStarted(startedAt);
8009
7825
  flushLedgerEvents(false);
8010
7826
  const runtimeDeadlineMs = nowMs() + STANDARD_PLAY_RUNTIME_LIMIT_MS;
8011
- // Run-scoped buffer for provider results that billed but whose durable
8012
- // receipt could not be marked completed even after the retry ladder. On
8013
- // terminal failure the buffer is built into a size-capped salvage record and
8014
- // attached to the run.failed ledger event so billed data is never memory-only.
7827
+ const leasedSheetNamespaces = new Set<string>();
8015
7828
  const receiptSalvage = createReceiptSalvageBuffer();
8016
7829
  const ctx = createMinimalWorkerCtx(
8017
7830
  req,
@@ -8020,7 +7833,9 @@ async function executeRunRequest(
8020
7833
  workflowStep,
8021
7834
  abortSignal,
8022
7835
  workerCallbacks,
7836
+ workBudgetMeter,
8023
7837
  runtimeDeadlineMs,
7838
+ leasedSheetNamespaces,
8024
7839
  receiptSalvage,
8025
7840
  );
8026
7841
  // Hard wall-clock cap on active user-code runtime. CF Workflows does not
@@ -8029,15 +7844,18 @@ async function executeRunRequest(
8029
7844
  // token expires. Aborting the controller surfaces cooperatively through the
8030
7845
  // same assertNotAborted checks used for harness cancellation.
8031
7846
  let runtimeLimitExceeded = false;
8032
- const runtimeDeadlineTimer = setTimeout(
8033
- () => {
8034
- runtimeLimitExceeded = true;
8035
- if (!abortSignal.aborted) {
8036
- abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
8037
- }
8038
- },
8039
- Math.max(1, runtimeDeadlineMs - nowMs()),
8040
- );
7847
+ const runtimeDeadlineTimer =
7848
+ workflowStep === undefined || workflowStep === null
7849
+ ? setTimeout(
7850
+ () => {
7851
+ runtimeLimitExceeded = true;
7852
+ if (!abortSignal.aborted) {
7853
+ abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
7854
+ }
7855
+ },
7856
+ Math.max(1, runtimeDeadlineMs - nowMs()),
7857
+ )
7858
+ : null;
8041
7859
  try {
8042
7860
  const playStartedAt = nowMs();
8043
7861
  const result = await (
@@ -8128,7 +7946,6 @@ async function executeRunRequest(
8128
7946
  }
8129
7947
  if (options?.persistResultDatasets) {
8130
7948
  await persistProjectedResultDatasets();
8131
- const parentSignal = startParentTerminalSignal();
8132
7949
  // Capped runs settle compute billing BEFORE declaring run.completed: a
8133
7950
  // per-run cap denial (422 billing_cap_exceeded) must fail the run as
8134
7951
  // its ONLY terminal. Flushing completed first opens a race — watchers
@@ -8163,6 +7980,7 @@ async function executeRunRequest(
8163
7980
  ms: nowMs() - terminalUpdateStartedAt,
8164
7981
  });
8165
7982
 
7983
+ const parentSignal = startParentTerminalSignal();
8166
7984
  if (!capped) {
8167
7985
  const billingStartedAt = nowMs();
8168
7986
  const billingPromise = finalizeWorkerComputeBilling({
@@ -8190,6 +8008,20 @@ async function executeRunRequest(
8190
8008
  }
8191
8009
  }
8192
8010
  await parentSignal;
8011
+ } else if (req.playCallGovernance) {
8012
+ const childTerminalStartedAt = nowMs();
8013
+ await flushTerminalLedgerEvents({
8014
+ type: 'run.completed',
8015
+ runId: req.runId,
8016
+ source: 'worker',
8017
+ occurredAt: nowMs(),
8018
+ result: terminalResult,
8019
+ });
8020
+ recordRunnerPerfTrace({
8021
+ req,
8022
+ phase: 'runner.child_terminal_ledger_append',
8023
+ ms: nowMs() - childTerminalStartedAt,
8024
+ });
8193
8025
  }
8194
8026
  await startParentTerminalSignal();
8195
8027
  recordRunnerPerfTrace({
@@ -8216,16 +8048,8 @@ async function executeRunRequest(
8216
8048
  stepLifecycle?.markStartedFailed(nowMs());
8217
8049
  // A runtime-limit abort is a timeout failure, not a user cancellation, so
8218
8050
  // it should be reported as run.failed with the limit message rather than
8219
- // run.cancelled. The deadline can also surface as a WorkflowAbortError
8220
- // thrown synchronously from the completeReceipt retry ladder BEFORE the
8221
- // runtimeDeadlineTimer macrotask flips the flag — match the limit message
8222
- // directly so that race can never misclassify a timeout as a cancel (which
8223
- // would silently drop the salvage buffer of billed-but-unpersisted rows).
8224
- const runtimeLimitError =
8225
- runtimeLimitExceeded ||
8226
- (error instanceof Error &&
8227
- error.message === STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
8228
- const aborted = isAbortLikeError(error) && !runtimeLimitError;
8051
+ // run.cancelled.
8052
+ const aborted = isAbortLikeError(error) && !runtimeLimitExceeded;
8229
8053
  if (aborted) {
8230
8054
  // Flip the controller so any concurrent user code observes the abort
8231
8055
  // through ctx.signal. We mark the run cancelled instead of failed.
@@ -8234,19 +8058,28 @@ async function executeRunRequest(
8234
8058
  );
8235
8059
  }
8236
8060
  const failure = normalizePlayRunFailure(error);
8237
- // Billed-but-unpersisted salvage rides on the run.failed event's `result`.
8238
- // The run STILL FAILS salvage is a loud marker, never a success fallback.
8061
+ const message = failure.message;
8062
+ // Controlled run-fatal teardown: release the runtime-sheet row leases and
8063
+ // work-receipt leases this attempt still holds so an immediate rerun is not
8064
+ // blocked for the full lease TTL. Best-effort but LOUD — a release failure
8065
+ // is logged and never masks the original run-fatal error. TTL expiry stays
8066
+ // the recovery path for true crashes (isolate death) where this never runs.
8067
+ await releaseRuntimeLeasesOnTeardown({
8068
+ req,
8069
+ leasedSheetNamespaces,
8070
+ emit: wrappedEmit,
8071
+ });
8072
+ const errorBilling = extractErrorBilling(error);
8239
8073
  const salvage: ReceiptSalvage | null = aborted
8240
8074
  ? null
8241
8075
  : receiptSalvage.build();
8242
- const message =
8076
+ const terminalMessage =
8243
8077
  salvage && salvage.totalEntries > 0
8244
- ? `${failure.message} ${receiptSalvageFailureSuffix(salvage)}`
8245
- : failure.message;
8246
- const errorBilling = extractErrorBilling(error);
8247
- if (options?.persistResultDatasets) {
8078
+ ? `${message} ${receiptSalvageFailureSuffix(salvage)}`
8079
+ : message;
8080
+ if (options?.persistResultDatasets || req.playCallGovernance) {
8248
8081
  appendRunLogLine(
8249
- `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(message)}`,
8082
+ `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(terminalMessage)}`,
8250
8083
  );
8251
8084
  const terminalUpdateStartedAt = nowMs();
8252
8085
  await flushTerminalLedgerEvents({
@@ -8254,64 +8087,70 @@ async function executeRunRequest(
8254
8087
  runId: req.runId,
8255
8088
  source: 'worker',
8256
8089
  occurredAt: nowMs(),
8257
- error: message,
8090
+ error: terminalMessage,
8258
8091
  result: aborted
8259
8092
  ? undefined
8260
8093
  : {
8261
8094
  success: false,
8262
8095
  status: 'failed',
8263
- error: message,
8096
+ error: terminalMessage,
8097
+ ...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
8264
8098
  errors: [
8265
8099
  {
8266
8100
  code: failure.code,
8267
8101
  phase: failure.phase,
8268
- message,
8102
+ message: terminalMessage,
8269
8103
  retryable: failure.retryable,
8270
8104
  ...(errorBilling ? { billing: errorBilling } : {}),
8271
8105
  ...(failure.cause ? { cause: failure.cause } : {}),
8272
8106
  },
8273
8107
  ],
8274
- ...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
8275
8108
  },
8276
8109
  });
8277
8110
  recordRunnerPerfTrace({
8278
8111
  req,
8279
- phase: aborted
8280
- ? 'runner.terminal_ledger_append_cancelled'
8281
- : 'runner.terminal_ledger_append_failed',
8112
+ phase: req.playCallGovernance
8113
+ ? aborted
8114
+ ? 'runner.child_terminal_ledger_append_cancelled'
8115
+ : 'runner.child_terminal_ledger_append_failed'
8116
+ : aborted
8117
+ ? 'runner.terminal_ledger_append_cancelled'
8118
+ : 'runner.terminal_ledger_append_failed',
8282
8119
  ms: nowMs() - terminalUpdateStartedAt,
8283
8120
  extra: {
8284
8121
  errorCode: failure.code,
8285
8122
  errorPhase: failure.phase,
8286
8123
  },
8287
8124
  });
8288
- const billingStartedAt = nowMs();
8289
- await finalizeWorkerComputeBilling({
8290
- req,
8291
- success: false,
8292
- actionEstimate: 4,
8293
- })
8294
- .catch((finalizeError) => {
8295
- console.error(
8296
- `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
8297
- finalizeError instanceof Error
8298
- ? finalizeError.message
8299
- : String(finalizeError)
8300
- }`,
8301
- );
8125
+ if (options?.persistResultDatasets) {
8126
+ const billingStartedAt = nowMs();
8127
+ await finalizeWorkerComputeBilling({
8128
+ req,
8129
+ success: false,
8130
+ actionEstimate: 4,
8302
8131
  })
8303
- .finally(() => {
8304
- recordRunnerPerfTrace({
8305
- req,
8306
- phase: 'runner.compute_billing_finalize_failed',
8307
- ms: nowMs() - billingStartedAt,
8132
+ .catch((finalizeError) => {
8133
+ console.error(
8134
+ `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
8135
+ finalizeError instanceof Error
8136
+ ? finalizeError.message
8137
+ : String(finalizeError)
8138
+ }`,
8139
+ );
8140
+ })
8141
+ .finally(() => {
8142
+ recordRunnerPerfTrace({
8143
+ req,
8144
+ phase: 'runner.compute_billing_finalize_failed',
8145
+ ms: nowMs() - billingStartedAt,
8146
+ });
8308
8147
  });
8309
- });
8148
+ }
8310
8149
  }
8311
8150
  await signalParentPlayTerminal({
8312
8151
  req,
8313
8152
  status: aborted ? 'cancelled' : 'failed',
8314
- error: message,
8153
+ error: terminalMessage,
8315
8154
  }).catch(() => null);
8316
8155
  recordRunnerPerfTrace({
8317
8156
  req,
@@ -8325,7 +8164,9 @@ async function executeRunRequest(
8325
8164
  await drainRunnerPerfTraces(req);
8326
8165
  throw error;
8327
8166
  } finally {
8328
- clearTimeout(runtimeDeadlineTimer);
8167
+ if (runtimeDeadlineTimer) {
8168
+ clearTimeout(runtimeDeadlineTimer);
8169
+ }
8329
8170
  }
8330
8171
  }
8331
8172
 
@@ -8417,10 +8258,12 @@ function runRequestFromWorkflowParams(
8417
8258
  orgId: String(params.orgId ?? ''),
8418
8259
  playName: String(params.playName ?? ''),
8419
8260
  userEmail: typeof params.userEmail === 'string' ? params.userEmail : null,
8261
+ force: params.force === true,
8262
+ forceToolRefresh: params.forceToolRefresh === true,
8263
+ runAttempt: normalizeWorkerRunAttempt(params.runAttempt),
8420
8264
  runtimeInput: isRecord(params.input)
8421
8265
  ? (params.input as Record<string, unknown>)
8422
8266
  : {},
8423
- force: params.forceToolRefresh === true,
8424
8267
  inlineCsv: isInlineCsv(params.inlineCsv) ? params.inlineCsv : null,
8425
8268
  inputFiles:
8426
8269
  inputFile && inputStorageKey
@@ -8478,6 +8321,14 @@ function runRequestFromWorkflowParams(
8478
8321
  params.coordinatorInternalToken.trim()
8479
8322
  ? params.coordinatorInternalToken.trim()
8480
8323
  : null,
8324
+ runtimeTestFaultHeader:
8325
+ typeof params.runtimeTestFaultHeader === 'string' &&
8326
+ params.runtimeTestFaultHeader.trim()
8327
+ ? params.runtimeTestFaultHeader.trim()
8328
+ : null,
8329
+ testPolicyOverrides: normalizeRuntimeTestPolicyOverrides(
8330
+ params.testPolicyOverrides,
8331
+ ),
8481
8332
  totalRows:
8482
8333
  typeof params.totalRows === 'number' && Number.isFinite(params.totalRows)
8483
8334
  ? params.totalRows
@@ -8547,6 +8398,7 @@ async function persistResultDatasets(
8547
8398
  rows: chunk.map((row) => ({ ...row })),
8548
8399
  runId: req.runId,
8549
8400
  inputOffset,
8401
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
8550
8402
  userEmail: req.userEmail,
8551
8403
  preloadedDbSessions: req.preloadedDbSessions ?? null,
8552
8404
  });
@@ -8583,6 +8435,7 @@ async function persistResultDatasets(
8583
8435
  rows: dataset.rows,
8584
8436
  runId: req.runId,
8585
8437
  inputOffset: 0,
8438
+ attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
8586
8439
  userEmail: req.userEmail,
8587
8440
  });
8588
8441
  }
@@ -8864,7 +8717,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
8864
8717
  try {
8865
8718
  const response = await fetchRuntimeApi(
8866
8719
  req.baseUrl,
8867
- '/api/v2/plays/internal/runtime',
8720
+ PLAY_RUNTIME_API_COMPAT_PATH,
8868
8721
  {
8869
8722
  method: 'POST',
8870
8723
  headers: {