deepline 0.1.181 → 0.1.183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
  2. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
  3. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +633 -107
  4. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
  5. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1932 -1838
  6. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
  7. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
  8. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +124 -12
  9. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
  10. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
  11. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
  12. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
  13. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
  14. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
  15. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
  16. package/dist/bundling-sources/sdk/src/client.ts +48 -5
  17. package/dist/bundling-sources/sdk/src/http.ts +23 -8
  18. package/dist/bundling-sources/sdk/src/play.ts +200 -26
  19. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
  20. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  21. package/dist/bundling-sources/sdk/src/types.ts +2 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +132 -10
  23. package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
  25. package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
  26. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
  27. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1857 -314
  28. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +106 -6
  29. package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
  33. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
  34. package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
  35. package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
  36. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
  37. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
  38. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
  40. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
  42. package/dist/bundling-sources/shared_libs/play-runtime/query-result-dataset.ts +120 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
  44. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
  45. package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
  46. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
  47. package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
  48. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2037 -262
  50. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
  52. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
  53. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
  54. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
  56. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
  57. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
  58. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
  59. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +139 -17
  60. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
  61. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
  62. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
  63. package/dist/cli/index.js +85 -23
  64. package/dist/cli/index.mjs +85 -23
  65. package/dist/index.d.mts +4 -1
  66. package/dist/index.d.ts +4 -1
  67. package/dist/index.js +274 -35
  68. package/dist/index.mjs +274 -35
  69. package/dist/plays/bundle-play-file.mjs +2 -2
  70. package/package.json +1 -1
@@ -23,12 +23,24 @@ import {
23
23
  } from './batch-runtime';
24
24
  import type { PlayQueueHint } from './governor/rate-state-backend';
25
25
  import type { MapRowOutcome } from './durability-store';
26
+ import type { WorkReceiptFailureKind } from './work-receipts';
26
27
  import {
27
28
  completedMapRowOutcome,
28
29
  failedMapRowOutcome,
29
30
  mapRowOutcomeRuntimeFields,
30
31
  resolveMapRowOutcomeKey,
31
32
  } from './map-row-outcome';
33
+ import { RuntimeSheetRowsBlockedError } from './runtime-sheet-errors';
34
+ import { buildChildRunId } from './child-run-id';
35
+ import type { PlayCallGovernanceSnapshot } from './scheduler-backend';
36
+ import {
37
+ RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER,
38
+ SYNTHETIC_RUN_HEADER,
39
+ } from './coordinator-headers';
40
+ export {
41
+ RuntimeSheetRowsBlockedError,
42
+ type RuntimeSheetBlockedRowDetail,
43
+ } from './runtime-sheet-errors';
32
44
  import {
33
45
  createDefaultGovernanceSnapshot,
34
46
  createPlayExecutionGovernor,
@@ -41,6 +53,7 @@ import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backen
41
53
  import { pacingPolicyForTool } from './pacing';
42
54
  import {
43
55
  cloneToolExecuteResultWithExecution,
56
+ attachToolResultListDataset,
44
57
  createToolExecuteResult,
45
58
  parseToolExecuteResponse,
46
59
  isToolExecuteResult,
@@ -63,24 +76,38 @@ import {
63
76
  TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
64
77
  classifyToolExecuteHttpFailure,
65
78
  createToolExecuteHttpFailureAttemptTracker,
79
+ parseToolExecuteAuthScopeChangedError,
80
+ ToolExecuteAuthScopeChangedError,
66
81
  } from './tool-execute-retry-policy';
67
82
  import {
68
83
  buildDurableCtxCallCacheKey,
84
+ buildDurableRunPlaySemanticKey,
85
+ buildDurableToolAggregateProviderIdempotencyKey,
86
+ buildDurableToolAggregateReceiptKey,
69
87
  buildDurableToolCallAuthScopeDigest,
70
88
  buildDurableToolCallCacheKey,
89
+ buildDurableToolProviderIdempotencyKey,
90
+ buildDurableToolReceiptPrefix,
71
91
  } from './durable-call-cache';
72
92
  import {
93
+ RuntimeReceiptLeaseLostError,
73
94
  RuntimeReceiptWaitTimeoutError,
74
95
  executeWithDurableRuntimeReceipt,
75
96
  resolveRuntimeToolReceiptWaitMaxAttempts,
97
+ runtimeReceiptFailureKindForError,
76
98
  runtimeReceiptOutput as durableRuntimeReceiptOutput,
77
99
  waitForCompletedRuntimeReceipt,
78
100
  type DurableReceiptExecutionStore,
79
101
  } from './durable-receipt-execution';
102
+ import {
103
+ QUERY_RESULT_DATASET_PAGE_SIZE,
104
+ isCustomerDbDatasetTool,
105
+ isQueryResultDatasetReadRequest,
106
+ isQueryResultDatasetTool,
107
+ } from './query-result-dataset';
80
108
  import { isRowIsolationExemptError } from './row-isolation';
81
109
  import {
82
110
  createRuntimePersistenceLatch,
83
- PROVIDER_DISPATCH_WAVE_SIZE,
84
111
  RuntimePersistenceCircuitOpenError,
85
112
  tripRuntimePersistenceLatch,
86
113
  type RuntimePersistenceLatch,
@@ -112,7 +139,9 @@ import { DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE } from './runtime-constraints';
112
139
  import {
113
140
  PlayExecutionSuspendedError,
114
141
  PlayRowExecutionSuspendedError,
142
+ isPlayExecutionSuspendedError,
115
143
  isPlayRowExecutionSuspendedError,
144
+ type PlayExecutionSuspension,
116
145
  } from './suspension';
117
146
  import {
118
147
  createSecretRedactionContext,
@@ -164,6 +193,7 @@ import {
164
193
  type StepProgramDatasetColumnInput,
165
194
  type StepProgramDatasetOptions,
166
195
  } from './step-program-dataset-builder';
196
+ import { readRuntimeSheetDatasetRows } from './runtime-api';
167
197
 
168
198
  /**
169
199
  * SECURITY: AsyncLocalStorage is per async execution, not a cross-run cache.
@@ -200,9 +230,118 @@ const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
200
230
  const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
201
231
  const FETCH_TRANSPORT_MAX_ATTEMPTS = 3;
202
232
  const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
233
+ const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100;
234
+ const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25;
203
235
  type SafeFetchModule = typeof import('@shared_libs/security/safe-fetch');
204
236
  let safeFetchModule: Promise<SafeFetchModule> | null = null;
205
237
 
238
+ export async function waitForNodeRuntimeMapRowsVisible(input: {
239
+ mapName: string;
240
+ tableNamespace: string;
241
+ runId?: string | null;
242
+ expectedRows: number;
243
+ updatedRows: number;
244
+ readVisibleRowCount: () => Promise<number>;
245
+ sleep?: (ms: number) => Promise<void>;
246
+ log?: (line: string) => void;
247
+ }): Promise<number> {
248
+ if (input.expectedRows <= 0) return 0;
249
+
250
+ const sleep =
251
+ input.sleep ??
252
+ ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
253
+ let lastVisibleRows = -1;
254
+ for (
255
+ let attempt = 1;
256
+ attempt <= NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS;
257
+ attempt += 1
258
+ ) {
259
+ lastVisibleRows = await input.readVisibleRowCount();
260
+ if (lastVisibleRows >= input.expectedRows) {
261
+ if (attempt > 1) {
262
+ input.log?.(
263
+ `Runtime sheet visibility barrier satisfied ctx.dataset("${input.mapName}") ` +
264
+ `after ${attempt} read(s): visible ${lastVisibleRows}/${input.expectedRows}; ` +
265
+ `wrote ${input.updatedRows}/${input.expectedRows}; run ${input.runId ?? 'unknown'}.`,
266
+ );
267
+ }
268
+ return lastVisibleRows;
269
+ }
270
+
271
+ if (attempt === 1) {
272
+ input.log?.(
273
+ `Runtime sheet visibility barrier waiting ctx.dataset("${input.mapName}"): ` +
274
+ `visible ${lastVisibleRows}/${input.expectedRows}; ` +
275
+ `wrote ${input.updatedRows}/${input.expectedRows}; run ${input.runId ?? 'unknown'}.`,
276
+ );
277
+ }
278
+ if (attempt < NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS) {
279
+ await sleep(NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS);
280
+ }
281
+ }
282
+
283
+ throw new Error(
284
+ `Runtime sheet visibility mismatch for ctx.dataset("${input.mapName}"): ` +
285
+ `expected ${input.expectedRows} terminal visible row(s), saw ${lastVisibleRows}; ` +
286
+ `write reported ${input.updatedRows}; run ${input.runId ?? 'unknown'}.`,
287
+ );
288
+ }
289
+
290
+ export async function reconcileNodeRuntimeMapResultsWithPersistedSheet(input: {
291
+ mapName: string;
292
+ tableNamespace: string;
293
+ runId?: string | null;
294
+ expectedRows: number;
295
+ currentRows: Record<string, unknown>[];
296
+ failedRowCount: number;
297
+ readPersistedRows: (input: {
298
+ limit: number;
299
+ offset: number;
300
+ rowMode: 'all';
301
+ }) => Promise<Record<string, unknown>[]>;
302
+ log?: (line: string) => void;
303
+ }): Promise<Record<string, unknown>[]> {
304
+ if (
305
+ input.failedRowCount > 0 ||
306
+ input.expectedRows <= 0 ||
307
+ input.currentRows.length >= input.expectedRows
308
+ ) {
309
+ return input.currentRows;
310
+ }
311
+
312
+ const persistedRows: Record<string, unknown>[] = [];
313
+ let offset = 0;
314
+ while (persistedRows.length < input.expectedRows) {
315
+ const page = await input.readPersistedRows({
316
+ limit: Math.min(10_000, input.expectedRows - persistedRows.length),
317
+ offset,
318
+ rowMode: 'all',
319
+ });
320
+ if (page.length === 0) break;
321
+ persistedRows.push(...page);
322
+ offset += page.length;
323
+ }
324
+
325
+ if (persistedRows.length < input.expectedRows) {
326
+ throw new Error(
327
+ `Runtime sheet finalization mismatch for ctx.dataset("${input.mapName}"): ` +
328
+ `expected ${input.expectedRows} terminal persisted row(s), saw ${persistedRows.length}; ` +
329
+ `in-memory Node map results reported ${input.currentRows.length}; run ${input.runId ?? 'unknown'}.`,
330
+ );
331
+ }
332
+
333
+ if (persistedRows.length > input.currentRows.length) {
334
+ input.log?.(
335
+ `Runtime sheet finalization reconciled ctx.dataset("${input.mapName}") ` +
336
+ `from ${input.currentRows.length}/${input.expectedRows} in-memory Node row(s) ` +
337
+ `to ${persistedRows.length} terminal persisted row(s).`,
338
+ );
339
+ return persistedRows;
340
+ }
341
+
342
+ return input.currentRows;
343
+ }
344
+
206
345
  export function resolveToolRuntimeTimeoutMs(
207
346
  toolId: string,
208
347
  requestedTimeoutMs?: number,
@@ -266,7 +405,47 @@ function publicToolResponseEnvelope(value: unknown): {
266
405
 
267
406
  const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
268
407
  const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
408
+ const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
269
409
  const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-response';
410
+ function recordOrNull(value: unknown): Record<string, unknown> | null {
411
+ return value && typeof value === 'object' && !Array.isArray(value)
412
+ ? (value as Record<string, unknown>)
413
+ : null;
414
+ }
415
+
416
+ function rowsFromUnknown(value: unknown): Record<string, unknown>[] {
417
+ if (!Array.isArray(value)) return [];
418
+ return value.map((row) =>
419
+ row && typeof row === 'object' && !Array.isArray(row)
420
+ ? (row as Record<string, unknown>)
421
+ : { value: row },
422
+ );
423
+ }
424
+
425
+ function finiteNonNegativeInteger(value: unknown): number | null {
426
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
427
+ return null;
428
+ }
429
+ return Math.floor(value);
430
+ }
431
+
432
+ function finitePositiveInteger(value: unknown): number | null {
433
+ const integer = finiteNonNegativeInteger(value);
434
+ return integer !== null && integer > 0 ? integer : null;
435
+ }
436
+
437
+ type ToolExecutionApiOptions = {
438
+ timeoutMs?: number;
439
+ durableCallReceiptKey?: string | null;
440
+ executionAuthScopeDigest?: string | null;
441
+ providerIdempotencyKey?: string | null;
442
+ customerDbDataset?: {
443
+ limit: number;
444
+ offset: number;
445
+ pageSize: number;
446
+ totalRows: number;
447
+ };
448
+ };
270
449
  const IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT = 25;
271
450
  const BATCH_SIZE_LOG_SAMPLE_LIMIT = 10;
272
451
  const STEP_PROGRAM_MAP_DEFINITION = Symbol('deepline.stepProgramMapDefinition');
@@ -303,6 +482,16 @@ function persistableMapRowBytes(row: PersistableMapRow): number {
303
482
  return JSON.stringify(row).length;
304
483
  }
305
484
 
485
+ function failedMapRowLogLabel(row: PersistableMapRow): string {
486
+ const rowId = row.data?.row_id;
487
+ if (typeof rowId === 'string' && rowId.trim()) return rowId.trim();
488
+ if (row.key?.trim()) return row.key.trim();
489
+ if (typeof row.inputIndex === 'number' && Number.isFinite(row.inputIndex)) {
490
+ return `input:${Math.floor(row.inputIndex)}`;
491
+ }
492
+ return 'unknown';
493
+ }
494
+
306
495
  type FieldMapRunResult = {
307
496
  completedRows: PersistableMapRow[];
308
497
  failedRows: PersistableMapRow[];
@@ -464,9 +653,7 @@ function durableCtxKey(input: {
464
653
  staleAfterSeconds?: number | null;
465
654
  }): string {
466
655
  if (input.operation === 'tool') {
467
- throw new Error(
468
- 'Durable ctx key is only for non-tool call boundaries; tools use buildDurableToolCallCacheKey.',
469
- );
656
+ throw new Error('Tool calls use tool receipt keys.');
470
657
  }
471
658
  return buildDurableCtxCallCacheKey({
472
659
  orgId: input.orgId,
@@ -712,6 +899,13 @@ export class PlayContextImpl {
712
899
  eventKey: string;
713
900
  timeoutMs: number;
714
901
  }> = [];
902
+ private pendingChildPlayBoundaries: Array<{
903
+ boundaryId: string;
904
+ childRunId: string;
905
+ childPlayName: string;
906
+ }> = [];
907
+ private activeScheduledChildPlayCalls = 0;
908
+ private scheduledChildPlayQuiescenceWaiters: Array<() => void> = [];
715
909
  private processedRowCount = 0;
716
910
  private sleepBoundaryIndex = 0;
717
911
  private fetchCallIndex = 0;
@@ -847,13 +1041,18 @@ export class PlayContextImpl {
847
1041
  options.workflowId ||
848
1042
  'anonymous-play';
849
1043
  const rootRunId = options.runId ?? options.workflowId ?? 'anonymous-run';
1044
+ if (options.requireSharedRateState && !options.rateState) {
1045
+ throw new Error(
1046
+ 'Shared rate-state backend is required for this Node runtime substrate.',
1047
+ );
1048
+ }
850
1049
  this.governor = createPlayExecutionGovernor({
851
1050
  adapter: 'cjs_node20',
852
1051
  scope: {
853
1052
  orgId: options.orgId ?? 'org',
854
1053
  rootRunId: options.runId ?? options.workflowId ?? 'run',
855
1054
  },
856
- rateState: new InMemoryRateStateBackend(),
1055
+ rateState: options.rateState ?? new InMemoryRateStateBackend(),
857
1056
  resolvePacing: createPacingResolver(options.getToolQueueHints),
858
1057
  // A root run keeps depth 0 (its first child play is depth 1) but still
859
1058
  // seeds its own play id into the ancestry/currentPlayId so the cycle guard
@@ -879,6 +1078,263 @@ export class PlayContextImpl {
879
1078
  return `${this.governance.currentRunId}:${localId}`;
880
1079
  }
881
1080
 
1081
+ private shouldLaunchChildPlaysThroughScheduler(): boolean {
1082
+ return (
1083
+ this.#options.requireSharedRateState === true &&
1084
+ this.#options.durableBoundaries === true
1085
+ );
1086
+ }
1087
+
1088
+ private childPlayBoundaryId(childRunId: string): string {
1089
+ return this.durableBoundaryId(`runPlay:${childRunId}`);
1090
+ }
1091
+
1092
+ private recordPendingChildPlayBoundary(input: {
1093
+ boundaryId: string;
1094
+ childRunId: string;
1095
+ childPlayName: string;
1096
+ }): void {
1097
+ this.pendingChildPlayBoundaries.push(input);
1098
+ }
1099
+
1100
+ private beginScheduledChildPlayCall(): { release: () => void } {
1101
+ this.activeScheduledChildPlayCalls += 1;
1102
+ let released = false;
1103
+ return {
1104
+ release: () => {
1105
+ if (released) return;
1106
+ released = true;
1107
+ this.activeScheduledChildPlayCalls = Math.max(
1108
+ 0,
1109
+ this.activeScheduledChildPlayCalls - 1,
1110
+ );
1111
+ if (this.activeScheduledChildPlayCalls !== 0) return;
1112
+ for (const resolve of this.scheduledChildPlayQuiescenceWaiters.splice(
1113
+ 0,
1114
+ )) {
1115
+ resolve();
1116
+ }
1117
+ },
1118
+ };
1119
+ }
1120
+
1121
+ private async waitForScheduledChildPlayQuiescence(): Promise<void> {
1122
+ if (this.activeScheduledChildPlayCalls === 0) return;
1123
+ await new Promise<void>((resolve) => {
1124
+ this.scheduledChildPlayQuiescenceWaiters.push(resolve);
1125
+ });
1126
+ }
1127
+
1128
+ private consumePendingChildPlaySuspension(): PlayExecutionSuspension | null {
1129
+ if (this.pendingChildPlayBoundaries.length === 0) return null;
1130
+ const boundaries = [
1131
+ ...new Map(
1132
+ this.pendingChildPlayBoundaries.map((boundary) => [
1133
+ boundary.boundaryId,
1134
+ boundary,
1135
+ ]),
1136
+ ).values(),
1137
+ ];
1138
+ this.pendingChildPlayBoundaries = [];
1139
+ if (boundaries.length === 1) {
1140
+ const boundary = boundaries[0]!;
1141
+ return {
1142
+ kind: 'child_play',
1143
+ boundaryId: boundary.boundaryId,
1144
+ childRunId: boundary.childRunId,
1145
+ childPlayName: boundary.childPlayName,
1146
+ };
1147
+ }
1148
+ return {
1149
+ kind: 'child_play_batch',
1150
+ boundaries,
1151
+ };
1152
+ }
1153
+
1154
+ private childPlayBoundaryOutput<TOutput>(input: {
1155
+ boundaryId: string;
1156
+ childRunId: string;
1157
+ childPlayName: string;
1158
+ }): { found: true; output: TOutput } | { found: false } {
1159
+ const existing = this.checkpoint.resolvedBoundaries?.[input.boundaryId];
1160
+ if (!existing || existing.kind !== 'child_play') return { found: false };
1161
+ if (existing.childRunId !== input.childRunId) return { found: false };
1162
+ if (existing.status === 'completed') {
1163
+ return { found: true, output: existing.output as TOutput };
1164
+ }
1165
+ const message =
1166
+ typeof existing.error === 'string' && existing.error.trim()
1167
+ ? existing.error.trim()
1168
+ : `Child play ${input.childPlayName} (${input.childRunId}) ${existing.status}.`;
1169
+ throw new Error(message);
1170
+ }
1171
+
1172
+ private recordPlayCallStep(input: {
1173
+ playId: string;
1174
+ description?: string | null;
1175
+ nestedSteps?: PlayStep[];
1176
+ }): void {
1177
+ const step = {
1178
+ type: 'play_call' as const,
1179
+ playId: input.playId,
1180
+ nestedSteps: input.nestedSteps ?? [],
1181
+ description: normalizeStepDescription(input.description ?? undefined),
1182
+ };
1183
+ if (this.activeDatasetStep) {
1184
+ this.activeDatasetStep.substeps.push(step);
1185
+ } else {
1186
+ this.steps.push(step);
1187
+ }
1188
+ }
1189
+
1190
+ private runtimeApiHeaders(): Record<string, string> {
1191
+ if (!this.#options.executorToken?.trim()) {
1192
+ throw new Error(
1193
+ 'ctx.runPlay scheduled child launch requires an executor token.',
1194
+ );
1195
+ }
1196
+ return {
1197
+ Authorization: `Bearer ${this.#options.executorToken.trim()}`,
1198
+ 'Content-Type': 'application/json',
1199
+ ...(this.#options.runtimeSchedulerSchema?.trim()
1200
+ ? {
1201
+ [RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER]:
1202
+ this.#options.runtimeSchedulerSchema.trim(),
1203
+ [SYNTHETIC_RUN_HEADER]: '1',
1204
+ }
1205
+ : {}),
1206
+ ...(this.#options.vercelProtectionBypassToken?.trim()
1207
+ ? {
1208
+ 'x-vercel-protection-bypass':
1209
+ this.#options.vercelProtectionBypassToken.trim(),
1210
+ }
1211
+ : {}),
1212
+ };
1213
+ }
1214
+
1215
+ private runtimeApiBaseUrl(): string {
1216
+ const baseUrl = this.#options.baseUrl?.trim();
1217
+ if (!baseUrl) {
1218
+ throw new Error('ctx.runPlay scheduled child launch requires baseUrl.');
1219
+ }
1220
+ return baseUrl.replace(/\/$/, '');
1221
+ }
1222
+
1223
+ private async launchScheduledChildPlay(input: {
1224
+ key: string;
1225
+ childRunId: string;
1226
+ childPlayName: string;
1227
+ childGovernance: GovernanceSnapshot;
1228
+ childInput: Record<string, unknown>;
1229
+ description?: string | null;
1230
+ }): Promise<void> {
1231
+ const parentPlayName =
1232
+ this.governance.currentPlayId ||
1233
+ this.#options.playName ||
1234
+ this.#options.playId ||
1235
+ 'anonymous-play';
1236
+ const governance: PlayCallGovernanceSnapshot = {
1237
+ rootRunId: this.governance.rootRunId,
1238
+ parentRunId: this.governance.currentRunId,
1239
+ parentPlayName,
1240
+ key: input.key,
1241
+ ancestryPlayIds: [...this.governance.ancestryPlayIds],
1242
+ callDepth: input.childGovernance.callDepth,
1243
+ playCallCount: input.childGovernance.playCallCount,
1244
+ toolCallCount: input.childGovernance.toolCallCount,
1245
+ retryCount: input.childGovernance.retryCount,
1246
+ descendantCount: input.childGovernance.descendantCount,
1247
+ waterfallStepExecutions: input.childGovernance.waterfallStepExecutions,
1248
+ };
1249
+ const response = await fetch(
1250
+ `${this.runtimeApiBaseUrl()}/api/v2/plays/run`,
1251
+ {
1252
+ method: 'POST',
1253
+ headers: this.runtimeApiHeaders(),
1254
+ body: JSON.stringify({
1255
+ name: input.childPlayName,
1256
+ workflowId: input.childRunId,
1257
+ input: input.childInput,
1258
+ profile: 'hatchet',
1259
+ waitForCompletionMs: 0,
1260
+ internalRunPlay: {
1261
+ ...governance,
1262
+ description: input.description ?? null,
1263
+ },
1264
+ }),
1265
+ },
1266
+ );
1267
+ if (!response.ok) {
1268
+ const body = (await response.json().catch(() => null)) as {
1269
+ error?: unknown;
1270
+ } | null;
1271
+ const message =
1272
+ typeof body?.error === 'string' && body.error.trim()
1273
+ ? body.error.trim()
1274
+ : response.statusText;
1275
+ throw new Error(
1276
+ `ctx.runPlay(${input.key}) failed to launch child play "${input.childPlayName}": ${message} (status ${response.status}).`,
1277
+ );
1278
+ }
1279
+ }
1280
+
1281
+ private async readScheduledChildTerminal<TOutput>(input: {
1282
+ childRunId: string;
1283
+ childPlayName: string;
1284
+ }): Promise<
1285
+ | { status: 'completed'; output: TOutput }
1286
+ | { status: 'failed' | 'cancelled'; error: string }
1287
+ | null
1288
+ > {
1289
+ const response = await fetch(
1290
+ `${this.runtimeApiBaseUrl()}/api/v2/internal/play-runtime`,
1291
+ {
1292
+ method: 'POST',
1293
+ headers: this.runtimeApiHeaders(),
1294
+ body: JSON.stringify({
1295
+ action: 'read_child_run_terminal_snapshot',
1296
+ parentRunId: this.governance.currentRunId,
1297
+ childRunId: input.childRunId,
1298
+ childPlayName: input.childPlayName,
1299
+ }),
1300
+ },
1301
+ );
1302
+ const body = (await response.json().catch(() => null)) as {
1303
+ state?: {
1304
+ data?: {
1305
+ status?: unknown;
1306
+ result?: unknown;
1307
+ error?: unknown;
1308
+ };
1309
+ };
1310
+ error?: unknown;
1311
+ } | null;
1312
+ if (!response.ok) {
1313
+ const message =
1314
+ typeof body?.error === 'string' && body.error.trim()
1315
+ ? body.error.trim()
1316
+ : response.statusText;
1317
+ throw new Error(
1318
+ `ctx.runPlay failed to read child terminal snapshot for "${input.childPlayName}": ${message} (status ${response.status}).`,
1319
+ );
1320
+ }
1321
+ const data = body?.state?.data;
1322
+ const status = String(data?.status ?? '').toLowerCase();
1323
+ if (status === 'completed') {
1324
+ return { status, output: data?.result as TOutput };
1325
+ }
1326
+ if (status === 'failed' || status === 'cancelled') {
1327
+ return {
1328
+ status,
1329
+ error:
1330
+ typeof data?.error === 'string' && data.error.trim()
1331
+ ? data.error.trim()
1332
+ : `Child play ${input.childPlayName} (${input.childRunId}) ${status}.`,
1333
+ };
1334
+ }
1335
+ return null;
1336
+ }
1337
+
882
1338
  private emitScopedRowUpdate(
883
1339
  key: string | null,
884
1340
  tableNamespace: string | null,
@@ -946,7 +1402,7 @@ export class PlayContextImpl {
946
1402
  this.#options.runId
947
1403
  ) {
948
1404
  const response = await fetch(
949
- `${this.#options.baseUrl.replace(/\/$/, '')}/api/v2/plays/internal/runtime`,
1405
+ `${this.#options.baseUrl.replace(/\/$/, '')}/api/v2/internal/play-runtime`,
950
1406
  {
951
1407
  method: 'POST',
952
1408
  headers: {
@@ -1058,6 +1514,7 @@ export class PlayContextImpl {
1058
1514
  runId: string,
1059
1515
  reclaimRunning = false,
1060
1516
  forceRefresh = false,
1517
+ forceFailedRefresh = false,
1061
1518
  ): Promise<RuntimeStepReceipt | null> {
1062
1519
  if (!this.#options.claimRuntimeStepReceipt) {
1063
1520
  return null;
@@ -1065,8 +1522,11 @@ export class PlayContextImpl {
1065
1522
  const claimed = await this.#options.claimRuntimeStepReceipt({
1066
1523
  key,
1067
1524
  runId,
1525
+ runAttempt: this.currentRunAttempt,
1526
+ leaseAware: true,
1068
1527
  ...(reclaimRunning ? { reclaimRunning: true } : {}),
1069
1528
  ...(forceRefresh ? { forceRefresh: true } : {}),
1529
+ ...(forceFailedRefresh ? { forceFailedRefresh: true } : {}),
1070
1530
  });
1071
1531
  if (!claimed || typeof claimed.key !== 'string' || !claimed.key.trim()) {
1072
1532
  return null;
@@ -1082,6 +1542,7 @@ export class PlayContextImpl {
1082
1542
  key: string,
1083
1543
  runId: string,
1084
1544
  output: unknown | null,
1545
+ leaseId?: string | null,
1085
1546
  ): Promise<RuntimeStepReceipt | null> {
1086
1547
  assertNoSecretTaint(output, 'ctx.step receipt output');
1087
1548
  if (!this.#options.completeRuntimeStepReceipt) {
@@ -1090,6 +1551,8 @@ export class PlayContextImpl {
1090
1551
  const completed = await this.#options.completeRuntimeStepReceipt({
1091
1552
  key,
1092
1553
  runId,
1554
+ runAttempt: this.currentRunAttempt,
1555
+ ...(leaseId ? { leaseId } : {}),
1093
1556
  output,
1094
1557
  });
1095
1558
  if (
@@ -1106,10 +1569,167 @@ export class PlayContextImpl {
1106
1569
  };
1107
1570
  }
1108
1571
 
1572
+ private async releaseRuntimeStepReceipt(
1573
+ key: string,
1574
+ runId: string,
1575
+ leaseId?: string | null,
1576
+ ): Promise<RuntimeStepReceipt | null> {
1577
+ if (!this.#options.releaseRuntimeStepReceipt) {
1578
+ return null;
1579
+ }
1580
+ const released = await this.#options.releaseRuntimeStepReceipt({
1581
+ key,
1582
+ runId,
1583
+ runAttempt: this.currentRunAttempt,
1584
+ ...(leaseId ? { leaseId } : {}),
1585
+ });
1586
+ if (!released || typeof released.key !== 'string' || !released.key.trim()) {
1587
+ return null;
1588
+ }
1589
+ return {
1590
+ ...released,
1591
+ key: released.key.trim(),
1592
+ runId: released.runId ?? null,
1593
+ };
1594
+ }
1595
+
1596
+ private async heartbeatRuntimeStepReceipt(
1597
+ key: string,
1598
+ runId: string,
1599
+ leaseId: string,
1600
+ ): Promise<RuntimeStepReceipt | null> {
1601
+ if (!this.#options.heartbeatRuntimeStepReceipts) {
1602
+ return null;
1603
+ }
1604
+ const [receipt] = await this.#options.heartbeatRuntimeStepReceipts({
1605
+ runId,
1606
+ runAttempt: this.currentRunAttempt,
1607
+ leaseId,
1608
+ keys: [key],
1609
+ });
1610
+ if (!receipt || typeof receipt.key !== 'string' || !receipt.key.trim()) {
1611
+ return null;
1612
+ }
1613
+ return {
1614
+ ...receipt,
1615
+ key: receipt.key.trim(),
1616
+ runId: receipt.runId ?? null,
1617
+ };
1618
+ }
1619
+
1620
+ private async assertRuntimeToolReceiptOwnership(
1621
+ requests: ToolCallRequest[],
1622
+ ): Promise<void> {
1623
+ const targets = requests.flatMap((request) => {
1624
+ const receiptKey = request.receiptKey?.trim() || null;
1625
+ if (!receiptKey) return [];
1626
+ const leaseId = request.receiptLeaseId?.trim() || null;
1627
+ if (!leaseId) return [];
1628
+ return [{ receiptKey, leaseId }];
1629
+ });
1630
+ if (targets.length === 0) return;
1631
+
1632
+ const byLeaseId = new Map<string, string[]>();
1633
+ for (const target of targets) {
1634
+ const keys = byLeaseId.get(target.leaseId) ?? [];
1635
+ keys.push(target.receiptKey);
1636
+ byLeaseId.set(target.leaseId, keys);
1637
+ }
1638
+
1639
+ if (this.#options.heartbeatRuntimeStepReceipts) {
1640
+ for (const [leaseId, keys] of byLeaseId) {
1641
+ const receipts = await this.#options.heartbeatRuntimeStepReceipts({
1642
+ runId: this.currentRunId,
1643
+ runAttempt: this.currentRunAttempt,
1644
+ leaseId,
1645
+ keys,
1646
+ });
1647
+ this.assertRuntimeToolReceiptHeartbeatResult(keys, leaseId, receipts);
1648
+ }
1649
+ return;
1650
+ }
1651
+
1652
+ if (
1653
+ !this.#options.getRuntimeStepReceipt &&
1654
+ !this.#options.getRuntimeStepReceipts
1655
+ ) {
1656
+ throw new RuntimeReceiptLeaseLostError({
1657
+ receiptKey: targets[0]?.receiptKey ?? 'unknown',
1658
+ runId: this.currentRunId,
1659
+ leaseId: targets[0]?.leaseId ?? 'unknown',
1660
+ });
1661
+ }
1662
+
1663
+ const latest = await this.getRuntimeStepReceipts(
1664
+ targets.map((target) => target.receiptKey),
1665
+ );
1666
+ for (const target of targets) {
1667
+ const receipt = latest.get(target.receiptKey);
1668
+ if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) {
1669
+ throw new RuntimeReceiptLeaseLostError({
1670
+ receiptKey: target.receiptKey,
1671
+ runId: this.currentRunId,
1672
+ leaseId: target.leaseId,
1673
+ });
1674
+ }
1675
+ }
1676
+ }
1677
+
1678
+ private assertRuntimeToolReceiptHeartbeatResult(
1679
+ keys: string[],
1680
+ leaseId: string,
1681
+ receipts: Array<RuntimeStepReceipt | null>,
1682
+ ): void {
1683
+ for (let index = 0; index < keys.length; index += 1) {
1684
+ const receipt = this.normalizeRuntimeStepReceipt(
1685
+ keys[index] ?? '',
1686
+ receipts[index],
1687
+ );
1688
+ if (!this.runtimeToolReceiptStillOwned(receipt, leaseId)) {
1689
+ throw new RuntimeReceiptLeaseLostError({
1690
+ receiptKey: keys[index] ?? 'unknown',
1691
+ runId: this.currentRunId,
1692
+ leaseId,
1693
+ });
1694
+ }
1695
+ }
1696
+ }
1697
+
1698
+ private runtimeToolReceiptStillOwned(
1699
+ receipt: RuntimeStepReceipt | null | undefined,
1700
+ leaseId: string,
1701
+ ): boolean {
1702
+ if (!receipt || receipt.status !== 'running') return false;
1703
+ if (receipt.leaseId !== leaseId) return false;
1704
+ const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
1705
+ return !ownerRunId || ownerRunId === this.currentRunId;
1706
+ }
1707
+
1708
+ private runtimeToolReceiptStillOwnedByCurrentAttempt(
1709
+ receipt: RuntimeStepReceipt | null | undefined,
1710
+ leaseId: string | null | undefined,
1711
+ ): boolean {
1712
+ if (!receipt || receipt.status !== 'running') return false;
1713
+ if (leaseId?.trim()) {
1714
+ if (receipt.leaseId !== leaseId.trim()) return false;
1715
+ } else if (receipt.leaseId != null) {
1716
+ return false;
1717
+ }
1718
+ const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
1719
+ if (ownerRunId !== this.currentRunId) return false;
1720
+ const ownerAttempt =
1721
+ typeof receipt.leaseOwnerAttempt === 'number'
1722
+ ? receipt.leaseOwnerAttempt
1723
+ : 0;
1724
+ return ownerAttempt === this.currentRunAttempt;
1725
+ }
1726
+
1109
1727
  private async failRuntimeStepReceipt(
1110
1728
  key: string,
1111
1729
  runId: string,
1112
1730
  error: string,
1731
+ leaseId?: string | null,
1732
+ failureKind?: WorkReceiptFailureKind | null,
1113
1733
  ): Promise<RuntimeStepReceipt | null> {
1114
1734
  if (!this.#options.failRuntimeStepReceipt) {
1115
1735
  return null;
@@ -1117,6 +1737,9 @@ export class PlayContextImpl {
1117
1737
  const failed = await this.#options.failRuntimeStepReceipt({
1118
1738
  key,
1119
1739
  runId,
1740
+ runAttempt: this.currentRunAttempt,
1741
+ ...(leaseId ? { leaseId } : {}),
1742
+ ...(failureKind ? { failureKind } : {}),
1120
1743
  error,
1121
1744
  });
1122
1745
  if (!failed || typeof failed.key !== 'string' || !failed.key.trim()) {
@@ -1176,6 +1799,7 @@ export class PlayContextImpl {
1176
1799
  runId: string,
1177
1800
  reclaimRunning = false,
1178
1801
  forceRefresh = false,
1802
+ forceFailedRefresh = false,
1179
1803
  ): Promise<Map<string, RuntimeStepReceipt>> {
1180
1804
  const uniqueKeys = [
1181
1805
  ...new Set(keys.map((key) => key.trim()).filter(Boolean)),
@@ -1185,8 +1809,11 @@ export class PlayContextImpl {
1185
1809
  ? await this.#options.claimRuntimeStepReceipts({
1186
1810
  keys: uniqueKeys,
1187
1811
  runId,
1812
+ runAttempt: this.currentRunAttempt,
1813
+ leaseAware: true,
1188
1814
  ...(reclaimRunning ? { reclaimRunning: true } : {}),
1189
1815
  ...(forceRefresh ? { forceRefresh: true } : {}),
1816
+ ...(forceFailedRefresh ? { forceFailedRefresh: true } : {}),
1190
1817
  })
1191
1818
  : await Promise.all(
1192
1819
  uniqueKeys.map((key) =>
@@ -1195,6 +1822,7 @@ export class PlayContextImpl {
1195
1822
  runId,
1196
1823
  reclaimRunning,
1197
1824
  forceRefresh,
1825
+ forceFailedRefresh,
1198
1826
  ),
1199
1827
  ),
1200
1828
  );
@@ -1210,26 +1838,44 @@ export class PlayContextImpl {
1210
1838
  }
1211
1839
 
1212
1840
  private async completeRuntimeStepReceipts(
1213
- receipts: Array<{ key: string; runId: string; output: unknown | null }>,
1841
+ receipts: Array<{
1842
+ key: string;
1843
+ runId: string;
1844
+ runAttempt?: number | null;
1845
+ output: unknown | null;
1846
+ leaseId?: string | null;
1847
+ }>,
1214
1848
  ): Promise<Map<string, RuntimeStepReceipt>> {
1215
1849
  const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
1216
1850
  if (normalizedInputs.length === 0) return new Map();
1217
1851
  for (const receipt of normalizedInputs) {
1218
1852
  assertNoSecretTaint(receipt.output, 'ctx.tool receipt output');
1219
1853
  }
1220
- const completed = this.#options.completeRuntimeStepReceipts
1221
- ? await this.#options.completeRuntimeStepReceipts({
1222
- receipts: normalizedInputs,
1223
- })
1224
- : await Promise.all(
1225
- normalizedInputs.map((receipt) =>
1226
- this.completeRuntimeStepReceipt(
1227
- receipt.key,
1228
- receipt.runId,
1229
- receipt.output,
1854
+ let completed: Array<RuntimeStepReceipt | null>;
1855
+ try {
1856
+ completed = this.#options.completeRuntimeStepReceipts
1857
+ ? await this.#options.completeRuntimeStepReceipts({
1858
+ receipts: normalizedInputs.map((receipt) => ({
1859
+ ...receipt,
1860
+ runAttempt: receipt.runAttempt ?? this.currentRunAttempt,
1861
+ })),
1862
+ })
1863
+ : await Promise.all(
1864
+ normalizedInputs.map((receipt) =>
1865
+ this.completeRuntimeStepReceipt(
1866
+ receipt.key,
1867
+ receipt.runId,
1868
+ receipt.output,
1869
+ receipt.leaseId,
1870
+ ),
1230
1871
  ),
1231
- ),
1232
- );
1872
+ );
1873
+ } catch (error) {
1874
+ await this.recoverRuntimeStepReceiptCompletionsAfterBulkError(
1875
+ normalizedInputs,
1876
+ );
1877
+ throw error;
1878
+ }
1233
1879
  const byKey = new Map<string, RuntimeStepReceipt>();
1234
1880
  for (let index = 0; index < completed.length; index += 1) {
1235
1881
  const normalized = this.normalizeRuntimeStepReceipt(
@@ -1238,17 +1884,123 @@ export class PlayContextImpl {
1238
1884
  );
1239
1885
  if (normalized) byKey.set(normalized.key, normalized);
1240
1886
  }
1887
+ await this.reconcileMissingRuntimeStepReceiptCompletions(
1888
+ normalizedInputs,
1889
+ byKey,
1890
+ );
1241
1891
  return byKey;
1242
1892
  }
1243
1893
 
1894
+ private async reconcileMissingRuntimeStepReceiptCompletions(
1895
+ receipts: Array<{
1896
+ key: string;
1897
+ runId: string;
1898
+ runAttempt?: number | null;
1899
+ output: unknown | null;
1900
+ leaseId?: string | null;
1901
+ }>,
1902
+ completedByKey: Map<string, RuntimeStepReceipt>,
1903
+ ): Promise<void> {
1904
+ if (
1905
+ receipts.length === completedByKey.size ||
1906
+ (!this.#options.getRuntimeStepReceipt &&
1907
+ !this.#options.getRuntimeStepReceipts)
1908
+ ) {
1909
+ return;
1910
+ }
1911
+ const missingKeys = receipts
1912
+ .map((receipt) => receipt.key.trim())
1913
+ .filter((key) => key && !completedByKey.has(key));
1914
+ if (missingKeys.length === 0) return;
1915
+ const latest = await this.getRuntimeStepReceipts(missingKeys);
1916
+ for (const key of missingKeys) {
1917
+ const recovered = latest.get(key);
1918
+ if (
1919
+ recovered?.status === 'completed' ||
1920
+ recovered?.status === 'skipped'
1921
+ ) {
1922
+ completedByKey.set(key, recovered);
1923
+ }
1924
+ }
1925
+ const recoveredCount = missingKeys.filter((key) =>
1926
+ completedByKey.has(key),
1927
+ ).length;
1928
+ if (recoveredCount > 0) {
1929
+ this.log(
1930
+ `Runtime tool receipt completion response reconciled ${recoveredCount}/${missingKeys.length} missing receipt(s) from durable store read-back.`,
1931
+ );
1932
+ }
1933
+ }
1934
+
1935
+ private async recoverRuntimeStepReceiptCompletionsAfterBulkError(
1936
+ receipts: Array<{
1937
+ key: string;
1938
+ runId: string;
1939
+ runAttempt?: number | null;
1940
+ output: unknown | null;
1941
+ leaseId?: string | null;
1942
+ }>,
1943
+ ): Promise<void> {
1944
+ if (
1945
+ receipts.length === 0 ||
1946
+ (!this.#options.getRuntimeStepReceipt &&
1947
+ !this.#options.getRuntimeStepReceipts)
1948
+ ) {
1949
+ return;
1950
+ }
1951
+ let latest: Map<string, RuntimeStepReceipt>;
1952
+ try {
1953
+ latest = await this.getRuntimeStepReceipts(
1954
+ receipts.map((receipt) => receipt.key),
1955
+ );
1956
+ } catch {
1957
+ return;
1958
+ }
1959
+ await Promise.allSettled(
1960
+ receipts.map(async (receipt) => {
1961
+ const recovered = latest.get(receipt.key);
1962
+ if (
1963
+ recovered?.status === 'completed' ||
1964
+ recovered?.status === 'skipped'
1965
+ ) {
1966
+ return;
1967
+ }
1968
+ if (
1969
+ !this.runtimeToolReceiptStillOwnedByCurrentAttempt(
1970
+ recovered,
1971
+ receipt.leaseId,
1972
+ )
1973
+ ) {
1974
+ return;
1975
+ }
1976
+ await this.completeRuntimeStepReceipt(
1977
+ receipt.key,
1978
+ receipt.runId,
1979
+ receipt.output,
1980
+ receipt.leaseId,
1981
+ );
1982
+ }),
1983
+ );
1984
+ }
1985
+
1244
1986
  private async failRuntimeStepReceipts(
1245
- receipts: Array<{ key: string; runId: string; error: string }>,
1987
+ receipts: Array<{
1988
+ key: string;
1989
+ runId: string;
1990
+ runAttempt?: number | null;
1991
+ error: string;
1992
+ leaseId?: string | null;
1993
+ failureKind?: WorkReceiptFailureKind | null;
1994
+ }>,
1246
1995
  ): Promise<Map<string, RuntimeStepReceipt>> {
1247
1996
  const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
1248
1997
  if (normalizedInputs.length === 0) return new Map();
1249
1998
  const failed = this.#options.failRuntimeStepReceipts
1250
1999
  ? await this.#options.failRuntimeStepReceipts({
1251
- receipts: normalizedInputs,
2000
+ receipts: normalizedInputs.map((receipt) => ({
2001
+ ...receipt,
2002
+ runAttempt: receipt.runAttempt ?? this.currentRunAttempt,
2003
+ })),
1252
2004
  })
1253
2005
  : await Promise.all(
1254
2006
  normalizedInputs.map((receipt) =>
@@ -1256,6 +2008,8 @@ export class PlayContextImpl {
1256
2008
  receipt.key,
1257
2009
  receipt.runId,
1258
2010
  receipt.error,
2011
+ receipt.leaseId,
2012
+ receipt.failureKind,
1259
2013
  ),
1260
2014
  ),
1261
2015
  );
@@ -1283,45 +2037,98 @@ export class PlayContextImpl {
1283
2037
  );
1284
2038
  }
1285
2039
 
1286
- private async durableToolCallCacheKey(input: {
2040
+ private async durableToolCallCacheKeyForScope(input: {
1287
2041
  toolId: string;
1288
2042
  requestInput: Record<string, unknown>;
2043
+ executionAuthScopeDigest?: string | null;
1289
2044
  staleAfterSeconds?: number | null;
2045
+ playLocalScope?: string | null;
1290
2046
  }): Promise<string> {
1291
2047
  const providerActionVersion =
1292
2048
  (await this.#options.getToolActionCacheVersion?.(input.toolId))?.trim() ??
1293
2049
  '';
2050
+ const executionAuthScopeDigest =
2051
+ input.executionAuthScopeDigest?.trim() ||
2052
+ (await this.resolveToolAuthScopeDigest(input.toolId))?.trim() ||
2053
+ null;
1294
2054
  return buildDurableToolCallCacheKey({
1295
2055
  orgId: this.#options.orgId,
1296
- playId: this.governance.currentPlayId,
2056
+ playLocalScope: input.playLocalScope,
1297
2057
  toolId: input.toolId,
1298
2058
  requestInput: input.requestInput,
1299
2059
  authScopeDigest: buildDurableToolCallAuthScopeDigest({
1300
2060
  orgId: this.#options.orgId,
1301
- userEmail: this.#options.userEmail,
1302
2061
  toolId: input.toolId,
2062
+ executionAuthScopeDigest,
1303
2063
  }),
1304
2064
  providerActionVersion,
1305
2065
  staleAfterSeconds: input.staleAfterSeconds,
1306
2066
  });
1307
2067
  }
1308
2068
 
2069
+ private async durableToolCallCacheKey(input: {
2070
+ toolId: string;
2071
+ requestInput: Record<string, unknown>;
2072
+ executionAuthScopeDigest?: string | null;
2073
+ staleAfterSeconds?: number | null;
2074
+ }): Promise<string> {
2075
+ return await this.durableToolCallCacheKeyForScope({
2076
+ ...input,
2077
+ playLocalScope: this.governance.currentPlayId,
2078
+ });
2079
+ }
2080
+
2081
+ private async resolveToolAuthScopeDigest(
2082
+ toolId: string,
2083
+ ): Promise<string | null> {
2084
+ const configured = await this.#options.getToolAuthScopeDigest?.(toolId);
2085
+ if (typeof configured === 'string' && configured.trim()) {
2086
+ return configured.trim();
2087
+ }
2088
+ return null;
2089
+ }
2090
+
2091
+ private invalidateToolAuthScopeDigest(toolId: string): void {
2092
+ this.#options.invalidateToolAuthScopeDigest?.(toolId);
2093
+ }
2094
+
1309
2095
  private durableReceiptExecutionStore(): DurableReceiptExecutionStore {
1310
2096
  return {
1311
2097
  enabled: Boolean(this.#options.claimRuntimeStepReceipt),
1312
2098
  get: (receiptKey) => this.getRuntimeStepReceipt(receiptKey),
1313
2099
  getMany: (receiptKeys) => this.getRuntimeStepReceipts(receiptKeys),
1314
- claim: (receiptKey, runId, reclaimRunning, forceRefresh) =>
2100
+ claim: (
2101
+ receiptKey,
2102
+ runId,
2103
+ reclaimRunning,
2104
+ forceRefresh,
2105
+ forceFailedRefresh,
2106
+ ) =>
1315
2107
  this.claimRuntimeStepReceipt(
1316
2108
  receiptKey,
1317
2109
  runId,
1318
2110
  reclaimRunning,
1319
2111
  forceRefresh,
2112
+ forceFailedRefresh,
2113
+ ),
2114
+ complete: (receiptKey, runId, output, leaseId) =>
2115
+ this.completeRuntimeStepReceipt(receiptKey, runId, output, leaseId),
2116
+ release: (receiptKey, runId, leaseId) =>
2117
+ this.releaseRuntimeStepReceipt(receiptKey, runId, leaseId),
2118
+ ...(this.#options.heartbeatRuntimeStepReceipts
2119
+ ? {
2120
+ heartbeat: (receiptKey, runId, leaseId) =>
2121
+ this.heartbeatRuntimeStepReceipt(receiptKey, runId, leaseId),
2122
+ }
2123
+ : {}),
2124
+ fail: (receiptKey, runId, error, leaseId, failureKind) =>
2125
+ this.failRuntimeStepReceipt(
2126
+ receiptKey,
2127
+ runId,
2128
+ error,
2129
+ leaseId,
2130
+ failureKind,
1320
2131
  ),
1321
- complete: (receiptKey, runId, output) =>
1322
- this.completeRuntimeStepReceipt(receiptKey, runId, output),
1323
- fail: (receiptKey, runId, error) =>
1324
- this.failRuntimeStepReceipt(receiptKey, runId, error),
1325
2132
  canPersistFailure: Boolean(this.#options.failRuntimeStepReceipt),
1326
2133
  canPersistCompletion: Boolean(this.#options.completeRuntimeStepReceipt),
1327
2134
  };
@@ -1363,7 +2170,8 @@ export class PlayContextImpl {
1363
2170
  source: DurableReceiptRecoverySource,
1364
2171
  ) => T;
1365
2172
  onClaimedResult?: (output: T, receiptKey: string) => T;
1366
- execute: () => Promise<T>;
2173
+ shouldPersistFailure?: (error: unknown) => boolean;
2174
+ execute: (context: { leaseId: string | null }) => Promise<T>;
1367
2175
  },
1368
2176
  ): Promise<T> {
1369
2177
  const receiptKey =
@@ -1392,6 +2200,7 @@ export class PlayContextImpl {
1392
2200
  markSkipped: opts.markSkipped,
1393
2201
  onRecovered: opts.onRecovered,
1394
2202
  onClaimedResult: opts.onClaimedResult,
2203
+ shouldPersistFailure: opts.shouldPersistFailure,
1395
2204
  formatError: (error) => this.formatRuntimeError(error),
1396
2205
  log: (message) => this.log(message),
1397
2206
  execute: opts.execute,
@@ -1402,6 +2211,13 @@ export class PlayContextImpl {
1402
2211
  return this.#options.runId ?? this.governance.currentRunId ?? 'unknown';
1403
2212
  }
1404
2213
 
2214
+ private get currentRunAttempt(): number {
2215
+ const attempt = this.#options.runAttempt;
2216
+ return typeof attempt === 'number' && Number.isFinite(attempt)
2217
+ ? Math.max(0, Math.floor(attempt))
2218
+ : 0;
2219
+ }
2220
+
1405
2221
  private emitScopedFieldMetaUpdate(input: {
1406
2222
  rowId: number;
1407
2223
  key: string | null;
@@ -1615,12 +2431,17 @@ export class PlayContextImpl {
1615
2431
 
1616
2432
  private effectiveToolCallCachePolicy(options?: ToolCallOptions): {
1617
2433
  force: boolean;
2434
+ forceFailedRefresh: boolean;
1618
2435
  staleAfterSeconds?: number | null;
1619
2436
  } {
1620
2437
  return {
1621
2438
  force:
1622
2439
  options?.force === true ||
1623
2440
  this.#options.cachePolicy?.forceToolRefresh === true,
2441
+ forceFailedRefresh:
2442
+ options?.force === true ||
2443
+ this.#options.cachePolicy?.forceToolRefresh === true ||
2444
+ this.#options.cachePolicy?.forceFailedToolRefresh === true,
1624
2445
  staleAfterSeconds: options?.staleAfterSeconds ?? null,
1625
2446
  };
1626
2447
  }
@@ -1641,6 +2462,19 @@ export class PlayContextImpl {
1641
2462
  return this.checkpoint.completedToolBatches[toolId]?.[rowCacheKey];
1642
2463
  }
1643
2464
 
2465
+ private getCachedToolResultCandidate(
2466
+ toolId: string,
2467
+ rowCacheKeys: readonly string[],
2468
+ ): { cacheKey: string; result: ToolBatchResult } | null {
2469
+ for (const rowCacheKey of rowCacheKeys) {
2470
+ const cached = this.getCachedToolResult(toolId, rowCacheKey);
2471
+ if (cached?.done) {
2472
+ return { cacheKey: rowCacheKey, result: cached };
2473
+ }
2474
+ }
2475
+ return null;
2476
+ }
2477
+
1644
2478
  private cacheToolResult(
1645
2479
  toolId: string,
1646
2480
  rowCacheKey: string,
@@ -1695,24 +2529,146 @@ export class PlayContextImpl {
1695
2529
  metadata?: ToolResultMetadataInput | null;
1696
2530
  meta?: Record<string, unknown>;
1697
2531
  execution: ToolResultExecutionMetadata;
2532
+ requestInput?: Record<string, unknown>;
1698
2533
  }): Promise<ToolExecuteResult> {
1699
2534
  if (isToolExecuteResult(input.result)) {
1700
- return cloneToolExecuteResultWithExecution(input.result, input.execution);
2535
+ return this.attachCustomerDbDatasetResult(
2536
+ input.toolId,
2537
+ input.requestInput,
2538
+ cloneToolExecuteResultWithExecution(input.result, input.execution),
2539
+ );
1701
2540
  }
1702
2541
  const publicToolResult = publicToolResponseEnvelope(input.result);
1703
- return createToolExecuteResult({
1704
- status: publicToolResult?.status ?? input.status,
1705
- jobId: input.jobId,
1706
- result: publicToolResult
1707
- ? {
1708
- data: publicToolResult.raw,
1709
- ...(publicToolResult.meta ? { meta: publicToolResult.meta } : {}),
1710
- }
1711
- : input.result,
1712
- metadata:
1713
- input.metadata ?? (await this.resolveToolResultMetadata(input.toolId)),
1714
- execution: input.execution,
1715
- meta: input.meta,
2542
+ return this.attachCustomerDbDatasetResult(
2543
+ input.toolId,
2544
+ input.requestInput,
2545
+ createToolExecuteResult({
2546
+ status: publicToolResult?.status ?? input.status,
2547
+ jobId: input.jobId,
2548
+ result: publicToolResult
2549
+ ? {
2550
+ data: publicToolResult.raw,
2551
+ ...(publicToolResult.meta ? { meta: publicToolResult.meta } : {}),
2552
+ }
2553
+ : input.result,
2554
+ metadata:
2555
+ input.metadata ??
2556
+ (await this.resolveToolResultMetadata(input.toolId)),
2557
+ execution: input.execution,
2558
+ meta: input.meta,
2559
+ }),
2560
+ );
2561
+ }
2562
+
2563
+ private attachCustomerDbDatasetResult(
2564
+ toolId: string,
2565
+ requestInput: Record<string, unknown> | undefined,
2566
+ wrapped: ToolExecuteResult,
2567
+ ): ToolExecuteResult {
2568
+ if (!isQueryResultDatasetTool(toolId)) return wrapped;
2569
+ const raw = recordOrNull(wrapped.toolResponse.raw);
2570
+ const dataset = recordOrNull(raw?.dataset);
2571
+ const rows = rowsFromUnknown(raw?.rows);
2572
+ const totalRows = finiteNonNegativeInteger(dataset?.total_rows);
2573
+ const sql =
2574
+ typeof requestInput?.sql === 'string'
2575
+ ? requestInput.sql
2576
+ : typeof requestInput?.query === 'string'
2577
+ ? requestInput.query
2578
+ : typeof raw?.sql === 'string'
2579
+ ? raw.sql
2580
+ : null;
2581
+ if (!dataset || totalRows === null || !sql) {
2582
+ return wrapped;
2583
+ }
2584
+ const originalRequestInput = requestInput as Record<string, unknown>;
2585
+ const datasetLimit =
2586
+ finitePositiveInteger(dataset.returned_limit) ?? totalRows;
2587
+ const previewRows = rows.slice(0, Math.min(rows.length, 25));
2588
+ const executionNonce =
2589
+ typeof wrapped.job_id === 'string' && wrapped.job_id.trim()
2590
+ ? wrapped.job_id.trim()
2591
+ : `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
2592
+ const datasetId = `tool-list:${sha256Hex(
2593
+ `${toolId}:${this.currentRunId}:${sql}:${datasetLimit}:${executionNonce}`,
2594
+ )}`;
2595
+ const fetchPage = async (
2596
+ offset: number,
2597
+ limit: number,
2598
+ ): Promise<Record<string, unknown>[]> => {
2599
+ if (limit <= 0 || offset >= totalRows) return [];
2600
+ const execution = await this.callToolExecutionAPI(
2601
+ toolId,
2602
+ originalRequestInput,
2603
+ {
2604
+ timeoutMs: resolveToolRuntimeTimeoutMs(toolId),
2605
+ customerDbDataset: {
2606
+ limit: datasetLimit,
2607
+ offset,
2608
+ pageSize: Math.min(limit, QUERY_RESULT_DATASET_PAGE_SIZE),
2609
+ totalRows,
2610
+ },
2611
+ },
2612
+ );
2613
+ const pageRaw =
2614
+ recordOrNull(execution.toolResponse)?.raw ??
2615
+ recordOrNull(execution.result)?.data ??
2616
+ execution.result;
2617
+ return rowsFromUnknown(recordOrNull(pageRaw)?.rows);
2618
+ };
2619
+ const collectRows = async (
2620
+ limit: number | undefined,
2621
+ ): Promise<Record<string, unknown>[]> => {
2622
+ const target = Math.min(limit ?? totalRows, totalRows, datasetLimit);
2623
+ const collected: Record<string, unknown>[] = [];
2624
+ for (
2625
+ let offset = 0;
2626
+ offset < target;
2627
+ offset += QUERY_RESULT_DATASET_PAGE_SIZE
2628
+ ) {
2629
+ collected.push(
2630
+ ...(await fetchPage(
2631
+ offset,
2632
+ Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, target - offset),
2633
+ )),
2634
+ );
2635
+ }
2636
+ return collected.slice(0, target);
2637
+ };
2638
+ const playDataset = createDeferredPlayDataset({
2639
+ datasetKind: 'csv',
2640
+ datasetId,
2641
+ count: Math.min(totalRows, datasetLimit),
2642
+ previewRows,
2643
+ sourceLabel: 'query result rows',
2644
+ tableNamespace: null,
2645
+ resolvers: {
2646
+ count: async () => Math.min(totalRows, datasetLimit),
2647
+ peek: async (limit) => collectRows(limit),
2648
+ materialize: async (limit) => collectRows(limit),
2649
+ iterate: () =>
2650
+ ({
2651
+ async *[Symbol.asyncIterator]() {
2652
+ const count = Math.min(totalRows, datasetLimit);
2653
+ for (
2654
+ let offset = 0;
2655
+ offset < count;
2656
+ offset += QUERY_RESULT_DATASET_PAGE_SIZE
2657
+ ) {
2658
+ yield* await fetchPage(
2659
+ offset,
2660
+ Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, count - offset),
2661
+ );
2662
+ }
2663
+ },
2664
+ }) as AsyncIterable<Record<string, unknown>>,
2665
+ },
2666
+ });
2667
+ return attachToolResultListDataset(wrapped, {
2668
+ name: 'rows',
2669
+ path: 'toolResponse.raw.rows',
2670
+ dataset: playDataset,
2671
+ count: Math.min(totalRows, datasetLimit),
1716
2672
  });
1717
2673
  }
1718
2674
 
@@ -1738,6 +2694,7 @@ export class PlayContextImpl {
1738
2694
  cacheKey,
1739
2695
  receiptKey,
1740
2696
  }),
2697
+ requestInput: request.input,
1741
2698
  });
1742
2699
  let completed: RuntimeStepReceipt | null | undefined = null;
1743
2700
  if (receiptKey) {
@@ -1747,6 +2704,7 @@ export class PlayContextImpl {
1747
2704
  {
1748
2705
  key: receiptKey,
1749
2706
  runId: this.currentRunId,
2707
+ leaseId: request.receiptLeaseId ?? null,
1750
2708
  output: serializeToolExecuteResult(wrapped),
1751
2709
  },
1752
2710
  ])
@@ -1791,6 +2749,7 @@ export class PlayContextImpl {
1791
2749
  cacheKey: entry.request.cacheKey,
1792
2750
  receiptKey: entry.request.receiptKey,
1793
2751
  }),
2752
+ requestInput: entry.request.input,
1794
2753
  }),
1795
2754
  })),
1796
2755
  );
@@ -1801,12 +2760,13 @@ export class PlayContextImpl {
1801
2760
  {
1802
2761
  key: receiptKey,
1803
2762
  runId: this.currentRunId,
2763
+ leaseId: entry.request.receiptLeaseId ?? null,
1804
2764
  output: serializeToolExecuteResult(entry.wrapped),
1805
2765
  },
1806
2766
  ]
1807
2767
  : [];
1808
2768
  });
1809
- let completedByKey: Map<string, RuntimeStepReceipt>;
2769
+ let completedByKey = new Map<string, RuntimeStepReceipt>();
1810
2770
  if (receiptInputs.length > 0) {
1811
2771
  try {
1812
2772
  completedByKey = await this.completeRuntimeStepReceipts(receiptInputs);
@@ -1814,10 +2774,15 @@ export class PlayContextImpl {
1814
2774
  // Completion failed AFTER successful (billed) tool calls: trip the
1815
2775
  // breaker so remaining rows stop dispatching new provider calls.
1816
2776
  tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
2777
+ await Promise.all(
2778
+ wrappedEntries.map((entry) =>
2779
+ this.rejectToolCall(toolId, entry.request, receiptError, {
2780
+ persistReceiptFailure: false,
2781
+ }),
2782
+ ),
2783
+ );
1817
2784
  throw receiptError;
1818
2785
  }
1819
- } else {
1820
- completedByKey = new Map<string, RuntimeStepReceipt>();
1821
2786
  }
1822
2787
  const results: unknown[] = [];
1823
2788
  for (const entry of wrappedEntries) {
@@ -1841,9 +2806,23 @@ export class PlayContextImpl {
1841
2806
  completed: RuntimeStepReceipt | null | undefined,
1842
2807
  ): Promise<unknown> {
1843
2808
  const cacheKey = request.cacheKey;
2809
+ const receiptKey = request.receiptKey?.trim() || null;
2810
+ const canPersistReceiptCompletion = Boolean(
2811
+ this.#options.completeRuntimeStepReceipt ||
2812
+ this.#options.completeRuntimeStepReceipts,
2813
+ );
2814
+ if (
2815
+ receiptKey &&
2816
+ canPersistReceiptCompletion &&
2817
+ completed?.status !== 'completed' &&
2818
+ completed?.status !== 'skipped'
2819
+ ) {
2820
+ throw new Error(
2821
+ `Durable tool call ${receiptKey} lost receipt ownership before completion.`,
2822
+ );
2823
+ }
1844
2824
  const finalWrapped =
1845
- (completed?.status === 'completed' || completed?.status === 'skipped') &&
1846
- completed.runId !== this.currentRunId
2825
+ completed?.status === 'completed' || completed?.status === 'skipped'
1847
2826
  ? await this.wrapToolExecutionResult({
1848
2827
  toolId,
1849
2828
  status:
@@ -1851,6 +2830,7 @@ export class PlayContextImpl {
1851
2830
  ? 'no_result'
1852
2831
  : 'completed',
1853
2832
  result: this.runtimeReceiptOutput(completed),
2833
+ requestInput: request.input,
1854
2834
  execution: toolExecutionMetadataForOutcome(
1855
2835
  completed.runId === this.currentRunId
1856
2836
  ? {
@@ -1861,13 +2841,15 @@ export class PlayContextImpl {
1861
2841
  : {
1862
2842
  kind: 'cache',
1863
2843
  cacheKey,
1864
- receiptKey: request.receiptKey ?? '',
1865
- attachedToReceiptKey: request.receiptKey,
2844
+ receiptKey: receiptKey ?? '',
2845
+ attachedToReceiptKey: receiptKey,
1866
2846
  },
1867
2847
  ),
1868
2848
  })
1869
2849
  : wrapped;
1870
- this.cacheToolResult(toolId, cacheKey, finalWrapped);
2850
+ if (request.cacheable !== false) {
2851
+ this.cacheToolResult(toolId, cacheKey, finalWrapped);
2852
+ }
1871
2853
 
1872
2854
  const resolver = this.toolCallResolvers.get(request.callId);
1873
2855
  if (resolver) {
@@ -1905,7 +2887,9 @@ export class PlayContextImpl {
1905
2887
  {
1906
2888
  key: receiptKey,
1907
2889
  runId: this.currentRunId,
2890
+ leaseId: request.receiptLeaseId ?? null,
1908
2891
  error: message,
2892
+ failureKind: runtimeReceiptFailureKindForError(error),
1909
2893
  },
1910
2894
  ]);
1911
2895
  } catch (receiptError) {
@@ -2189,6 +3173,7 @@ export class PlayContextImpl {
2189
3173
  this.toOutputRow(item as Record<string, unknown>),
2190
3174
  );
2191
3175
  itemOriginalIndexes = materializedItems.map((_item, index) => index);
3176
+ const mapStartSeedRowsByKey = new Map<string, Record<string, unknown>>();
2192
3177
  if (this.#options.onMapStart) {
2193
3178
  const shouldPassRowKey = explicitKeyResolver != null;
2194
3179
  // toSerializableCsvAliasedRow (not a plain spread): projected CSV
@@ -2210,21 +3195,22 @@ export class PlayContextImpl {
2210
3195
  playName: this.#options.playName,
2211
3196
  playId: this.#options.playId,
2212
3197
  runId: this.#options.runId,
3198
+ executorToken: this.#options.executorToken,
2213
3199
  staticPipeline: this.#options.staticPipeline,
2214
- forceRefresh:
2215
- this.#options.cachePolicy?.forceToolRefresh === true,
3200
+ forceRefresh: this.#options.cachePolicy?.forceToolRefresh === true,
2216
3201
  },
2217
3202
  );
2218
3203
  resolvedTableNamespace = normalizeTableNamespace(
2219
3204
  mapStartResult.tableNamespace,
2220
3205
  );
3206
+ if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
3207
+ throw new RuntimeSheetRowsBlockedError({
3208
+ tableNamespace: resolvedTableNamespace,
3209
+ blockedRows: mapStartResult.blockedRows ?? [],
3210
+ });
3211
+ }
2221
3212
  const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
2222
3213
  resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
2223
- const pendingKeys = new Set(
2224
- mapStartResult.pendingRows.map((row, index) =>
2225
- persistedRowIdentity(row, index),
2226
- ),
2227
- );
2228
3214
  const pendingRowsByKey = new Map<string, Record<string, unknown>>();
2229
3215
  for (
2230
3216
  let index = 0;
@@ -2235,15 +3221,20 @@ export class PlayContextImpl {
2235
3221
  const rowKey = persistedRowIdentity(row, index);
2236
3222
  if (rowKey) pendingRowsByKey.set(rowKey, row);
2237
3223
  }
2238
- const pendingItems = materializedItems
3224
+ for (const row of mapStartResult.completedRows ?? []) {
3225
+ const rowKey = persistedRowIdentity(row);
3226
+ if (!rowKey) continue;
3227
+ mapStartSeedRowsByKey.set(rowKey, row);
3228
+ }
3229
+ const seededItems = materializedItems
2239
3230
  .map((item, index) => ({
2240
3231
  row: this.toOutputRow(item as Record<string, unknown>),
2241
3232
  index,
2242
3233
  }))
2243
- .filter(({ row, index }) => pendingKeys.has(rowIdentity(row, index)))
2244
3234
  .map(({ row, index }) => {
2245
3235
  const rowKey = rowIdentity(row, index);
2246
- const persisted = pendingRowsByKey.get(rowKey);
3236
+ const persisted =
3237
+ pendingRowsByKey.get(rowKey) ?? mapStartSeedRowsByKey.get(rowKey);
2247
3238
  // Persisted rows lose the non-enumerable CSV alias projections on
2248
3239
  // the sheet round-trip. Merge the persisted data/meta over the
2249
3240
  // in-memory row so key functions and column resolvers keep seeing
@@ -2253,8 +3244,8 @@ export class PlayContextImpl {
2253
3244
  index,
2254
3245
  };
2255
3246
  });
2256
- itemsToProcess = pendingItems.map((item) => item.row);
2257
- itemOriginalIndexes = pendingItems.map((item) => item.index);
3247
+ itemsToProcess = seededItems.map((item) => item.row);
3248
+ itemOriginalIndexes = seededItems.map((item) => item.index);
2258
3249
  }
2259
3250
 
2260
3251
  const mapScope = this.createMapExecutionScope({
@@ -2325,6 +3316,7 @@ export class PlayContextImpl {
2325
3316
  playName: this.#options.playName,
2326
3317
  playId: this.#options.playId,
2327
3318
  runId: this.#options.runId,
3319
+ executorToken: this.#options.executorToken,
2328
3320
  tableNamespace: resolvedTableNamespace,
2329
3321
  rows: chunk,
2330
3322
  outputFields: datasetColumnNames.filter((field) =>
@@ -2410,7 +3402,7 @@ export class PlayContextImpl {
2410
3402
  const directCompletedResults = mapResult.completedRows.map((row) =>
2411
3403
  this.toPublicOutputRow(row.data),
2412
3404
  );
2413
- const results =
3405
+ let results =
2414
3406
  mapResult.failedRows.length === 0 &&
2415
3407
  directCompletedResults.length === rawItems.length
2416
3408
  ? directCompletedResults
@@ -2463,6 +3455,49 @@ export class PlayContextImpl {
2463
3455
  }
2464
3456
  this.activeMapCellMeta = null;
2465
3457
 
3458
+ if (this.#options.onMapRowsCompleted) {
3459
+ results = await reconcileNodeRuntimeMapResultsWithPersistedSheet({
3460
+ mapName: normalizedMapNamespace,
3461
+ tableNamespace: resolvedTableNamespace,
3462
+ runId: this.#options.runId,
3463
+ expectedRows: totalInputCount,
3464
+ currentRows: results,
3465
+ failedRowCount: mapResult.failedRows.length,
3466
+ log: (line) => this.log(line),
3467
+ readPersistedRows: async (readInput) => {
3468
+ if (
3469
+ !this.#options.baseUrl ||
3470
+ !this.#options.executorToken ||
3471
+ !this.#options.playName ||
3472
+ !this.#options.runId
3473
+ ) {
3474
+ throw new Error(
3475
+ `Runtime sheet finalization mismatch for ctx.dataset("${normalizedMapNamespace}"): ` +
3476
+ 'cannot verify terminal persisted rows because the Node runtime context is missing ' +
3477
+ 'baseUrl, executorToken, playName, or runId.',
3478
+ );
3479
+ }
3480
+ const result = await readRuntimeSheetDatasetRows(
3481
+ {
3482
+ baseUrl: this.#options.baseUrl,
3483
+ executorToken: this.#options.executorToken,
3484
+ playName: this.#options.playName,
3485
+ userEmail: this.#options.userEmail,
3486
+ runId: this.#options.runId,
3487
+ },
3488
+ {
3489
+ tableNamespace: resolvedTableNamespace,
3490
+ runId: this.#options.runId,
3491
+ rowMode: readInput.rowMode,
3492
+ limit: readInput.limit,
3493
+ offset: readInput.offset,
3494
+ },
3495
+ );
3496
+ return result.rows;
3497
+ },
3498
+ });
3499
+ }
3500
+
2466
3501
  return createDeferredPlayDataset({
2467
3502
  datasetKind: 'map',
2468
3503
  datasetId: createRuntimeDatasetId(
@@ -2739,7 +3774,7 @@ export class PlayContextImpl {
2739
3774
  this.lastDatasetStep = datasetStep;
2740
3775
  this.activeDatasetStep = null;
2741
3776
  this.log(
2742
- `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} duplicate keys skipped)`,
3777
+ `Map completed: ${results.length + completedRows} results (${results.length} succeeded, 0 failed, ${completedRows} duplicate keys skipped)`,
2743
3778
  );
2744
3779
  return {
2745
3780
  completedRows: results.map((row, index) =>
@@ -3013,6 +4048,17 @@ export class PlayContextImpl {
3013
4048
  }
3014
4049
  return result.value;
3015
4050
  });
4051
+ const accountedRowIndexes = new Set<number>();
4052
+ for (const row of completedRowsToPersist) {
4053
+ if (typeof row.inputIndex === 'number') {
4054
+ accountedRowIndexes.add(row.inputIndex);
4055
+ }
4056
+ }
4057
+ for (const row of failedRowsToPersist) {
4058
+ if (typeof row.inputIndex === 'number') {
4059
+ accountedRowIndexes.add(row.inputIndex);
4060
+ }
4061
+ }
3016
4062
  if (this.pendingRowEventBoundaries.length > 0) {
3017
4063
  const completedRowKeys = new Set<string>();
3018
4064
  for (let index = 0; index < settledResults.length; index += 1) {
@@ -3025,6 +4071,7 @@ export class PlayContextImpl {
3025
4071
  if (key) {
3026
4072
  completedRowKeys.add(key);
3027
4073
  }
4074
+ accountedRowIndexes.add(executionRowIndex(index));
3028
4075
  continue;
3029
4076
  }
3030
4077
  const row = result as Record<string, unknown>;
@@ -3068,6 +4115,19 @@ export class PlayContextImpl {
3068
4115
  boundaries: uniqueBoundaries,
3069
4116
  });
3070
4117
  }
4118
+ if (accountedRowIndexes.size !== items.length) {
4119
+ const expectedRowIndexes = items.map((_item, index) =>
4120
+ executionRowIndex(index),
4121
+ );
4122
+ const missingRowIndexes = expectedRowIndexes.filter(
4123
+ (index) => !accountedRowIndexes.has(index),
4124
+ );
4125
+ throw new Error(
4126
+ `Runtime map execution accounting mismatch for ctx.dataset("${normalizedTableNamespace}"): ` +
4127
+ `expected ${items.length} row(s), accounted for ${accountedRowIndexes.size}; ` +
4128
+ `missing input index(es): ${missingRowIndexes.slice(0, 10).join(', ') || 'unknown'}.`,
4129
+ );
4130
+ }
3071
4131
  const results = settledResults.filter(
3072
4132
  (result): result is Record<string, unknown> =>
3073
4133
  result !== WAITING_ROW && result !== FAILED_ROW,
@@ -3079,8 +4139,18 @@ export class PlayContextImpl {
3079
4139
  emitEventType: 'map.completed',
3080
4140
  });
3081
4141
  }
4142
+ for (const failedRow of failedRowsToPersist.slice(0, 3)) {
4143
+ this.log(
4144
+ `row ${failedMapRowLogLabel(failedRow)} failed: ${failedRow.error ?? 'unknown error'}`,
4145
+ );
4146
+ }
4147
+ if (failedRowsToPersist.length > 3) {
4148
+ this.log(
4149
+ `${failedRowsToPersist.length - 3} additional row failure(s) omitted from map log`,
4150
+ );
4151
+ }
3082
4152
  this.log(
3083
- `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} duplicate keys skipped)`,
4153
+ `Map completed: ${results.length + completedRows} results (${results.length} succeeded, ${failedRowsToPersist.length} failed, ${completedRows} duplicate keys skipped)`,
3084
4154
  );
3085
4155
  return {
3086
4156
  completedRows: [...completedRowsToPersist].sort(
@@ -3390,6 +4460,13 @@ export class PlayContextImpl {
3390
4460
  if (!hasPendingWork) {
3391
4461
  const status = await raceSettled();
3392
4462
  if (status === 'done' && this.toolCallQueue.length === 0) {
4463
+ if (!this.activeDatasetStep) {
4464
+ const childPlaySuspension =
4465
+ this.consumePendingChildPlaySuspension();
4466
+ if (childPlaySuspension) {
4467
+ throw new PlayExecutionSuspendedError(childPlaySuspension);
4468
+ }
4469
+ }
3393
4470
  break;
3394
4471
  }
3395
4472
 
@@ -3486,16 +4563,23 @@ export class PlayContextImpl {
3486
4563
  toolId,
3487
4564
  requestInput: input,
3488
4565
  });
3489
- const durableCacheKey = await this.durableToolCallCacheKey({
4566
+ let executionAuthScopeDigest =
4567
+ (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
4568
+ const cacheableToolResult = !isQueryResultDatasetReadRequest(toolId, input);
4569
+ let durableCacheKey = await this.durableToolCallCacheKey({
3490
4570
  toolId,
3491
4571
  requestInput: input,
4572
+ executionAuthScopeDigest,
3492
4573
  staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
3493
4574
  });
4575
+ const checkpointCacheKeys = [durableCacheKey];
3494
4576
  const eventWaitHandler =
3495
4577
  (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
3496
4578
  const store = rowContext.getStore();
3497
4579
 
3498
- const executeTool = async (): Promise<unknown> => {
4580
+ const executeTool = async (context?: {
4581
+ leaseId?: string | null;
4582
+ }): Promise<unknown> => {
3499
4583
  if (eventWaitHandler) {
3500
4584
  return this.executeIntegrationEventWaitTool(
3501
4585
  toolId,
@@ -3518,18 +4602,21 @@ export class PlayContextImpl {
3518
4602
 
3519
4603
  if (!store) {
3520
4604
  const directCacheKey = durableCacheKey;
3521
- const cached = toolCachePolicy.force
4605
+ const cached = !cacheableToolResult
3522
4606
  ? null
3523
- : this.getCachedToolResult(toolId, directCacheKey);
3524
- if (cached?.done) {
3525
- this.log(`Tool cache hit: ${toolId} exact payload`);
4607
+ : toolCachePolicy.force
4608
+ ? null
4609
+ : this.getCachedToolResultCandidate(toolId, checkpointCacheKeys);
4610
+ if (cached) {
4611
+ this.log(`Calling tool: ${toolId} recovered from checkpoint`);
3526
4612
  return await this.wrapToolExecutionResult({
3527
4613
  toolId,
3528
- status: cached.result == null ? 'no_result' : 'completed',
3529
- result: cached.result,
4614
+ status: cached.result.result == null ? 'no_result' : 'completed',
4615
+ result: cached.result.result,
4616
+ requestInput: input,
3530
4617
  execution: toolExecutionMetadataForOutcome({
3531
4618
  kind: 'checkpoint',
3532
- cacheKey: directCacheKey,
4619
+ cacheKey: cached.cacheKey,
3533
4620
  }),
3534
4621
  });
3535
4622
  }
@@ -3539,6 +4626,17 @@ export class PlayContextImpl {
3539
4626
  : `Calling tool: ${toolId}`,
3540
4627
  );
3541
4628
  const execution = await this.callToolExecutionAPI(toolId, input, {
4629
+ ...(cacheableToolResult
4630
+ ? {
4631
+ durableCallReceiptKey: directCacheKey,
4632
+ executionAuthScopeDigest,
4633
+ providerIdempotencyKey: this.providerIdempotencyKeyForToolCall({
4634
+ cacheKey: directCacheKey,
4635
+ force: toolCachePolicy.force,
4636
+ leaseId: context?.leaseId ?? null,
4637
+ }),
4638
+ }
4639
+ : {}),
3542
4640
  timeoutMs: resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs),
3543
4641
  });
3544
4642
  const wrapped = await this.wrapToolExecutionResult({
@@ -3548,19 +4646,22 @@ export class PlayContextImpl {
3548
4646
  result: execution.result,
3549
4647
  metadata: execution.metadata,
3550
4648
  meta: execution.meta,
4649
+ requestInput: input,
3551
4650
  execution: toolExecutionMetadataForOutcome({
3552
4651
  kind: 'live',
3553
4652
  cacheKey: directCacheKey,
3554
4653
  }),
3555
4654
  });
3556
- this.checkpoint.completedToolBatches[toolId] = {
3557
- ...(this.checkpoint.completedToolBatches[toolId] ?? {}),
3558
- [directCacheKey]: {
3559
- done: true,
3560
- result: wrapped,
3561
- },
3562
- };
3563
- this.#options.onBatchComplete?.(this.checkpoint);
4655
+ if (cacheableToolResult) {
4656
+ this.checkpoint.completedToolBatches[toolId] = {
4657
+ ...(this.checkpoint.completedToolBatches[toolId] ?? {}),
4658
+ [directCacheKey]: {
4659
+ done: true,
4660
+ result: wrapped,
4661
+ },
4662
+ };
4663
+ this.#options.onBatchComplete?.(this.checkpoint);
4664
+ }
3564
4665
  return wrapped;
3565
4666
  }
3566
4667
 
@@ -3581,18 +4682,21 @@ export class PlayContextImpl {
3581
4682
  }
3582
4683
 
3583
4684
  const toolResultCacheKey = durableCacheKey;
3584
- const cached = toolCachePolicy.force
4685
+ const cached = !cacheableToolResult
3585
4686
  ? null
3586
- : this.getCachedToolResult(toolId, toolResultCacheKey);
3587
- if (cached?.done) {
3588
- this.log(` Row ${rowId} ${toolId}: cache hit exact payload`);
4687
+ : toolCachePolicy.force
4688
+ ? null
4689
+ : this.getCachedToolResultCandidate(toolId, checkpointCacheKeys);
4690
+ if (cached) {
4691
+ this.log(` Row ${rowId} ${toolId}: recovered from checkpoint`);
3589
4692
  return await this.wrapToolExecutionResult({
3590
4693
  toolId,
3591
- status: cached.result == null ? 'no_result' : 'completed',
3592
- result: cached.result,
4694
+ status: cached.result.result == null ? 'no_result' : 'completed',
4695
+ result: cached.result.result,
4696
+ requestInput: input,
3593
4697
  execution: toolExecutionMetadataForOutcome({
3594
4698
  kind: 'checkpoint',
3595
- cacheKey: toolResultCacheKey,
4699
+ cacheKey: cached.cacheKey,
3596
4700
  }),
3597
4701
  });
3598
4702
  }
@@ -3624,8 +4728,11 @@ export class PlayContextImpl {
3624
4728
  this.toolCallQueue.push({
3625
4729
  callId,
3626
4730
  cacheKey: toolResultCacheKey,
3627
- receiptKey: durableCacheKey,
4731
+ cacheable: cacheableToolResult,
4732
+ receiptKey: cacheableToolResult ? durableCacheKey : null,
4733
+ executionAuthScopeDigest,
3628
4734
  force: toolCachePolicy.force,
4735
+ forceFailedRefresh: toolCachePolicy.forceFailedRefresh,
3629
4736
  rowId,
3630
4737
  fieldName,
3631
4738
  toolId,
@@ -3641,45 +4748,85 @@ export class PlayContextImpl {
3641
4748
  });
3642
4749
  };
3643
4750
 
3644
- if (store) {
4751
+ if (store || !cacheableToolResult) {
3645
4752
  return await executeTool();
3646
4753
  }
3647
4754
 
3648
- return this.executeWithRuntimeReceipt(
3649
- 'tool',
3650
- normalizedKey,
3651
- this.currentRunId,
3652
- {
3653
- receiptKey: durableCacheKey,
3654
- semanticKey: toolRequestIdentity,
3655
- force: toolCachePolicy.force,
3656
- staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
3657
- markSkipped: () => {
3658
- this.log(
3659
- `Tool cache hit: ${toolId} exact payload (${normalizedKey}:${toolRequestIdentity})`,
3660
- );
3661
- },
3662
- onClaimedResult: (output, receiptKey) =>
3663
- markToolExecuteResultExecutionOutcome(output, {
3664
- kind: 'live',
3665
- receiptKey,
3666
- }),
3667
- onRecovered: (output, _receipt, source) =>
3668
- markToolExecuteResultExecutionOutcome(
3669
- output,
3670
- toolExecutionOutcomeForDurableReceipt({
3671
- source,
3672
- receiptKey: durableCacheKey,
3673
- }),
3674
- ),
3675
- runningReceiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
3676
- typeof options?.receiptWaitMs === 'number'
3677
- ? { max_wait_ms: options.receiptWaitMs }
3678
- : input,
3679
- ),
3680
- execute: executeTool,
3681
- },
3682
- );
4755
+ for (
4756
+ let authScopeAttempt = 0;
4757
+ authScopeAttempt < 2;
4758
+ authScopeAttempt += 1
4759
+ ) {
4760
+ try {
4761
+ return await this.executeWithRuntimeReceipt(
4762
+ 'tool',
4763
+ normalizedKey,
4764
+ this.currentRunId,
4765
+ {
4766
+ receiptKey: durableCacheKey,
4767
+ semanticKey: toolRequestIdentity,
4768
+ force: toolCachePolicy.force,
4769
+ staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
4770
+ markSkipped: () => {
4771
+ this.log(
4772
+ `ctx.tools.execute(${toolId}): no-op due completed receipt ${toolRequestIdentity} (label: ${normalizedKey})`,
4773
+ );
4774
+ },
4775
+ onClaimedResult: (output, receiptKey) =>
4776
+ markToolExecuteResultExecutionOutcome(output, {
4777
+ kind: 'live',
4778
+ receiptKey,
4779
+ }),
4780
+ onRecovered: (output, _receipt, source) =>
4781
+ markToolExecuteResultExecutionOutcome(
4782
+ output,
4783
+ toolExecutionOutcomeForDurableReceipt({
4784
+ source,
4785
+ receiptKey: durableCacheKey,
4786
+ }),
4787
+ ),
4788
+ shouldPersistFailure: (error) =>
4789
+ !(error instanceof ToolExecuteAuthScopeChangedError),
4790
+ execute: ({ leaseId }) => executeTool({ leaseId }),
4791
+ runningReceiptWaitMaxAttempts:
4792
+ resolveRuntimeToolReceiptWaitMaxAttempts(
4793
+ typeof options?.receiptWaitMs === 'number'
4794
+ ? { max_wait_ms: options.receiptWaitMs }
4795
+ : input,
4796
+ ),
4797
+ },
4798
+ );
4799
+ } catch (error) {
4800
+ if (
4801
+ !(error instanceof ToolExecuteAuthScopeChangedError) ||
4802
+ authScopeAttempt > 0
4803
+ ) {
4804
+ throw error;
4805
+ }
4806
+ // Make the bounded auth-scope re-claim observable in run logs instead
4807
+ // of a silent continuation: the credential identity changed after the
4808
+ // receipt key was prepared, so we evict the cached digest, re-resolve,
4809
+ // and re-claim a fresh receipt under the new scope before retrying once.
4810
+ this.log(
4811
+ `ctx.tools.execute(${toolId}): auth_scope_changed_reclaim ` +
4812
+ `(label: ${normalizedKey}); credential identity changed mid-run, ` +
4813
+ `re-resolving auth scope and re-claiming a fresh receipt under the ` +
4814
+ `new scope (attempt ${authScopeAttempt + 1}/2).`,
4815
+ );
4816
+ this.invalidateToolAuthScopeDigest(toolId);
4817
+ executionAuthScopeDigest =
4818
+ (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
4819
+ durableCacheKey = await this.durableToolCallCacheKey({
4820
+ toolId,
4821
+ requestInput: input,
4822
+ executionAuthScopeDigest,
4823
+ staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
4824
+ });
4825
+ checkpointCacheKeys[0] = durableCacheKey;
4826
+ }
4827
+ }
4828
+
4829
+ throw new ToolExecuteAuthScopeChangedError();
3683
4830
  }
3684
4831
 
3685
4832
  private async executeIntegrationEventWaitTool(
@@ -3805,163 +4952,349 @@ export class PlayContextImpl {
3805
4952
  if (!resolvedName.trim()) {
3806
4953
  throw new Error('ctx.runPlay(...) requires a resolvable play name.');
3807
4954
  }
4955
+ const scheduledChildPlayCall = this.shouldLaunchChildPlaysThroughScheduler()
4956
+ ? this.beginScheduledChildPlayCall()
4957
+ : null;
3808
4958
 
3809
- if (!this.#options.resolvePlay) {
3810
- throw new Error(
3811
- 'ctx.runPlay(...) is unavailable because no play resolver was configured.',
3812
- );
3813
- }
3814
- const resolvePlay = this.#options.resolvePlay;
3815
- const resolvedPlay = await resolvePlay(resolvedName);
3816
- if (!resolvedPlay) {
3817
- throw new Error(
3818
- `Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`,
3819
- );
3820
- }
3821
- const childRevisionFingerprint =
3822
- resolvedPlayRevisionFingerprint(resolvedPlay);
3823
-
3824
- const store = rowContext.getStore();
3825
- const executePlayCall = async (): Promise<TOutput> => {
3826
- // Reserve depth + per-parent + descendant/play-call budget on the parent
3827
- // and fork the lineage-global snapshot for the child. forkChild owns the
3828
- // cycle guard, depth cap, per-parent cap, and play/descendant charges, so
3829
- // a recovered receipt (which skips this body) never consumes budget. The
3830
- // child run id stays deterministic — it matches the prior post-increment
3831
- // playCallCount — so receipt keys and replay are stable.
3832
- const childRunId = `${this.governance.rootRunId}:${resolvedName}:${this.governor.snapshot().playCallCount + 1}`;
3833
- const childGovernance = this.governor.forkChild({
4959
+ try {
4960
+ if (!this.#options.resolvePlay) {
4961
+ throw new Error(
4962
+ 'ctx.runPlay(...) is unavailable because no play resolver was configured.',
4963
+ );
4964
+ }
4965
+ const resolvePlay = this.#options.resolvePlay;
4966
+ const resolvedPlay = await resolvePlay(resolvedName);
4967
+ if (!resolvedPlay) {
4968
+ throw new Error(
4969
+ `Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`,
4970
+ );
4971
+ }
4972
+ const childRevisionFingerprint =
4973
+ resolvedPlayRevisionFingerprint(resolvedPlay);
4974
+ const runPlayRowScope = rowContext.getStore();
4975
+ const runPlaySemanticKey = buildDurableRunPlaySemanticKey({
3834
4976
  childPlayName: resolvedName,
3835
- childRunId,
4977
+ childRevisionFingerprint,
4978
+ input,
4979
+ rowScope: runPlayRowScope
4980
+ ? {
4981
+ fieldName: runPlayRowScope.fieldName,
4982
+ rowId: runPlayRowScope.rowId,
4983
+ rowKey: runPlayRowScope.rowKey ?? null,
4984
+ tableNamespace: runPlayRowScope.tableNamespace ?? null,
4985
+ }
4986
+ : null,
3836
4987
  });
3837
- const childPlaySlot = await this.governor.acquireChildPlaySlot();
3838
- const rowStore = rowContext.getStore();
3839
- const producer = {
3840
- kind: 'play' as const,
3841
- id: normalizedKey,
3842
- playId: resolvedName,
3843
- displayName: displayNameFromProducerId(resolvedName),
3844
- runId: childRunId,
3845
- };
3846
4988
 
3847
- try {
3848
- if (rowStore) {
3849
- this.emitScopedFieldMetaUpdate({
3850
- rowId: rowStore.rowId,
3851
- key: rowStore.rowKey ?? null,
3852
- tableNamespace: rowStore.tableNamespace ?? null,
3853
- fieldName: rowStore.fieldName,
3854
- status: 'running',
3855
- rowStatus: 'running',
3856
- stage: resolvedName,
3857
- provider: 'deepline_native',
3858
- error: null,
3859
- producer,
3860
- dataPatch: {},
3861
- });
3862
- }
3863
- const childContext = createPlayContext({
3864
- ...this.#options,
3865
- playId: resolvedName,
3866
- playName: resolvedName,
3867
- runId: childGovernance.currentRunId,
3868
- staticPipeline: resolvedPlay.staticPipeline ?? null,
3869
- checkpoint: this.checkpoint,
3870
- governance: childGovernance,
3871
- getRuntimeStepReceipt: undefined,
3872
- getRuntimeStepReceipts: undefined,
3873
- claimRuntimeStepReceipt: undefined,
3874
- claimRuntimeStepReceipts: undefined,
3875
- completeRuntimeStepReceipt: undefined,
3876
- completeRuntimeStepReceipts: undefined,
3877
- failRuntimeStepReceipt: undefined,
3878
- failRuntimeStepReceipts: undefined,
3879
- skipRuntimeStepReceipt: undefined,
3880
- });
3881
- const childExecution = this.executeResolvedPlay(
3882
- resolvedPlay,
3883
- childContext,
4989
+ const executePlayCall = async (): Promise<TOutput> => {
4990
+ // Reserve depth + per-parent + descendant/play-call budget on the parent
4991
+ // and fork the lineage-global snapshot for the child. forkChild owns the
4992
+ // cycle guard, depth cap, per-parent cap, and play/descendant charges, so
4993
+ // a recovered receipt (which skips this body) never consumes budget.
4994
+ //
4995
+ // The child run id is derived through the shared canonical builder so the
4996
+ // in-process runtime and the workers_edge coordinator produce the same
4997
+ // FORMAT (`play/<slug>/run/child-<digest>`) via the same code. The digest
4998
+ // is deterministic across replay because every input (parent lineage,
4999
+ // call key, input) is stable. Per-substrate difference: in-process has no
5000
+ // coordinator idempotency key and no compiled child graphHash, so it uses
5001
+ // the semantic fallback with graphHash omitted; workers_edge feeds the
5002
+ // durable child idempotency key. See child-run-id.ts.
5003
+ const childRunId = buildChildRunId({
5004
+ childPlayName: resolvedName,
5005
+ parentRunId: this.currentRunId,
5006
+ parentPlayName: this.#options.playName,
5007
+ key: normalizedKey,
5008
+ runPlaySemanticKey,
3884
5009
  input,
3885
- );
3886
- await childContext.drainQueuedWork([childExecution]);
3887
- const result = await childExecution;
3888
- if (rowStore) {
3889
- this.emitScopedFieldMetaUpdate({
3890
- rowId: rowStore.rowId,
3891
- key: rowStore.rowKey ?? null,
3892
- tableNamespace: rowStore.tableNamespace ?? null,
3893
- fieldName: rowStore.fieldName,
3894
- status: 'completed',
3895
- rowStatus: 'running',
3896
- stage: resolvedName,
3897
- provider: 'deepline_native',
3898
- error: null,
3899
- producer,
3900
- dataPatch: {},
3901
- });
3902
- }
3903
- const step = {
3904
- type: 'play_call' as const,
5010
+ graphHash: null,
5011
+ });
5012
+ const childGovernance = this.governor.forkChild({
5013
+ childPlayName: resolvedName,
5014
+ childRunId,
5015
+ });
5016
+ const childPlaySlot = await this.governor.acquireChildSubmitSlot();
5017
+ const rowStore = rowContext.getStore();
5018
+ const producer = {
5019
+ kind: 'play' as const,
5020
+ id: normalizedKey,
3905
5021
  playId: resolvedName,
3906
- nestedSteps: childContext.getSteps(),
3907
- description: normalizeStepDescription(options?.description),
5022
+ displayName: displayNameFromProducerId(resolvedName),
5023
+ runId: childRunId,
3908
5024
  };
3909
- if (this.activeDatasetStep) {
3910
- this.activeDatasetStep.substeps.push(step);
3911
- } else {
3912
- this.steps.push(step);
3913
- }
3914
- return result as TOutput;
3915
- } catch (error) {
3916
- if (rowStore) {
3917
- this.emitScopedFieldMetaUpdate({
3918
- rowId: rowStore.rowId,
3919
- key: rowStore.rowKey ?? null,
3920
- tableNamespace: rowStore.tableNamespace ?? null,
3921
- fieldName: rowStore.fieldName,
3922
- status: 'failed',
3923
- rowStatus: 'running',
3924
- stage: resolvedName,
3925
- provider: 'deepline_native',
3926
- error: this.formatRuntimeError(error),
3927
- producer,
3928
- dataPatch: {},
5025
+
5026
+ try {
5027
+ if (rowStore) {
5028
+ this.emitScopedFieldMetaUpdate({
5029
+ rowId: rowStore.rowId,
5030
+ key: rowStore.rowKey ?? null,
5031
+ tableNamespace: rowStore.tableNamespace ?? null,
5032
+ fieldName: rowStore.fieldName,
5033
+ status: 'running',
5034
+ rowStatus: 'running',
5035
+ stage: resolvedName,
5036
+ provider: 'deepline_native',
5037
+ error: null,
5038
+ producer,
5039
+ dataPatch: {},
5040
+ });
5041
+ }
5042
+ if (this.shouldLaunchChildPlaysThroughScheduler()) {
5043
+ const boundaryId = this.childPlayBoundaryId(
5044
+ childGovernance.currentRunId,
5045
+ );
5046
+ const checkpointResult = this.childPlayBoundaryOutput<TOutput>({
5047
+ boundaryId,
5048
+ childRunId: childGovernance.currentRunId,
5049
+ childPlayName: resolvedName,
5050
+ });
5051
+ if (checkpointResult.found) {
5052
+ if (rowStore) {
5053
+ this.emitScopedFieldMetaUpdate({
5054
+ rowId: rowStore.rowId,
5055
+ key: rowStore.rowKey ?? null,
5056
+ tableNamespace: rowStore.tableNamespace ?? null,
5057
+ fieldName: rowStore.fieldName,
5058
+ status: 'completed',
5059
+ rowStatus: 'running',
5060
+ stage: resolvedName,
5061
+ provider: 'deepline_native',
5062
+ error: null,
5063
+ producer,
5064
+ dataPatch: {},
5065
+ });
5066
+ }
5067
+ this.recordPlayCallStep({
5068
+ playId: resolvedName,
5069
+ description: options?.description,
5070
+ });
5071
+ return checkpointResult.output;
5072
+ }
5073
+
5074
+ await this.launchScheduledChildPlay({
5075
+ key: normalizedKey,
5076
+ childRunId: childGovernance.currentRunId,
5077
+ childPlayName: resolvedName,
5078
+ childGovernance,
5079
+ childInput: input,
5080
+ description: options?.description ?? null,
5081
+ });
5082
+ childPlaySlot.release();
5083
+ const terminal = await this.readScheduledChildTerminal<TOutput>({
5084
+ childRunId: childGovernance.currentRunId,
5085
+ childPlayName: resolvedName,
5086
+ });
5087
+ if (terminal?.status === 'completed') {
5088
+ this.checkpoint.resolvedBoundaries = {
5089
+ ...(this.checkpoint.resolvedBoundaries ?? {}),
5090
+ [boundaryId]: {
5091
+ kind: 'child_play',
5092
+ childRunId: childGovernance.currentRunId,
5093
+ childPlayName: resolvedName,
5094
+ status: 'completed',
5095
+ output: terminal.output,
5096
+ completedAt: Date.now(),
5097
+ },
5098
+ };
5099
+ if (rowStore) {
5100
+ this.emitScopedFieldMetaUpdate({
5101
+ rowId: rowStore.rowId,
5102
+ key: rowStore.rowKey ?? null,
5103
+ tableNamespace: rowStore.tableNamespace ?? null,
5104
+ fieldName: rowStore.fieldName,
5105
+ status: 'completed',
5106
+ rowStatus: 'running',
5107
+ stage: resolvedName,
5108
+ provider: 'deepline_native',
5109
+ error: null,
5110
+ producer,
5111
+ dataPatch: {},
5112
+ });
5113
+ }
5114
+ this.recordPlayCallStep({
5115
+ playId: resolvedName,
5116
+ description: options?.description,
5117
+ });
5118
+ return terminal.output;
5119
+ }
5120
+ if (terminal) {
5121
+ throw new Error(terminal.error);
5122
+ }
5123
+
5124
+ const suspension = {
5125
+ kind: 'child_play',
5126
+ boundaryId,
5127
+ childRunId: childGovernance.currentRunId,
5128
+ childPlayName: resolvedName,
5129
+ } as const;
5130
+ this.recordPendingChildPlayBoundary(suspension);
5131
+ scheduledChildPlayCall?.release();
5132
+ await this.waitForScheduledChildPlayQuiescence();
5133
+ throw new PlayExecutionSuspendedError(
5134
+ this.consumePendingChildPlaySuspension() ?? suspension,
5135
+ );
5136
+ }
5137
+ const childContext = createPlayContext({
5138
+ ...this.#options,
5139
+ playId: resolvedName,
5140
+ playName: resolvedName,
5141
+ runId: childGovernance.currentRunId,
5142
+ executorToken: await this.mintChildExecutorToken({
5143
+ parentRunId: this.currentRunId,
5144
+ parentPlayName: this.#options.playName,
5145
+ childRunId: childGovernance.currentRunId,
5146
+ childPlayName: resolvedName,
5147
+ }),
5148
+ staticPipeline: resolvedPlay.staticPipeline ?? null,
5149
+ checkpoint: this.checkpoint,
5150
+ governance: childGovernance,
5151
+ getRuntimeStepReceipt: undefined,
5152
+ getRuntimeStepReceipts: undefined,
5153
+ claimRuntimeStepReceipt: undefined,
5154
+ claimRuntimeStepReceipts: undefined,
5155
+ completeRuntimeStepReceipt: undefined,
5156
+ completeRuntimeStepReceipts: undefined,
5157
+ failRuntimeStepReceipt: undefined,
5158
+ failRuntimeStepReceipts: undefined,
5159
+ heartbeatRuntimeStepReceipts: undefined,
5160
+ skipRuntimeStepReceipt: undefined,
3929
5161
  });
5162
+ const childExecution = this.executeResolvedPlay(
5163
+ resolvedPlay,
5164
+ childContext,
5165
+ input,
5166
+ );
5167
+ await childContext.drainQueuedWork([childExecution]);
5168
+ const result = await childExecution;
5169
+ if (rowStore) {
5170
+ this.emitScopedFieldMetaUpdate({
5171
+ rowId: rowStore.rowId,
5172
+ key: rowStore.rowKey ?? null,
5173
+ tableNamespace: rowStore.tableNamespace ?? null,
5174
+ fieldName: rowStore.fieldName,
5175
+ status: 'completed',
5176
+ rowStatus: 'running',
5177
+ stage: resolvedName,
5178
+ provider: 'deepline_native',
5179
+ error: null,
5180
+ producer,
5181
+ dataPatch: {},
5182
+ });
5183
+ }
5184
+ const step = {
5185
+ type: 'play_call' as const,
5186
+ playId: resolvedName,
5187
+ nestedSteps: childContext.getSteps(),
5188
+ description: normalizeStepDescription(options?.description),
5189
+ };
5190
+ if (this.activeDatasetStep) {
5191
+ this.activeDatasetStep.substeps.push(step);
5192
+ } else {
5193
+ this.steps.push(step);
5194
+ }
5195
+ return result as TOutput;
5196
+ } catch (error) {
5197
+ if (isPlayExecutionSuspendedError(error)) {
5198
+ throw error;
5199
+ }
5200
+ if (rowStore) {
5201
+ this.emitScopedFieldMetaUpdate({
5202
+ rowId: rowStore.rowId,
5203
+ key: rowStore.rowKey ?? null,
5204
+ tableNamespace: rowStore.tableNamespace ?? null,
5205
+ fieldName: rowStore.fieldName,
5206
+ status: 'failed',
5207
+ rowStatus: 'running',
5208
+ stage: resolvedName,
5209
+ provider: 'deepline_native',
5210
+ error: this.formatRuntimeError(error),
5211
+ producer,
5212
+ dataPatch: {},
5213
+ });
5214
+ }
5215
+ throw error;
5216
+ } finally {
5217
+ childPlaySlot.release();
3930
5218
  }
3931
- throw error;
3932
- } finally {
3933
- childPlaySlot.release();
3934
- }
3935
- };
5219
+ };
3936
5220
 
3937
- if (store) {
3938
- return await executePlayCall();
5221
+ return await this.executeWithRuntimeReceipt<TOutput>(
5222
+ 'runPlay',
5223
+ normalizedKey,
5224
+ this.currentRunId,
5225
+ {
5226
+ semanticKey: runPlaySemanticKey,
5227
+ staleAfterSeconds: options?.staleAfterSeconds,
5228
+ markSkipped: () => {
5229
+ this.log(
5230
+ `ctx.runPlay(${normalizedKey}): no-op due completed receipt`,
5231
+ );
5232
+ },
5233
+ repairRunningReceiptForSameRunAfterWaitTimeout: true,
5234
+ runningReceiptWaitMaxAttempts:
5235
+ resolveRuntimeToolReceiptWaitMaxAttempts(input),
5236
+ execute: executePlayCall,
5237
+ },
5238
+ );
5239
+ } finally {
5240
+ scheduledChildPlayCall?.release();
3939
5241
  }
5242
+ }
3940
5243
 
3941
- return this.executeWithRuntimeReceipt<TOutput>(
3942
- 'runPlay',
3943
- normalizedKey,
3944
- this.currentRunId,
5244
+ private async mintChildExecutorToken(input: {
5245
+ parentRunId: string;
5246
+ parentPlayName?: string | null;
5247
+ childRunId: string;
5248
+ childPlayName: string;
5249
+ }): Promise<string | undefined> {
5250
+ if (!this.#options.executorToken || !this.#options.baseUrl) {
5251
+ return this.#options.executorToken;
5252
+ }
5253
+ const parentPlayName = input.parentPlayName?.trim();
5254
+ if (!parentPlayName) {
5255
+ return this.#options.executorToken;
5256
+ }
5257
+ const response = await fetch(
5258
+ `${this.#options.baseUrl.replace(/\/$/, '')}/api/v2/internal/play-runtime/child-executor-token`,
3945
5259
  {
3946
- semanticKey: stableDigest(
3947
- stableStringify({
3948
- childPlayName: resolvedName,
3949
- childRevisionFingerprint,
3950
- input,
3951
- }),
3952
- ),
3953
- staleAfterSeconds: options?.staleAfterSeconds,
3954
- markSkipped: () => {
3955
- this.log(
3956
- `ctx.runPlay(${normalizedKey}): no-op due completed receipt`,
3957
- );
5260
+ method: 'POST',
5261
+ headers: {
5262
+ Authorization: `Bearer ${this.#options.executorToken}`,
5263
+ 'Content-Type': 'application/json',
5264
+ ...(this.#options.vercelProtectionBypassToken?.trim()
5265
+ ? {
5266
+ 'x-vercel-protection-bypass':
5267
+ this.#options.vercelProtectionBypassToken.trim(),
5268
+ }
5269
+ : {}),
3958
5270
  },
3959
- repairRunningReceiptForSameRunAfterWaitTimeout: true,
3960
- runningReceiptWaitMaxAttempts:
3961
- resolveRuntimeToolReceiptWaitMaxAttempts(input),
3962
- execute: executePlayCall,
5271
+ body: JSON.stringify({
5272
+ parentRunId: input.parentRunId,
5273
+ parentPlayName,
5274
+ childRunId: input.childRunId,
5275
+ childPlayName: input.childPlayName,
5276
+ }),
3963
5277
  },
3964
5278
  );
5279
+ const body = (await response.json().catch(() => null)) as {
5280
+ executorToken?: unknown;
5281
+ error?: unknown;
5282
+ } | null;
5283
+ if (!response.ok) {
5284
+ const message =
5285
+ typeof body?.error === 'string' && body.error.trim()
5286
+ ? body.error.trim()
5287
+ : response.statusText;
5288
+ throw new Error(
5289
+ `ctx.runPlay failed to mint child executor token for "${input.childPlayName}": ${message} (status ${response.status}).`,
5290
+ );
5291
+ }
5292
+ if (typeof body?.executorToken !== 'string' || !body.executorToken.trim()) {
5293
+ throw new Error(
5294
+ `ctx.runPlay failed to mint child executor token for "${input.childPlayName}": response did not include executorToken.`,
5295
+ );
5296
+ }
5297
+ return body.executorToken.trim();
3965
5298
  }
3966
5299
 
3967
5300
  /**
@@ -4347,17 +5680,21 @@ export class PlayContextImpl {
4347
5680
  await Promise.all(
4348
5681
  [...byTool.entries()].map(async ([toolId, requests]) => {
4349
5682
  this.log(`Executing tool batch ${toolId}: ${requests.length} calls`);
5683
+ const liveStepResults = new Map<string, unknown>();
4350
5684
 
4351
5685
  const recordToolStep = (stepRequests: ToolCallRequest[]): void => {
4352
5686
  if (stepRequests.length === 0) return;
4353
5687
  const stepResults: PlayStepRowResult[] = stepRequests.map((req) => {
4354
- const cachedResult = this.getCachedToolResult(toolId, req.cacheKey);
5688
+ const hasLiveResult = liveStepResults.has(req.callId);
5689
+ const result = hasLiveResult
5690
+ ? liveStepResults.get(req.callId)
5691
+ : this.getCachedToolResult(toolId, req.cacheKey)?.result;
4355
5692
  return {
4356
5693
  rowId: req.rowId,
4357
- status: cachedResult?.result != null ? 'completed' : 'failed',
4358
- success: cachedResult?.result != null,
4359
- value: cachedResult?.result,
4360
- error: cachedResult?.result != null ? null : 'Tool call failed',
5694
+ status: result != null ? 'completed' : 'failed',
5695
+ success: result != null,
5696
+ value: result,
5697
+ error: result != null ? null : 'Tool call failed',
4361
5698
  };
4362
5699
  });
4363
5700
  const toolStep = {
@@ -4377,9 +5714,12 @@ export class PlayContextImpl {
4377
5714
  const pendingRequests: ToolCallRequest[] = [];
4378
5715
  const recoveredRequests: ToolCallRequest[] = [];
4379
5716
  for (const req of requests) {
4380
- const cached = req.force
4381
- ? undefined
4382
- : this.getCachedToolResult(toolId, req.cacheKey);
5717
+ const cached =
5718
+ req.cacheable === false
5719
+ ? undefined
5720
+ : req.force
5721
+ ? undefined
5722
+ : this.getCachedToolResult(toolId, req.cacheKey);
4383
5723
  if (cached?.done) {
4384
5724
  this.log(` Row ${req.rowId} ${toolId}: recovered from checkpoint`);
4385
5725
  const resolver = this.toolCallResolvers.get(req.callId);
@@ -4411,7 +5751,9 @@ export class PlayContextImpl {
4411
5751
  attachedToReceiptKey: followerReceiptKey,
4412
5752
  })
4413
5753
  : result;
4414
- this.cacheToolResult(toolId, follower.cacheKey, followerResult);
5754
+ if (follower.cacheable !== false) {
5755
+ this.cacheToolResult(toolId, follower.cacheKey, followerResult);
5756
+ }
4415
5757
  const resolver = this.toolCallResolvers.get(follower.callId);
4416
5758
  if (resolver) {
4417
5759
  resolver.resolve(followerResult);
@@ -4439,7 +5781,11 @@ export class PlayContextImpl {
4439
5781
  const followers = liveFollowersByOwnerCallId.get(owner.callId) ?? [];
4440
5782
  liveFollowersByOwnerCallId.delete(owner.callId);
4441
5783
  for (const request of [owner, ...followers]) {
4442
- await this.rejectToolCall(toolId, request, error);
5784
+ await this.rejectToolCall(toolId, request, error, {
5785
+ persistReceiptFailure: !(
5786
+ error instanceof ToolExecuteAuthScopeChangedError
5787
+ ),
5788
+ });
4443
5789
  }
4444
5790
  };
4445
5791
  const requestsWithLiveFollowers = (
@@ -4494,7 +5840,11 @@ export class PlayContextImpl {
4494
5840
  ) {
4495
5841
  const requestsByReceiptKey = new Map<
4496
5842
  string,
4497
- { requests: ToolCallRequest[]; forceRefresh: boolean }
5843
+ {
5844
+ requests: ToolCallRequest[];
5845
+ forceRefresh: boolean;
5846
+ forceFailedRefresh: boolean;
5847
+ }
4498
5848
  >();
4499
5849
  const localOnlyRequests: ToolCallRequest[] = [];
4500
5850
  for (const request of pendingRequests) {
@@ -4506,17 +5856,37 @@ export class PlayContextImpl {
4506
5856
  const group = requestsByReceiptKey.get(receiptKey) ?? {
4507
5857
  requests: [],
4508
5858
  forceRefresh: false,
5859
+ forceFailedRefresh: false,
4509
5860
  };
4510
5861
  group.requests.push(request);
4511
5862
  group.forceRefresh ||= request.force === true;
5863
+ group.forceFailedRefresh ||= request.forceFailedRefresh === true;
4512
5864
  requestsByReceiptKey.set(receiptKey, group);
4513
5865
  }
4514
5866
  const normalReceiptKeys = [...requestsByReceiptKey]
4515
- .filter(([, group]) => !group.forceRefresh)
5867
+ .filter(
5868
+ ([, group]) => !group.forceRefresh && !group.forceFailedRefresh,
5869
+ )
4516
5870
  .map(([receiptKey]) => receiptKey);
4517
5871
  const forcedReceiptKeys = [...requestsByReceiptKey]
4518
5872
  .filter(([, group]) => group.forceRefresh)
4519
5873
  .map(([receiptKey]) => receiptKey);
5874
+ const failedRefreshReceiptKeys = [...requestsByReceiptKey]
5875
+ .filter(
5876
+ ([, group]) => !group.forceRefresh && group.forceFailedRefresh,
5877
+ )
5878
+ .map(([receiptKey]) => receiptKey);
5879
+ const forcedExisting = forcedReceiptKeys.length
5880
+ ? await this.getRuntimeStepReceipts(forcedReceiptKeys)
5881
+ : new Map<string, RuntimeStepReceipt>();
5882
+ const forcedRunningReceiptKeys = new Set(
5883
+ [...forcedExisting]
5884
+ .filter(([, receipt]) => receipt.status === 'running')
5885
+ .map(([receiptKey]) => receiptKey),
5886
+ );
5887
+ const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
5888
+ (receiptKey) => !forcedRunningReceiptKeys.has(receiptKey),
5889
+ );
4520
5890
  const claims =
4521
5891
  normalReceiptKeys.length > 0
4522
5892
  ? await this.claimRuntimeStepReceipts(
@@ -4525,17 +5895,30 @@ export class PlayContextImpl {
4525
5895
  )
4526
5896
  : new Map<string, RuntimeStepReceipt>();
4527
5897
  const forcedClaims =
4528
- forcedReceiptKeys.length > 0
5898
+ claimableForcedReceiptKeys.length > 0
4529
5899
  ? await this.claimRuntimeStepReceipts(
4530
- forcedReceiptKeys,
5900
+ claimableForcedReceiptKeys,
4531
5901
  this.currentRunId,
5902
+ false,
4532
5903
  true,
5904
+ )
5905
+ : new Map<string, RuntimeStepReceipt>();
5906
+ const failedRefreshClaims =
5907
+ failedRefreshReceiptKeys.length > 0
5908
+ ? await this.claimRuntimeStepReceipts(
5909
+ failedRefreshReceiptKeys,
5910
+ this.currentRunId,
5911
+ false,
5912
+ false,
4533
5913
  true,
4534
5914
  )
4535
5915
  : new Map<string, RuntimeStepReceipt>();
4536
5916
  for (const [receiptKey, claim] of forcedClaims) {
4537
5917
  claims.set(receiptKey, claim);
4538
5918
  }
5919
+ for (const [receiptKey, claim] of failedRefreshClaims) {
5920
+ claims.set(receiptKey, claim);
5921
+ }
4539
5922
  const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
4540
5923
  const durableRecoveredRequests: ToolCallRequest[] = [];
4541
5924
  const resolveRequestsFromReceipt = async (
@@ -4547,9 +5930,7 @@ export class PlayContextImpl {
4547
5930
  if (!request) return;
4548
5931
  const receiptKey = request.receiptKey?.trim() || null;
4549
5932
  if (!receiptKey) {
4550
- throw new Error(
4551
- `Durable tool receipt ${source} recovery requires a receipt key.`,
4552
- );
5933
+ throw new Error(`${source} tool recovery needs receipt key.`);
4553
5934
  }
4554
5935
  const wrapped = await this.wrapToolExecutionResult({
4555
5936
  toolId,
@@ -4558,6 +5939,7 @@ export class PlayContextImpl {
4558
5939
  ? 'no_result'
4559
5940
  : 'completed',
4560
5941
  result: this.runtimeReceiptOutput(receipt),
5942
+ requestInput: request.input,
4561
5943
  execution: toolExecutionMetadataForOutcome({
4562
5944
  kind: source,
4563
5945
  cacheKey: request.cacheKey,
@@ -4602,6 +5984,7 @@ export class PlayContextImpl {
4602
5984
  const reclaimExistingRunningReceiptAfterWait = async (
4603
5985
  receiptKey: string,
4604
5986
  requestsForKey: ToolCallRequest[],
5987
+ forceAfterWait = false,
4605
5988
  ): Promise<void> => {
4606
5989
  const waitMaxAttempts =
4607
5990
  requestsForKey.length > 0
@@ -4616,15 +5999,18 @@ export class PlayContextImpl {
4616
5999
  )
4617
6000
  : undefined;
4618
6001
  try {
4619
- await resolveRequestsFromReceipt(
4620
- requestsForKey,
4621
- await this.waitForCompletedRuntimeToolReceipt(
4622
- receiptKey,
4623
- waitMaxAttempts,
4624
- ),
4625
- 'in_flight',
6002
+ const completed = await this.waitForCompletedRuntimeToolReceipt(
6003
+ receiptKey,
6004
+ waitMaxAttempts,
4626
6005
  );
4627
- return;
6006
+ if (!forceAfterWait) {
6007
+ await resolveRequestsFromReceipt(
6008
+ requestsForKey,
6009
+ completed,
6010
+ 'in_flight',
6011
+ );
6012
+ return;
6013
+ }
4628
6014
  } catch (error) {
4629
6015
  if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
4630
6016
  for (const request of requestsForKey) {
@@ -4640,6 +6026,7 @@ export class PlayContextImpl {
4640
6026
  [receiptKey],
4641
6027
  this.currentRunId,
4642
6028
  true,
6029
+ forceAfterWait,
4643
6030
  )
4644
6031
  ).get(receiptKey);
4645
6032
  if (
@@ -4670,14 +6057,31 @@ export class PlayContextImpl {
4670
6057
  ) {
4671
6058
  const [owner, ...waiters] = requestsForKey;
4672
6059
  if (!owner) return;
6060
+ if (!reclaimed.leaseId) {
6061
+ throw new RuntimeReceiptLeaseLostError({
6062
+ receiptKey,
6063
+ runId: this.currentRunId,
6064
+ leaseId: 'missing',
6065
+ });
6066
+ }
6067
+ owner.receiptLeaseId = reclaimed.leaseId ?? null;
4673
6068
  if (waiters.length > 0) {
4674
6069
  liveFollowersByOwnerCallId.set(owner.callId, waiters);
4675
6070
  }
4676
6071
  try {
6072
+ await this.assertRuntimeToolReceiptOwnership([owner]);
4677
6073
  const execution = await this.callToolExecutionAPI(
4678
6074
  toolId,
4679
6075
  owner.input,
4680
6076
  {
6077
+ durableCallReceiptKey: receiptKey,
6078
+ executionAuthScopeDigest: owner.executionAuthScopeDigest,
6079
+ providerIdempotencyKey:
6080
+ this.providerIdempotencyKeyForToolCall({
6081
+ cacheKey: owner.cacheKey,
6082
+ force: owner.force === true,
6083
+ leaseId: owner.receiptLeaseId,
6084
+ }),
4681
6085
  timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([owner]),
4682
6086
  },
4683
6087
  );
@@ -4689,6 +6093,7 @@ export class PlayContextImpl {
4689
6093
  execution?.jobId,
4690
6094
  execution?.meta,
4691
6095
  );
6096
+ liveStepResults.set(owner.callId, result);
4692
6097
  resolveLiveFollowers(owner, result);
4693
6098
  recordToolStep([owner]);
4694
6099
  this.#options.onBatchComplete?.(this.checkpoint);
@@ -4711,8 +6116,13 @@ export class PlayContextImpl {
4711
6116
  const requestsForKey = group.requests;
4712
6117
  const claim = claims.get(receiptKey);
4713
6118
  if (!claim) {
4714
- const [owner, ...waiters] = requestsForKey;
4715
- claimOwnerWithLiveFollowers(owner, waiters);
6119
+ durableExistingRunningHandlers.push(
6120
+ reclaimExistingRunningReceiptAfterWait(
6121
+ receiptKey,
6122
+ requestsForKey,
6123
+ group.forceRefresh,
6124
+ ),
6125
+ );
4716
6126
  continue;
4717
6127
  }
4718
6128
  if (claim?.status === 'completed' || claim?.status === 'skipped') {
@@ -4735,6 +6145,16 @@ export class PlayContextImpl {
4735
6145
  claim.claimState !== 'existing'
4736
6146
  ) {
4737
6147
  const [owner, ...waiters] = requestsForKey;
6148
+ if (!claim.leaseId) {
6149
+ throw new RuntimeReceiptLeaseLostError({
6150
+ receiptKey,
6151
+ runId: this.currentRunId,
6152
+ leaseId: 'missing',
6153
+ });
6154
+ }
6155
+ if (owner) {
6156
+ owner.receiptLeaseId = claim.leaseId ?? null;
6157
+ }
4738
6158
  claimOwnerWithLiveFollowers(owner, waiters);
4739
6159
  continue;
4740
6160
  }
@@ -4746,6 +6166,7 @@ export class PlayContextImpl {
4746
6166
  reclaimExistingRunningReceiptAfterWait(
4747
6167
  receiptKey,
4748
6168
  requestsForKey,
6169
+ group.forceRefresh,
4749
6170
  ),
4750
6171
  );
4751
6172
  continue;
@@ -4754,6 +6175,7 @@ export class PlayContextImpl {
4754
6175
  reclaimExistingRunningReceiptAfterWait(
4755
6176
  receiptKey,
4756
6177
  requestsForKey,
6178
+ group.forceRefresh,
4757
6179
  ),
4758
6180
  );
4759
6181
  }
@@ -4778,18 +6200,13 @@ export class PlayContextImpl {
4778
6200
  compiledBatches.length > 0
4779
6201
  ? await this.governor.suggestedParallelism(
4780
6202
  compiledBatches[0]!.batchOperation,
4781
- 4,
6203
+ this.governor.policy.pacing
6204
+ .workerToolBatchDefaultParallelism,
4782
6205
  )
4783
- : 4;
6206
+ : this.governor.policy.pacing.workerToolBatchDefaultParallelism;
4784
6207
  await executeChunkedRequests({
4785
6208
  requests: compiledBatches,
4786
6209
  batchSize,
4787
- // Bounded dispatch wave (postgres_fast parity with workers_edge):
4788
- // cap the provider calls (counted by batch member size) a single
4789
- // chunk dispatches before the latch is read again, so a
4790
- // persistence failure prevents the rest of a large batch group.
4791
- maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
4792
- weightOf: (batch) => batch.memberRequests.length,
4793
6210
  execute: async (batch) => {
4794
6211
  // Circuit breaker: skip dispatching this batch's provider call
4795
6212
  // once a persistence failure has occurred in this run.
@@ -4800,10 +6217,42 @@ export class PlayContextImpl {
4800
6217
  this.persistenceLatch,
4801
6218
  );
4802
6219
  }
6220
+ await this.assertRuntimeToolReceiptOwnership(
6221
+ batch.memberRequests,
6222
+ );
6223
+ const receiptKeys = batch.memberRequests.map(
6224
+ (request) => request.cacheKey,
6225
+ );
6226
+ const aggregateReceiptKey = buildDurableToolAggregateReceiptKey(
6227
+ {
6228
+ receiptKeys,
6229
+ prefix: 'batch',
6230
+ aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
6231
+ orgId: this.#options.orgId,
6232
+ toolId: batch.batchOperation,
6233
+ }),
6234
+ },
6235
+ );
4803
6236
  return await this.callToolAPI(
4804
6237
  batch.batchOperation,
4805
6238
  batch.batchPayload,
4806
6239
  {
6240
+ durableCallReceiptKey: aggregateReceiptKey,
6241
+ executionAuthScopeDigest:
6242
+ batch.memberRequests[0]?.executionAuthScopeDigest ?? null,
6243
+ providerIdempotencyKey:
6244
+ buildDurableToolAggregateProviderIdempotencyKey({
6245
+ aggregateReceiptKey,
6246
+ receiptKeys,
6247
+ providerIdempotencyKeys: batch.memberRequests.map(
6248
+ (request) =>
6249
+ this.providerIdempotencyKeyForToolCall({
6250
+ cacheKey: request.cacheKey,
6251
+ force: request.force === true,
6252
+ leaseId: request.receiptLeaseId,
6253
+ }),
6254
+ ),
6255
+ }),
4807
6256
  timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
4808
6257
  batch.memberRequests,
4809
6258
  ),
@@ -4837,6 +6286,7 @@ export class PlayContextImpl {
4837
6286
  index += 1
4838
6287
  ) {
4839
6288
  const request = entry.request.memberRequests[index]!;
6289
+ liveStepResults.set(request.callId, resolvedResults[index]);
4840
6290
  resolveLiveFollowers(request, resolvedResults[index]);
4841
6291
  }
4842
6292
  }
@@ -4848,13 +6298,9 @@ export class PlayContextImpl {
4848
6298
  },
4849
6299
  });
4850
6300
  } else {
4851
- // Bounded dispatch wave (postgres_fast parity with workers_edge):
4852
- // an unbatched tool group can carry a whole chunk's rows, so cap the
4853
- // provider calls dispatched before the latch is read again to the
4854
- // wave size. Respect a lower provider-pacing suggestion when present.
4855
- const batchSize = Math.min(
4856
- await this.governor.suggestedParallelism(toolId, 50),
4857
- PROVIDER_DISPATCH_WAVE_SIZE,
6301
+ const batchSize = await this.governor.suggestedParallelism(
6302
+ toolId,
6303
+ this.governor.policy.pacing.suggestedMaxParallelism,
4858
6304
  );
4859
6305
  for (
4860
6306
  let start = 0;
@@ -4877,10 +6323,24 @@ export class PlayContextImpl {
4877
6323
  return;
4878
6324
  }
4879
6325
  try {
6326
+ await this.assertRuntimeToolReceiptOwnership([request]);
4880
6327
  const execution = await this.callToolExecutionAPI(
4881
6328
  toolId,
4882
6329
  request.input,
4883
6330
  {
6331
+ ...(request.receiptKey
6332
+ ? {
6333
+ durableCallReceiptKey: request.receiptKey,
6334
+ executionAuthScopeDigest:
6335
+ request.executionAuthScopeDigest,
6336
+ providerIdempotencyKey:
6337
+ this.providerIdempotencyKeyForToolCall({
6338
+ cacheKey: request.receiptKey,
6339
+ force: request.force === true,
6340
+ leaseId: request.receiptLeaseId,
6341
+ }),
6342
+ }
6343
+ : {}),
4884
6344
  timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
4885
6345
  request,
4886
6346
  ]),
@@ -4894,6 +6354,7 @@ export class PlayContextImpl {
4894
6354
  execution?.jobId,
4895
6355
  execution?.meta,
4896
6356
  );
6357
+ liveStepResults.set(request.callId, result);
4897
6358
  resolveLiveFollowers(request, result);
4898
6359
  } catch (error) {
4899
6360
  await rejectWithLiveFollowers(request, error);
@@ -4989,7 +6450,7 @@ export class PlayContextImpl {
4989
6450
  private async callToolAPI(
4990
6451
  toolId: string,
4991
6452
  input: Record<string, unknown>,
4992
- options?: { timeoutMs?: number },
6453
+ options?: ToolExecutionApiOptions,
4993
6454
  ): Promise<unknown> {
4994
6455
  const execution = await this.callToolExecutionAPI(toolId, input, options);
4995
6456
  if (execution.toolResponse && 'raw' in execution.toolResponse) {
@@ -5003,10 +6464,27 @@ export class PlayContextImpl {
5003
6464
  : execution.result;
5004
6465
  }
5005
6466
 
6467
+ private providerIdempotencyKeyForToolCall(input: {
6468
+ cacheKey: string;
6469
+ force?: boolean;
6470
+ leaseId?: string | null;
6471
+ }): string {
6472
+ const fallbackAttemptId =
6473
+ input.force === true && !input.leaseId
6474
+ ? `${this.currentRunId}:${crypto.randomUUID()}`
6475
+ : null;
6476
+ return buildDurableToolProviderIdempotencyKey({
6477
+ receiptKey: input.cacheKey,
6478
+ force: input.force,
6479
+ receiptLeaseId: input.leaseId,
6480
+ fallbackAttemptId,
6481
+ });
6482
+ }
6483
+
5006
6484
  private async callToolExecutionAPI(
5007
6485
  toolId: string,
5008
6486
  input: Record<string, unknown>,
5009
- options?: { timeoutMs?: number },
6487
+ options?: ToolExecutionApiOptions,
5010
6488
  ): Promise<ParsedToolExecuteResponse> {
5011
6489
  if (!this.#options.executorToken || !this.#options.baseUrl) {
5012
6490
  throw new Error(
@@ -5045,6 +6523,20 @@ export class PlayContextImpl {
5045
6523
 
5046
6524
  while (true) {
5047
6525
  let response: Response;
6526
+ const durableCallReceiptKey =
6527
+ options?.durableCallReceiptKey?.trim() || null;
6528
+ const executionAuthScopeDigest =
6529
+ durableCallReceiptKey && options?.executionAuthScopeDigest
6530
+ ? options.executionAuthScopeDigest.trim() || null
6531
+ : durableCallReceiptKey
6532
+ ? ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ??
6533
+ null)
6534
+ : null;
6535
+ const providerIdempotencyKey =
6536
+ options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
6537
+ const deeplineRequestId = providerIdempotencyKey
6538
+ ? `ctx-tool-${stableDigest(providerIdempotencyKey).slice(0, 32)}`
6539
+ : null;
5048
6540
  const abortController = timeoutMs ? new AbortController() : null;
5049
6541
  const timeoutHandle = timeoutMs
5050
6542
  ? setTimeout(() => {
@@ -5064,7 +6556,14 @@ export class PlayContextImpl {
5064
6556
  Authorization: `Bearer ${this.#options.executorToken}`,
5065
6557
  [EXECUTE_RESPONSE_CONTRACT_HEADER]:
5066
6558
  V2_EXECUTE_RESPONSE_CONTRACT,
6559
+ [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
5067
6560
  [EXECUTE_TOOL_METADATA_HEADER]: 'true',
6561
+ ...(deeplineRequestId
6562
+ ? {
6563
+ 'x-deepline-request-id': deeplineRequestId,
6564
+ 'x-deepline-idempotency-key': providerIdempotencyKey!,
6565
+ }
6566
+ : {}),
5068
6567
  ...(this.#options.vercelProtectionBypassToken?.trim()
5069
6568
  ? {
5070
6569
  'x-vercel-protection-bypass':
@@ -5074,7 +6573,44 @@ export class PlayContextImpl {
5074
6573
  },
5075
6574
  body: JSON.stringify({
5076
6575
  payload: input,
5077
- metadata: { parent_run_id: this.#options.runId },
6576
+ metadata: {
6577
+ parent_run_id: this.#options.runId,
6578
+ ...(durableCallReceiptKey
6579
+ ? {
6580
+ durable_call_receipt_key: durableCallReceiptKey,
6581
+ ...(executionAuthScopeDigest
6582
+ ? {
6583
+ execution_auth_scope_digest:
6584
+ executionAuthScopeDigest,
6585
+ }
6586
+ : {}),
6587
+ }
6588
+ : {}),
6589
+ ...(options?.customerDbDataset
6590
+ ? {
6591
+ query_result_dataset: {
6592
+ limit: options.customerDbDataset.limit,
6593
+ offset: options.customerDbDataset.offset,
6594
+ page_size: options.customerDbDataset.pageSize,
6595
+ total_rows: options.customerDbDataset.totalRows,
6596
+ },
6597
+ ...(isCustomerDbDatasetTool(toolId)
6598
+ ? {
6599
+ customer_db_dataset: {
6600
+ limit: options.customerDbDataset.limit,
6601
+ offset: options.customerDbDataset.offset,
6602
+ page_size: options.customerDbDataset.pageSize,
6603
+ total_rows:
6604
+ options.customerDbDataset.totalRows,
6605
+ },
6606
+ }
6607
+ : {}),
6608
+ }
6609
+ : {}),
6610
+ ...(providerIdempotencyKey
6611
+ ? { provider_idempotency_key: providerIdempotencyKey }
6612
+ : {}),
6613
+ },
5078
6614
  ...(this.#options.integrationMode
5079
6615
  ? { integration_mode: this.#options.integrationMode }
5080
6616
  : {}),
@@ -5118,6 +6654,13 @@ export class PlayContextImpl {
5118
6654
 
5119
6655
  if (!response.ok) {
5120
6656
  const text = await response.text();
6657
+ const authScopeChanged = parseToolExecuteAuthScopeChangedError({
6658
+ status: response.status,
6659
+ bodyText: text,
6660
+ });
6661
+ if (authScopeChanged) {
6662
+ throw authScopeChanged;
6663
+ }
5121
6664
  const httpFailureAttempt = httpFailureAttempts.next({
5122
6665
  toolId,
5123
6666
  status: response.status,