deepline 0.1.153 → 0.1.155

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +15 -0
  2. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1180 -825
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts +34 -18
  4. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +41 -0
  5. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +143 -8
  6. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +104 -0
  7. package/dist/bundling-sources/sdk/src/index.ts +0 -1
  8. package/dist/bundling-sources/sdk/src/play.ts +3 -48
  9. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +27 -2
  10. package/dist/bundling-sources/sdk/src/release.ts +6 -4
  11. package/dist/bundling-sources/sdk/src/worker-play-entry.ts +0 -10
  12. package/dist/bundling-sources/shared_libs/play-data-plane/index.ts +0 -1
  13. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +87 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +0 -59
  15. package/dist/bundling-sources/shared_libs/play-runtime/cell-staleness.ts +0 -253
  16. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +805 -1570
  17. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +47 -74
  18. package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +36 -14
  19. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +145 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +284 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/postgres-json.ts +12 -5
  22. package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts +78 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +10 -45
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +1 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +923 -535
  26. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +45 -76
  27. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +12 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/step-program-dataset-builder.ts +1 -14
  29. package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts +159 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/tool-result-types.ts +4 -1
  31. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +32 -0
  32. package/dist/bundling-sources/shared_libs/plays/definition.ts +4 -2
  33. package/dist/bundling-sources/shared_libs/plays/runtime-validation.ts +3 -14
  34. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +1 -43
  35. package/dist/cli/index.js +1305 -401
  36. package/dist/cli/index.mjs +1273 -363
  37. package/dist/{compiler-manifest-BjoRENv9.d.ts → compiler-manifest-DW1flrHk.d.mts} +0 -9
  38. package/dist/{compiler-manifest-BjoRENv9.d.mts → compiler-manifest-DW1flrHk.d.ts} +0 -9
  39. package/dist/index.d.mts +9 -38
  40. package/dist/index.d.ts +9 -38
  41. package/dist/index.js +26 -13
  42. package/dist/index.mjs +26 -13
  43. package/dist/plays/bundle-play-file.d.mts +2 -2
  44. package/dist/plays/bundle-play-file.d.ts +2 -2
  45. package/package.json +1 -1
  46. package/dist/bundling-sources/shared_libs/play-data-plane/cell-policy.ts +0 -76
  47. package/dist/bundling-sources/shared_libs/play-runtime/progress-emitter.ts +0 -197
  48. package/dist/bundling-sources/shared_libs/play-runtime/waterfall-replay.ts +0 -79
@@ -3,11 +3,9 @@
3
3
  *
4
4
  * Batching model:
5
5
  * 1. ctx.dataset("table_key", rows).withColumn("field", resolver).run() starts all row column resolvers concurrently
6
- * 2. ctx.waterfall() calls inside field resolvers QUEUE requests (don't execute)
7
- * 3. ctx.tools.execute() calls inside field resolvers also QUEUE
8
- * 4. After all rows have queued, executeBatchedWaterfalls() runs provider-by-provider
9
- * 5. Each provider batch = real HTTP call to /api/v2/integrations/{toolId}/execute
10
- * 6. Results resolve suspended row promises, rows complete
6
+ * 2. ctx.tools.execute() calls inside field resolvers QUEUE requests
7
+ * 3. After all rows have queued, executeBatchedToolCalls() runs provider calls
8
+ * 4. Results resolve suspended row promises, rows complete
11
9
  *
12
10
  * Temporal integration:
13
11
  * - checkpoint: recovered from heartbeatDetails on retry (skip completed batches)
@@ -22,9 +20,6 @@ import type { PlayDataset, PlayDatasetInput } from '@shared_libs/plays/dataset';
22
20
  import {
23
21
  compileRequestsWithStrategy,
24
22
  executeChunkedRequests,
25
- executeWaterfallProviders,
26
- formatChunkExecutionError,
27
- resolveWaterfallToolId,
28
23
  } from './batch-runtime';
29
24
  import type { PlayQueueHint } from './governor/rate-state-backend';
30
25
  import type { MapRowOutcome } from './durability-store';
@@ -54,15 +49,34 @@ import {
54
49
  deserializeToolExecuteResult,
55
50
  type ParsedToolExecuteResponse,
56
51
  type ToolExecuteResult,
52
+ type ToolResultExecutionMetadata,
57
53
  type ToolResultMetadataInput,
58
54
  } from './tool-result';
55
+ import {
56
+ markToolExecuteResultExecutionOutcome,
57
+ toolExecutionMetadataForOutcome,
58
+ toolExecutionOutcomeForDurableReceipt,
59
+ type DurableReceiptRecoverySource,
60
+ type ToolExecutionOutcome,
61
+ } from './tool-execution-outcome';
59
62
  import {
60
63
  TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
61
64
  classifyToolExecuteHttpFailure,
62
65
  createToolExecuteHttpFailureAttemptTracker,
63
66
  } from './tool-execute-retry-policy';
67
+ import {
68
+ buildDurableCtxCallCacheKey,
69
+ buildDurableToolCallAuthScopeDigest,
70
+ buildDurableToolCallCacheKey,
71
+ } from './durable-call-cache';
72
+ import {
73
+ RuntimeReceiptWaitTimeoutError,
74
+ executeWithDurableRuntimeReceipt,
75
+ runtimeReceiptOutput as durableRuntimeReceiptOutput,
76
+ waitForCompletedRuntimeReceipt,
77
+ type DurableReceiptExecutionStore,
78
+ } from './durable-receipt-execution';
64
79
  import { isRowIsolationExemptError } from './row-isolation';
65
- import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline';
66
80
  import { createRuntimeDatasetId } from './dataset-id';
67
81
  import { dedupeExplicitMapKeyRows } from './map-row-identity';
68
82
  import {
@@ -77,18 +91,8 @@ import {
77
91
  import {
78
92
  DEEPLINE_CELL_META_FIELD,
79
93
  previousCellFromValue,
80
- resolveCompletedCellStalenessMeta,
81
- resolveReusableCellMetaForCurrentPolicy,
82
- shouldRecomputeCell,
83
- type AuthoredCellStalenessPolicyByField,
84
- type CellStalenessMeta,
85
- type CellStalenessPolicyByField,
86
94
  type PreviousCell,
87
95
  } from './cell-staleness';
88
- import {
89
- authoredCellPolicyMap,
90
- normalizeAuthoredCellPolicyMap,
91
- } from '../play-data-plane/cell-policy';
92
96
  import {
93
97
  cloneCsvAliasedRow,
94
98
  stripCsvProjectedFields,
@@ -122,9 +126,6 @@ import {
122
126
  import type {
123
127
  CsvOptions,
124
128
  RowState,
125
- WaterfallRequest,
126
- WaterfallOptions,
127
- InlineWaterfallSpec,
128
129
  DatasetOptions,
129
130
  ToolCallRequest,
130
131
  ToolBatchResult,
@@ -202,15 +203,43 @@ function isUnsafeOutboundUrlError(error: unknown): boolean {
202
203
  return error instanceof Error && error.name === 'UnsafeOutboundUrlError';
203
204
  }
204
205
 
206
+ function publicToolResponseEnvelope(value: unknown): {
207
+ status: string;
208
+ raw: unknown;
209
+ meta?: Record<string, unknown>;
210
+ } | null {
211
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
212
+ return null;
213
+ }
214
+ const record = value as Record<string, unknown>;
215
+ if (typeof record.status !== 'string') return null;
216
+ const toolResponse = record.toolResponse;
217
+ if (
218
+ !toolResponse ||
219
+ typeof toolResponse !== 'object' ||
220
+ Array.isArray(toolResponse) ||
221
+ !Object.prototype.hasOwnProperty.call(toolResponse, 'raw')
222
+ ) {
223
+ return null;
224
+ }
225
+ const response = toolResponse as Record<string, unknown>;
226
+ return {
227
+ status: record.status,
228
+ raw: response.raw,
229
+ ...(response.meta &&
230
+ typeof response.meta === 'object' &&
231
+ !Array.isArray(response.meta)
232
+ ? { meta: response.meta as Record<string, unknown> }
233
+ : {}),
234
+ };
235
+ }
236
+
205
237
  const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
206
238
  const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
207
239
  const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-response';
208
240
  const IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT = 25;
209
- const WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT = 10;
210
- const WATERFALL_ROW_MATCH_LOG_INTERVAL = 1_000;
211
241
  const BATCH_SIZE_LOG_SAMPLE_LIMIT = 10;
212
242
  const STEP_PROGRAM_MAP_DEFINITION = Symbol('deepline.stepProgramMapDefinition');
213
- const CELL_STALENESS_POLICIES = Symbol('deepline.cellStalenessPolicies');
214
243
 
215
244
  function shouldPersistMapCellField(fieldName: string): boolean {
216
245
  return !fieldName.startsWith('_') || fieldName === '_metadata';
@@ -396,25 +425,6 @@ function stableDigest(value: string): string {
396
425
 
397
426
  type DurableCtxOperation = 'step' | 'tool' | 'fetch' | 'runPlay';
398
427
 
399
- function staleBucketForSeconds(
400
- staleAfterSeconds?: number | null,
401
- nowMs = Date.now(),
402
- ): string | null {
403
- if (staleAfterSeconds === undefined || staleAfterSeconds === null) {
404
- return null;
405
- }
406
- if (
407
- !Number.isFinite(staleAfterSeconds) ||
408
- !Number.isInteger(staleAfterSeconds) ||
409
- staleAfterSeconds <= 0
410
- ) {
411
- throw new Error(
412
- 'staleAfterSeconds must be a positive whole number of seconds.',
413
- );
414
- }
415
- return `${staleAfterSeconds}:${Math.floor(nowMs / (staleAfterSeconds * 1000))}`;
416
- }
417
-
418
428
  function durableCtxKey(input: {
419
429
  orgId?: string | null;
420
430
  playId: string;
@@ -423,19 +433,19 @@ function durableCtxKey(input: {
423
433
  semanticKey?: string | null;
424
434
  staleAfterSeconds?: number | null;
425
435
  }): string {
426
- const orgId = input.orgId?.trim() || 'org';
427
- const playId = input.playId.trim() || 'play';
428
- const digest = stableDigest(
429
- JSON.stringify({
430
- orgId,
431
- playId,
432
- operation: input.operation,
433
- id: input.id,
434
- semanticKey: input.semanticKey ?? null,
435
- staleBucket: staleBucketForSeconds(input.staleAfterSeconds),
436
- }),
437
- );
438
- return `ctx:${orgId}:${playId}:${input.operation}:${input.id}:${digest}`;
436
+ if (input.operation === 'tool') {
437
+ throw new Error(
438
+ 'Durable ctx key is only for non-tool call boundaries; tools use buildDurableToolCallCacheKey.',
439
+ );
440
+ }
441
+ return buildDurableCtxCallCacheKey({
442
+ orgId: input.orgId,
443
+ playId: input.playId,
444
+ kind: input.operation,
445
+ id: input.id,
446
+ semanticKey: input.semanticKey,
447
+ staleAfterSeconds: input.staleAfterSeconds,
448
+ });
439
449
  }
440
450
 
441
451
  function resolvedPlayRevisionFingerprint(play: ResolvedPlayExecution): string {
@@ -642,102 +652,8 @@ function createPacingResolver(
642
652
  };
643
653
  }
644
654
 
645
- function isInlineWaterfallSpec(value: unknown): value is InlineWaterfallSpec {
646
- if (!value || typeof value !== 'object' || Array.isArray(value)) {
647
- return false;
648
- }
649
- const candidate = value as InlineWaterfallSpec;
650
- return (
651
- typeof candidate.id === 'string' &&
652
- typeof candidate.output === 'string' &&
653
- typeof candidate.minResults === 'number' &&
654
- candidate.minResults >= 1 &&
655
- Array.isArray(candidate.steps) &&
656
- candidate.steps.length > 0
657
- );
658
- }
659
-
660
- function isInlineWaterfallCodeStep(
661
- step: InlineWaterfallSpec['steps'][number],
662
- ): step is Extract<InlineWaterfallSpec['steps'][number], { kind: 'code' }> {
663
- return step.kind === 'code';
664
- }
665
-
666
- function isInlineWaterfallToolStep(
667
- step: InlineWaterfallSpec['steps'][number],
668
- ): step is Extract<InlineWaterfallSpec['steps'][number], { toolId: string }> {
669
- return !isInlineWaterfallCodeStep(step);
670
- }
671
-
672
- function extractInlineWaterfallCodeStepValue(
673
- output: string,
674
- result: unknown,
675
- ): unknown {
676
- if (
677
- result &&
678
- typeof result === 'object' &&
679
- !Array.isArray(result) &&
680
- output in result
681
- ) {
682
- return (result as Record<string, unknown>)[output] ?? null;
683
- }
684
- return result ?? null;
685
- }
686
-
687
- function isMeaningfulValue(value: unknown): boolean {
688
- if (value == null) return false;
689
- if (typeof value === 'string') return value.trim().length > 0;
690
- if (Array.isArray(value)) return value.length > 0;
691
- if (typeof value === 'object') return Object.keys(value).length > 0;
692
- return true;
693
- }
694
-
695
- function getValueAtPath(value: unknown, path: string): unknown {
696
- let current = value;
697
- for (const part of path.split('.').filter(Boolean)) {
698
- if (!current || typeof current !== 'object') return undefined;
699
- current = (current as Record<string, unknown>)[part];
700
- }
701
- return current;
702
- }
703
-
704
- async function extractWaterfallOutputValue(
705
- toolId: string,
706
- output: string,
707
- result: unknown,
708
- getToolTargetGetters?: (
709
- toolId: string,
710
- output: string,
711
- ) => Promise<readonly string[]>,
712
- ): Promise<unknown> {
713
- const getters = getToolTargetGetters
714
- ? await getToolTargetGetters(toolId, output)
715
- : [];
716
- for (const getter of getters) {
717
- const value = getValueAtPath(result, getter);
718
- if (isMeaningfulValue(value)) {
719
- return value;
720
- }
721
- }
722
-
723
- const directOutputValue = getValueAtPath(result, output);
724
- if (directOutputValue !== undefined) {
725
- return isMeaningfulValue(directOutputValue) ? directOutputValue : null;
726
- }
727
-
728
- if (output === 'people' || output === 'companies') {
729
- if (Array.isArray(result)) return result;
730
- const direct = getValueAtPath(result, output);
731
- if (Array.isArray(direct)) return direct;
732
- }
733
-
734
- return isMeaningfulValue(result) ? result : null;
735
- }
736
-
737
655
  export class PlayContextImpl {
738
656
  private rowStates = new Map<number, RowState>();
739
- private resolvers = new Map<string, (value: unknown) => void>();
740
- private waterfallQueue = new Map<string, WaterfallRequest[]>();
741
657
  private toolCallQueue: ToolCallRequest[] = [];
742
658
  private toolCallResolvers = new Map<
743
659
  string,
@@ -767,14 +683,12 @@ export class PlayContextImpl {
767
683
  timeoutMs: number;
768
684
  }> = [];
769
685
  private processedRowCount = 0;
770
- private directToolCallIndex = 0;
771
686
  private sleepBoundaryIndex = 0;
772
687
  private fetchCallIndex = 0;
773
688
  private readonly secretRedactor: SecretRedactionContext =
774
689
  createSecretRedactionContext();
775
690
  private mapInvocationIndex = 0;
776
691
  private readonly stepCallIndexByKey = new Map<string, number>();
777
- private readonly waterfallMatchLogCounts = new Map<string, number>();
778
692
  /**
779
693
  * The single source of every concurrency, budget, and pacing decision for this
780
694
  * run-attempt. Tool/play/row slots are blocking semaphores; budgets are charged
@@ -1093,6 +1007,7 @@ export class PlayContextImpl {
1093
1007
  key: string,
1094
1008
  runId: string,
1095
1009
  reclaimRunning = false,
1010
+ forceRefresh = false,
1096
1011
  ): Promise<RuntimeStepReceipt | null> {
1097
1012
  if (!this.#options.claimRuntimeStepReceipt) {
1098
1013
  return null;
@@ -1101,6 +1016,7 @@ export class PlayContextImpl {
1101
1016
  key,
1102
1017
  runId,
1103
1018
  ...(reclaimRunning ? { reclaimRunning: true } : {}),
1019
+ ...(forceRefresh ? { forceRefresh: true } : {}),
1104
1020
  });
1105
1021
  if (!claimed || typeof claimed.key !== 'string' || !claimed.key.trim()) {
1106
1022
  return null;
@@ -1163,122 +1079,267 @@ export class PlayContextImpl {
1163
1079
  };
1164
1080
  }
1165
1081
 
1082
+ private normalizeRuntimeStepReceipt(
1083
+ key: string,
1084
+ receipt: RuntimeStepReceipt | null | undefined,
1085
+ ): RuntimeStepReceipt | null {
1086
+ if (!receipt) return null;
1087
+ if (typeof receipt.key === 'string' && receipt.key.trim()) {
1088
+ return {
1089
+ ...receipt,
1090
+ key: receipt.key.trim(),
1091
+ runId: receipt.runId ?? null,
1092
+ };
1093
+ }
1094
+ return {
1095
+ ...receipt,
1096
+ key: key.trim(),
1097
+ runId: receipt.runId ?? null,
1098
+ };
1099
+ }
1100
+
1101
+ private async getRuntimeStepReceipts(
1102
+ keys: string[],
1103
+ ): Promise<Map<string, RuntimeStepReceipt>> {
1104
+ const uniqueKeys = [
1105
+ ...new Set(keys.map((key) => key.trim()).filter(Boolean)),
1106
+ ];
1107
+ if (uniqueKeys.length === 0) return new Map();
1108
+ const receipts = this.#options.getRuntimeStepReceipts
1109
+ ? await this.#options.getRuntimeStepReceipts({ keys: uniqueKeys })
1110
+ : await Promise.all(
1111
+ uniqueKeys.map((key) => this.getRuntimeStepReceipt(key)),
1112
+ );
1113
+ const byKey = new Map<string, RuntimeStepReceipt>();
1114
+ for (let index = 0; index < receipts.length; index += 1) {
1115
+ const normalized = this.normalizeRuntimeStepReceipt(
1116
+ uniqueKeys[index] ?? '',
1117
+ receipts[index],
1118
+ );
1119
+ if (normalized) byKey.set(normalized.key, normalized);
1120
+ }
1121
+ return byKey;
1122
+ }
1123
+
1124
+ private async claimRuntimeStepReceipts(
1125
+ keys: string[],
1126
+ runId: string,
1127
+ reclaimRunning = false,
1128
+ forceRefresh = false,
1129
+ ): Promise<Map<string, RuntimeStepReceipt>> {
1130
+ const uniqueKeys = [
1131
+ ...new Set(keys.map((key) => key.trim()).filter(Boolean)),
1132
+ ];
1133
+ if (uniqueKeys.length === 0) return new Map();
1134
+ const receipts = this.#options.claimRuntimeStepReceipts
1135
+ ? await this.#options.claimRuntimeStepReceipts({
1136
+ keys: uniqueKeys,
1137
+ runId,
1138
+ ...(reclaimRunning ? { reclaimRunning: true } : {}),
1139
+ ...(forceRefresh ? { forceRefresh: true } : {}),
1140
+ })
1141
+ : await Promise.all(
1142
+ uniqueKeys.map((key) =>
1143
+ this.claimRuntimeStepReceipt(
1144
+ key,
1145
+ runId,
1146
+ reclaimRunning,
1147
+ forceRefresh,
1148
+ ),
1149
+ ),
1150
+ );
1151
+ const byKey = new Map<string, RuntimeStepReceipt>();
1152
+ for (let index = 0; index < receipts.length; index += 1) {
1153
+ const normalized = this.normalizeRuntimeStepReceipt(
1154
+ uniqueKeys[index] ?? '',
1155
+ receipts[index],
1156
+ );
1157
+ if (normalized) byKey.set(normalized.key, normalized);
1158
+ }
1159
+ return byKey;
1160
+ }
1161
+
1162
+ private async completeRuntimeStepReceipts(
1163
+ receipts: Array<{ key: string; runId: string; output: unknown | null }>,
1164
+ ): Promise<Map<string, RuntimeStepReceipt>> {
1165
+ const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
1166
+ if (normalizedInputs.length === 0) return new Map();
1167
+ for (const receipt of normalizedInputs) {
1168
+ assertNoSecretTaint(receipt.output, 'ctx.tool receipt output');
1169
+ }
1170
+ const completed = this.#options.completeRuntimeStepReceipts
1171
+ ? await this.#options.completeRuntimeStepReceipts({
1172
+ receipts: normalizedInputs,
1173
+ })
1174
+ : await Promise.all(
1175
+ normalizedInputs.map((receipt) =>
1176
+ this.completeRuntimeStepReceipt(
1177
+ receipt.key,
1178
+ receipt.runId,
1179
+ receipt.output,
1180
+ ),
1181
+ ),
1182
+ );
1183
+ const byKey = new Map<string, RuntimeStepReceipt>();
1184
+ for (let index = 0; index < completed.length; index += 1) {
1185
+ const normalized = this.normalizeRuntimeStepReceipt(
1186
+ normalizedInputs[index]?.key ?? '',
1187
+ completed[index],
1188
+ );
1189
+ if (normalized) byKey.set(normalized.key, normalized);
1190
+ }
1191
+ return byKey;
1192
+ }
1193
+
1194
+ private async failRuntimeStepReceipts(
1195
+ receipts: Array<{ key: string; runId: string; error: string }>,
1196
+ ): Promise<Map<string, RuntimeStepReceipt>> {
1197
+ const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
1198
+ if (normalizedInputs.length === 0) return new Map();
1199
+ const failed = this.#options.failRuntimeStepReceipts
1200
+ ? await this.#options.failRuntimeStepReceipts({
1201
+ receipts: normalizedInputs,
1202
+ })
1203
+ : await Promise.all(
1204
+ normalizedInputs.map((receipt) =>
1205
+ this.failRuntimeStepReceipt(
1206
+ receipt.key,
1207
+ receipt.runId,
1208
+ receipt.error,
1209
+ ),
1210
+ ),
1211
+ );
1212
+ const byKey = new Map<string, RuntimeStepReceipt>();
1213
+ for (let index = 0; index < failed.length; index += 1) {
1214
+ const normalized = this.normalizeRuntimeStepReceipt(
1215
+ normalizedInputs[index]?.key ?? '',
1216
+ failed[index],
1217
+ );
1218
+ if (normalized) byKey.set(normalized.key, normalized);
1219
+ }
1220
+ return byKey;
1221
+ }
1222
+
1223
+ private async seedForcedRuntimeStepReceipts(keys: string[]): Promise<void> {
1224
+ const receiptKeys = [
1225
+ ...new Set(keys.map((key) => key.trim()).filter(Boolean)),
1226
+ ];
1227
+ if (receiptKeys.length === 0) return;
1228
+ await this.claimRuntimeStepReceipts(
1229
+ receiptKeys,
1230
+ this.currentRunId,
1231
+ true,
1232
+ true,
1233
+ );
1234
+ }
1235
+
1236
+ private async durableToolCallCacheKey(input: {
1237
+ toolId: string;
1238
+ requestInput: Record<string, unknown>;
1239
+ staleAfterSeconds?: number | null;
1240
+ }): Promise<string> {
1241
+ const providerActionVersion =
1242
+ (await this.#options.getToolActionCacheVersion?.(input.toolId))?.trim() ??
1243
+ '';
1244
+ return buildDurableToolCallCacheKey({
1245
+ orgId: this.#options.orgId,
1246
+ playId: this.governance.currentPlayId,
1247
+ toolId: input.toolId,
1248
+ requestInput: input.requestInput,
1249
+ authScopeDigest: buildDurableToolCallAuthScopeDigest({
1250
+ orgId: this.#options.orgId,
1251
+ userEmail: this.#options.userEmail,
1252
+ toolId: input.toolId,
1253
+ }),
1254
+ providerActionVersion,
1255
+ staleAfterSeconds: input.staleAfterSeconds,
1256
+ });
1257
+ }
1258
+
1259
+ private durableReceiptExecutionStore(): DurableReceiptExecutionStore {
1260
+ return {
1261
+ enabled: Boolean(this.#options.claimRuntimeStepReceipt),
1262
+ get: (receiptKey) => this.getRuntimeStepReceipt(receiptKey),
1263
+ getMany: (receiptKeys) => this.getRuntimeStepReceipts(receiptKeys),
1264
+ claim: (receiptKey, runId, reclaimRunning, forceRefresh) =>
1265
+ this.claimRuntimeStepReceipt(
1266
+ receiptKey,
1267
+ runId,
1268
+ reclaimRunning,
1269
+ forceRefresh,
1270
+ ),
1271
+ complete: (receiptKey, runId, output) =>
1272
+ this.completeRuntimeStepReceipt(receiptKey, runId, output),
1273
+ fail: (receiptKey, runId, error) =>
1274
+ this.failRuntimeStepReceipt(receiptKey, runId, error),
1275
+ canPersistFailure: Boolean(this.#options.failRuntimeStepReceipt),
1276
+ canPersistCompletion: Boolean(this.#options.completeRuntimeStepReceipt),
1277
+ };
1278
+ }
1279
+
1280
+ private runtimeReceiptOutput<T>(receipt: RuntimeStepReceipt): T {
1281
+ return durableRuntimeReceiptOutput<T>(receipt);
1282
+ }
1283
+
1284
+ private async waitForCompletedRuntimeToolReceipt(
1285
+ key: string,
1286
+ ): Promise<RuntimeStepReceipt> {
1287
+ return await waitForCompletedRuntimeReceipt({
1288
+ receiptKey: key,
1289
+ store: this.durableReceiptExecutionStore(),
1290
+ });
1291
+ }
1292
+
1166
1293
  private async executeWithRuntimeReceipt<T>(
1167
1294
  operation: DurableCtxOperation,
1168
1295
  id: string,
1169
1296
  runId: string,
1170
1297
  opts: {
1171
1298
  force?: boolean;
1299
+ receiptKey?: string | null;
1172
1300
  semanticKey?: string | null;
1173
1301
  staleAfterSeconds?: number | null;
1174
1302
  repairRunningReceiptForSameRun?: boolean;
1303
+ repairRunningReceiptForSameRunAfterWaitTimeout?: boolean;
1175
1304
  reclaimRunning?: boolean;
1176
1305
  markSkipped?: (output: T) => Promise<void> | void;
1306
+ onRecovered?: (
1307
+ output: T,
1308
+ receipt: RuntimeStepReceipt,
1309
+ source: DurableReceiptRecoverySource,
1310
+ ) => T;
1311
+ onClaimedResult?: (output: T, receiptKey: string) => T;
1177
1312
  execute: () => Promise<T>;
1178
1313
  },
1179
1314
  ): Promise<T> {
1180
- if (!this.#options.claimRuntimeStepReceipt) {
1181
- return await opts.execute();
1182
- }
1183
- if (opts.force === true) {
1184
- return await opts.execute();
1185
- }
1186
- const receiptKey = durableCtxKey({
1187
- orgId: this.#options.orgId,
1188
- playId: this.governance.currentPlayId,
1315
+ const receiptKey =
1316
+ opts.receiptKey?.trim() ||
1317
+ durableCtxKey({
1318
+ orgId: this.#options.orgId,
1319
+ playId: this.governance.currentPlayId,
1320
+ operation,
1321
+ id,
1322
+ semanticKey: opts.semanticKey,
1323
+ staleAfterSeconds: opts.staleAfterSeconds,
1324
+ });
1325
+ return await executeWithDurableRuntimeReceipt<T>({
1189
1326
  operation,
1190
1327
  id,
1191
- semanticKey: opts.semanticKey,
1192
- staleAfterSeconds: opts.staleAfterSeconds,
1193
- });
1194
- const recoverCompletedReceipt = async (
1195
- receipt: RuntimeStepReceipt,
1196
- ): Promise<T> => {
1197
- this.log(`ctx.${operation}(${id}): recovered result from receipt`);
1198
- if (receipt.output === undefined) {
1199
- return receipt.output as T;
1200
- }
1201
- // Tool results are persisted in their serialized form (live getters and
1202
- // the non-enumerable toolOutput/_metadata fields do not survive JSON
1203
- // storage). Rehydrate so a recovered `ctx.tools.execute` result behaves
1204
- // exactly like a live one — same contract as dataset cell persistence.
1205
- const output = isSerializedToolExecuteResult(receipt.output)
1206
- ? (deserializeToolExecuteResult(receipt.output) as T)
1207
- : (receipt.output as T);
1208
- if (opts.markSkipped) {
1209
- await opts.markSkipped(output);
1210
- }
1211
- return output;
1212
- };
1213
- const claimed = await this.claimRuntimeStepReceipt(
1214
- receiptKey,
1215
1328
  runId,
1216
- opts.reclaimRunning === true,
1217
- );
1218
- if (claimed?.status === 'completed' || claimed?.status === 'skipped') {
1219
- return await recoverCompletedReceipt(claimed);
1220
- }
1221
- if (!claimed) {
1222
- const latest = await this.getRuntimeStepReceipt(receiptKey);
1223
- if (latest?.status === 'completed' || latest?.status === 'skipped') {
1224
- return await recoverCompletedReceipt(latest);
1225
- }
1226
- if (latest?.status === 'running') {
1227
- // Running receipts are in-flight telemetry, not locks. Execute in
1228
- // parallel and let completion reconcile against the latest receipt.
1229
- } else {
1230
- if (latest?.status === 'failed') {
1231
- throw new Error(
1232
- `ctx.${operation}(${id}): receipt is failed and could not be claimed: ${latest.error ?? 'unknown error'}.`,
1233
- );
1234
- }
1235
- throw new Error(
1236
- `ctx.${operation}(${id}): receipt claim did not return execution ownership.`,
1237
- );
1238
- }
1239
- } else if (claimed.status === 'running') {
1240
- // Existing running receipts do not grant exclusive execution ownership.
1241
- // The completion path decides whether this attempt is still the newest.
1242
- } else if (claimed.status === 'failed') {
1243
- throw new Error(
1244
- `ctx.${operation}(${id}): receipt is failed and could not be claimed: ${claimed.error ?? 'unknown error'}.`,
1245
- );
1246
- }
1247
-
1248
- let result: T;
1249
- try {
1250
- result = await opts.execute();
1251
- assertNoSecretTaint(result, `ctx.${operation} result`);
1252
- } catch (error) {
1253
- const failed = await this.failRuntimeStepReceipt(
1254
- receiptKey,
1255
- runId,
1256
- this.formatRuntimeError(error),
1257
- );
1258
- if (!failed && this.#options.failRuntimeStepReceipt) {
1259
- throw new Error(
1260
- `ctx.${operation}(${id}): execution failed and failed receipt could not be persisted: ${this.formatRuntimeError(error)}`,
1261
- );
1262
- }
1263
- throw error;
1264
- }
1265
- // Persist tool results in their serialized form so the recovery path can
1266
- // rebuild the live ToolExecuteResult wrapper (see recoverCompletedReceipt).
1267
- const completed = await this.completeRuntimeStepReceipt(
1268
1329
  receiptKey,
1269
- runId,
1270
- isToolExecuteResult(result) ? serializeToolExecuteResult(result) : result,
1271
- );
1272
- if (!completed && this.#options.completeRuntimeStepReceipt) {
1273
- return result;
1274
- }
1275
- if (
1276
- completed &&
1277
- (completed.status === 'completed' || completed.status === 'skipped')
1278
- ) {
1279
- return await recoverCompletedReceipt(completed);
1280
- }
1281
- return result;
1330
+ store: this.durableReceiptExecutionStore(),
1331
+ force: opts.force,
1332
+ repairRunningReceiptForSameRun: opts.repairRunningReceiptForSameRun,
1333
+ repairRunningReceiptForSameRunAfterWaitTimeout:
1334
+ opts.repairRunningReceiptForSameRunAfterWaitTimeout,
1335
+ reclaimRunning: opts.reclaimRunning,
1336
+ markSkipped: opts.markSkipped,
1337
+ onRecovered: opts.onRecovered,
1338
+ onClaimedResult: opts.onClaimedResult,
1339
+ formatError: (error) => this.formatRuntimeError(error),
1340
+ log: (message) => this.log(message),
1341
+ execute: opts.execute,
1342
+ });
1282
1343
  }
1283
1344
 
1284
1345
  private get currentRunId(): string {
@@ -1402,29 +1463,6 @@ export class PlayContextImpl {
1402
1463
  });
1403
1464
  }
1404
1465
 
1405
- private emitQueuedInlineWaterfallSteps(
1406
- rowId: number,
1407
- key: string | null,
1408
- tableNamespace: string | null,
1409
- spec: InlineWaterfallSpec,
1410
- ): void {
1411
- for (const step of spec.steps) {
1412
- this.emitCellUpdate({
1413
- rowId,
1414
- key,
1415
- tableNamespace,
1416
- columnName: sqlSafePlayColumnName(`${spec.id}.${step.id}`),
1417
- status: 'queued',
1418
- stage: step.id,
1419
- provider: isInlineWaterfallToolStep(step) ? step.toolId : 'code',
1420
- producer: isInlineWaterfallToolStep(step)
1421
- ? { kind: 'tool', toolId: step.toolId, displayName: step.toolId }
1422
- : { kind: 'code', id: step.id, displayName: step.id },
1423
- value: null,
1424
- });
1425
- }
1426
- }
1427
-
1428
1466
  private isCompletedFieldValue(value: unknown): boolean {
1429
1467
  return (
1430
1468
  value !== null &&
@@ -1433,48 +1471,6 @@ export class PlayContextImpl {
1433
1471
  );
1434
1472
  }
1435
1473
 
1436
- private existingFieldReuseDecision(
1437
- row: Record<string, unknown>,
1438
- fieldName: string,
1439
- policies?: CellStalenessPolicyByField,
1440
- authoredPolicies?: AuthoredCellStalenessPolicyByField,
1441
- ): {
1442
- reuse: boolean;
1443
- completedAt?: number;
1444
- stalenessMeta?: Pick<CellStalenessMeta, 'staleAfterSeconds' | 'staleAt'>;
1445
- } {
1446
- const hasValue =
1447
- Object.prototype.hasOwnProperty.call(row, fieldName) &&
1448
- this.isCompletedFieldValue(row[fieldName]);
1449
- const meta = this.cellMetaForField(row, fieldName);
1450
- const completedAt =
1451
- typeof meta?.completedAt === 'number' && Number.isFinite(meta.completedAt)
1452
- ? meta.completedAt
1453
- : undefined;
1454
- const stalenessMeta = resolveReusableCellMetaForCurrentPolicy({
1455
- hasValue,
1456
- value: row[fieldName],
1457
- meta,
1458
- policy: authoredPolicies?.[fieldName],
1459
- });
1460
- const decisionMeta =
1461
- Object.keys(stalenessMeta).length > 0
1462
- ? { ...meta, ...stalenessMeta }
1463
- : meta;
1464
- const reuse =
1465
- shouldRecomputeCell({
1466
- hasValue,
1467
- value: row[fieldName],
1468
- meta: decisionMeta,
1469
- policy: policies?.[fieldName],
1470
- }).action === 'reuse';
1471
- return {
1472
- reuse,
1473
- ...(completedAt !== undefined ? { completedAt } : {}),
1474
- ...(Object.keys(stalenessMeta).length > 0 ? { stalenessMeta } : {}),
1475
- };
1476
- }
1477
-
1478
1474
  // --- Tool-result cells survive the dataset persist/resume boundary ---
1479
1475
  // A ToolExecuteResult carries live getter methods that cannot be serialized
1480
1476
  // to durable storage. We store its serialized form on write and rehydrate it
@@ -1544,24 +1540,6 @@ export class PlayContextImpl {
1544
1540
  : null;
1545
1541
  }
1546
1542
 
1547
- private cellPoliciesForMapDefinition(
1548
- definition: MapFieldDefinition<Record<string, unknown>>,
1549
- ): CellStalenessPolicyByField {
1550
- const raw = (definition as Record<symbol, unknown>)[
1551
- CELL_STALENESS_POLICIES
1552
- ];
1553
- return normalizeAuthoredCellPolicyMap(raw);
1554
- }
1555
-
1556
- private authoredCellPoliciesForMapDefinition(
1557
- definition: MapFieldDefinition<Record<string, unknown>>,
1558
- ): AuthoredCellStalenessPolicyByField {
1559
- const raw = (definition as Record<symbol, unknown>)[
1560
- CELL_STALENESS_POLICIES
1561
- ];
1562
- return authoredCellPolicyMap(raw);
1563
- }
1564
-
1565
1543
  private toVisibleDataPatch(
1566
1544
  fields: Record<string, unknown>,
1567
1545
  ): Record<string, unknown> {
@@ -1579,31 +1557,14 @@ export class PlayContextImpl {
1579
1557
  return String(error);
1580
1558
  }
1581
1559
 
1582
- private logWaterfallMatch(input: {
1583
- queueKey: string;
1584
- rowId: number;
1585
- provider: string;
1586
- }): void {
1587
- const count = (this.waterfallMatchLogCounts.get(input.queueKey) ?? 0) + 1;
1588
- this.waterfallMatchLogCounts.set(input.queueKey, count);
1589
-
1590
- if (count <= WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT) {
1591
- this.log(
1592
- ` Row ${input.rowId}: found with ${input.provider} (${count}/${WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT} sample)`,
1593
- );
1594
- if (count === WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT) {
1595
- this.log(
1596
- ` Further per-row matches for ${input.queueKey} will be summarized every ${WATERFALL_ROW_MATCH_LOG_INTERVAL.toLocaleString()} hits.`,
1597
- );
1598
- }
1599
- return;
1600
- }
1601
-
1602
- if (count % WATERFALL_ROW_MATCH_LOG_INTERVAL === 0) {
1603
- this.log(
1604
- ` ${count.toLocaleString()} rows matched for ${input.queueKey}; latest row ${input.rowId} with ${input.provider}`,
1605
- );
1606
- }
1560
+ private effectiveToolCallCachePolicy(options?: ToolCallOptions): {
1561
+ force: boolean;
1562
+ staleAfterSeconds?: number | null;
1563
+ } {
1564
+ return {
1565
+ force: options?.force === true,
1566
+ staleAfterSeconds: options?.staleAfterSeconds ?? null,
1567
+ };
1607
1568
  }
1608
1569
 
1609
1570
  private summarizeBatchSizes(sizes: readonly number[]): string {
@@ -1675,30 +1636,24 @@ export class PlayContextImpl {
1675
1636
  result: unknown;
1676
1637
  metadata?: ToolResultMetadataInput | null;
1677
1638
  meta?: Record<string, unknown>;
1678
- cached: boolean;
1679
- source: 'live' | 'checkpoint' | 'cache';
1680
- cacheKey?: string;
1639
+ execution: ToolResultExecutionMetadata;
1681
1640
  }): Promise<ToolExecuteResult> {
1682
1641
  if (isToolExecuteResult(input.result)) {
1683
- return cloneToolExecuteResultWithExecution(input.result, {
1684
- idempotent: true,
1685
- cached: input.cached,
1686
- source: input.source,
1687
- ...(input.cacheKey ? { cacheKey: input.cacheKey } : {}),
1688
- });
1642
+ return cloneToolExecuteResultWithExecution(input.result, input.execution);
1689
1643
  }
1644
+ const publicToolResult = publicToolResponseEnvelope(input.result);
1690
1645
  return createToolExecuteResult({
1691
- status: input.status,
1646
+ status: publicToolResult?.status ?? input.status,
1692
1647
  jobId: input.jobId,
1693
- result: input.result,
1648
+ result: publicToolResult
1649
+ ? {
1650
+ data: publicToolResult.raw,
1651
+ ...(publicToolResult.meta ? { meta: publicToolResult.meta } : {}),
1652
+ }
1653
+ : input.result,
1694
1654
  metadata:
1695
1655
  input.metadata ?? (await this.resolveToolResultMetadata(input.toolId)),
1696
- execution: {
1697
- idempotent: true,
1698
- cached: input.cached,
1699
- source: input.source,
1700
- ...(input.cacheKey ? { cacheKey: input.cacheKey } : {}),
1701
- },
1656
+ execution: input.execution,
1702
1657
  meta: input.meta,
1703
1658
  });
1704
1659
  }
@@ -1710,13 +1665,9 @@ export class PlayContextImpl {
1710
1665
  metadata?: ToolResultMetadataInput | null,
1711
1666
  jobId?: string,
1712
1667
  meta?: Record<string, unknown>,
1713
- ): Promise<void> {
1714
- const cacheKey = this.buildToolResultCacheKey({
1715
- rowId: request.rowId,
1716
- tableNamespace: request.tableNamespace,
1717
- rowKey: request.rowKey ?? undefined,
1718
- callId: request.callId,
1719
- });
1668
+ ): Promise<unknown> {
1669
+ const cacheKey = request.cacheKey;
1670
+ const receiptKey = request.receiptKey?.trim() || null;
1720
1671
  const wrapped = await this.wrapToolExecutionResult({
1721
1672
  toolId,
1722
1673
  status: result == null ? 'no_result' : 'completed',
@@ -1724,15 +1675,128 @@ export class PlayContextImpl {
1724
1675
  result,
1725
1676
  metadata,
1726
1677
  meta,
1727
- cached: false,
1728
- source: 'live',
1729
- cacheKey,
1678
+ execution: toolExecutionMetadataForOutcome({
1679
+ kind: 'live',
1680
+ cacheKey,
1681
+ receiptKey,
1682
+ }),
1683
+ });
1684
+ const completed = receiptKey
1685
+ ? (
1686
+ await this.completeRuntimeStepReceipts([
1687
+ {
1688
+ key: receiptKey,
1689
+ runId: this.currentRunId,
1690
+ output: serializeToolExecuteResult(wrapped),
1691
+ },
1692
+ ])
1693
+ ).get(receiptKey)
1694
+ : null;
1695
+ return await this.finalizeResolvedToolCall(
1696
+ toolId,
1697
+ request,
1698
+ wrapped,
1699
+ completed,
1700
+ );
1701
+ }
1702
+
1703
+ private async resolveToolCallBatchResults(
1704
+ toolId: string,
1705
+ entries: Array<{
1706
+ request: ToolCallRequest;
1707
+ result: unknown | null;
1708
+ metadata?: ToolResultMetadataInput | null;
1709
+ jobId?: string;
1710
+ meta?: Record<string, unknown>;
1711
+ }>,
1712
+ ): Promise<unknown[]> {
1713
+ const wrappedEntries = await Promise.all(
1714
+ entries.map(async (entry) => ({
1715
+ ...entry,
1716
+ wrapped: await this.wrapToolExecutionResult({
1717
+ toolId,
1718
+ status: entry.result == null ? 'no_result' : 'completed',
1719
+ jobId: entry.jobId,
1720
+ result: entry.result,
1721
+ metadata: entry.metadata,
1722
+ meta: entry.meta,
1723
+ execution: toolExecutionMetadataForOutcome({
1724
+ kind: 'live',
1725
+ cacheKey: entry.request.cacheKey,
1726
+ receiptKey: entry.request.receiptKey,
1727
+ }),
1728
+ }),
1729
+ })),
1730
+ );
1731
+ const receiptInputs = wrappedEntries.flatMap((entry) => {
1732
+ const receiptKey = entry.request.receiptKey?.trim() || null;
1733
+ return receiptKey
1734
+ ? [
1735
+ {
1736
+ key: receiptKey,
1737
+ runId: this.currentRunId,
1738
+ output: serializeToolExecuteResult(entry.wrapped),
1739
+ },
1740
+ ]
1741
+ : [];
1730
1742
  });
1731
- this.cacheToolResult(toolId, cacheKey, wrapped);
1743
+ const completedByKey =
1744
+ receiptInputs.length > 0
1745
+ ? await this.completeRuntimeStepReceipts(receiptInputs)
1746
+ : new Map<string, RuntimeStepReceipt>();
1747
+ const results: unknown[] = [];
1748
+ for (const entry of wrappedEntries) {
1749
+ const receiptKey = entry.request.receiptKey?.trim() || null;
1750
+ results.push(
1751
+ await this.finalizeResolvedToolCall(
1752
+ toolId,
1753
+ entry.request,
1754
+ entry.wrapped,
1755
+ receiptKey ? completedByKey.get(receiptKey) : null,
1756
+ ),
1757
+ );
1758
+ }
1759
+ return results;
1760
+ }
1761
+
1762
+ private async finalizeResolvedToolCall(
1763
+ toolId: string,
1764
+ request: ToolCallRequest,
1765
+ wrapped: ToolExecuteResult,
1766
+ completed: RuntimeStepReceipt | null | undefined,
1767
+ ): Promise<unknown> {
1768
+ const cacheKey = request.cacheKey;
1769
+ const finalWrapped =
1770
+ (completed?.status === 'completed' || completed?.status === 'skipped') &&
1771
+ completed.runId !== this.currentRunId
1772
+ ? await this.wrapToolExecutionResult({
1773
+ toolId,
1774
+ status:
1775
+ completed.output === null || completed.output === undefined
1776
+ ? 'no_result'
1777
+ : 'completed',
1778
+ result: this.runtimeReceiptOutput(completed),
1779
+ execution: toolExecutionMetadataForOutcome(
1780
+ completed.runId === this.currentRunId
1781
+ ? {
1782
+ kind: 'live',
1783
+ cacheKey,
1784
+ receiptKey: request.receiptKey,
1785
+ }
1786
+ : {
1787
+ kind: 'cache',
1788
+ cacheKey,
1789
+ receiptKey: request.receiptKey ?? '',
1790
+ attachedToReceiptKey: request.receiptKey,
1791
+ },
1792
+ ),
1793
+ })
1794
+ : wrapped;
1795
+ this.cacheToolResult(toolId, cacheKey, finalWrapped);
1732
1796
 
1733
1797
  const resolver = this.toolCallResolvers.get(request.callId);
1734
1798
  if (resolver) {
1735
- resolver.resolve(wrapped);
1799
+ resolver.resolve(finalWrapped);
1736
1800
  this.toolCallResolvers.delete(request.callId);
1737
1801
  }
1738
1802
 
@@ -1748,17 +1812,39 @@ export class PlayContextImpl {
1748
1812
  error: null,
1749
1813
  dataPatch: {},
1750
1814
  });
1815
+ return finalWrapped;
1751
1816
  }
1752
1817
 
1753
- private rejectToolCall(
1818
+ private async rejectToolCall(
1754
1819
  toolId: string,
1755
1820
  request: ToolCallRequest,
1756
1821
  error: unknown,
1757
- ): void {
1822
+ options?: { persistReceiptFailure?: boolean },
1823
+ ): Promise<void> {
1758
1824
  const message = this.formatRuntimeError(error);
1825
+ let rejectionError = error;
1826
+ const receiptKey = request.receiptKey?.trim() || null;
1827
+ if (receiptKey && options?.persistReceiptFailure !== false) {
1828
+ try {
1829
+ await this.failRuntimeStepReceipts([
1830
+ {
1831
+ key: receiptKey,
1832
+ runId: this.currentRunId,
1833
+ error: message,
1834
+ },
1835
+ ]);
1836
+ } catch (receiptError) {
1837
+ rejectionError = new AggregateError(
1838
+ [error, receiptError],
1839
+ 'Tool call failed and durable receipt could not be marked failed',
1840
+ );
1841
+ }
1842
+ }
1759
1843
  const resolver = this.toolCallResolvers.get(request.callId);
1760
1844
  if (resolver) {
1761
- resolver.reject(error instanceof Error ? error : new Error(message));
1845
+ resolver.reject(
1846
+ rejectionError instanceof Error ? rejectionError : new Error(message),
1847
+ );
1762
1848
  this.toolCallResolvers.delete(request.callId);
1763
1849
  }
1764
1850
 
@@ -1875,22 +1961,6 @@ export class PlayContextImpl {
1875
1961
  options?: DatasetOptions<T>,
1876
1962
  ): Promise<PlayDataset<Record<string, unknown>>> {
1877
1963
  const definition = this.stepProgramToMapDefinition(program);
1878
- Object.defineProperty(definition, CELL_STALENESS_POLICIES, {
1879
- value: Object.fromEntries(
1880
- program.steps.map((step) => [
1881
- step.name,
1882
- {
1883
- ...(step.recompute === true ? { recompute: true } : {}),
1884
- ...(step.recomputeOnError === true
1885
- ? { recomputeOnError: true }
1886
- : {}),
1887
- ...(step.staleAfterSeconds === undefined
1888
- ? {}
1889
- : { staleAfterSeconds: step.staleAfterSeconds }),
1890
- },
1891
- ]),
1892
- ) satisfies AuthoredCellStalenessPolicyByField,
1893
- });
1894
1964
  return this.runMapDefinition(key, items, definition, options);
1895
1965
  }
1896
1966
 
@@ -1940,7 +2010,6 @@ export class PlayContextImpl {
1940
2010
  let rawItems: Record<string, unknown>[] = [];
1941
2011
  let itemsToProcess: Array<Record<string, unknown>> = [];
1942
2012
  let itemOriginalIndexes: number[] = [];
1943
- let completedItemsByKey: Map<string, Record<string, unknown>> | null = null;
1944
2013
  const datasetColumnNames = Object.keys(input);
1945
2014
  const stripFieldOutputs = (row: Record<string, unknown>) => {
1946
2015
  const stripped = cloneCsvAliasedRow(row);
@@ -2066,7 +2135,6 @@ export class PlayContextImpl {
2066
2135
  playId: this.#options.playId,
2067
2136
  runId: this.#options.runId,
2068
2137
  staticPipeline: this.#options.staticPipeline,
2069
- cellPolicies: this.cellPoliciesForMapDefinition(input),
2070
2138
  },
2071
2139
  );
2072
2140
  resolvedTableNamespace = normalizeTableNamespace(
@@ -2109,21 +2177,6 @@ export class PlayContextImpl {
2109
2177
  });
2110
2178
  itemsToProcess = pendingItems.map((item) => item.row);
2111
2179
  itemOriginalIndexes = pendingItems.map((item) => item.index);
2112
- if (mapStartResult.completedRows.length > 0) {
2113
- completedItemsByKey = new Map();
2114
- for (
2115
- let index = 0;
2116
- index < mapStartResult.completedRows.length;
2117
- index += 1
2118
- ) {
2119
- const row = mapStartResult.completedRows[index]!;
2120
- const rowKey = persistedRowIdentity(row, index);
2121
- if (rowKey) completedItemsByKey.set(rowKey, row);
2122
- }
2123
- this.log(
2124
- `Resuming: ${mapStartResult.completedRows.length} already completed, ${itemsToProcess.length} pending`,
2125
- );
2126
- }
2127
2180
  }
2128
2181
 
2129
2182
  const mapScope = this.createMapExecutionScope({
@@ -2131,8 +2184,7 @@ export class PlayContextImpl {
2131
2184
  artifactTableNamespace: resolvedTableNamespace,
2132
2185
  explicitKey: explicitKeyResolver,
2133
2186
  });
2134
- const completedRowKeys =
2135
- completedItemsByKey != null ? [...completedItemsByKey.keys()] : [];
2187
+ const completedRowKeys: string[] = [];
2136
2188
  const pendingRowKeys = itemsToProcess.map((item, index) =>
2137
2189
  rowIdentity(this.toOutputRow(item), itemOriginalIndexes[index] ?? index),
2138
2190
  );
@@ -2230,7 +2282,7 @@ export class PlayContextImpl {
2230
2282
  options?.description,
2231
2283
  {
2232
2284
  totalRows: totalInputCount,
2233
- completedRows: (completedItemsByKey?.size ?? 0) + duplicateReuseCount,
2285
+ completedRows: duplicateReuseCount,
2234
2286
  },
2235
2287
  {
2236
2288
  onRowError: options?.onRowError,
@@ -2274,7 +2326,6 @@ export class PlayContextImpl {
2274
2326
  this.toPublicOutputRow(row.data),
2275
2327
  );
2276
2328
  const results =
2277
- !completedItemsByKey &&
2278
2329
  mapResult.failedRows.length === 0 &&
2279
2330
  directCompletedResults.length === rawItems.length
2280
2331
  ? directCompletedResults
@@ -2284,15 +2335,11 @@ export class PlayContextImpl {
2284
2335
  return [];
2285
2336
  }
2286
2337
  return [
2287
- this.toPublicOutputRow(
2288
- resultsByKey.get(rowKey) ??
2289
- completedItemsByKey?.get(rowKey) ??
2290
- rawItem,
2291
- ),
2338
+ this.toPublicOutputRow(resultsByKey.get(rowKey) ?? rawItem),
2292
2339
  ];
2293
2340
  });
2294
2341
  const executedCount = rowsToExecute.length;
2295
- const reusedCount = Math.max(0, totalInputCount - executedCount);
2342
+ const reusedCount = duplicateReuseCount;
2296
2343
 
2297
2344
  // Persist executed rows to the tenant runtime sheet — the sheet is the
2298
2345
  // source of truth, not this in-memory results array. Chunked by rows AND
@@ -2403,9 +2450,6 @@ export class PlayContextImpl {
2403
2450
  },
2404
2451
  ): Promise<FieldMapRunResult> {
2405
2452
  const fieldEntries = Object.entries(definition);
2406
- const cellPolicies = this.cellPoliciesForMapDefinition(definition);
2407
- const authoredCellPolicies =
2408
- this.authoredCellPoliciesForMapDefinition(definition);
2409
2453
  const datasetColumnNames = fieldEntries.map(([fieldName]) => fieldName);
2410
2454
  const visibleFields = fieldEntries
2411
2455
  .map(([fieldName]) => fieldName)
@@ -2454,7 +2498,7 @@ export class PlayContextImpl {
2454
2498
 
2455
2499
  if (completedRows > 0 || pendingRows !== totalRows) {
2456
2500
  this.log(
2457
- `Starting map over ${totalRows} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace}; ${completedRows} already satisfied; ${pendingRows} pending)`,
2501
+ `Starting map over ${totalRows} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace}; ${completedRows} duplicate keys skipped; ${pendingRows} pending)`,
2458
2502
  );
2459
2503
  } else {
2460
2504
  this.log(
@@ -2587,8 +2631,6 @@ export class PlayContextImpl {
2587
2631
  visibleFields,
2588
2632
  normalizedTableNamespace,
2589
2633
  executionRowKey,
2590
- cellPolicies,
2591
- authoredCellPolicies,
2592
2634
  );
2593
2635
  // One batched frame update for the whole pure map. The per-row loop
2594
2636
  // this replaces ran AFTER every row had already computed, emitting 150k
@@ -2612,7 +2654,7 @@ export class PlayContextImpl {
2612
2654
  this.lastDatasetStep = datasetStep;
2613
2655
  this.activeDatasetStep = null;
2614
2656
  this.log(
2615
- `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} already satisfied)`,
2657
+ `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} duplicate keys skipped)`,
2616
2658
  );
2617
2659
  return {
2618
2660
  completedRows: results.map((row, index) =>
@@ -2674,32 +2716,6 @@ export class PlayContextImpl {
2674
2716
  try {
2675
2717
  for (const [fieldName, resolver] of fieldEntries) {
2676
2718
  activeFieldName = fieldName;
2677
- const reuseDecision = this.existingFieldReuseDecision(
2678
- baseRow,
2679
- fieldName,
2680
- cellPolicies,
2681
- authoredCellPolicies,
2682
- );
2683
- if (reuseDecision.reuse) {
2684
- const reusedValue = baseRow[fieldName];
2685
- computedFields[fieldName] = reusedValue;
2686
- this.rowStates.get(idx)?.results.set(fieldName, reusedValue);
2687
- this.emitScopedFieldMetaUpdate({
2688
- rowId: idx,
2689
- key: rowKey,
2690
- tableNamespace: normalizedTableNamespace,
2691
- fieldName,
2692
- status: 'cached',
2693
- reused: true,
2694
- ...(reuseDecision.completedAt !== undefined
2695
- ? { completedAt: reuseDecision.completedAt }
2696
- : {}),
2697
- ...(reuseDecision.stalenessMeta ?? {}),
2698
- dataPatch: {},
2699
- });
2700
- continue;
2701
- }
2702
-
2703
2719
  this.emitScopedFieldMetaUpdate({
2704
2720
  rowId: idx,
2705
2721
  key: rowKey,
@@ -2805,12 +2821,6 @@ export class PlayContextImpl {
2805
2821
  const cellValue = this.serializeCellValue(value);
2806
2822
  computedFields[fieldName] = cellValue;
2807
2823
  this.rowStates.get(idx)?.results.set(fieldName, cellValue);
2808
- const completedAt = Date.now();
2809
- const stalenessMeta = resolveCompletedCellStalenessMeta({
2810
- policy: authoredCellPolicies[fieldName],
2811
- value,
2812
- completedAt,
2813
- });
2814
2824
  this.emitScopedFieldMetaUpdate({
2815
2825
  rowId: idx,
2816
2826
  key: rowKey,
@@ -2818,8 +2828,7 @@ export class PlayContextImpl {
2818
2828
  fieldName,
2819
2829
  status: 'completed',
2820
2830
  stage: 'completed',
2821
- completedAt,
2822
- ...stalenessMeta,
2831
+ completedAt: Date.now(),
2823
2832
  dataPatch: shouldPersistMapCellField(fieldName)
2824
2833
  ? { [fieldName]: cellValue }
2825
2834
  : {},
@@ -2986,7 +2995,7 @@ export class PlayContextImpl {
2986
2995
  });
2987
2996
  }
2988
2997
  this.log(
2989
- `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} already satisfied)`,
2998
+ `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} duplicate keys skipped)`,
2990
2999
  );
2991
3000
  return {
2992
3001
  completedRows: [...completedRowsToPersist].sort(
@@ -3157,9 +3166,7 @@ export class PlayContextImpl {
3157
3166
 
3158
3167
  const source = Function.prototype.toString.call(resolver);
3159
3168
  return (
3160
- !source.includes('.tools.execute(') &&
3161
- !source.includes('.waterfall(') &&
3162
- !source.includes('.runPlay(')
3169
+ !source.includes('.tools.execute(') && !source.includes('.runPlay(')
3163
3170
  );
3164
3171
  });
3165
3172
  }
@@ -3170,8 +3177,6 @@ export class PlayContextImpl {
3170
3177
  visibleFields: string[],
3171
3178
  tableNamespace: string,
3172
3179
  rowIdentity: (row: Record<string, unknown>, index: number) => string,
3173
- cellPolicies?: CellStalenessPolicyByField,
3174
- authoredCellPolicies?: AuthoredCellStalenessPolicyByField,
3175
3180
  ): Promise<Array<Record<string, unknown>>> {
3176
3181
  const results: Array<Record<string, unknown>> = [];
3177
3182
  this.pureMapExecutionActive = true;
@@ -3195,25 +3200,6 @@ export class PlayContextImpl {
3195
3200
  try {
3196
3201
  for (const [fieldName, resolver] of fieldEntries) {
3197
3202
  activeFieldName = fieldName;
3198
- const reuseDecision = this.existingFieldReuseDecision(
3199
- baseRow,
3200
- fieldName,
3201
- cellPolicies,
3202
- authoredCellPolicies,
3203
- );
3204
- if (reuseDecision.reuse) {
3205
- computedFields[fieldName] = baseRow[fieldName];
3206
- rowCellMetaPatch[fieldName] = {
3207
- status: 'cached',
3208
- reused: true,
3209
- ...(reuseDecision.completedAt !== undefined
3210
- ? { completedAt: reuseDecision.completedAt }
3211
- : {}),
3212
- ...(reuseDecision.stalenessMeta ?? {}),
3213
- };
3214
- continue;
3215
- }
3216
-
3217
3203
  const value = await this.resolveMapFieldValue(
3218
3204
  resolver,
3219
3205
  item,
@@ -3224,20 +3210,13 @@ export class PlayContextImpl {
3224
3210
  this.previousCellForField(baseRow, fieldName),
3225
3211
  );
3226
3212
  computedFields[fieldName] = this.serializeCellValue(value);
3227
- const completedAt = Date.now();
3228
- const stalenessMeta = resolveCompletedCellStalenessMeta({
3229
- policy: authoredCellPolicies?.[fieldName],
3230
- value,
3231
- completedAt,
3232
- });
3233
3213
  if (shouldPersistMapCellField(fieldName)) {
3234
3214
  rowDataPatch[fieldName] = computedFields[fieldName];
3235
3215
  }
3236
3216
  rowCellMetaPatch[fieldName] = {
3237
3217
  status: 'completed',
3238
3218
  stage: 'completed',
3239
- completedAt,
3240
- ...stalenessMeta,
3219
+ completedAt: Date.now(),
3241
3220
  };
3242
3221
  }
3243
3222
 
@@ -3294,14 +3273,13 @@ export class PlayContextImpl {
3294
3273
  private initializeRowStates(items: readonly unknown[]): void {
3295
3274
  for (let idx = 0; idx < items.length; idx += 1) {
3296
3275
  this.rowStates.set(idx, {
3297
- waterfalls: new Map(),
3298
3276
  results: new Map(),
3299
3277
  });
3300
3278
  }
3301
3279
  }
3302
3280
 
3303
3281
  private async drainQueuedWork<T>(promises: Promise<T>[]): Promise<void> {
3304
- // Drain loop: each pass resolves queued waterfalls/tool calls.
3282
+ // Drain loop: each pass resolves queued tool calls.
3305
3283
  // When a batch resolves, rows resume and may queue MORE calls
3306
3284
  // (e.g. row does tools.execute('a') then tools.execute('b') sequentially).
3307
3285
  // We keep looping until nothing new is queued and all rows finish.
@@ -3311,7 +3289,7 @@ export class PlayContextImpl {
3311
3289
  // Promise.allSettled(promises) while there is no timer/socket/file handle can
3312
3290
  // let Node exit 0 before main() emits the result envelope. That presents as
3313
3291
  // "Local play runner produced no result" even though the real bug was a
3314
- // map row waiting for queued tool/waterfall work. Always race row settlement
3292
+ // map row waiting for queued tool work. Always race row settlement
3315
3293
  // against a small timer and re-check the queues instead of blocking forever
3316
3294
  // on promises that may be waiting for this drain loop to do the next batch.
3317
3295
  const raceSettled = () =>
@@ -3322,16 +3300,11 @@ export class PlayContextImpl {
3322
3300
 
3323
3301
  let pass = 0;
3324
3302
  while (true) {
3325
- const hasPendingWork =
3326
- this.waterfallQueue.size > 0 || this.toolCallQueue.length > 0;
3303
+ const hasPendingWork = this.toolCallQueue.length > 0;
3327
3304
 
3328
3305
  if (!hasPendingWork) {
3329
3306
  const status = await raceSettled();
3330
- if (
3331
- status === 'done' &&
3332
- this.waterfallQueue.size === 0 &&
3333
- this.toolCallQueue.length === 0
3334
- ) {
3307
+ if (status === 'done' && this.toolCallQueue.length === 0) {
3335
3308
  break;
3336
3309
  }
3337
3310
 
@@ -3346,13 +3319,9 @@ export class PlayContextImpl {
3346
3319
  pass++;
3347
3320
  this.log(` Batch pass ${pass}`);
3348
3321
  this.log(
3349
- ` Queue depth before drain: waterfalls=${this.waterfallQueue.size} tool_calls=${this.toolCallQueue.length}`,
3322
+ ` Queue depth before drain: tool_calls=${this.toolCallQueue.length}`,
3350
3323
  );
3351
3324
 
3352
- if (this.waterfallQueue.size > 0) {
3353
- await this.executeBatchedWaterfalls();
3354
- }
3355
-
3356
3325
  if (this.toolCallQueue.length > 0) {
3357
3326
  await this.executeBatchedToolCalls();
3358
3327
  }
@@ -3420,99 +3389,6 @@ export class PlayContextImpl {
3420
3389
  );
3421
3390
  }
3422
3391
 
3423
- async waterfall(
3424
- toolNameOrSpec: string | InlineWaterfallSpec,
3425
- input: Record<string, unknown>,
3426
- opts?: WaterfallOptions,
3427
- ): Promise<unknown | null> {
3428
- const store = rowContext.getStore();
3429
- const toolName =
3430
- typeof toolNameOrSpec === 'string' ? toolNameOrSpec : toolNameOrSpec.id;
3431
- const baseQueueKey =
3432
- typeof toolNameOrSpec === 'string'
3433
- ? toolNameOrSpec
3434
- : `inline:${toolNameOrSpec.id}`;
3435
- const inlineSpec = isInlineWaterfallSpec(toolNameOrSpec)
3436
- ? toolNameOrSpec
3437
- : undefined;
3438
-
3439
- if (this.pureMapExecutionActive && !store) {
3440
- throw new Error(
3441
- 'ctx.waterfall() cannot run inside the pure-JS fast path. Call it directly in the map definition so the batching runtime can stay enabled.',
3442
- );
3443
- }
3444
-
3445
- if (!store) {
3446
- return this.executeWaterfallDirect(toolNameOrSpec, input, opts);
3447
- }
3448
-
3449
- const rowId = store.rowId;
3450
- const fieldName = store.fieldName;
3451
- const queueKey = store.tableNamespace?.trim()
3452
- ? `${baseQueueKey}:${store.tableNamespace.trim()}`
3453
- : baseQueueKey;
3454
- const rowState = this.rowStates.get(rowId);
3455
- if (rowState && !rowState.waterfalls.has(toolName)) {
3456
- rowState.waterfalls.set(toolName, {
3457
- status: 'pending',
3458
- providerIndex: 0,
3459
- });
3460
- }
3461
-
3462
- // Check if this was already resolved in a previous attempt (checkpoint)
3463
- const resolved = this.checkpoint.resolvedWaterfalls[queueKey];
3464
- if (resolved && rowId in resolved) {
3465
- this.log(` Row ${rowId} ${toolName}: recovered from checkpoint`);
3466
- return resolved[rowId];
3467
- }
3468
-
3469
- return new Promise((resolve) => {
3470
- const resolverId = `${rowId}-${queueKey}`;
3471
- this.resolvers.set(resolverId, resolve);
3472
-
3473
- this.emitScopedFieldMetaUpdate({
3474
- rowId,
3475
- key: store?.rowKey ?? null,
3476
- tableNamespace: store?.tableNamespace ?? null,
3477
- fieldName,
3478
- status: 'running',
3479
- rowStatus: 'running',
3480
- stage: toolName,
3481
- provider: null,
3482
- error: null,
3483
- dataPatch: {},
3484
- });
3485
- if (inlineSpec) {
3486
- // Inline waterfalls have fully compiled child stages, so we can publish a
3487
- // real queued state for each step immediately instead of forcing the grid
3488
- // to guess from the row's broader status.
3489
- this.emitQueuedInlineWaterfallSteps(
3490
- rowId,
3491
- store?.rowKey ?? null,
3492
- store?.tableNamespace ?? null,
3493
- inlineSpec,
3494
- );
3495
- }
3496
-
3497
- if (!this.waterfallQueue.has(queueKey)) {
3498
- this.waterfallQueue.set(queueKey, []);
3499
- }
3500
- this.waterfallQueue.get(queueKey)!.push({
3501
- rowId,
3502
- fieldName,
3503
- tableNamespace: store?.tableNamespace,
3504
- rowKey: store?.rowKey ?? null,
3505
- key: queueKey,
3506
- toolName,
3507
- input,
3508
- providerIndex: 0,
3509
- opts,
3510
- description: normalizeStepDescription(opts?.description),
3511
- ...(inlineSpec ? { spec: inlineSpec } : {}),
3512
- });
3513
- });
3514
- }
3515
-
3516
3392
  private async executeTool(
3517
3393
  key: string,
3518
3394
  toolId: string,
@@ -3520,17 +3396,16 @@ export class PlayContextImpl {
3520
3396
  options?: ToolCallOptions,
3521
3397
  ): Promise<unknown> {
3522
3398
  const normalizedKey = this.normalizeContextKey(key, 'tool');
3399
+ const toolCachePolicy = this.effectiveToolCallCachePolicy(options);
3523
3400
  const toolRequestIdentity = deriveToolRequestIdentity({
3524
3401
  toolId,
3525
3402
  requestInput: input,
3526
3403
  });
3527
- const directCallId = [
3528
- 'direct',
3529
- normalizedKey,
3404
+ const durableCacheKey = await this.durableToolCallCacheKey({
3530
3405
  toolId,
3531
- toolRequestIdentity,
3532
- ].join(':');
3533
-
3406
+ requestInput: input,
3407
+ staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
3408
+ });
3534
3409
  const eventWaitHandler =
3535
3410
  (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
3536
3411
  const store = rowContext.getStore();
@@ -3557,13 +3432,8 @@ export class PlayContextImpl {
3557
3432
  }
3558
3433
 
3559
3434
  if (!store) {
3560
- const directRowId = -(this.directToolCallIndex + 1);
3561
- this.directToolCallIndex += 1;
3562
- const directCacheKey = this.buildToolResultCacheKey({
3563
- rowId: directRowId,
3564
- callId: directCallId,
3565
- });
3566
- const cached = options?.force
3435
+ const directCacheKey = durableCacheKey;
3436
+ const cached = toolCachePolicy.force
3567
3437
  ? null
3568
3438
  : this.getCachedToolResult(toolId, directCacheKey);
3569
3439
  if (cached?.done) {
@@ -3572,9 +3442,10 @@ export class PlayContextImpl {
3572
3442
  toolId,
3573
3443
  status: cached.result == null ? 'no_result' : 'completed',
3574
3444
  result: cached.result,
3575
- cached: true,
3576
- source: 'checkpoint',
3577
- cacheKey: directCacheKey,
3445
+ execution: toolExecutionMetadataForOutcome({
3446
+ kind: 'checkpoint',
3447
+ cacheKey: directCacheKey,
3448
+ }),
3578
3449
  });
3579
3450
  }
3580
3451
  this.log(`Calling tool: ${toolId}`);
@@ -3586,9 +3457,10 @@ export class PlayContextImpl {
3586
3457
  result: execution.result,
3587
3458
  metadata: execution.metadata,
3588
3459
  meta: execution.meta,
3589
- cached: false,
3590
- source: 'live',
3591
- cacheKey: directCacheKey,
3460
+ execution: toolExecutionMetadataForOutcome({
3461
+ kind: 'live',
3462
+ cacheKey: directCacheKey,
3463
+ }),
3592
3464
  });
3593
3465
  this.checkpoint.completedToolBatches[toolId] = {
3594
3466
  ...(this.checkpoint.completedToolBatches[toolId] ?? {}),
@@ -3617,13 +3489,8 @@ export class PlayContextImpl {
3617
3489
  );
3618
3490
  }
3619
3491
 
3620
- const toolResultCacheKey = this.buildToolResultCacheKey({
3621
- rowId,
3622
- tableNamespace: store.tableNamespace,
3623
- rowKey: store.rowKey,
3624
- callId,
3625
- });
3626
- const cached = options?.force
3492
+ const toolResultCacheKey = durableCacheKey;
3493
+ const cached = toolCachePolicy.force
3627
3494
  ? null
3628
3495
  : this.getCachedToolResult(toolId, toolResultCacheKey);
3629
3496
  if (cached?.done) {
@@ -3632,9 +3499,10 @@ export class PlayContextImpl {
3632
3499
  toolId,
3633
3500
  status: cached.result == null ? 'no_result' : 'completed',
3634
3501
  result: cached.result,
3635
- cached: true,
3636
- source: 'checkpoint',
3637
- cacheKey: toolResultCacheKey,
3502
+ execution: toolExecutionMetadataForOutcome({
3503
+ kind: 'checkpoint',
3504
+ cacheKey: toolResultCacheKey,
3505
+ }),
3638
3506
  });
3639
3507
  }
3640
3508
 
@@ -3660,6 +3528,9 @@ export class PlayContextImpl {
3660
3528
  });
3661
3529
  this.toolCallQueue.push({
3662
3530
  callId,
3531
+ cacheKey: toolResultCacheKey,
3532
+ receiptKey: durableCacheKey,
3533
+ force: toolCachePolicy.force,
3663
3534
  rowId,
3664
3535
  fieldName,
3665
3536
  toolId,
@@ -3680,15 +3551,28 @@ export class PlayContextImpl {
3680
3551
  normalizedKey,
3681
3552
  this.currentRunId,
3682
3553
  {
3554
+ receiptKey: durableCacheKey,
3683
3555
  semanticKey: toolRequestIdentity,
3684
- force: options?.force === true,
3685
- repairRunningReceiptForSameRun: true,
3686
- staleAfterSeconds: options?.staleAfterSeconds,
3556
+ force: toolCachePolicy.force,
3557
+ staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
3687
3558
  markSkipped: () => {
3688
3559
  this.log(
3689
3560
  `ctx.tools.execute(${toolId}): no-op due completed receipt ${toolRequestIdentity} (label: ${normalizedKey})`,
3690
3561
  );
3691
3562
  },
3563
+ onClaimedResult: (output, receiptKey) =>
3564
+ markToolExecuteResultExecutionOutcome(output, {
3565
+ kind: 'live',
3566
+ receiptKey,
3567
+ }),
3568
+ onRecovered: (output, _receipt, source) =>
3569
+ markToolExecuteResultExecutionOutcome(
3570
+ output,
3571
+ toolExecutionOutcomeForDurableReceipt({
3572
+ source,
3573
+ receiptKey: durableCacheKey,
3574
+ }),
3575
+ ),
3692
3576
  execute: executeTool,
3693
3577
  },
3694
3578
  );
@@ -3743,9 +3627,10 @@ export class PlayContextImpl {
3743
3627
  toolId,
3744
3628
  status: 'completed',
3745
3629
  result: existing.output,
3746
- cached: true,
3747
- source: 'checkpoint',
3748
- cacheKey: `integration_event:${preparedBoundary.boundaryId}`,
3630
+ execution: toolExecutionMetadataForOutcome({
3631
+ kind: 'checkpoint',
3632
+ cacheKey: `integration_event:${preparedBoundary.boundaryId}`,
3633
+ }),
3749
3634
  });
3750
3635
  }
3751
3636
 
@@ -3880,9 +3765,13 @@ export class PlayContextImpl {
3880
3765
  checkpoint: this.checkpoint,
3881
3766
  governance: childGovernance,
3882
3767
  getRuntimeStepReceipt: undefined,
3768
+ getRuntimeStepReceipts: undefined,
3883
3769
  claimRuntimeStepReceipt: undefined,
3770
+ claimRuntimeStepReceipts: undefined,
3884
3771
  completeRuntimeStepReceipt: undefined,
3772
+ completeRuntimeStepReceipts: undefined,
3885
3773
  failRuntimeStepReceipt: undefined,
3774
+ failRuntimeStepReceipts: undefined,
3886
3775
  skipRuntimeStepReceipt: undefined,
3887
3776
  });
3888
3777
  const childExecution = this.executeResolvedPlay(
@@ -3957,13 +3846,13 @@ export class PlayContextImpl {
3957
3846
  input,
3958
3847
  }),
3959
3848
  ),
3960
- repairRunningReceiptForSameRun: true,
3961
3849
  staleAfterSeconds: options?.staleAfterSeconds,
3962
3850
  markSkipped: () => {
3963
3851
  this.log(
3964
3852
  `ctx.runPlay(${normalizedKey}): no-op due completed receipt`,
3965
3853
  );
3966
3854
  },
3855
+ repairRunningReceiptForSameRunAfterWaitTimeout: true,
3967
3856
  execute: executePlayCall,
3968
3857
  },
3969
3858
  );
@@ -4273,16 +4162,23 @@ export class PlayContextImpl {
4273
4162
  return output;
4274
4163
  };
4275
4164
 
4276
- if (rowStore) {
4277
- return await executeStep();
4278
- }
4279
-
4280
4165
  return this.executeWithRuntimeReceipt<T>(
4281
4166
  'step',
4282
4167
  normalizedKey,
4283
4168
  this.currentRunId,
4284
4169
  {
4285
- semanticKey: options?.semanticKey,
4170
+ semanticKey: rowStore
4171
+ ? stableDigest(
4172
+ stableStringify({
4173
+ scope: 'row',
4174
+ tableNamespace: rowStore.tableNamespace ?? null,
4175
+ rowKey: rowStore.rowKey ?? null,
4176
+ rowId: rowStore.rowKey ? null : rowStore.rowId,
4177
+ fieldName: rowStore.fieldName ?? null,
4178
+ callIndex,
4179
+ }),
4180
+ )
4181
+ : options?.semanticKey,
4286
4182
  staleAfterSeconds: options?.staleAfterSeconds,
4287
4183
  markSkipped: (output) => {
4288
4184
  this.log(`ctx.step(${normalizedKey}): no-op due completed receipt`);
@@ -4324,809 +4220,11 @@ export class PlayContextImpl {
4324
4220
  }
4325
4221
 
4326
4222
  getStats(): Record<string, unknown> {
4327
- const stats: Record<
4328
- string,
4329
- { total: number; complete: number; failed: number }
4330
- > = {};
4331
- for (const [, rowState] of this.rowStates) {
4332
- for (const [toolName, wState] of rowState.waterfalls) {
4333
- if (!stats[toolName])
4334
- stats[toolName] = { total: 0, complete: 0, failed: 0 };
4335
- stats[toolName].total++;
4336
- if (wState.status === 'complete') stats[toolName].complete++;
4337
- if (wState.status === 'failed') stats[toolName].failed++;
4338
- }
4339
- }
4340
4223
  return {
4341
4224
  rowsProcessed: Math.max(this.rowStates.size, this.processedRowCount),
4342
- waterfalls: stats,
4343
4225
  };
4344
4226
  }
4345
4227
 
4346
- // ——— Batched waterfall execution (the core engine) ———
4347
-
4348
- private async executeBatchedWaterfalls(): Promise<void> {
4349
- const queuedWaterfalls = this.waterfallQueue;
4350
- this.waterfallQueue = new Map();
4351
- this.log(`Executing batched waterfalls for ${queuedWaterfalls.size} tools`);
4352
-
4353
- for (const [queueKey, requests] of queuedWaterfalls) {
4354
- const inlineSpec = requests[0]?.spec;
4355
- if (inlineSpec) {
4356
- await this.executeInlineWaterfall(queueKey, inlineSpec, requests);
4357
- continue;
4358
- }
4359
-
4360
- const toolName = requests[0]?.toolName ?? queueKey;
4361
- const providers = requests[0]?.opts?.providers ?? [
4362
- 'hunter',
4363
- 'leadmagic',
4364
- 'pdl',
4365
- 'dropcontact',
4366
- 'prospeo',
4367
- ];
4368
-
4369
- this.log(
4370
- `Processing waterfall ${toolName}: ${requests.length} rows, providers: ${providers.join(', ')}`,
4371
- );
4372
-
4373
- if (!this.checkpoint.resolvedWaterfalls[queueKey]) {
4374
- this.checkpoint.resolvedWaterfalls[queueKey] = {};
4375
- }
4376
-
4377
- await executeWaterfallProviders<WaterfallRequest, unknown>({
4378
- providers,
4379
- getPendingRequests: () =>
4380
- requests.filter((req) => {
4381
- const wState = this.rowStates
4382
- .get(req.rowId)
4383
- ?.waterfalls.get(toolName);
4384
- return wState?.status === 'pending';
4385
- }),
4386
- getCachedResults: (provider) => {
4387
- const batchKey = `${queueKey}:${provider}`;
4388
- const cached = this.checkpoint.completedBatches[batchKey];
4389
- if (!cached) return null;
4390
- this.log(` ${provider}: skipping (recovered from checkpoint)`);
4391
- return cached.map((entry) => ({
4392
- request: requests.find((request) => request.rowId === entry.rowId)!,
4393
- result: entry.result,
4394
- }));
4395
- },
4396
- storeCachedResults: (provider, results) => {
4397
- const batchKey = `${queueKey}:${provider}`;
4398
- this.checkpoint.completedBatches[batchKey] = results.map((entry) => ({
4399
- rowId: entry.request.rowId,
4400
- result: entry.result,
4401
- }));
4402
- },
4403
- executeProviderRequests: async (provider, pending) => {
4404
- const providerToolId = resolveWaterfallToolId(provider, toolName);
4405
- const strategy =
4406
- this.#options.getBatchOperationStrategy?.(providerToolId) ?? null;
4407
- this.log(` ${provider}: ${pending.length} pending rows`);
4408
- this.governor.chargeBudget('retry', pending.length);
4409
-
4410
- if (strategy) {
4411
- const compiledBatches = compileRequestsWithStrategy({
4412
- requests: pending,
4413
- strategy,
4414
- getPayload: (request: WaterfallRequest) => request.input,
4415
- });
4416
- const flattenedResults: Array<{
4417
- request: WaterfallRequest;
4418
- result: unknown | null;
4419
- }> = [];
4420
-
4421
- await executeChunkedRequests({
4422
- requests: compiledBatches,
4423
- batchSize:
4424
- compiledBatches.length > 0
4425
- ? await this.governor.suggestedParallelism(
4426
- compiledBatches[0]!.batchOperation,
4427
- 4,
4428
- )
4429
- : 4,
4430
- execute: async (batch) =>
4431
- await this.callToolAPI(
4432
- batch.batchOperation,
4433
- batch.batchPayload,
4434
- ),
4435
- onRequestError: (batch, error) => {
4436
- this.log(
4437
- ` ${provider}: batch of ${batch.memberRequests.length} request(s) failed: ` +
4438
- `${formatChunkExecutionError(error)} (rows recorded as misses; the waterfall continues)`,
4439
- );
4440
- },
4441
- onChunkComplete: async (chunkResults) => {
4442
- for (const entry of chunkResults) {
4443
- const splitResults =
4444
- entry.result != null
4445
- ? entry.request.splitResults(entry.result)
4446
- : entry.request.memberRequests.map(() => null);
4447
-
4448
- for (
4449
- let index = 0;
4450
- index < entry.request.memberRequests.length;
4451
- index += 1
4452
- ) {
4453
- flattenedResults.push({
4454
- request: entry.request.memberRequests[index]!,
4455
- result: splitResults[index] ?? null,
4456
- });
4457
- }
4458
- }
4459
- this.#options.onBatchComplete?.(this.checkpoint);
4460
- },
4461
- });
4462
-
4463
- return flattenedResults;
4464
- }
4465
-
4466
- const chunkResults: Array<{
4467
- request: WaterfallRequest;
4468
- result: unknown | null;
4469
- }> = [];
4470
- await executeChunkedRequests<WaterfallRequest, unknown>({
4471
- requests: pending,
4472
- batchSize: await this.governor.suggestedParallelism(
4473
- providerToolId,
4474
- 50,
4475
- ),
4476
- execute: async (request) =>
4477
- await this.callToolAPI(providerToolId, request.input),
4478
- onRequestError: (request, error) => {
4479
- this.log(
4480
- ` ${provider}: row ${request.rowId} failed: ` +
4481
- `${formatChunkExecutionError(error)} (recorded as a miss; the waterfall continues)`,
4482
- );
4483
- },
4484
- onChunkComplete: async (results) => {
4485
- chunkResults.push(...results);
4486
- this.#options.onBatchComplete?.(this.checkpoint);
4487
- },
4488
- });
4489
- return chunkResults;
4490
- },
4491
- onHit: (provider, request, result) => {
4492
- this.resolveWaterfall(
4493
- queueKey,
4494
- toolName,
4495
- request.rowId,
4496
- result,
4497
- provider,
4498
- request.rowKey ?? null,
4499
- request.tableNamespace ?? null,
4500
- request.fieldName,
4501
- );
4502
- },
4503
- onMiss: (_provider, request) => {
4504
- const wState = this.rowStates
4505
- .get(request.rowId)
4506
- ?.waterfalls.get(toolName);
4507
- if (wState) wState.providerIndex++;
4508
- this.log(` Row ${request.rowId}: miss`);
4509
- },
4510
- onProviderComplete: () => {
4511
- this.#options.onBatchComplete?.(this.checkpoint);
4512
- },
4513
- });
4514
-
4515
- const stepResults: PlayStepRowResult[] = requests.map((req) => {
4516
- const wState = this.rowStates.get(req.rowId)?.waterfalls.get(toolName);
4517
- const success = wState?.status === 'complete';
4518
- return {
4519
- rowId: req.rowId,
4520
- status: success ? 'completed' : 'missed',
4521
- success,
4522
- value: wState?.result,
4523
- error: success ? null : (wState?.error ?? null),
4524
- };
4525
- });
4526
- const waterfallStep = {
4527
- type: 'waterfall' as const,
4528
- tool: toolName,
4529
- providers,
4530
- results: stepResults,
4531
- description: normalizeStepDescription(requests[0]?.description),
4532
- };
4533
- if (this.activeDatasetStep) {
4534
- this.activeDatasetStep.substeps.push(waterfallStep);
4535
- } else {
4536
- this.steps.push(waterfallStep);
4537
- }
4538
-
4539
- for (const req of requests) {
4540
- const wState = this.rowStates.get(req.rowId)?.waterfalls.get(toolName);
4541
- if (wState?.status === 'pending') {
4542
- wState.status = 'failed';
4543
- wState.error = 'All providers exhausted';
4544
- this.checkpoint.resolvedWaterfalls[queueKey]![req.rowId] = null;
4545
-
4546
- const resolver = this.resolvers.get(`${req.rowId}-${queueKey}`);
4547
- if (resolver) {
4548
- resolver(null);
4549
- this.resolvers.delete(`${req.rowId}-${queueKey}`);
4550
- }
4551
- this.emitScopedFieldMetaUpdate({
4552
- rowId: req.rowId,
4553
- key: req.rowKey ?? null,
4554
- tableNamespace: req.tableNamespace ?? null,
4555
- fieldName: req.fieldName,
4556
- status: 'failed',
4557
- rowStatus: 'running',
4558
- stage: toolName,
4559
- provider: null,
4560
- error: 'All providers exhausted',
4561
- dataPatch: {},
4562
- });
4563
- this.log(` Row ${req.rowId}: all providers exhausted`);
4564
- }
4565
- }
4566
- }
4567
- }
4568
-
4569
- private async executeInlineWaterfall(
4570
- queueKey: string,
4571
- spec: InlineWaterfallSpec,
4572
- requests: WaterfallRequest[],
4573
- ): Promise<void> {
4574
- if (!this.checkpoint.resolvedWaterfalls[queueKey]) {
4575
- this.checkpoint.resolvedWaterfalls[queueKey] = {};
4576
- }
4577
-
4578
- const pendingRows = new Set<number>(
4579
- requests
4580
- .filter(
4581
- (req) =>
4582
- this.rowStates.get(req.rowId)?.waterfalls.get(spec.id)?.status ===
4583
- 'pending',
4584
- )
4585
- .map((req) => req.rowId),
4586
- );
4587
- const resultsByRow = new Map<number, unknown[]>();
4588
- const stepResults: Array<{
4589
- id: string;
4590
- kind?: 'tool' | 'code';
4591
- toolId?: string;
4592
- results: PlayStepRowResult[];
4593
- }> = [];
4594
- const stepColumnNames = spec.steps.map((s) =>
4595
- sqlSafePlayColumnName(`${spec.id}.${s.id}`),
4596
- );
4597
- const resolvedInChunkRowIds = new Set<number>();
4598
- let stepIdx = 0;
4599
-
4600
- for (const step of spec.steps) {
4601
- const stepColumnName = sqlSafePlayColumnName(`${spec.id}.${step.id}`);
4602
- const stepProvider = isInlineWaterfallToolStep(step)
4603
- ? step.toolId
4604
- : 'code';
4605
- if (pendingRows.size === 0) {
4606
- const skippedResults: PlayStepRowResult[] = requests.map((request) => {
4607
- this.emitCellUpdate({
4608
- rowId: request.rowId,
4609
- key: request.rowKey ?? null,
4610
- tableNamespace: request.tableNamespace ?? null,
4611
- columnName: stepColumnName,
4612
- status: 'skipped',
4613
- stage: step.id,
4614
- provider: stepProvider,
4615
- value: null,
4616
- });
4617
- return {
4618
- rowId: request.rowId,
4619
- status: 'skipped',
4620
- success: false,
4621
- value: null,
4622
- error: null,
4623
- };
4624
- });
4625
- stepResults.push({
4626
- id: step.id,
4627
- kind: isInlineWaterfallCodeStep(step) ? 'code' : 'tool',
4628
- toolId: stepProvider,
4629
- results: skippedResults,
4630
- });
4631
- stepIdx++;
4632
- continue;
4633
- }
4634
- this.governor.chargeBudget('waterfallStep', pendingRows.size);
4635
- const pendingRowIds = new Set<number>(pendingRows);
4636
- const stepRequests = requests.filter((req) =>
4637
- pendingRowIds.has(req.rowId),
4638
- );
4639
- const perRowResultsByRowId = new Map<number, PlayStepRowResult>();
4640
- if (isInlineWaterfallCodeStep(step)) {
4641
- this.log(
4642
- ` Inline waterfall ${spec.id} -> ${step.id}: ${stepRequests.length} pending rows (code)`,
4643
- );
4644
- await executeChunkedRequests<WaterfallRequest, unknown>({
4645
- requests: stepRequests,
4646
- batchSize: await this.governor.suggestedParallelism(
4647
- `code:${spec.id}:${step.id}`,
4648
- 20,
4649
- ),
4650
- onRequestError: (request, error) => {
4651
- this.log(
4652
- ` Inline waterfall ${spec.id} -> ${step.id}: row ${request.rowId} failed: ` +
4653
- `${formatChunkExecutionError(error)} (recorded as a miss; the waterfall continues)`,
4654
- );
4655
- },
4656
- execute: async (request) => {
4657
- const codeStepCtx = {
4658
- tools: {
4659
- execute: async (
4660
- requestOrKey:
4661
- | { tool: string; input: Record<string, unknown> }
4662
- | string,
4663
- toolId?: string,
4664
- payload?: Record<string, unknown>,
4665
- ) => {
4666
- if (typeof requestOrKey === 'object') {
4667
- return await this.callToolAPI(
4668
- requestOrKey.tool,
4669
- requestOrKey.input,
4670
- );
4671
- }
4672
- if (!toolId || !payload) {
4673
- throw new Error(
4674
- 'inline waterfall ctx.tools.execute requires a tool and input.',
4675
- );
4676
- }
4677
- return await this.callToolAPI(toolId, payload);
4678
- },
4679
- },
4680
- };
4681
- return await step.run(request.input, codeStepCtx);
4682
- },
4683
- onChunkComplete: async (chunkResults) => {
4684
- for (const entry of chunkResults) {
4685
- const matchedValue = extractInlineWaterfallCodeStepValue(
4686
- spec.output,
4687
- entry.result,
4688
- );
4689
- const bucket = resultsByRow.get(entry.request.rowId) ?? [];
4690
- const nextBucket = Array.isArray(matchedValue)
4691
- ? [...bucket, ...matchedValue]
4692
- : matchedValue != null
4693
- ? [...bucket, matchedValue]
4694
- : bucket;
4695
- resultsByRow.set(entry.request.rowId, nextBucket);
4696
- const satisfied = nextBucket.length >= spec.minResults;
4697
- perRowResultsByRowId.set(entry.request.rowId, {
4698
- rowId: entry.request.rowId,
4699
- status: matchedValue != null ? 'completed' : 'missed',
4700
- success: matchedValue != null,
4701
- value: matchedValue,
4702
- provider: matchedValue != null ? 'code' : undefined,
4703
- error: null,
4704
- });
4705
- if (satisfied) {
4706
- const finalValue =
4707
- spec.minResults === 1 ? (nextBucket[0] ?? null) : nextBucket;
4708
- this.resolveWaterfall(
4709
- queueKey,
4710
- spec.id,
4711
- entry.request.rowId,
4712
- finalValue,
4713
- 'code',
4714
- entry.request.rowKey ?? null,
4715
- entry.request.tableNamespace ?? null,
4716
- entry.request.fieldName,
4717
- );
4718
- pendingRows.delete(entry.request.rowId);
4719
- this.emitCellUpdate({
4720
- rowId: entry.request.rowId,
4721
- key: entry.request.rowKey ?? null,
4722
- tableNamespace: entry.request.tableNamespace ?? null,
4723
- columnName: stepColumnName,
4724
- status: 'completed',
4725
- stage: step.id,
4726
- provider: 'code',
4727
- value: matchedValue,
4728
- });
4729
- for (let ri = stepIdx + 1; ri < spec.steps.length; ri++) {
4730
- const skippedStep = spec.steps[ri]!;
4731
- this.emitCellUpdate({
4732
- rowId: entry.request.rowId,
4733
- key: entry.request.rowKey ?? null,
4734
- tableNamespace: entry.request.tableNamespace ?? null,
4735
- columnName: stepColumnNames[ri]!,
4736
- status: 'skipped',
4737
- stage: skippedStep.id,
4738
- provider: isInlineWaterfallToolStep(skippedStep)
4739
- ? skippedStep.toolId
4740
- : 'code',
4741
- value: null,
4742
- });
4743
- }
4744
- resolvedInChunkRowIds.add(entry.request.rowId);
4745
- }
4746
- }
4747
- this.#options.onBatchComplete?.(this.checkpoint);
4748
- },
4749
- });
4750
- } else {
4751
- const strategy =
4752
- this.#options.getBatchOperationStrategy?.(step.toolId) ?? null;
4753
- this.log(
4754
- ` Inline waterfall ${spec.id} -> ${step.id}: ${stepRequests.length} pending rows`,
4755
- );
4756
- for (const request of stepRequests) {
4757
- this.emitCellUpdate({
4758
- rowId: request.rowId,
4759
- key: request.rowKey ?? null,
4760
- tableNamespace: request.tableNamespace ?? null,
4761
- columnName: stepColumnName,
4762
- status: 'running',
4763
- stage: step.id,
4764
- provider: step.toolId,
4765
- value: null,
4766
- });
4767
- }
4768
-
4769
- if (strategy) {
4770
- const compiledBatches = compileRequestsWithStrategy({
4771
- requests: stepRequests,
4772
- strategy,
4773
- getPayload: (request: WaterfallRequest) =>
4774
- step.mapInput(request.input),
4775
- });
4776
- this.log(
4777
- ` ${step.toolId}: compiled ${compiledBatches.length} batch request(s)` +
4778
- ` (${this.summarizeBatchSizes(
4779
- compiledBatches.map((batch) => batch.memberRequests.length),
4780
- )})`,
4781
- );
4782
- await executeChunkedRequests({
4783
- requests: compiledBatches,
4784
- batchSize:
4785
- compiledBatches.length > 0
4786
- ? await this.governor.suggestedParallelism(
4787
- compiledBatches[0]!.batchOperation,
4788
- 4,
4789
- )
4790
- : 4,
4791
- execute: async (batch) =>
4792
- await this.callToolAPI(batch.batchOperation, batch.batchPayload),
4793
- onRequestError: (batch, error) => {
4794
- this.log(
4795
- ` ${step.toolId}: batch of ${batch.memberRequests.length} request(s) failed: ` +
4796
- `${formatChunkExecutionError(error)} (rows recorded as misses; the waterfall continues)`,
4797
- );
4798
- },
4799
- onChunkComplete: async (chunkResults) => {
4800
- for (const entry of chunkResults) {
4801
- const splitResults =
4802
- entry.result != null
4803
- ? entry.request.splitResults(entry.result)
4804
- : entry.request.memberRequests.map(() => null);
4805
- for (
4806
- let index = 0;
4807
- index < entry.request.memberRequests.length;
4808
- index += 1
4809
- ) {
4810
- const request = entry.request.memberRequests[index]!;
4811
- const rawResult = splitResults[index] ?? null;
4812
- const matchedValue = await extractWaterfallOutputValue(
4813
- step.toolId,
4814
- spec.output,
4815
- rawResult,
4816
- this.#options.getToolTargetGetters,
4817
- );
4818
- const bucket = resultsByRow.get(request.rowId) ?? [];
4819
- const nextBucket = Array.isArray(matchedValue)
4820
- ? [...bucket, ...matchedValue]
4821
- : matchedValue != null
4822
- ? [...bucket, matchedValue]
4823
- : bucket;
4824
- resultsByRow.set(request.rowId, nextBucket);
4825
- const satisfied = nextBucket.length >= spec.minResults;
4826
- perRowResultsByRowId.set(request.rowId, {
4827
- rowId: request.rowId,
4828
- status: matchedValue != null ? 'completed' : 'missed',
4829
- success: matchedValue != null,
4830
- value: matchedValue,
4831
- provider: matchedValue != null ? step.toolId : undefined,
4832
- error: null,
4833
- });
4834
- if (satisfied) {
4835
- const finalValue =
4836
- spec.minResults === 1
4837
- ? (nextBucket[0] ?? null)
4838
- : nextBucket;
4839
- this.resolveWaterfall(
4840
- queueKey,
4841
- spec.id,
4842
- request.rowId,
4843
- finalValue,
4844
- step.toolId,
4845
- request.rowKey ?? null,
4846
- request.tableNamespace ?? null,
4847
- request.fieldName,
4848
- );
4849
- pendingRows.delete(request.rowId);
4850
- this.emitCellUpdate({
4851
- rowId: request.rowId,
4852
- key: request.rowKey ?? null,
4853
- tableNamespace: request.tableNamespace ?? null,
4854
- columnName: stepColumnNames[stepIdx]!,
4855
- status: 'completed',
4856
- stage: step.id,
4857
- provider: step.toolId,
4858
- value: matchedValue,
4859
- });
4860
- for (let ri = stepIdx + 1; ri < spec.steps.length; ri++) {
4861
- const skippedStep = spec.steps[ri]!;
4862
- this.emitCellUpdate({
4863
- rowId: request.rowId,
4864
- key: request.rowKey ?? null,
4865
- tableNamespace: request.tableNamespace ?? null,
4866
- columnName: stepColumnNames[ri]!,
4867
- status: 'skipped',
4868
- stage: skippedStep.id,
4869
- provider: isInlineWaterfallToolStep(skippedStep)
4870
- ? skippedStep.toolId
4871
- : 'code',
4872
- value: null,
4873
- });
4874
- }
4875
- resolvedInChunkRowIds.add(request.rowId);
4876
- }
4877
- }
4878
- }
4879
- this.#options.onBatchComplete?.(this.checkpoint);
4880
- },
4881
- });
4882
- } else {
4883
- this.log(
4884
- ` ${step.toolId}: executing ${stepRequests.length} unbatched request(s)`,
4885
- );
4886
- await executeChunkedRequests<WaterfallRequest, unknown>({
4887
- requests: stepRequests,
4888
- batchSize: await this.governor.suggestedParallelism(
4889
- step.toolId,
4890
- 50,
4891
- ),
4892
- execute: async (request) =>
4893
- await this.callToolAPI(step.toolId, step.mapInput(request.input)),
4894
- onRequestError: (request, error) => {
4895
- this.log(
4896
- ` ${step.toolId}: row ${request.rowId} failed: ` +
4897
- `${formatChunkExecutionError(error)} (recorded as a miss; the waterfall continues)`,
4898
- );
4899
- },
4900
- onChunkComplete: async (chunkResults) => {
4901
- for (const entry of chunkResults) {
4902
- const matchedValue = await extractWaterfallOutputValue(
4903
- step.toolId,
4904
- spec.output,
4905
- entry.result,
4906
- this.#options.getToolTargetGetters,
4907
- );
4908
- const bucket = resultsByRow.get(entry.request.rowId) ?? [];
4909
- const nextBucket = Array.isArray(matchedValue)
4910
- ? [...bucket, ...matchedValue]
4911
- : matchedValue != null
4912
- ? [...bucket, matchedValue]
4913
- : bucket;
4914
- resultsByRow.set(entry.request.rowId, nextBucket);
4915
- const satisfied = nextBucket.length >= spec.minResults;
4916
- perRowResultsByRowId.set(entry.request.rowId, {
4917
- rowId: entry.request.rowId,
4918
- status: matchedValue != null ? 'completed' : 'missed',
4919
- success: matchedValue != null,
4920
- value: matchedValue,
4921
- provider: matchedValue != null ? step.toolId : undefined,
4922
- error: null,
4923
- });
4924
- if (satisfied) {
4925
- const finalValue =
4926
- spec.minResults === 1
4927
- ? (nextBucket[0] ?? null)
4928
- : nextBucket;
4929
- this.resolveWaterfall(
4930
- queueKey,
4931
- spec.id,
4932
- entry.request.rowId,
4933
- finalValue,
4934
- step.toolId,
4935
- entry.request.rowKey ?? null,
4936
- entry.request.tableNamespace ?? null,
4937
- entry.request.fieldName,
4938
- );
4939
- pendingRows.delete(entry.request.rowId);
4940
- this.emitCellUpdate({
4941
- rowId: entry.request.rowId,
4942
- key: entry.request.rowKey ?? null,
4943
- tableNamespace: entry.request.tableNamespace ?? null,
4944
- columnName: stepColumnNames[stepIdx]!,
4945
- status: 'completed',
4946
- stage: step.id,
4947
- provider: step.toolId,
4948
- value: matchedValue,
4949
- });
4950
- for (let ri = stepIdx + 1; ri < spec.steps.length; ri++) {
4951
- const skippedStep = spec.steps[ri]!;
4952
- this.emitCellUpdate({
4953
- rowId: entry.request.rowId,
4954
- key: entry.request.rowKey ?? null,
4955
- tableNamespace: entry.request.tableNamespace ?? null,
4956
- columnName: stepColumnNames[ri]!,
4957
- status: 'skipped',
4958
- stage: skippedStep.id,
4959
- provider: isInlineWaterfallToolStep(skippedStep)
4960
- ? skippedStep.toolId
4961
- : 'code',
4962
- value: null,
4963
- });
4964
- }
4965
- resolvedInChunkRowIds.add(entry.request.rowId);
4966
- }
4967
- }
4968
- this.#options.onBatchComplete?.(this.checkpoint);
4969
- },
4970
- });
4971
- }
4972
- }
4973
-
4974
- const perRowResults: PlayStepRowResult[] = requests.map((request) => {
4975
- if (resolvedInChunkRowIds.has(request.rowId)) {
4976
- const existing = perRowResultsByRowId.get(request.rowId);
4977
- return (
4978
- existing ?? {
4979
- rowId: request.rowId,
4980
- status: 'completed' as const,
4981
- success: true,
4982
- value: null,
4983
- error: null,
4984
- }
4985
- );
4986
- }
4987
- const existing = perRowResultsByRowId.get(request.rowId);
4988
- if (existing) {
4989
- this.emitCellUpdate({
4990
- rowId: request.rowId,
4991
- key: request.rowKey ?? null,
4992
- tableNamespace: request.tableNamespace ?? null,
4993
- columnName: stepColumnName,
4994
- status: existing.status,
4995
- stage: step.id,
4996
- provider: existing.provider ?? stepProvider,
4997
- error: existing.error ?? null,
4998
- value: existing.value ?? null,
4999
- });
5000
- return existing;
5001
- }
5002
- if (!pendingRowIds.has(request.rowId)) {
5003
- this.emitCellUpdate({
5004
- rowId: request.rowId,
5005
- key: request.rowKey ?? null,
5006
- tableNamespace: request.tableNamespace ?? null,
5007
- columnName: stepColumnName,
5008
- status: 'skipped',
5009
- stage: step.id,
5010
- provider: stepProvider,
5011
- value: null,
5012
- });
5013
- return {
5014
- rowId: request.rowId,
5015
- status: 'skipped',
5016
- success: false,
5017
- value: null,
5018
- error: null,
5019
- };
5020
- }
5021
- this.emitCellUpdate({
5022
- rowId: request.rowId,
5023
- key: request.rowKey ?? null,
5024
- tableNamespace: request.tableNamespace ?? null,
5025
- columnName: stepColumnName,
5026
- status: 'missed',
5027
- stage: step.id,
5028
- provider: stepProvider,
5029
- value: null,
5030
- });
5031
- return {
5032
- rowId: request.rowId,
5033
- status: 'missed',
5034
- success: false,
5035
- value: null,
5036
- error: null,
5037
- };
5038
- });
5039
-
5040
- stepResults.push({
5041
- id: step.id,
5042
- kind: isInlineWaterfallCodeStep(step) ? 'code' : 'tool',
5043
- toolId: stepProvider,
5044
- // Persist only a bounded preview in the in-memory execution trace.
5045
- // The authoritative per-row state already lives in Neon/live progress.
5046
- results: compactRowResultsPreview(perRowResults),
5047
- });
5048
- stepIdx++;
5049
- }
5050
-
5051
- for (const req of requests) {
5052
- const wState = this.rowStates.get(req.rowId)?.waterfalls.get(spec.id);
5053
- if (wState?.status === 'pending') {
5054
- wState.status = 'failed';
5055
- wState.error = 'All waterfall steps exhausted';
5056
- this.checkpoint.resolvedWaterfalls[queueKey]![req.rowId] = null;
5057
- const resolver = this.resolvers.get(`${req.rowId}-${queueKey}`);
5058
- if (resolver) {
5059
- resolver(null);
5060
- this.resolvers.delete(`${req.rowId}-${queueKey}`);
5061
- }
5062
- }
5063
- }
5064
-
5065
- const groupResults: PlayStepRowResult[] = requests.map((req) => {
5066
- const wState = this.rowStates.get(req.rowId)?.waterfalls.get(spec.id);
5067
- const success = wState?.status === 'complete';
5068
- return {
5069
- rowId: req.rowId,
5070
- status: success ? 'completed' : 'missed',
5071
- success,
5072
- value: wState?.result,
5073
- error: success ? null : (wState?.error ?? null),
5074
- };
5075
- });
5076
-
5077
- const waterfallStep = {
5078
- type: 'waterfall' as const,
5079
- id: spec.id,
5080
- output: spec.output,
5081
- minResults: spec.minResults,
5082
- steps: stepResults,
5083
- results: compactRowResultsPreview(groupResults),
5084
- description: normalizeStepDescription(requests[0]?.description),
5085
- };
5086
- if (this.activeDatasetStep) {
5087
- this.activeDatasetStep.substeps.push(waterfallStep);
5088
- } else {
5089
- this.steps.push(waterfallStep);
5090
- }
5091
- }
5092
-
5093
- private resolveWaterfall(
5094
- queueKey: string,
5095
- toolName: string,
5096
- rowId: number,
5097
- result: unknown,
5098
- provider: string,
5099
- rowKey: string | null,
5100
- tableNamespace: string | null,
5101
- fieldName?: string,
5102
- ): void {
5103
- const wState = this.rowStates.get(rowId)?.waterfalls.get(toolName);
5104
- if (!wState || wState.status !== 'pending') return;
5105
-
5106
- wState.status = 'complete';
5107
- wState.result = result;
5108
- this.checkpoint.resolvedWaterfalls[queueKey]![rowId] = result;
5109
-
5110
- const resolver = this.resolvers.get(`${rowId}-${queueKey}`);
5111
- if (resolver) {
5112
- resolver(result);
5113
- this.resolvers.delete(`${rowId}-${queueKey}`);
5114
- }
5115
- this.emitScopedFieldMetaUpdate({
5116
- rowId,
5117
- key: rowKey,
5118
- tableNamespace,
5119
- fieldName,
5120
- status: 'running',
5121
- rowStatus: 'running',
5122
- stage: toolName,
5123
- provider,
5124
- error: null,
5125
- dataPatch: {},
5126
- });
5127
- this.logWaterfallMatch({ queueKey, rowId, provider });
5128
- }
5129
-
5130
4228
  // ——— Batched tool call execution ———
5131
4229
 
5132
4230
  private async executeBatchedToolCalls(): Promise<void> {
@@ -5147,15 +4245,7 @@ export class PlayContextImpl {
5147
4245
  const recordToolStep = (stepRequests: ToolCallRequest[]): void => {
5148
4246
  if (stepRequests.length === 0) return;
5149
4247
  const stepResults: PlayStepRowResult[] = stepRequests.map((req) => {
5150
- const cachedResult = this.getCachedToolResult(
5151
- toolId,
5152
- this.buildToolResultCacheKey({
5153
- rowId: req.rowId,
5154
- tableNamespace: req.tableNamespace,
5155
- rowKey: req.rowKey ?? undefined,
5156
- callId: req.callId,
5157
- }),
5158
- );
4248
+ const cachedResult = this.getCachedToolResult(toolId, req.cacheKey);
5159
4249
  return {
5160
4250
  rowId: req.rowId,
5161
4251
  status: cachedResult?.result != null ? 'completed' : 'failed',
@@ -5181,15 +4271,7 @@ export class PlayContextImpl {
5181
4271
  const pendingRequests: ToolCallRequest[] = [];
5182
4272
  const recoveredRequests: ToolCallRequest[] = [];
5183
4273
  for (const req of requests) {
5184
- const cached = this.getCachedToolResult(
5185
- toolId,
5186
- this.buildToolResultCacheKey({
5187
- rowId: req.rowId,
5188
- tableNamespace: req.tableNamespace,
5189
- rowKey: req.rowKey ?? undefined,
5190
- callId: req.callId,
5191
- }),
5192
- );
4274
+ const cached = this.getCachedToolResult(toolId, req.cacheKey);
5193
4275
  if (cached?.done) {
5194
4276
  this.log(` Row ${req.rowId} ${toolId}: recovered from checkpoint`);
5195
4277
  const resolver = this.toolCallResolvers.get(req.callId);
@@ -5204,6 +4286,240 @@ export class PlayContextImpl {
5204
4286
  }
5205
4287
  recordToolStep(recoveredRequests);
5206
4288
 
4289
+ const liveFollowersByOwnerCallId = new Map<string, ToolCallRequest[]>();
4290
+ const resolveLiveFollowers = (
4291
+ owner: ToolCallRequest,
4292
+ result: unknown,
4293
+ ): void => {
4294
+ const followers = liveFollowersByOwnerCallId.get(owner.callId);
4295
+ if (!followers || followers.length === 0) return;
4296
+ liveFollowersByOwnerCallId.delete(owner.callId);
4297
+ for (const follower of followers) {
4298
+ const followerReceiptKey = follower.receiptKey?.trim() || null;
4299
+ const followerResult = followerReceiptKey
4300
+ ? markToolExecuteResultExecutionOutcome(result, {
4301
+ kind: 'in_flight',
4302
+ receiptKey: followerReceiptKey,
4303
+ attachedToReceiptKey: followerReceiptKey,
4304
+ })
4305
+ : result;
4306
+ this.cacheToolResult(toolId, follower.cacheKey, followerResult);
4307
+ const resolver = this.toolCallResolvers.get(follower.callId);
4308
+ if (resolver) {
4309
+ resolver.resolve(followerResult);
4310
+ this.toolCallResolvers.delete(follower.callId);
4311
+ }
4312
+ this.emitScopedFieldMetaUpdate({
4313
+ rowId: follower.rowId,
4314
+ key: follower.rowKey ?? null,
4315
+ tableNamespace: follower.tableNamespace ?? null,
4316
+ fieldName: follower.fieldName,
4317
+ status: 'cached',
4318
+ rowStatus: 'running',
4319
+ stage: toolId,
4320
+ provider: null,
4321
+ error: null,
4322
+ reused: true,
4323
+ dataPatch: {},
4324
+ });
4325
+ }
4326
+ };
4327
+ const rejectWithLiveFollowers = async (
4328
+ owner: ToolCallRequest,
4329
+ error: unknown,
4330
+ ): Promise<void> => {
4331
+ const followers = liveFollowersByOwnerCallId.get(owner.callId) ?? [];
4332
+ liveFollowersByOwnerCallId.delete(owner.callId);
4333
+ for (const request of [owner, ...followers]) {
4334
+ await this.rejectToolCall(toolId, request, error);
4335
+ }
4336
+ };
4337
+
4338
+ const durableWaiters: Promise<void>[] = [];
4339
+ if (
4340
+ pendingRequests.length > 0 &&
4341
+ (this.#options.claimRuntimeStepReceipts ||
4342
+ this.#options.claimRuntimeStepReceipt)
4343
+ ) {
4344
+ const requestsByReceiptKey = new Map<
4345
+ string,
4346
+ { requests: ToolCallRequest[]; forceRefresh: boolean }
4347
+ >();
4348
+ const localOnlyRequests: ToolCallRequest[] = [];
4349
+ for (const request of pendingRequests) {
4350
+ const receiptKey = request.receiptKey?.trim() || null;
4351
+ if (!receiptKey) {
4352
+ localOnlyRequests.push(request);
4353
+ continue;
4354
+ }
4355
+ const group = requestsByReceiptKey.get(receiptKey) ?? {
4356
+ requests: [],
4357
+ forceRefresh: false,
4358
+ };
4359
+ group.requests.push(request);
4360
+ group.forceRefresh ||= request.force === true;
4361
+ requestsByReceiptKey.set(receiptKey, group);
4362
+ }
4363
+ const normalReceiptKeys = [...requestsByReceiptKey]
4364
+ .filter(([, group]) => !group.forceRefresh)
4365
+ .map(([receiptKey]) => receiptKey);
4366
+ const forcedReceiptKeys = [...requestsByReceiptKey]
4367
+ .filter(([, group]) => group.forceRefresh)
4368
+ .map(([receiptKey]) => receiptKey);
4369
+ const claims =
4370
+ normalReceiptKeys.length > 0
4371
+ ? await this.claimRuntimeStepReceipts(
4372
+ normalReceiptKeys,
4373
+ this.currentRunId,
4374
+ )
4375
+ : new Map<string, RuntimeStepReceipt>();
4376
+ const forcedClaims =
4377
+ forcedReceiptKeys.length > 0
4378
+ ? await this.claimRuntimeStepReceipts(
4379
+ forcedReceiptKeys,
4380
+ this.currentRunId,
4381
+ true,
4382
+ true,
4383
+ )
4384
+ : new Map<string, RuntimeStepReceipt>();
4385
+ for (const [receiptKey, claim] of forcedClaims) {
4386
+ claims.set(receiptKey, claim);
4387
+ }
4388
+ const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
4389
+ const durableRecoveredRequests: ToolCallRequest[] = [];
4390
+ const resolveRequestsFromReceipt = async (
4391
+ requestsForKey: ToolCallRequest[],
4392
+ receipt: RuntimeStepReceipt,
4393
+ source: 'cache' | 'in_flight',
4394
+ ): Promise<void> => {
4395
+ const request = requestsForKey[0];
4396
+ if (!request) return;
4397
+ const receiptKey = request.receiptKey?.trim() || null;
4398
+ if (!receiptKey) {
4399
+ throw new Error(
4400
+ `Durable tool receipt ${source} recovery requires a receipt key.`,
4401
+ );
4402
+ }
4403
+ const wrapped = await this.wrapToolExecutionResult({
4404
+ toolId,
4405
+ status:
4406
+ receipt.output === null || receipt.output === undefined
4407
+ ? 'no_result'
4408
+ : 'completed',
4409
+ result: this.runtimeReceiptOutput(receipt),
4410
+ execution: toolExecutionMetadataForOutcome({
4411
+ kind: source,
4412
+ cacheKey: request.cacheKey,
4413
+ receiptKey,
4414
+ attachedToReceiptKey: receiptKey,
4415
+ }),
4416
+ });
4417
+ this.cacheToolResult(toolId, request.cacheKey, wrapped);
4418
+ for (const waitingRequest of requestsForKey) {
4419
+ const resolver = this.toolCallResolvers.get(
4420
+ waitingRequest.callId,
4421
+ );
4422
+ if (resolver) {
4423
+ resolver.resolve(wrapped);
4424
+ this.toolCallResolvers.delete(waitingRequest.callId);
4425
+ }
4426
+ this.emitScopedFieldMetaUpdate({
4427
+ rowId: waitingRequest.rowId,
4428
+ key: waitingRequest.rowKey ?? null,
4429
+ tableNamespace: waitingRequest.tableNamespace ?? null,
4430
+ fieldName: waitingRequest.fieldName,
4431
+ status: 'cached',
4432
+ rowStatus: 'running',
4433
+ stage: toolId,
4434
+ provider: null,
4435
+ error: null,
4436
+ reused: true,
4437
+ dataPatch: {},
4438
+ });
4439
+ }
4440
+ };
4441
+ const claimOwnerWithLiveFollowers = (
4442
+ owner: ToolCallRequest | undefined,
4443
+ followers: ToolCallRequest[],
4444
+ ): void => {
4445
+ if (!owner) return;
4446
+ claimedRequests.push(owner);
4447
+ if (followers.length > 0) {
4448
+ liveFollowersByOwnerCallId.set(owner.callId, followers);
4449
+ }
4450
+ };
4451
+ const waitForReceiptOrTimeout = async (
4452
+ receiptKey: string,
4453
+ requestsForKey: ToolCallRequest[],
4454
+ ): Promise<void> => {
4455
+ try {
4456
+ await resolveRequestsFromReceipt(
4457
+ requestsForKey,
4458
+ await this.waitForCompletedRuntimeToolReceipt(receiptKey),
4459
+ 'in_flight',
4460
+ );
4461
+ return;
4462
+ } catch (error) {
4463
+ if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
4464
+ for (const request of requestsForKey) {
4465
+ await this.rejectToolCall(toolId, request, error, {
4466
+ persistReceiptFailure: false,
4467
+ });
4468
+ }
4469
+ return;
4470
+ }
4471
+ }
4472
+ for (const request of requestsForKey) {
4473
+ await this.rejectToolCall(
4474
+ toolId,
4475
+ request,
4476
+ new RuntimeReceiptWaitTimeoutError(receiptKey),
4477
+ { persistReceiptFailure: false },
4478
+ );
4479
+ }
4480
+ };
4481
+
4482
+ for (const [receiptKey, group] of requestsByReceiptKey) {
4483
+ const requestsForKey = group.requests;
4484
+ const claim = claims.get(receiptKey);
4485
+ if (!claim) {
4486
+ const [owner, ...waiters] = requestsForKey;
4487
+ claimOwnerWithLiveFollowers(owner, waiters);
4488
+ continue;
4489
+ }
4490
+ if (claim?.status === 'completed' || claim?.status === 'skipped') {
4491
+ await resolveRequestsFromReceipt(requestsForKey, claim, 'cache');
4492
+ durableRecoveredRequests.push(...requestsForKey);
4493
+ continue;
4494
+ }
4495
+ if (claim?.status === 'failed') {
4496
+ for (const request of requestsForKey) {
4497
+ await this.rejectToolCall(
4498
+ toolId,
4499
+ request,
4500
+ new Error(claim.error ?? 'Durable tool call failed.'),
4501
+ );
4502
+ }
4503
+ continue;
4504
+ }
4505
+ if (
4506
+ claim?.status === 'running' &&
4507
+ claim.claimState !== 'existing'
4508
+ ) {
4509
+ const [owner, ...waiters] = requestsForKey;
4510
+ claimOwnerWithLiveFollowers(owner, waiters);
4511
+ continue;
4512
+ }
4513
+ durableWaiters.push(
4514
+ waitForReceiptOrTimeout(receiptKey, requestsForKey),
4515
+ );
4516
+ }
4517
+
4518
+ pendingRequests.length = 0;
4519
+ pendingRequests.push(...claimedRequests);
4520
+ recordToolStep(durableRecoveredRequests);
4521
+ }
4522
+
5207
4523
  if (pendingRequests.length > 0) {
5208
4524
  const strategy =
5209
4525
  this.#options.getBatchOperationStrategy?.(toolId) ?? null;
@@ -5234,7 +4550,7 @@ export class PlayContextImpl {
5234
4550
  for (const entry of chunkResults) {
5235
4551
  if (entry.error !== undefined) {
5236
4552
  for (const request of entry.request.memberRequests) {
5237
- this.rejectToolCall(toolId, request, entry.error);
4553
+ await rejectWithLiveFollowers(request, entry.error);
5238
4554
  }
5239
4555
  continue;
5240
4556
  }
@@ -5242,6 +4558,14 @@ export class PlayContextImpl {
5242
4558
  entry.result != null
5243
4559
  ? entry.request.splitResults(entry.result)
5244
4560
  : entry.request.memberRequests.map(() => null);
4561
+ const resolvedResults =
4562
+ await this.resolveToolCallBatchResults(
4563
+ toolId,
4564
+ entry.request.memberRequests.map((request, index) => ({
4565
+ request,
4566
+ result: splitResults[index] ?? null,
4567
+ })),
4568
+ );
5245
4569
 
5246
4570
  for (
5247
4571
  let index = 0;
@@ -5249,11 +4573,7 @@ export class PlayContextImpl {
5249
4573
  index += 1
5250
4574
  ) {
5251
4575
  const request = entry.request.memberRequests[index]!;
5252
- await this.resolveToolCall(
5253
- toolId,
5254
- request,
5255
- splitResults[index] ?? null,
5256
- );
4576
+ resolveLiveFollowers(request, resolvedResults[index]);
5257
4577
  }
5258
4578
  }
5259
4579
 
@@ -5281,7 +4601,7 @@ export class PlayContextImpl {
5281
4601
  toolId,
5282
4602
  request.input,
5283
4603
  );
5284
- await this.resolveToolCall(
4604
+ const result = await this.resolveToolCall(
5285
4605
  toolId,
5286
4606
  request,
5287
4607
  execution?.result ?? null,
@@ -5289,8 +4609,9 @@ export class PlayContextImpl {
5289
4609
  execution?.jobId,
5290
4610
  execution?.meta,
5291
4611
  );
4612
+ resolveLiveFollowers(request, result);
5292
4613
  } catch (error) {
5293
- this.rejectToolCall(toolId, request, error);
4614
+ await rejectWithLiveFollowers(request, error);
5294
4615
  }
5295
4616
  }),
5296
4617
  );
@@ -5300,6 +4621,9 @@ export class PlayContextImpl {
5300
4621
  }
5301
4622
  }
5302
4623
  }
4624
+ if (durableWaiters.length > 0) {
4625
+ await Promise.allSettled(durableWaiters);
4626
+ }
5303
4627
  }),
5304
4628
  );
5305
4629
  }
@@ -5377,95 +4701,6 @@ export class PlayContextImpl {
5377
4701
  )(ctx, input);
5378
4702
  }
5379
4703
 
5380
- // ——— Direct execution (outside map context) ———
5381
-
5382
- private async executeWaterfallDirect(
5383
- toolNameOrSpec: string | InlineWaterfallSpec,
5384
- input: Record<string, unknown>,
5385
- opts?: WaterfallOptions,
5386
- ): Promise<unknown | null> {
5387
- if (isInlineWaterfallSpec(toolNameOrSpec)) {
5388
- this.log(`Direct inline waterfall: ${toolNameOrSpec.id}`);
5389
- const collected: unknown[] = [];
5390
- for (const step of toolNameOrSpec.steps) {
5391
- this.governor.chargeBudget('waterfallStep');
5392
- const matched = isInlineWaterfallCodeStep(step)
5393
- ? extractInlineWaterfallCodeStepValue(
5394
- toolNameOrSpec.output,
5395
- await step.run(input, {
5396
- tools: {
5397
- execute: async (
5398
- requestOrKey:
5399
- | { tool: string; input: Record<string, unknown> }
5400
- | string,
5401
- toolId?: string,
5402
- payload?: Record<string, unknown>,
5403
- ) => {
5404
- if (typeof requestOrKey === 'object') {
5405
- return await this.callToolAPI(
5406
- requestOrKey.tool,
5407
- requestOrKey.input,
5408
- );
5409
- }
5410
- if (!toolId || !payload) {
5411
- throw new Error(
5412
- 'inline waterfall ctx.tools.execute requires a tool and input.',
5413
- );
5414
- }
5415
- return await this.callToolAPI(toolId, payload);
5416
- },
5417
- },
5418
- }),
5419
- )
5420
- : await extractWaterfallOutputValue(
5421
- step.toolId,
5422
- toolNameOrSpec.output,
5423
- await this.callToolAPI(step.toolId, step.mapInput(input)),
5424
- this.#options.getToolTargetGetters,
5425
- );
5426
- if (Array.isArray(matched)) {
5427
- collected.push(...matched);
5428
- } else if (matched != null) {
5429
- collected.push(matched);
5430
- }
5431
- if (collected.length >= toolNameOrSpec.minResults) {
5432
- return toolNameOrSpec.minResults === 1
5433
- ? (collected[0] ?? null)
5434
- : collected;
5435
- }
5436
- }
5437
- return null;
5438
- }
5439
-
5440
- const toolName = toolNameOrSpec;
5441
- this.log(`Direct waterfall: ${toolName}`);
5442
- const providers = opts?.providers ?? ['hunter', 'leadmagic', 'pdl'];
5443
-
5444
- for (const provider of providers) {
5445
- this.log(` Trying ${provider}`);
5446
- try {
5447
- this.governor.chargeBudget('retry');
5448
- const result = await this.callToolAPI(
5449
- resolveWaterfallToolId(provider, toolName),
5450
- input,
5451
- );
5452
- if (
5453
- result != null &&
5454
- typeof result === 'object' &&
5455
- Object.keys(result as object).length > 0
5456
- ) {
5457
- this.log(` Found with ${provider}`);
5458
- return result;
5459
- }
5460
- } catch {
5461
- this.log(` Failed with ${provider}`);
5462
- }
5463
- }
5464
-
5465
- this.log(` All providers exhausted`);
5466
- return null;
5467
- }
5468
-
5469
4704
  private async callToolAPI(
5470
4705
  toolId: string,
5471
4706
  input: Record<string, unknown>,