deepline 0.1.12 → 0.1.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +14 -6
  2. package/dist/cli/index.js +1298 -711
  3. package/dist/cli/index.mjs +1294 -707
  4. package/dist/index.d.mts +199 -23
  5. package/dist/index.d.ts +199 -23
  6. package/dist/index.js +219 -13
  7. package/dist/index.mjs +219 -13
  8. package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +68 -12
  9. package/dist/repo/apps/play-runner-workers/src/entry.ts +241 -51
  10. package/dist/repo/sdk/src/client.ts +237 -0
  11. package/dist/repo/sdk/src/config.ts +125 -8
  12. package/dist/repo/sdk/src/http.ts +10 -2
  13. package/dist/repo/sdk/src/play.ts +19 -36
  14. package/dist/repo/sdk/src/plays/bundle-play-file.ts +22 -8
  15. package/dist/repo/sdk/src/plays/local-file-discovery.ts +207 -160
  16. package/dist/repo/sdk/src/types.ts +25 -0
  17. package/dist/repo/sdk/src/version.ts +2 -2
  18. package/dist/repo/shared_libs/play-runtime/tool-result.ts +237 -145
  19. package/dist/repo/shared_libs/plays/bundling/index.ts +206 -229
  20. package/dist/repo/shared_libs/plays/dataset.ts +28 -0
  21. package/package.json +5 -4
  22. package/dist/cli/index.js.map +0 -1
  23. package/dist/cli/index.mjs.map +0 -1
  24. package/dist/index.js.map +0 -1
  25. package/dist/index.mjs.map +0 -1
  26. package/dist/repo/apps/play-runner-workers/src/runtime/README.md +0 -21
  27. package/dist/repo/apps/play-runner-workers/src/runtime/batching.ts +0 -177
  28. package/dist/repo/apps/play-runner-workers/src/runtime/execution-plan.ts +0 -52
  29. package/dist/repo/apps/play-runner-workers/src/runtime/tool-batch.ts +0 -100
  30. package/dist/repo/sdk/src/cli/commands/auth.ts +0 -500
  31. package/dist/repo/sdk/src/cli/commands/billing.ts +0 -188
  32. package/dist/repo/sdk/src/cli/commands/csv.ts +0 -123
  33. package/dist/repo/sdk/src/cli/commands/db.ts +0 -119
  34. package/dist/repo/sdk/src/cli/commands/feedback.ts +0 -40
  35. package/dist/repo/sdk/src/cli/commands/org.ts +0 -117
  36. package/dist/repo/sdk/src/cli/commands/play.ts +0 -3441
  37. package/dist/repo/sdk/src/cli/commands/tools.ts +0 -687
  38. package/dist/repo/sdk/src/cli/dataset-stats.ts +0 -415
  39. package/dist/repo/sdk/src/cli/index.ts +0 -148
  40. package/dist/repo/sdk/src/cli/progress.ts +0 -149
  41. package/dist/repo/sdk/src/cli/skills-sync.ts +0 -141
  42. package/dist/repo/sdk/src/cli/trace.ts +0 -61
  43. package/dist/repo/sdk/src/cli/utils.ts +0 -145
  44. package/dist/repo/sdk/src/compat.ts +0 -77
  45. package/dist/repo/shared_libs/observability/node-tracing.ts +0 -129
  46. package/dist/repo/shared_libs/observability/tracing.ts +0 -98
  47. package/dist/repo/shared_libs/play-runtime/context.ts +0 -4242
  48. package/dist/repo/shared_libs/play-runtime/ctx-contract.ts +0 -250
  49. package/dist/repo/shared_libs/play-runtime/ctx-types.ts +0 -725
  50. package/dist/repo/shared_libs/play-runtime/dataset-id.ts +0 -10
  51. package/dist/repo/shared_libs/play-runtime/db-session-crypto.ts +0 -304
  52. package/dist/repo/shared_libs/play-runtime/db-session.ts +0 -462
  53. package/dist/repo/shared_libs/play-runtime/live-events.ts +0 -214
  54. package/dist/repo/shared_libs/play-runtime/live-state-contract.ts +0 -50
  55. package/dist/repo/shared_libs/play-runtime/map-execution-frame.ts +0 -114
  56. package/dist/repo/shared_libs/play-runtime/map-row-identity.ts +0 -158
  57. package/dist/repo/shared_libs/play-runtime/progress-emitter.ts +0 -172
  58. package/dist/repo/shared_libs/play-runtime/protocol.ts +0 -121
  59. package/dist/repo/shared_libs/play-runtime/public-play-contract.ts +0 -42
  60. package/dist/repo/shared_libs/play-runtime/result-normalization.ts +0 -33
  61. package/dist/repo/shared_libs/play-runtime/runtime-api.ts +0 -1873
  62. package/dist/repo/shared_libs/play-runtime/runtime-constraints.ts +0 -2
  63. package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +0 -201
  64. package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-pg.ts +0 -48
  65. package/dist/repo/shared_libs/play-runtime/runtime-pg-driver.ts +0 -84
  66. package/dist/repo/shared_libs/play-runtime/static-pipeline-types.ts +0 -147
  67. package/dist/repo/shared_libs/play-runtime/suspension.ts +0 -68
  68. package/dist/repo/shared_libs/play-runtime/tracing.ts +0 -31
  69. package/dist/repo/shared_libs/play-runtime/waterfall-replay.ts +0 -75
  70. package/dist/repo/shared_libs/play-runtime/worker-api-types.ts +0 -140
  71. package/dist/repo/shared_libs/plays/artifact-transport.ts +0 -14
  72. package/dist/repo/shared_libs/plays/artifact-types.ts +0 -49
  73. package/dist/repo/shared_libs/plays/compiler-manifest.ts +0 -186
  74. package/dist/repo/shared_libs/plays/definition.ts +0 -264
  75. package/dist/repo/shared_libs/plays/file-refs.ts +0 -11
  76. package/dist/repo/shared_libs/plays/rate-limit-scheduler.ts +0 -206
  77. package/dist/repo/shared_libs/plays/resolve-static-pipeline.ts +0 -164
  78. package/dist/repo/shared_libs/plays/runtime-validation.ts +0 -395
  79. package/dist/repo/shared_libs/temporal/constants.ts +0 -39
  80. package/dist/repo/shared_libs/temporal/preview-config.ts +0 -153
@@ -1,4242 +0,0 @@
1
- /**
2
- * PlayContextImpl — the cloud execution engine.
3
- *
4
- * Batching model:
5
- * 1. ctx.map("table_key", rows, { field: resolver }) starts all row field resolvers concurrently
6
- * 2. ctx.waterfall() calls inside field resolvers QUEUE requests (don't execute)
7
- * 3. ctx.tools.execute() calls inside field resolvers also QUEUE
8
- * 4. After all rows have queued, executeBatchedWaterfalls() runs provider-by-provider
9
- * 5. Each provider batch = real HTTP call to /api/v2/integrations/{toolId}/execute
10
- * 6. Results resolve suspended row promises, rows complete
11
- *
12
- * Temporal integration:
13
- * - checkpoint: recovered from heartbeatDetails on retry (skip completed batches)
14
- * - onBatchComplete: called after each provider batch (heartbeat checkpoint to Temporal)
15
- */
16
- import { AsyncLocalStorage } from 'async_hooks';
17
- import {
18
- createDeferredPlayDataset,
19
- materializePlayDatasetInput,
20
- } from '@shared_libs/plays/dataset';
21
- import type { PlayDataset, PlayDatasetInput } from '@shared_libs/plays/dataset';
22
- import {
23
- compileRequestsWithStrategy,
24
- executeChunkedRequests,
25
- executeWaterfallProviders,
26
- resolveWaterfallToolId,
27
- } from './batch-runtime';
28
- import { PlayRateLimitScheduler } from '@shared_libs/plays/rate-limit-scheduler';
29
- import {
30
- cloneToolExecuteResultWithExecution,
31
- createToolExecuteResult,
32
- isToolExecuteResult,
33
- type ToolExecuteResult,
34
- type ToolResultMetadataInput,
35
- } from './tool-result';
36
- import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline';
37
- import { createRuntimeDatasetId } from './dataset-id';
38
- import {
39
- derivePlayRowIdentity,
40
- derivePlayRowIdentityFromKey,
41
- MAP_KEY_NAMESPACE_MAX_LENGTH,
42
- normalizeTableNamespace,
43
- resolveStaleMapTableNamespace,
44
- } from '@shared_libs/plays/row-identity';
45
- import { cloneCsvAliasedRow, stripCsvProjectedFields } from './csv-rename';
46
- import { setSpanAttributes, withActiveSpan } from './tracing';
47
- import { DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE } from './runtime-constraints';
48
- import {
49
- PlayExecutionSuspendedError,
50
- PlayRowExecutionSuspendedError,
51
- isPlayRowExecutionSuspendedError,
52
- } from './suspension';
53
- import type {
54
- CsvOptions,
55
- RowState,
56
- WaterfallRequest,
57
- WaterfallOptions,
58
- InlineWaterfallSpec,
59
- MapOptions,
60
- ToolCallRequest,
61
- ToolBatchResult,
62
- ContextOptions,
63
- PlayCallOptions,
64
- PlayCheckpoint,
65
- PlayStep,
66
- PlayStepRowResult,
67
- PlayRowUpdate,
68
- MapFieldDefinition,
69
- MapFieldResolver,
70
- ToolCallOptions,
71
- PlayExecutionGovernanceLimits,
72
- PlayExecutionGovernanceState,
73
- ResolvedPlayExecution,
74
- PlayFetchResponse,
75
- MapExecutionScope,
76
- MapExecutionFrame,
77
- PlayExecutionEvent,
78
- IntegrationEventWaitHandler,
79
- RuntimeStepReceipt,
80
- RuntimeStepProgram,
81
- RuntimeStepProgramStep,
82
- RuntimeConditionalStepResolver,
83
- } from './ctx-types';
84
-
85
- /**
86
- * SECURITY: AsyncLocalStorage is per async execution, not a cross-run cache.
87
- *
88
- * Keep only row-scoped metadata here. Do not expand this store with credentials,
89
- * provider responses, org-wide mutable caches, or anything that must not cross
90
- * workflow boundaries if worker scheduling/interleaving changes.
91
- */
92
- const rowContext = new AsyncLocalStorage<{
93
- rowId: number;
94
- fieldName?: string;
95
- tableNamespace?: string;
96
- rowKey?: string;
97
- mapScope?: MapExecutionScope;
98
- }>();
99
- const PROGRESS_HEARTBEAT_INTERVAL_MS = 1_000;
100
- const PURE_JS_HEARTBEAT_ROW_INTERVAL = 250;
101
- const TOOL_RETRY_AFTER_FALLBACK_MS = 1_000;
102
- const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
103
- const TOOL_TRANSIENT_HTTP_MAX_ATTEMPTS = 3;
104
- const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
105
- const IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT = 25;
106
- const WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT = 10;
107
- const WATERFALL_ROW_MATCH_LOG_INTERVAL = 1_000;
108
- const BATCH_SIZE_LOG_SAMPLE_LIMIT = 10;
109
- const DEFAULT_GOVERNANCE_LIMITS: PlayExecutionGovernanceLimits = {
110
- maxPlayCallDepth: 6,
111
- maxPlayCallCount: 32,
112
- maxToolCallCount: 500,
113
- maxWaterfallStepExecutions: null,
114
- maxRetryCount: 200,
115
- maxDescendants: 64,
116
- maxChildPlayCallsPerParent: 16,
117
- maxConcurrentPlayCalls: 16,
118
- maxConcurrentToolCalls: 32,
119
- };
120
- const STEP_PROGRAM_MAP_DEFINITION = Symbol('deepline.stepProgramMapDefinition');
121
-
122
- function stableFetchHash(value: string): string {
123
- let hash = 0;
124
- for (let index = 0; index < value.length; index += 1) {
125
- hash = (hash * 31 + value.charCodeAt(index)) >>> 0;
126
- }
127
- return hash.toString(36);
128
- }
129
-
130
- function compactRowResultsPreview<T>(rows: T[]): T[] {
131
- if (rows.length <= IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT) {
132
- return rows;
133
- }
134
- return rows.slice(0, IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT);
135
- }
136
-
137
- function isRuntimeStepProgram(value: unknown): value is RuntimeStepProgram {
138
- return (
139
- !!value &&
140
- typeof value === 'object' &&
141
- (value as { kind?: unknown }).kind === 'steps' &&
142
- Array.isArray((value as { steps?: unknown }).steps)
143
- );
144
- }
145
-
146
- function isRuntimeConditionalStepResolver(
147
- value: unknown,
148
- ): value is RuntimeConditionalStepResolver {
149
- return (
150
- !!value &&
151
- typeof value === 'object' &&
152
- (value as { kind?: unknown }).kind === 'conditional' &&
153
- typeof (value as { when?: unknown }).when === 'function' &&
154
- typeof (value as { run?: unknown }).run === 'function'
155
- );
156
- }
157
-
158
- class RuntimeMapBuilder<T extends Record<string, unknown>> {
159
- private readonly program: RuntimeStepProgram = {
160
- kind: 'steps',
161
- steps: [],
162
- };
163
-
164
- constructor(
165
- private readonly ctx: PlayContextImpl,
166
- private readonly key: string,
167
- private readonly items: PlayDatasetInput<T>,
168
- ) {}
169
-
170
- step(name: string, resolver: RuntimeStepProgramStep['resolver']): this {
171
- if (!name.trim()) {
172
- throw new Error(
173
- 'ctx.map(...).step(name, ...) requires a non-empty step name.',
174
- );
175
- }
176
- this.program.steps = [...this.program.steps, { name, resolver }];
177
- return this;
178
- }
179
-
180
- run(options?: MapOptions<T>): Promise<PlayDataset<Record<string, unknown>>> {
181
- return this.ctx.runStepProgramMap(
182
- this.key,
183
- this.items,
184
- this.program,
185
- options,
186
- );
187
- }
188
- }
189
-
190
- const WAITING_ROW = Symbol('deepline.waiting_row');
191
-
192
- function normalizeFetchHeaders(
193
- headers: RequestInit['headers'],
194
- ): Record<string, string> {
195
- if (!headers) return {};
196
- if (headers instanceof Headers) {
197
- return Object.fromEntries(
198
- [...headers.entries()].map(([key, value]) => [key.toLowerCase(), value]),
199
- );
200
- }
201
- if (Array.isArray(headers)) {
202
- return Object.fromEntries(
203
- headers.map(([key, value]) => [key.toLowerCase(), value]),
204
- );
205
- }
206
- return Object.fromEntries(
207
- Object.entries(headers).map(([key, value]) => [
208
- key.toLowerCase(),
209
- String(value),
210
- ]),
211
- );
212
- }
213
-
214
- function parseRetryAfterMs(header: string | null): number {
215
- if (!header) {
216
- return TOOL_RETRY_AFTER_FALLBACK_MS;
217
- }
218
-
219
- const seconds = Number(header);
220
- if (Number.isFinite(seconds) && seconds > 0) {
221
- return Math.ceil(seconds * 1000);
222
- }
223
-
224
- const retryAt = Date.parse(header);
225
- if (Number.isFinite(retryAt)) {
226
- return Math.max(1, retryAt - Date.now());
227
- }
228
-
229
- return TOOL_RETRY_AFTER_FALLBACK_MS;
230
- }
231
-
232
- function parseJsonOrNull(bodyText: string): unknown | null {
233
- if (!bodyText.trim()) return null;
234
- try {
235
- return JSON.parse(bodyText) as unknown;
236
- } catch {
237
- return null;
238
- }
239
- }
240
-
241
- type ToolExecutionResponse = {
242
- status: string;
243
- result: unknown;
244
- metadata: ToolResultMetadataInput | null;
245
- };
246
-
247
- function parseExecuteToolMetadata(
248
- toolId: string,
249
- data: Record<string, unknown>,
250
- ): ToolResultMetadataInput | null {
251
- const metadata = data._metadata;
252
- if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata)) {
253
- return null;
254
- }
255
- const tool = (metadata as Record<string, unknown>).tool;
256
- if (!tool || typeof tool !== 'object' || Array.isArray(tool)) return null;
257
- const record = tool as Record<string, unknown>;
258
- const metadataToolId =
259
- typeof record.toolId === 'string' && record.toolId.trim()
260
- ? record.toolId
261
- : toolId;
262
- const readGetters = (value: unknown): Record<string, readonly string[]> => {
263
- if (!value || typeof value !== 'object' || Array.isArray(value)) return {};
264
- return Object.fromEntries(
265
- Object.entries(value as Record<string, unknown>).flatMap(([key, paths]) => {
266
- if (!Array.isArray(paths)) return [];
267
- const normalized = paths.filter(
268
- (path): path is string => typeof path === 'string' && path.trim().length > 0,
269
- );
270
- return normalized.length > 0 ? [[key, normalized]] : [];
271
- }),
272
- );
273
- };
274
- const listExtractorPaths = Array.isArray(record.listExtractorPaths)
275
- ? record.listExtractorPaths.filter(
276
- (path): path is string => typeof path === 'string' && path.trim().length > 0,
277
- )
278
- : [];
279
- return {
280
- toolId: metadataToolId,
281
- resultIdentityGetters: readGetters(record.resultIdentityGetters),
282
- listExtractorPaths,
283
- listIdentityGetters: readGetters(record.listIdentityGetters),
284
- };
285
- }
286
-
287
- function assertJsonSerializableStepOutput(
288
- stepId: string,
289
- output: unknown,
290
- ): void {
291
- try {
292
- JSON.stringify(output);
293
- } catch (error) {
294
- throw new Error(
295
- `ctx.step(${stepId}) returned a value that cannot be checkpointed as JSON: ${
296
- error instanceof Error ? error.message : String(error)
297
- }`,
298
- );
299
- }
300
- }
301
-
302
- function normalizeStepDescription(
303
- value: string | undefined,
304
- ): string | undefined {
305
- const trimmed = value?.trim();
306
- return trimmed ? trimmed : undefined;
307
- }
308
-
309
- function emptyCheckpoint(): PlayCheckpoint {
310
- return {
311
- completedBatches: {},
312
- completedToolBatches: {},
313
- resolvedWaterfalls: {},
314
- mapFrames: {},
315
- };
316
- }
317
-
318
- function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
319
- return {
320
- ...frame,
321
- completedRowKeys: [...frame.completedRowKeys],
322
- pendingRowKeys: [...frame.pendingRowKeys],
323
- };
324
- }
325
-
326
- function createDefaultGovernanceState(
327
- options: ContextOptions,
328
- ): PlayExecutionGovernanceState {
329
- const playId = options.playId ?? options.workflowId ?? 'anonymous-play';
330
- const runId = options.runId ?? options.workflowId ?? 'anonymous-run';
331
- return {
332
- rootPlayId: playId,
333
- rootRunId: runId,
334
- currentPlayId: playId,
335
- currentRunId: runId,
336
- ancestryPlayIds: [playId],
337
- ancestryRunIds: [runId],
338
- callDepth: 0,
339
- playCallCount: 0,
340
- toolCallCount: 0,
341
- waterfallStepExecutions: 0,
342
- retryCount: 0,
343
- descendantCount: 0,
344
- parentChildCalls: {},
345
- inFlightPlayCalls: 0,
346
- inFlightPlayCallsByPlayId: {},
347
- inFlightToolCalls: 0,
348
- limits: DEFAULT_GOVERNANCE_LIMITS,
349
- };
350
- }
351
-
352
- function isInlineWaterfallSpec(value: unknown): value is InlineWaterfallSpec {
353
- if (!value || typeof value !== 'object' || Array.isArray(value)) {
354
- return false;
355
- }
356
- const candidate = value as InlineWaterfallSpec;
357
- return (
358
- typeof candidate.id === 'string' &&
359
- typeof candidate.output === 'string' &&
360
- typeof candidate.minResults === 'number' &&
361
- candidate.minResults >= 1 &&
362
- Array.isArray(candidate.steps) &&
363
- candidate.steps.length > 0
364
- );
365
- }
366
-
367
- function isInlineWaterfallCodeStep(
368
- step: InlineWaterfallSpec['steps'][number],
369
- ): step is Extract<InlineWaterfallSpec['steps'][number], { kind: 'code' }> {
370
- return step.kind === 'code';
371
- }
372
-
373
- function isInlineWaterfallToolStep(
374
- step: InlineWaterfallSpec['steps'][number],
375
- ): step is Extract<InlineWaterfallSpec['steps'][number], { toolId: string }> {
376
- return !isInlineWaterfallCodeStep(step);
377
- }
378
-
379
- function extractInlineWaterfallCodeStepValue(
380
- output: string,
381
- result: unknown,
382
- ): unknown {
383
- if (
384
- result &&
385
- typeof result === 'object' &&
386
- !Array.isArray(result) &&
387
- output in result
388
- ) {
389
- return (result as Record<string, unknown>)[output] ?? null;
390
- }
391
- return result ?? null;
392
- }
393
-
394
- function isMeaningfulValue(value: unknown): boolean {
395
- if (value == null) return false;
396
- if (typeof value === 'string') return value.trim().length > 0;
397
- if (Array.isArray(value)) return value.length > 0;
398
- if (typeof value === 'object') return Object.keys(value).length > 0;
399
- return true;
400
- }
401
-
402
- function getValueAtPath(value: unknown, path: string): unknown {
403
- let current = value;
404
- for (const part of path.split('.').filter(Boolean)) {
405
- if (!current || typeof current !== 'object') return undefined;
406
- current = (current as Record<string, unknown>)[part];
407
- }
408
- return current;
409
- }
410
-
411
- async function extractWaterfallOutputValue(
412
- toolId: string,
413
- output: string,
414
- result: unknown,
415
- getToolResultIdentityGetters?: (
416
- toolId: string,
417
- output: string,
418
- ) => Promise<readonly string[]>,
419
- ): Promise<unknown> {
420
- const getters = getToolResultIdentityGetters
421
- ? await getToolResultIdentityGetters(toolId, output)
422
- : [];
423
- for (const getter of getters) {
424
- const value = getValueAtPath(result, getter);
425
- if (isMeaningfulValue(value)) {
426
- return value;
427
- }
428
- }
429
-
430
- const directOutputValue = getValueAtPath(result, output);
431
- if (directOutputValue !== undefined) {
432
- return isMeaningfulValue(directOutputValue) ? directOutputValue : null;
433
- }
434
-
435
- if (output === 'people' || output === 'companies') {
436
- if (Array.isArray(result)) return result;
437
- const direct = getValueAtPath(result, output);
438
- if (Array.isArray(direct)) return direct;
439
- }
440
-
441
- return isMeaningfulValue(result) ? result : null;
442
- }
443
-
444
- export class PlayContextImpl {
445
- private rowStates = new Map<number, RowState>();
446
- private resolvers = new Map<string, (value: unknown) => void>();
447
- private waterfallQueue = new Map<string, WaterfallRequest[]>();
448
- private toolCallQueue: ToolCallRequest[] = [];
449
- private toolCallResolvers = new Map<string, (value: unknown) => void>();
450
- private options: ContextOptions;
451
- private logBuffer: string[] = [];
452
- private checkpoint: PlayCheckpoint;
453
- private steps: PlayStep[] = [];
454
- private explicitMapInvocationKeys = new Set<string>();
455
- /** The map step currently being built — substeps go here instead of top-level. */
456
- private activeMapStep: Extract<PlayStep, { type: 'map' }> | null = null;
457
- /** Last completed map step — for post-map recordStep calls (e.g. run_javascript). */
458
- private lastMapStep: Extract<PlayStep, { type: 'map' }> | null = null;
459
- private pureMapExecutionActive = false;
460
- private lastProgressHeartbeatAt = 0;
461
- private pendingRowEventBoundaries: Array<{
462
- boundaryId: string;
463
- eventKey: string;
464
- timeoutMs: number;
465
- }> = [];
466
- private processedRowCount = 0;
467
- private directToolCallIndex = 0;
468
- private sleepBoundaryIndex = 0;
469
- private fetchCallIndex = 0;
470
- private mapInvocationIndex = 0;
471
- private readonly stepCallIndexByKey = new Map<string, number>();
472
- private readonly waterfallMatchLogCounts = new Map<string, number>();
473
- private readonly rateLimitScheduler: PlayRateLimitScheduler;
474
- private readonly governance: PlayExecutionGovernanceState;
475
- readonly tools = {
476
- execute: <TOutput = unknown>(
477
- requestOrKey:
478
- | {
479
- id: string;
480
- tool: string;
481
- input: Record<string, unknown>;
482
- description?: string;
483
- }
484
- | string,
485
- toolId?: string,
486
- input?: Record<string, unknown>,
487
- options?: ToolCallOptions,
488
- ): Promise<TOutput> => {
489
- if (typeof requestOrKey === 'object') {
490
- return this.executeTool(
491
- requestOrKey.id,
492
- requestOrKey.tool,
493
- requestOrKey.input,
494
- requestOrKey.description
495
- ? { description: requestOrKey.description }
496
- : undefined,
497
- ) as Promise<TOutput>;
498
- }
499
- if (!toolId || !input) {
500
- throw new Error(
501
- 'ctx.tools.execute(key, toolId, input) requires a tool ID and input.',
502
- );
503
- }
504
- return this.executeTool(
505
- requestOrKey,
506
- toolId,
507
- input,
508
- options,
509
- ) as Promise<TOutput>;
510
- },
511
- };
512
-
513
- tool<TOutput = unknown>(
514
- key: string,
515
- toolId: string,
516
- input: Record<string, unknown>,
517
- options?: ToolCallOptions,
518
- ): Promise<TOutput> {
519
- return this.tools.execute<TOutput>(key, toolId, input, options);
520
- }
521
-
522
- constructor(options: ContextOptions) {
523
- this.options = options;
524
- this.rateLimitScheduler = new PlayRateLimitScheduler({
525
- getQueueHints: options.getToolQueueHints,
526
- });
527
- this.checkpoint = options.checkpoint ?? emptyCheckpoint();
528
- this.governance = {
529
- ...(options.governance ?? createDefaultGovernanceState(options)),
530
- inFlightPlayCallsByPlayId:
531
- options.governance?.inFlightPlayCallsByPlayId ?? {},
532
- };
533
- }
534
-
535
- private durableBoundaryId(localId: string): string {
536
- // Durable boundaries live in one checkpoint for the whole root execution.
537
- // Nested plays and concurrent child calls therefore need a stable run scope
538
- // in the key, otherwise two children can both produce e.g. "sleep-0-25"
539
- // and replay the wrong boundary or never observe completion.
540
- return `${this.governance.currentRunId}:${localId}`;
541
- }
542
-
543
- private emitScopedRowUpdate(
544
- key: string | null,
545
- tableNamespace: string | null,
546
- update: Omit<PlayRowUpdate, 'key'>,
547
- ): void {
548
- const rowScope = rowContext.getStore()?.mapScope;
549
- if (rowScope && key) {
550
- this.emitExecutionEvent({
551
- type: 'map.row.updated',
552
- mapInvocationId: rowScope.mapInvocationId,
553
- mapNodeId: rowScope.mapNodeId ?? null,
554
- logicalNamespace: rowScope.logicalNamespace,
555
- artifactTableNamespace: rowScope.artifactTableNamespace,
556
- rowKey: key,
557
- rowStatus: update.status,
558
- fieldName: rowContext.getStore()?.fieldName ?? null,
559
- stage: update.stage ?? null,
560
- provider: update.provider ?? null,
561
- at: Date.now(),
562
- });
563
- }
564
- if (!key || !this.options.onRowUpdate) {
565
- return;
566
- }
567
- void this.options.onRowUpdate({
568
- ...update,
569
- key,
570
- tableNamespace,
571
- });
572
- }
573
-
574
- private emitExecutionEvent(event: PlayExecutionEvent): void {
575
- if (!this.options.onExecutionEvent) {
576
- return;
577
- }
578
- void this.options.onExecutionEvent(event);
579
- }
580
-
581
- private setMapFrame(frame: MapExecutionFrame): void {
582
- this.checkpoint.mapFrames = {
583
- ...(this.checkpoint.mapFrames ?? {}),
584
- [frame.mapInvocationId]: cloneMapFrame(frame),
585
- };
586
- }
587
-
588
- private createMapExecutionScope(input: {
589
- logicalNamespace: string;
590
- artifactTableNamespace: string;
591
- mapNodeId?: string | null;
592
- explicitKey?:
593
- | ((row: Record<string, unknown>, index: number) => string)
594
- | null;
595
- }): MapExecutionScope {
596
- const mapInvocationId = `${input.logicalNamespace}:${this.mapInvocationIndex}`;
597
- this.mapInvocationIndex += 1;
598
- const explicitKey = input.explicitKey ?? null;
599
- return {
600
- mapInvocationId,
601
- mapNodeId: input.mapNodeId ?? null,
602
- logicalNamespace: input.logicalNamespace,
603
- artifactTableNamespace: input.artifactTableNamespace,
604
- rowIdentity: (row, index) =>
605
- explicitKey
606
- ? derivePlayRowIdentityFromKey(
607
- explicitKey(row, index ?? 0),
608
- input.artifactTableNamespace,
609
- )
610
- : derivePlayRowIdentity(
611
- stripCsvProjectedFields(row),
612
- input.artifactTableNamespace,
613
- ),
614
- };
615
- }
616
-
617
- private normalizeContextKey(key: string, operation: string): string {
618
- const normalizedKey = key.trim();
619
- if (!normalizedKey) {
620
- throw new Error(`ctx.${operation} requires a non-empty key.`);
621
- }
622
- return normalizedKey;
623
- }
624
-
625
- private async getRuntimeStepReceipt(
626
- key: string,
627
- ): Promise<RuntimeStepReceipt | null> {
628
- if (!this.options.getRuntimeStepReceipt) {
629
- return null;
630
- }
631
- const receipt = await this.options.getRuntimeStepReceipt({ key });
632
- if (receipt && typeof receipt.key === 'string' && receipt.key.trim()) {
633
- return {
634
- ...receipt,
635
- key: receipt.key.trim(),
636
- runId: receipt.runId ?? null,
637
- };
638
- }
639
- return receipt
640
- ? {
641
- key: key.trim(),
642
- status: 'pending',
643
- }
644
- : null;
645
- }
646
-
647
- private async claimRuntimeStepReceipt(
648
- key: string,
649
- runId: string,
650
- ): Promise<RuntimeStepReceipt | null> {
651
- if (!this.options.claimRuntimeStepReceipt) {
652
- return null;
653
- }
654
- const claimed = await this.options.claimRuntimeStepReceipt({
655
- key,
656
- runId,
657
- });
658
- if (!claimed || typeof claimed.key !== 'string' || !claimed.key.trim()) {
659
- return null;
660
- }
661
- return {
662
- ...claimed,
663
- key: claimed.key.trim(),
664
- runId: claimed.runId ?? null,
665
- };
666
- }
667
-
668
- private async completeRuntimeStepReceipt(
669
- key: string,
670
- runId: string,
671
- output: unknown | null,
672
- ): Promise<RuntimeStepReceipt | null> {
673
- if (!this.options.completeRuntimeStepReceipt) {
674
- return null;
675
- }
676
- const completed = await this.options.completeRuntimeStepReceipt({
677
- key,
678
- runId,
679
- output,
680
- });
681
- if (
682
- !completed ||
683
- typeof completed.key !== 'string' ||
684
- !completed.key.trim()
685
- ) {
686
- return null;
687
- }
688
- return {
689
- ...completed,
690
- key: completed.key.trim(),
691
- runId: completed.runId ?? null,
692
- };
693
- }
694
-
695
- private async failRuntimeStepReceipt(
696
- key: string,
697
- runId: string,
698
- error: string,
699
- ): Promise<RuntimeStepReceipt | null> {
700
- if (!this.options.failRuntimeStepReceipt) {
701
- return null;
702
- }
703
- const failed = await this.options.failRuntimeStepReceipt({
704
- key,
705
- runId,
706
- error,
707
- });
708
- if (!failed || typeof failed.key !== 'string' || !failed.key.trim()) {
709
- return null;
710
- }
711
- return {
712
- ...failed,
713
- key: failed.key.trim(),
714
- runId: failed.runId ?? null,
715
- };
716
- }
717
-
718
- private async executeWithRuntimeReceipt<T>(
719
- operation: string,
720
- key: string,
721
- runId: string,
722
- opts: {
723
- markSkipped?: (output: T) => Promise<void> | void;
724
- execute: () => Promise<T>;
725
- },
726
- ): Promise<T> {
727
- const existing = await this.getRuntimeStepReceipt(key);
728
- if (existing?.status === 'completed' || existing?.status === 'skipped') {
729
- this.log(`ctx.${operation}(${key}): recovered result from receipt`);
730
- if (existing.output === undefined) {
731
- return existing.output as T;
732
- }
733
- if (opts.markSkipped) {
734
- await opts.markSkipped(existing.output as T);
735
- }
736
- return existing.output as T;
737
- }
738
-
739
- const claimed = await this.claimRuntimeStepReceipt(key, runId);
740
- if (!claimed) {
741
- if (existing?.status === 'running' || existing?.status === 'failed') {
742
- const runningRun = existing.runId?.trim() ?? 'unknown';
743
- if (existing.status === 'running' && existing.runId === runId) {
744
- // Fall through and attempt execution; likely a replay of the same run
745
- // where row-level checkpoints already recorded but external receipt
746
- // state was not visible.
747
- } else {
748
- throw new Error(
749
- `ctx.${operation}(${key}): another run is already executing this key (${runningRun}).`,
750
- );
751
- }
752
- }
753
- }
754
-
755
- try {
756
- const result = await opts.execute();
757
- await this.completeRuntimeStepReceipt(key, runId, result);
758
- return result;
759
- } catch (error) {
760
- await this.failRuntimeStepReceipt(
761
- key,
762
- runId,
763
- this.formatRuntimeError(error),
764
- );
765
- throw error;
766
- }
767
- }
768
-
769
- private get currentRunId(): string {
770
- return this.options.runId ?? this.governance.currentRunId ?? 'unknown';
771
- }
772
-
773
- private emitScopedFieldMetaUpdate(input: {
774
- rowId: number;
775
- key: string | null;
776
- tableNamespace: string | null;
777
- fieldName?: string | null;
778
- status:
779
- | 'queued'
780
- | 'running'
781
- | 'completed'
782
- | 'failed'
783
- | 'cached'
784
- | 'missed'
785
- | 'skipped';
786
- rowStatus?: PlayRowUpdate['status'];
787
- stage?: string | null;
788
- provider?: string | null;
789
- error?: string | null;
790
- reused?: boolean;
791
- dataPatch?: Record<string, unknown>;
792
- }): void {
793
- if (!input.fieldName) {
794
- this.emitScopedRowUpdate(input.key, input.tableNamespace, {
795
- rowId: input.rowId,
796
- status: input.rowStatus,
797
- stage: input.stage ?? null,
798
- provider: input.provider ?? null,
799
- error: input.error ?? null,
800
- dataPatch: input.dataPatch ?? {},
801
- });
802
- return;
803
- }
804
-
805
- this.emitScopedRowUpdate(input.key, input.tableNamespace, {
806
- rowId: input.rowId,
807
- status: input.rowStatus,
808
- stage: input.stage ?? null,
809
- provider: input.provider ?? null,
810
- error: input.error ?? null,
811
- dataPatch: input.dataPatch ?? {},
812
- cellMetaPatch: {
813
- [input.fieldName]: {
814
- status: input.status,
815
- stage: input.stage ?? null,
816
- provider: input.provider ?? null,
817
- error: input.error ?? null,
818
- ...(input.reused !== undefined ? { reused: input.reused } : {}),
819
- },
820
- },
821
- });
822
- }
823
-
824
- private emitCellUpdate(input: {
825
- rowId: number;
826
- key: string | null;
827
- tableNamespace: string | null;
828
- columnName: string;
829
- status:
830
- | 'queued'
831
- | 'running'
832
- | 'completed'
833
- | 'failed'
834
- | 'cached'
835
- | 'missed'
836
- | 'skipped';
837
- rowStatus?: PlayRowUpdate['status'];
838
- stage?: string | null;
839
- provider?: string | null;
840
- error?: string | null;
841
- value?: unknown;
842
- }): void {
843
- this.emitScopedRowUpdate(input.key, input.tableNamespace, {
844
- rowId: input.rowId,
845
- status: input.rowStatus,
846
- stage: input.stage ?? null,
847
- provider: input.provider ?? null,
848
- error: input.error ?? null,
849
- dataPatch:
850
- input.value === undefined ? {} : { [input.columnName]: input.value },
851
- cellMetaPatch: {
852
- [input.columnName]: {
853
- status: input.status,
854
- stage: input.stage ?? null,
855
- provider: input.provider ?? null,
856
- error: input.error ?? null,
857
- },
858
- },
859
- });
860
- }
861
-
862
- private emitQueuedInlineWaterfallSteps(
863
- rowId: number,
864
- key: string | null,
865
- tableNamespace: string | null,
866
- spec: InlineWaterfallSpec,
867
- ): void {
868
- for (const step of spec.steps) {
869
- this.emitCellUpdate({
870
- rowId,
871
- key,
872
- tableNamespace,
873
- columnName: sqlSafePlayColumnName(`${spec.id}.${step.id}`),
874
- status: 'queued',
875
- stage: step.id,
876
- provider: isInlineWaterfallToolStep(step) ? step.toolId : 'code',
877
- value: null,
878
- });
879
- }
880
- }
881
-
882
- private isCompletedFieldValue(value: unknown): boolean {
883
- return (
884
- value !== null &&
885
- value !== undefined &&
886
- !(typeof value === 'string' && value.length === 0)
887
- );
888
- }
889
-
890
- private shouldReuseExistingFieldValue(
891
- row: Record<string, unknown>,
892
- fieldName: string,
893
- ): boolean {
894
- return (
895
- Object.prototype.hasOwnProperty.call(row, fieldName) &&
896
- this.isCompletedFieldValue(row[fieldName])
897
- );
898
- }
899
-
900
- private toVisibleDataPatch(
901
- fields: Record<string, unknown>,
902
- ): Record<string, unknown> {
903
- return Object.fromEntries(
904
- Object.entries(fields).filter(
905
- ([fieldName]) => !fieldName.startsWith('_'),
906
- ),
907
- );
908
- }
909
-
910
- private formatRuntimeError(error: unknown): string {
911
- if (error instanceof Error) {
912
- return error.message;
913
- }
914
- return String(error);
915
- }
916
-
917
- private logWaterfallMatch(input: {
918
- queueKey: string;
919
- rowId: number;
920
- provider: string;
921
- }): void {
922
- const count = (this.waterfallMatchLogCounts.get(input.queueKey) ?? 0) + 1;
923
- this.waterfallMatchLogCounts.set(input.queueKey, count);
924
-
925
- if (count <= WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT) {
926
- this.log(
927
- ` Row ${input.rowId}: found with ${input.provider} (${count}/${WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT} sample)`,
928
- );
929
- if (count === WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT) {
930
- this.log(
931
- ` Further per-row matches for ${input.queueKey} will be summarized every ${WATERFALL_ROW_MATCH_LOG_INTERVAL.toLocaleString()} hits.`,
932
- );
933
- }
934
- return;
935
- }
936
-
937
- if (count % WATERFALL_ROW_MATCH_LOG_INTERVAL === 0) {
938
- this.log(
939
- ` ${count.toLocaleString()} rows matched for ${input.queueKey}; latest row ${input.rowId} with ${input.provider}`,
940
- );
941
- }
942
- }
943
-
944
- private summarizeBatchSizes(sizes: readonly number[]): string {
945
- if (sizes.length <= BATCH_SIZE_LOG_SAMPLE_LIMIT) {
946
- return sizes.join(', ');
947
- }
948
- const sample = sizes.slice(0, BATCH_SIZE_LOG_SAMPLE_LIMIT).join(', ');
949
- const remaining = sizes.length - BATCH_SIZE_LOG_SAMPLE_LIMIT;
950
- return `${sample}, ... +${remaining} more`;
951
- }
952
-
953
- private bumpCounter(
954
- field:
955
- | 'playCallCount'
956
- | 'toolCallCount'
957
- | 'waterfallStepExecutions'
958
- | 'retryCount'
959
- | 'descendantCount',
960
- amount = 1,
961
- ): void {
962
- this.governance[field] += amount;
963
- const limits = this.governance.limits;
964
- if (
965
- field === 'playCallCount' &&
966
- this.governance.playCallCount > limits.maxPlayCallCount
967
- ) {
968
- throw new Error(
969
- `Root play-call budget exceeded (${this.governance.playCallCount}/${limits.maxPlayCallCount}).`,
970
- );
971
- }
972
- if (
973
- field === 'toolCallCount' &&
974
- this.governance.toolCallCount > limits.maxToolCallCount
975
- ) {
976
- throw new Error(
977
- `Root tool-call budget exceeded (${this.governance.toolCallCount}/${limits.maxToolCallCount}).`,
978
- );
979
- }
980
- if (
981
- field === 'waterfallStepExecutions' &&
982
- typeof limits.maxWaterfallStepExecutions === 'number' &&
983
- this.governance.waterfallStepExecutions >
984
- limits.maxWaterfallStepExecutions
985
- ) {
986
- throw new Error(
987
- `Root waterfall-step budget exceeded (${this.governance.waterfallStepExecutions}/${limits.maxWaterfallStepExecutions}).`,
988
- );
989
- }
990
- if (
991
- field === 'retryCount' &&
992
- this.governance.retryCount > limits.maxRetryCount
993
- ) {
994
- throw new Error(
995
- `Root retry budget exceeded (${this.governance.retryCount}/${limits.maxRetryCount}).`,
996
- );
997
- }
998
- if (
999
- field === 'descendantCount' &&
1000
- this.governance.descendantCount > limits.maxDescendants
1001
- ) {
1002
- throw new Error(
1003
- `Root descendant budget exceeded (${this.governance.descendantCount}/${limits.maxDescendants}).`,
1004
- );
1005
- }
1006
- }
1007
-
1008
- private acquirePlayConcurrency(playId: string): void {
1009
- this.governance.inFlightPlayCalls += 1;
1010
- const inFlightByPlayId = (this.governance.inFlightPlayCallsByPlayId ??= {});
1011
- inFlightByPlayId[playId] = (inFlightByPlayId[playId] ?? 0) + 1;
1012
- }
1013
-
1014
- private releasePlayConcurrency(playId: string): void {
1015
- this.governance.inFlightPlayCalls = Math.max(
1016
- 0,
1017
- this.governance.inFlightPlayCalls - 1,
1018
- );
1019
- const inFlightByPlayId = (this.governance.inFlightPlayCallsByPlayId ??= {});
1020
- const nextPlayCount = Math.max(0, (inFlightByPlayId[playId] ?? 0) - 1);
1021
- if (nextPlayCount === 0) {
1022
- delete inFlightByPlayId[playId];
1023
- } else {
1024
- inFlightByPlayId[playId] = nextPlayCount;
1025
- }
1026
- }
1027
-
1028
- private assertNoDuplicateConcurrentMapBackedPlay(): void {
1029
- if (this.governance.callDepth === 0) {
1030
- return;
1031
- }
1032
- const inFlightCount =
1033
- this.governance.inFlightPlayCallsByPlayId?.[
1034
- this.governance.currentPlayId
1035
- ] ?? 0;
1036
- if (inFlightCount <= 1) {
1037
- return;
1038
- }
1039
- throw new Error(
1040
- `Concurrent map-backed play call blocked for ${this.governance.currentPlayId}. ` +
1041
- 'A child play that uses ctx.map() cannot run more than once at the same time because its map tables share durable row identity. ' +
1042
- 'Run these child play calls sequentially, or give each concurrent branch a different child play/table contract.',
1043
- );
1044
- }
1045
-
1046
- private acquireToolConcurrency(): void {
1047
- this.governance.inFlightToolCalls += 1;
1048
- if (
1049
- this.governance.inFlightToolCalls >
1050
- this.governance.limits.maxConcurrentToolCalls
1051
- ) {
1052
- this.governance.inFlightToolCalls -= 1;
1053
- throw new Error(
1054
- `Concurrent tool-call limit exceeded (${this.governance.limits.maxConcurrentToolCalls}).`,
1055
- );
1056
- }
1057
- }
1058
-
1059
- private releaseToolConcurrency(): void {
1060
- this.governance.inFlightToolCalls = Math.max(
1061
- 0,
1062
- this.governance.inFlightToolCalls - 1,
1063
- );
1064
- }
1065
-
1066
- private getCachedToolResult(
1067
- toolId: string,
1068
- rowCacheKey: string,
1069
- ): ToolBatchResult | undefined {
1070
- return this.checkpoint.completedToolBatches[toolId]?.[rowCacheKey];
1071
- }
1072
-
1073
- private cacheToolResult(
1074
- toolId: string,
1075
- rowCacheKey: string,
1076
- result: unknown | null,
1077
- ): void {
1078
- if (!this.checkpoint.completedToolBatches[toolId]) {
1079
- this.checkpoint.completedToolBatches[toolId] = {};
1080
- }
1081
- this.checkpoint.completedToolBatches[toolId]![rowCacheKey] = {
1082
- done: true,
1083
- result,
1084
- };
1085
- }
1086
-
1087
- private buildToolResultCacheKey(input: {
1088
- rowId: number;
1089
- tableNamespace?: string;
1090
- rowKey?: string;
1091
- callId?: string;
1092
- }): string {
1093
- const scope = this.governance.currentRunId || this.options.runId || 'run';
1094
- if (input.callId?.trim()) {
1095
- return `${scope}:${input.callId.trim()}`;
1096
- }
1097
- if (input.rowKey?.trim()) {
1098
- return `${scope}:${input.rowKey.trim()}`;
1099
- }
1100
- if (input.tableNamespace?.trim()) {
1101
- return `${scope}:${input.tableNamespace.trim()}:${input.rowId}`;
1102
- }
1103
- return `${scope}:direct:${input.rowId}`;
1104
- }
1105
-
1106
- private async resolveToolResultMetadata(
1107
- toolId: string,
1108
- ): Promise<ToolResultMetadataInput> {
1109
- const metadata = await this.options.getToolResultMetadata?.(toolId);
1110
- return {
1111
- toolId,
1112
- resultIdentityGetters: metadata?.resultIdentityGetters ?? {},
1113
- listExtractorPaths: metadata?.listExtractorPaths ?? [],
1114
- listIdentityGetters: metadata?.listIdentityGetters ?? {},
1115
- };
1116
- }
1117
-
1118
- private async wrapToolExecutionResult(input: {
1119
- toolId: string;
1120
- status: string;
1121
- result: unknown;
1122
- metadata?: ToolResultMetadataInput | null;
1123
- cached: boolean;
1124
- source: 'live' | 'checkpoint' | 'cache';
1125
- cacheKey?: string;
1126
- }): Promise<ToolExecuteResult> {
1127
- if (isToolExecuteResult(input.result)) {
1128
- return cloneToolExecuteResultWithExecution(input.result, {
1129
- idempotent: true,
1130
- cached: input.cached,
1131
- source: input.source,
1132
- ...(input.cacheKey ? { cacheKey: input.cacheKey } : {}),
1133
- });
1134
- }
1135
- return createToolExecuteResult({
1136
- status: input.status,
1137
- result: input.result,
1138
- metadata: input.metadata ?? (await this.resolveToolResultMetadata(input.toolId)),
1139
- execution: {
1140
- idempotent: true,
1141
- cached: input.cached,
1142
- source: input.source,
1143
- ...(input.cacheKey ? { cacheKey: input.cacheKey } : {}),
1144
- },
1145
- });
1146
- }
1147
-
1148
- private async resolveToolCall(
1149
- toolId: string,
1150
- request: ToolCallRequest,
1151
- result: unknown | null,
1152
- metadata?: ToolResultMetadataInput | null,
1153
- ): Promise<void> {
1154
- const cacheKey = this.buildToolResultCacheKey({
1155
- rowId: request.rowId,
1156
- tableNamespace: request.tableNamespace,
1157
- rowKey: request.rowKey ?? undefined,
1158
- callId: request.callId,
1159
- });
1160
- const wrapped = await this.wrapToolExecutionResult({
1161
- toolId,
1162
- status: result == null ? 'no_result' : 'completed',
1163
- result,
1164
- metadata,
1165
- cached: false,
1166
- source: 'live',
1167
- cacheKey,
1168
- });
1169
- this.cacheToolResult(toolId, cacheKey, wrapped);
1170
-
1171
- const resolver = this.toolCallResolvers.get(request.callId);
1172
- if (resolver) {
1173
- resolver(wrapped);
1174
- this.toolCallResolvers.delete(request.callId);
1175
- }
1176
-
1177
- this.emitScopedFieldMetaUpdate({
1178
- rowId: request.rowId,
1179
- key: request.rowKey ?? null,
1180
- tableNamespace: request.tableNamespace ?? null,
1181
- fieldName: request.fieldName,
1182
- status: 'running',
1183
- rowStatus: 'running',
1184
- stage: toolId,
1185
- provider: null,
1186
- error: null,
1187
- dataPatch: {},
1188
- });
1189
- }
1190
-
1191
- private pulseProgressHeartbeat(force = false): void {
1192
- if (!this.options.onBatchComplete) {
1193
- return;
1194
- }
1195
- const now = Date.now();
1196
- if (
1197
- !force &&
1198
- now - this.lastProgressHeartbeatAt < PROGRESS_HEARTBEAT_INTERVAL_MS
1199
- ) {
1200
- return;
1201
- }
1202
- this.lastProgressHeartbeatAt = now;
1203
- this.options.onBatchComplete(this.checkpoint);
1204
- }
1205
-
1206
- // ——— Public ctx API ———
1207
-
1208
- async csv(
1209
- path: string,
1210
- _options?: CsvOptions,
1211
- ): Promise<PlayDataset<Record<string, unknown>>> {
1212
- void _options;
1213
- // In cloud mode, CSV data is passed in — path is just a label
1214
- // The activity loads the actual data before creating the ctx
1215
- throw new Error(
1216
- `ctx.csv("${path}") is handled by the workflow. ` +
1217
- `CSV data is loaded as a Temporal activity and passed to your play via input.`,
1218
- );
1219
- }
1220
-
1221
- map<T extends Record<string, unknown>>(
1222
- key: string,
1223
- items: PlayDatasetInput<T>,
1224
- ): RuntimeMapBuilder<T>;
1225
- map<T extends Record<string, unknown>>(
1226
- key: string,
1227
- items: PlayDatasetInput<T>,
1228
- input: RuntimeStepProgram,
1229
- options?: MapOptions<T>,
1230
- ): Promise<PlayDataset<Record<string, unknown>>>;
1231
- map<
1232
- T extends Record<string, unknown>,
1233
- TColumns extends Record<string, unknown> = Record<string, unknown>,
1234
- >(
1235
- key: string,
1236
- items: PlayDatasetInput<T>,
1237
- input?: MapFieldDefinition<T, TColumns> | RuntimeStepProgram,
1238
- options?: MapOptions<T>,
1239
- ): RuntimeMapBuilder<T> | Promise<PlayDataset<Record<string, unknown>>> {
1240
- if (rowContext.getStore()) {
1241
- throw new Error(
1242
- 'Nested ctx.map() is not supported. Flatten your fields into one map, or keep custom per-row logic inside a single map field.',
1243
- );
1244
- }
1245
- this.assertNoDuplicateConcurrentMapBackedPlay();
1246
-
1247
- if (input === undefined) {
1248
- return new RuntimeMapBuilder(this, key, items);
1249
- }
1250
-
1251
- if (isRuntimeStepProgram(input)) {
1252
- return this.runStepProgramMap(key, items, input, options);
1253
- }
1254
-
1255
- throw new Error(
1256
- 'ctx.map(key, rows, fields, options) was removed. Use ctx.map(key, rows).step(...).run(options).',
1257
- );
1258
- }
1259
-
1260
- async runStepProgramMap<T extends Record<string, unknown>>(
1261
- key: string,
1262
- items: PlayDatasetInput<T>,
1263
- program: RuntimeStepProgram,
1264
- options?: MapOptions<T>,
1265
- ): Promise<PlayDataset<Record<string, unknown>>> {
1266
- return this.runMapDefinition(
1267
- key,
1268
- items,
1269
- this.stepProgramToMapDefinition(program),
1270
- options,
1271
- );
1272
- }
1273
-
1274
- private async runMapDefinition<
1275
- T extends Record<string, unknown>,
1276
- TColumns extends Record<string, unknown> = Record<string, unknown>,
1277
- >(
1278
- key: string,
1279
- items: PlayDatasetInput<T>,
1280
- input: MapFieldDefinition<T, TColumns>,
1281
- options?: MapOptions<T>,
1282
- ): Promise<PlayDataset<Record<string, unknown>>> {
1283
- if (rowContext.getStore()) {
1284
- throw new Error(
1285
- 'Nested ctx.map() is not supported. Flatten your fields into one map, or keep custom per-row logic inside a single map field.',
1286
- );
1287
- }
1288
- this.assertNoDuplicateConcurrentMapBackedPlay();
1289
-
1290
- const normalizedMapKey = this.normalizeContextKey(key, 'map');
1291
-
1292
- const normalizedMapNamespace = normalizeTableNamespace(normalizedMapKey);
1293
- const staleAfterSeconds = options?.staleAfterSeconds ?? null;
1294
- try {
1295
- resolveStaleMapTableNamespace(normalizedMapNamespace, staleAfterSeconds);
1296
- } catch (error) {
1297
- throw new Error(
1298
- error instanceof Error
1299
- ? `${error.message} Example: ctx.map(\"${normalizedMapNamespace}\", rows, { company: ... }, { staleAfterSeconds: 86400 }).`
1300
- : `ctx.map() key must normalize to <= ${MAP_KEY_NAMESPACE_MAX_LENGTH} characters.`,
1301
- );
1302
- }
1303
- if (this.explicitMapInvocationKeys.has(normalizedMapNamespace)) {
1304
- throw new Error(
1305
- `Duplicate ctx.map() key "${normalizedMapNamespace}" in the same play. ` +
1306
- 'Each ctx.map() call must use a distinct idempotency key.',
1307
- );
1308
- }
1309
- this.explicitMapInvocationKeys.add(normalizedMapNamespace);
1310
-
1311
- let resolvedTableNamespace = resolveStaleMapTableNamespace(
1312
- normalizedMapNamespace,
1313
- staleAfterSeconds,
1314
- );
1315
- let totalInputCount = 0;
1316
- let rawItems: Record<string, unknown>[] = [];
1317
- let itemsToProcess: Array<Record<string, unknown>> = [];
1318
- let completedItemsByKey: Map<string, Record<string, unknown>> | null = null;
1319
- const mapFieldNames = Object.keys(input);
1320
- const stripFieldOutputs = (row: Record<string, unknown>) =>
1321
- Object.fromEntries(
1322
- Object.entries(row).filter(
1323
- ([fieldName]) => !mapFieldNames.includes(fieldName),
1324
- ),
1325
- );
1326
-
1327
- const userKeyOption = options?.key;
1328
- let explicitKeyResolver:
1329
- | ((row: Record<string, unknown>, index: number) => string)
1330
- | null = null;
1331
- if (userKeyOption !== undefined) {
1332
- explicitKeyResolver = (row, index) => {
1333
- const inputRow = stripFieldOutputs(row);
1334
- const raw =
1335
- typeof userKeyOption === 'function'
1336
- ? (
1337
- userKeyOption as (
1338
- row: Record<string, unknown>,
1339
- index: number,
1340
- ) => string | number | readonly unknown[]
1341
- )(inputRow, index)
1342
- : Array.isArray(userKeyOption)
1343
- ? userKeyOption.map((fieldName) => inputRow[fieldName])
1344
- : inputRow[String(userKeyOption)];
1345
- const parts = Array.isArray(raw) ? raw : [raw];
1346
- if (parts.some((part) => part === null || part === undefined)) {
1347
- throw new Error(
1348
- `ctx.map("${normalizedMapNamespace}") key returned null or undefined for row ${index}. ` +
1349
- 'Return a non-empty string or number derived from a stable input column (e.g. row.email ?? row.domain).',
1350
- );
1351
- }
1352
- const normalizedParts = parts.map((part) => {
1353
- if (typeof part === 'number') {
1354
- return Number.isFinite(part) ? String(part) : '';
1355
- }
1356
- return String(part).trim();
1357
- });
1358
- if (normalizedParts.some((part) => !part)) {
1359
- throw new Error(
1360
- `ctx.map("${normalizedMapNamespace}") key returned an empty value for row ${index}. ` +
1361
- 'Return a non-empty string or finite number derived from a stable input column.',
1362
- );
1363
- }
1364
- return normalizedParts.length === 1
1365
- ? normalizedParts[0]!
1366
- : JSON.stringify(normalizedParts);
1367
- };
1368
- }
1369
-
1370
- const rowIdentity = (row: Record<string, unknown>, index = 0) =>
1371
- explicitKeyResolver
1372
- ? derivePlayRowIdentityFromKey(
1373
- explicitKeyResolver(row, index),
1374
- resolvedTableNamespace,
1375
- )
1376
- : derivePlayRowIdentity(
1377
- stripCsvProjectedFields(stripFieldOutputs(row)),
1378
- resolvedTableNamespace,
1379
- );
1380
-
1381
- const materializedItems = await materializePlayDatasetInput(items);
1382
- totalInputCount = materializedItems.length;
1383
- rawItems = materializedItems.map((item) => this.toOutputRow(item));
1384
- itemsToProcess = materializedItems.map((item) =>
1385
- this.toOutputRow(item as Record<string, unknown>),
1386
- );
1387
-
1388
- if (explicitKeyResolver) {
1389
- const seenKeys = new Map<string, number>();
1390
- for (let index = 0; index < rawItems.length; index += 1) {
1391
- const keyValue = explicitKeyResolver(rawItems[index]!, index);
1392
- const previousIndex = seenKeys.get(keyValue);
1393
- if (previousIndex !== undefined) {
1394
- throw new Error(
1395
- `ctx.map("${normalizedMapNamespace}") key function produced duplicate value "${keyValue}" for rows ${previousIndex} and ${index}. ` +
1396
- 'Each row must produce a unique key. Combine columns (e.g. `${row.email}|${row.company}`) or pick a column that is unique per row.',
1397
- );
1398
- }
1399
- seenKeys.set(keyValue, index);
1400
- }
1401
- }
1402
- if (this.options.onMapStart) {
1403
- const mapStartRows = explicitKeyResolver
1404
- ? rawItems.map((row, index) => ({
1405
- ...row,
1406
- __deeplineRowKey: rowIdentity(row, index),
1407
- }))
1408
- : rawItems;
1409
- const mapStartResult = await this.options.onMapStart(
1410
- mapStartRows,
1411
- resolvedTableNamespace,
1412
- {
1413
- playName: this.options.playName,
1414
- playId: this.options.playId,
1415
- runId: this.options.runId,
1416
- staticPipeline: this.options.staticPipeline,
1417
- },
1418
- );
1419
- resolvedTableNamespace = normalizeTableNamespace(
1420
- mapStartResult.tableNamespace,
1421
- );
1422
- const persistedRowIdentity = (
1423
- row: Record<string, unknown>,
1424
- index = 0,
1425
- ) =>
1426
- typeof row.__deeplineRowKey === 'string'
1427
- ? row.__deeplineRowKey
1428
- : rowIdentity(row, index);
1429
- const pendingKeys = new Set(
1430
- mapStartResult.pendingRows.map((row, index) =>
1431
- persistedRowIdentity(row, index),
1432
- ),
1433
- );
1434
- itemsToProcess = materializedItems
1435
- .map((item, index) => ({
1436
- row: this.toOutputRow(item as Record<string, unknown>),
1437
- index,
1438
- }))
1439
- .filter(({ row, index }) => pendingKeys.has(rowIdentity(row, index)))
1440
- .map(({ row }) => row);
1441
- if (mapStartResult.completedRows.length > 0) {
1442
- completedItemsByKey = new Map();
1443
- for (let index = 0; index < mapStartResult.completedRows.length; index += 1) {
1444
- const row = mapStartResult.completedRows[index]!;
1445
- const rowKey = persistedRowIdentity(row, index);
1446
- if (rowKey) completedItemsByKey.set(rowKey, row);
1447
- }
1448
- this.log(
1449
- `Resuming: ${mapStartResult.completedRows.length} already completed, ${itemsToProcess.length} pending`,
1450
- );
1451
- }
1452
- }
1453
-
1454
- const mapScope = this.createMapExecutionScope({
1455
- logicalNamespace: normalizedMapNamespace,
1456
- artifactTableNamespace: resolvedTableNamespace,
1457
- explicitKey: explicitKeyResolver,
1458
- });
1459
- const completedRowKeys =
1460
- completedItemsByKey != null ? [...completedItemsByKey.keys()] : [];
1461
- const pendingRowKeys = itemsToProcess.map((item, index) =>
1462
- mapScope.rowIdentity(this.toOutputRow(item), index),
1463
- );
1464
- this.setMapFrame({
1465
- mapInvocationId: mapScope.mapInvocationId,
1466
- mapNodeId: mapScope.mapNodeId ?? null,
1467
- logicalNamespace: mapScope.logicalNamespace,
1468
- artifactTableNamespace: mapScope.artifactTableNamespace,
1469
- status: 'running',
1470
- totalRows: totalInputCount,
1471
- completedRowKeys,
1472
- pendingRowKeys,
1473
- startedAt: Date.now(),
1474
- updatedAt: Date.now(),
1475
- });
1476
- this.emitExecutionEvent({
1477
- type: 'map.started',
1478
- mapInvocationId: mapScope.mapInvocationId,
1479
- mapNodeId: mapScope.mapNodeId ?? null,
1480
- logicalNamespace: mapScope.logicalNamespace,
1481
- artifactTableNamespace: mapScope.artifactTableNamespace,
1482
- totalRows: totalInputCount,
1483
- completedRows: completedRowKeys.length,
1484
- pendingRows: pendingRowKeys.length,
1485
- at: Date.now(),
1486
- });
1487
-
1488
- const mapResults = await this.runFieldMap(
1489
- itemsToProcess,
1490
- mapScope,
1491
- input as MapFieldDefinition<Record<string, unknown>>,
1492
- options?.description,
1493
- {
1494
- totalRows: totalInputCount,
1495
- completedRows: completedItemsByKey?.size ?? 0,
1496
- },
1497
- );
1498
-
1499
- const results =
1500
- completedItemsByKey && completedItemsByKey.size > 0
1501
- ? (() => {
1502
- const resultsByKey = new Map<string, Record<string, unknown>>();
1503
- for (let index = 0; index < mapResults.length; index += 1) {
1504
- const row = mapResults[index]!;
1505
- const rowIdentityRow = this.toOutputRow(
1506
- itemsToProcess[index] as Record<string, unknown>,
1507
- );
1508
- const rowKey = rowIdentity(rowIdentityRow, index);
1509
- if (rowKey) resultsByKey.set(rowKey, row);
1510
- }
1511
- return rawItems.map((rawItem, index) => {
1512
- const rowKey = rowIdentity(rawItem, index);
1513
- return this.toPublicOutputRow(
1514
- resultsByKey.get(rowKey) ??
1515
- completedItemsByKey!.get(rowKey) ??
1516
- rawItem,
1517
- );
1518
- });
1519
- })()
1520
- : mapResults.map((row) => this.toPublicOutputRow(row));
1521
-
1522
- return createDeferredPlayDataset({
1523
- datasetKind: 'map',
1524
- datasetId: createRuntimeDatasetId(
1525
- this.options.playName ?? this.options.playId ?? 'play',
1526
- resolvedTableNamespace,
1527
- ),
1528
- count: results.length,
1529
- previewRows: results.slice(0, 5),
1530
- tableNamespace: resolvedTableNamespace,
1531
- resolvers: {
1532
- count: async () => results.length,
1533
- peek: async (limit) => results.slice(0, Math.max(0, limit)),
1534
- materialize: async (limit) =>
1535
- limit === undefined
1536
- ? [...results]
1537
- : results.slice(0, Math.max(0, limit)),
1538
- iterate: () =>
1539
- ({
1540
- async *[Symbol.asyncIterator]() {
1541
- for (const row of results) {
1542
- yield row;
1543
- }
1544
- },
1545
- }) as AsyncIterable<Record<string, unknown>>,
1546
- },
1547
- });
1548
- }
1549
-
1550
- private async runFieldMap<T extends Record<string, unknown>>(
1551
- items: T[],
1552
- mapScope: MapExecutionScope,
1553
- definition: MapFieldDefinition<T>,
1554
- description?: string,
1555
- executionSummary?: {
1556
- totalRows: number;
1557
- completedRows: number;
1558
- },
1559
- runtimeOptions?: {
1560
- emitTerminalEvent?: boolean;
1561
- },
1562
- ): Promise<Array<Record<string, unknown>>> {
1563
- const fieldEntries = Object.entries(definition);
1564
- const mapFieldNames = fieldEntries.map(([fieldName]) => fieldName);
1565
- const visibleFields = fieldEntries
1566
- .map(([fieldName]) => fieldName)
1567
- .filter((fieldName) => !fieldName.startsWith('_'));
1568
- const normalizedTableNamespace = mapScope.artifactTableNamespace;
1569
- const rowIdentity = (row: Record<string, unknown>, index = 0) =>
1570
- mapScope.rowIdentity(
1571
- stripCsvProjectedFields(
1572
- Object.fromEntries(
1573
- Object.entries(row).filter(
1574
- ([fieldName]) => !mapFieldNames.includes(fieldName),
1575
- ),
1576
- ),
1577
- ),
1578
- index,
1579
- );
1580
-
1581
- const totalRows = Math.max(
1582
- executionSummary?.totalRows ?? items.length,
1583
- items.length,
1584
- );
1585
- const completedRows = Math.min(
1586
- executionSummary?.completedRows ?? 0,
1587
- totalRows,
1588
- );
1589
- const pendingRows = Math.max(0, items.length);
1590
- const emitTerminalEvent = runtimeOptions?.emitTerminalEvent !== false;
1591
-
1592
- if (completedRows > 0 || pendingRows !== totalRows) {
1593
- this.log(
1594
- `Starting map over ${totalRows} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace}; ${completedRows} already satisfied; ${pendingRows} pending)`,
1595
- );
1596
- } else {
1597
- this.log(
1598
- `Starting map over ${items.length} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace})`,
1599
- );
1600
- }
1601
- this.processedRowCount = items.length;
1602
- const mapStep: Extract<PlayStep, { type: 'map' }> = {
1603
- type: 'map',
1604
- items: items.length,
1605
- fields: visibleFields,
1606
- substeps: [],
1607
- description: normalizeStepDescription(description),
1608
- };
1609
- this.steps.push(mapStep);
1610
- this.activeMapStep = mapStep;
1611
-
1612
- const updateMapFrameProgress = (input: {
1613
- status?: MapExecutionFrame['status'];
1614
- completedRowKey?: string | null;
1615
- pendingRowKey?: string | null;
1616
- activeBoundaryId?: string | null;
1617
- failedDelta?: number;
1618
- emitEventType?: PlayExecutionEvent['type'];
1619
- }) => {
1620
- const existing =
1621
- this.checkpoint.mapFrames?.[mapScope.mapInvocationId] ?? null;
1622
- if (!existing) {
1623
- return;
1624
- }
1625
- const completedRowKeys = new Set(existing.completedRowKeys);
1626
- const pendingRowKeys = new Set(existing.pendingRowKeys);
1627
- if (input.completedRowKey?.trim()) {
1628
- completedRowKeys.add(input.completedRowKey.trim());
1629
- pendingRowKeys.delete(input.completedRowKey.trim());
1630
- }
1631
- if (input.pendingRowKey?.trim()) {
1632
- pendingRowKeys.add(input.pendingRowKey.trim());
1633
- }
1634
- const nextFrame: MapExecutionFrame = {
1635
- ...existing,
1636
- status: input.status ?? existing.status,
1637
- completedRowKeys: [...completedRowKeys],
1638
- pendingRowKeys: [...pendingRowKeys],
1639
- ...(input.activeBoundaryId !== undefined
1640
- ? { activeBoundaryId: input.activeBoundaryId }
1641
- : {}),
1642
- updatedAt: Date.now(),
1643
- };
1644
- this.setMapFrame(nextFrame);
1645
- if (input.emitEventType) {
1646
- this.emitExecutionEvent({
1647
- type: input.emitEventType,
1648
- mapInvocationId: mapScope.mapInvocationId,
1649
- mapNodeId: mapScope.mapNodeId ?? null,
1650
- logicalNamespace: mapScope.logicalNamespace,
1651
- artifactTableNamespace: mapScope.artifactTableNamespace,
1652
- completedRows: nextFrame.completedRowKeys.length,
1653
- failedRows: Math.max(
1654
- 0,
1655
- totalRows -
1656
- nextFrame.completedRowKeys.length -
1657
- nextFrame.pendingRowKeys.length,
1658
- ),
1659
- totalRows,
1660
- at: Date.now(),
1661
- } as PlayExecutionEvent);
1662
- }
1663
- };
1664
-
1665
- if (this.canUsePureJsMapFastPath(definition)) {
1666
- const results = await this.runPureFieldMap(
1667
- items,
1668
- fieldEntries,
1669
- visibleFields,
1670
- normalizedTableNamespace,
1671
- (row, index) => mapScope.rowIdentity(row, index),
1672
- );
1673
- for (let index = 0; index < results.length; index += 1) {
1674
- const row = results[index]!;
1675
- updateMapFrameProgress({
1676
- completedRowKey: rowIdentity(row, index),
1677
- });
1678
- }
1679
- if (emitTerminalEvent) {
1680
- updateMapFrameProgress({
1681
- status: 'completed',
1682
- activeBoundaryId: null,
1683
- emitEventType: 'map.completed',
1684
- });
1685
- }
1686
- this.lastMapStep = mapStep;
1687
- this.activeMapStep = null;
1688
- this.log(
1689
- `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} already satisfied)`,
1690
- );
1691
- return results;
1692
- }
1693
-
1694
- this.initializeRowStates(items);
1695
-
1696
- const promises = items.map(async (item, idx) => {
1697
- const baseRow = this.toOutputRow(item);
1698
- const rowKey = rowIdentity(baseRow, idx);
1699
- const computedFields: Record<string, unknown> = {};
1700
- let activeFieldName: string | null = null;
1701
-
1702
- try {
1703
- for (const [fieldName, resolver] of fieldEntries) {
1704
- activeFieldName = fieldName;
1705
- if (this.shouldReuseExistingFieldValue(baseRow, fieldName)) {
1706
- const reusedValue = baseRow[fieldName];
1707
- computedFields[fieldName] = reusedValue;
1708
- this.rowStates.get(idx)?.results.set(fieldName, reusedValue);
1709
- this.emitScopedFieldMetaUpdate({
1710
- rowId: idx,
1711
- key: rowKey,
1712
- tableNamespace: normalizedTableNamespace,
1713
- fieldName,
1714
- status: 'cached',
1715
- reused: true,
1716
- dataPatch: {},
1717
- });
1718
- continue;
1719
- }
1720
-
1721
- this.emitScopedFieldMetaUpdate({
1722
- rowId: idx,
1723
- key: rowKey,
1724
- tableNamespace: normalizedTableNamespace,
1725
- fieldName,
1726
- status: 'running',
1727
- rowStatus: 'running',
1728
- stage: fieldName,
1729
- dataPatch: {},
1730
- });
1731
-
1732
- const value = await rowContext.run(
1733
- {
1734
- rowId: idx,
1735
- fieldName,
1736
- tableNamespace: normalizedTableNamespace,
1737
- rowKey,
1738
- mapScope,
1739
- },
1740
- async () =>
1741
- await this.resolveMapFieldValue(
1742
- resolver,
1743
- item,
1744
- cloneCsvAliasedRow(baseRow, computedFields),
1745
- idx,
1746
- ),
1747
- );
1748
- computedFields[fieldName] = value;
1749
- this.rowStates.get(idx)?.results.set(fieldName, value);
1750
- this.emitScopedFieldMetaUpdate({
1751
- rowId: idx,
1752
- key: rowKey,
1753
- tableNamespace: normalizedTableNamespace,
1754
- fieldName,
1755
- status: 'completed',
1756
- stage: 'completed',
1757
- dataPatch: fieldName.startsWith('_') ? {} : { [fieldName]: value },
1758
- });
1759
- }
1760
-
1761
- const merged = cloneCsvAliasedRow(baseRow, computedFields);
1762
- activeFieldName = null;
1763
- updateMapFrameProgress({
1764
- completedRowKey: rowKey,
1765
- });
1766
- this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
1767
- rowId: idx,
1768
- status: 'completed',
1769
- stage: 'completed',
1770
- provider: null,
1771
- error: null,
1772
- dataPatch: {},
1773
- });
1774
- return this.toPublicOutputRow(merged);
1775
- } catch (error) {
1776
- if (isPlayRowExecutionSuspendedError(error)) {
1777
- this.pendingRowEventBoundaries.push(error.boundary);
1778
- this.rowStates.get(idx)!.status = 'waiting';
1779
- updateMapFrameProgress({
1780
- pendingRowKey: rowKey,
1781
- activeBoundaryId: error.boundary.boundaryId,
1782
- emitEventType: 'map.suspended',
1783
- });
1784
- const fieldName = rowContext.getStore()?.fieldName ?? activeFieldName;
1785
- this.emitScopedFieldMetaUpdate({
1786
- rowId: idx,
1787
- key: rowKey,
1788
- tableNamespace: normalizedTableNamespace,
1789
- fieldName,
1790
- status: 'running',
1791
- rowStatus: 'running',
1792
- stage: 'waiting_for_event',
1793
- provider: null,
1794
- error: null,
1795
- dataPatch: {},
1796
- });
1797
- return WAITING_ROW;
1798
- }
1799
- const fieldName = rowContext.getStore()?.fieldName;
1800
- updateMapFrameProgress({
1801
- status: 'failed',
1802
- emitEventType: 'map.failed',
1803
- });
1804
- this.emitScopedFieldMetaUpdate({
1805
- rowId: idx,
1806
- key: rowKey,
1807
- tableNamespace: normalizedTableNamespace,
1808
- fieldName: fieldName ?? activeFieldName,
1809
- status: 'failed',
1810
- rowStatus: 'failed',
1811
- stage: 'failed',
1812
- provider: null,
1813
- error: this.formatRuntimeError(error),
1814
- dataPatch: {},
1815
- });
1816
- throw error;
1817
- }
1818
- });
1819
-
1820
- await this.drainQueuedWork(promises);
1821
- this.lastMapStep = mapStep;
1822
- this.activeMapStep = null;
1823
-
1824
- const settledResults = await Promise.all(promises);
1825
- if (this.pendingRowEventBoundaries.length > 0) {
1826
- const completedRowKeys = new Set<string>();
1827
- for (let index = 0; index < settledResults.length; index += 1) {
1828
- const result = settledResults[index];
1829
- const rawItem = this.toOutputRow(
1830
- items[index] as Record<string, unknown>,
1831
- );
1832
- if (result === WAITING_ROW) {
1833
- const key = rowIdentity(rawItem, index);
1834
- if (key) {
1835
- completedRowKeys.add(key);
1836
- }
1837
- continue;
1838
- }
1839
- const row = result as Record<string, unknown>;
1840
- const key = rowIdentity(row, index);
1841
- if (key) completedRowKeys.add(key);
1842
- }
1843
- this.setMapFrame({
1844
- ...(this.checkpoint.mapFrames?.[mapScope.mapInvocationId] ?? {
1845
- mapInvocationId: mapScope.mapInvocationId,
1846
- logicalNamespace: mapScope.logicalNamespace,
1847
- artifactTableNamespace: mapScope.artifactTableNamespace,
1848
- status: 'suspended' as const,
1849
- totalRows,
1850
- completedRowKeys: [...completedRowKeys],
1851
- pendingRowKeys: items.map((item, index) =>
1852
- rowIdentity(this.toOutputRow(item as Record<string, unknown>), index),
1853
- ),
1854
- startedAt: Date.now(),
1855
- updatedAt: Date.now(),
1856
- }),
1857
- status: 'suspended',
1858
- completedRowKeys: [...completedRowKeys],
1859
- updatedAt: Date.now(),
1860
- });
1861
- const uniqueBoundaries = [
1862
- ...new Map(
1863
- this.pendingRowEventBoundaries.map((boundary) => [
1864
- boundary.boundaryId,
1865
- boundary,
1866
- ]),
1867
- ).values(),
1868
- ];
1869
- this.pendingRowEventBoundaries = [];
1870
- this.options.onBatchComplete?.(this.checkpoint);
1871
- throw new PlayExecutionSuspendedError({
1872
- kind: 'integration_event_batch',
1873
- boundaries: uniqueBoundaries,
1874
- });
1875
- }
1876
- const results = settledResults.filter(
1877
- (result): result is Record<string, unknown> => result !== WAITING_ROW,
1878
- );
1879
- for (let index = 0; index < results.length; index += 1) {
1880
- const row = results[index]!;
1881
- updateMapFrameProgress({
1882
- completedRowKey: rowIdentity(row, index),
1883
- });
1884
- }
1885
- if (emitTerminalEvent) {
1886
- updateMapFrameProgress({
1887
- status: 'completed',
1888
- activeBoundaryId: null,
1889
- emitEventType: 'map.completed',
1890
- });
1891
- }
1892
- this.log(
1893
- `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} already satisfied)`,
1894
- );
1895
- return results;
1896
- }
1897
-
1898
- private stepProgramToMapDefinition(
1899
- program: RuntimeStepProgram,
1900
- ): MapFieldDefinition<Record<string, unknown>> {
1901
- const definition: MapFieldDefinition<Record<string, unknown>> = {};
1902
- for (const step of program.steps) {
1903
- const resolver: MapFieldResolver<
1904
- Record<string, unknown>,
1905
- Record<string, unknown>
1906
- > = async (
1907
- _row: Record<string, unknown>,
1908
- _ctx: unknown,
1909
- currentRow: Record<string, unknown>,
1910
- index: number,
1911
- ) =>
1912
- await this.executeStepProgramStep(step, currentRow, index, [step.name]);
1913
- definition[step.name] = resolver;
1914
- }
1915
- Object.defineProperty(definition, STEP_PROGRAM_MAP_DEFINITION, {
1916
- value: true,
1917
- enumerable: false,
1918
- });
1919
- return definition;
1920
- }
1921
-
1922
- private async executeStepProgram(
1923
- program: RuntimeStepProgram,
1924
- row: Record<string, unknown>,
1925
- index: number,
1926
- path: string[],
1927
- ): Promise<unknown> {
1928
- let currentRow = cloneCsvAliasedRow(row);
1929
- const produced: Record<string, unknown> = {};
1930
- for (const step of program.steps) {
1931
- const value = await this.executeStepProgramStep(step, currentRow, index, [
1932
- ...path,
1933
- step.name,
1934
- ]);
1935
- produced[step.name] = value;
1936
- currentRow = cloneCsvAliasedRow(currentRow, { [step.name]: value });
1937
- }
1938
- if (typeof program.returnResolver === 'function') {
1939
- return await program.returnResolver(currentRow, this, index);
1940
- }
1941
- return produced;
1942
- }
1943
-
1944
- private async executeStepProgramStep(
1945
- step: RuntimeStepProgramStep,
1946
- currentRow: Record<string, unknown>,
1947
- index: number,
1948
- path: string[],
1949
- ): Promise<unknown> {
1950
- const resolver = step.resolver;
1951
- const store = rowContext.getStore();
1952
- const nestedFieldName = path.join('.');
1953
- const runWithStepScope = async (run: () => Promise<unknown>) => {
1954
- if (!store) return await run();
1955
- return await rowContext.run(
1956
- {
1957
- ...store,
1958
- fieldName: nestedFieldName,
1959
- },
1960
- run,
1961
- );
1962
- };
1963
-
1964
- if (isRuntimeStepProgram(resolver)) {
1965
- return await runWithStepScope(
1966
- async () =>
1967
- await this.executeStepProgram(resolver, currentRow, index, path),
1968
- );
1969
- }
1970
-
1971
- if (isRuntimeConditionalStepResolver(resolver)) {
1972
- const shouldRun = await resolver.when(currentRow, index);
1973
- if (!shouldRun) {
1974
- const elseValue = Object.prototype.hasOwnProperty.call(
1975
- resolver,
1976
- 'elseValue',
1977
- )
1978
- ? resolver.elseValue
1979
- : null;
1980
- if (store) {
1981
- this.emitScopedFieldMetaUpdate({
1982
- rowId: store.rowId,
1983
- key: store.rowKey ?? null,
1984
- tableNamespace: store.tableNamespace ?? null,
1985
- fieldName: nestedFieldName,
1986
- status: 'skipped',
1987
- rowStatus: 'running',
1988
- stage: 'skipped',
1989
- provider: null,
1990
- error: null,
1991
- dataPatch: {},
1992
- });
1993
- }
1994
- return elseValue;
1995
- }
1996
- return await runWithStepScope(
1997
- async () => await resolver.run(currentRow, this, index),
1998
- );
1999
- }
2000
-
2001
- if (typeof resolver !== 'function') {
2002
- return resolver;
2003
- }
2004
-
2005
- return await runWithStepScope(
2006
- async () => await resolver(currentRow, this, index),
2007
- );
2008
- }
2009
-
2010
- private canUsePureJsMapFastPath<T>(
2011
- definition: MapFieldDefinition<T>,
2012
- ): boolean {
2013
- if (
2014
- (definition as Record<PropertyKey, unknown>)[STEP_PROGRAM_MAP_DEFINITION]
2015
- ) {
2016
- return false;
2017
- }
2018
- return Object.values(definition).every((resolver) => {
2019
- if (typeof resolver !== 'function') {
2020
- return true;
2021
- }
2022
-
2023
- const source = Function.prototype.toString.call(resolver);
2024
- return (
2025
- !source.includes('.tools.execute(') && !source.includes('.waterfall(')
2026
- );
2027
- });
2028
- }
2029
-
2030
- private async runPureFieldMap<T>(
2031
- items: T[],
2032
- fieldEntries: [string, MapFieldDefinition<T>[string]][],
2033
- visibleFields: string[],
2034
- tableNamespace: string,
2035
- rowIdentity: (row: Record<string, unknown>, index: number) => string,
2036
- ): Promise<Array<Record<string, unknown>>> {
2037
- const results: Array<Record<string, unknown>> = [];
2038
- this.pureMapExecutionActive = true;
2039
-
2040
- try {
2041
- for (let index = 0; index < items.length; index += 1) {
2042
- const item = items[index]!;
2043
- const baseRow = this.toOutputRow(item);
2044
- const computedFields: Record<string, unknown> = {};
2045
- let activeFieldName: string | null = null;
2046
-
2047
- try {
2048
- for (const [fieldName, resolver] of fieldEntries) {
2049
- activeFieldName = fieldName;
2050
- if (this.shouldReuseExistingFieldValue(baseRow, fieldName)) {
2051
- computedFields[fieldName] = baseRow[fieldName];
2052
- this.emitScopedRowUpdate(rowIdentity(baseRow, index), tableNamespace, {
2053
- rowId: index,
2054
- status: undefined,
2055
- stage: null,
2056
- provider: null,
2057
- error: null,
2058
- dataPatch: {},
2059
- cellMetaPatch: {
2060
- [fieldName]: {
2061
- status: 'cached',
2062
- reused: true,
2063
- },
2064
- },
2065
- });
2066
- continue;
2067
- }
2068
-
2069
- this.emitScopedRowUpdate(rowIdentity(baseRow, index), tableNamespace, {
2070
- rowId: index,
2071
- status: 'running',
2072
- stage: fieldName,
2073
- provider: null,
2074
- error: null,
2075
- dataPatch: {},
2076
- cellMetaPatch: {
2077
- [fieldName]: {
2078
- status: 'running',
2079
- stage: fieldName,
2080
- },
2081
- },
2082
- });
2083
-
2084
- computedFields[fieldName] = await this.resolveMapFieldValue(
2085
- resolver,
2086
- item,
2087
- cloneCsvAliasedRow(baseRow, computedFields),
2088
- index,
2089
- );
2090
- this.emitScopedRowUpdate(rowIdentity(baseRow, index), tableNamespace, {
2091
- rowId: index,
2092
- status: undefined,
2093
- stage: 'completed',
2094
- provider: null,
2095
- error: null,
2096
- dataPatch: fieldName.startsWith('_')
2097
- ? {}
2098
- : { [fieldName]: computedFields[fieldName] },
2099
- cellMetaPatch: {
2100
- [fieldName]: {
2101
- status: 'completed',
2102
- stage: 'completed',
2103
- },
2104
- },
2105
- });
2106
- }
2107
-
2108
- results.push(
2109
- this.toPublicOutputRow(
2110
- cloneCsvAliasedRow(baseRow, computedFields),
2111
- ),
2112
- );
2113
-
2114
- const rowKey = rowIdentity(baseRow, index);
2115
- activeFieldName = null;
2116
- this.emitScopedRowUpdate(rowKey, tableNamespace, {
2117
- rowId: index,
2118
- status: 'completed',
2119
- stage: 'completed',
2120
- provider: null,
2121
- error: null,
2122
- dataPatch: {},
2123
- });
2124
- } catch (error) {
2125
- const rowKey = rowIdentity(baseRow, index);
2126
- this.emitScopedRowUpdate(rowKey, tableNamespace, {
2127
- rowId: index,
2128
- status: 'failed',
2129
- stage: 'failed',
2130
- provider: null,
2131
- error: this.formatRuntimeError(error),
2132
- dataPatch: {},
2133
- cellMetaPatch: {
2134
- [String(activeFieldName ?? '__unknown')]: {
2135
- status: 'failed',
2136
- stage: 'failed',
2137
- error: this.formatRuntimeError(error),
2138
- },
2139
- },
2140
- });
2141
- throw error;
2142
- }
2143
-
2144
- if ((index + 1) % PURE_JS_HEARTBEAT_ROW_INTERVAL === 0) {
2145
- this.pulseProgressHeartbeat();
2146
- }
2147
- }
2148
- } finally {
2149
- this.pulseProgressHeartbeat(true);
2150
- this.pureMapExecutionActive = false;
2151
- }
2152
-
2153
- return results;
2154
- }
2155
-
2156
- private initializeRowStates<T>(items: T[]): void {
2157
- items.forEach((item, idx) => {
2158
- this.rowStates.set(idx, {
2159
- id: idx,
2160
- input: item,
2161
- status: 'pending',
2162
- waterfalls: new Map(),
2163
- toolCalls: new Map(),
2164
- results: new Map(),
2165
- });
2166
- });
2167
- }
2168
-
2169
- private async drainQueuedWork<T>(promises: Promise<T>[]): Promise<void> {
2170
- // Drain loop: each pass resolves queued waterfalls/tool calls.
2171
- // When a batch resolves, rows resume and may queue MORE calls
2172
- // (e.g. row does tools.execute('a') then tools.execute('b') sequentially).
2173
- // We keep looping until nothing new is queued and all rows finish.
2174
- //
2175
- // Important: an unresolved Promise by itself does not keep Node alive. The
2176
- // play runner is a short-lived child process, so parking on
2177
- // Promise.allSettled(promises) while there is no timer/socket/file handle can
2178
- // let Node exit 0 before main() emits the result envelope. That presents as
2179
- // "Local play runner produced no result" even though the real bug was a
2180
- // map row waiting for queued tool/waterfall work. Always race row settlement
2181
- // against a small timer and re-check the queues instead of blocking forever
2182
- // on promises that may be waiting for this drain loop to do the next batch.
2183
- const raceSettled = () =>
2184
- Promise.race([
2185
- Promise.allSettled(promises).then(() => 'done' as const),
2186
- new Promise<'pending'>((r) => setTimeout(() => r('pending'), 50)),
2187
- ]);
2188
-
2189
- let pass = 0;
2190
- while (true) {
2191
- const hasPendingWork =
2192
- this.waterfallQueue.size > 0 || this.toolCallQueue.length > 0;
2193
-
2194
- if (!hasPendingWork) {
2195
- const status = await raceSettled();
2196
- if (
2197
- status === 'done' &&
2198
- this.waterfallQueue.size === 0 &&
2199
- this.toolCallQueue.length === 0
2200
- ) {
2201
- break;
2202
- }
2203
-
2204
- // Promises are still running, or they settled while scheduling one last
2205
- // batch. Give resolver microtasks a turn, then loop back and drain any
2206
- // newly queued work. This timer is deliberately tiny; it is here for
2207
- // correctness/liveness, not throughput throttling.
2208
- await new Promise((r) => setTimeout(r, 10));
2209
- continue;
2210
- }
2211
-
2212
- pass++;
2213
- this.log(` Batch pass ${pass}`);
2214
- this.log(
2215
- ` Queue depth before drain: waterfalls=${this.waterfallQueue.size} tool_calls=${this.toolCallQueue.length}`,
2216
- );
2217
-
2218
- if (this.waterfallQueue.size > 0) {
2219
- await this.executeBatchedWaterfalls();
2220
- }
2221
-
2222
- if (this.toolCallQueue.length > 0) {
2223
- await this.executeBatchedToolCalls();
2224
- }
2225
- }
2226
- }
2227
-
2228
- private async resolveMapFieldValue<T>(
2229
- resolver: MapFieldResolver<T>,
2230
- row: T,
2231
- fields: Record<string, unknown>,
2232
- index: number,
2233
- ): Promise<unknown> {
2234
- if (typeof resolver !== 'function') {
2235
- return resolver;
2236
- }
2237
-
2238
- return await resolver(row, this, fields, index);
2239
- }
2240
-
2241
- private toOutputRow<T>(item: T): Record<string, unknown> {
2242
- if (item != null && typeof item === 'object' && !Array.isArray(item)) {
2243
- return item as Record<string, unknown>;
2244
- }
2245
-
2246
- return { value: item };
2247
- }
2248
-
2249
- private toPublicOutputRow(row: Record<string, unknown>): Record<string, unknown> {
2250
- const stripped = stripCsvProjectedFields(row);
2251
- return Object.fromEntries(
2252
- Object.entries(stripped).filter(([fieldName]) => !fieldName.startsWith('_')),
2253
- );
2254
- }
2255
-
2256
- async waterfall(
2257
- toolNameOrSpec: string | InlineWaterfallSpec,
2258
- input: Record<string, unknown>,
2259
- opts?: WaterfallOptions,
2260
- ): Promise<unknown | null> {
2261
- const store = rowContext.getStore();
2262
- const toolName =
2263
- typeof toolNameOrSpec === 'string' ? toolNameOrSpec : toolNameOrSpec.id;
2264
- const baseQueueKey =
2265
- typeof toolNameOrSpec === 'string'
2266
- ? toolNameOrSpec
2267
- : `inline:${toolNameOrSpec.id}`;
2268
- const inlineSpec = isInlineWaterfallSpec(toolNameOrSpec)
2269
- ? toolNameOrSpec
2270
- : undefined;
2271
-
2272
- if (this.pureMapExecutionActive && !store) {
2273
- throw new Error(
2274
- 'ctx.waterfall() cannot run inside the pure-JS fast path. Call it directly in the map definition so the batching runtime can stay enabled.',
2275
- );
2276
- }
2277
-
2278
- if (!store) {
2279
- return this.executeWaterfallDirect(toolNameOrSpec, input, opts);
2280
- }
2281
-
2282
- const rowId = store.rowId;
2283
- const fieldName = store.fieldName;
2284
- const queueKey = store.tableNamespace?.trim()
2285
- ? `${baseQueueKey}:${store.tableNamespace.trim()}`
2286
- : baseQueueKey;
2287
- const rowState = this.rowStates.get(rowId);
2288
- if (rowState && !rowState.waterfalls.has(toolName)) {
2289
- rowState.waterfalls.set(toolName, {
2290
- status: 'pending',
2291
- providerIndex: 0,
2292
- });
2293
- }
2294
-
2295
- // Check if this was already resolved in a previous attempt (checkpoint)
2296
- const resolved = this.checkpoint.resolvedWaterfalls[queueKey];
2297
- if (resolved && rowId in resolved) {
2298
- this.log(` Row ${rowId} ${toolName}: recovered from checkpoint`);
2299
- return resolved[rowId];
2300
- }
2301
-
2302
- return new Promise((resolve) => {
2303
- const resolverId = `${rowId}-${queueKey}`;
2304
- this.resolvers.set(resolverId, resolve);
2305
-
2306
- this.emitScopedFieldMetaUpdate({
2307
- rowId,
2308
- key: store?.rowKey ?? null,
2309
- tableNamespace: store?.tableNamespace ?? null,
2310
- fieldName,
2311
- status: 'running',
2312
- rowStatus: 'running',
2313
- stage: toolName,
2314
- provider: null,
2315
- error: null,
2316
- dataPatch: {},
2317
- });
2318
- if (inlineSpec) {
2319
- // Inline waterfalls have fully compiled child stages, so we can publish a
2320
- // real queued state for each step immediately instead of forcing the grid
2321
- // to guess from the row's broader status.
2322
- this.emitQueuedInlineWaterfallSteps(
2323
- rowId,
2324
- store?.rowKey ?? null,
2325
- store?.tableNamespace ?? null,
2326
- inlineSpec,
2327
- );
2328
- }
2329
-
2330
- if (!this.waterfallQueue.has(queueKey)) {
2331
- this.waterfallQueue.set(queueKey, []);
2332
- }
2333
- this.waterfallQueue.get(queueKey)!.push({
2334
- rowId,
2335
- fieldName,
2336
- tableNamespace: store?.tableNamespace,
2337
- rowKey: store?.rowKey ?? null,
2338
- key: queueKey,
2339
- toolName,
2340
- input,
2341
- providerIndex: 0,
2342
- opts,
2343
- description: normalizeStepDescription(opts?.description),
2344
- ...(inlineSpec ? { spec: inlineSpec } : {}),
2345
- });
2346
- });
2347
- }
2348
-
2349
- private async executeTool(
2350
- key: string,
2351
- toolId: string,
2352
- input: Record<string, unknown>,
2353
- options?: ToolCallOptions,
2354
- ): Promise<unknown> {
2355
- const normalizedKey = this.normalizeContextKey(key, 'tool');
2356
-
2357
- const eventWaitHandler =
2358
- (await this.options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
2359
- const store = rowContext.getStore();
2360
-
2361
- const executeTool = async (): Promise<unknown> => {
2362
- if (eventWaitHandler) {
2363
- return this.executeIntegrationEventWaitTool(
2364
- toolId,
2365
- input,
2366
- eventWaitHandler,
2367
- );
2368
- }
2369
- if (toolId === 'run_javascript') {
2370
- throw new Error(DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE);
2371
- }
2372
- this.bumpCounter('toolCallCount');
2373
-
2374
- if (this.pureMapExecutionActive && !store) {
2375
- throw new Error(
2376
- 'ctx.tools.execute() cannot run inside the pure-JS fast path. Call it directly in the map definition so the batching runtime can stay enabled.',
2377
- );
2378
- }
2379
-
2380
- if (!store) {
2381
- const directRowId = -(this.directToolCallIndex + 1);
2382
- this.directToolCallIndex += 1;
2383
- const directCacheKey = this.buildToolResultCacheKey({ rowId: directRowId });
2384
- const cached = this.getCachedToolResult(toolId, directCacheKey);
2385
- if (cached?.done) {
2386
- this.log(`Calling tool: ${toolId} recovered from checkpoint`);
2387
- return await this.wrapToolExecutionResult({
2388
- toolId,
2389
- status: cached.result == null ? 'no_result' : 'completed',
2390
- result: cached.result,
2391
- cached: true,
2392
- source: 'checkpoint',
2393
- cacheKey: directCacheKey,
2394
- });
2395
- }
2396
- this.log(`Calling tool: ${toolId}`);
2397
- const execution = await this.callToolExecutionAPI(toolId, input);
2398
- const wrapped = await this.wrapToolExecutionResult({
2399
- toolId,
2400
- status: execution.status,
2401
- result: execution.result,
2402
- metadata: execution.metadata,
2403
- cached: false,
2404
- source: 'live',
2405
- cacheKey: directCacheKey,
2406
- });
2407
- this.checkpoint.completedToolBatches[toolId] = {
2408
- ...(this.checkpoint.completedToolBatches[toolId] ?? {}),
2409
- [directCacheKey]: {
2410
- done: true,
2411
- result: wrapped,
2412
- },
2413
- };
2414
- this.options.onBatchComplete?.(this.checkpoint);
2415
- return wrapped;
2416
- }
2417
-
2418
- const rowId = store.rowId;
2419
- const fieldName = store.fieldName;
2420
- const callId = [
2421
- store.tableNamespace?.trim() || 'map',
2422
- store.rowKey?.trim() || String(rowId),
2423
- normalizedKey,
2424
- toolId,
2425
- ].join(':');
2426
- if (this.toolCallResolvers.has(callId)) {
2427
- throw new Error(
2428
- `ctx.tools.execute("${normalizedKey}") was called more than once concurrently for the same row. ` +
2429
- 'Use a unique id for each row-scoped tool call.',
2430
- );
2431
- }
2432
-
2433
- const cached = this.getCachedToolResult(
2434
- toolId,
2435
- this.buildToolResultCacheKey({
2436
- rowId,
2437
- tableNamespace: store.tableNamespace,
2438
- rowKey: store.rowKey,
2439
- callId,
2440
- }),
2441
- );
2442
- if (cached?.done) {
2443
- this.log(` Row ${rowId} ${toolId}: recovered from checkpoint`);
2444
- return await this.wrapToolExecutionResult({
2445
- toolId,
2446
- status: cached.result == null ? 'no_result' : 'completed',
2447
- result: cached.result,
2448
- cached: true,
2449
- source: 'checkpoint',
2450
- cacheKey: this.buildToolResultCacheKey({
2451
- rowId,
2452
- tableNamespace: store.tableNamespace,
2453
- rowKey: store.rowKey,
2454
- callId,
2455
- }),
2456
- });
2457
- }
2458
-
2459
- return new Promise((resolve) => {
2460
- this.toolCallResolvers.set(callId, resolve);
2461
- this.emitScopedFieldMetaUpdate({
2462
- rowId,
2463
- key: store.rowKey ?? null,
2464
- tableNamespace: store.tableNamespace ?? null,
2465
- fieldName,
2466
- status: 'running',
2467
- rowStatus: 'running',
2468
- stage: toolId,
2469
- provider: null,
2470
- error: null,
2471
- dataPatch: {},
2472
- });
2473
- this.toolCallQueue.push({
2474
- callId,
2475
- rowId,
2476
- fieldName,
2477
- toolId,
2478
- input,
2479
- tableNamespace: store.tableNamespace,
2480
- rowKey: store.rowKey ?? null,
2481
- description: normalizeStepDescription(options?.description),
2482
- });
2483
- });
2484
- };
2485
-
2486
- if (store) {
2487
- return await executeTool();
2488
- }
2489
-
2490
- return this.executeWithRuntimeReceipt(
2491
- 'tool',
2492
- normalizedKey,
2493
- this.currentRunId,
2494
- {
2495
- markSkipped: () => {
2496
- this.log(
2497
- `ctx.tools.execute(${toolId}): no-op due completed receipt ${normalizedKey}`,
2498
- );
2499
- },
2500
- execute: executeTool,
2501
- },
2502
- );
2503
- }
2504
-
2505
- private async executeIntegrationEventWaitTool(
2506
- toolId: string,
2507
- input: Record<string, unknown>,
2508
- handler: IntegrationEventWaitHandler,
2509
- ): Promise<unknown> {
2510
- if (!this.options.durableBoundaries) {
2511
- throw new Error(`${toolId} requires durable play boundaries.`);
2512
- }
2513
-
2514
- const rowScope = rowContext.getStore();
2515
- const eventContext = {
2516
- playId: this.options.playId,
2517
- runId: this.options.runId,
2518
- workflowId: this.options.workflowId,
2519
- orgId: this.options.orgId,
2520
- executorToken: this.options.executorToken,
2521
- };
2522
- const basePreparedBoundary = await handler.prepare({
2523
- payload: input,
2524
- context: eventContext,
2525
- });
2526
- const preparedBoundary = rowScope
2527
- ? {
2528
- ...basePreparedBoundary,
2529
- boundaryId: `map-row-${stableFetchHash(
2530
- JSON.stringify({
2531
- tableNamespace: rowScope.tableNamespace,
2532
- rowKey: rowScope.rowKey,
2533
- fieldName: rowScope.fieldName,
2534
- boundaryId: basePreparedBoundary.boundaryId,
2535
- eventKey: basePreparedBoundary.eventKey,
2536
- }),
2537
- )}`,
2538
- }
2539
- : basePreparedBoundary;
2540
-
2541
- const existing =
2542
- this.checkpoint.resolvedBoundaries?.[preparedBoundary.boundaryId];
2543
- if (existing?.kind === 'integration_event' && 'output' in existing) {
2544
- this.log(
2545
- `Integration event ${preparedBoundary.boundaryId}: recovered response from checkpoint`,
2546
- );
2547
- return existing.output;
2548
- }
2549
-
2550
- const boundary = await handler.arm({
2551
- payload: input,
2552
- context: eventContext,
2553
- boundary: preparedBoundary,
2554
- });
2555
-
2556
- this.log(
2557
- `Armed ${handler.provider} integration event wait: ${boundary.eventKey}`,
2558
- );
2559
- this.checkpoint.resolvedBoundaries = {
2560
- ...(this.checkpoint.resolvedBoundaries ?? {}),
2561
- [boundary.boundaryId]: {
2562
- kind: 'integration_event',
2563
- eventKey: boundary.eventKey,
2564
- timeoutMs: boundary.timeoutMs,
2565
- provider: boundary.provider,
2566
- toolId: boundary.toolId,
2567
- ...(rowScope
2568
- ? {
2569
- scope: {
2570
- type: 'map_row' as const,
2571
- tableNamespace: rowScope.tableNamespace,
2572
- rowKey: rowScope.rowKey,
2573
- rowIndex: rowScope.rowId,
2574
- fieldName: rowScope.fieldName,
2575
- },
2576
- }
2577
- : { scope: { type: 'workflow' as const } }),
2578
- ...(boundary.messageRef ? { messageRef: boundary.messageRef } : {}),
2579
- },
2580
- };
2581
- this.options.onBatchComplete?.(this.checkpoint);
2582
-
2583
- if (rowScope) {
2584
- throw new PlayRowExecutionSuspendedError({
2585
- boundaryId: boundary.boundaryId,
2586
- eventKey: boundary.eventKey,
2587
- timeoutMs: boundary.timeoutMs,
2588
- });
2589
- }
2590
-
2591
- throw new PlayExecutionSuspendedError({
2592
- kind: 'integration_event',
2593
- boundaryId: boundary.boundaryId,
2594
- eventKey: boundary.eventKey,
2595
- timeoutMs: boundary.timeoutMs,
2596
- });
2597
- }
2598
-
2599
- async runPlay(
2600
- key: string,
2601
- playRef: string | { playName?: string; name?: string },
2602
- input: Record<string, unknown>,
2603
- options?: PlayCallOptions,
2604
- ): Promise<unknown> {
2605
- const normalizedKey = this.normalizeContextKey(key, 'runPlay');
2606
- const resolvedName =
2607
- typeof playRef === 'string'
2608
- ? playRef
2609
- : typeof playRef.playName === 'string'
2610
- ? playRef.playName
2611
- : (playRef.name ?? '');
2612
-
2613
- if (!resolvedName.trim()) {
2614
- throw new Error('ctx.runPlay(...) requires a resolvable play name.');
2615
- }
2616
-
2617
- if (!this.options.resolvePlay) {
2618
- throw new Error(
2619
- 'ctx.runPlay(...) is unavailable because no play resolver was configured.',
2620
- );
2621
- }
2622
-
2623
- if (this.governance.ancestryPlayIds.includes(resolvedName)) {
2624
- const chain = [...this.governance.ancestryPlayIds, resolvedName].join(
2625
- ' -> ',
2626
- );
2627
- throw new Error(`Recursive play graph detected: ${chain}`);
2628
- }
2629
-
2630
- const nextDepth = this.governance.callDepth + 1;
2631
- if (nextDepth > this.governance.limits.maxPlayCallDepth) {
2632
- throw new Error(
2633
- `Play-call depth exceeded (${nextDepth}/${this.governance.limits.maxPlayCallDepth}) while calling ${resolvedName}.`,
2634
- );
2635
- }
2636
-
2637
- const parentKey = this.governance.currentPlayId;
2638
- const nextParentCalls =
2639
- (this.governance.parentChildCalls[parentKey] ?? 0) + 1;
2640
- if (nextParentCalls > this.governance.limits.maxChildPlayCallsPerParent) {
2641
- throw new Error(
2642
- `Child play-call cap exceeded for ${parentKey} (${nextParentCalls}/${this.governance.limits.maxChildPlayCallsPerParent}).`,
2643
- );
2644
- }
2645
-
2646
- const executePlayCall = async (): Promise<unknown> => {
2647
- this.bumpCounter('playCallCount');
2648
- this.bumpCounter('descendantCount');
2649
- this.governance.parentChildCalls[parentKey] = nextParentCalls;
2650
- this.acquirePlayConcurrency(resolvedName);
2651
-
2652
- try {
2653
- const resolvePlay = this.options.resolvePlay;
2654
- if (!resolvePlay) {
2655
- throw new Error(
2656
- 'ctx.runPlay(...) is unavailable because no play resolver was configured.',
2657
- );
2658
- }
2659
- const resolvedPlay = await resolvePlay(resolvedName);
2660
- if (!resolvedPlay) {
2661
- throw new Error(
2662
- `Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`,
2663
- );
2664
- }
2665
-
2666
- const childGovernance: PlayExecutionGovernanceState = {
2667
- ...this.governance,
2668
- currentPlayId: resolvedName,
2669
- currentRunId: `${this.governance.rootRunId}:${resolvedName}:${this.governance.playCallCount}`,
2670
- ancestryPlayIds: [...this.governance.ancestryPlayIds, resolvedName],
2671
- ancestryRunIds: [
2672
- ...this.governance.ancestryRunIds,
2673
- `${this.governance.rootRunId}:${resolvedName}:${this.governance.playCallCount}`,
2674
- ],
2675
- callDepth: nextDepth,
2676
- };
2677
- const childContext = createPlayContext({
2678
- ...this.options,
2679
- playId: resolvedName,
2680
- playName: resolvedName,
2681
- runId: childGovernance.currentRunId,
2682
- staticPipeline: resolvedPlay.staticPipeline ?? null,
2683
- checkpoint: this.checkpoint,
2684
- governance: childGovernance,
2685
- getRuntimeStepReceipt: undefined,
2686
- claimRuntimeStepReceipt: undefined,
2687
- completeRuntimeStepReceipt: undefined,
2688
- failRuntimeStepReceipt: undefined,
2689
- skipRuntimeStepReceipt: undefined,
2690
- });
2691
- const childExecution = this.executeResolvedPlay(
2692
- resolvedPlay,
2693
- childContext,
2694
- input,
2695
- );
2696
- await childContext.drainQueuedWork([childExecution]);
2697
- const result = await childExecution;
2698
- const step = {
2699
- type: 'play_call' as const,
2700
- playId: resolvedName,
2701
- nestedSteps: childContext.getSteps(),
2702
- description: normalizeStepDescription(options?.description),
2703
- };
2704
- if (this.activeMapStep) {
2705
- this.activeMapStep.substeps.push(step);
2706
- } else {
2707
- this.steps.push(step);
2708
- }
2709
- return result;
2710
- } finally {
2711
- this.releasePlayConcurrency(resolvedName);
2712
- }
2713
- };
2714
-
2715
- if (rowContext.getStore()) {
2716
- return await executePlayCall();
2717
- }
2718
-
2719
- return this.executeWithRuntimeReceipt<unknown>(
2720
- 'runPlay',
2721
- normalizedKey,
2722
- this.currentRunId,
2723
- {
2724
- markSkipped: () => {
2725
- this.log(
2726
- `ctx.runPlay(${normalizedKey}): no-op due completed receipt`,
2727
- );
2728
- },
2729
- execute: executePlayCall,
2730
- },
2731
- );
2732
- }
2733
-
2734
- /**
2735
- * Extract a list from a tool result.
2736
- * e.g. ctx.extractList(result, 'people', ['first_name', 'last_name', 'email'])
2737
- */
2738
- extractList(
2739
- result: unknown,
2740
- listPath: string,
2741
- fields?: string[],
2742
- ): Record<string, unknown>[] {
2743
- if (result == null || typeof result !== 'object') return [];
2744
-
2745
- let list: unknown = result;
2746
- for (const key of listPath.split('.')) {
2747
- if (list == null || typeof list !== 'object') return [];
2748
- list = (list as Record<string, unknown>)[key];
2749
- }
2750
-
2751
- if (!Array.isArray(list)) return [];
2752
-
2753
- if (!fields || fields.length === 0) {
2754
- return list.filter(
2755
- (item): item is Record<string, unknown> =>
2756
- item != null && typeof item === 'object',
2757
- );
2758
- }
2759
-
2760
- return list
2761
- .filter(
2762
- (item): item is Record<string, unknown> =>
2763
- item != null && typeof item === 'object',
2764
- )
2765
- .map((item) => {
2766
- const picked: Record<string, unknown> = {};
2767
- for (const field of fields) {
2768
- if (field in item) picked[field] = item[field];
2769
- }
2770
- return picked;
2771
- });
2772
- }
2773
-
2774
- log(msg: string): void {
2775
- const line = `[${new Date().toISOString()}] ${msg}`;
2776
- this.logBuffer.push(line);
2777
- this.options.onLog?.(line);
2778
- if (this.options.verbose) console.log(line);
2779
- }
2780
-
2781
- async sleep(ms: number): Promise<void> {
2782
- if (this.options.durableBoundaries) {
2783
- const delayMs = Math.max(0, Math.round(ms));
2784
- const boundaryId = this.durableBoundaryId(
2785
- `sleep-${this.sleepBoundaryIndex}-${delayMs}`,
2786
- );
2787
- this.sleepBoundaryIndex += 1;
2788
- const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
2789
- if (existing?.kind === 'sleep' && existing.completedAt !== undefined) {
2790
- return;
2791
- }
2792
- this.checkpoint.resolvedBoundaries = {
2793
- ...(this.checkpoint.resolvedBoundaries ?? {}),
2794
- [boundaryId]: {
2795
- kind: 'sleep',
2796
- delayMs,
2797
- },
2798
- };
2799
- this.options.onBatchComplete?.(this.checkpoint);
2800
- throw new PlayExecutionSuspendedError({
2801
- kind: 'sleep',
2802
- boundaryId,
2803
- delayMs,
2804
- });
2805
- }
2806
- return new Promise((resolve) => setTimeout(resolve, ms));
2807
- }
2808
-
2809
- async fetch(
2810
- key: string,
2811
- input: string | URL,
2812
- init: RequestInit = {},
2813
- ): Promise<PlayFetchResponse> {
2814
- const normalizedKey = this.normalizeContextKey(key, 'fetch');
2815
- if (rowContext.getStore()) {
2816
- throw new Error(
2817
- 'ctx.fetch() must run outside ctx.map(); use ctx.tools.execute(...) for row-level external requests so Deepline can batch and checkpoint them.',
2818
- );
2819
- }
2820
-
2821
- return this.executeWithRuntimeReceipt<PlayFetchResponse>(
2822
- 'fetch',
2823
- normalizedKey,
2824
- this.currentRunId,
2825
- {
2826
- execute: async () => {
2827
- const url = input.toString();
2828
- const method = (init.method ?? 'GET').toUpperCase();
2829
- const headers = normalizeFetchHeaders(init.headers);
2830
- const boundaryId = this.durableBoundaryId(
2831
- `fetch-${this.fetchCallIndex}-${stableFetchHash(
2832
- JSON.stringify({
2833
- url,
2834
- method,
2835
- headers,
2836
- body: typeof init.body === 'string' ? init.body : null,
2837
- }),
2838
- )}`,
2839
- );
2840
- this.fetchCallIndex += 1;
2841
-
2842
- const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
2843
- if (existing?.kind === 'fetch' && 'output' in existing) {
2844
- this.log(`ctx.fetch(${url}): recovered response from checkpoint`);
2845
- return existing.output as PlayFetchResponse;
2846
- }
2847
-
2848
- if (!['GET', 'HEAD', 'OPTIONS'].includes(method)) {
2849
- const hasIdempotencyKey =
2850
- headers['idempotency-key'] !== undefined ||
2851
- headers['x-idempotency-key'] !== undefined;
2852
- if (!hasIdempotencyKey) {
2853
- throw new Error(
2854
- `ctx.fetch(${method} ${url}) needs an Idempotency-Key header. Durable plays can replay after waits/retries; add an idempotency key or wrap the side effect in a Deepline integration tool.`,
2855
- );
2856
- }
2857
- }
2858
-
2859
- const response = await fetch(url, init);
2860
- const bodyText = await response.text();
2861
- const output: PlayFetchResponse = {
2862
- ok: response.ok,
2863
- status: response.status,
2864
- statusText: response.statusText,
2865
- url: response.url,
2866
- headers: Object.fromEntries(response.headers.entries()),
2867
- bodyText,
2868
- json: parseJsonOrNull(bodyText),
2869
- };
2870
-
2871
- this.checkpoint.resolvedBoundaries = {
2872
- ...(this.checkpoint.resolvedBoundaries ?? {}),
2873
- [boundaryId]: {
2874
- kind: 'fetch',
2875
- url,
2876
- method,
2877
- output,
2878
- completedAt: Date.now(),
2879
- },
2880
- };
2881
- this.options.onBatchComplete?.(this.checkpoint);
2882
- return output;
2883
- },
2884
- markSkipped: (output) => {
2885
- this.log(
2886
- `ctx.fetch(${output?.url ?? ''}): no-op due completed receipt`,
2887
- );
2888
- },
2889
- },
2890
- );
2891
- }
2892
-
2893
- async step<T>(key: string, run: () => T | Promise<T>): Promise<T> {
2894
- const normalizedKey = this.normalizeContextKey(key, 'step');
2895
- if (!normalizedKey.trim()) {
2896
- throw new Error('ctx.step(key, fn) requires a non-empty stable step id.');
2897
- }
2898
-
2899
- const rowStore = rowContext.getStore();
2900
- const scope = rowStore ? `row-${rowStore.rowId}` : 'workflow';
2901
- const callIndexKey = `${scope}:${normalizedKey}`;
2902
- const callIndex = this.stepCallIndexByKey.get(callIndexKey) ?? 0;
2903
- this.stepCallIndexByKey.set(callIndexKey, callIndex + 1);
2904
- const boundarySuffix = callIndex === 0 ? '' : `:${callIndex}`;
2905
- const boundaryId = this.durableBoundaryId(
2906
- `step-${scope}:${normalizedKey}${boundarySuffix}`,
2907
- );
2908
- const executeStep = async (): Promise<T> => {
2909
- const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
2910
- if (existing?.kind === 'step' && 'output' in existing) {
2911
- this.log(
2912
- `ctx.step(${normalizedKey}): recovered result from checkpoint`,
2913
- );
2914
- return existing.output as T;
2915
- }
2916
-
2917
- const output = await run();
2918
- assertJsonSerializableStepOutput(normalizedKey, output);
2919
- this.checkpoint.resolvedBoundaries = {
2920
- ...(this.checkpoint.resolvedBoundaries ?? {}),
2921
- [boundaryId]: {
2922
- kind: 'step',
2923
- stepId: normalizedKey,
2924
- output,
2925
- completedAt: Date.now(),
2926
- },
2927
- };
2928
- this.options.onBatchComplete?.(this.checkpoint);
2929
- return output;
2930
- };
2931
-
2932
- if (rowStore) {
2933
- return await executeStep();
2934
- }
2935
-
2936
- return this.executeWithRuntimeReceipt<T>(
2937
- 'step',
2938
- normalizedKey,
2939
- this.currentRunId,
2940
- {
2941
- markSkipped: (output) => {
2942
- this.log(`ctx.step(${normalizedKey}): no-op due completed receipt`);
2943
- assertJsonSerializableStepOutput(normalizedKey, output);
2944
- },
2945
- execute: executeStep,
2946
- },
2947
- );
2948
- }
2949
-
2950
- getLogs(): string[] {
2951
- return this.logBuffer;
2952
- }
2953
-
2954
- getCheckpoint(): PlayCheckpoint {
2955
- return this.checkpoint;
2956
- }
2957
-
2958
- getSteps(): PlayStep[] {
2959
- return this.steps;
2960
- }
2961
-
2962
- recordStep(step: PlayStep): void {
2963
- const isSubstepType =
2964
- step.type === 'waterfall' ||
2965
- step.type === 'tool' ||
2966
- step.type === 'run_javascript';
2967
- const targetMap = this.activeMapStep ?? this.lastMapStep;
2968
- if (targetMap && isSubstepType) {
2969
- targetMap.substeps.push(step);
2970
- } else {
2971
- this.steps.push(step);
2972
- }
2973
- }
2974
-
2975
- recordReturn(outputRows: number): void {
2976
- this.lastMapStep = null; // No more substeps expected
2977
- this.steps.push({ type: 'return', outputRows });
2978
- }
2979
-
2980
- getStats(): Record<string, unknown> {
2981
- const stats: Record<
2982
- string,
2983
- { total: number; complete: number; failed: number }
2984
- > = {};
2985
- for (const [, rowState] of this.rowStates) {
2986
- for (const [toolName, wState] of rowState.waterfalls) {
2987
- if (!stats[toolName])
2988
- stats[toolName] = { total: 0, complete: 0, failed: 0 };
2989
- stats[toolName].total++;
2990
- if (wState.status === 'complete') stats[toolName].complete++;
2991
- if (wState.status === 'failed') stats[toolName].failed++;
2992
- }
2993
- }
2994
- return {
2995
- rowsProcessed: Math.max(this.rowStates.size, this.processedRowCount),
2996
- waterfalls: stats,
2997
- };
2998
- }
2999
-
3000
- // ——— Batched waterfall execution (the core engine) ———
3001
-
3002
- private async executeBatchedWaterfalls(): Promise<void> {
3003
- const queuedWaterfalls = this.waterfallQueue;
3004
- this.waterfallQueue = new Map();
3005
- this.log(`Executing batched waterfalls for ${queuedWaterfalls.size} tools`);
3006
-
3007
- for (const [queueKey, requests] of queuedWaterfalls) {
3008
- const inlineSpec = requests[0]?.spec;
3009
- if (inlineSpec) {
3010
- await this.executeInlineWaterfall(queueKey, inlineSpec, requests);
3011
- continue;
3012
- }
3013
-
3014
- const toolName = requests[0]?.toolName ?? queueKey;
3015
- const providers = requests[0]?.opts?.providers ?? [
3016
- 'hunter',
3017
- 'leadmagic',
3018
- 'pdl',
3019
- 'dropcontact',
3020
- 'prospeo',
3021
- ];
3022
-
3023
- this.log(
3024
- `Processing waterfall ${toolName}: ${requests.length} rows, providers: ${providers.join(', ')}`,
3025
- );
3026
-
3027
- if (!this.checkpoint.resolvedWaterfalls[queueKey]) {
3028
- this.checkpoint.resolvedWaterfalls[queueKey] = {};
3029
- }
3030
-
3031
- await executeWaterfallProviders<WaterfallRequest, unknown>({
3032
- providers,
3033
- getPendingRequests: () =>
3034
- requests.filter((req) => {
3035
- const wState = this.rowStates
3036
- .get(req.rowId)
3037
- ?.waterfalls.get(toolName);
3038
- return wState?.status === 'pending';
3039
- }),
3040
- getCachedResults: (provider) => {
3041
- const batchKey = `${queueKey}:${provider}`;
3042
- const cached = this.checkpoint.completedBatches[batchKey];
3043
- if (!cached) return null;
3044
- this.log(` ${provider}: skipping (recovered from checkpoint)`);
3045
- return cached.map((entry) => ({
3046
- request: requests.find((request) => request.rowId === entry.rowId)!,
3047
- result: entry.result,
3048
- }));
3049
- },
3050
- storeCachedResults: (provider, results) => {
3051
- const batchKey = `${queueKey}:${provider}`;
3052
- this.checkpoint.completedBatches[batchKey] = results.map((entry) => ({
3053
- rowId: entry.request.rowId,
3054
- result: entry.result,
3055
- }));
3056
- },
3057
- executeProviderRequests: async (provider, pending) => {
3058
- const providerToolId = resolveWaterfallToolId(provider, toolName);
3059
- const strategy =
3060
- this.options.getBatchOperationStrategy?.(providerToolId) ?? null;
3061
- this.log(` ${provider}: ${pending.length} pending rows`);
3062
- this.bumpCounter('retryCount', pending.length);
3063
-
3064
- if (strategy) {
3065
- const compiledBatches = compileRequestsWithStrategy({
3066
- requests: pending,
3067
- strategy,
3068
- getPayload: (request: WaterfallRequest) => request.input,
3069
- });
3070
- const flattenedResults: Array<{
3071
- request: WaterfallRequest;
3072
- result: unknown | null;
3073
- }> = [];
3074
-
3075
- await executeChunkedRequests({
3076
- requests: compiledBatches,
3077
- batchSize:
3078
- compiledBatches.length > 0
3079
- ? await this.rateLimitScheduler.getSuggestedParallelism(
3080
- compiledBatches[0]!.batchOperation,
3081
- 4,
3082
- )
3083
- : 4,
3084
- execute: async (batch) =>
3085
- await this.callToolAPI(
3086
- batch.batchOperation,
3087
- batch.batchPayload,
3088
- ),
3089
- onChunkComplete: async (chunkResults) => {
3090
- for (const entry of chunkResults) {
3091
- const splitResults =
3092
- entry.result != null
3093
- ? entry.request.splitResults(entry.result)
3094
- : entry.request.memberRequests.map(() => null);
3095
-
3096
- for (
3097
- let index = 0;
3098
- index < entry.request.memberRequests.length;
3099
- index += 1
3100
- ) {
3101
- flattenedResults.push({
3102
- request: entry.request.memberRequests[index]!,
3103
- result: splitResults[index] ?? null,
3104
- });
3105
- }
3106
- }
3107
- this.options.onBatchComplete?.(this.checkpoint);
3108
- },
3109
- });
3110
-
3111
- return flattenedResults;
3112
- }
3113
-
3114
- const chunkResults: Array<{
3115
- request: WaterfallRequest;
3116
- result: unknown | null;
3117
- }> = [];
3118
- await executeChunkedRequests<WaterfallRequest, unknown>({
3119
- requests: pending,
3120
- batchSize: await this.rateLimitScheduler.getSuggestedParallelism(
3121
- providerToolId,
3122
- 50,
3123
- ),
3124
- execute: async (request) =>
3125
- await this.callToolAPI(providerToolId, request.input),
3126
- onChunkComplete: async (results) => {
3127
- chunkResults.push(...results);
3128
- this.options.onBatchComplete?.(this.checkpoint);
3129
- },
3130
- });
3131
- return chunkResults;
3132
- },
3133
- onHit: (provider, request, result) => {
3134
- this.resolveWaterfall(
3135
- queueKey,
3136
- toolName,
3137
- request.rowId,
3138
- result,
3139
- provider,
3140
- request.rowKey ?? null,
3141
- request.tableNamespace ?? null,
3142
- request.fieldName,
3143
- );
3144
- },
3145
- onMiss: (_provider, request) => {
3146
- const wState = this.rowStates
3147
- .get(request.rowId)
3148
- ?.waterfalls.get(toolName);
3149
- if (wState) wState.providerIndex++;
3150
- this.log(` Row ${request.rowId}: miss`);
3151
- },
3152
- onProviderComplete: () => {
3153
- this.options.onBatchComplete?.(this.checkpoint);
3154
- },
3155
- });
3156
-
3157
- const stepResults: PlayStepRowResult[] = requests.map((req) => {
3158
- const wState = this.rowStates.get(req.rowId)?.waterfalls.get(toolName);
3159
- const success = wState?.status === 'complete';
3160
- return {
3161
- rowId: req.rowId,
3162
- status: success ? 'completed' : 'missed',
3163
- success,
3164
- value: wState?.result,
3165
- error: success ? null : (wState?.error ?? null),
3166
- };
3167
- });
3168
- const waterfallStep = {
3169
- type: 'waterfall' as const,
3170
- tool: toolName,
3171
- providers,
3172
- results: stepResults,
3173
- description: normalizeStepDescription(requests[0]?.description),
3174
- };
3175
- if (this.activeMapStep) {
3176
- this.activeMapStep.substeps.push(waterfallStep);
3177
- } else {
3178
- this.steps.push(waterfallStep);
3179
- }
3180
-
3181
- for (const req of requests) {
3182
- const wState = this.rowStates.get(req.rowId)?.waterfalls.get(toolName);
3183
- if (wState?.status === 'pending') {
3184
- wState.status = 'failed';
3185
- wState.error = 'All providers exhausted';
3186
- this.checkpoint.resolvedWaterfalls[queueKey]![req.rowId] = null;
3187
-
3188
- const resolver = this.resolvers.get(`${req.rowId}-${queueKey}`);
3189
- if (resolver) {
3190
- resolver(null);
3191
- this.resolvers.delete(`${req.rowId}-${queueKey}`);
3192
- }
3193
- this.emitScopedFieldMetaUpdate({
3194
- rowId: req.rowId,
3195
- key: req.rowKey ?? null,
3196
- tableNamespace: req.tableNamespace ?? null,
3197
- fieldName: req.fieldName,
3198
- status: 'failed',
3199
- rowStatus: 'running',
3200
- stage: toolName,
3201
- provider: null,
3202
- error: 'All providers exhausted',
3203
- dataPatch: {},
3204
- });
3205
- this.log(` Row ${req.rowId}: all providers exhausted`);
3206
- }
3207
- }
3208
- }
3209
- }
3210
-
3211
- private async executeInlineWaterfall(
3212
- queueKey: string,
3213
- spec: InlineWaterfallSpec,
3214
- requests: WaterfallRequest[],
3215
- ): Promise<void> {
3216
- if (!this.checkpoint.resolvedWaterfalls[queueKey]) {
3217
- this.checkpoint.resolvedWaterfalls[queueKey] = {};
3218
- }
3219
-
3220
- const pendingRows = new Set<number>(
3221
- requests
3222
- .filter(
3223
- (req) =>
3224
- this.rowStates.get(req.rowId)?.waterfalls.get(spec.id)?.status ===
3225
- 'pending',
3226
- )
3227
- .map((req) => req.rowId),
3228
- );
3229
- const resultsByRow = new Map<number, unknown[]>();
3230
- const stepResults: Array<{
3231
- id: string;
3232
- kind?: 'tool' | 'code';
3233
- toolId?: string;
3234
- results: PlayStepRowResult[];
3235
- }> = [];
3236
- const stepColumnNames = spec.steps.map((s) =>
3237
- sqlSafePlayColumnName(`${spec.id}.${s.id}`),
3238
- );
3239
- const resolvedInChunkRowIds = new Set<number>();
3240
- let stepIdx = 0;
3241
-
3242
- for (const step of spec.steps) {
3243
- const stepColumnName = sqlSafePlayColumnName(`${spec.id}.${step.id}`);
3244
- const stepProvider = isInlineWaterfallToolStep(step)
3245
- ? step.toolId
3246
- : 'code';
3247
- if (pendingRows.size === 0) {
3248
- const skippedResults: PlayStepRowResult[] = requests.map((request) => {
3249
- this.emitCellUpdate({
3250
- rowId: request.rowId,
3251
- key: request.rowKey ?? null,
3252
- tableNamespace: request.tableNamespace ?? null,
3253
- columnName: stepColumnName,
3254
- status: 'skipped',
3255
- stage: step.id,
3256
- provider: stepProvider,
3257
- value: null,
3258
- });
3259
- return {
3260
- rowId: request.rowId,
3261
- status: 'skipped',
3262
- success: false,
3263
- value: null,
3264
- error: null,
3265
- };
3266
- });
3267
- stepResults.push({
3268
- id: step.id,
3269
- kind: isInlineWaterfallCodeStep(step) ? 'code' : 'tool',
3270
- toolId: stepProvider,
3271
- results: skippedResults,
3272
- });
3273
- stepIdx++;
3274
- continue;
3275
- }
3276
- this.bumpCounter('waterfallStepExecutions', pendingRows.size);
3277
- const pendingRowIds = new Set<number>(pendingRows);
3278
- const stepRequests = requests.filter((req) =>
3279
- pendingRowIds.has(req.rowId),
3280
- );
3281
- const perRowResultsByRowId = new Map<number, PlayStepRowResult>();
3282
- if (isInlineWaterfallCodeStep(step)) {
3283
- this.log(
3284
- ` Inline waterfall ${spec.id} -> ${step.id}: ${stepRequests.length} pending rows (code)`,
3285
- );
3286
- await executeChunkedRequests<WaterfallRequest, unknown>({
3287
- requests: stepRequests,
3288
- batchSize: await this.rateLimitScheduler.getSuggestedParallelism(
3289
- `code:${spec.id}:${step.id}`,
3290
- 20,
3291
- ),
3292
- execute: async (request) => {
3293
- const codeStepCtx = {
3294
- tools: {
3295
- execute: async (
3296
- requestOrKey:
3297
- | { tool: string; input: Record<string, unknown> }
3298
- | string,
3299
- toolId?: string,
3300
- payload?: Record<string, unknown>,
3301
- ) => {
3302
- if (typeof requestOrKey === 'object') {
3303
- return await this.callToolAPI(
3304
- requestOrKey.tool,
3305
- requestOrKey.input,
3306
- );
3307
- }
3308
- if (!toolId || !payload) {
3309
- throw new Error(
3310
- 'inline waterfall ctx.tools.execute requires a tool and input.',
3311
- );
3312
- }
3313
- return await this.callToolAPI(toolId, payload);
3314
- },
3315
- },
3316
- };
3317
- return await step.run(request.input, codeStepCtx);
3318
- },
3319
- onChunkComplete: async (chunkResults) => {
3320
- for (const entry of chunkResults) {
3321
- const matchedValue = extractInlineWaterfallCodeStepValue(
3322
- spec.output,
3323
- entry.result,
3324
- );
3325
- const bucket = resultsByRow.get(entry.request.rowId) ?? [];
3326
- const nextBucket = Array.isArray(matchedValue)
3327
- ? [...bucket, ...matchedValue]
3328
- : matchedValue != null
3329
- ? [...bucket, matchedValue]
3330
- : bucket;
3331
- resultsByRow.set(entry.request.rowId, nextBucket);
3332
- const satisfied = nextBucket.length >= spec.minResults;
3333
- perRowResultsByRowId.set(entry.request.rowId, {
3334
- rowId: entry.request.rowId,
3335
- status: matchedValue != null ? 'completed' : 'missed',
3336
- success: matchedValue != null,
3337
- value: matchedValue,
3338
- provider: matchedValue != null ? 'code' : undefined,
3339
- error: null,
3340
- });
3341
- if (satisfied) {
3342
- const finalValue =
3343
- spec.minResults === 1 ? (nextBucket[0] ?? null) : nextBucket;
3344
- this.resolveWaterfall(
3345
- queueKey,
3346
- spec.id,
3347
- entry.request.rowId,
3348
- finalValue,
3349
- 'code',
3350
- entry.request.rowKey ?? null,
3351
- entry.request.tableNamespace ?? null,
3352
- entry.request.fieldName,
3353
- );
3354
- pendingRows.delete(entry.request.rowId);
3355
- this.emitCellUpdate({
3356
- rowId: entry.request.rowId,
3357
- key: entry.request.rowKey ?? null,
3358
- tableNamespace: entry.request.tableNamespace ?? null,
3359
- columnName: stepColumnName,
3360
- status: 'completed',
3361
- stage: step.id,
3362
- provider: 'code',
3363
- value: matchedValue,
3364
- });
3365
- for (let ri = stepIdx + 1; ri < spec.steps.length; ri++) {
3366
- const skippedStep = spec.steps[ri]!;
3367
- this.emitCellUpdate({
3368
- rowId: entry.request.rowId,
3369
- key: entry.request.rowKey ?? null,
3370
- tableNamespace: entry.request.tableNamespace ?? null,
3371
- columnName: stepColumnNames[ri]!,
3372
- status: 'skipped',
3373
- stage: skippedStep.id,
3374
- provider: isInlineWaterfallToolStep(skippedStep)
3375
- ? skippedStep.toolId
3376
- : 'code',
3377
- value: null,
3378
- });
3379
- }
3380
- resolvedInChunkRowIds.add(entry.request.rowId);
3381
- }
3382
- }
3383
- this.options.onBatchComplete?.(this.checkpoint);
3384
- },
3385
- });
3386
- } else {
3387
- const strategy =
3388
- this.options.getBatchOperationStrategy?.(step.toolId) ?? null;
3389
- this.log(
3390
- ` Inline waterfall ${spec.id} -> ${step.id}: ${stepRequests.length} pending rows`,
3391
- );
3392
- for (const request of stepRequests) {
3393
- this.emitCellUpdate({
3394
- rowId: request.rowId,
3395
- key: request.rowKey ?? null,
3396
- tableNamespace: request.tableNamespace ?? null,
3397
- columnName: stepColumnName,
3398
- status: 'running',
3399
- stage: step.id,
3400
- provider: step.toolId,
3401
- value: null,
3402
- });
3403
- }
3404
-
3405
- if (strategy) {
3406
- const compiledBatches = compileRequestsWithStrategy({
3407
- requests: stepRequests,
3408
- strategy,
3409
- getPayload: (request: WaterfallRequest) =>
3410
- step.mapInput(request.input),
3411
- });
3412
- this.log(
3413
- ` ${step.toolId}: compiled ${compiledBatches.length} batch request(s)` +
3414
- ` (${this.summarizeBatchSizes(
3415
- compiledBatches.map((batch) => batch.memberRequests.length),
3416
- )})`,
3417
- );
3418
- await executeChunkedRequests({
3419
- requests: compiledBatches,
3420
- batchSize:
3421
- compiledBatches.length > 0
3422
- ? await this.rateLimitScheduler.getSuggestedParallelism(
3423
- compiledBatches[0]!.batchOperation,
3424
- 4,
3425
- )
3426
- : 4,
3427
- execute: async (batch) =>
3428
- await this.callToolAPI(batch.batchOperation, batch.batchPayload),
3429
- onChunkComplete: async (chunkResults) => {
3430
- for (const entry of chunkResults) {
3431
- const splitResults =
3432
- entry.result != null
3433
- ? entry.request.splitResults(entry.result)
3434
- : entry.request.memberRequests.map(() => null);
3435
- for (
3436
- let index = 0;
3437
- index < entry.request.memberRequests.length;
3438
- index += 1
3439
- ) {
3440
- const request = entry.request.memberRequests[index]!;
3441
- const rawResult = splitResults[index] ?? null;
3442
- const matchedValue = await extractWaterfallOutputValue(
3443
- step.toolId,
3444
- spec.output,
3445
- rawResult,
3446
- this.options.getToolResultIdentityGetters,
3447
- );
3448
- const bucket = resultsByRow.get(request.rowId) ?? [];
3449
- const nextBucket = Array.isArray(matchedValue)
3450
- ? [...bucket, ...matchedValue]
3451
- : matchedValue != null
3452
- ? [...bucket, matchedValue]
3453
- : bucket;
3454
- resultsByRow.set(request.rowId, nextBucket);
3455
- const satisfied = nextBucket.length >= spec.minResults;
3456
- perRowResultsByRowId.set(request.rowId, {
3457
- rowId: request.rowId,
3458
- status: matchedValue != null ? 'completed' : 'missed',
3459
- success: matchedValue != null,
3460
- value: matchedValue,
3461
- provider: matchedValue != null ? step.toolId : undefined,
3462
- error: null,
3463
- });
3464
- if (satisfied) {
3465
- const finalValue =
3466
- spec.minResults === 1
3467
- ? (nextBucket[0] ?? null)
3468
- : nextBucket;
3469
- this.resolveWaterfall(
3470
- queueKey,
3471
- spec.id,
3472
- request.rowId,
3473
- finalValue,
3474
- step.toolId,
3475
- request.rowKey ?? null,
3476
- request.tableNamespace ?? null,
3477
- request.fieldName,
3478
- );
3479
- pendingRows.delete(request.rowId);
3480
- this.emitCellUpdate({
3481
- rowId: request.rowId,
3482
- key: request.rowKey ?? null,
3483
- tableNamespace: request.tableNamespace ?? null,
3484
- columnName: stepColumnNames[stepIdx]!,
3485
- status: 'completed',
3486
- stage: step.id,
3487
- provider: step.toolId,
3488
- value: matchedValue,
3489
- });
3490
- for (let ri = stepIdx + 1; ri < spec.steps.length; ri++) {
3491
- const skippedStep = spec.steps[ri]!;
3492
- this.emitCellUpdate({
3493
- rowId: request.rowId,
3494
- key: request.rowKey ?? null,
3495
- tableNamespace: request.tableNamespace ?? null,
3496
- columnName: stepColumnNames[ri]!,
3497
- status: 'skipped',
3498
- stage: skippedStep.id,
3499
- provider: isInlineWaterfallToolStep(skippedStep)
3500
- ? skippedStep.toolId
3501
- : 'code',
3502
- value: null,
3503
- });
3504
- }
3505
- resolvedInChunkRowIds.add(request.rowId);
3506
- }
3507
- }
3508
- }
3509
- this.options.onBatchComplete?.(this.checkpoint);
3510
- },
3511
- });
3512
- } else {
3513
- this.log(
3514
- ` ${step.toolId}: executing ${stepRequests.length} unbatched request(s)`,
3515
- );
3516
- await executeChunkedRequests<WaterfallRequest, unknown>({
3517
- requests: stepRequests,
3518
- batchSize: await this.rateLimitScheduler.getSuggestedParallelism(
3519
- step.toolId,
3520
- 50,
3521
- ),
3522
- execute: async (request) =>
3523
- await this.callToolAPI(step.toolId, step.mapInput(request.input)),
3524
- onChunkComplete: async (chunkResults) => {
3525
- for (const entry of chunkResults) {
3526
- const matchedValue = await extractWaterfallOutputValue(
3527
- step.toolId,
3528
- spec.output,
3529
- entry.result,
3530
- this.options.getToolResultIdentityGetters,
3531
- );
3532
- const bucket = resultsByRow.get(entry.request.rowId) ?? [];
3533
- const nextBucket = Array.isArray(matchedValue)
3534
- ? [...bucket, ...matchedValue]
3535
- : matchedValue != null
3536
- ? [...bucket, matchedValue]
3537
- : bucket;
3538
- resultsByRow.set(entry.request.rowId, nextBucket);
3539
- const satisfied = nextBucket.length >= spec.minResults;
3540
- perRowResultsByRowId.set(entry.request.rowId, {
3541
- rowId: entry.request.rowId,
3542
- status: matchedValue != null ? 'completed' : 'missed',
3543
- success: matchedValue != null,
3544
- value: matchedValue,
3545
- provider: matchedValue != null ? step.toolId : undefined,
3546
- error: null,
3547
- });
3548
- if (satisfied) {
3549
- const finalValue =
3550
- spec.minResults === 1
3551
- ? (nextBucket[0] ?? null)
3552
- : nextBucket;
3553
- this.resolveWaterfall(
3554
- queueKey,
3555
- spec.id,
3556
- entry.request.rowId,
3557
- finalValue,
3558
- step.toolId,
3559
- entry.request.rowKey ?? null,
3560
- entry.request.tableNamespace ?? null,
3561
- entry.request.fieldName,
3562
- );
3563
- pendingRows.delete(entry.request.rowId);
3564
- this.emitCellUpdate({
3565
- rowId: entry.request.rowId,
3566
- key: entry.request.rowKey ?? null,
3567
- tableNamespace: entry.request.tableNamespace ?? null,
3568
- columnName: stepColumnNames[stepIdx]!,
3569
- status: 'completed',
3570
- stage: step.id,
3571
- provider: step.toolId,
3572
- value: matchedValue,
3573
- });
3574
- for (let ri = stepIdx + 1; ri < spec.steps.length; ri++) {
3575
- const skippedStep = spec.steps[ri]!;
3576
- this.emitCellUpdate({
3577
- rowId: entry.request.rowId,
3578
- key: entry.request.rowKey ?? null,
3579
- tableNamespace: entry.request.tableNamespace ?? null,
3580
- columnName: stepColumnNames[ri]!,
3581
- status: 'skipped',
3582
- stage: skippedStep.id,
3583
- provider: isInlineWaterfallToolStep(skippedStep)
3584
- ? skippedStep.toolId
3585
- : 'code',
3586
- value: null,
3587
- });
3588
- }
3589
- resolvedInChunkRowIds.add(entry.request.rowId);
3590
- }
3591
- }
3592
- this.options.onBatchComplete?.(this.checkpoint);
3593
- },
3594
- });
3595
- }
3596
- }
3597
-
3598
- const perRowResults: PlayStepRowResult[] = requests.map((request) => {
3599
- if (resolvedInChunkRowIds.has(request.rowId)) {
3600
- const existing = perRowResultsByRowId.get(request.rowId);
3601
- return (
3602
- existing ?? {
3603
- rowId: request.rowId,
3604
- status: 'completed' as const,
3605
- success: true,
3606
- value: null,
3607
- error: null,
3608
- }
3609
- );
3610
- }
3611
- const existing = perRowResultsByRowId.get(request.rowId);
3612
- if (existing) {
3613
- this.emitCellUpdate({
3614
- rowId: request.rowId,
3615
- key: request.rowKey ?? null,
3616
- tableNamespace: request.tableNamespace ?? null,
3617
- columnName: stepColumnName,
3618
- status: existing.status,
3619
- stage: step.id,
3620
- provider: existing.provider ?? stepProvider,
3621
- error: existing.error ?? null,
3622
- value: existing.value ?? null,
3623
- });
3624
- return existing;
3625
- }
3626
- if (!pendingRowIds.has(request.rowId)) {
3627
- this.emitCellUpdate({
3628
- rowId: request.rowId,
3629
- key: request.rowKey ?? null,
3630
- tableNamespace: request.tableNamespace ?? null,
3631
- columnName: stepColumnName,
3632
- status: 'skipped',
3633
- stage: step.id,
3634
- provider: stepProvider,
3635
- value: null,
3636
- });
3637
- return {
3638
- rowId: request.rowId,
3639
- status: 'skipped',
3640
- success: false,
3641
- value: null,
3642
- error: null,
3643
- };
3644
- }
3645
- this.emitCellUpdate({
3646
- rowId: request.rowId,
3647
- key: request.rowKey ?? null,
3648
- tableNamespace: request.tableNamespace ?? null,
3649
- columnName: stepColumnName,
3650
- status: 'missed',
3651
- stage: step.id,
3652
- provider: stepProvider,
3653
- value: null,
3654
- });
3655
- return {
3656
- rowId: request.rowId,
3657
- status: 'missed',
3658
- success: false,
3659
- value: null,
3660
- error: null,
3661
- };
3662
- });
3663
-
3664
- stepResults.push({
3665
- id: step.id,
3666
- kind: isInlineWaterfallCodeStep(step) ? 'code' : 'tool',
3667
- toolId: stepProvider,
3668
- // Persist only a bounded preview in the in-memory execution trace.
3669
- // The authoritative per-row state already lives in Neon/live progress.
3670
- results: compactRowResultsPreview(perRowResults),
3671
- });
3672
- stepIdx++;
3673
- }
3674
-
3675
- for (const req of requests) {
3676
- const wState = this.rowStates.get(req.rowId)?.waterfalls.get(spec.id);
3677
- if (wState?.status === 'pending') {
3678
- wState.status = 'failed';
3679
- wState.error = 'All waterfall steps exhausted';
3680
- this.checkpoint.resolvedWaterfalls[queueKey]![req.rowId] = null;
3681
- const resolver = this.resolvers.get(`${req.rowId}-${queueKey}`);
3682
- if (resolver) {
3683
- resolver(null);
3684
- this.resolvers.delete(`${req.rowId}-${queueKey}`);
3685
- }
3686
- }
3687
- }
3688
-
3689
- const groupResults: PlayStepRowResult[] = requests.map((req) => {
3690
- const wState = this.rowStates.get(req.rowId)?.waterfalls.get(spec.id);
3691
- const success = wState?.status === 'complete';
3692
- return {
3693
- rowId: req.rowId,
3694
- status: success ? 'completed' : 'missed',
3695
- success,
3696
- value: wState?.result,
3697
- error: success ? null : (wState?.error ?? null),
3698
- };
3699
- });
3700
-
3701
- const waterfallStep = {
3702
- type: 'waterfall' as const,
3703
- id: spec.id,
3704
- output: spec.output,
3705
- minResults: spec.minResults,
3706
- steps: stepResults,
3707
- results: compactRowResultsPreview(groupResults),
3708
- description: normalizeStepDescription(requests[0]?.description),
3709
- };
3710
- if (this.activeMapStep) {
3711
- this.activeMapStep.substeps.push(waterfallStep);
3712
- } else {
3713
- this.steps.push(waterfallStep);
3714
- }
3715
- }
3716
-
3717
- private resolveWaterfall(
3718
- queueKey: string,
3719
- toolName: string,
3720
- rowId: number,
3721
- result: unknown,
3722
- provider: string,
3723
- rowKey: string | null,
3724
- tableNamespace: string | null,
3725
- fieldName?: string,
3726
- ): void {
3727
- const wState = this.rowStates.get(rowId)?.waterfalls.get(toolName);
3728
- if (!wState || wState.status !== 'pending') return;
3729
-
3730
- wState.status = 'complete';
3731
- wState.result = result;
3732
- this.checkpoint.resolvedWaterfalls[queueKey]![rowId] = result;
3733
-
3734
- const resolver = this.resolvers.get(`${rowId}-${queueKey}`);
3735
- if (resolver) {
3736
- resolver(result);
3737
- this.resolvers.delete(`${rowId}-${queueKey}`);
3738
- }
3739
- this.emitScopedFieldMetaUpdate({
3740
- rowId,
3741
- key: rowKey,
3742
- tableNamespace,
3743
- fieldName,
3744
- status: 'running',
3745
- rowStatus: 'running',
3746
- stage: toolName,
3747
- provider,
3748
- error: null,
3749
- dataPatch: {},
3750
- });
3751
- this.logWaterfallMatch({ queueKey, rowId, provider });
3752
- }
3753
-
3754
- // ——— Batched tool call execution ———
3755
-
3756
- private async executeBatchedToolCalls(): Promise<void> {
3757
- const queuedToolCalls = this.toolCallQueue;
3758
- this.toolCallQueue = [];
3759
-
3760
- // Group by toolId
3761
- const byTool = new Map<string, ToolCallRequest[]>();
3762
- for (const req of queuedToolCalls) {
3763
- if (!byTool.has(req.toolId)) byTool.set(req.toolId, []);
3764
- byTool.get(req.toolId)!.push(req);
3765
- }
3766
-
3767
- await Promise.all(
3768
- [...byTool.entries()].map(async ([toolId, requests]) => {
3769
- this.log(`Executing tool batch ${toolId}: ${requests.length} calls`);
3770
-
3771
- const pendingRequests: ToolCallRequest[] = [];
3772
- for (const req of requests) {
3773
- const cached = this.getCachedToolResult(
3774
- toolId,
3775
- this.buildToolResultCacheKey({
3776
- rowId: req.rowId,
3777
- tableNamespace: req.tableNamespace,
3778
- rowKey: req.rowKey ?? undefined,
3779
- callId: req.callId,
3780
- }),
3781
- );
3782
- if (cached?.done) {
3783
- this.log(` Row ${req.rowId} ${toolId}: recovered from checkpoint`);
3784
- const resolver = this.toolCallResolvers.get(req.callId);
3785
- if (resolver) {
3786
- resolver(cached.result);
3787
- this.toolCallResolvers.delete(req.callId);
3788
- }
3789
- } else {
3790
- pendingRequests.push(req);
3791
- }
3792
- }
3793
-
3794
- if (pendingRequests.length > 0) {
3795
- const strategy =
3796
- this.options.getBatchOperationStrategy?.(toolId) ?? null;
3797
-
3798
- if (strategy) {
3799
- const compiledBatches = compileRequestsWithStrategy({
3800
- requests: pendingRequests,
3801
- strategy,
3802
- getPayload: (request: ToolCallRequest) => request.input,
3803
- });
3804
-
3805
- await executeChunkedRequests({
3806
- requests: compiledBatches,
3807
- batchSize:
3808
- compiledBatches.length > 0
3809
- ? await this.rateLimitScheduler.getSuggestedParallelism(
3810
- compiledBatches[0]!.batchOperation,
3811
- 4,
3812
- )
3813
- : 4,
3814
- execute: async (batch) =>
3815
- await this.callToolAPI(
3816
- batch.batchOperation,
3817
- batch.batchPayload,
3818
- ),
3819
- onChunkComplete: async (chunkResults) => {
3820
- for (const entry of chunkResults) {
3821
- const splitResults =
3822
- entry.result != null
3823
- ? entry.request.splitResults(entry.result)
3824
- : entry.request.memberRequests.map(() => null);
3825
-
3826
- for (
3827
- let index = 0;
3828
- index < entry.request.memberRequests.length;
3829
- index += 1
3830
- ) {
3831
- const request = entry.request.memberRequests[index]!;
3832
- await this.resolveToolCall(
3833
- toolId,
3834
- request,
3835
- splitResults[index] ?? null,
3836
- );
3837
- }
3838
- }
3839
-
3840
- this.options.onBatchComplete?.(this.checkpoint);
3841
- },
3842
- });
3843
- } else {
3844
- await executeChunkedRequests<ToolCallRequest, ToolExecutionResponse>({
3845
- requests: pendingRequests,
3846
- batchSize: await this.rateLimitScheduler.getSuggestedParallelism(
3847
- toolId,
3848
- 50,
3849
- ),
3850
- execute: async (request) =>
3851
- await this.callToolExecutionAPI(toolId, request.input),
3852
- onChunkComplete: async (chunkResults) => {
3853
- for (const entry of chunkResults) {
3854
- await this.resolveToolCall(
3855
- toolId,
3856
- entry.request,
3857
- entry.result?.result ?? null,
3858
- entry.result?.metadata ?? null,
3859
- );
3860
- }
3861
-
3862
- this.options.onBatchComplete?.(this.checkpoint);
3863
- },
3864
- });
3865
- }
3866
- }
3867
-
3868
- // Record step — nest under map if inside one
3869
- const stepResults: PlayStepRowResult[] = requests.map((req) => {
3870
- const cachedResult = this.getCachedToolResult(
3871
- toolId,
3872
- this.buildToolResultCacheKey({
3873
- rowId: req.rowId,
3874
- tableNamespace: req.tableNamespace,
3875
- rowKey: req.rowKey ?? undefined,
3876
- callId: req.callId,
3877
- }),
3878
- );
3879
- return {
3880
- rowId: req.rowId,
3881
- status: cachedResult?.result != null ? 'completed' : 'failed',
3882
- success: cachedResult?.result != null,
3883
- value: cachedResult?.result,
3884
- error: cachedResult?.result != null ? null : 'Tool call failed',
3885
- };
3886
- });
3887
- const toolStep = {
3888
- type: 'tool' as const,
3889
- toolId,
3890
- // Keep the step trace preview-sized for large map pages.
3891
- results: compactRowResultsPreview(stepResults),
3892
- description: normalizeStepDescription(requests[0]?.description),
3893
- };
3894
- if (this.activeMapStep) {
3895
- this.activeMapStep.substeps.push(toolStep);
3896
- } else {
3897
- this.steps.push(toolStep);
3898
- }
3899
- }),
3900
- );
3901
- }
3902
-
3903
- private async executeResolvedPlay(
3904
- resolvedPlay: ResolvedPlayExecution,
3905
- ctx: PlayContextImpl,
3906
- input: Record<string, unknown>,
3907
- ): Promise<unknown> {
3908
- if (resolvedPlay.definition) {
3909
- if (!this.options.executeStructuredPlayDefinition) {
3910
- throw new Error(
3911
- `Play "${resolvedPlay.playId}" is a structured play, but this runtime did not provide a structured play executor.`,
3912
- );
3913
- }
3914
- return await this.options.executeStructuredPlayDefinition({
3915
- definition: resolvedPlay.definition,
3916
- ctx,
3917
- rows: [],
3918
- playInput: input,
3919
- });
3920
- }
3921
-
3922
- if (resolvedPlay.codeFormat === 'cjs_module') {
3923
- const artifact = resolvedPlay.artifact;
3924
- if (!artifact) {
3925
- throw new Error(
3926
- `Play "${resolvedPlay.playId}" is missing a bundled artifact.`,
3927
- );
3928
- }
3929
- const runtimeModule =
3930
- (await import('node:module')) as unknown as typeof import('node:module') & {
3931
- Module: typeof import('node:module').Module & {
3932
- _nodeModulePaths: (from: string) => string[];
3933
- };
3934
- };
3935
- const compiled = new runtimeModule.Module(artifact.virtualFilename);
3936
- compiled.filename = artifact.virtualFilename;
3937
- compiled.paths = runtimeModule.Module._nodeModulePaths(process.cwd());
3938
- (
3939
- compiled as import('node:module').Module & {
3940
- _compile: (code: string, filename: string) => void;
3941
- }
3942
- )._compile(artifact.bundledCode, artifact.virtualFilename);
3943
- const candidate =
3944
- typeof compiled.exports === 'function'
3945
- ? compiled.exports
3946
- : (compiled.exports as { default?: unknown }).default;
3947
- if (typeof candidate !== 'function') {
3948
- throw new Error(
3949
- `Play "${resolvedPlay.playId}" does not export a callable default.`,
3950
- );
3951
- }
3952
- return await (
3953
- candidate as (
3954
- ctx: unknown,
3955
- runtimeInput: Record<string, unknown>,
3956
- ) => Promise<unknown>
3957
- )(ctx, input);
3958
- }
3959
-
3960
- const code = resolvedPlay.code ?? resolvedPlay.sourceCode;
3961
- if (!code?.trim()) {
3962
- throw new Error(
3963
- `Play "${resolvedPlay.playId}" is missing executable source.`,
3964
- );
3965
- }
3966
- return await new Function(
3967
- 'ctx',
3968
- 'input',
3969
- `
3970
- const __playFn = ${code};
3971
- return __playFn(ctx, input);
3972
- `,
3973
- )(ctx, input);
3974
- }
3975
-
3976
- // ——— Direct execution (outside map context) ———
3977
-
3978
- private async executeWaterfallDirect(
3979
- toolNameOrSpec: string | InlineWaterfallSpec,
3980
- input: Record<string, unknown>,
3981
- opts?: WaterfallOptions,
3982
- ): Promise<unknown | null> {
3983
- if (isInlineWaterfallSpec(toolNameOrSpec)) {
3984
- this.log(`Direct inline waterfall: ${toolNameOrSpec.id}`);
3985
- const collected: unknown[] = [];
3986
- for (const step of toolNameOrSpec.steps) {
3987
- this.bumpCounter('waterfallStepExecutions');
3988
- const matched = isInlineWaterfallCodeStep(step)
3989
- ? extractInlineWaterfallCodeStepValue(
3990
- toolNameOrSpec.output,
3991
- await step.run(input, {
3992
- tools: {
3993
- execute: async (
3994
- requestOrKey:
3995
- | { tool: string; input: Record<string, unknown> }
3996
- | string,
3997
- toolId?: string,
3998
- payload?: Record<string, unknown>,
3999
- ) => {
4000
- if (typeof requestOrKey === 'object') {
4001
- return await this.callToolAPI(
4002
- requestOrKey.tool,
4003
- requestOrKey.input,
4004
- );
4005
- }
4006
- if (!toolId || !payload) {
4007
- throw new Error(
4008
- 'inline waterfall ctx.tools.execute requires a tool and input.',
4009
- );
4010
- }
4011
- return await this.callToolAPI(toolId, payload);
4012
- },
4013
- },
4014
- }),
4015
- )
4016
- : await extractWaterfallOutputValue(
4017
- step.toolId,
4018
- toolNameOrSpec.output,
4019
- await this.callToolAPI(step.toolId, step.mapInput(input)),
4020
- this.options.getToolResultIdentityGetters,
4021
- );
4022
- if (Array.isArray(matched)) {
4023
- collected.push(...matched);
4024
- } else if (matched != null) {
4025
- collected.push(matched);
4026
- }
4027
- if (collected.length >= toolNameOrSpec.minResults) {
4028
- return toolNameOrSpec.minResults === 1
4029
- ? (collected[0] ?? null)
4030
- : collected;
4031
- }
4032
- }
4033
- return null;
4034
- }
4035
-
4036
- const toolName = toolNameOrSpec;
4037
- this.log(`Direct waterfall: ${toolName}`);
4038
- const providers = opts?.providers ?? ['hunter', 'leadmagic', 'pdl'];
4039
-
4040
- for (const provider of providers) {
4041
- this.log(` Trying ${provider}`);
4042
- try {
4043
- this.bumpCounter('retryCount');
4044
- const result = await this.callToolAPI(
4045
- resolveWaterfallToolId(provider, toolName),
4046
- input,
4047
- );
4048
- if (
4049
- result != null &&
4050
- typeof result === 'object' &&
4051
- Object.keys(result as object).length > 0
4052
- ) {
4053
- this.log(` Found with ${provider}`);
4054
- return result;
4055
- }
4056
- } catch {
4057
- this.log(` Failed with ${provider}`);
4058
- }
4059
- }
4060
-
4061
- this.log(` All providers exhausted`);
4062
- return null;
4063
- }
4064
-
4065
- private async callToolAPI(
4066
- toolId: string,
4067
- input: Record<string, unknown>,
4068
- ): Promise<unknown> {
4069
- const execution = await this.callToolExecutionAPI(toolId, input);
4070
- return execution.result != null
4071
- && typeof execution.result === 'object'
4072
- && !Array.isArray(execution.result)
4073
- && 'data' in execution.result
4074
- ? execution.result.data
4075
- : execution.result;
4076
- }
4077
-
4078
- private async callToolExecutionAPI(
4079
- toolId: string,
4080
- input: Record<string, unknown>,
4081
- ): Promise<ToolExecutionResponse> {
4082
- if (!this.options.executorToken || !this.options.baseUrl) {
4083
- throw new Error(
4084
- 'executorToken and baseUrl are required for tool API calls (cloud execution only)',
4085
- );
4086
- }
4087
- const url = `${this.options.baseUrl}/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
4088
-
4089
- this.acquireToolConcurrency();
4090
- try {
4091
- return await withActiveSpan(
4092
- 'plays.tool.execute',
4093
- {
4094
- tracer: 'deepline.plays',
4095
- attributes: {
4096
- 'plays.play_name': this.options.playId ?? 'anonymous-play',
4097
- 'plays.workflow_id': this.options.workflowId ?? '',
4098
- 'plays.run_id': this.options.runId ?? this.options.workflowId ?? '',
4099
- 'plays.tool_id': toolId,
4100
- },
4101
- },
4102
- async (span) =>
4103
- await this.rateLimitScheduler.run(toolId, async () => {
4104
- let rateLimitAttempt = 0;
4105
-
4106
- while (true) {
4107
- const response = await fetch(url, {
4108
- method: 'POST',
4109
- headers: {
4110
- 'Content-Type': 'application/json',
4111
- Authorization: `Bearer ${this.options.executorToken}`,
4112
- [EXECUTE_TOOL_METADATA_HEADER]: 'true',
4113
- },
4114
- body: JSON.stringify({ payload: input }),
4115
- });
4116
-
4117
- span.setAttribute('plays.http_status_code', response.status);
4118
-
4119
- if (response.status === 429) {
4120
- rateLimitAttempt += 1;
4121
- this.bumpCounter('retryCount');
4122
- const retryAfterMs = parseRetryAfterMs(
4123
- response.headers.get('retry-after'),
4124
- );
4125
- span.setAttribute(
4126
- 'plays.rate_limit_retry_after_ms',
4127
- retryAfterMs,
4128
- );
4129
- span.setAttribute('plays.rate_limit_attempt', rateLimitAttempt);
4130
- this.log(
4131
- `Tool ${toolId} rate limited; retrying after ${retryAfterMs}ms`,
4132
- );
4133
- await this.sleepWithCheckpointHeartbeat(retryAfterMs);
4134
- continue;
4135
- }
4136
-
4137
- if (!response.ok) {
4138
- const text = await response.text();
4139
- if (
4140
- response.status >= 500 &&
4141
- response.status < 600 &&
4142
- rateLimitAttempt + 1 < TOOL_TRANSIENT_HTTP_MAX_ATTEMPTS
4143
- ) {
4144
- rateLimitAttempt += 1;
4145
- this.bumpCounter('retryCount');
4146
- const retryAfterMs = parseRetryAfterMs(
4147
- response.headers.get('retry-after'),
4148
- );
4149
- span.setAttribute(
4150
- 'plays.transient_http_retry_after_ms',
4151
- retryAfterMs,
4152
- );
4153
- span.setAttribute(
4154
- 'plays.transient_http_attempt',
4155
- rateLimitAttempt,
4156
- );
4157
- this.log(
4158
- `Tool ${toolId} returned ${response.status}; retrying after ${retryAfterMs}ms`,
4159
- );
4160
- await this.sleepWithCheckpointHeartbeat(retryAfterMs);
4161
- continue;
4162
- }
4163
- const failureDetail = (() => {
4164
- try {
4165
- const parsed = JSON.parse(text) as Record<string, unknown>;
4166
- const detail =
4167
- typeof parsed.message === 'string'
4168
- ? parsed.message
4169
- : typeof parsed.error === 'string'
4170
- ? parsed.error
4171
- : text;
4172
- const code =
4173
- typeof parsed.code === 'string'
4174
- ? ` code=${parsed.code}`
4175
- : '';
4176
- const requestId =
4177
- typeof parsed.request_id === 'string'
4178
- ? ` requestId=${parsed.request_id}`
4179
- : typeof parsed.requestId === 'string'
4180
- ? ` requestId=${parsed.requestId}`
4181
- : '';
4182
- return `${detail}${code}${requestId}`;
4183
- } catch {
4184
- return text;
4185
- }
4186
- })();
4187
- const message = `Tool ${toolId} failed (${response.status}): ${failureDetail}`;
4188
- this.log(message);
4189
- throw new Error(message);
4190
- }
4191
-
4192
- const data = (await response.json()) as Record<string, unknown>;
4193
- const result = data.result ?? data;
4194
- const status =
4195
- typeof data.status === 'string'
4196
- ? data.status
4197
- : result == null
4198
- ? 'no_result'
4199
- : 'completed';
4200
- setSpanAttributes(span, {
4201
- 'plays.tool_result_kind':
4202
- result == null
4203
- ? 'null'
4204
- : Array.isArray(result)
4205
- ? 'array'
4206
- : typeof result,
4207
- });
4208
- return {
4209
- status,
4210
- result,
4211
- metadata: parseExecuteToolMetadata(toolId, data),
4212
- };
4213
- }
4214
- }),
4215
- );
4216
- } finally {
4217
- this.releaseToolConcurrency();
4218
- }
4219
- }
4220
-
4221
- private async sleepWithCheckpointHeartbeat(ms: number): Promise<void> {
4222
- const waitMs = Math.max(1, Math.ceil(ms));
4223
- const startedAt = Date.now();
4224
-
4225
- while (Date.now() - startedAt < waitMs) {
4226
- this.options.onBatchComplete?.(this.checkpoint);
4227
- const remainingMs = waitMs - (Date.now() - startedAt);
4228
- await new Promise((resolve) =>
4229
- setTimeout(
4230
- resolve,
4231
- Math.min(TOOL_RETRY_HEARTBEAT_INTERVAL_MS, Math.max(1, remainingMs)),
4232
- ),
4233
- );
4234
- }
4235
-
4236
- this.options.onBatchComplete?.(this.checkpoint);
4237
- }
4238
- }
4239
-
4240
- export function createPlayContext(options: ContextOptions): PlayContextImpl {
4241
- return new PlayContextImpl(options);
4242
- }