deepline 0.1.152 → 0.1.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +46 -6
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1180 -825
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts +34 -18
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +41 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +143 -8
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +104 -0
- package/dist/bundling-sources/sdk/src/index.ts +0 -1
- package/dist/bundling-sources/sdk/src/play.ts +3 -48
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +27 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/worker-play-entry.ts +0 -10
- package/dist/bundling-sources/shared_libs/play-data-plane/index.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +87 -0
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +0 -59
- package/dist/bundling-sources/shared_libs/play-runtime/cell-staleness.ts +0 -253
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +805 -1570
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +47 -74
- package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +36 -14
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +145 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +284 -0
- package/dist/bundling-sources/shared_libs/play-runtime/postgres-json.ts +12 -5
- package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts +78 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +10 -45
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +923 -535
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +58 -78
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +12 -1
- package/dist/bundling-sources/shared_libs/play-runtime/step-program-dataset-builder.ts +1 -14
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts +159 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result-types.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +32 -0
- package/dist/bundling-sources/shared_libs/plays/definition.ts +4 -2
- package/dist/bundling-sources/shared_libs/plays/runtime-validation.ts +3 -14
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +1 -43
- package/dist/cli/index.js +1301 -399
- package/dist/cli/index.mjs +1269 -361
- package/dist/{compiler-manifest-BjoRENv9.d.ts → compiler-manifest-DW1flrHk.d.mts} +0 -9
- package/dist/{compiler-manifest-BjoRENv9.d.mts → compiler-manifest-DW1flrHk.d.ts} +0 -9
- package/dist/index.d.mts +9 -38
- package/dist/index.d.ts +9 -38
- package/dist/index.js +22 -11
- package/dist/index.mjs +22 -11
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-data-plane/cell-policy.ts +0 -76
- package/dist/bundling-sources/shared_libs/play-runtime/progress-emitter.ts +0 -197
- package/dist/bundling-sources/shared_libs/play-runtime/waterfall-replay.ts +0 -79
|
@@ -3,11 +3,9 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Batching model:
|
|
5
5
|
* 1. ctx.dataset("table_key", rows).withColumn("field", resolver).run() starts all row column resolvers concurrently
|
|
6
|
-
* 2. ctx.
|
|
7
|
-
* 3.
|
|
8
|
-
* 4.
|
|
9
|
-
* 5. Each provider batch = real HTTP call to /api/v2/integrations/{toolId}/execute
|
|
10
|
-
* 6. Results resolve suspended row promises, rows complete
|
|
6
|
+
* 2. ctx.tools.execute() calls inside field resolvers QUEUE requests
|
|
7
|
+
* 3. After all rows have queued, executeBatchedToolCalls() runs provider calls
|
|
8
|
+
* 4. Results resolve suspended row promises, rows complete
|
|
11
9
|
*
|
|
12
10
|
* Temporal integration:
|
|
13
11
|
* - checkpoint: recovered from heartbeatDetails on retry (skip completed batches)
|
|
@@ -22,9 +20,6 @@ import type { PlayDataset, PlayDatasetInput } from '@shared_libs/plays/dataset';
|
|
|
22
20
|
import {
|
|
23
21
|
compileRequestsWithStrategy,
|
|
24
22
|
executeChunkedRequests,
|
|
25
|
-
executeWaterfallProviders,
|
|
26
|
-
formatChunkExecutionError,
|
|
27
|
-
resolveWaterfallToolId,
|
|
28
23
|
} from './batch-runtime';
|
|
29
24
|
import type { PlayQueueHint } from './governor/rate-state-backend';
|
|
30
25
|
import type { MapRowOutcome } from './durability-store';
|
|
@@ -54,15 +49,34 @@ import {
|
|
|
54
49
|
deserializeToolExecuteResult,
|
|
55
50
|
type ParsedToolExecuteResponse,
|
|
56
51
|
type ToolExecuteResult,
|
|
52
|
+
type ToolResultExecutionMetadata,
|
|
57
53
|
type ToolResultMetadataInput,
|
|
58
54
|
} from './tool-result';
|
|
55
|
+
import {
|
|
56
|
+
markToolExecuteResultExecutionOutcome,
|
|
57
|
+
toolExecutionMetadataForOutcome,
|
|
58
|
+
toolExecutionOutcomeForDurableReceipt,
|
|
59
|
+
type DurableReceiptRecoverySource,
|
|
60
|
+
type ToolExecutionOutcome,
|
|
61
|
+
} from './tool-execution-outcome';
|
|
59
62
|
import {
|
|
60
63
|
TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
|
|
61
64
|
classifyToolExecuteHttpFailure,
|
|
62
65
|
createToolExecuteHttpFailureAttemptTracker,
|
|
63
66
|
} from './tool-execute-retry-policy';
|
|
67
|
+
import {
|
|
68
|
+
buildDurableCtxCallCacheKey,
|
|
69
|
+
buildDurableToolCallAuthScopeDigest,
|
|
70
|
+
buildDurableToolCallCacheKey,
|
|
71
|
+
} from './durable-call-cache';
|
|
72
|
+
import {
|
|
73
|
+
RuntimeReceiptWaitTimeoutError,
|
|
74
|
+
executeWithDurableRuntimeReceipt,
|
|
75
|
+
runtimeReceiptOutput as durableRuntimeReceiptOutput,
|
|
76
|
+
waitForCompletedRuntimeReceipt,
|
|
77
|
+
type DurableReceiptExecutionStore,
|
|
78
|
+
} from './durable-receipt-execution';
|
|
64
79
|
import { isRowIsolationExemptError } from './row-isolation';
|
|
65
|
-
import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline';
|
|
66
80
|
import { createRuntimeDatasetId } from './dataset-id';
|
|
67
81
|
import { dedupeExplicitMapKeyRows } from './map-row-identity';
|
|
68
82
|
import {
|
|
@@ -77,18 +91,8 @@ import {
|
|
|
77
91
|
import {
|
|
78
92
|
DEEPLINE_CELL_META_FIELD,
|
|
79
93
|
previousCellFromValue,
|
|
80
|
-
resolveCompletedCellStalenessMeta,
|
|
81
|
-
resolveReusableCellMetaForCurrentPolicy,
|
|
82
|
-
shouldRecomputeCell,
|
|
83
|
-
type AuthoredCellStalenessPolicyByField,
|
|
84
|
-
type CellStalenessMeta,
|
|
85
|
-
type CellStalenessPolicyByField,
|
|
86
94
|
type PreviousCell,
|
|
87
95
|
} from './cell-staleness';
|
|
88
|
-
import {
|
|
89
|
-
authoredCellPolicyMap,
|
|
90
|
-
normalizeAuthoredCellPolicyMap,
|
|
91
|
-
} from '../play-data-plane/cell-policy';
|
|
92
96
|
import {
|
|
93
97
|
cloneCsvAliasedRow,
|
|
94
98
|
stripCsvProjectedFields,
|
|
@@ -122,9 +126,6 @@ import {
|
|
|
122
126
|
import type {
|
|
123
127
|
CsvOptions,
|
|
124
128
|
RowState,
|
|
125
|
-
WaterfallRequest,
|
|
126
|
-
WaterfallOptions,
|
|
127
|
-
InlineWaterfallSpec,
|
|
128
129
|
DatasetOptions,
|
|
129
130
|
ToolCallRequest,
|
|
130
131
|
ToolBatchResult,
|
|
@@ -202,15 +203,43 @@ function isUnsafeOutboundUrlError(error: unknown): boolean {
|
|
|
202
203
|
return error instanceof Error && error.name === 'UnsafeOutboundUrlError';
|
|
203
204
|
}
|
|
204
205
|
|
|
206
|
+
function publicToolResponseEnvelope(value: unknown): {
|
|
207
|
+
status: string;
|
|
208
|
+
raw: unknown;
|
|
209
|
+
meta?: Record<string, unknown>;
|
|
210
|
+
} | null {
|
|
211
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
const record = value as Record<string, unknown>;
|
|
215
|
+
if (typeof record.status !== 'string') return null;
|
|
216
|
+
const toolResponse = record.toolResponse;
|
|
217
|
+
if (
|
|
218
|
+
!toolResponse ||
|
|
219
|
+
typeof toolResponse !== 'object' ||
|
|
220
|
+
Array.isArray(toolResponse) ||
|
|
221
|
+
!Object.prototype.hasOwnProperty.call(toolResponse, 'raw')
|
|
222
|
+
) {
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
const response = toolResponse as Record<string, unknown>;
|
|
226
|
+
return {
|
|
227
|
+
status: record.status,
|
|
228
|
+
raw: response.raw,
|
|
229
|
+
...(response.meta &&
|
|
230
|
+
typeof response.meta === 'object' &&
|
|
231
|
+
!Array.isArray(response.meta)
|
|
232
|
+
? { meta: response.meta as Record<string, unknown> }
|
|
233
|
+
: {}),
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
|
|
205
237
|
const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
|
|
206
238
|
const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
|
|
207
239
|
const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-response';
|
|
208
240
|
const IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT = 25;
|
|
209
|
-
const WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT = 10;
|
|
210
|
-
const WATERFALL_ROW_MATCH_LOG_INTERVAL = 1_000;
|
|
211
241
|
const BATCH_SIZE_LOG_SAMPLE_LIMIT = 10;
|
|
212
242
|
const STEP_PROGRAM_MAP_DEFINITION = Symbol('deepline.stepProgramMapDefinition');
|
|
213
|
-
const CELL_STALENESS_POLICIES = Symbol('deepline.cellStalenessPolicies');
|
|
214
243
|
|
|
215
244
|
function shouldPersistMapCellField(fieldName: string): boolean {
|
|
216
245
|
return !fieldName.startsWith('_') || fieldName === '_metadata';
|
|
@@ -396,25 +425,6 @@ function stableDigest(value: string): string {
|
|
|
396
425
|
|
|
397
426
|
type DurableCtxOperation = 'step' | 'tool' | 'fetch' | 'runPlay';
|
|
398
427
|
|
|
399
|
-
function staleBucketForSeconds(
|
|
400
|
-
staleAfterSeconds?: number | null,
|
|
401
|
-
nowMs = Date.now(),
|
|
402
|
-
): string | null {
|
|
403
|
-
if (staleAfterSeconds === undefined || staleAfterSeconds === null) {
|
|
404
|
-
return null;
|
|
405
|
-
}
|
|
406
|
-
if (
|
|
407
|
-
!Number.isFinite(staleAfterSeconds) ||
|
|
408
|
-
!Number.isInteger(staleAfterSeconds) ||
|
|
409
|
-
staleAfterSeconds <= 0
|
|
410
|
-
) {
|
|
411
|
-
throw new Error(
|
|
412
|
-
'staleAfterSeconds must be a positive whole number of seconds.',
|
|
413
|
-
);
|
|
414
|
-
}
|
|
415
|
-
return `${staleAfterSeconds}:${Math.floor(nowMs / (staleAfterSeconds * 1000))}`;
|
|
416
|
-
}
|
|
417
|
-
|
|
418
428
|
function durableCtxKey(input: {
|
|
419
429
|
orgId?: string | null;
|
|
420
430
|
playId: string;
|
|
@@ -423,19 +433,19 @@ function durableCtxKey(input: {
|
|
|
423
433
|
semanticKey?: string | null;
|
|
424
434
|
staleAfterSeconds?: number | null;
|
|
425
435
|
}): string {
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
436
|
+
if (input.operation === 'tool') {
|
|
437
|
+
throw new Error(
|
|
438
|
+
'Durable ctx key is only for non-tool call boundaries; tools use buildDurableToolCallCacheKey.',
|
|
439
|
+
);
|
|
440
|
+
}
|
|
441
|
+
return buildDurableCtxCallCacheKey({
|
|
442
|
+
orgId: input.orgId,
|
|
443
|
+
playId: input.playId,
|
|
444
|
+
kind: input.operation,
|
|
445
|
+
id: input.id,
|
|
446
|
+
semanticKey: input.semanticKey,
|
|
447
|
+
staleAfterSeconds: input.staleAfterSeconds,
|
|
448
|
+
});
|
|
439
449
|
}
|
|
440
450
|
|
|
441
451
|
function resolvedPlayRevisionFingerprint(play: ResolvedPlayExecution): string {
|
|
@@ -642,102 +652,8 @@ function createPacingResolver(
|
|
|
642
652
|
};
|
|
643
653
|
}
|
|
644
654
|
|
|
645
|
-
function isInlineWaterfallSpec(value: unknown): value is InlineWaterfallSpec {
|
|
646
|
-
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
647
|
-
return false;
|
|
648
|
-
}
|
|
649
|
-
const candidate = value as InlineWaterfallSpec;
|
|
650
|
-
return (
|
|
651
|
-
typeof candidate.id === 'string' &&
|
|
652
|
-
typeof candidate.output === 'string' &&
|
|
653
|
-
typeof candidate.minResults === 'number' &&
|
|
654
|
-
candidate.minResults >= 1 &&
|
|
655
|
-
Array.isArray(candidate.steps) &&
|
|
656
|
-
candidate.steps.length > 0
|
|
657
|
-
);
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
function isInlineWaterfallCodeStep(
|
|
661
|
-
step: InlineWaterfallSpec['steps'][number],
|
|
662
|
-
): step is Extract<InlineWaterfallSpec['steps'][number], { kind: 'code' }> {
|
|
663
|
-
return step.kind === 'code';
|
|
664
|
-
}
|
|
665
|
-
|
|
666
|
-
function isInlineWaterfallToolStep(
|
|
667
|
-
step: InlineWaterfallSpec['steps'][number],
|
|
668
|
-
): step is Extract<InlineWaterfallSpec['steps'][number], { toolId: string }> {
|
|
669
|
-
return !isInlineWaterfallCodeStep(step);
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
function extractInlineWaterfallCodeStepValue(
|
|
673
|
-
output: string,
|
|
674
|
-
result: unknown,
|
|
675
|
-
): unknown {
|
|
676
|
-
if (
|
|
677
|
-
result &&
|
|
678
|
-
typeof result === 'object' &&
|
|
679
|
-
!Array.isArray(result) &&
|
|
680
|
-
output in result
|
|
681
|
-
) {
|
|
682
|
-
return (result as Record<string, unknown>)[output] ?? null;
|
|
683
|
-
}
|
|
684
|
-
return result ?? null;
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
function isMeaningfulValue(value: unknown): boolean {
|
|
688
|
-
if (value == null) return false;
|
|
689
|
-
if (typeof value === 'string') return value.trim().length > 0;
|
|
690
|
-
if (Array.isArray(value)) return value.length > 0;
|
|
691
|
-
if (typeof value === 'object') return Object.keys(value).length > 0;
|
|
692
|
-
return true;
|
|
693
|
-
}
|
|
694
|
-
|
|
695
|
-
function getValueAtPath(value: unknown, path: string): unknown {
|
|
696
|
-
let current = value;
|
|
697
|
-
for (const part of path.split('.').filter(Boolean)) {
|
|
698
|
-
if (!current || typeof current !== 'object') return undefined;
|
|
699
|
-
current = (current as Record<string, unknown>)[part];
|
|
700
|
-
}
|
|
701
|
-
return current;
|
|
702
|
-
}
|
|
703
|
-
|
|
704
|
-
async function extractWaterfallOutputValue(
|
|
705
|
-
toolId: string,
|
|
706
|
-
output: string,
|
|
707
|
-
result: unknown,
|
|
708
|
-
getToolTargetGetters?: (
|
|
709
|
-
toolId: string,
|
|
710
|
-
output: string,
|
|
711
|
-
) => Promise<readonly string[]>,
|
|
712
|
-
): Promise<unknown> {
|
|
713
|
-
const getters = getToolTargetGetters
|
|
714
|
-
? await getToolTargetGetters(toolId, output)
|
|
715
|
-
: [];
|
|
716
|
-
for (const getter of getters) {
|
|
717
|
-
const value = getValueAtPath(result, getter);
|
|
718
|
-
if (isMeaningfulValue(value)) {
|
|
719
|
-
return value;
|
|
720
|
-
}
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
const directOutputValue = getValueAtPath(result, output);
|
|
724
|
-
if (directOutputValue !== undefined) {
|
|
725
|
-
return isMeaningfulValue(directOutputValue) ? directOutputValue : null;
|
|
726
|
-
}
|
|
727
|
-
|
|
728
|
-
if (output === 'people' || output === 'companies') {
|
|
729
|
-
if (Array.isArray(result)) return result;
|
|
730
|
-
const direct = getValueAtPath(result, output);
|
|
731
|
-
if (Array.isArray(direct)) return direct;
|
|
732
|
-
}
|
|
733
|
-
|
|
734
|
-
return isMeaningfulValue(result) ? result : null;
|
|
735
|
-
}
|
|
736
|
-
|
|
737
655
|
export class PlayContextImpl {
|
|
738
656
|
private rowStates = new Map<number, RowState>();
|
|
739
|
-
private resolvers = new Map<string, (value: unknown) => void>();
|
|
740
|
-
private waterfallQueue = new Map<string, WaterfallRequest[]>();
|
|
741
657
|
private toolCallQueue: ToolCallRequest[] = [];
|
|
742
658
|
private toolCallResolvers = new Map<
|
|
743
659
|
string,
|
|
@@ -767,14 +683,12 @@ export class PlayContextImpl {
|
|
|
767
683
|
timeoutMs: number;
|
|
768
684
|
}> = [];
|
|
769
685
|
private processedRowCount = 0;
|
|
770
|
-
private directToolCallIndex = 0;
|
|
771
686
|
private sleepBoundaryIndex = 0;
|
|
772
687
|
private fetchCallIndex = 0;
|
|
773
688
|
private readonly secretRedactor: SecretRedactionContext =
|
|
774
689
|
createSecretRedactionContext();
|
|
775
690
|
private mapInvocationIndex = 0;
|
|
776
691
|
private readonly stepCallIndexByKey = new Map<string, number>();
|
|
777
|
-
private readonly waterfallMatchLogCounts = new Map<string, number>();
|
|
778
692
|
/**
|
|
779
693
|
* The single source of every concurrency, budget, and pacing decision for this
|
|
780
694
|
* run-attempt. Tool/play/row slots are blocking semaphores; budgets are charged
|
|
@@ -1093,6 +1007,7 @@ export class PlayContextImpl {
|
|
|
1093
1007
|
key: string,
|
|
1094
1008
|
runId: string,
|
|
1095
1009
|
reclaimRunning = false,
|
|
1010
|
+
forceRefresh = false,
|
|
1096
1011
|
): Promise<RuntimeStepReceipt | null> {
|
|
1097
1012
|
if (!this.#options.claimRuntimeStepReceipt) {
|
|
1098
1013
|
return null;
|
|
@@ -1101,6 +1016,7 @@ export class PlayContextImpl {
|
|
|
1101
1016
|
key,
|
|
1102
1017
|
runId,
|
|
1103
1018
|
...(reclaimRunning ? { reclaimRunning: true } : {}),
|
|
1019
|
+
...(forceRefresh ? { forceRefresh: true } : {}),
|
|
1104
1020
|
});
|
|
1105
1021
|
if (!claimed || typeof claimed.key !== 'string' || !claimed.key.trim()) {
|
|
1106
1022
|
return null;
|
|
@@ -1163,122 +1079,267 @@ export class PlayContextImpl {
|
|
|
1163
1079
|
};
|
|
1164
1080
|
}
|
|
1165
1081
|
|
|
1082
|
+
private normalizeRuntimeStepReceipt(
|
|
1083
|
+
key: string,
|
|
1084
|
+
receipt: RuntimeStepReceipt | null | undefined,
|
|
1085
|
+
): RuntimeStepReceipt | null {
|
|
1086
|
+
if (!receipt) return null;
|
|
1087
|
+
if (typeof receipt.key === 'string' && receipt.key.trim()) {
|
|
1088
|
+
return {
|
|
1089
|
+
...receipt,
|
|
1090
|
+
key: receipt.key.trim(),
|
|
1091
|
+
runId: receipt.runId ?? null,
|
|
1092
|
+
};
|
|
1093
|
+
}
|
|
1094
|
+
return {
|
|
1095
|
+
...receipt,
|
|
1096
|
+
key: key.trim(),
|
|
1097
|
+
runId: receipt.runId ?? null,
|
|
1098
|
+
};
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
private async getRuntimeStepReceipts(
|
|
1102
|
+
keys: string[],
|
|
1103
|
+
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
1104
|
+
const uniqueKeys = [
|
|
1105
|
+
...new Set(keys.map((key) => key.trim()).filter(Boolean)),
|
|
1106
|
+
];
|
|
1107
|
+
if (uniqueKeys.length === 0) return new Map();
|
|
1108
|
+
const receipts = this.#options.getRuntimeStepReceipts
|
|
1109
|
+
? await this.#options.getRuntimeStepReceipts({ keys: uniqueKeys })
|
|
1110
|
+
: await Promise.all(
|
|
1111
|
+
uniqueKeys.map((key) => this.getRuntimeStepReceipt(key)),
|
|
1112
|
+
);
|
|
1113
|
+
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
1114
|
+
for (let index = 0; index < receipts.length; index += 1) {
|
|
1115
|
+
const normalized = this.normalizeRuntimeStepReceipt(
|
|
1116
|
+
uniqueKeys[index] ?? '',
|
|
1117
|
+
receipts[index],
|
|
1118
|
+
);
|
|
1119
|
+
if (normalized) byKey.set(normalized.key, normalized);
|
|
1120
|
+
}
|
|
1121
|
+
return byKey;
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
private async claimRuntimeStepReceipts(
|
|
1125
|
+
keys: string[],
|
|
1126
|
+
runId: string,
|
|
1127
|
+
reclaimRunning = false,
|
|
1128
|
+
forceRefresh = false,
|
|
1129
|
+
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
1130
|
+
const uniqueKeys = [
|
|
1131
|
+
...new Set(keys.map((key) => key.trim()).filter(Boolean)),
|
|
1132
|
+
];
|
|
1133
|
+
if (uniqueKeys.length === 0) return new Map();
|
|
1134
|
+
const receipts = this.#options.claimRuntimeStepReceipts
|
|
1135
|
+
? await this.#options.claimRuntimeStepReceipts({
|
|
1136
|
+
keys: uniqueKeys,
|
|
1137
|
+
runId,
|
|
1138
|
+
...(reclaimRunning ? { reclaimRunning: true } : {}),
|
|
1139
|
+
...(forceRefresh ? { forceRefresh: true } : {}),
|
|
1140
|
+
})
|
|
1141
|
+
: await Promise.all(
|
|
1142
|
+
uniqueKeys.map((key) =>
|
|
1143
|
+
this.claimRuntimeStepReceipt(
|
|
1144
|
+
key,
|
|
1145
|
+
runId,
|
|
1146
|
+
reclaimRunning,
|
|
1147
|
+
forceRefresh,
|
|
1148
|
+
),
|
|
1149
|
+
),
|
|
1150
|
+
);
|
|
1151
|
+
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
1152
|
+
for (let index = 0; index < receipts.length; index += 1) {
|
|
1153
|
+
const normalized = this.normalizeRuntimeStepReceipt(
|
|
1154
|
+
uniqueKeys[index] ?? '',
|
|
1155
|
+
receipts[index],
|
|
1156
|
+
);
|
|
1157
|
+
if (normalized) byKey.set(normalized.key, normalized);
|
|
1158
|
+
}
|
|
1159
|
+
return byKey;
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
private async completeRuntimeStepReceipts(
|
|
1163
|
+
receipts: Array<{ key: string; runId: string; output: unknown | null }>,
|
|
1164
|
+
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
1165
|
+
const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
|
|
1166
|
+
if (normalizedInputs.length === 0) return new Map();
|
|
1167
|
+
for (const receipt of normalizedInputs) {
|
|
1168
|
+
assertNoSecretTaint(receipt.output, 'ctx.tool receipt output');
|
|
1169
|
+
}
|
|
1170
|
+
const completed = this.#options.completeRuntimeStepReceipts
|
|
1171
|
+
? await this.#options.completeRuntimeStepReceipts({
|
|
1172
|
+
receipts: normalizedInputs,
|
|
1173
|
+
})
|
|
1174
|
+
: await Promise.all(
|
|
1175
|
+
normalizedInputs.map((receipt) =>
|
|
1176
|
+
this.completeRuntimeStepReceipt(
|
|
1177
|
+
receipt.key,
|
|
1178
|
+
receipt.runId,
|
|
1179
|
+
receipt.output,
|
|
1180
|
+
),
|
|
1181
|
+
),
|
|
1182
|
+
);
|
|
1183
|
+
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
1184
|
+
for (let index = 0; index < completed.length; index += 1) {
|
|
1185
|
+
const normalized = this.normalizeRuntimeStepReceipt(
|
|
1186
|
+
normalizedInputs[index]?.key ?? '',
|
|
1187
|
+
completed[index],
|
|
1188
|
+
);
|
|
1189
|
+
if (normalized) byKey.set(normalized.key, normalized);
|
|
1190
|
+
}
|
|
1191
|
+
return byKey;
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
private async failRuntimeStepReceipts(
|
|
1195
|
+
receipts: Array<{ key: string; runId: string; error: string }>,
|
|
1196
|
+
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
1197
|
+
const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
|
|
1198
|
+
if (normalizedInputs.length === 0) return new Map();
|
|
1199
|
+
const failed = this.#options.failRuntimeStepReceipts
|
|
1200
|
+
? await this.#options.failRuntimeStepReceipts({
|
|
1201
|
+
receipts: normalizedInputs,
|
|
1202
|
+
})
|
|
1203
|
+
: await Promise.all(
|
|
1204
|
+
normalizedInputs.map((receipt) =>
|
|
1205
|
+
this.failRuntimeStepReceipt(
|
|
1206
|
+
receipt.key,
|
|
1207
|
+
receipt.runId,
|
|
1208
|
+
receipt.error,
|
|
1209
|
+
),
|
|
1210
|
+
),
|
|
1211
|
+
);
|
|
1212
|
+
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
1213
|
+
for (let index = 0; index < failed.length; index += 1) {
|
|
1214
|
+
const normalized = this.normalizeRuntimeStepReceipt(
|
|
1215
|
+
normalizedInputs[index]?.key ?? '',
|
|
1216
|
+
failed[index],
|
|
1217
|
+
);
|
|
1218
|
+
if (normalized) byKey.set(normalized.key, normalized);
|
|
1219
|
+
}
|
|
1220
|
+
return byKey;
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
private async seedForcedRuntimeStepReceipts(keys: string[]): Promise<void> {
|
|
1224
|
+
const receiptKeys = [
|
|
1225
|
+
...new Set(keys.map((key) => key.trim()).filter(Boolean)),
|
|
1226
|
+
];
|
|
1227
|
+
if (receiptKeys.length === 0) return;
|
|
1228
|
+
await this.claimRuntimeStepReceipts(
|
|
1229
|
+
receiptKeys,
|
|
1230
|
+
this.currentRunId,
|
|
1231
|
+
true,
|
|
1232
|
+
true,
|
|
1233
|
+
);
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
private async durableToolCallCacheKey(input: {
|
|
1237
|
+
toolId: string;
|
|
1238
|
+
requestInput: Record<string, unknown>;
|
|
1239
|
+
staleAfterSeconds?: number | null;
|
|
1240
|
+
}): Promise<string> {
|
|
1241
|
+
const providerActionVersion =
|
|
1242
|
+
(await this.#options.getToolActionCacheVersion?.(input.toolId))?.trim() ??
|
|
1243
|
+
'';
|
|
1244
|
+
return buildDurableToolCallCacheKey({
|
|
1245
|
+
orgId: this.#options.orgId,
|
|
1246
|
+
playId: this.governance.currentPlayId,
|
|
1247
|
+
toolId: input.toolId,
|
|
1248
|
+
requestInput: input.requestInput,
|
|
1249
|
+
authScopeDigest: buildDurableToolCallAuthScopeDigest({
|
|
1250
|
+
orgId: this.#options.orgId,
|
|
1251
|
+
userEmail: this.#options.userEmail,
|
|
1252
|
+
toolId: input.toolId,
|
|
1253
|
+
}),
|
|
1254
|
+
providerActionVersion,
|
|
1255
|
+
staleAfterSeconds: input.staleAfterSeconds,
|
|
1256
|
+
});
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
private durableReceiptExecutionStore(): DurableReceiptExecutionStore {
|
|
1260
|
+
return {
|
|
1261
|
+
enabled: Boolean(this.#options.claimRuntimeStepReceipt),
|
|
1262
|
+
get: (receiptKey) => this.getRuntimeStepReceipt(receiptKey),
|
|
1263
|
+
getMany: (receiptKeys) => this.getRuntimeStepReceipts(receiptKeys),
|
|
1264
|
+
claim: (receiptKey, runId, reclaimRunning, forceRefresh) =>
|
|
1265
|
+
this.claimRuntimeStepReceipt(
|
|
1266
|
+
receiptKey,
|
|
1267
|
+
runId,
|
|
1268
|
+
reclaimRunning,
|
|
1269
|
+
forceRefresh,
|
|
1270
|
+
),
|
|
1271
|
+
complete: (receiptKey, runId, output) =>
|
|
1272
|
+
this.completeRuntimeStepReceipt(receiptKey, runId, output),
|
|
1273
|
+
fail: (receiptKey, runId, error) =>
|
|
1274
|
+
this.failRuntimeStepReceipt(receiptKey, runId, error),
|
|
1275
|
+
canPersistFailure: Boolean(this.#options.failRuntimeStepReceipt),
|
|
1276
|
+
canPersistCompletion: Boolean(this.#options.completeRuntimeStepReceipt),
|
|
1277
|
+
};
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
private runtimeReceiptOutput<T>(receipt: RuntimeStepReceipt): T {
|
|
1281
|
+
return durableRuntimeReceiptOutput<T>(receipt);
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
private async waitForCompletedRuntimeToolReceipt(
|
|
1285
|
+
key: string,
|
|
1286
|
+
): Promise<RuntimeStepReceipt> {
|
|
1287
|
+
return await waitForCompletedRuntimeReceipt({
|
|
1288
|
+
receiptKey: key,
|
|
1289
|
+
store: this.durableReceiptExecutionStore(),
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1166
1293
|
private async executeWithRuntimeReceipt<T>(
|
|
1167
1294
|
operation: DurableCtxOperation,
|
|
1168
1295
|
id: string,
|
|
1169
1296
|
runId: string,
|
|
1170
1297
|
opts: {
|
|
1171
1298
|
force?: boolean;
|
|
1299
|
+
receiptKey?: string | null;
|
|
1172
1300
|
semanticKey?: string | null;
|
|
1173
1301
|
staleAfterSeconds?: number | null;
|
|
1174
1302
|
repairRunningReceiptForSameRun?: boolean;
|
|
1303
|
+
repairRunningReceiptForSameRunAfterWaitTimeout?: boolean;
|
|
1175
1304
|
reclaimRunning?: boolean;
|
|
1176
1305
|
markSkipped?: (output: T) => Promise<void> | void;
|
|
1306
|
+
onRecovered?: (
|
|
1307
|
+
output: T,
|
|
1308
|
+
receipt: RuntimeStepReceipt,
|
|
1309
|
+
source: DurableReceiptRecoverySource,
|
|
1310
|
+
) => T;
|
|
1311
|
+
onClaimedResult?: (output: T, receiptKey: string) => T;
|
|
1177
1312
|
execute: () => Promise<T>;
|
|
1178
1313
|
},
|
|
1179
1314
|
): Promise<T> {
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1315
|
+
const receiptKey =
|
|
1316
|
+
opts.receiptKey?.trim() ||
|
|
1317
|
+
durableCtxKey({
|
|
1318
|
+
orgId: this.#options.orgId,
|
|
1319
|
+
playId: this.governance.currentPlayId,
|
|
1320
|
+
operation,
|
|
1321
|
+
id,
|
|
1322
|
+
semanticKey: opts.semanticKey,
|
|
1323
|
+
staleAfterSeconds: opts.staleAfterSeconds,
|
|
1324
|
+
});
|
|
1325
|
+
return await executeWithDurableRuntimeReceipt<T>({
|
|
1189
1326
|
operation,
|
|
1190
1327
|
id,
|
|
1191
|
-
semanticKey: opts.semanticKey,
|
|
1192
|
-
staleAfterSeconds: opts.staleAfterSeconds,
|
|
1193
|
-
});
|
|
1194
|
-
const recoverCompletedReceipt = async (
|
|
1195
|
-
receipt: RuntimeStepReceipt,
|
|
1196
|
-
): Promise<T> => {
|
|
1197
|
-
this.log(`ctx.${operation}(${id}): recovered result from receipt`);
|
|
1198
|
-
if (receipt.output === undefined) {
|
|
1199
|
-
return receipt.output as T;
|
|
1200
|
-
}
|
|
1201
|
-
// Tool results are persisted in their serialized form (live getters and
|
|
1202
|
-
// the non-enumerable toolOutput/_metadata fields do not survive JSON
|
|
1203
|
-
// storage). Rehydrate so a recovered `ctx.tools.execute` result behaves
|
|
1204
|
-
// exactly like a live one — same contract as dataset cell persistence.
|
|
1205
|
-
const output = isSerializedToolExecuteResult(receipt.output)
|
|
1206
|
-
? (deserializeToolExecuteResult(receipt.output) as T)
|
|
1207
|
-
: (receipt.output as T);
|
|
1208
|
-
if (opts.markSkipped) {
|
|
1209
|
-
await opts.markSkipped(output);
|
|
1210
|
-
}
|
|
1211
|
-
return output;
|
|
1212
|
-
};
|
|
1213
|
-
const claimed = await this.claimRuntimeStepReceipt(
|
|
1214
|
-
receiptKey,
|
|
1215
1328
|
runId,
|
|
1216
|
-
opts.reclaimRunning === true,
|
|
1217
|
-
);
|
|
1218
|
-
if (claimed?.status === 'completed' || claimed?.status === 'skipped') {
|
|
1219
|
-
return await recoverCompletedReceipt(claimed);
|
|
1220
|
-
}
|
|
1221
|
-
if (!claimed) {
|
|
1222
|
-
const latest = await this.getRuntimeStepReceipt(receiptKey);
|
|
1223
|
-
if (latest?.status === 'completed' || latest?.status === 'skipped') {
|
|
1224
|
-
return await recoverCompletedReceipt(latest);
|
|
1225
|
-
}
|
|
1226
|
-
if (latest?.status === 'running') {
|
|
1227
|
-
// Running receipts are in-flight telemetry, not locks. Execute in
|
|
1228
|
-
// parallel and let completion reconcile against the latest receipt.
|
|
1229
|
-
} else {
|
|
1230
|
-
if (latest?.status === 'failed') {
|
|
1231
|
-
throw new Error(
|
|
1232
|
-
`ctx.${operation}(${id}): receipt is failed and could not be claimed: ${latest.error ?? 'unknown error'}.`,
|
|
1233
|
-
);
|
|
1234
|
-
}
|
|
1235
|
-
throw new Error(
|
|
1236
|
-
`ctx.${operation}(${id}): receipt claim did not return execution ownership.`,
|
|
1237
|
-
);
|
|
1238
|
-
}
|
|
1239
|
-
} else if (claimed.status === 'running') {
|
|
1240
|
-
// Existing running receipts do not grant exclusive execution ownership.
|
|
1241
|
-
// The completion path decides whether this attempt is still the newest.
|
|
1242
|
-
} else if (claimed.status === 'failed') {
|
|
1243
|
-
throw new Error(
|
|
1244
|
-
`ctx.${operation}(${id}): receipt is failed and could not be claimed: ${claimed.error ?? 'unknown error'}.`,
|
|
1245
|
-
);
|
|
1246
|
-
}
|
|
1247
|
-
|
|
1248
|
-
let result: T;
|
|
1249
|
-
try {
|
|
1250
|
-
result = await opts.execute();
|
|
1251
|
-
assertNoSecretTaint(result, `ctx.${operation} result`);
|
|
1252
|
-
} catch (error) {
|
|
1253
|
-
const failed = await this.failRuntimeStepReceipt(
|
|
1254
|
-
receiptKey,
|
|
1255
|
-
runId,
|
|
1256
|
-
this.formatRuntimeError(error),
|
|
1257
|
-
);
|
|
1258
|
-
if (!failed && this.#options.failRuntimeStepReceipt) {
|
|
1259
|
-
throw new Error(
|
|
1260
|
-
`ctx.${operation}(${id}): execution failed and failed receipt could not be persisted: ${this.formatRuntimeError(error)}`,
|
|
1261
|
-
);
|
|
1262
|
-
}
|
|
1263
|
-
throw error;
|
|
1264
|
-
}
|
|
1265
|
-
// Persist tool results in their serialized form so the recovery path can
|
|
1266
|
-
// rebuild the live ToolExecuteResult wrapper (see recoverCompletedReceipt).
|
|
1267
|
-
const completed = await this.completeRuntimeStepReceipt(
|
|
1268
1329
|
receiptKey,
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1330
|
+
store: this.durableReceiptExecutionStore(),
|
|
1331
|
+
force: opts.force,
|
|
1332
|
+
repairRunningReceiptForSameRun: opts.repairRunningReceiptForSameRun,
|
|
1333
|
+
repairRunningReceiptForSameRunAfterWaitTimeout:
|
|
1334
|
+
opts.repairRunningReceiptForSameRunAfterWaitTimeout,
|
|
1335
|
+
reclaimRunning: opts.reclaimRunning,
|
|
1336
|
+
markSkipped: opts.markSkipped,
|
|
1337
|
+
onRecovered: opts.onRecovered,
|
|
1338
|
+
onClaimedResult: opts.onClaimedResult,
|
|
1339
|
+
formatError: (error) => this.formatRuntimeError(error),
|
|
1340
|
+
log: (message) => this.log(message),
|
|
1341
|
+
execute: opts.execute,
|
|
1342
|
+
});
|
|
1282
1343
|
}
|
|
1283
1344
|
|
|
1284
1345
|
private get currentRunId(): string {
|
|
@@ -1402,29 +1463,6 @@ export class PlayContextImpl {
|
|
|
1402
1463
|
});
|
|
1403
1464
|
}
|
|
1404
1465
|
|
|
1405
|
-
private emitQueuedInlineWaterfallSteps(
|
|
1406
|
-
rowId: number,
|
|
1407
|
-
key: string | null,
|
|
1408
|
-
tableNamespace: string | null,
|
|
1409
|
-
spec: InlineWaterfallSpec,
|
|
1410
|
-
): void {
|
|
1411
|
-
for (const step of spec.steps) {
|
|
1412
|
-
this.emitCellUpdate({
|
|
1413
|
-
rowId,
|
|
1414
|
-
key,
|
|
1415
|
-
tableNamespace,
|
|
1416
|
-
columnName: sqlSafePlayColumnName(`${spec.id}.${step.id}`),
|
|
1417
|
-
status: 'queued',
|
|
1418
|
-
stage: step.id,
|
|
1419
|
-
provider: isInlineWaterfallToolStep(step) ? step.toolId : 'code',
|
|
1420
|
-
producer: isInlineWaterfallToolStep(step)
|
|
1421
|
-
? { kind: 'tool', toolId: step.toolId, displayName: step.toolId }
|
|
1422
|
-
: { kind: 'code', id: step.id, displayName: step.id },
|
|
1423
|
-
value: null,
|
|
1424
|
-
});
|
|
1425
|
-
}
|
|
1426
|
-
}
|
|
1427
|
-
|
|
1428
1466
|
private isCompletedFieldValue(value: unknown): boolean {
|
|
1429
1467
|
return (
|
|
1430
1468
|
value !== null &&
|
|
@@ -1433,48 +1471,6 @@ export class PlayContextImpl {
|
|
|
1433
1471
|
);
|
|
1434
1472
|
}
|
|
1435
1473
|
|
|
1436
|
-
private existingFieldReuseDecision(
|
|
1437
|
-
row: Record<string, unknown>,
|
|
1438
|
-
fieldName: string,
|
|
1439
|
-
policies?: CellStalenessPolicyByField,
|
|
1440
|
-
authoredPolicies?: AuthoredCellStalenessPolicyByField,
|
|
1441
|
-
): {
|
|
1442
|
-
reuse: boolean;
|
|
1443
|
-
completedAt?: number;
|
|
1444
|
-
stalenessMeta?: Pick<CellStalenessMeta, 'staleAfterSeconds' | 'staleAt'>;
|
|
1445
|
-
} {
|
|
1446
|
-
const hasValue =
|
|
1447
|
-
Object.prototype.hasOwnProperty.call(row, fieldName) &&
|
|
1448
|
-
this.isCompletedFieldValue(row[fieldName]);
|
|
1449
|
-
const meta = this.cellMetaForField(row, fieldName);
|
|
1450
|
-
const completedAt =
|
|
1451
|
-
typeof meta?.completedAt === 'number' && Number.isFinite(meta.completedAt)
|
|
1452
|
-
? meta.completedAt
|
|
1453
|
-
: undefined;
|
|
1454
|
-
const stalenessMeta = resolveReusableCellMetaForCurrentPolicy({
|
|
1455
|
-
hasValue,
|
|
1456
|
-
value: row[fieldName],
|
|
1457
|
-
meta,
|
|
1458
|
-
policy: authoredPolicies?.[fieldName],
|
|
1459
|
-
});
|
|
1460
|
-
const decisionMeta =
|
|
1461
|
-
Object.keys(stalenessMeta).length > 0
|
|
1462
|
-
? { ...meta, ...stalenessMeta }
|
|
1463
|
-
: meta;
|
|
1464
|
-
const reuse =
|
|
1465
|
-
shouldRecomputeCell({
|
|
1466
|
-
hasValue,
|
|
1467
|
-
value: row[fieldName],
|
|
1468
|
-
meta: decisionMeta,
|
|
1469
|
-
policy: policies?.[fieldName],
|
|
1470
|
-
}).action === 'reuse';
|
|
1471
|
-
return {
|
|
1472
|
-
reuse,
|
|
1473
|
-
...(completedAt !== undefined ? { completedAt } : {}),
|
|
1474
|
-
...(Object.keys(stalenessMeta).length > 0 ? { stalenessMeta } : {}),
|
|
1475
|
-
};
|
|
1476
|
-
}
|
|
1477
|
-
|
|
1478
1474
|
// --- Tool-result cells survive the dataset persist/resume boundary ---
|
|
1479
1475
|
// A ToolExecuteResult carries live getter methods that cannot be serialized
|
|
1480
1476
|
// to durable storage. We store its serialized form on write and rehydrate it
|
|
@@ -1544,24 +1540,6 @@ export class PlayContextImpl {
|
|
|
1544
1540
|
: null;
|
|
1545
1541
|
}
|
|
1546
1542
|
|
|
1547
|
-
private cellPoliciesForMapDefinition(
|
|
1548
|
-
definition: MapFieldDefinition<Record<string, unknown>>,
|
|
1549
|
-
): CellStalenessPolicyByField {
|
|
1550
|
-
const raw = (definition as Record<symbol, unknown>)[
|
|
1551
|
-
CELL_STALENESS_POLICIES
|
|
1552
|
-
];
|
|
1553
|
-
return normalizeAuthoredCellPolicyMap(raw);
|
|
1554
|
-
}
|
|
1555
|
-
|
|
1556
|
-
private authoredCellPoliciesForMapDefinition(
|
|
1557
|
-
definition: MapFieldDefinition<Record<string, unknown>>,
|
|
1558
|
-
): AuthoredCellStalenessPolicyByField {
|
|
1559
|
-
const raw = (definition as Record<symbol, unknown>)[
|
|
1560
|
-
CELL_STALENESS_POLICIES
|
|
1561
|
-
];
|
|
1562
|
-
return authoredCellPolicyMap(raw);
|
|
1563
|
-
}
|
|
1564
|
-
|
|
1565
1543
|
private toVisibleDataPatch(
|
|
1566
1544
|
fields: Record<string, unknown>,
|
|
1567
1545
|
): Record<string, unknown> {
|
|
@@ -1579,31 +1557,14 @@ export class PlayContextImpl {
|
|
|
1579
1557
|
return String(error);
|
|
1580
1558
|
}
|
|
1581
1559
|
|
|
1582
|
-
private
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
if (count <= WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT) {
|
|
1591
|
-
this.log(
|
|
1592
|
-
` Row ${input.rowId}: found with ${input.provider} (${count}/${WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT} sample)`,
|
|
1593
|
-
);
|
|
1594
|
-
if (count === WATERFALL_ROW_MATCH_LOG_SAMPLE_LIMIT) {
|
|
1595
|
-
this.log(
|
|
1596
|
-
` Further per-row matches for ${input.queueKey} will be summarized every ${WATERFALL_ROW_MATCH_LOG_INTERVAL.toLocaleString()} hits.`,
|
|
1597
|
-
);
|
|
1598
|
-
}
|
|
1599
|
-
return;
|
|
1600
|
-
}
|
|
1601
|
-
|
|
1602
|
-
if (count % WATERFALL_ROW_MATCH_LOG_INTERVAL === 0) {
|
|
1603
|
-
this.log(
|
|
1604
|
-
` ${count.toLocaleString()} rows matched for ${input.queueKey}; latest row ${input.rowId} with ${input.provider}`,
|
|
1605
|
-
);
|
|
1606
|
-
}
|
|
1560
|
+
private effectiveToolCallCachePolicy(options?: ToolCallOptions): {
|
|
1561
|
+
force: boolean;
|
|
1562
|
+
staleAfterSeconds?: number | null;
|
|
1563
|
+
} {
|
|
1564
|
+
return {
|
|
1565
|
+
force: options?.force === true,
|
|
1566
|
+
staleAfterSeconds: options?.staleAfterSeconds ?? null,
|
|
1567
|
+
};
|
|
1607
1568
|
}
|
|
1608
1569
|
|
|
1609
1570
|
private summarizeBatchSizes(sizes: readonly number[]): string {
|
|
@@ -1675,30 +1636,24 @@ export class PlayContextImpl {
|
|
|
1675
1636
|
result: unknown;
|
|
1676
1637
|
metadata?: ToolResultMetadataInput | null;
|
|
1677
1638
|
meta?: Record<string, unknown>;
|
|
1678
|
-
|
|
1679
|
-
source: 'live' | 'checkpoint' | 'cache';
|
|
1680
|
-
cacheKey?: string;
|
|
1639
|
+
execution: ToolResultExecutionMetadata;
|
|
1681
1640
|
}): Promise<ToolExecuteResult> {
|
|
1682
1641
|
if (isToolExecuteResult(input.result)) {
|
|
1683
|
-
return cloneToolExecuteResultWithExecution(input.result,
|
|
1684
|
-
idempotent: true,
|
|
1685
|
-
cached: input.cached,
|
|
1686
|
-
source: input.source,
|
|
1687
|
-
...(input.cacheKey ? { cacheKey: input.cacheKey } : {}),
|
|
1688
|
-
});
|
|
1642
|
+
return cloneToolExecuteResultWithExecution(input.result, input.execution);
|
|
1689
1643
|
}
|
|
1644
|
+
const publicToolResult = publicToolResponseEnvelope(input.result);
|
|
1690
1645
|
return createToolExecuteResult({
|
|
1691
|
-
status: input.status,
|
|
1646
|
+
status: publicToolResult?.status ?? input.status,
|
|
1692
1647
|
jobId: input.jobId,
|
|
1693
|
-
result:
|
|
1648
|
+
result: publicToolResult
|
|
1649
|
+
? {
|
|
1650
|
+
data: publicToolResult.raw,
|
|
1651
|
+
...(publicToolResult.meta ? { meta: publicToolResult.meta } : {}),
|
|
1652
|
+
}
|
|
1653
|
+
: input.result,
|
|
1694
1654
|
metadata:
|
|
1695
1655
|
input.metadata ?? (await this.resolveToolResultMetadata(input.toolId)),
|
|
1696
|
-
execution:
|
|
1697
|
-
idempotent: true,
|
|
1698
|
-
cached: input.cached,
|
|
1699
|
-
source: input.source,
|
|
1700
|
-
...(input.cacheKey ? { cacheKey: input.cacheKey } : {}),
|
|
1701
|
-
},
|
|
1656
|
+
execution: input.execution,
|
|
1702
1657
|
meta: input.meta,
|
|
1703
1658
|
});
|
|
1704
1659
|
}
|
|
@@ -1710,13 +1665,9 @@ export class PlayContextImpl {
|
|
|
1710
1665
|
metadata?: ToolResultMetadataInput | null,
|
|
1711
1666
|
jobId?: string,
|
|
1712
1667
|
meta?: Record<string, unknown>,
|
|
1713
|
-
): Promise<
|
|
1714
|
-
const cacheKey =
|
|
1715
|
-
|
|
1716
|
-
tableNamespace: request.tableNamespace,
|
|
1717
|
-
rowKey: request.rowKey ?? undefined,
|
|
1718
|
-
callId: request.callId,
|
|
1719
|
-
});
|
|
1668
|
+
): Promise<unknown> {
|
|
1669
|
+
const cacheKey = request.cacheKey;
|
|
1670
|
+
const receiptKey = request.receiptKey?.trim() || null;
|
|
1720
1671
|
const wrapped = await this.wrapToolExecutionResult({
|
|
1721
1672
|
toolId,
|
|
1722
1673
|
status: result == null ? 'no_result' : 'completed',
|
|
@@ -1724,15 +1675,128 @@ export class PlayContextImpl {
|
|
|
1724
1675
|
result,
|
|
1725
1676
|
metadata,
|
|
1726
1677
|
meta,
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1678
|
+
execution: toolExecutionMetadataForOutcome({
|
|
1679
|
+
kind: 'live',
|
|
1680
|
+
cacheKey,
|
|
1681
|
+
receiptKey,
|
|
1682
|
+
}),
|
|
1683
|
+
});
|
|
1684
|
+
const completed = receiptKey
|
|
1685
|
+
? (
|
|
1686
|
+
await this.completeRuntimeStepReceipts([
|
|
1687
|
+
{
|
|
1688
|
+
key: receiptKey,
|
|
1689
|
+
runId: this.currentRunId,
|
|
1690
|
+
output: serializeToolExecuteResult(wrapped),
|
|
1691
|
+
},
|
|
1692
|
+
])
|
|
1693
|
+
).get(receiptKey)
|
|
1694
|
+
: null;
|
|
1695
|
+
return await this.finalizeResolvedToolCall(
|
|
1696
|
+
toolId,
|
|
1697
|
+
request,
|
|
1698
|
+
wrapped,
|
|
1699
|
+
completed,
|
|
1700
|
+
);
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
private async resolveToolCallBatchResults(
|
|
1704
|
+
toolId: string,
|
|
1705
|
+
entries: Array<{
|
|
1706
|
+
request: ToolCallRequest;
|
|
1707
|
+
result: unknown | null;
|
|
1708
|
+
metadata?: ToolResultMetadataInput | null;
|
|
1709
|
+
jobId?: string;
|
|
1710
|
+
meta?: Record<string, unknown>;
|
|
1711
|
+
}>,
|
|
1712
|
+
): Promise<unknown[]> {
|
|
1713
|
+
const wrappedEntries = await Promise.all(
|
|
1714
|
+
entries.map(async (entry) => ({
|
|
1715
|
+
...entry,
|
|
1716
|
+
wrapped: await this.wrapToolExecutionResult({
|
|
1717
|
+
toolId,
|
|
1718
|
+
status: entry.result == null ? 'no_result' : 'completed',
|
|
1719
|
+
jobId: entry.jobId,
|
|
1720
|
+
result: entry.result,
|
|
1721
|
+
metadata: entry.metadata,
|
|
1722
|
+
meta: entry.meta,
|
|
1723
|
+
execution: toolExecutionMetadataForOutcome({
|
|
1724
|
+
kind: 'live',
|
|
1725
|
+
cacheKey: entry.request.cacheKey,
|
|
1726
|
+
receiptKey: entry.request.receiptKey,
|
|
1727
|
+
}),
|
|
1728
|
+
}),
|
|
1729
|
+
})),
|
|
1730
|
+
);
|
|
1731
|
+
const receiptInputs = wrappedEntries.flatMap((entry) => {
|
|
1732
|
+
const receiptKey = entry.request.receiptKey?.trim() || null;
|
|
1733
|
+
return receiptKey
|
|
1734
|
+
? [
|
|
1735
|
+
{
|
|
1736
|
+
key: receiptKey,
|
|
1737
|
+
runId: this.currentRunId,
|
|
1738
|
+
output: serializeToolExecuteResult(entry.wrapped),
|
|
1739
|
+
},
|
|
1740
|
+
]
|
|
1741
|
+
: [];
|
|
1730
1742
|
});
|
|
1731
|
-
|
|
1743
|
+
const completedByKey =
|
|
1744
|
+
receiptInputs.length > 0
|
|
1745
|
+
? await this.completeRuntimeStepReceipts(receiptInputs)
|
|
1746
|
+
: new Map<string, RuntimeStepReceipt>();
|
|
1747
|
+
const results: unknown[] = [];
|
|
1748
|
+
for (const entry of wrappedEntries) {
|
|
1749
|
+
const receiptKey = entry.request.receiptKey?.trim() || null;
|
|
1750
|
+
results.push(
|
|
1751
|
+
await this.finalizeResolvedToolCall(
|
|
1752
|
+
toolId,
|
|
1753
|
+
entry.request,
|
|
1754
|
+
entry.wrapped,
|
|
1755
|
+
receiptKey ? completedByKey.get(receiptKey) : null,
|
|
1756
|
+
),
|
|
1757
|
+
);
|
|
1758
|
+
}
|
|
1759
|
+
return results;
|
|
1760
|
+
}
|
|
1761
|
+
|
|
1762
|
+
private async finalizeResolvedToolCall(
|
|
1763
|
+
toolId: string,
|
|
1764
|
+
request: ToolCallRequest,
|
|
1765
|
+
wrapped: ToolExecuteResult,
|
|
1766
|
+
completed: RuntimeStepReceipt | null | undefined,
|
|
1767
|
+
): Promise<unknown> {
|
|
1768
|
+
const cacheKey = request.cacheKey;
|
|
1769
|
+
const finalWrapped =
|
|
1770
|
+
(completed?.status === 'completed' || completed?.status === 'skipped') &&
|
|
1771
|
+
completed.runId !== this.currentRunId
|
|
1772
|
+
? await this.wrapToolExecutionResult({
|
|
1773
|
+
toolId,
|
|
1774
|
+
status:
|
|
1775
|
+
completed.output === null || completed.output === undefined
|
|
1776
|
+
? 'no_result'
|
|
1777
|
+
: 'completed',
|
|
1778
|
+
result: this.runtimeReceiptOutput(completed),
|
|
1779
|
+
execution: toolExecutionMetadataForOutcome(
|
|
1780
|
+
completed.runId === this.currentRunId
|
|
1781
|
+
? {
|
|
1782
|
+
kind: 'live',
|
|
1783
|
+
cacheKey,
|
|
1784
|
+
receiptKey: request.receiptKey,
|
|
1785
|
+
}
|
|
1786
|
+
: {
|
|
1787
|
+
kind: 'cache',
|
|
1788
|
+
cacheKey,
|
|
1789
|
+
receiptKey: request.receiptKey ?? '',
|
|
1790
|
+
attachedToReceiptKey: request.receiptKey,
|
|
1791
|
+
},
|
|
1792
|
+
),
|
|
1793
|
+
})
|
|
1794
|
+
: wrapped;
|
|
1795
|
+
this.cacheToolResult(toolId, cacheKey, finalWrapped);
|
|
1732
1796
|
|
|
1733
1797
|
const resolver = this.toolCallResolvers.get(request.callId);
|
|
1734
1798
|
if (resolver) {
|
|
1735
|
-
resolver.resolve(
|
|
1799
|
+
resolver.resolve(finalWrapped);
|
|
1736
1800
|
this.toolCallResolvers.delete(request.callId);
|
|
1737
1801
|
}
|
|
1738
1802
|
|
|
@@ -1748,17 +1812,39 @@ export class PlayContextImpl {
|
|
|
1748
1812
|
error: null,
|
|
1749
1813
|
dataPatch: {},
|
|
1750
1814
|
});
|
|
1815
|
+
return finalWrapped;
|
|
1751
1816
|
}
|
|
1752
1817
|
|
|
1753
|
-
private rejectToolCall(
|
|
1818
|
+
private async rejectToolCall(
|
|
1754
1819
|
toolId: string,
|
|
1755
1820
|
request: ToolCallRequest,
|
|
1756
1821
|
error: unknown,
|
|
1757
|
-
|
|
1822
|
+
options?: { persistReceiptFailure?: boolean },
|
|
1823
|
+
): Promise<void> {
|
|
1758
1824
|
const message = this.formatRuntimeError(error);
|
|
1825
|
+
let rejectionError = error;
|
|
1826
|
+
const receiptKey = request.receiptKey?.trim() || null;
|
|
1827
|
+
if (receiptKey && options?.persistReceiptFailure !== false) {
|
|
1828
|
+
try {
|
|
1829
|
+
await this.failRuntimeStepReceipts([
|
|
1830
|
+
{
|
|
1831
|
+
key: receiptKey,
|
|
1832
|
+
runId: this.currentRunId,
|
|
1833
|
+
error: message,
|
|
1834
|
+
},
|
|
1835
|
+
]);
|
|
1836
|
+
} catch (receiptError) {
|
|
1837
|
+
rejectionError = new AggregateError(
|
|
1838
|
+
[error, receiptError],
|
|
1839
|
+
'Tool call failed and durable receipt could not be marked failed',
|
|
1840
|
+
);
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1759
1843
|
const resolver = this.toolCallResolvers.get(request.callId);
|
|
1760
1844
|
if (resolver) {
|
|
1761
|
-
resolver.reject(
|
|
1845
|
+
resolver.reject(
|
|
1846
|
+
rejectionError instanceof Error ? rejectionError : new Error(message),
|
|
1847
|
+
);
|
|
1762
1848
|
this.toolCallResolvers.delete(request.callId);
|
|
1763
1849
|
}
|
|
1764
1850
|
|
|
@@ -1875,22 +1961,6 @@ export class PlayContextImpl {
|
|
|
1875
1961
|
options?: DatasetOptions<T>,
|
|
1876
1962
|
): Promise<PlayDataset<Record<string, unknown>>> {
|
|
1877
1963
|
const definition = this.stepProgramToMapDefinition(program);
|
|
1878
|
-
Object.defineProperty(definition, CELL_STALENESS_POLICIES, {
|
|
1879
|
-
value: Object.fromEntries(
|
|
1880
|
-
program.steps.map((step) => [
|
|
1881
|
-
step.name,
|
|
1882
|
-
{
|
|
1883
|
-
...(step.recompute === true ? { recompute: true } : {}),
|
|
1884
|
-
...(step.recomputeOnError === true
|
|
1885
|
-
? { recomputeOnError: true }
|
|
1886
|
-
: {}),
|
|
1887
|
-
...(step.staleAfterSeconds === undefined
|
|
1888
|
-
? {}
|
|
1889
|
-
: { staleAfterSeconds: step.staleAfterSeconds }),
|
|
1890
|
-
},
|
|
1891
|
-
]),
|
|
1892
|
-
) satisfies AuthoredCellStalenessPolicyByField,
|
|
1893
|
-
});
|
|
1894
1964
|
return this.runMapDefinition(key, items, definition, options);
|
|
1895
1965
|
}
|
|
1896
1966
|
|
|
@@ -1940,7 +2010,6 @@ export class PlayContextImpl {
|
|
|
1940
2010
|
let rawItems: Record<string, unknown>[] = [];
|
|
1941
2011
|
let itemsToProcess: Array<Record<string, unknown>> = [];
|
|
1942
2012
|
let itemOriginalIndexes: number[] = [];
|
|
1943
|
-
let completedItemsByKey: Map<string, Record<string, unknown>> | null = null;
|
|
1944
2013
|
const datasetColumnNames = Object.keys(input);
|
|
1945
2014
|
const stripFieldOutputs = (row: Record<string, unknown>) => {
|
|
1946
2015
|
const stripped = cloneCsvAliasedRow(row);
|
|
@@ -2066,7 +2135,6 @@ export class PlayContextImpl {
|
|
|
2066
2135
|
playId: this.#options.playId,
|
|
2067
2136
|
runId: this.#options.runId,
|
|
2068
2137
|
staticPipeline: this.#options.staticPipeline,
|
|
2069
|
-
cellPolicies: this.cellPoliciesForMapDefinition(input),
|
|
2070
2138
|
},
|
|
2071
2139
|
);
|
|
2072
2140
|
resolvedTableNamespace = normalizeTableNamespace(
|
|
@@ -2109,21 +2177,6 @@ export class PlayContextImpl {
|
|
|
2109
2177
|
});
|
|
2110
2178
|
itemsToProcess = pendingItems.map((item) => item.row);
|
|
2111
2179
|
itemOriginalIndexes = pendingItems.map((item) => item.index);
|
|
2112
|
-
if (mapStartResult.completedRows.length > 0) {
|
|
2113
|
-
completedItemsByKey = new Map();
|
|
2114
|
-
for (
|
|
2115
|
-
let index = 0;
|
|
2116
|
-
index < mapStartResult.completedRows.length;
|
|
2117
|
-
index += 1
|
|
2118
|
-
) {
|
|
2119
|
-
const row = mapStartResult.completedRows[index]!;
|
|
2120
|
-
const rowKey = persistedRowIdentity(row, index);
|
|
2121
|
-
if (rowKey) completedItemsByKey.set(rowKey, row);
|
|
2122
|
-
}
|
|
2123
|
-
this.log(
|
|
2124
|
-
`Resuming: ${mapStartResult.completedRows.length} already completed, ${itemsToProcess.length} pending`,
|
|
2125
|
-
);
|
|
2126
|
-
}
|
|
2127
2180
|
}
|
|
2128
2181
|
|
|
2129
2182
|
const mapScope = this.createMapExecutionScope({
|
|
@@ -2131,8 +2184,7 @@ export class PlayContextImpl {
|
|
|
2131
2184
|
artifactTableNamespace: resolvedTableNamespace,
|
|
2132
2185
|
explicitKey: explicitKeyResolver,
|
|
2133
2186
|
});
|
|
2134
|
-
const completedRowKeys =
|
|
2135
|
-
completedItemsByKey != null ? [...completedItemsByKey.keys()] : [];
|
|
2187
|
+
const completedRowKeys: string[] = [];
|
|
2136
2188
|
const pendingRowKeys = itemsToProcess.map((item, index) =>
|
|
2137
2189
|
rowIdentity(this.toOutputRow(item), itemOriginalIndexes[index] ?? index),
|
|
2138
2190
|
);
|
|
@@ -2230,7 +2282,7 @@ export class PlayContextImpl {
|
|
|
2230
2282
|
options?.description,
|
|
2231
2283
|
{
|
|
2232
2284
|
totalRows: totalInputCount,
|
|
2233
|
-
completedRows:
|
|
2285
|
+
completedRows: duplicateReuseCount,
|
|
2234
2286
|
},
|
|
2235
2287
|
{
|
|
2236
2288
|
onRowError: options?.onRowError,
|
|
@@ -2274,7 +2326,6 @@ export class PlayContextImpl {
|
|
|
2274
2326
|
this.toPublicOutputRow(row.data),
|
|
2275
2327
|
);
|
|
2276
2328
|
const results =
|
|
2277
|
-
!completedItemsByKey &&
|
|
2278
2329
|
mapResult.failedRows.length === 0 &&
|
|
2279
2330
|
directCompletedResults.length === rawItems.length
|
|
2280
2331
|
? directCompletedResults
|
|
@@ -2284,15 +2335,11 @@ export class PlayContextImpl {
|
|
|
2284
2335
|
return [];
|
|
2285
2336
|
}
|
|
2286
2337
|
return [
|
|
2287
|
-
this.toPublicOutputRow(
|
|
2288
|
-
resultsByKey.get(rowKey) ??
|
|
2289
|
-
completedItemsByKey?.get(rowKey) ??
|
|
2290
|
-
rawItem,
|
|
2291
|
-
),
|
|
2338
|
+
this.toPublicOutputRow(resultsByKey.get(rowKey) ?? rawItem),
|
|
2292
2339
|
];
|
|
2293
2340
|
});
|
|
2294
2341
|
const executedCount = rowsToExecute.length;
|
|
2295
|
-
const reusedCount =
|
|
2342
|
+
const reusedCount = duplicateReuseCount;
|
|
2296
2343
|
|
|
2297
2344
|
// Persist executed rows to the tenant runtime sheet — the sheet is the
|
|
2298
2345
|
// source of truth, not this in-memory results array. Chunked by rows AND
|
|
@@ -2403,9 +2450,6 @@ export class PlayContextImpl {
|
|
|
2403
2450
|
},
|
|
2404
2451
|
): Promise<FieldMapRunResult> {
|
|
2405
2452
|
const fieldEntries = Object.entries(definition);
|
|
2406
|
-
const cellPolicies = this.cellPoliciesForMapDefinition(definition);
|
|
2407
|
-
const authoredCellPolicies =
|
|
2408
|
-
this.authoredCellPoliciesForMapDefinition(definition);
|
|
2409
2453
|
const datasetColumnNames = fieldEntries.map(([fieldName]) => fieldName);
|
|
2410
2454
|
const visibleFields = fieldEntries
|
|
2411
2455
|
.map(([fieldName]) => fieldName)
|
|
@@ -2454,7 +2498,7 @@ export class PlayContextImpl {
|
|
|
2454
2498
|
|
|
2455
2499
|
if (completedRows > 0 || pendingRows !== totalRows) {
|
|
2456
2500
|
this.log(
|
|
2457
|
-
`Starting map over ${totalRows} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace}; ${completedRows}
|
|
2501
|
+
`Starting map over ${totalRows} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace}; ${completedRows} duplicate keys skipped; ${pendingRows} pending)`,
|
|
2458
2502
|
);
|
|
2459
2503
|
} else {
|
|
2460
2504
|
this.log(
|
|
@@ -2587,8 +2631,6 @@ export class PlayContextImpl {
|
|
|
2587
2631
|
visibleFields,
|
|
2588
2632
|
normalizedTableNamespace,
|
|
2589
2633
|
executionRowKey,
|
|
2590
|
-
cellPolicies,
|
|
2591
|
-
authoredCellPolicies,
|
|
2592
2634
|
);
|
|
2593
2635
|
// One batched frame update for the whole pure map. The per-row loop
|
|
2594
2636
|
// this replaces ran AFTER every row had already computed, emitting 150k
|
|
@@ -2612,7 +2654,7 @@ export class PlayContextImpl {
|
|
|
2612
2654
|
this.lastDatasetStep = datasetStep;
|
|
2613
2655
|
this.activeDatasetStep = null;
|
|
2614
2656
|
this.log(
|
|
2615
|
-
`Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows}
|
|
2657
|
+
`Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} duplicate keys skipped)`,
|
|
2616
2658
|
);
|
|
2617
2659
|
return {
|
|
2618
2660
|
completedRows: results.map((row, index) =>
|
|
@@ -2674,32 +2716,6 @@ export class PlayContextImpl {
|
|
|
2674
2716
|
try {
|
|
2675
2717
|
for (const [fieldName, resolver] of fieldEntries) {
|
|
2676
2718
|
activeFieldName = fieldName;
|
|
2677
|
-
const reuseDecision = this.existingFieldReuseDecision(
|
|
2678
|
-
baseRow,
|
|
2679
|
-
fieldName,
|
|
2680
|
-
cellPolicies,
|
|
2681
|
-
authoredCellPolicies,
|
|
2682
|
-
);
|
|
2683
|
-
if (reuseDecision.reuse) {
|
|
2684
|
-
const reusedValue = baseRow[fieldName];
|
|
2685
|
-
computedFields[fieldName] = reusedValue;
|
|
2686
|
-
this.rowStates.get(idx)?.results.set(fieldName, reusedValue);
|
|
2687
|
-
this.emitScopedFieldMetaUpdate({
|
|
2688
|
-
rowId: idx,
|
|
2689
|
-
key: rowKey,
|
|
2690
|
-
tableNamespace: normalizedTableNamespace,
|
|
2691
|
-
fieldName,
|
|
2692
|
-
status: 'cached',
|
|
2693
|
-
reused: true,
|
|
2694
|
-
...(reuseDecision.completedAt !== undefined
|
|
2695
|
-
? { completedAt: reuseDecision.completedAt }
|
|
2696
|
-
: {}),
|
|
2697
|
-
...(reuseDecision.stalenessMeta ?? {}),
|
|
2698
|
-
dataPatch: {},
|
|
2699
|
-
});
|
|
2700
|
-
continue;
|
|
2701
|
-
}
|
|
2702
|
-
|
|
2703
2719
|
this.emitScopedFieldMetaUpdate({
|
|
2704
2720
|
rowId: idx,
|
|
2705
2721
|
key: rowKey,
|
|
@@ -2805,12 +2821,6 @@ export class PlayContextImpl {
|
|
|
2805
2821
|
const cellValue = this.serializeCellValue(value);
|
|
2806
2822
|
computedFields[fieldName] = cellValue;
|
|
2807
2823
|
this.rowStates.get(idx)?.results.set(fieldName, cellValue);
|
|
2808
|
-
const completedAt = Date.now();
|
|
2809
|
-
const stalenessMeta = resolveCompletedCellStalenessMeta({
|
|
2810
|
-
policy: authoredCellPolicies[fieldName],
|
|
2811
|
-
value,
|
|
2812
|
-
completedAt,
|
|
2813
|
-
});
|
|
2814
2824
|
this.emitScopedFieldMetaUpdate({
|
|
2815
2825
|
rowId: idx,
|
|
2816
2826
|
key: rowKey,
|
|
@@ -2818,8 +2828,7 @@ export class PlayContextImpl {
|
|
|
2818
2828
|
fieldName,
|
|
2819
2829
|
status: 'completed',
|
|
2820
2830
|
stage: 'completed',
|
|
2821
|
-
completedAt,
|
|
2822
|
-
...stalenessMeta,
|
|
2831
|
+
completedAt: Date.now(),
|
|
2823
2832
|
dataPatch: shouldPersistMapCellField(fieldName)
|
|
2824
2833
|
? { [fieldName]: cellValue }
|
|
2825
2834
|
: {},
|
|
@@ -2986,7 +2995,7 @@ export class PlayContextImpl {
|
|
|
2986
2995
|
});
|
|
2987
2996
|
}
|
|
2988
2997
|
this.log(
|
|
2989
|
-
`Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows}
|
|
2998
|
+
`Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} duplicate keys skipped)`,
|
|
2990
2999
|
);
|
|
2991
3000
|
return {
|
|
2992
3001
|
completedRows: [...completedRowsToPersist].sort(
|
|
@@ -3157,9 +3166,7 @@ export class PlayContextImpl {
|
|
|
3157
3166
|
|
|
3158
3167
|
const source = Function.prototype.toString.call(resolver);
|
|
3159
3168
|
return (
|
|
3160
|
-
!source.includes('.tools.execute(') &&
|
|
3161
|
-
!source.includes('.waterfall(') &&
|
|
3162
|
-
!source.includes('.runPlay(')
|
|
3169
|
+
!source.includes('.tools.execute(') && !source.includes('.runPlay(')
|
|
3163
3170
|
);
|
|
3164
3171
|
});
|
|
3165
3172
|
}
|
|
@@ -3170,8 +3177,6 @@ export class PlayContextImpl {
|
|
|
3170
3177
|
visibleFields: string[],
|
|
3171
3178
|
tableNamespace: string,
|
|
3172
3179
|
rowIdentity: (row: Record<string, unknown>, index: number) => string,
|
|
3173
|
-
cellPolicies?: CellStalenessPolicyByField,
|
|
3174
|
-
authoredCellPolicies?: AuthoredCellStalenessPolicyByField,
|
|
3175
3180
|
): Promise<Array<Record<string, unknown>>> {
|
|
3176
3181
|
const results: Array<Record<string, unknown>> = [];
|
|
3177
3182
|
this.pureMapExecutionActive = true;
|
|
@@ -3195,25 +3200,6 @@ export class PlayContextImpl {
|
|
|
3195
3200
|
try {
|
|
3196
3201
|
for (const [fieldName, resolver] of fieldEntries) {
|
|
3197
3202
|
activeFieldName = fieldName;
|
|
3198
|
-
const reuseDecision = this.existingFieldReuseDecision(
|
|
3199
|
-
baseRow,
|
|
3200
|
-
fieldName,
|
|
3201
|
-
cellPolicies,
|
|
3202
|
-
authoredCellPolicies,
|
|
3203
|
-
);
|
|
3204
|
-
if (reuseDecision.reuse) {
|
|
3205
|
-
computedFields[fieldName] = baseRow[fieldName];
|
|
3206
|
-
rowCellMetaPatch[fieldName] = {
|
|
3207
|
-
status: 'cached',
|
|
3208
|
-
reused: true,
|
|
3209
|
-
...(reuseDecision.completedAt !== undefined
|
|
3210
|
-
? { completedAt: reuseDecision.completedAt }
|
|
3211
|
-
: {}),
|
|
3212
|
-
...(reuseDecision.stalenessMeta ?? {}),
|
|
3213
|
-
};
|
|
3214
|
-
continue;
|
|
3215
|
-
}
|
|
3216
|
-
|
|
3217
3203
|
const value = await this.resolveMapFieldValue(
|
|
3218
3204
|
resolver,
|
|
3219
3205
|
item,
|
|
@@ -3224,20 +3210,13 @@ export class PlayContextImpl {
|
|
|
3224
3210
|
this.previousCellForField(baseRow, fieldName),
|
|
3225
3211
|
);
|
|
3226
3212
|
computedFields[fieldName] = this.serializeCellValue(value);
|
|
3227
|
-
const completedAt = Date.now();
|
|
3228
|
-
const stalenessMeta = resolveCompletedCellStalenessMeta({
|
|
3229
|
-
policy: authoredCellPolicies?.[fieldName],
|
|
3230
|
-
value,
|
|
3231
|
-
completedAt,
|
|
3232
|
-
});
|
|
3233
3213
|
if (shouldPersistMapCellField(fieldName)) {
|
|
3234
3214
|
rowDataPatch[fieldName] = computedFields[fieldName];
|
|
3235
3215
|
}
|
|
3236
3216
|
rowCellMetaPatch[fieldName] = {
|
|
3237
3217
|
status: 'completed',
|
|
3238
3218
|
stage: 'completed',
|
|
3239
|
-
completedAt,
|
|
3240
|
-
...stalenessMeta,
|
|
3219
|
+
completedAt: Date.now(),
|
|
3241
3220
|
};
|
|
3242
3221
|
}
|
|
3243
3222
|
|
|
@@ -3294,14 +3273,13 @@ export class PlayContextImpl {
|
|
|
3294
3273
|
private initializeRowStates(items: readonly unknown[]): void {
|
|
3295
3274
|
for (let idx = 0; idx < items.length; idx += 1) {
|
|
3296
3275
|
this.rowStates.set(idx, {
|
|
3297
|
-
waterfalls: new Map(),
|
|
3298
3276
|
results: new Map(),
|
|
3299
3277
|
});
|
|
3300
3278
|
}
|
|
3301
3279
|
}
|
|
3302
3280
|
|
|
3303
3281
|
private async drainQueuedWork<T>(promises: Promise<T>[]): Promise<void> {
|
|
3304
|
-
// Drain loop: each pass resolves queued
|
|
3282
|
+
// Drain loop: each pass resolves queued tool calls.
|
|
3305
3283
|
// When a batch resolves, rows resume and may queue MORE calls
|
|
3306
3284
|
// (e.g. row does tools.execute('a') then tools.execute('b') sequentially).
|
|
3307
3285
|
// We keep looping until nothing new is queued and all rows finish.
|
|
@@ -3311,7 +3289,7 @@ export class PlayContextImpl {
|
|
|
3311
3289
|
// Promise.allSettled(promises) while there is no timer/socket/file handle can
|
|
3312
3290
|
// let Node exit 0 before main() emits the result envelope. That presents as
|
|
3313
3291
|
// "Local play runner produced no result" even though the real bug was a
|
|
3314
|
-
// map row waiting for queued tool
|
|
3292
|
+
// map row waiting for queued tool work. Always race row settlement
|
|
3315
3293
|
// against a small timer and re-check the queues instead of blocking forever
|
|
3316
3294
|
// on promises that may be waiting for this drain loop to do the next batch.
|
|
3317
3295
|
const raceSettled = () =>
|
|
@@ -3322,16 +3300,11 @@ export class PlayContextImpl {
|
|
|
3322
3300
|
|
|
3323
3301
|
let pass = 0;
|
|
3324
3302
|
while (true) {
|
|
3325
|
-
const hasPendingWork =
|
|
3326
|
-
this.waterfallQueue.size > 0 || this.toolCallQueue.length > 0;
|
|
3303
|
+
const hasPendingWork = this.toolCallQueue.length > 0;
|
|
3327
3304
|
|
|
3328
3305
|
if (!hasPendingWork) {
|
|
3329
3306
|
const status = await raceSettled();
|
|
3330
|
-
if (
|
|
3331
|
-
status === 'done' &&
|
|
3332
|
-
this.waterfallQueue.size === 0 &&
|
|
3333
|
-
this.toolCallQueue.length === 0
|
|
3334
|
-
) {
|
|
3307
|
+
if (status === 'done' && this.toolCallQueue.length === 0) {
|
|
3335
3308
|
break;
|
|
3336
3309
|
}
|
|
3337
3310
|
|
|
@@ -3346,13 +3319,9 @@ export class PlayContextImpl {
|
|
|
3346
3319
|
pass++;
|
|
3347
3320
|
this.log(` Batch pass ${pass}`);
|
|
3348
3321
|
this.log(
|
|
3349
|
-
` Queue depth before drain:
|
|
3322
|
+
` Queue depth before drain: tool_calls=${this.toolCallQueue.length}`,
|
|
3350
3323
|
);
|
|
3351
3324
|
|
|
3352
|
-
if (this.waterfallQueue.size > 0) {
|
|
3353
|
-
await this.executeBatchedWaterfalls();
|
|
3354
|
-
}
|
|
3355
|
-
|
|
3356
3325
|
if (this.toolCallQueue.length > 0) {
|
|
3357
3326
|
await this.executeBatchedToolCalls();
|
|
3358
3327
|
}
|
|
@@ -3420,99 +3389,6 @@ export class PlayContextImpl {
|
|
|
3420
3389
|
);
|
|
3421
3390
|
}
|
|
3422
3391
|
|
|
3423
|
-
async waterfall(
|
|
3424
|
-
toolNameOrSpec: string | InlineWaterfallSpec,
|
|
3425
|
-
input: Record<string, unknown>,
|
|
3426
|
-
opts?: WaterfallOptions,
|
|
3427
|
-
): Promise<unknown | null> {
|
|
3428
|
-
const store = rowContext.getStore();
|
|
3429
|
-
const toolName =
|
|
3430
|
-
typeof toolNameOrSpec === 'string' ? toolNameOrSpec : toolNameOrSpec.id;
|
|
3431
|
-
const baseQueueKey =
|
|
3432
|
-
typeof toolNameOrSpec === 'string'
|
|
3433
|
-
? toolNameOrSpec
|
|
3434
|
-
: `inline:${toolNameOrSpec.id}`;
|
|
3435
|
-
const inlineSpec = isInlineWaterfallSpec(toolNameOrSpec)
|
|
3436
|
-
? toolNameOrSpec
|
|
3437
|
-
: undefined;
|
|
3438
|
-
|
|
3439
|
-
if (this.pureMapExecutionActive && !store) {
|
|
3440
|
-
throw new Error(
|
|
3441
|
-
'ctx.waterfall() cannot run inside the pure-JS fast path. Call it directly in the map definition so the batching runtime can stay enabled.',
|
|
3442
|
-
);
|
|
3443
|
-
}
|
|
3444
|
-
|
|
3445
|
-
if (!store) {
|
|
3446
|
-
return this.executeWaterfallDirect(toolNameOrSpec, input, opts);
|
|
3447
|
-
}
|
|
3448
|
-
|
|
3449
|
-
const rowId = store.rowId;
|
|
3450
|
-
const fieldName = store.fieldName;
|
|
3451
|
-
const queueKey = store.tableNamespace?.trim()
|
|
3452
|
-
? `${baseQueueKey}:${store.tableNamespace.trim()}`
|
|
3453
|
-
: baseQueueKey;
|
|
3454
|
-
const rowState = this.rowStates.get(rowId);
|
|
3455
|
-
if (rowState && !rowState.waterfalls.has(toolName)) {
|
|
3456
|
-
rowState.waterfalls.set(toolName, {
|
|
3457
|
-
status: 'pending',
|
|
3458
|
-
providerIndex: 0,
|
|
3459
|
-
});
|
|
3460
|
-
}
|
|
3461
|
-
|
|
3462
|
-
// Check if this was already resolved in a previous attempt (checkpoint)
|
|
3463
|
-
const resolved = this.checkpoint.resolvedWaterfalls[queueKey];
|
|
3464
|
-
if (resolved && rowId in resolved) {
|
|
3465
|
-
this.log(` Row ${rowId} ${toolName}: recovered from checkpoint`);
|
|
3466
|
-
return resolved[rowId];
|
|
3467
|
-
}
|
|
3468
|
-
|
|
3469
|
-
return new Promise((resolve) => {
|
|
3470
|
-
const resolverId = `${rowId}-${queueKey}`;
|
|
3471
|
-
this.resolvers.set(resolverId, resolve);
|
|
3472
|
-
|
|
3473
|
-
this.emitScopedFieldMetaUpdate({
|
|
3474
|
-
rowId,
|
|
3475
|
-
key: store?.rowKey ?? null,
|
|
3476
|
-
tableNamespace: store?.tableNamespace ?? null,
|
|
3477
|
-
fieldName,
|
|
3478
|
-
status: 'running',
|
|
3479
|
-
rowStatus: 'running',
|
|
3480
|
-
stage: toolName,
|
|
3481
|
-
provider: null,
|
|
3482
|
-
error: null,
|
|
3483
|
-
dataPatch: {},
|
|
3484
|
-
});
|
|
3485
|
-
if (inlineSpec) {
|
|
3486
|
-
// Inline waterfalls have fully compiled child stages, so we can publish a
|
|
3487
|
-
// real queued state for each step immediately instead of forcing the grid
|
|
3488
|
-
// to guess from the row's broader status.
|
|
3489
|
-
this.emitQueuedInlineWaterfallSteps(
|
|
3490
|
-
rowId,
|
|
3491
|
-
store?.rowKey ?? null,
|
|
3492
|
-
store?.tableNamespace ?? null,
|
|
3493
|
-
inlineSpec,
|
|
3494
|
-
);
|
|
3495
|
-
}
|
|
3496
|
-
|
|
3497
|
-
if (!this.waterfallQueue.has(queueKey)) {
|
|
3498
|
-
this.waterfallQueue.set(queueKey, []);
|
|
3499
|
-
}
|
|
3500
|
-
this.waterfallQueue.get(queueKey)!.push({
|
|
3501
|
-
rowId,
|
|
3502
|
-
fieldName,
|
|
3503
|
-
tableNamespace: store?.tableNamespace,
|
|
3504
|
-
rowKey: store?.rowKey ?? null,
|
|
3505
|
-
key: queueKey,
|
|
3506
|
-
toolName,
|
|
3507
|
-
input,
|
|
3508
|
-
providerIndex: 0,
|
|
3509
|
-
opts,
|
|
3510
|
-
description: normalizeStepDescription(opts?.description),
|
|
3511
|
-
...(inlineSpec ? { spec: inlineSpec } : {}),
|
|
3512
|
-
});
|
|
3513
|
-
});
|
|
3514
|
-
}
|
|
3515
|
-
|
|
3516
3392
|
private async executeTool(
|
|
3517
3393
|
key: string,
|
|
3518
3394
|
toolId: string,
|
|
@@ -3520,17 +3396,16 @@ export class PlayContextImpl {
|
|
|
3520
3396
|
options?: ToolCallOptions,
|
|
3521
3397
|
): Promise<unknown> {
|
|
3522
3398
|
const normalizedKey = this.normalizeContextKey(key, 'tool');
|
|
3399
|
+
const toolCachePolicy = this.effectiveToolCallCachePolicy(options);
|
|
3523
3400
|
const toolRequestIdentity = deriveToolRequestIdentity({
|
|
3524
3401
|
toolId,
|
|
3525
3402
|
requestInput: input,
|
|
3526
3403
|
});
|
|
3527
|
-
const
|
|
3528
|
-
'direct',
|
|
3529
|
-
normalizedKey,
|
|
3404
|
+
const durableCacheKey = await this.durableToolCallCacheKey({
|
|
3530
3405
|
toolId,
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3406
|
+
requestInput: input,
|
|
3407
|
+
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
3408
|
+
});
|
|
3534
3409
|
const eventWaitHandler =
|
|
3535
3410
|
(await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
|
|
3536
3411
|
const store = rowContext.getStore();
|
|
@@ -3557,13 +3432,8 @@ export class PlayContextImpl {
|
|
|
3557
3432
|
}
|
|
3558
3433
|
|
|
3559
3434
|
if (!store) {
|
|
3560
|
-
const
|
|
3561
|
-
|
|
3562
|
-
const directCacheKey = this.buildToolResultCacheKey({
|
|
3563
|
-
rowId: directRowId,
|
|
3564
|
-
callId: directCallId,
|
|
3565
|
-
});
|
|
3566
|
-
const cached = options?.force
|
|
3435
|
+
const directCacheKey = durableCacheKey;
|
|
3436
|
+
const cached = toolCachePolicy.force
|
|
3567
3437
|
? null
|
|
3568
3438
|
: this.getCachedToolResult(toolId, directCacheKey);
|
|
3569
3439
|
if (cached?.done) {
|
|
@@ -3572,9 +3442,10 @@ export class PlayContextImpl {
|
|
|
3572
3442
|
toolId,
|
|
3573
3443
|
status: cached.result == null ? 'no_result' : 'completed',
|
|
3574
3444
|
result: cached.result,
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3445
|
+
execution: toolExecutionMetadataForOutcome({
|
|
3446
|
+
kind: 'checkpoint',
|
|
3447
|
+
cacheKey: directCacheKey,
|
|
3448
|
+
}),
|
|
3578
3449
|
});
|
|
3579
3450
|
}
|
|
3580
3451
|
this.log(`Calling tool: ${toolId}`);
|
|
@@ -3586,9 +3457,10 @@ export class PlayContextImpl {
|
|
|
3586
3457
|
result: execution.result,
|
|
3587
3458
|
metadata: execution.metadata,
|
|
3588
3459
|
meta: execution.meta,
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3460
|
+
execution: toolExecutionMetadataForOutcome({
|
|
3461
|
+
kind: 'live',
|
|
3462
|
+
cacheKey: directCacheKey,
|
|
3463
|
+
}),
|
|
3592
3464
|
});
|
|
3593
3465
|
this.checkpoint.completedToolBatches[toolId] = {
|
|
3594
3466
|
...(this.checkpoint.completedToolBatches[toolId] ?? {}),
|
|
@@ -3617,13 +3489,8 @@ export class PlayContextImpl {
|
|
|
3617
3489
|
);
|
|
3618
3490
|
}
|
|
3619
3491
|
|
|
3620
|
-
const toolResultCacheKey =
|
|
3621
|
-
|
|
3622
|
-
tableNamespace: store.tableNamespace,
|
|
3623
|
-
rowKey: store.rowKey,
|
|
3624
|
-
callId,
|
|
3625
|
-
});
|
|
3626
|
-
const cached = options?.force
|
|
3492
|
+
const toolResultCacheKey = durableCacheKey;
|
|
3493
|
+
const cached = toolCachePolicy.force
|
|
3627
3494
|
? null
|
|
3628
3495
|
: this.getCachedToolResult(toolId, toolResultCacheKey);
|
|
3629
3496
|
if (cached?.done) {
|
|
@@ -3632,9 +3499,10 @@ export class PlayContextImpl {
|
|
|
3632
3499
|
toolId,
|
|
3633
3500
|
status: cached.result == null ? 'no_result' : 'completed',
|
|
3634
3501
|
result: cached.result,
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3502
|
+
execution: toolExecutionMetadataForOutcome({
|
|
3503
|
+
kind: 'checkpoint',
|
|
3504
|
+
cacheKey: toolResultCacheKey,
|
|
3505
|
+
}),
|
|
3638
3506
|
});
|
|
3639
3507
|
}
|
|
3640
3508
|
|
|
@@ -3660,6 +3528,9 @@ export class PlayContextImpl {
|
|
|
3660
3528
|
});
|
|
3661
3529
|
this.toolCallQueue.push({
|
|
3662
3530
|
callId,
|
|
3531
|
+
cacheKey: toolResultCacheKey,
|
|
3532
|
+
receiptKey: durableCacheKey,
|
|
3533
|
+
force: toolCachePolicy.force,
|
|
3663
3534
|
rowId,
|
|
3664
3535
|
fieldName,
|
|
3665
3536
|
toolId,
|
|
@@ -3680,15 +3551,28 @@ export class PlayContextImpl {
|
|
|
3680
3551
|
normalizedKey,
|
|
3681
3552
|
this.currentRunId,
|
|
3682
3553
|
{
|
|
3554
|
+
receiptKey: durableCacheKey,
|
|
3683
3555
|
semanticKey: toolRequestIdentity,
|
|
3684
|
-
force:
|
|
3685
|
-
|
|
3686
|
-
staleAfterSeconds: options?.staleAfterSeconds,
|
|
3556
|
+
force: toolCachePolicy.force,
|
|
3557
|
+
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
3687
3558
|
markSkipped: () => {
|
|
3688
3559
|
this.log(
|
|
3689
3560
|
`ctx.tools.execute(${toolId}): no-op due completed receipt ${toolRequestIdentity} (label: ${normalizedKey})`,
|
|
3690
3561
|
);
|
|
3691
3562
|
},
|
|
3563
|
+
onClaimedResult: (output, receiptKey) =>
|
|
3564
|
+
markToolExecuteResultExecutionOutcome(output, {
|
|
3565
|
+
kind: 'live',
|
|
3566
|
+
receiptKey,
|
|
3567
|
+
}),
|
|
3568
|
+
onRecovered: (output, _receipt, source) =>
|
|
3569
|
+
markToolExecuteResultExecutionOutcome(
|
|
3570
|
+
output,
|
|
3571
|
+
toolExecutionOutcomeForDurableReceipt({
|
|
3572
|
+
source,
|
|
3573
|
+
receiptKey: durableCacheKey,
|
|
3574
|
+
}),
|
|
3575
|
+
),
|
|
3692
3576
|
execute: executeTool,
|
|
3693
3577
|
},
|
|
3694
3578
|
);
|
|
@@ -3743,9 +3627,10 @@ export class PlayContextImpl {
|
|
|
3743
3627
|
toolId,
|
|
3744
3628
|
status: 'completed',
|
|
3745
3629
|
result: existing.output,
|
|
3746
|
-
|
|
3747
|
-
|
|
3748
|
-
|
|
3630
|
+
execution: toolExecutionMetadataForOutcome({
|
|
3631
|
+
kind: 'checkpoint',
|
|
3632
|
+
cacheKey: `integration_event:${preparedBoundary.boundaryId}`,
|
|
3633
|
+
}),
|
|
3749
3634
|
});
|
|
3750
3635
|
}
|
|
3751
3636
|
|
|
@@ -3880,9 +3765,13 @@ export class PlayContextImpl {
|
|
|
3880
3765
|
checkpoint: this.checkpoint,
|
|
3881
3766
|
governance: childGovernance,
|
|
3882
3767
|
getRuntimeStepReceipt: undefined,
|
|
3768
|
+
getRuntimeStepReceipts: undefined,
|
|
3883
3769
|
claimRuntimeStepReceipt: undefined,
|
|
3770
|
+
claimRuntimeStepReceipts: undefined,
|
|
3884
3771
|
completeRuntimeStepReceipt: undefined,
|
|
3772
|
+
completeRuntimeStepReceipts: undefined,
|
|
3885
3773
|
failRuntimeStepReceipt: undefined,
|
|
3774
|
+
failRuntimeStepReceipts: undefined,
|
|
3886
3775
|
skipRuntimeStepReceipt: undefined,
|
|
3887
3776
|
});
|
|
3888
3777
|
const childExecution = this.executeResolvedPlay(
|
|
@@ -3957,13 +3846,13 @@ export class PlayContextImpl {
|
|
|
3957
3846
|
input,
|
|
3958
3847
|
}),
|
|
3959
3848
|
),
|
|
3960
|
-
repairRunningReceiptForSameRun: true,
|
|
3961
3849
|
staleAfterSeconds: options?.staleAfterSeconds,
|
|
3962
3850
|
markSkipped: () => {
|
|
3963
3851
|
this.log(
|
|
3964
3852
|
`ctx.runPlay(${normalizedKey}): no-op due completed receipt`,
|
|
3965
3853
|
);
|
|
3966
3854
|
},
|
|
3855
|
+
repairRunningReceiptForSameRunAfterWaitTimeout: true,
|
|
3967
3856
|
execute: executePlayCall,
|
|
3968
3857
|
},
|
|
3969
3858
|
);
|
|
@@ -4273,16 +4162,23 @@ export class PlayContextImpl {
|
|
|
4273
4162
|
return output;
|
|
4274
4163
|
};
|
|
4275
4164
|
|
|
4276
|
-
if (rowStore) {
|
|
4277
|
-
return await executeStep();
|
|
4278
|
-
}
|
|
4279
|
-
|
|
4280
4165
|
return this.executeWithRuntimeReceipt<T>(
|
|
4281
4166
|
'step',
|
|
4282
4167
|
normalizedKey,
|
|
4283
4168
|
this.currentRunId,
|
|
4284
4169
|
{
|
|
4285
|
-
semanticKey:
|
|
4170
|
+
semanticKey: rowStore
|
|
4171
|
+
? stableDigest(
|
|
4172
|
+
stableStringify({
|
|
4173
|
+
scope: 'row',
|
|
4174
|
+
tableNamespace: rowStore.tableNamespace ?? null,
|
|
4175
|
+
rowKey: rowStore.rowKey ?? null,
|
|
4176
|
+
rowId: rowStore.rowKey ? null : rowStore.rowId,
|
|
4177
|
+
fieldName: rowStore.fieldName ?? null,
|
|
4178
|
+
callIndex,
|
|
4179
|
+
}),
|
|
4180
|
+
)
|
|
4181
|
+
: options?.semanticKey,
|
|
4286
4182
|
staleAfterSeconds: options?.staleAfterSeconds,
|
|
4287
4183
|
markSkipped: (output) => {
|
|
4288
4184
|
this.log(`ctx.step(${normalizedKey}): no-op due completed receipt`);
|
|
@@ -4324,809 +4220,11 @@ export class PlayContextImpl {
|
|
|
4324
4220
|
}
|
|
4325
4221
|
|
|
4326
4222
|
getStats(): Record<string, unknown> {
|
|
4327
|
-
const stats: Record<
|
|
4328
|
-
string,
|
|
4329
|
-
{ total: number; complete: number; failed: number }
|
|
4330
|
-
> = {};
|
|
4331
|
-
for (const [, rowState] of this.rowStates) {
|
|
4332
|
-
for (const [toolName, wState] of rowState.waterfalls) {
|
|
4333
|
-
if (!stats[toolName])
|
|
4334
|
-
stats[toolName] = { total: 0, complete: 0, failed: 0 };
|
|
4335
|
-
stats[toolName].total++;
|
|
4336
|
-
if (wState.status === 'complete') stats[toolName].complete++;
|
|
4337
|
-
if (wState.status === 'failed') stats[toolName].failed++;
|
|
4338
|
-
}
|
|
4339
|
-
}
|
|
4340
4223
|
return {
|
|
4341
4224
|
rowsProcessed: Math.max(this.rowStates.size, this.processedRowCount),
|
|
4342
|
-
waterfalls: stats,
|
|
4343
4225
|
};
|
|
4344
4226
|
}
|
|
4345
4227
|
|
|
4346
|
-
// ——— Batched waterfall execution (the core engine) ———
|
|
4347
|
-
|
|
4348
|
-
private async executeBatchedWaterfalls(): Promise<void> {
|
|
4349
|
-
const queuedWaterfalls = this.waterfallQueue;
|
|
4350
|
-
this.waterfallQueue = new Map();
|
|
4351
|
-
this.log(`Executing batched waterfalls for ${queuedWaterfalls.size} tools`);
|
|
4352
|
-
|
|
4353
|
-
for (const [queueKey, requests] of queuedWaterfalls) {
|
|
4354
|
-
const inlineSpec = requests[0]?.spec;
|
|
4355
|
-
if (inlineSpec) {
|
|
4356
|
-
await this.executeInlineWaterfall(queueKey, inlineSpec, requests);
|
|
4357
|
-
continue;
|
|
4358
|
-
}
|
|
4359
|
-
|
|
4360
|
-
const toolName = requests[0]?.toolName ?? queueKey;
|
|
4361
|
-
const providers = requests[0]?.opts?.providers ?? [
|
|
4362
|
-
'hunter',
|
|
4363
|
-
'leadmagic',
|
|
4364
|
-
'pdl',
|
|
4365
|
-
'dropcontact',
|
|
4366
|
-
'prospeo',
|
|
4367
|
-
];
|
|
4368
|
-
|
|
4369
|
-
this.log(
|
|
4370
|
-
`Processing waterfall ${toolName}: ${requests.length} rows, providers: ${providers.join(', ')}`,
|
|
4371
|
-
);
|
|
4372
|
-
|
|
4373
|
-
if (!this.checkpoint.resolvedWaterfalls[queueKey]) {
|
|
4374
|
-
this.checkpoint.resolvedWaterfalls[queueKey] = {};
|
|
4375
|
-
}
|
|
4376
|
-
|
|
4377
|
-
await executeWaterfallProviders<WaterfallRequest, unknown>({
|
|
4378
|
-
providers,
|
|
4379
|
-
getPendingRequests: () =>
|
|
4380
|
-
requests.filter((req) => {
|
|
4381
|
-
const wState = this.rowStates
|
|
4382
|
-
.get(req.rowId)
|
|
4383
|
-
?.waterfalls.get(toolName);
|
|
4384
|
-
return wState?.status === 'pending';
|
|
4385
|
-
}),
|
|
4386
|
-
getCachedResults: (provider) => {
|
|
4387
|
-
const batchKey = `${queueKey}:${provider}`;
|
|
4388
|
-
const cached = this.checkpoint.completedBatches[batchKey];
|
|
4389
|
-
if (!cached) return null;
|
|
4390
|
-
this.log(` ${provider}: skipping (recovered from checkpoint)`);
|
|
4391
|
-
return cached.map((entry) => ({
|
|
4392
|
-
request: requests.find((request) => request.rowId === entry.rowId)!,
|
|
4393
|
-
result: entry.result,
|
|
4394
|
-
}));
|
|
4395
|
-
},
|
|
4396
|
-
storeCachedResults: (provider, results) => {
|
|
4397
|
-
const batchKey = `${queueKey}:${provider}`;
|
|
4398
|
-
this.checkpoint.completedBatches[batchKey] = results.map((entry) => ({
|
|
4399
|
-
rowId: entry.request.rowId,
|
|
4400
|
-
result: entry.result,
|
|
4401
|
-
}));
|
|
4402
|
-
},
|
|
4403
|
-
executeProviderRequests: async (provider, pending) => {
|
|
4404
|
-
const providerToolId = resolveWaterfallToolId(provider, toolName);
|
|
4405
|
-
const strategy =
|
|
4406
|
-
this.#options.getBatchOperationStrategy?.(providerToolId) ?? null;
|
|
4407
|
-
this.log(` ${provider}: ${pending.length} pending rows`);
|
|
4408
|
-
this.governor.chargeBudget('retry', pending.length);
|
|
4409
|
-
|
|
4410
|
-
if (strategy) {
|
|
4411
|
-
const compiledBatches = compileRequestsWithStrategy({
|
|
4412
|
-
requests: pending,
|
|
4413
|
-
strategy,
|
|
4414
|
-
getPayload: (request: WaterfallRequest) => request.input,
|
|
4415
|
-
});
|
|
4416
|
-
const flattenedResults: Array<{
|
|
4417
|
-
request: WaterfallRequest;
|
|
4418
|
-
result: unknown | null;
|
|
4419
|
-
}> = [];
|
|
4420
|
-
|
|
4421
|
-
await executeChunkedRequests({
|
|
4422
|
-
requests: compiledBatches,
|
|
4423
|
-
batchSize:
|
|
4424
|
-
compiledBatches.length > 0
|
|
4425
|
-
? await this.governor.suggestedParallelism(
|
|
4426
|
-
compiledBatches[0]!.batchOperation,
|
|
4427
|
-
4,
|
|
4428
|
-
)
|
|
4429
|
-
: 4,
|
|
4430
|
-
execute: async (batch) =>
|
|
4431
|
-
await this.callToolAPI(
|
|
4432
|
-
batch.batchOperation,
|
|
4433
|
-
batch.batchPayload,
|
|
4434
|
-
),
|
|
4435
|
-
onRequestError: (batch, error) => {
|
|
4436
|
-
this.log(
|
|
4437
|
-
` ${provider}: batch of ${batch.memberRequests.length} request(s) failed: ` +
|
|
4438
|
-
`${formatChunkExecutionError(error)} (rows recorded as misses; the waterfall continues)`,
|
|
4439
|
-
);
|
|
4440
|
-
},
|
|
4441
|
-
onChunkComplete: async (chunkResults) => {
|
|
4442
|
-
for (const entry of chunkResults) {
|
|
4443
|
-
const splitResults =
|
|
4444
|
-
entry.result != null
|
|
4445
|
-
? entry.request.splitResults(entry.result)
|
|
4446
|
-
: entry.request.memberRequests.map(() => null);
|
|
4447
|
-
|
|
4448
|
-
for (
|
|
4449
|
-
let index = 0;
|
|
4450
|
-
index < entry.request.memberRequests.length;
|
|
4451
|
-
index += 1
|
|
4452
|
-
) {
|
|
4453
|
-
flattenedResults.push({
|
|
4454
|
-
request: entry.request.memberRequests[index]!,
|
|
4455
|
-
result: splitResults[index] ?? null,
|
|
4456
|
-
});
|
|
4457
|
-
}
|
|
4458
|
-
}
|
|
4459
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
4460
|
-
},
|
|
4461
|
-
});
|
|
4462
|
-
|
|
4463
|
-
return flattenedResults;
|
|
4464
|
-
}
|
|
4465
|
-
|
|
4466
|
-
const chunkResults: Array<{
|
|
4467
|
-
request: WaterfallRequest;
|
|
4468
|
-
result: unknown | null;
|
|
4469
|
-
}> = [];
|
|
4470
|
-
await executeChunkedRequests<WaterfallRequest, unknown>({
|
|
4471
|
-
requests: pending,
|
|
4472
|
-
batchSize: await this.governor.suggestedParallelism(
|
|
4473
|
-
providerToolId,
|
|
4474
|
-
50,
|
|
4475
|
-
),
|
|
4476
|
-
execute: async (request) =>
|
|
4477
|
-
await this.callToolAPI(providerToolId, request.input),
|
|
4478
|
-
onRequestError: (request, error) => {
|
|
4479
|
-
this.log(
|
|
4480
|
-
` ${provider}: row ${request.rowId} failed: ` +
|
|
4481
|
-
`${formatChunkExecutionError(error)} (recorded as a miss; the waterfall continues)`,
|
|
4482
|
-
);
|
|
4483
|
-
},
|
|
4484
|
-
onChunkComplete: async (results) => {
|
|
4485
|
-
chunkResults.push(...results);
|
|
4486
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
4487
|
-
},
|
|
4488
|
-
});
|
|
4489
|
-
return chunkResults;
|
|
4490
|
-
},
|
|
4491
|
-
onHit: (provider, request, result) => {
|
|
4492
|
-
this.resolveWaterfall(
|
|
4493
|
-
queueKey,
|
|
4494
|
-
toolName,
|
|
4495
|
-
request.rowId,
|
|
4496
|
-
result,
|
|
4497
|
-
provider,
|
|
4498
|
-
request.rowKey ?? null,
|
|
4499
|
-
request.tableNamespace ?? null,
|
|
4500
|
-
request.fieldName,
|
|
4501
|
-
);
|
|
4502
|
-
},
|
|
4503
|
-
onMiss: (_provider, request) => {
|
|
4504
|
-
const wState = this.rowStates
|
|
4505
|
-
.get(request.rowId)
|
|
4506
|
-
?.waterfalls.get(toolName);
|
|
4507
|
-
if (wState) wState.providerIndex++;
|
|
4508
|
-
this.log(` Row ${request.rowId}: miss`);
|
|
4509
|
-
},
|
|
4510
|
-
onProviderComplete: () => {
|
|
4511
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
4512
|
-
},
|
|
4513
|
-
});
|
|
4514
|
-
|
|
4515
|
-
const stepResults: PlayStepRowResult[] = requests.map((req) => {
|
|
4516
|
-
const wState = this.rowStates.get(req.rowId)?.waterfalls.get(toolName);
|
|
4517
|
-
const success = wState?.status === 'complete';
|
|
4518
|
-
return {
|
|
4519
|
-
rowId: req.rowId,
|
|
4520
|
-
status: success ? 'completed' : 'missed',
|
|
4521
|
-
success,
|
|
4522
|
-
value: wState?.result,
|
|
4523
|
-
error: success ? null : (wState?.error ?? null),
|
|
4524
|
-
};
|
|
4525
|
-
});
|
|
4526
|
-
const waterfallStep = {
|
|
4527
|
-
type: 'waterfall' as const,
|
|
4528
|
-
tool: toolName,
|
|
4529
|
-
providers,
|
|
4530
|
-
results: stepResults,
|
|
4531
|
-
description: normalizeStepDescription(requests[0]?.description),
|
|
4532
|
-
};
|
|
4533
|
-
if (this.activeDatasetStep) {
|
|
4534
|
-
this.activeDatasetStep.substeps.push(waterfallStep);
|
|
4535
|
-
} else {
|
|
4536
|
-
this.steps.push(waterfallStep);
|
|
4537
|
-
}
|
|
4538
|
-
|
|
4539
|
-
for (const req of requests) {
|
|
4540
|
-
const wState = this.rowStates.get(req.rowId)?.waterfalls.get(toolName);
|
|
4541
|
-
if (wState?.status === 'pending') {
|
|
4542
|
-
wState.status = 'failed';
|
|
4543
|
-
wState.error = 'All providers exhausted';
|
|
4544
|
-
this.checkpoint.resolvedWaterfalls[queueKey]![req.rowId] = null;
|
|
4545
|
-
|
|
4546
|
-
const resolver = this.resolvers.get(`${req.rowId}-${queueKey}`);
|
|
4547
|
-
if (resolver) {
|
|
4548
|
-
resolver(null);
|
|
4549
|
-
this.resolvers.delete(`${req.rowId}-${queueKey}`);
|
|
4550
|
-
}
|
|
4551
|
-
this.emitScopedFieldMetaUpdate({
|
|
4552
|
-
rowId: req.rowId,
|
|
4553
|
-
key: req.rowKey ?? null,
|
|
4554
|
-
tableNamespace: req.tableNamespace ?? null,
|
|
4555
|
-
fieldName: req.fieldName,
|
|
4556
|
-
status: 'failed',
|
|
4557
|
-
rowStatus: 'running',
|
|
4558
|
-
stage: toolName,
|
|
4559
|
-
provider: null,
|
|
4560
|
-
error: 'All providers exhausted',
|
|
4561
|
-
dataPatch: {},
|
|
4562
|
-
});
|
|
4563
|
-
this.log(` Row ${req.rowId}: all providers exhausted`);
|
|
4564
|
-
}
|
|
4565
|
-
}
|
|
4566
|
-
}
|
|
4567
|
-
}
|
|
4568
|
-
|
|
4569
|
-
private async executeInlineWaterfall(
|
|
4570
|
-
queueKey: string,
|
|
4571
|
-
spec: InlineWaterfallSpec,
|
|
4572
|
-
requests: WaterfallRequest[],
|
|
4573
|
-
): Promise<void> {
|
|
4574
|
-
if (!this.checkpoint.resolvedWaterfalls[queueKey]) {
|
|
4575
|
-
this.checkpoint.resolvedWaterfalls[queueKey] = {};
|
|
4576
|
-
}
|
|
4577
|
-
|
|
4578
|
-
const pendingRows = new Set<number>(
|
|
4579
|
-
requests
|
|
4580
|
-
.filter(
|
|
4581
|
-
(req) =>
|
|
4582
|
-
this.rowStates.get(req.rowId)?.waterfalls.get(spec.id)?.status ===
|
|
4583
|
-
'pending',
|
|
4584
|
-
)
|
|
4585
|
-
.map((req) => req.rowId),
|
|
4586
|
-
);
|
|
4587
|
-
const resultsByRow = new Map<number, unknown[]>();
|
|
4588
|
-
const stepResults: Array<{
|
|
4589
|
-
id: string;
|
|
4590
|
-
kind?: 'tool' | 'code';
|
|
4591
|
-
toolId?: string;
|
|
4592
|
-
results: PlayStepRowResult[];
|
|
4593
|
-
}> = [];
|
|
4594
|
-
const stepColumnNames = spec.steps.map((s) =>
|
|
4595
|
-
sqlSafePlayColumnName(`${spec.id}.${s.id}`),
|
|
4596
|
-
);
|
|
4597
|
-
const resolvedInChunkRowIds = new Set<number>();
|
|
4598
|
-
let stepIdx = 0;
|
|
4599
|
-
|
|
4600
|
-
for (const step of spec.steps) {
|
|
4601
|
-
const stepColumnName = sqlSafePlayColumnName(`${spec.id}.${step.id}`);
|
|
4602
|
-
const stepProvider = isInlineWaterfallToolStep(step)
|
|
4603
|
-
? step.toolId
|
|
4604
|
-
: 'code';
|
|
4605
|
-
if (pendingRows.size === 0) {
|
|
4606
|
-
const skippedResults: PlayStepRowResult[] = requests.map((request) => {
|
|
4607
|
-
this.emitCellUpdate({
|
|
4608
|
-
rowId: request.rowId,
|
|
4609
|
-
key: request.rowKey ?? null,
|
|
4610
|
-
tableNamespace: request.tableNamespace ?? null,
|
|
4611
|
-
columnName: stepColumnName,
|
|
4612
|
-
status: 'skipped',
|
|
4613
|
-
stage: step.id,
|
|
4614
|
-
provider: stepProvider,
|
|
4615
|
-
value: null,
|
|
4616
|
-
});
|
|
4617
|
-
return {
|
|
4618
|
-
rowId: request.rowId,
|
|
4619
|
-
status: 'skipped',
|
|
4620
|
-
success: false,
|
|
4621
|
-
value: null,
|
|
4622
|
-
error: null,
|
|
4623
|
-
};
|
|
4624
|
-
});
|
|
4625
|
-
stepResults.push({
|
|
4626
|
-
id: step.id,
|
|
4627
|
-
kind: isInlineWaterfallCodeStep(step) ? 'code' : 'tool',
|
|
4628
|
-
toolId: stepProvider,
|
|
4629
|
-
results: skippedResults,
|
|
4630
|
-
});
|
|
4631
|
-
stepIdx++;
|
|
4632
|
-
continue;
|
|
4633
|
-
}
|
|
4634
|
-
this.governor.chargeBudget('waterfallStep', pendingRows.size);
|
|
4635
|
-
const pendingRowIds = new Set<number>(pendingRows);
|
|
4636
|
-
const stepRequests = requests.filter((req) =>
|
|
4637
|
-
pendingRowIds.has(req.rowId),
|
|
4638
|
-
);
|
|
4639
|
-
const perRowResultsByRowId = new Map<number, PlayStepRowResult>();
|
|
4640
|
-
if (isInlineWaterfallCodeStep(step)) {
|
|
4641
|
-
this.log(
|
|
4642
|
-
` Inline waterfall ${spec.id} -> ${step.id}: ${stepRequests.length} pending rows (code)`,
|
|
4643
|
-
);
|
|
4644
|
-
await executeChunkedRequests<WaterfallRequest, unknown>({
|
|
4645
|
-
requests: stepRequests,
|
|
4646
|
-
batchSize: await this.governor.suggestedParallelism(
|
|
4647
|
-
`code:${spec.id}:${step.id}`,
|
|
4648
|
-
20,
|
|
4649
|
-
),
|
|
4650
|
-
onRequestError: (request, error) => {
|
|
4651
|
-
this.log(
|
|
4652
|
-
` Inline waterfall ${spec.id} -> ${step.id}: row ${request.rowId} failed: ` +
|
|
4653
|
-
`${formatChunkExecutionError(error)} (recorded as a miss; the waterfall continues)`,
|
|
4654
|
-
);
|
|
4655
|
-
},
|
|
4656
|
-
execute: async (request) => {
|
|
4657
|
-
const codeStepCtx = {
|
|
4658
|
-
tools: {
|
|
4659
|
-
execute: async (
|
|
4660
|
-
requestOrKey:
|
|
4661
|
-
| { tool: string; input: Record<string, unknown> }
|
|
4662
|
-
| string,
|
|
4663
|
-
toolId?: string,
|
|
4664
|
-
payload?: Record<string, unknown>,
|
|
4665
|
-
) => {
|
|
4666
|
-
if (typeof requestOrKey === 'object') {
|
|
4667
|
-
return await this.callToolAPI(
|
|
4668
|
-
requestOrKey.tool,
|
|
4669
|
-
requestOrKey.input,
|
|
4670
|
-
);
|
|
4671
|
-
}
|
|
4672
|
-
if (!toolId || !payload) {
|
|
4673
|
-
throw new Error(
|
|
4674
|
-
'inline waterfall ctx.tools.execute requires a tool and input.',
|
|
4675
|
-
);
|
|
4676
|
-
}
|
|
4677
|
-
return await this.callToolAPI(toolId, payload);
|
|
4678
|
-
},
|
|
4679
|
-
},
|
|
4680
|
-
};
|
|
4681
|
-
return await step.run(request.input, codeStepCtx);
|
|
4682
|
-
},
|
|
4683
|
-
onChunkComplete: async (chunkResults) => {
|
|
4684
|
-
for (const entry of chunkResults) {
|
|
4685
|
-
const matchedValue = extractInlineWaterfallCodeStepValue(
|
|
4686
|
-
spec.output,
|
|
4687
|
-
entry.result,
|
|
4688
|
-
);
|
|
4689
|
-
const bucket = resultsByRow.get(entry.request.rowId) ?? [];
|
|
4690
|
-
const nextBucket = Array.isArray(matchedValue)
|
|
4691
|
-
? [...bucket, ...matchedValue]
|
|
4692
|
-
: matchedValue != null
|
|
4693
|
-
? [...bucket, matchedValue]
|
|
4694
|
-
: bucket;
|
|
4695
|
-
resultsByRow.set(entry.request.rowId, nextBucket);
|
|
4696
|
-
const satisfied = nextBucket.length >= spec.minResults;
|
|
4697
|
-
perRowResultsByRowId.set(entry.request.rowId, {
|
|
4698
|
-
rowId: entry.request.rowId,
|
|
4699
|
-
status: matchedValue != null ? 'completed' : 'missed',
|
|
4700
|
-
success: matchedValue != null,
|
|
4701
|
-
value: matchedValue,
|
|
4702
|
-
provider: matchedValue != null ? 'code' : undefined,
|
|
4703
|
-
error: null,
|
|
4704
|
-
});
|
|
4705
|
-
if (satisfied) {
|
|
4706
|
-
const finalValue =
|
|
4707
|
-
spec.minResults === 1 ? (nextBucket[0] ?? null) : nextBucket;
|
|
4708
|
-
this.resolveWaterfall(
|
|
4709
|
-
queueKey,
|
|
4710
|
-
spec.id,
|
|
4711
|
-
entry.request.rowId,
|
|
4712
|
-
finalValue,
|
|
4713
|
-
'code',
|
|
4714
|
-
entry.request.rowKey ?? null,
|
|
4715
|
-
entry.request.tableNamespace ?? null,
|
|
4716
|
-
entry.request.fieldName,
|
|
4717
|
-
);
|
|
4718
|
-
pendingRows.delete(entry.request.rowId);
|
|
4719
|
-
this.emitCellUpdate({
|
|
4720
|
-
rowId: entry.request.rowId,
|
|
4721
|
-
key: entry.request.rowKey ?? null,
|
|
4722
|
-
tableNamespace: entry.request.tableNamespace ?? null,
|
|
4723
|
-
columnName: stepColumnName,
|
|
4724
|
-
status: 'completed',
|
|
4725
|
-
stage: step.id,
|
|
4726
|
-
provider: 'code',
|
|
4727
|
-
value: matchedValue,
|
|
4728
|
-
});
|
|
4729
|
-
for (let ri = stepIdx + 1; ri < spec.steps.length; ri++) {
|
|
4730
|
-
const skippedStep = spec.steps[ri]!;
|
|
4731
|
-
this.emitCellUpdate({
|
|
4732
|
-
rowId: entry.request.rowId,
|
|
4733
|
-
key: entry.request.rowKey ?? null,
|
|
4734
|
-
tableNamespace: entry.request.tableNamespace ?? null,
|
|
4735
|
-
columnName: stepColumnNames[ri]!,
|
|
4736
|
-
status: 'skipped',
|
|
4737
|
-
stage: skippedStep.id,
|
|
4738
|
-
provider: isInlineWaterfallToolStep(skippedStep)
|
|
4739
|
-
? skippedStep.toolId
|
|
4740
|
-
: 'code',
|
|
4741
|
-
value: null,
|
|
4742
|
-
});
|
|
4743
|
-
}
|
|
4744
|
-
resolvedInChunkRowIds.add(entry.request.rowId);
|
|
4745
|
-
}
|
|
4746
|
-
}
|
|
4747
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
4748
|
-
},
|
|
4749
|
-
});
|
|
4750
|
-
} else {
|
|
4751
|
-
const strategy =
|
|
4752
|
-
this.#options.getBatchOperationStrategy?.(step.toolId) ?? null;
|
|
4753
|
-
this.log(
|
|
4754
|
-
` Inline waterfall ${spec.id} -> ${step.id}: ${stepRequests.length} pending rows`,
|
|
4755
|
-
);
|
|
4756
|
-
for (const request of stepRequests) {
|
|
4757
|
-
this.emitCellUpdate({
|
|
4758
|
-
rowId: request.rowId,
|
|
4759
|
-
key: request.rowKey ?? null,
|
|
4760
|
-
tableNamespace: request.tableNamespace ?? null,
|
|
4761
|
-
columnName: stepColumnName,
|
|
4762
|
-
status: 'running',
|
|
4763
|
-
stage: step.id,
|
|
4764
|
-
provider: step.toolId,
|
|
4765
|
-
value: null,
|
|
4766
|
-
});
|
|
4767
|
-
}
|
|
4768
|
-
|
|
4769
|
-
if (strategy) {
|
|
4770
|
-
const compiledBatches = compileRequestsWithStrategy({
|
|
4771
|
-
requests: stepRequests,
|
|
4772
|
-
strategy,
|
|
4773
|
-
getPayload: (request: WaterfallRequest) =>
|
|
4774
|
-
step.mapInput(request.input),
|
|
4775
|
-
});
|
|
4776
|
-
this.log(
|
|
4777
|
-
` ${step.toolId}: compiled ${compiledBatches.length} batch request(s)` +
|
|
4778
|
-
` (${this.summarizeBatchSizes(
|
|
4779
|
-
compiledBatches.map((batch) => batch.memberRequests.length),
|
|
4780
|
-
)})`,
|
|
4781
|
-
);
|
|
4782
|
-
await executeChunkedRequests({
|
|
4783
|
-
requests: compiledBatches,
|
|
4784
|
-
batchSize:
|
|
4785
|
-
compiledBatches.length > 0
|
|
4786
|
-
? await this.governor.suggestedParallelism(
|
|
4787
|
-
compiledBatches[0]!.batchOperation,
|
|
4788
|
-
4,
|
|
4789
|
-
)
|
|
4790
|
-
: 4,
|
|
4791
|
-
execute: async (batch) =>
|
|
4792
|
-
await this.callToolAPI(batch.batchOperation, batch.batchPayload),
|
|
4793
|
-
onRequestError: (batch, error) => {
|
|
4794
|
-
this.log(
|
|
4795
|
-
` ${step.toolId}: batch of ${batch.memberRequests.length} request(s) failed: ` +
|
|
4796
|
-
`${formatChunkExecutionError(error)} (rows recorded as misses; the waterfall continues)`,
|
|
4797
|
-
);
|
|
4798
|
-
},
|
|
4799
|
-
onChunkComplete: async (chunkResults) => {
|
|
4800
|
-
for (const entry of chunkResults) {
|
|
4801
|
-
const splitResults =
|
|
4802
|
-
entry.result != null
|
|
4803
|
-
? entry.request.splitResults(entry.result)
|
|
4804
|
-
: entry.request.memberRequests.map(() => null);
|
|
4805
|
-
for (
|
|
4806
|
-
let index = 0;
|
|
4807
|
-
index < entry.request.memberRequests.length;
|
|
4808
|
-
index += 1
|
|
4809
|
-
) {
|
|
4810
|
-
const request = entry.request.memberRequests[index]!;
|
|
4811
|
-
const rawResult = splitResults[index] ?? null;
|
|
4812
|
-
const matchedValue = await extractWaterfallOutputValue(
|
|
4813
|
-
step.toolId,
|
|
4814
|
-
spec.output,
|
|
4815
|
-
rawResult,
|
|
4816
|
-
this.#options.getToolTargetGetters,
|
|
4817
|
-
);
|
|
4818
|
-
const bucket = resultsByRow.get(request.rowId) ?? [];
|
|
4819
|
-
const nextBucket = Array.isArray(matchedValue)
|
|
4820
|
-
? [...bucket, ...matchedValue]
|
|
4821
|
-
: matchedValue != null
|
|
4822
|
-
? [...bucket, matchedValue]
|
|
4823
|
-
: bucket;
|
|
4824
|
-
resultsByRow.set(request.rowId, nextBucket);
|
|
4825
|
-
const satisfied = nextBucket.length >= spec.minResults;
|
|
4826
|
-
perRowResultsByRowId.set(request.rowId, {
|
|
4827
|
-
rowId: request.rowId,
|
|
4828
|
-
status: matchedValue != null ? 'completed' : 'missed',
|
|
4829
|
-
success: matchedValue != null,
|
|
4830
|
-
value: matchedValue,
|
|
4831
|
-
provider: matchedValue != null ? step.toolId : undefined,
|
|
4832
|
-
error: null,
|
|
4833
|
-
});
|
|
4834
|
-
if (satisfied) {
|
|
4835
|
-
const finalValue =
|
|
4836
|
-
spec.minResults === 1
|
|
4837
|
-
? (nextBucket[0] ?? null)
|
|
4838
|
-
: nextBucket;
|
|
4839
|
-
this.resolveWaterfall(
|
|
4840
|
-
queueKey,
|
|
4841
|
-
spec.id,
|
|
4842
|
-
request.rowId,
|
|
4843
|
-
finalValue,
|
|
4844
|
-
step.toolId,
|
|
4845
|
-
request.rowKey ?? null,
|
|
4846
|
-
request.tableNamespace ?? null,
|
|
4847
|
-
request.fieldName,
|
|
4848
|
-
);
|
|
4849
|
-
pendingRows.delete(request.rowId);
|
|
4850
|
-
this.emitCellUpdate({
|
|
4851
|
-
rowId: request.rowId,
|
|
4852
|
-
key: request.rowKey ?? null,
|
|
4853
|
-
tableNamespace: request.tableNamespace ?? null,
|
|
4854
|
-
columnName: stepColumnNames[stepIdx]!,
|
|
4855
|
-
status: 'completed',
|
|
4856
|
-
stage: step.id,
|
|
4857
|
-
provider: step.toolId,
|
|
4858
|
-
value: matchedValue,
|
|
4859
|
-
});
|
|
4860
|
-
for (let ri = stepIdx + 1; ri < spec.steps.length; ri++) {
|
|
4861
|
-
const skippedStep = spec.steps[ri]!;
|
|
4862
|
-
this.emitCellUpdate({
|
|
4863
|
-
rowId: request.rowId,
|
|
4864
|
-
key: request.rowKey ?? null,
|
|
4865
|
-
tableNamespace: request.tableNamespace ?? null,
|
|
4866
|
-
columnName: stepColumnNames[ri]!,
|
|
4867
|
-
status: 'skipped',
|
|
4868
|
-
stage: skippedStep.id,
|
|
4869
|
-
provider: isInlineWaterfallToolStep(skippedStep)
|
|
4870
|
-
? skippedStep.toolId
|
|
4871
|
-
: 'code',
|
|
4872
|
-
value: null,
|
|
4873
|
-
});
|
|
4874
|
-
}
|
|
4875
|
-
resolvedInChunkRowIds.add(request.rowId);
|
|
4876
|
-
}
|
|
4877
|
-
}
|
|
4878
|
-
}
|
|
4879
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
4880
|
-
},
|
|
4881
|
-
});
|
|
4882
|
-
} else {
|
|
4883
|
-
this.log(
|
|
4884
|
-
` ${step.toolId}: executing ${stepRequests.length} unbatched request(s)`,
|
|
4885
|
-
);
|
|
4886
|
-
await executeChunkedRequests<WaterfallRequest, unknown>({
|
|
4887
|
-
requests: stepRequests,
|
|
4888
|
-
batchSize: await this.governor.suggestedParallelism(
|
|
4889
|
-
step.toolId,
|
|
4890
|
-
50,
|
|
4891
|
-
),
|
|
4892
|
-
execute: async (request) =>
|
|
4893
|
-
await this.callToolAPI(step.toolId, step.mapInput(request.input)),
|
|
4894
|
-
onRequestError: (request, error) => {
|
|
4895
|
-
this.log(
|
|
4896
|
-
` ${step.toolId}: row ${request.rowId} failed: ` +
|
|
4897
|
-
`${formatChunkExecutionError(error)} (recorded as a miss; the waterfall continues)`,
|
|
4898
|
-
);
|
|
4899
|
-
},
|
|
4900
|
-
onChunkComplete: async (chunkResults) => {
|
|
4901
|
-
for (const entry of chunkResults) {
|
|
4902
|
-
const matchedValue = await extractWaterfallOutputValue(
|
|
4903
|
-
step.toolId,
|
|
4904
|
-
spec.output,
|
|
4905
|
-
entry.result,
|
|
4906
|
-
this.#options.getToolTargetGetters,
|
|
4907
|
-
);
|
|
4908
|
-
const bucket = resultsByRow.get(entry.request.rowId) ?? [];
|
|
4909
|
-
const nextBucket = Array.isArray(matchedValue)
|
|
4910
|
-
? [...bucket, ...matchedValue]
|
|
4911
|
-
: matchedValue != null
|
|
4912
|
-
? [...bucket, matchedValue]
|
|
4913
|
-
: bucket;
|
|
4914
|
-
resultsByRow.set(entry.request.rowId, nextBucket);
|
|
4915
|
-
const satisfied = nextBucket.length >= spec.minResults;
|
|
4916
|
-
perRowResultsByRowId.set(entry.request.rowId, {
|
|
4917
|
-
rowId: entry.request.rowId,
|
|
4918
|
-
status: matchedValue != null ? 'completed' : 'missed',
|
|
4919
|
-
success: matchedValue != null,
|
|
4920
|
-
value: matchedValue,
|
|
4921
|
-
provider: matchedValue != null ? step.toolId : undefined,
|
|
4922
|
-
error: null,
|
|
4923
|
-
});
|
|
4924
|
-
if (satisfied) {
|
|
4925
|
-
const finalValue =
|
|
4926
|
-
spec.minResults === 1
|
|
4927
|
-
? (nextBucket[0] ?? null)
|
|
4928
|
-
: nextBucket;
|
|
4929
|
-
this.resolveWaterfall(
|
|
4930
|
-
queueKey,
|
|
4931
|
-
spec.id,
|
|
4932
|
-
entry.request.rowId,
|
|
4933
|
-
finalValue,
|
|
4934
|
-
step.toolId,
|
|
4935
|
-
entry.request.rowKey ?? null,
|
|
4936
|
-
entry.request.tableNamespace ?? null,
|
|
4937
|
-
entry.request.fieldName,
|
|
4938
|
-
);
|
|
4939
|
-
pendingRows.delete(entry.request.rowId);
|
|
4940
|
-
this.emitCellUpdate({
|
|
4941
|
-
rowId: entry.request.rowId,
|
|
4942
|
-
key: entry.request.rowKey ?? null,
|
|
4943
|
-
tableNamespace: entry.request.tableNamespace ?? null,
|
|
4944
|
-
columnName: stepColumnNames[stepIdx]!,
|
|
4945
|
-
status: 'completed',
|
|
4946
|
-
stage: step.id,
|
|
4947
|
-
provider: step.toolId,
|
|
4948
|
-
value: matchedValue,
|
|
4949
|
-
});
|
|
4950
|
-
for (let ri = stepIdx + 1; ri < spec.steps.length; ri++) {
|
|
4951
|
-
const skippedStep = spec.steps[ri]!;
|
|
4952
|
-
this.emitCellUpdate({
|
|
4953
|
-
rowId: entry.request.rowId,
|
|
4954
|
-
key: entry.request.rowKey ?? null,
|
|
4955
|
-
tableNamespace: entry.request.tableNamespace ?? null,
|
|
4956
|
-
columnName: stepColumnNames[ri]!,
|
|
4957
|
-
status: 'skipped',
|
|
4958
|
-
stage: skippedStep.id,
|
|
4959
|
-
provider: isInlineWaterfallToolStep(skippedStep)
|
|
4960
|
-
? skippedStep.toolId
|
|
4961
|
-
: 'code',
|
|
4962
|
-
value: null,
|
|
4963
|
-
});
|
|
4964
|
-
}
|
|
4965
|
-
resolvedInChunkRowIds.add(entry.request.rowId);
|
|
4966
|
-
}
|
|
4967
|
-
}
|
|
4968
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
4969
|
-
},
|
|
4970
|
-
});
|
|
4971
|
-
}
|
|
4972
|
-
}
|
|
4973
|
-
|
|
4974
|
-
const perRowResults: PlayStepRowResult[] = requests.map((request) => {
|
|
4975
|
-
if (resolvedInChunkRowIds.has(request.rowId)) {
|
|
4976
|
-
const existing = perRowResultsByRowId.get(request.rowId);
|
|
4977
|
-
return (
|
|
4978
|
-
existing ?? {
|
|
4979
|
-
rowId: request.rowId,
|
|
4980
|
-
status: 'completed' as const,
|
|
4981
|
-
success: true,
|
|
4982
|
-
value: null,
|
|
4983
|
-
error: null,
|
|
4984
|
-
}
|
|
4985
|
-
);
|
|
4986
|
-
}
|
|
4987
|
-
const existing = perRowResultsByRowId.get(request.rowId);
|
|
4988
|
-
if (existing) {
|
|
4989
|
-
this.emitCellUpdate({
|
|
4990
|
-
rowId: request.rowId,
|
|
4991
|
-
key: request.rowKey ?? null,
|
|
4992
|
-
tableNamespace: request.tableNamespace ?? null,
|
|
4993
|
-
columnName: stepColumnName,
|
|
4994
|
-
status: existing.status,
|
|
4995
|
-
stage: step.id,
|
|
4996
|
-
provider: existing.provider ?? stepProvider,
|
|
4997
|
-
error: existing.error ?? null,
|
|
4998
|
-
value: existing.value ?? null,
|
|
4999
|
-
});
|
|
5000
|
-
return existing;
|
|
5001
|
-
}
|
|
5002
|
-
if (!pendingRowIds.has(request.rowId)) {
|
|
5003
|
-
this.emitCellUpdate({
|
|
5004
|
-
rowId: request.rowId,
|
|
5005
|
-
key: request.rowKey ?? null,
|
|
5006
|
-
tableNamespace: request.tableNamespace ?? null,
|
|
5007
|
-
columnName: stepColumnName,
|
|
5008
|
-
status: 'skipped',
|
|
5009
|
-
stage: step.id,
|
|
5010
|
-
provider: stepProvider,
|
|
5011
|
-
value: null,
|
|
5012
|
-
});
|
|
5013
|
-
return {
|
|
5014
|
-
rowId: request.rowId,
|
|
5015
|
-
status: 'skipped',
|
|
5016
|
-
success: false,
|
|
5017
|
-
value: null,
|
|
5018
|
-
error: null,
|
|
5019
|
-
};
|
|
5020
|
-
}
|
|
5021
|
-
this.emitCellUpdate({
|
|
5022
|
-
rowId: request.rowId,
|
|
5023
|
-
key: request.rowKey ?? null,
|
|
5024
|
-
tableNamespace: request.tableNamespace ?? null,
|
|
5025
|
-
columnName: stepColumnName,
|
|
5026
|
-
status: 'missed',
|
|
5027
|
-
stage: step.id,
|
|
5028
|
-
provider: stepProvider,
|
|
5029
|
-
value: null,
|
|
5030
|
-
});
|
|
5031
|
-
return {
|
|
5032
|
-
rowId: request.rowId,
|
|
5033
|
-
status: 'missed',
|
|
5034
|
-
success: false,
|
|
5035
|
-
value: null,
|
|
5036
|
-
error: null,
|
|
5037
|
-
};
|
|
5038
|
-
});
|
|
5039
|
-
|
|
5040
|
-
stepResults.push({
|
|
5041
|
-
id: step.id,
|
|
5042
|
-
kind: isInlineWaterfallCodeStep(step) ? 'code' : 'tool',
|
|
5043
|
-
toolId: stepProvider,
|
|
5044
|
-
// Persist only a bounded preview in the in-memory execution trace.
|
|
5045
|
-
// The authoritative per-row state already lives in Neon/live progress.
|
|
5046
|
-
results: compactRowResultsPreview(perRowResults),
|
|
5047
|
-
});
|
|
5048
|
-
stepIdx++;
|
|
5049
|
-
}
|
|
5050
|
-
|
|
5051
|
-
for (const req of requests) {
|
|
5052
|
-
const wState = this.rowStates.get(req.rowId)?.waterfalls.get(spec.id);
|
|
5053
|
-
if (wState?.status === 'pending') {
|
|
5054
|
-
wState.status = 'failed';
|
|
5055
|
-
wState.error = 'All waterfall steps exhausted';
|
|
5056
|
-
this.checkpoint.resolvedWaterfalls[queueKey]![req.rowId] = null;
|
|
5057
|
-
const resolver = this.resolvers.get(`${req.rowId}-${queueKey}`);
|
|
5058
|
-
if (resolver) {
|
|
5059
|
-
resolver(null);
|
|
5060
|
-
this.resolvers.delete(`${req.rowId}-${queueKey}`);
|
|
5061
|
-
}
|
|
5062
|
-
}
|
|
5063
|
-
}
|
|
5064
|
-
|
|
5065
|
-
const groupResults: PlayStepRowResult[] = requests.map((req) => {
|
|
5066
|
-
const wState = this.rowStates.get(req.rowId)?.waterfalls.get(spec.id);
|
|
5067
|
-
const success = wState?.status === 'complete';
|
|
5068
|
-
return {
|
|
5069
|
-
rowId: req.rowId,
|
|
5070
|
-
status: success ? 'completed' : 'missed',
|
|
5071
|
-
success,
|
|
5072
|
-
value: wState?.result,
|
|
5073
|
-
error: success ? null : (wState?.error ?? null),
|
|
5074
|
-
};
|
|
5075
|
-
});
|
|
5076
|
-
|
|
5077
|
-
const waterfallStep = {
|
|
5078
|
-
type: 'waterfall' as const,
|
|
5079
|
-
id: spec.id,
|
|
5080
|
-
output: spec.output,
|
|
5081
|
-
minResults: spec.minResults,
|
|
5082
|
-
steps: stepResults,
|
|
5083
|
-
results: compactRowResultsPreview(groupResults),
|
|
5084
|
-
description: normalizeStepDescription(requests[0]?.description),
|
|
5085
|
-
};
|
|
5086
|
-
if (this.activeDatasetStep) {
|
|
5087
|
-
this.activeDatasetStep.substeps.push(waterfallStep);
|
|
5088
|
-
} else {
|
|
5089
|
-
this.steps.push(waterfallStep);
|
|
5090
|
-
}
|
|
5091
|
-
}
|
|
5092
|
-
|
|
5093
|
-
private resolveWaterfall(
|
|
5094
|
-
queueKey: string,
|
|
5095
|
-
toolName: string,
|
|
5096
|
-
rowId: number,
|
|
5097
|
-
result: unknown,
|
|
5098
|
-
provider: string,
|
|
5099
|
-
rowKey: string | null,
|
|
5100
|
-
tableNamespace: string | null,
|
|
5101
|
-
fieldName?: string,
|
|
5102
|
-
): void {
|
|
5103
|
-
const wState = this.rowStates.get(rowId)?.waterfalls.get(toolName);
|
|
5104
|
-
if (!wState || wState.status !== 'pending') return;
|
|
5105
|
-
|
|
5106
|
-
wState.status = 'complete';
|
|
5107
|
-
wState.result = result;
|
|
5108
|
-
this.checkpoint.resolvedWaterfalls[queueKey]![rowId] = result;
|
|
5109
|
-
|
|
5110
|
-
const resolver = this.resolvers.get(`${rowId}-${queueKey}`);
|
|
5111
|
-
if (resolver) {
|
|
5112
|
-
resolver(result);
|
|
5113
|
-
this.resolvers.delete(`${rowId}-${queueKey}`);
|
|
5114
|
-
}
|
|
5115
|
-
this.emitScopedFieldMetaUpdate({
|
|
5116
|
-
rowId,
|
|
5117
|
-
key: rowKey,
|
|
5118
|
-
tableNamespace,
|
|
5119
|
-
fieldName,
|
|
5120
|
-
status: 'running',
|
|
5121
|
-
rowStatus: 'running',
|
|
5122
|
-
stage: toolName,
|
|
5123
|
-
provider,
|
|
5124
|
-
error: null,
|
|
5125
|
-
dataPatch: {},
|
|
5126
|
-
});
|
|
5127
|
-
this.logWaterfallMatch({ queueKey, rowId, provider });
|
|
5128
|
-
}
|
|
5129
|
-
|
|
5130
4228
|
// ——— Batched tool call execution ———
|
|
5131
4229
|
|
|
5132
4230
|
private async executeBatchedToolCalls(): Promise<void> {
|
|
@@ -5147,15 +4245,7 @@ export class PlayContextImpl {
|
|
|
5147
4245
|
const recordToolStep = (stepRequests: ToolCallRequest[]): void => {
|
|
5148
4246
|
if (stepRequests.length === 0) return;
|
|
5149
4247
|
const stepResults: PlayStepRowResult[] = stepRequests.map((req) => {
|
|
5150
|
-
const cachedResult = this.getCachedToolResult(
|
|
5151
|
-
toolId,
|
|
5152
|
-
this.buildToolResultCacheKey({
|
|
5153
|
-
rowId: req.rowId,
|
|
5154
|
-
tableNamespace: req.tableNamespace,
|
|
5155
|
-
rowKey: req.rowKey ?? undefined,
|
|
5156
|
-
callId: req.callId,
|
|
5157
|
-
}),
|
|
5158
|
-
);
|
|
4248
|
+
const cachedResult = this.getCachedToolResult(toolId, req.cacheKey);
|
|
5159
4249
|
return {
|
|
5160
4250
|
rowId: req.rowId,
|
|
5161
4251
|
status: cachedResult?.result != null ? 'completed' : 'failed',
|
|
@@ -5181,15 +4271,7 @@ export class PlayContextImpl {
|
|
|
5181
4271
|
const pendingRequests: ToolCallRequest[] = [];
|
|
5182
4272
|
const recoveredRequests: ToolCallRequest[] = [];
|
|
5183
4273
|
for (const req of requests) {
|
|
5184
|
-
const cached = this.getCachedToolResult(
|
|
5185
|
-
toolId,
|
|
5186
|
-
this.buildToolResultCacheKey({
|
|
5187
|
-
rowId: req.rowId,
|
|
5188
|
-
tableNamespace: req.tableNamespace,
|
|
5189
|
-
rowKey: req.rowKey ?? undefined,
|
|
5190
|
-
callId: req.callId,
|
|
5191
|
-
}),
|
|
5192
|
-
);
|
|
4274
|
+
const cached = this.getCachedToolResult(toolId, req.cacheKey);
|
|
5193
4275
|
if (cached?.done) {
|
|
5194
4276
|
this.log(` Row ${req.rowId} ${toolId}: recovered from checkpoint`);
|
|
5195
4277
|
const resolver = this.toolCallResolvers.get(req.callId);
|
|
@@ -5204,6 +4286,240 @@ export class PlayContextImpl {
|
|
|
5204
4286
|
}
|
|
5205
4287
|
recordToolStep(recoveredRequests);
|
|
5206
4288
|
|
|
4289
|
+
const liveFollowersByOwnerCallId = new Map<string, ToolCallRequest[]>();
|
|
4290
|
+
const resolveLiveFollowers = (
|
|
4291
|
+
owner: ToolCallRequest,
|
|
4292
|
+
result: unknown,
|
|
4293
|
+
): void => {
|
|
4294
|
+
const followers = liveFollowersByOwnerCallId.get(owner.callId);
|
|
4295
|
+
if (!followers || followers.length === 0) return;
|
|
4296
|
+
liveFollowersByOwnerCallId.delete(owner.callId);
|
|
4297
|
+
for (const follower of followers) {
|
|
4298
|
+
const followerReceiptKey = follower.receiptKey?.trim() || null;
|
|
4299
|
+
const followerResult = followerReceiptKey
|
|
4300
|
+
? markToolExecuteResultExecutionOutcome(result, {
|
|
4301
|
+
kind: 'in_flight',
|
|
4302
|
+
receiptKey: followerReceiptKey,
|
|
4303
|
+
attachedToReceiptKey: followerReceiptKey,
|
|
4304
|
+
})
|
|
4305
|
+
: result;
|
|
4306
|
+
this.cacheToolResult(toolId, follower.cacheKey, followerResult);
|
|
4307
|
+
const resolver = this.toolCallResolvers.get(follower.callId);
|
|
4308
|
+
if (resolver) {
|
|
4309
|
+
resolver.resolve(followerResult);
|
|
4310
|
+
this.toolCallResolvers.delete(follower.callId);
|
|
4311
|
+
}
|
|
4312
|
+
this.emitScopedFieldMetaUpdate({
|
|
4313
|
+
rowId: follower.rowId,
|
|
4314
|
+
key: follower.rowKey ?? null,
|
|
4315
|
+
tableNamespace: follower.tableNamespace ?? null,
|
|
4316
|
+
fieldName: follower.fieldName,
|
|
4317
|
+
status: 'cached',
|
|
4318
|
+
rowStatus: 'running',
|
|
4319
|
+
stage: toolId,
|
|
4320
|
+
provider: null,
|
|
4321
|
+
error: null,
|
|
4322
|
+
reused: true,
|
|
4323
|
+
dataPatch: {},
|
|
4324
|
+
});
|
|
4325
|
+
}
|
|
4326
|
+
};
|
|
4327
|
+
const rejectWithLiveFollowers = async (
|
|
4328
|
+
owner: ToolCallRequest,
|
|
4329
|
+
error: unknown,
|
|
4330
|
+
): Promise<void> => {
|
|
4331
|
+
const followers = liveFollowersByOwnerCallId.get(owner.callId) ?? [];
|
|
4332
|
+
liveFollowersByOwnerCallId.delete(owner.callId);
|
|
4333
|
+
for (const request of [owner, ...followers]) {
|
|
4334
|
+
await this.rejectToolCall(toolId, request, error);
|
|
4335
|
+
}
|
|
4336
|
+
};
|
|
4337
|
+
|
|
4338
|
+
const durableWaiters: Promise<void>[] = [];
|
|
4339
|
+
if (
|
|
4340
|
+
pendingRequests.length > 0 &&
|
|
4341
|
+
(this.#options.claimRuntimeStepReceipts ||
|
|
4342
|
+
this.#options.claimRuntimeStepReceipt)
|
|
4343
|
+
) {
|
|
4344
|
+
const requestsByReceiptKey = new Map<
|
|
4345
|
+
string,
|
|
4346
|
+
{ requests: ToolCallRequest[]; forceRefresh: boolean }
|
|
4347
|
+
>();
|
|
4348
|
+
const localOnlyRequests: ToolCallRequest[] = [];
|
|
4349
|
+
for (const request of pendingRequests) {
|
|
4350
|
+
const receiptKey = request.receiptKey?.trim() || null;
|
|
4351
|
+
if (!receiptKey) {
|
|
4352
|
+
localOnlyRequests.push(request);
|
|
4353
|
+
continue;
|
|
4354
|
+
}
|
|
4355
|
+
const group = requestsByReceiptKey.get(receiptKey) ?? {
|
|
4356
|
+
requests: [],
|
|
4357
|
+
forceRefresh: false,
|
|
4358
|
+
};
|
|
4359
|
+
group.requests.push(request);
|
|
4360
|
+
group.forceRefresh ||= request.force === true;
|
|
4361
|
+
requestsByReceiptKey.set(receiptKey, group);
|
|
4362
|
+
}
|
|
4363
|
+
const normalReceiptKeys = [...requestsByReceiptKey]
|
|
4364
|
+
.filter(([, group]) => !group.forceRefresh)
|
|
4365
|
+
.map(([receiptKey]) => receiptKey);
|
|
4366
|
+
const forcedReceiptKeys = [...requestsByReceiptKey]
|
|
4367
|
+
.filter(([, group]) => group.forceRefresh)
|
|
4368
|
+
.map(([receiptKey]) => receiptKey);
|
|
4369
|
+
const claims =
|
|
4370
|
+
normalReceiptKeys.length > 0
|
|
4371
|
+
? await this.claimRuntimeStepReceipts(
|
|
4372
|
+
normalReceiptKeys,
|
|
4373
|
+
this.currentRunId,
|
|
4374
|
+
)
|
|
4375
|
+
: new Map<string, RuntimeStepReceipt>();
|
|
4376
|
+
const forcedClaims =
|
|
4377
|
+
forcedReceiptKeys.length > 0
|
|
4378
|
+
? await this.claimRuntimeStepReceipts(
|
|
4379
|
+
forcedReceiptKeys,
|
|
4380
|
+
this.currentRunId,
|
|
4381
|
+
true,
|
|
4382
|
+
true,
|
|
4383
|
+
)
|
|
4384
|
+
: new Map<string, RuntimeStepReceipt>();
|
|
4385
|
+
for (const [receiptKey, claim] of forcedClaims) {
|
|
4386
|
+
claims.set(receiptKey, claim);
|
|
4387
|
+
}
|
|
4388
|
+
const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
|
|
4389
|
+
const durableRecoveredRequests: ToolCallRequest[] = [];
|
|
4390
|
+
const resolveRequestsFromReceipt = async (
|
|
4391
|
+
requestsForKey: ToolCallRequest[],
|
|
4392
|
+
receipt: RuntimeStepReceipt,
|
|
4393
|
+
source: 'cache' | 'in_flight',
|
|
4394
|
+
): Promise<void> => {
|
|
4395
|
+
const request = requestsForKey[0];
|
|
4396
|
+
if (!request) return;
|
|
4397
|
+
const receiptKey = request.receiptKey?.trim() || null;
|
|
4398
|
+
if (!receiptKey) {
|
|
4399
|
+
throw new Error(
|
|
4400
|
+
`Durable tool receipt ${source} recovery requires a receipt key.`,
|
|
4401
|
+
);
|
|
4402
|
+
}
|
|
4403
|
+
const wrapped = await this.wrapToolExecutionResult({
|
|
4404
|
+
toolId,
|
|
4405
|
+
status:
|
|
4406
|
+
receipt.output === null || receipt.output === undefined
|
|
4407
|
+
? 'no_result'
|
|
4408
|
+
: 'completed',
|
|
4409
|
+
result: this.runtimeReceiptOutput(receipt),
|
|
4410
|
+
execution: toolExecutionMetadataForOutcome({
|
|
4411
|
+
kind: source,
|
|
4412
|
+
cacheKey: request.cacheKey,
|
|
4413
|
+
receiptKey,
|
|
4414
|
+
attachedToReceiptKey: receiptKey,
|
|
4415
|
+
}),
|
|
4416
|
+
});
|
|
4417
|
+
this.cacheToolResult(toolId, request.cacheKey, wrapped);
|
|
4418
|
+
for (const waitingRequest of requestsForKey) {
|
|
4419
|
+
const resolver = this.toolCallResolvers.get(
|
|
4420
|
+
waitingRequest.callId,
|
|
4421
|
+
);
|
|
4422
|
+
if (resolver) {
|
|
4423
|
+
resolver.resolve(wrapped);
|
|
4424
|
+
this.toolCallResolvers.delete(waitingRequest.callId);
|
|
4425
|
+
}
|
|
4426
|
+
this.emitScopedFieldMetaUpdate({
|
|
4427
|
+
rowId: waitingRequest.rowId,
|
|
4428
|
+
key: waitingRequest.rowKey ?? null,
|
|
4429
|
+
tableNamespace: waitingRequest.tableNamespace ?? null,
|
|
4430
|
+
fieldName: waitingRequest.fieldName,
|
|
4431
|
+
status: 'cached',
|
|
4432
|
+
rowStatus: 'running',
|
|
4433
|
+
stage: toolId,
|
|
4434
|
+
provider: null,
|
|
4435
|
+
error: null,
|
|
4436
|
+
reused: true,
|
|
4437
|
+
dataPatch: {},
|
|
4438
|
+
});
|
|
4439
|
+
}
|
|
4440
|
+
};
|
|
4441
|
+
const claimOwnerWithLiveFollowers = (
|
|
4442
|
+
owner: ToolCallRequest | undefined,
|
|
4443
|
+
followers: ToolCallRequest[],
|
|
4444
|
+
): void => {
|
|
4445
|
+
if (!owner) return;
|
|
4446
|
+
claimedRequests.push(owner);
|
|
4447
|
+
if (followers.length > 0) {
|
|
4448
|
+
liveFollowersByOwnerCallId.set(owner.callId, followers);
|
|
4449
|
+
}
|
|
4450
|
+
};
|
|
4451
|
+
const waitForReceiptOrTimeout = async (
|
|
4452
|
+
receiptKey: string,
|
|
4453
|
+
requestsForKey: ToolCallRequest[],
|
|
4454
|
+
): Promise<void> => {
|
|
4455
|
+
try {
|
|
4456
|
+
await resolveRequestsFromReceipt(
|
|
4457
|
+
requestsForKey,
|
|
4458
|
+
await this.waitForCompletedRuntimeToolReceipt(receiptKey),
|
|
4459
|
+
'in_flight',
|
|
4460
|
+
);
|
|
4461
|
+
return;
|
|
4462
|
+
} catch (error) {
|
|
4463
|
+
if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
|
|
4464
|
+
for (const request of requestsForKey) {
|
|
4465
|
+
await this.rejectToolCall(toolId, request, error, {
|
|
4466
|
+
persistReceiptFailure: false,
|
|
4467
|
+
});
|
|
4468
|
+
}
|
|
4469
|
+
return;
|
|
4470
|
+
}
|
|
4471
|
+
}
|
|
4472
|
+
for (const request of requestsForKey) {
|
|
4473
|
+
await this.rejectToolCall(
|
|
4474
|
+
toolId,
|
|
4475
|
+
request,
|
|
4476
|
+
new RuntimeReceiptWaitTimeoutError(receiptKey),
|
|
4477
|
+
{ persistReceiptFailure: false },
|
|
4478
|
+
);
|
|
4479
|
+
}
|
|
4480
|
+
};
|
|
4481
|
+
|
|
4482
|
+
for (const [receiptKey, group] of requestsByReceiptKey) {
|
|
4483
|
+
const requestsForKey = group.requests;
|
|
4484
|
+
const claim = claims.get(receiptKey);
|
|
4485
|
+
if (!claim) {
|
|
4486
|
+
const [owner, ...waiters] = requestsForKey;
|
|
4487
|
+
claimOwnerWithLiveFollowers(owner, waiters);
|
|
4488
|
+
continue;
|
|
4489
|
+
}
|
|
4490
|
+
if (claim?.status === 'completed' || claim?.status === 'skipped') {
|
|
4491
|
+
await resolveRequestsFromReceipt(requestsForKey, claim, 'cache');
|
|
4492
|
+
durableRecoveredRequests.push(...requestsForKey);
|
|
4493
|
+
continue;
|
|
4494
|
+
}
|
|
4495
|
+
if (claim?.status === 'failed') {
|
|
4496
|
+
for (const request of requestsForKey) {
|
|
4497
|
+
await this.rejectToolCall(
|
|
4498
|
+
toolId,
|
|
4499
|
+
request,
|
|
4500
|
+
new Error(claim.error ?? 'Durable tool call failed.'),
|
|
4501
|
+
);
|
|
4502
|
+
}
|
|
4503
|
+
continue;
|
|
4504
|
+
}
|
|
4505
|
+
if (
|
|
4506
|
+
claim?.status === 'running' &&
|
|
4507
|
+
claim.claimState !== 'existing'
|
|
4508
|
+
) {
|
|
4509
|
+
const [owner, ...waiters] = requestsForKey;
|
|
4510
|
+
claimOwnerWithLiveFollowers(owner, waiters);
|
|
4511
|
+
continue;
|
|
4512
|
+
}
|
|
4513
|
+
durableWaiters.push(
|
|
4514
|
+
waitForReceiptOrTimeout(receiptKey, requestsForKey),
|
|
4515
|
+
);
|
|
4516
|
+
}
|
|
4517
|
+
|
|
4518
|
+
pendingRequests.length = 0;
|
|
4519
|
+
pendingRequests.push(...claimedRequests);
|
|
4520
|
+
recordToolStep(durableRecoveredRequests);
|
|
4521
|
+
}
|
|
4522
|
+
|
|
5207
4523
|
if (pendingRequests.length > 0) {
|
|
5208
4524
|
const strategy =
|
|
5209
4525
|
this.#options.getBatchOperationStrategy?.(toolId) ?? null;
|
|
@@ -5234,7 +4550,7 @@ export class PlayContextImpl {
|
|
|
5234
4550
|
for (const entry of chunkResults) {
|
|
5235
4551
|
if (entry.error !== undefined) {
|
|
5236
4552
|
for (const request of entry.request.memberRequests) {
|
|
5237
|
-
|
|
4553
|
+
await rejectWithLiveFollowers(request, entry.error);
|
|
5238
4554
|
}
|
|
5239
4555
|
continue;
|
|
5240
4556
|
}
|
|
@@ -5242,6 +4558,14 @@ export class PlayContextImpl {
|
|
|
5242
4558
|
entry.result != null
|
|
5243
4559
|
? entry.request.splitResults(entry.result)
|
|
5244
4560
|
: entry.request.memberRequests.map(() => null);
|
|
4561
|
+
const resolvedResults =
|
|
4562
|
+
await this.resolveToolCallBatchResults(
|
|
4563
|
+
toolId,
|
|
4564
|
+
entry.request.memberRequests.map((request, index) => ({
|
|
4565
|
+
request,
|
|
4566
|
+
result: splitResults[index] ?? null,
|
|
4567
|
+
})),
|
|
4568
|
+
);
|
|
5245
4569
|
|
|
5246
4570
|
for (
|
|
5247
4571
|
let index = 0;
|
|
@@ -5249,11 +4573,7 @@ export class PlayContextImpl {
|
|
|
5249
4573
|
index += 1
|
|
5250
4574
|
) {
|
|
5251
4575
|
const request = entry.request.memberRequests[index]!;
|
|
5252
|
-
|
|
5253
|
-
toolId,
|
|
5254
|
-
request,
|
|
5255
|
-
splitResults[index] ?? null,
|
|
5256
|
-
);
|
|
4576
|
+
resolveLiveFollowers(request, resolvedResults[index]);
|
|
5257
4577
|
}
|
|
5258
4578
|
}
|
|
5259
4579
|
|
|
@@ -5281,7 +4601,7 @@ export class PlayContextImpl {
|
|
|
5281
4601
|
toolId,
|
|
5282
4602
|
request.input,
|
|
5283
4603
|
);
|
|
5284
|
-
await this.resolveToolCall(
|
|
4604
|
+
const result = await this.resolveToolCall(
|
|
5285
4605
|
toolId,
|
|
5286
4606
|
request,
|
|
5287
4607
|
execution?.result ?? null,
|
|
@@ -5289,8 +4609,9 @@ export class PlayContextImpl {
|
|
|
5289
4609
|
execution?.jobId,
|
|
5290
4610
|
execution?.meta,
|
|
5291
4611
|
);
|
|
4612
|
+
resolveLiveFollowers(request, result);
|
|
5292
4613
|
} catch (error) {
|
|
5293
|
-
|
|
4614
|
+
await rejectWithLiveFollowers(request, error);
|
|
5294
4615
|
}
|
|
5295
4616
|
}),
|
|
5296
4617
|
);
|
|
@@ -5300,6 +4621,9 @@ export class PlayContextImpl {
|
|
|
5300
4621
|
}
|
|
5301
4622
|
}
|
|
5302
4623
|
}
|
|
4624
|
+
if (durableWaiters.length > 0) {
|
|
4625
|
+
await Promise.allSettled(durableWaiters);
|
|
4626
|
+
}
|
|
5303
4627
|
}),
|
|
5304
4628
|
);
|
|
5305
4629
|
}
|
|
@@ -5377,95 +4701,6 @@ export class PlayContextImpl {
|
|
|
5377
4701
|
)(ctx, input);
|
|
5378
4702
|
}
|
|
5379
4703
|
|
|
5380
|
-
// ——— Direct execution (outside map context) ———
|
|
5381
|
-
|
|
5382
|
-
private async executeWaterfallDirect(
|
|
5383
|
-
toolNameOrSpec: string | InlineWaterfallSpec,
|
|
5384
|
-
input: Record<string, unknown>,
|
|
5385
|
-
opts?: WaterfallOptions,
|
|
5386
|
-
): Promise<unknown | null> {
|
|
5387
|
-
if (isInlineWaterfallSpec(toolNameOrSpec)) {
|
|
5388
|
-
this.log(`Direct inline waterfall: ${toolNameOrSpec.id}`);
|
|
5389
|
-
const collected: unknown[] = [];
|
|
5390
|
-
for (const step of toolNameOrSpec.steps) {
|
|
5391
|
-
this.governor.chargeBudget('waterfallStep');
|
|
5392
|
-
const matched = isInlineWaterfallCodeStep(step)
|
|
5393
|
-
? extractInlineWaterfallCodeStepValue(
|
|
5394
|
-
toolNameOrSpec.output,
|
|
5395
|
-
await step.run(input, {
|
|
5396
|
-
tools: {
|
|
5397
|
-
execute: async (
|
|
5398
|
-
requestOrKey:
|
|
5399
|
-
| { tool: string; input: Record<string, unknown> }
|
|
5400
|
-
| string,
|
|
5401
|
-
toolId?: string,
|
|
5402
|
-
payload?: Record<string, unknown>,
|
|
5403
|
-
) => {
|
|
5404
|
-
if (typeof requestOrKey === 'object') {
|
|
5405
|
-
return await this.callToolAPI(
|
|
5406
|
-
requestOrKey.tool,
|
|
5407
|
-
requestOrKey.input,
|
|
5408
|
-
);
|
|
5409
|
-
}
|
|
5410
|
-
if (!toolId || !payload) {
|
|
5411
|
-
throw new Error(
|
|
5412
|
-
'inline waterfall ctx.tools.execute requires a tool and input.',
|
|
5413
|
-
);
|
|
5414
|
-
}
|
|
5415
|
-
return await this.callToolAPI(toolId, payload);
|
|
5416
|
-
},
|
|
5417
|
-
},
|
|
5418
|
-
}),
|
|
5419
|
-
)
|
|
5420
|
-
: await extractWaterfallOutputValue(
|
|
5421
|
-
step.toolId,
|
|
5422
|
-
toolNameOrSpec.output,
|
|
5423
|
-
await this.callToolAPI(step.toolId, step.mapInput(input)),
|
|
5424
|
-
this.#options.getToolTargetGetters,
|
|
5425
|
-
);
|
|
5426
|
-
if (Array.isArray(matched)) {
|
|
5427
|
-
collected.push(...matched);
|
|
5428
|
-
} else if (matched != null) {
|
|
5429
|
-
collected.push(matched);
|
|
5430
|
-
}
|
|
5431
|
-
if (collected.length >= toolNameOrSpec.minResults) {
|
|
5432
|
-
return toolNameOrSpec.minResults === 1
|
|
5433
|
-
? (collected[0] ?? null)
|
|
5434
|
-
: collected;
|
|
5435
|
-
}
|
|
5436
|
-
}
|
|
5437
|
-
return null;
|
|
5438
|
-
}
|
|
5439
|
-
|
|
5440
|
-
const toolName = toolNameOrSpec;
|
|
5441
|
-
this.log(`Direct waterfall: ${toolName}`);
|
|
5442
|
-
const providers = opts?.providers ?? ['hunter', 'leadmagic', 'pdl'];
|
|
5443
|
-
|
|
5444
|
-
for (const provider of providers) {
|
|
5445
|
-
this.log(` Trying ${provider}`);
|
|
5446
|
-
try {
|
|
5447
|
-
this.governor.chargeBudget('retry');
|
|
5448
|
-
const result = await this.callToolAPI(
|
|
5449
|
-
resolveWaterfallToolId(provider, toolName),
|
|
5450
|
-
input,
|
|
5451
|
-
);
|
|
5452
|
-
if (
|
|
5453
|
-
result != null &&
|
|
5454
|
-
typeof result === 'object' &&
|
|
5455
|
-
Object.keys(result as object).length > 0
|
|
5456
|
-
) {
|
|
5457
|
-
this.log(` Found with ${provider}`);
|
|
5458
|
-
return result;
|
|
5459
|
-
}
|
|
5460
|
-
} catch {
|
|
5461
|
-
this.log(` Failed with ${provider}`);
|
|
5462
|
-
}
|
|
5463
|
-
}
|
|
5464
|
-
|
|
5465
|
-
this.log(` All providers exhausted`);
|
|
5466
|
-
return null;
|
|
5467
|
-
}
|
|
5468
|
-
|
|
5469
4704
|
private async callToolAPI(
|
|
5470
4705
|
toolId: string,
|
|
5471
4706
|
input: Record<string, unknown>,
|