deepline 0.1.181 → 0.1.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +633 -107
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1932 -1838
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +124 -12
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
- package/dist/bundling-sources/sdk/src/client.ts +48 -5
- package/dist/bundling-sources/sdk/src/http.ts +23 -8
- package/dist/bundling-sources/sdk/src/play.ts +200 -26
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +132 -10
- package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
- package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1857 -314
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +106 -6
- package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
- package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
- package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/query-result-dataset.ts +120 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
- package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
- package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2037 -262
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +139 -17
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
- package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
- package/dist/cli/index.js +85 -23
- package/dist/cli/index.mjs +85 -23
- package/dist/index.d.mts +4 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.js +274 -35
- package/dist/index.mjs +274 -35
- package/dist/plays/bundle-play-file.mjs +2 -2
- package/package.json +1 -1
|
@@ -23,12 +23,24 @@ import {
|
|
|
23
23
|
} from './batch-runtime';
|
|
24
24
|
import type { PlayQueueHint } from './governor/rate-state-backend';
|
|
25
25
|
import type { MapRowOutcome } from './durability-store';
|
|
26
|
+
import type { WorkReceiptFailureKind } from './work-receipts';
|
|
26
27
|
import {
|
|
27
28
|
completedMapRowOutcome,
|
|
28
29
|
failedMapRowOutcome,
|
|
29
30
|
mapRowOutcomeRuntimeFields,
|
|
30
31
|
resolveMapRowOutcomeKey,
|
|
31
32
|
} from './map-row-outcome';
|
|
33
|
+
import { RuntimeSheetRowsBlockedError } from './runtime-sheet-errors';
|
|
34
|
+
import { buildChildRunId } from './child-run-id';
|
|
35
|
+
import type { PlayCallGovernanceSnapshot } from './scheduler-backend';
|
|
36
|
+
import {
|
|
37
|
+
RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER,
|
|
38
|
+
SYNTHETIC_RUN_HEADER,
|
|
39
|
+
} from './coordinator-headers';
|
|
40
|
+
export {
|
|
41
|
+
RuntimeSheetRowsBlockedError,
|
|
42
|
+
type RuntimeSheetBlockedRowDetail,
|
|
43
|
+
} from './runtime-sheet-errors';
|
|
32
44
|
import {
|
|
33
45
|
createDefaultGovernanceSnapshot,
|
|
34
46
|
createPlayExecutionGovernor,
|
|
@@ -41,6 +53,7 @@ import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backen
|
|
|
41
53
|
import { pacingPolicyForTool } from './pacing';
|
|
42
54
|
import {
|
|
43
55
|
cloneToolExecuteResultWithExecution,
|
|
56
|
+
attachToolResultListDataset,
|
|
44
57
|
createToolExecuteResult,
|
|
45
58
|
parseToolExecuteResponse,
|
|
46
59
|
isToolExecuteResult,
|
|
@@ -63,24 +76,38 @@ import {
|
|
|
63
76
|
TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
|
|
64
77
|
classifyToolExecuteHttpFailure,
|
|
65
78
|
createToolExecuteHttpFailureAttemptTracker,
|
|
79
|
+
parseToolExecuteAuthScopeChangedError,
|
|
80
|
+
ToolExecuteAuthScopeChangedError,
|
|
66
81
|
} from './tool-execute-retry-policy';
|
|
67
82
|
import {
|
|
68
83
|
buildDurableCtxCallCacheKey,
|
|
84
|
+
buildDurableRunPlaySemanticKey,
|
|
85
|
+
buildDurableToolAggregateProviderIdempotencyKey,
|
|
86
|
+
buildDurableToolAggregateReceiptKey,
|
|
69
87
|
buildDurableToolCallAuthScopeDigest,
|
|
70
88
|
buildDurableToolCallCacheKey,
|
|
89
|
+
buildDurableToolProviderIdempotencyKey,
|
|
90
|
+
buildDurableToolReceiptPrefix,
|
|
71
91
|
} from './durable-call-cache';
|
|
72
92
|
import {
|
|
93
|
+
RuntimeReceiptLeaseLostError,
|
|
73
94
|
RuntimeReceiptWaitTimeoutError,
|
|
74
95
|
executeWithDurableRuntimeReceipt,
|
|
75
96
|
resolveRuntimeToolReceiptWaitMaxAttempts,
|
|
97
|
+
runtimeReceiptFailureKindForError,
|
|
76
98
|
runtimeReceiptOutput as durableRuntimeReceiptOutput,
|
|
77
99
|
waitForCompletedRuntimeReceipt,
|
|
78
100
|
type DurableReceiptExecutionStore,
|
|
79
101
|
} from './durable-receipt-execution';
|
|
102
|
+
import {
|
|
103
|
+
QUERY_RESULT_DATASET_PAGE_SIZE,
|
|
104
|
+
isCustomerDbDatasetTool,
|
|
105
|
+
isQueryResultDatasetReadRequest,
|
|
106
|
+
isQueryResultDatasetTool,
|
|
107
|
+
} from './query-result-dataset';
|
|
80
108
|
import { isRowIsolationExemptError } from './row-isolation';
|
|
81
109
|
import {
|
|
82
110
|
createRuntimePersistenceLatch,
|
|
83
|
-
PROVIDER_DISPATCH_WAVE_SIZE,
|
|
84
111
|
RuntimePersistenceCircuitOpenError,
|
|
85
112
|
tripRuntimePersistenceLatch,
|
|
86
113
|
type RuntimePersistenceLatch,
|
|
@@ -112,7 +139,9 @@ import { DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE } from './runtime-constraints';
|
|
|
112
139
|
import {
|
|
113
140
|
PlayExecutionSuspendedError,
|
|
114
141
|
PlayRowExecutionSuspendedError,
|
|
142
|
+
isPlayExecutionSuspendedError,
|
|
115
143
|
isPlayRowExecutionSuspendedError,
|
|
144
|
+
type PlayExecutionSuspension,
|
|
116
145
|
} from './suspension';
|
|
117
146
|
import {
|
|
118
147
|
createSecretRedactionContext,
|
|
@@ -164,6 +193,7 @@ import {
|
|
|
164
193
|
type StepProgramDatasetColumnInput,
|
|
165
194
|
type StepProgramDatasetOptions,
|
|
166
195
|
} from './step-program-dataset-builder';
|
|
196
|
+
import { readRuntimeSheetDatasetRows } from './runtime-api';
|
|
167
197
|
|
|
168
198
|
/**
|
|
169
199
|
* SECURITY: AsyncLocalStorage is per async execution, not a cross-run cache.
|
|
@@ -200,9 +230,118 @@ const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
|
200
230
|
const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
|
|
201
231
|
const FETCH_TRANSPORT_MAX_ATTEMPTS = 3;
|
|
202
232
|
const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
|
|
233
|
+
const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100;
|
|
234
|
+
const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25;
|
|
203
235
|
type SafeFetchModule = typeof import('@shared_libs/security/safe-fetch');
|
|
204
236
|
let safeFetchModule: Promise<SafeFetchModule> | null = null;
|
|
205
237
|
|
|
238
|
+
export async function waitForNodeRuntimeMapRowsVisible(input: {
|
|
239
|
+
mapName: string;
|
|
240
|
+
tableNamespace: string;
|
|
241
|
+
runId?: string | null;
|
|
242
|
+
expectedRows: number;
|
|
243
|
+
updatedRows: number;
|
|
244
|
+
readVisibleRowCount: () => Promise<number>;
|
|
245
|
+
sleep?: (ms: number) => Promise<void>;
|
|
246
|
+
log?: (line: string) => void;
|
|
247
|
+
}): Promise<number> {
|
|
248
|
+
if (input.expectedRows <= 0) return 0;
|
|
249
|
+
|
|
250
|
+
const sleep =
|
|
251
|
+
input.sleep ??
|
|
252
|
+
((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
|
253
|
+
let lastVisibleRows = -1;
|
|
254
|
+
for (
|
|
255
|
+
let attempt = 1;
|
|
256
|
+
attempt <= NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS;
|
|
257
|
+
attempt += 1
|
|
258
|
+
) {
|
|
259
|
+
lastVisibleRows = await input.readVisibleRowCount();
|
|
260
|
+
if (lastVisibleRows >= input.expectedRows) {
|
|
261
|
+
if (attempt > 1) {
|
|
262
|
+
input.log?.(
|
|
263
|
+
`Runtime sheet visibility barrier satisfied ctx.dataset("${input.mapName}") ` +
|
|
264
|
+
`after ${attempt} read(s): visible ${lastVisibleRows}/${input.expectedRows}; ` +
|
|
265
|
+
`wrote ${input.updatedRows}/${input.expectedRows}; run ${input.runId ?? 'unknown'}.`,
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
return lastVisibleRows;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
if (attempt === 1) {
|
|
272
|
+
input.log?.(
|
|
273
|
+
`Runtime sheet visibility barrier waiting ctx.dataset("${input.mapName}"): ` +
|
|
274
|
+
`visible ${lastVisibleRows}/${input.expectedRows}; ` +
|
|
275
|
+
`wrote ${input.updatedRows}/${input.expectedRows}; run ${input.runId ?? 'unknown'}.`,
|
|
276
|
+
);
|
|
277
|
+
}
|
|
278
|
+
if (attempt < NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS) {
|
|
279
|
+
await sleep(NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
throw new Error(
|
|
284
|
+
`Runtime sheet visibility mismatch for ctx.dataset("${input.mapName}"): ` +
|
|
285
|
+
`expected ${input.expectedRows} terminal visible row(s), saw ${lastVisibleRows}; ` +
|
|
286
|
+
`write reported ${input.updatedRows}; run ${input.runId ?? 'unknown'}.`,
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
export async function reconcileNodeRuntimeMapResultsWithPersistedSheet(input: {
|
|
291
|
+
mapName: string;
|
|
292
|
+
tableNamespace: string;
|
|
293
|
+
runId?: string | null;
|
|
294
|
+
expectedRows: number;
|
|
295
|
+
currentRows: Record<string, unknown>[];
|
|
296
|
+
failedRowCount: number;
|
|
297
|
+
readPersistedRows: (input: {
|
|
298
|
+
limit: number;
|
|
299
|
+
offset: number;
|
|
300
|
+
rowMode: 'all';
|
|
301
|
+
}) => Promise<Record<string, unknown>[]>;
|
|
302
|
+
log?: (line: string) => void;
|
|
303
|
+
}): Promise<Record<string, unknown>[]> {
|
|
304
|
+
if (
|
|
305
|
+
input.failedRowCount > 0 ||
|
|
306
|
+
input.expectedRows <= 0 ||
|
|
307
|
+
input.currentRows.length >= input.expectedRows
|
|
308
|
+
) {
|
|
309
|
+
return input.currentRows;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const persistedRows: Record<string, unknown>[] = [];
|
|
313
|
+
let offset = 0;
|
|
314
|
+
while (persistedRows.length < input.expectedRows) {
|
|
315
|
+
const page = await input.readPersistedRows({
|
|
316
|
+
limit: Math.min(10_000, input.expectedRows - persistedRows.length),
|
|
317
|
+
offset,
|
|
318
|
+
rowMode: 'all',
|
|
319
|
+
});
|
|
320
|
+
if (page.length === 0) break;
|
|
321
|
+
persistedRows.push(...page);
|
|
322
|
+
offset += page.length;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
if (persistedRows.length < input.expectedRows) {
|
|
326
|
+
throw new Error(
|
|
327
|
+
`Runtime sheet finalization mismatch for ctx.dataset("${input.mapName}"): ` +
|
|
328
|
+
`expected ${input.expectedRows} terminal persisted row(s), saw ${persistedRows.length}; ` +
|
|
329
|
+
`in-memory Node map results reported ${input.currentRows.length}; run ${input.runId ?? 'unknown'}.`,
|
|
330
|
+
);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
if (persistedRows.length > input.currentRows.length) {
|
|
334
|
+
input.log?.(
|
|
335
|
+
`Runtime sheet finalization reconciled ctx.dataset("${input.mapName}") ` +
|
|
336
|
+
`from ${input.currentRows.length}/${input.expectedRows} in-memory Node row(s) ` +
|
|
337
|
+
`to ${persistedRows.length} terminal persisted row(s).`,
|
|
338
|
+
);
|
|
339
|
+
return persistedRows;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
return input.currentRows;
|
|
343
|
+
}
|
|
344
|
+
|
|
206
345
|
export function resolveToolRuntimeTimeoutMs(
|
|
207
346
|
toolId: string,
|
|
208
347
|
requestedTimeoutMs?: number,
|
|
@@ -266,7 +405,47 @@ function publicToolResponseEnvelope(value: unknown): {
|
|
|
266
405
|
|
|
267
406
|
const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
|
|
268
407
|
const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
|
|
408
|
+
const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
|
|
269
409
|
const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-response';
|
|
410
|
+
function recordOrNull(value: unknown): Record<string, unknown> | null {
|
|
411
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
412
|
+
? (value as Record<string, unknown>)
|
|
413
|
+
: null;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function rowsFromUnknown(value: unknown): Record<string, unknown>[] {
|
|
417
|
+
if (!Array.isArray(value)) return [];
|
|
418
|
+
return value.map((row) =>
|
|
419
|
+
row && typeof row === 'object' && !Array.isArray(row)
|
|
420
|
+
? (row as Record<string, unknown>)
|
|
421
|
+
: { value: row },
|
|
422
|
+
);
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
function finiteNonNegativeInteger(value: unknown): number | null {
|
|
426
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
427
|
+
return null;
|
|
428
|
+
}
|
|
429
|
+
return Math.floor(value);
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function finitePositiveInteger(value: unknown): number | null {
|
|
433
|
+
const integer = finiteNonNegativeInteger(value);
|
|
434
|
+
return integer !== null && integer > 0 ? integer : null;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
type ToolExecutionApiOptions = {
|
|
438
|
+
timeoutMs?: number;
|
|
439
|
+
durableCallReceiptKey?: string | null;
|
|
440
|
+
executionAuthScopeDigest?: string | null;
|
|
441
|
+
providerIdempotencyKey?: string | null;
|
|
442
|
+
customerDbDataset?: {
|
|
443
|
+
limit: number;
|
|
444
|
+
offset: number;
|
|
445
|
+
pageSize: number;
|
|
446
|
+
totalRows: number;
|
|
447
|
+
};
|
|
448
|
+
};
|
|
270
449
|
const IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT = 25;
|
|
271
450
|
const BATCH_SIZE_LOG_SAMPLE_LIMIT = 10;
|
|
272
451
|
const STEP_PROGRAM_MAP_DEFINITION = Symbol('deepline.stepProgramMapDefinition');
|
|
@@ -303,6 +482,16 @@ function persistableMapRowBytes(row: PersistableMapRow): number {
|
|
|
303
482
|
return JSON.stringify(row).length;
|
|
304
483
|
}
|
|
305
484
|
|
|
485
|
+
function failedMapRowLogLabel(row: PersistableMapRow): string {
|
|
486
|
+
const rowId = row.data?.row_id;
|
|
487
|
+
if (typeof rowId === 'string' && rowId.trim()) return rowId.trim();
|
|
488
|
+
if (row.key?.trim()) return row.key.trim();
|
|
489
|
+
if (typeof row.inputIndex === 'number' && Number.isFinite(row.inputIndex)) {
|
|
490
|
+
return `input:${Math.floor(row.inputIndex)}`;
|
|
491
|
+
}
|
|
492
|
+
return 'unknown';
|
|
493
|
+
}
|
|
494
|
+
|
|
306
495
|
type FieldMapRunResult = {
|
|
307
496
|
completedRows: PersistableMapRow[];
|
|
308
497
|
failedRows: PersistableMapRow[];
|
|
@@ -464,9 +653,7 @@ function durableCtxKey(input: {
|
|
|
464
653
|
staleAfterSeconds?: number | null;
|
|
465
654
|
}): string {
|
|
466
655
|
if (input.operation === 'tool') {
|
|
467
|
-
throw new Error(
|
|
468
|
-
'Durable ctx key is only for non-tool call boundaries; tools use buildDurableToolCallCacheKey.',
|
|
469
|
-
);
|
|
656
|
+
throw new Error('Tool calls use tool receipt keys.');
|
|
470
657
|
}
|
|
471
658
|
return buildDurableCtxCallCacheKey({
|
|
472
659
|
orgId: input.orgId,
|
|
@@ -712,6 +899,13 @@ export class PlayContextImpl {
|
|
|
712
899
|
eventKey: string;
|
|
713
900
|
timeoutMs: number;
|
|
714
901
|
}> = [];
|
|
902
|
+
private pendingChildPlayBoundaries: Array<{
|
|
903
|
+
boundaryId: string;
|
|
904
|
+
childRunId: string;
|
|
905
|
+
childPlayName: string;
|
|
906
|
+
}> = [];
|
|
907
|
+
private activeScheduledChildPlayCalls = 0;
|
|
908
|
+
private scheduledChildPlayQuiescenceWaiters: Array<() => void> = [];
|
|
715
909
|
private processedRowCount = 0;
|
|
716
910
|
private sleepBoundaryIndex = 0;
|
|
717
911
|
private fetchCallIndex = 0;
|
|
@@ -847,13 +1041,18 @@ export class PlayContextImpl {
|
|
|
847
1041
|
options.workflowId ||
|
|
848
1042
|
'anonymous-play';
|
|
849
1043
|
const rootRunId = options.runId ?? options.workflowId ?? 'anonymous-run';
|
|
1044
|
+
if (options.requireSharedRateState && !options.rateState) {
|
|
1045
|
+
throw new Error(
|
|
1046
|
+
'Shared rate-state backend is required for this Node runtime substrate.',
|
|
1047
|
+
);
|
|
1048
|
+
}
|
|
850
1049
|
this.governor = createPlayExecutionGovernor({
|
|
851
1050
|
adapter: 'cjs_node20',
|
|
852
1051
|
scope: {
|
|
853
1052
|
orgId: options.orgId ?? 'org',
|
|
854
1053
|
rootRunId: options.runId ?? options.workflowId ?? 'run',
|
|
855
1054
|
},
|
|
856
|
-
rateState: new InMemoryRateStateBackend(),
|
|
1055
|
+
rateState: options.rateState ?? new InMemoryRateStateBackend(),
|
|
857
1056
|
resolvePacing: createPacingResolver(options.getToolQueueHints),
|
|
858
1057
|
// A root run keeps depth 0 (its first child play is depth 1) but still
|
|
859
1058
|
// seeds its own play id into the ancestry/currentPlayId so the cycle guard
|
|
@@ -879,6 +1078,263 @@ export class PlayContextImpl {
|
|
|
879
1078
|
return `${this.governance.currentRunId}:${localId}`;
|
|
880
1079
|
}
|
|
881
1080
|
|
|
1081
|
+
private shouldLaunchChildPlaysThroughScheduler(): boolean {
|
|
1082
|
+
return (
|
|
1083
|
+
this.#options.requireSharedRateState === true &&
|
|
1084
|
+
this.#options.durableBoundaries === true
|
|
1085
|
+
);
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
private childPlayBoundaryId(childRunId: string): string {
|
|
1089
|
+
return this.durableBoundaryId(`runPlay:${childRunId}`);
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
private recordPendingChildPlayBoundary(input: {
|
|
1093
|
+
boundaryId: string;
|
|
1094
|
+
childRunId: string;
|
|
1095
|
+
childPlayName: string;
|
|
1096
|
+
}): void {
|
|
1097
|
+
this.pendingChildPlayBoundaries.push(input);
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
private beginScheduledChildPlayCall(): { release: () => void } {
|
|
1101
|
+
this.activeScheduledChildPlayCalls += 1;
|
|
1102
|
+
let released = false;
|
|
1103
|
+
return {
|
|
1104
|
+
release: () => {
|
|
1105
|
+
if (released) return;
|
|
1106
|
+
released = true;
|
|
1107
|
+
this.activeScheduledChildPlayCalls = Math.max(
|
|
1108
|
+
0,
|
|
1109
|
+
this.activeScheduledChildPlayCalls - 1,
|
|
1110
|
+
);
|
|
1111
|
+
if (this.activeScheduledChildPlayCalls !== 0) return;
|
|
1112
|
+
for (const resolve of this.scheduledChildPlayQuiescenceWaiters.splice(
|
|
1113
|
+
0,
|
|
1114
|
+
)) {
|
|
1115
|
+
resolve();
|
|
1116
|
+
}
|
|
1117
|
+
},
|
|
1118
|
+
};
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
private async waitForScheduledChildPlayQuiescence(): Promise<void> {
|
|
1122
|
+
if (this.activeScheduledChildPlayCalls === 0) return;
|
|
1123
|
+
await new Promise<void>((resolve) => {
|
|
1124
|
+
this.scheduledChildPlayQuiescenceWaiters.push(resolve);
|
|
1125
|
+
});
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
private consumePendingChildPlaySuspension(): PlayExecutionSuspension | null {
|
|
1129
|
+
if (this.pendingChildPlayBoundaries.length === 0) return null;
|
|
1130
|
+
const boundaries = [
|
|
1131
|
+
...new Map(
|
|
1132
|
+
this.pendingChildPlayBoundaries.map((boundary) => [
|
|
1133
|
+
boundary.boundaryId,
|
|
1134
|
+
boundary,
|
|
1135
|
+
]),
|
|
1136
|
+
).values(),
|
|
1137
|
+
];
|
|
1138
|
+
this.pendingChildPlayBoundaries = [];
|
|
1139
|
+
if (boundaries.length === 1) {
|
|
1140
|
+
const boundary = boundaries[0]!;
|
|
1141
|
+
return {
|
|
1142
|
+
kind: 'child_play',
|
|
1143
|
+
boundaryId: boundary.boundaryId,
|
|
1144
|
+
childRunId: boundary.childRunId,
|
|
1145
|
+
childPlayName: boundary.childPlayName,
|
|
1146
|
+
};
|
|
1147
|
+
}
|
|
1148
|
+
return {
|
|
1149
|
+
kind: 'child_play_batch',
|
|
1150
|
+
boundaries,
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
private childPlayBoundaryOutput<TOutput>(input: {
|
|
1155
|
+
boundaryId: string;
|
|
1156
|
+
childRunId: string;
|
|
1157
|
+
childPlayName: string;
|
|
1158
|
+
}): { found: true; output: TOutput } | { found: false } {
|
|
1159
|
+
const existing = this.checkpoint.resolvedBoundaries?.[input.boundaryId];
|
|
1160
|
+
if (!existing || existing.kind !== 'child_play') return { found: false };
|
|
1161
|
+
if (existing.childRunId !== input.childRunId) return { found: false };
|
|
1162
|
+
if (existing.status === 'completed') {
|
|
1163
|
+
return { found: true, output: existing.output as TOutput };
|
|
1164
|
+
}
|
|
1165
|
+
const message =
|
|
1166
|
+
typeof existing.error === 'string' && existing.error.trim()
|
|
1167
|
+
? existing.error.trim()
|
|
1168
|
+
: `Child play ${input.childPlayName} (${input.childRunId}) ${existing.status}.`;
|
|
1169
|
+
throw new Error(message);
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
private recordPlayCallStep(input: {
|
|
1173
|
+
playId: string;
|
|
1174
|
+
description?: string | null;
|
|
1175
|
+
nestedSteps?: PlayStep[];
|
|
1176
|
+
}): void {
|
|
1177
|
+
const step = {
|
|
1178
|
+
type: 'play_call' as const,
|
|
1179
|
+
playId: input.playId,
|
|
1180
|
+
nestedSteps: input.nestedSteps ?? [],
|
|
1181
|
+
description: normalizeStepDescription(input.description ?? undefined),
|
|
1182
|
+
};
|
|
1183
|
+
if (this.activeDatasetStep) {
|
|
1184
|
+
this.activeDatasetStep.substeps.push(step);
|
|
1185
|
+
} else {
|
|
1186
|
+
this.steps.push(step);
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
private runtimeApiHeaders(): Record<string, string> {
|
|
1191
|
+
if (!this.#options.executorToken?.trim()) {
|
|
1192
|
+
throw new Error(
|
|
1193
|
+
'ctx.runPlay scheduled child launch requires an executor token.',
|
|
1194
|
+
);
|
|
1195
|
+
}
|
|
1196
|
+
return {
|
|
1197
|
+
Authorization: `Bearer ${this.#options.executorToken.trim()}`,
|
|
1198
|
+
'Content-Type': 'application/json',
|
|
1199
|
+
...(this.#options.runtimeSchedulerSchema?.trim()
|
|
1200
|
+
? {
|
|
1201
|
+
[RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER]:
|
|
1202
|
+
this.#options.runtimeSchedulerSchema.trim(),
|
|
1203
|
+
[SYNTHETIC_RUN_HEADER]: '1',
|
|
1204
|
+
}
|
|
1205
|
+
: {}),
|
|
1206
|
+
...(this.#options.vercelProtectionBypassToken?.trim()
|
|
1207
|
+
? {
|
|
1208
|
+
'x-vercel-protection-bypass':
|
|
1209
|
+
this.#options.vercelProtectionBypassToken.trim(),
|
|
1210
|
+
}
|
|
1211
|
+
: {}),
|
|
1212
|
+
};
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
private runtimeApiBaseUrl(): string {
|
|
1216
|
+
const baseUrl = this.#options.baseUrl?.trim();
|
|
1217
|
+
if (!baseUrl) {
|
|
1218
|
+
throw new Error('ctx.runPlay scheduled child launch requires baseUrl.');
|
|
1219
|
+
}
|
|
1220
|
+
return baseUrl.replace(/\/$/, '');
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
private async launchScheduledChildPlay(input: {
|
|
1224
|
+
key: string;
|
|
1225
|
+
childRunId: string;
|
|
1226
|
+
childPlayName: string;
|
|
1227
|
+
childGovernance: GovernanceSnapshot;
|
|
1228
|
+
childInput: Record<string, unknown>;
|
|
1229
|
+
description?: string | null;
|
|
1230
|
+
}): Promise<void> {
|
|
1231
|
+
const parentPlayName =
|
|
1232
|
+
this.governance.currentPlayId ||
|
|
1233
|
+
this.#options.playName ||
|
|
1234
|
+
this.#options.playId ||
|
|
1235
|
+
'anonymous-play';
|
|
1236
|
+
const governance: PlayCallGovernanceSnapshot = {
|
|
1237
|
+
rootRunId: this.governance.rootRunId,
|
|
1238
|
+
parentRunId: this.governance.currentRunId,
|
|
1239
|
+
parentPlayName,
|
|
1240
|
+
key: input.key,
|
|
1241
|
+
ancestryPlayIds: [...this.governance.ancestryPlayIds],
|
|
1242
|
+
callDepth: input.childGovernance.callDepth,
|
|
1243
|
+
playCallCount: input.childGovernance.playCallCount,
|
|
1244
|
+
toolCallCount: input.childGovernance.toolCallCount,
|
|
1245
|
+
retryCount: input.childGovernance.retryCount,
|
|
1246
|
+
descendantCount: input.childGovernance.descendantCount,
|
|
1247
|
+
waterfallStepExecutions: input.childGovernance.waterfallStepExecutions,
|
|
1248
|
+
};
|
|
1249
|
+
const response = await fetch(
|
|
1250
|
+
`${this.runtimeApiBaseUrl()}/api/v2/plays/run`,
|
|
1251
|
+
{
|
|
1252
|
+
method: 'POST',
|
|
1253
|
+
headers: this.runtimeApiHeaders(),
|
|
1254
|
+
body: JSON.stringify({
|
|
1255
|
+
name: input.childPlayName,
|
|
1256
|
+
workflowId: input.childRunId,
|
|
1257
|
+
input: input.childInput,
|
|
1258
|
+
profile: 'hatchet',
|
|
1259
|
+
waitForCompletionMs: 0,
|
|
1260
|
+
internalRunPlay: {
|
|
1261
|
+
...governance,
|
|
1262
|
+
description: input.description ?? null,
|
|
1263
|
+
},
|
|
1264
|
+
}),
|
|
1265
|
+
},
|
|
1266
|
+
);
|
|
1267
|
+
if (!response.ok) {
|
|
1268
|
+
const body = (await response.json().catch(() => null)) as {
|
|
1269
|
+
error?: unknown;
|
|
1270
|
+
} | null;
|
|
1271
|
+
const message =
|
|
1272
|
+
typeof body?.error === 'string' && body.error.trim()
|
|
1273
|
+
? body.error.trim()
|
|
1274
|
+
: response.statusText;
|
|
1275
|
+
throw new Error(
|
|
1276
|
+
`ctx.runPlay(${input.key}) failed to launch child play "${input.childPlayName}": ${message} (status ${response.status}).`,
|
|
1277
|
+
);
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
private async readScheduledChildTerminal<TOutput>(input: {
|
|
1282
|
+
childRunId: string;
|
|
1283
|
+
childPlayName: string;
|
|
1284
|
+
}): Promise<
|
|
1285
|
+
| { status: 'completed'; output: TOutput }
|
|
1286
|
+
| { status: 'failed' | 'cancelled'; error: string }
|
|
1287
|
+
| null
|
|
1288
|
+
> {
|
|
1289
|
+
const response = await fetch(
|
|
1290
|
+
`${this.runtimeApiBaseUrl()}/api/v2/internal/play-runtime`,
|
|
1291
|
+
{
|
|
1292
|
+
method: 'POST',
|
|
1293
|
+
headers: this.runtimeApiHeaders(),
|
|
1294
|
+
body: JSON.stringify({
|
|
1295
|
+
action: 'read_child_run_terminal_snapshot',
|
|
1296
|
+
parentRunId: this.governance.currentRunId,
|
|
1297
|
+
childRunId: input.childRunId,
|
|
1298
|
+
childPlayName: input.childPlayName,
|
|
1299
|
+
}),
|
|
1300
|
+
},
|
|
1301
|
+
);
|
|
1302
|
+
const body = (await response.json().catch(() => null)) as {
|
|
1303
|
+
state?: {
|
|
1304
|
+
data?: {
|
|
1305
|
+
status?: unknown;
|
|
1306
|
+
result?: unknown;
|
|
1307
|
+
error?: unknown;
|
|
1308
|
+
};
|
|
1309
|
+
};
|
|
1310
|
+
error?: unknown;
|
|
1311
|
+
} | null;
|
|
1312
|
+
if (!response.ok) {
|
|
1313
|
+
const message =
|
|
1314
|
+
typeof body?.error === 'string' && body.error.trim()
|
|
1315
|
+
? body.error.trim()
|
|
1316
|
+
: response.statusText;
|
|
1317
|
+
throw new Error(
|
|
1318
|
+
`ctx.runPlay failed to read child terminal snapshot for "${input.childPlayName}": ${message} (status ${response.status}).`,
|
|
1319
|
+
);
|
|
1320
|
+
}
|
|
1321
|
+
const data = body?.state?.data;
|
|
1322
|
+
const status = String(data?.status ?? '').toLowerCase();
|
|
1323
|
+
if (status === 'completed') {
|
|
1324
|
+
return { status, output: data?.result as TOutput };
|
|
1325
|
+
}
|
|
1326
|
+
if (status === 'failed' || status === 'cancelled') {
|
|
1327
|
+
return {
|
|
1328
|
+
status,
|
|
1329
|
+
error:
|
|
1330
|
+
typeof data?.error === 'string' && data.error.trim()
|
|
1331
|
+
? data.error.trim()
|
|
1332
|
+
: `Child play ${input.childPlayName} (${input.childRunId}) ${status}.`,
|
|
1333
|
+
};
|
|
1334
|
+
}
|
|
1335
|
+
return null;
|
|
1336
|
+
}
|
|
1337
|
+
|
|
882
1338
|
private emitScopedRowUpdate(
|
|
883
1339
|
key: string | null,
|
|
884
1340
|
tableNamespace: string | null,
|
|
@@ -946,7 +1402,7 @@ export class PlayContextImpl {
|
|
|
946
1402
|
this.#options.runId
|
|
947
1403
|
) {
|
|
948
1404
|
const response = await fetch(
|
|
949
|
-
`${this.#options.baseUrl.replace(/\/$/, '')}/api/v2/
|
|
1405
|
+
`${this.#options.baseUrl.replace(/\/$/, '')}/api/v2/internal/play-runtime`,
|
|
950
1406
|
{
|
|
951
1407
|
method: 'POST',
|
|
952
1408
|
headers: {
|
|
@@ -1058,6 +1514,7 @@ export class PlayContextImpl {
|
|
|
1058
1514
|
runId: string,
|
|
1059
1515
|
reclaimRunning = false,
|
|
1060
1516
|
forceRefresh = false,
|
|
1517
|
+
forceFailedRefresh = false,
|
|
1061
1518
|
): Promise<RuntimeStepReceipt | null> {
|
|
1062
1519
|
if (!this.#options.claimRuntimeStepReceipt) {
|
|
1063
1520
|
return null;
|
|
@@ -1065,8 +1522,11 @@ export class PlayContextImpl {
|
|
|
1065
1522
|
const claimed = await this.#options.claimRuntimeStepReceipt({
|
|
1066
1523
|
key,
|
|
1067
1524
|
runId,
|
|
1525
|
+
runAttempt: this.currentRunAttempt,
|
|
1526
|
+
leaseAware: true,
|
|
1068
1527
|
...(reclaimRunning ? { reclaimRunning: true } : {}),
|
|
1069
1528
|
...(forceRefresh ? { forceRefresh: true } : {}),
|
|
1529
|
+
...(forceFailedRefresh ? { forceFailedRefresh: true } : {}),
|
|
1070
1530
|
});
|
|
1071
1531
|
if (!claimed || typeof claimed.key !== 'string' || !claimed.key.trim()) {
|
|
1072
1532
|
return null;
|
|
@@ -1082,6 +1542,7 @@ export class PlayContextImpl {
|
|
|
1082
1542
|
key: string,
|
|
1083
1543
|
runId: string,
|
|
1084
1544
|
output: unknown | null,
|
|
1545
|
+
leaseId?: string | null,
|
|
1085
1546
|
): Promise<RuntimeStepReceipt | null> {
|
|
1086
1547
|
assertNoSecretTaint(output, 'ctx.step receipt output');
|
|
1087
1548
|
if (!this.#options.completeRuntimeStepReceipt) {
|
|
@@ -1090,6 +1551,8 @@ export class PlayContextImpl {
|
|
|
1090
1551
|
const completed = await this.#options.completeRuntimeStepReceipt({
|
|
1091
1552
|
key,
|
|
1092
1553
|
runId,
|
|
1554
|
+
runAttempt: this.currentRunAttempt,
|
|
1555
|
+
...(leaseId ? { leaseId } : {}),
|
|
1093
1556
|
output,
|
|
1094
1557
|
});
|
|
1095
1558
|
if (
|
|
@@ -1106,10 +1569,167 @@ export class PlayContextImpl {
|
|
|
1106
1569
|
};
|
|
1107
1570
|
}
|
|
1108
1571
|
|
|
1572
|
+
private async releaseRuntimeStepReceipt(
|
|
1573
|
+
key: string,
|
|
1574
|
+
runId: string,
|
|
1575
|
+
leaseId?: string | null,
|
|
1576
|
+
): Promise<RuntimeStepReceipt | null> {
|
|
1577
|
+
if (!this.#options.releaseRuntimeStepReceipt) {
|
|
1578
|
+
return null;
|
|
1579
|
+
}
|
|
1580
|
+
const released = await this.#options.releaseRuntimeStepReceipt({
|
|
1581
|
+
key,
|
|
1582
|
+
runId,
|
|
1583
|
+
runAttempt: this.currentRunAttempt,
|
|
1584
|
+
...(leaseId ? { leaseId } : {}),
|
|
1585
|
+
});
|
|
1586
|
+
if (!released || typeof released.key !== 'string' || !released.key.trim()) {
|
|
1587
|
+
return null;
|
|
1588
|
+
}
|
|
1589
|
+
return {
|
|
1590
|
+
...released,
|
|
1591
|
+
key: released.key.trim(),
|
|
1592
|
+
runId: released.runId ?? null,
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
|
|
1596
|
+
private async heartbeatRuntimeStepReceipt(
|
|
1597
|
+
key: string,
|
|
1598
|
+
runId: string,
|
|
1599
|
+
leaseId: string,
|
|
1600
|
+
): Promise<RuntimeStepReceipt | null> {
|
|
1601
|
+
if (!this.#options.heartbeatRuntimeStepReceipts) {
|
|
1602
|
+
return null;
|
|
1603
|
+
}
|
|
1604
|
+
const [receipt] = await this.#options.heartbeatRuntimeStepReceipts({
|
|
1605
|
+
runId,
|
|
1606
|
+
runAttempt: this.currentRunAttempt,
|
|
1607
|
+
leaseId,
|
|
1608
|
+
keys: [key],
|
|
1609
|
+
});
|
|
1610
|
+
if (!receipt || typeof receipt.key !== 'string' || !receipt.key.trim()) {
|
|
1611
|
+
return null;
|
|
1612
|
+
}
|
|
1613
|
+
return {
|
|
1614
|
+
...receipt,
|
|
1615
|
+
key: receipt.key.trim(),
|
|
1616
|
+
runId: receipt.runId ?? null,
|
|
1617
|
+
};
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
private async assertRuntimeToolReceiptOwnership(
|
|
1621
|
+
requests: ToolCallRequest[],
|
|
1622
|
+
): Promise<void> {
|
|
1623
|
+
const targets = requests.flatMap((request) => {
|
|
1624
|
+
const receiptKey = request.receiptKey?.trim() || null;
|
|
1625
|
+
if (!receiptKey) return [];
|
|
1626
|
+
const leaseId = request.receiptLeaseId?.trim() || null;
|
|
1627
|
+
if (!leaseId) return [];
|
|
1628
|
+
return [{ receiptKey, leaseId }];
|
|
1629
|
+
});
|
|
1630
|
+
if (targets.length === 0) return;
|
|
1631
|
+
|
|
1632
|
+
const byLeaseId = new Map<string, string[]>();
|
|
1633
|
+
for (const target of targets) {
|
|
1634
|
+
const keys = byLeaseId.get(target.leaseId) ?? [];
|
|
1635
|
+
keys.push(target.receiptKey);
|
|
1636
|
+
byLeaseId.set(target.leaseId, keys);
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
if (this.#options.heartbeatRuntimeStepReceipts) {
|
|
1640
|
+
for (const [leaseId, keys] of byLeaseId) {
|
|
1641
|
+
const receipts = await this.#options.heartbeatRuntimeStepReceipts({
|
|
1642
|
+
runId: this.currentRunId,
|
|
1643
|
+
runAttempt: this.currentRunAttempt,
|
|
1644
|
+
leaseId,
|
|
1645
|
+
keys,
|
|
1646
|
+
});
|
|
1647
|
+
this.assertRuntimeToolReceiptHeartbeatResult(keys, leaseId, receipts);
|
|
1648
|
+
}
|
|
1649
|
+
return;
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
if (
|
|
1653
|
+
!this.#options.getRuntimeStepReceipt &&
|
|
1654
|
+
!this.#options.getRuntimeStepReceipts
|
|
1655
|
+
) {
|
|
1656
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
1657
|
+
receiptKey: targets[0]?.receiptKey ?? 'unknown',
|
|
1658
|
+
runId: this.currentRunId,
|
|
1659
|
+
leaseId: targets[0]?.leaseId ?? 'unknown',
|
|
1660
|
+
});
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
const latest = await this.getRuntimeStepReceipts(
|
|
1664
|
+
targets.map((target) => target.receiptKey),
|
|
1665
|
+
);
|
|
1666
|
+
for (const target of targets) {
|
|
1667
|
+
const receipt = latest.get(target.receiptKey);
|
|
1668
|
+
if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) {
|
|
1669
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
1670
|
+
receiptKey: target.receiptKey,
|
|
1671
|
+
runId: this.currentRunId,
|
|
1672
|
+
leaseId: target.leaseId,
|
|
1673
|
+
});
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
private assertRuntimeToolReceiptHeartbeatResult(
|
|
1679
|
+
keys: string[],
|
|
1680
|
+
leaseId: string,
|
|
1681
|
+
receipts: Array<RuntimeStepReceipt | null>,
|
|
1682
|
+
): void {
|
|
1683
|
+
for (let index = 0; index < keys.length; index += 1) {
|
|
1684
|
+
const receipt = this.normalizeRuntimeStepReceipt(
|
|
1685
|
+
keys[index] ?? '',
|
|
1686
|
+
receipts[index],
|
|
1687
|
+
);
|
|
1688
|
+
if (!this.runtimeToolReceiptStillOwned(receipt, leaseId)) {
|
|
1689
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
1690
|
+
receiptKey: keys[index] ?? 'unknown',
|
|
1691
|
+
runId: this.currentRunId,
|
|
1692
|
+
leaseId,
|
|
1693
|
+
});
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
private runtimeToolReceiptStillOwned(
|
|
1699
|
+
receipt: RuntimeStepReceipt | null | undefined,
|
|
1700
|
+
leaseId: string,
|
|
1701
|
+
): boolean {
|
|
1702
|
+
if (!receipt || receipt.status !== 'running') return false;
|
|
1703
|
+
if (receipt.leaseId !== leaseId) return false;
|
|
1704
|
+
const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
|
|
1705
|
+
return !ownerRunId || ownerRunId === this.currentRunId;
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
private runtimeToolReceiptStillOwnedByCurrentAttempt(
|
|
1709
|
+
receipt: RuntimeStepReceipt | null | undefined,
|
|
1710
|
+
leaseId: string | null | undefined,
|
|
1711
|
+
): boolean {
|
|
1712
|
+
if (!receipt || receipt.status !== 'running') return false;
|
|
1713
|
+
if (leaseId?.trim()) {
|
|
1714
|
+
if (receipt.leaseId !== leaseId.trim()) return false;
|
|
1715
|
+
} else if (receipt.leaseId != null) {
|
|
1716
|
+
return false;
|
|
1717
|
+
}
|
|
1718
|
+
const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
|
|
1719
|
+
if (ownerRunId !== this.currentRunId) return false;
|
|
1720
|
+
const ownerAttempt =
|
|
1721
|
+
typeof receipt.leaseOwnerAttempt === 'number'
|
|
1722
|
+
? receipt.leaseOwnerAttempt
|
|
1723
|
+
: 0;
|
|
1724
|
+
return ownerAttempt === this.currentRunAttempt;
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1109
1727
|
private async failRuntimeStepReceipt(
|
|
1110
1728
|
key: string,
|
|
1111
1729
|
runId: string,
|
|
1112
1730
|
error: string,
|
|
1731
|
+
leaseId?: string | null,
|
|
1732
|
+
failureKind?: WorkReceiptFailureKind | null,
|
|
1113
1733
|
): Promise<RuntimeStepReceipt | null> {
|
|
1114
1734
|
if (!this.#options.failRuntimeStepReceipt) {
|
|
1115
1735
|
return null;
|
|
@@ -1117,6 +1737,9 @@ export class PlayContextImpl {
|
|
|
1117
1737
|
const failed = await this.#options.failRuntimeStepReceipt({
|
|
1118
1738
|
key,
|
|
1119
1739
|
runId,
|
|
1740
|
+
runAttempt: this.currentRunAttempt,
|
|
1741
|
+
...(leaseId ? { leaseId } : {}),
|
|
1742
|
+
...(failureKind ? { failureKind } : {}),
|
|
1120
1743
|
error,
|
|
1121
1744
|
});
|
|
1122
1745
|
if (!failed || typeof failed.key !== 'string' || !failed.key.trim()) {
|
|
@@ -1176,6 +1799,7 @@ export class PlayContextImpl {
|
|
|
1176
1799
|
runId: string,
|
|
1177
1800
|
reclaimRunning = false,
|
|
1178
1801
|
forceRefresh = false,
|
|
1802
|
+
forceFailedRefresh = false,
|
|
1179
1803
|
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
1180
1804
|
const uniqueKeys = [
|
|
1181
1805
|
...new Set(keys.map((key) => key.trim()).filter(Boolean)),
|
|
@@ -1185,8 +1809,11 @@ export class PlayContextImpl {
|
|
|
1185
1809
|
? await this.#options.claimRuntimeStepReceipts({
|
|
1186
1810
|
keys: uniqueKeys,
|
|
1187
1811
|
runId,
|
|
1812
|
+
runAttempt: this.currentRunAttempt,
|
|
1813
|
+
leaseAware: true,
|
|
1188
1814
|
...(reclaimRunning ? { reclaimRunning: true } : {}),
|
|
1189
1815
|
...(forceRefresh ? { forceRefresh: true } : {}),
|
|
1816
|
+
...(forceFailedRefresh ? { forceFailedRefresh: true } : {}),
|
|
1190
1817
|
})
|
|
1191
1818
|
: await Promise.all(
|
|
1192
1819
|
uniqueKeys.map((key) =>
|
|
@@ -1195,6 +1822,7 @@ export class PlayContextImpl {
|
|
|
1195
1822
|
runId,
|
|
1196
1823
|
reclaimRunning,
|
|
1197
1824
|
forceRefresh,
|
|
1825
|
+
forceFailedRefresh,
|
|
1198
1826
|
),
|
|
1199
1827
|
),
|
|
1200
1828
|
);
|
|
@@ -1210,26 +1838,44 @@ export class PlayContextImpl {
|
|
|
1210
1838
|
}
|
|
1211
1839
|
|
|
1212
1840
|
private async completeRuntimeStepReceipts(
|
|
1213
|
-
receipts: Array<{
|
|
1841
|
+
receipts: Array<{
|
|
1842
|
+
key: string;
|
|
1843
|
+
runId: string;
|
|
1844
|
+
runAttempt?: number | null;
|
|
1845
|
+
output: unknown | null;
|
|
1846
|
+
leaseId?: string | null;
|
|
1847
|
+
}>,
|
|
1214
1848
|
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
1215
1849
|
const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
|
|
1216
1850
|
if (normalizedInputs.length === 0) return new Map();
|
|
1217
1851
|
for (const receipt of normalizedInputs) {
|
|
1218
1852
|
assertNoSecretTaint(receipt.output, 'ctx.tool receipt output');
|
|
1219
1853
|
}
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1854
|
+
let completed: Array<RuntimeStepReceipt | null>;
|
|
1855
|
+
try {
|
|
1856
|
+
completed = this.#options.completeRuntimeStepReceipts
|
|
1857
|
+
? await this.#options.completeRuntimeStepReceipts({
|
|
1858
|
+
receipts: normalizedInputs.map((receipt) => ({
|
|
1859
|
+
...receipt,
|
|
1860
|
+
runAttempt: receipt.runAttempt ?? this.currentRunAttempt,
|
|
1861
|
+
})),
|
|
1862
|
+
})
|
|
1863
|
+
: await Promise.all(
|
|
1864
|
+
normalizedInputs.map((receipt) =>
|
|
1865
|
+
this.completeRuntimeStepReceipt(
|
|
1866
|
+
receipt.key,
|
|
1867
|
+
receipt.runId,
|
|
1868
|
+
receipt.output,
|
|
1869
|
+
receipt.leaseId,
|
|
1870
|
+
),
|
|
1230
1871
|
),
|
|
1231
|
-
)
|
|
1232
|
-
|
|
1872
|
+
);
|
|
1873
|
+
} catch (error) {
|
|
1874
|
+
await this.recoverRuntimeStepReceiptCompletionsAfterBulkError(
|
|
1875
|
+
normalizedInputs,
|
|
1876
|
+
);
|
|
1877
|
+
throw error;
|
|
1878
|
+
}
|
|
1233
1879
|
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
1234
1880
|
for (let index = 0; index < completed.length; index += 1) {
|
|
1235
1881
|
const normalized = this.normalizeRuntimeStepReceipt(
|
|
@@ -1238,17 +1884,123 @@ export class PlayContextImpl {
|
|
|
1238
1884
|
);
|
|
1239
1885
|
if (normalized) byKey.set(normalized.key, normalized);
|
|
1240
1886
|
}
|
|
1887
|
+
await this.reconcileMissingRuntimeStepReceiptCompletions(
|
|
1888
|
+
normalizedInputs,
|
|
1889
|
+
byKey,
|
|
1890
|
+
);
|
|
1241
1891
|
return byKey;
|
|
1242
1892
|
}
|
|
1243
1893
|
|
|
1894
|
+
private async reconcileMissingRuntimeStepReceiptCompletions(
|
|
1895
|
+
receipts: Array<{
|
|
1896
|
+
key: string;
|
|
1897
|
+
runId: string;
|
|
1898
|
+
runAttempt?: number | null;
|
|
1899
|
+
output: unknown | null;
|
|
1900
|
+
leaseId?: string | null;
|
|
1901
|
+
}>,
|
|
1902
|
+
completedByKey: Map<string, RuntimeStepReceipt>,
|
|
1903
|
+
): Promise<void> {
|
|
1904
|
+
if (
|
|
1905
|
+
receipts.length === completedByKey.size ||
|
|
1906
|
+
(!this.#options.getRuntimeStepReceipt &&
|
|
1907
|
+
!this.#options.getRuntimeStepReceipts)
|
|
1908
|
+
) {
|
|
1909
|
+
return;
|
|
1910
|
+
}
|
|
1911
|
+
const missingKeys = receipts
|
|
1912
|
+
.map((receipt) => receipt.key.trim())
|
|
1913
|
+
.filter((key) => key && !completedByKey.has(key));
|
|
1914
|
+
if (missingKeys.length === 0) return;
|
|
1915
|
+
const latest = await this.getRuntimeStepReceipts(missingKeys);
|
|
1916
|
+
for (const key of missingKeys) {
|
|
1917
|
+
const recovered = latest.get(key);
|
|
1918
|
+
if (
|
|
1919
|
+
recovered?.status === 'completed' ||
|
|
1920
|
+
recovered?.status === 'skipped'
|
|
1921
|
+
) {
|
|
1922
|
+
completedByKey.set(key, recovered);
|
|
1923
|
+
}
|
|
1924
|
+
}
|
|
1925
|
+
const recoveredCount = missingKeys.filter((key) =>
|
|
1926
|
+
completedByKey.has(key),
|
|
1927
|
+
).length;
|
|
1928
|
+
if (recoveredCount > 0) {
|
|
1929
|
+
this.log(
|
|
1930
|
+
`Runtime tool receipt completion response reconciled ${recoveredCount}/${missingKeys.length} missing receipt(s) from durable store read-back.`,
|
|
1931
|
+
);
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
private async recoverRuntimeStepReceiptCompletionsAfterBulkError(
|
|
1936
|
+
receipts: Array<{
|
|
1937
|
+
key: string;
|
|
1938
|
+
runId: string;
|
|
1939
|
+
runAttempt?: number | null;
|
|
1940
|
+
output: unknown | null;
|
|
1941
|
+
leaseId?: string | null;
|
|
1942
|
+
}>,
|
|
1943
|
+
): Promise<void> {
|
|
1944
|
+
if (
|
|
1945
|
+
receipts.length === 0 ||
|
|
1946
|
+
(!this.#options.getRuntimeStepReceipt &&
|
|
1947
|
+
!this.#options.getRuntimeStepReceipts)
|
|
1948
|
+
) {
|
|
1949
|
+
return;
|
|
1950
|
+
}
|
|
1951
|
+
let latest: Map<string, RuntimeStepReceipt>;
|
|
1952
|
+
try {
|
|
1953
|
+
latest = await this.getRuntimeStepReceipts(
|
|
1954
|
+
receipts.map((receipt) => receipt.key),
|
|
1955
|
+
);
|
|
1956
|
+
} catch {
|
|
1957
|
+
return;
|
|
1958
|
+
}
|
|
1959
|
+
await Promise.allSettled(
|
|
1960
|
+
receipts.map(async (receipt) => {
|
|
1961
|
+
const recovered = latest.get(receipt.key);
|
|
1962
|
+
if (
|
|
1963
|
+
recovered?.status === 'completed' ||
|
|
1964
|
+
recovered?.status === 'skipped'
|
|
1965
|
+
) {
|
|
1966
|
+
return;
|
|
1967
|
+
}
|
|
1968
|
+
if (
|
|
1969
|
+
!this.runtimeToolReceiptStillOwnedByCurrentAttempt(
|
|
1970
|
+
recovered,
|
|
1971
|
+
receipt.leaseId,
|
|
1972
|
+
)
|
|
1973
|
+
) {
|
|
1974
|
+
return;
|
|
1975
|
+
}
|
|
1976
|
+
await this.completeRuntimeStepReceipt(
|
|
1977
|
+
receipt.key,
|
|
1978
|
+
receipt.runId,
|
|
1979
|
+
receipt.output,
|
|
1980
|
+
receipt.leaseId,
|
|
1981
|
+
);
|
|
1982
|
+
}),
|
|
1983
|
+
);
|
|
1984
|
+
}
|
|
1985
|
+
|
|
1244
1986
|
private async failRuntimeStepReceipts(
|
|
1245
|
-
receipts: Array<{
|
|
1987
|
+
receipts: Array<{
|
|
1988
|
+
key: string;
|
|
1989
|
+
runId: string;
|
|
1990
|
+
runAttempt?: number | null;
|
|
1991
|
+
error: string;
|
|
1992
|
+
leaseId?: string | null;
|
|
1993
|
+
failureKind?: WorkReceiptFailureKind | null;
|
|
1994
|
+
}>,
|
|
1246
1995
|
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
1247
1996
|
const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
|
|
1248
1997
|
if (normalizedInputs.length === 0) return new Map();
|
|
1249
1998
|
const failed = this.#options.failRuntimeStepReceipts
|
|
1250
1999
|
? await this.#options.failRuntimeStepReceipts({
|
|
1251
|
-
receipts: normalizedInputs
|
|
2000
|
+
receipts: normalizedInputs.map((receipt) => ({
|
|
2001
|
+
...receipt,
|
|
2002
|
+
runAttempt: receipt.runAttempt ?? this.currentRunAttempt,
|
|
2003
|
+
})),
|
|
1252
2004
|
})
|
|
1253
2005
|
: await Promise.all(
|
|
1254
2006
|
normalizedInputs.map((receipt) =>
|
|
@@ -1256,6 +2008,8 @@ export class PlayContextImpl {
|
|
|
1256
2008
|
receipt.key,
|
|
1257
2009
|
receipt.runId,
|
|
1258
2010
|
receipt.error,
|
|
2011
|
+
receipt.leaseId,
|
|
2012
|
+
receipt.failureKind,
|
|
1259
2013
|
),
|
|
1260
2014
|
),
|
|
1261
2015
|
);
|
|
@@ -1283,45 +2037,98 @@ export class PlayContextImpl {
|
|
|
1283
2037
|
);
|
|
1284
2038
|
}
|
|
1285
2039
|
|
|
1286
|
-
private async
|
|
2040
|
+
private async durableToolCallCacheKeyForScope(input: {
|
|
1287
2041
|
toolId: string;
|
|
1288
2042
|
requestInput: Record<string, unknown>;
|
|
2043
|
+
executionAuthScopeDigest?: string | null;
|
|
1289
2044
|
staleAfterSeconds?: number | null;
|
|
2045
|
+
playLocalScope?: string | null;
|
|
1290
2046
|
}): Promise<string> {
|
|
1291
2047
|
const providerActionVersion =
|
|
1292
2048
|
(await this.#options.getToolActionCacheVersion?.(input.toolId))?.trim() ??
|
|
1293
2049
|
'';
|
|
2050
|
+
const executionAuthScopeDigest =
|
|
2051
|
+
input.executionAuthScopeDigest?.trim() ||
|
|
2052
|
+
(await this.resolveToolAuthScopeDigest(input.toolId))?.trim() ||
|
|
2053
|
+
null;
|
|
1294
2054
|
return buildDurableToolCallCacheKey({
|
|
1295
2055
|
orgId: this.#options.orgId,
|
|
1296
|
-
|
|
2056
|
+
playLocalScope: input.playLocalScope,
|
|
1297
2057
|
toolId: input.toolId,
|
|
1298
2058
|
requestInput: input.requestInput,
|
|
1299
2059
|
authScopeDigest: buildDurableToolCallAuthScopeDigest({
|
|
1300
2060
|
orgId: this.#options.orgId,
|
|
1301
|
-
userEmail: this.#options.userEmail,
|
|
1302
2061
|
toolId: input.toolId,
|
|
2062
|
+
executionAuthScopeDigest,
|
|
1303
2063
|
}),
|
|
1304
2064
|
providerActionVersion,
|
|
1305
2065
|
staleAfterSeconds: input.staleAfterSeconds,
|
|
1306
2066
|
});
|
|
1307
2067
|
}
|
|
1308
2068
|
|
|
2069
|
+
private async durableToolCallCacheKey(input: {
|
|
2070
|
+
toolId: string;
|
|
2071
|
+
requestInput: Record<string, unknown>;
|
|
2072
|
+
executionAuthScopeDigest?: string | null;
|
|
2073
|
+
staleAfterSeconds?: number | null;
|
|
2074
|
+
}): Promise<string> {
|
|
2075
|
+
return await this.durableToolCallCacheKeyForScope({
|
|
2076
|
+
...input,
|
|
2077
|
+
playLocalScope: this.governance.currentPlayId,
|
|
2078
|
+
});
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
private async resolveToolAuthScopeDigest(
|
|
2082
|
+
toolId: string,
|
|
2083
|
+
): Promise<string | null> {
|
|
2084
|
+
const configured = await this.#options.getToolAuthScopeDigest?.(toolId);
|
|
2085
|
+
if (typeof configured === 'string' && configured.trim()) {
|
|
2086
|
+
return configured.trim();
|
|
2087
|
+
}
|
|
2088
|
+
return null;
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
private invalidateToolAuthScopeDigest(toolId: string): void {
|
|
2092
|
+
this.#options.invalidateToolAuthScopeDigest?.(toolId);
|
|
2093
|
+
}
|
|
2094
|
+
|
|
1309
2095
|
private durableReceiptExecutionStore(): DurableReceiptExecutionStore {
|
|
1310
2096
|
return {
|
|
1311
2097
|
enabled: Boolean(this.#options.claimRuntimeStepReceipt),
|
|
1312
2098
|
get: (receiptKey) => this.getRuntimeStepReceipt(receiptKey),
|
|
1313
2099
|
getMany: (receiptKeys) => this.getRuntimeStepReceipts(receiptKeys),
|
|
1314
|
-
claim: (
|
|
2100
|
+
claim: (
|
|
2101
|
+
receiptKey,
|
|
2102
|
+
runId,
|
|
2103
|
+
reclaimRunning,
|
|
2104
|
+
forceRefresh,
|
|
2105
|
+
forceFailedRefresh,
|
|
2106
|
+
) =>
|
|
1315
2107
|
this.claimRuntimeStepReceipt(
|
|
1316
2108
|
receiptKey,
|
|
1317
2109
|
runId,
|
|
1318
2110
|
reclaimRunning,
|
|
1319
2111
|
forceRefresh,
|
|
2112
|
+
forceFailedRefresh,
|
|
2113
|
+
),
|
|
2114
|
+
complete: (receiptKey, runId, output, leaseId) =>
|
|
2115
|
+
this.completeRuntimeStepReceipt(receiptKey, runId, output, leaseId),
|
|
2116
|
+
release: (receiptKey, runId, leaseId) =>
|
|
2117
|
+
this.releaseRuntimeStepReceipt(receiptKey, runId, leaseId),
|
|
2118
|
+
...(this.#options.heartbeatRuntimeStepReceipts
|
|
2119
|
+
? {
|
|
2120
|
+
heartbeat: (receiptKey, runId, leaseId) =>
|
|
2121
|
+
this.heartbeatRuntimeStepReceipt(receiptKey, runId, leaseId),
|
|
2122
|
+
}
|
|
2123
|
+
: {}),
|
|
2124
|
+
fail: (receiptKey, runId, error, leaseId, failureKind) =>
|
|
2125
|
+
this.failRuntimeStepReceipt(
|
|
2126
|
+
receiptKey,
|
|
2127
|
+
runId,
|
|
2128
|
+
error,
|
|
2129
|
+
leaseId,
|
|
2130
|
+
failureKind,
|
|
1320
2131
|
),
|
|
1321
|
-
complete: (receiptKey, runId, output) =>
|
|
1322
|
-
this.completeRuntimeStepReceipt(receiptKey, runId, output),
|
|
1323
|
-
fail: (receiptKey, runId, error) =>
|
|
1324
|
-
this.failRuntimeStepReceipt(receiptKey, runId, error),
|
|
1325
2132
|
canPersistFailure: Boolean(this.#options.failRuntimeStepReceipt),
|
|
1326
2133
|
canPersistCompletion: Boolean(this.#options.completeRuntimeStepReceipt),
|
|
1327
2134
|
};
|
|
@@ -1363,7 +2170,8 @@ export class PlayContextImpl {
|
|
|
1363
2170
|
source: DurableReceiptRecoverySource,
|
|
1364
2171
|
) => T;
|
|
1365
2172
|
onClaimedResult?: (output: T, receiptKey: string) => T;
|
|
1366
|
-
|
|
2173
|
+
shouldPersistFailure?: (error: unknown) => boolean;
|
|
2174
|
+
execute: (context: { leaseId: string | null }) => Promise<T>;
|
|
1367
2175
|
},
|
|
1368
2176
|
): Promise<T> {
|
|
1369
2177
|
const receiptKey =
|
|
@@ -1392,6 +2200,7 @@ export class PlayContextImpl {
|
|
|
1392
2200
|
markSkipped: opts.markSkipped,
|
|
1393
2201
|
onRecovered: opts.onRecovered,
|
|
1394
2202
|
onClaimedResult: opts.onClaimedResult,
|
|
2203
|
+
shouldPersistFailure: opts.shouldPersistFailure,
|
|
1395
2204
|
formatError: (error) => this.formatRuntimeError(error),
|
|
1396
2205
|
log: (message) => this.log(message),
|
|
1397
2206
|
execute: opts.execute,
|
|
@@ -1402,6 +2211,13 @@ export class PlayContextImpl {
|
|
|
1402
2211
|
return this.#options.runId ?? this.governance.currentRunId ?? 'unknown';
|
|
1403
2212
|
}
|
|
1404
2213
|
|
|
2214
|
+
private get currentRunAttempt(): number {
|
|
2215
|
+
const attempt = this.#options.runAttempt;
|
|
2216
|
+
return typeof attempt === 'number' && Number.isFinite(attempt)
|
|
2217
|
+
? Math.max(0, Math.floor(attempt))
|
|
2218
|
+
: 0;
|
|
2219
|
+
}
|
|
2220
|
+
|
|
1405
2221
|
private emitScopedFieldMetaUpdate(input: {
|
|
1406
2222
|
rowId: number;
|
|
1407
2223
|
key: string | null;
|
|
@@ -1615,12 +2431,17 @@ export class PlayContextImpl {
|
|
|
1615
2431
|
|
|
1616
2432
|
private effectiveToolCallCachePolicy(options?: ToolCallOptions): {
|
|
1617
2433
|
force: boolean;
|
|
2434
|
+
forceFailedRefresh: boolean;
|
|
1618
2435
|
staleAfterSeconds?: number | null;
|
|
1619
2436
|
} {
|
|
1620
2437
|
return {
|
|
1621
2438
|
force:
|
|
1622
2439
|
options?.force === true ||
|
|
1623
2440
|
this.#options.cachePolicy?.forceToolRefresh === true,
|
|
2441
|
+
forceFailedRefresh:
|
|
2442
|
+
options?.force === true ||
|
|
2443
|
+
this.#options.cachePolicy?.forceToolRefresh === true ||
|
|
2444
|
+
this.#options.cachePolicy?.forceFailedToolRefresh === true,
|
|
1624
2445
|
staleAfterSeconds: options?.staleAfterSeconds ?? null,
|
|
1625
2446
|
};
|
|
1626
2447
|
}
|
|
@@ -1641,6 +2462,19 @@ export class PlayContextImpl {
|
|
|
1641
2462
|
return this.checkpoint.completedToolBatches[toolId]?.[rowCacheKey];
|
|
1642
2463
|
}
|
|
1643
2464
|
|
|
2465
|
+
private getCachedToolResultCandidate(
|
|
2466
|
+
toolId: string,
|
|
2467
|
+
rowCacheKeys: readonly string[],
|
|
2468
|
+
): { cacheKey: string; result: ToolBatchResult } | null {
|
|
2469
|
+
for (const rowCacheKey of rowCacheKeys) {
|
|
2470
|
+
const cached = this.getCachedToolResult(toolId, rowCacheKey);
|
|
2471
|
+
if (cached?.done) {
|
|
2472
|
+
return { cacheKey: rowCacheKey, result: cached };
|
|
2473
|
+
}
|
|
2474
|
+
}
|
|
2475
|
+
return null;
|
|
2476
|
+
}
|
|
2477
|
+
|
|
1644
2478
|
private cacheToolResult(
|
|
1645
2479
|
toolId: string,
|
|
1646
2480
|
rowCacheKey: string,
|
|
@@ -1695,24 +2529,146 @@ export class PlayContextImpl {
|
|
|
1695
2529
|
metadata?: ToolResultMetadataInput | null;
|
|
1696
2530
|
meta?: Record<string, unknown>;
|
|
1697
2531
|
execution: ToolResultExecutionMetadata;
|
|
2532
|
+
requestInput?: Record<string, unknown>;
|
|
1698
2533
|
}): Promise<ToolExecuteResult> {
|
|
1699
2534
|
if (isToolExecuteResult(input.result)) {
|
|
1700
|
-
return
|
|
2535
|
+
return this.attachCustomerDbDatasetResult(
|
|
2536
|
+
input.toolId,
|
|
2537
|
+
input.requestInput,
|
|
2538
|
+
cloneToolExecuteResultWithExecution(input.result, input.execution),
|
|
2539
|
+
);
|
|
1701
2540
|
}
|
|
1702
2541
|
const publicToolResult = publicToolResponseEnvelope(input.result);
|
|
1703
|
-
return
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
2542
|
+
return this.attachCustomerDbDatasetResult(
|
|
2543
|
+
input.toolId,
|
|
2544
|
+
input.requestInput,
|
|
2545
|
+
createToolExecuteResult({
|
|
2546
|
+
status: publicToolResult?.status ?? input.status,
|
|
2547
|
+
jobId: input.jobId,
|
|
2548
|
+
result: publicToolResult
|
|
2549
|
+
? {
|
|
2550
|
+
data: publicToolResult.raw,
|
|
2551
|
+
...(publicToolResult.meta ? { meta: publicToolResult.meta } : {}),
|
|
2552
|
+
}
|
|
2553
|
+
: input.result,
|
|
2554
|
+
metadata:
|
|
2555
|
+
input.metadata ??
|
|
2556
|
+
(await this.resolveToolResultMetadata(input.toolId)),
|
|
2557
|
+
execution: input.execution,
|
|
2558
|
+
meta: input.meta,
|
|
2559
|
+
}),
|
|
2560
|
+
);
|
|
2561
|
+
}
|
|
2562
|
+
|
|
2563
|
+
private attachCustomerDbDatasetResult(
|
|
2564
|
+
toolId: string,
|
|
2565
|
+
requestInput: Record<string, unknown> | undefined,
|
|
2566
|
+
wrapped: ToolExecuteResult,
|
|
2567
|
+
): ToolExecuteResult {
|
|
2568
|
+
if (!isQueryResultDatasetTool(toolId)) return wrapped;
|
|
2569
|
+
const raw = recordOrNull(wrapped.toolResponse.raw);
|
|
2570
|
+
const dataset = recordOrNull(raw?.dataset);
|
|
2571
|
+
const rows = rowsFromUnknown(raw?.rows);
|
|
2572
|
+
const totalRows = finiteNonNegativeInteger(dataset?.total_rows);
|
|
2573
|
+
const sql =
|
|
2574
|
+
typeof requestInput?.sql === 'string'
|
|
2575
|
+
? requestInput.sql
|
|
2576
|
+
: typeof requestInput?.query === 'string'
|
|
2577
|
+
? requestInput.query
|
|
2578
|
+
: typeof raw?.sql === 'string'
|
|
2579
|
+
? raw.sql
|
|
2580
|
+
: null;
|
|
2581
|
+
if (!dataset || totalRows === null || !sql) {
|
|
2582
|
+
return wrapped;
|
|
2583
|
+
}
|
|
2584
|
+
const originalRequestInput = requestInput as Record<string, unknown>;
|
|
2585
|
+
const datasetLimit =
|
|
2586
|
+
finitePositiveInteger(dataset.returned_limit) ?? totalRows;
|
|
2587
|
+
const previewRows = rows.slice(0, Math.min(rows.length, 25));
|
|
2588
|
+
const executionNonce =
|
|
2589
|
+
typeof wrapped.job_id === 'string' && wrapped.job_id.trim()
|
|
2590
|
+
? wrapped.job_id.trim()
|
|
2591
|
+
: `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
2592
|
+
const datasetId = `tool-list:${sha256Hex(
|
|
2593
|
+
`${toolId}:${this.currentRunId}:${sql}:${datasetLimit}:${executionNonce}`,
|
|
2594
|
+
)}`;
|
|
2595
|
+
const fetchPage = async (
|
|
2596
|
+
offset: number,
|
|
2597
|
+
limit: number,
|
|
2598
|
+
): Promise<Record<string, unknown>[]> => {
|
|
2599
|
+
if (limit <= 0 || offset >= totalRows) return [];
|
|
2600
|
+
const execution = await this.callToolExecutionAPI(
|
|
2601
|
+
toolId,
|
|
2602
|
+
originalRequestInput,
|
|
2603
|
+
{
|
|
2604
|
+
timeoutMs: resolveToolRuntimeTimeoutMs(toolId),
|
|
2605
|
+
customerDbDataset: {
|
|
2606
|
+
limit: datasetLimit,
|
|
2607
|
+
offset,
|
|
2608
|
+
pageSize: Math.min(limit, QUERY_RESULT_DATASET_PAGE_SIZE),
|
|
2609
|
+
totalRows,
|
|
2610
|
+
},
|
|
2611
|
+
},
|
|
2612
|
+
);
|
|
2613
|
+
const pageRaw =
|
|
2614
|
+
recordOrNull(execution.toolResponse)?.raw ??
|
|
2615
|
+
recordOrNull(execution.result)?.data ??
|
|
2616
|
+
execution.result;
|
|
2617
|
+
return rowsFromUnknown(recordOrNull(pageRaw)?.rows);
|
|
2618
|
+
};
|
|
2619
|
+
const collectRows = async (
|
|
2620
|
+
limit: number | undefined,
|
|
2621
|
+
): Promise<Record<string, unknown>[]> => {
|
|
2622
|
+
const target = Math.min(limit ?? totalRows, totalRows, datasetLimit);
|
|
2623
|
+
const collected: Record<string, unknown>[] = [];
|
|
2624
|
+
for (
|
|
2625
|
+
let offset = 0;
|
|
2626
|
+
offset < target;
|
|
2627
|
+
offset += QUERY_RESULT_DATASET_PAGE_SIZE
|
|
2628
|
+
) {
|
|
2629
|
+
collected.push(
|
|
2630
|
+
...(await fetchPage(
|
|
2631
|
+
offset,
|
|
2632
|
+
Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, target - offset),
|
|
2633
|
+
)),
|
|
2634
|
+
);
|
|
2635
|
+
}
|
|
2636
|
+
return collected.slice(0, target);
|
|
2637
|
+
};
|
|
2638
|
+
const playDataset = createDeferredPlayDataset({
|
|
2639
|
+
datasetKind: 'csv',
|
|
2640
|
+
datasetId,
|
|
2641
|
+
count: Math.min(totalRows, datasetLimit),
|
|
2642
|
+
previewRows,
|
|
2643
|
+
sourceLabel: 'query result rows',
|
|
2644
|
+
tableNamespace: null,
|
|
2645
|
+
resolvers: {
|
|
2646
|
+
count: async () => Math.min(totalRows, datasetLimit),
|
|
2647
|
+
peek: async (limit) => collectRows(limit),
|
|
2648
|
+
materialize: async (limit) => collectRows(limit),
|
|
2649
|
+
iterate: () =>
|
|
2650
|
+
({
|
|
2651
|
+
async *[Symbol.asyncIterator]() {
|
|
2652
|
+
const count = Math.min(totalRows, datasetLimit);
|
|
2653
|
+
for (
|
|
2654
|
+
let offset = 0;
|
|
2655
|
+
offset < count;
|
|
2656
|
+
offset += QUERY_RESULT_DATASET_PAGE_SIZE
|
|
2657
|
+
) {
|
|
2658
|
+
yield* await fetchPage(
|
|
2659
|
+
offset,
|
|
2660
|
+
Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, count - offset),
|
|
2661
|
+
);
|
|
2662
|
+
}
|
|
2663
|
+
},
|
|
2664
|
+
}) as AsyncIterable<Record<string, unknown>>,
|
|
2665
|
+
},
|
|
2666
|
+
});
|
|
2667
|
+
return attachToolResultListDataset(wrapped, {
|
|
2668
|
+
name: 'rows',
|
|
2669
|
+
path: 'toolResponse.raw.rows',
|
|
2670
|
+
dataset: playDataset,
|
|
2671
|
+
count: Math.min(totalRows, datasetLimit),
|
|
1716
2672
|
});
|
|
1717
2673
|
}
|
|
1718
2674
|
|
|
@@ -1738,6 +2694,7 @@ export class PlayContextImpl {
|
|
|
1738
2694
|
cacheKey,
|
|
1739
2695
|
receiptKey,
|
|
1740
2696
|
}),
|
|
2697
|
+
requestInput: request.input,
|
|
1741
2698
|
});
|
|
1742
2699
|
let completed: RuntimeStepReceipt | null | undefined = null;
|
|
1743
2700
|
if (receiptKey) {
|
|
@@ -1747,6 +2704,7 @@ export class PlayContextImpl {
|
|
|
1747
2704
|
{
|
|
1748
2705
|
key: receiptKey,
|
|
1749
2706
|
runId: this.currentRunId,
|
|
2707
|
+
leaseId: request.receiptLeaseId ?? null,
|
|
1750
2708
|
output: serializeToolExecuteResult(wrapped),
|
|
1751
2709
|
},
|
|
1752
2710
|
])
|
|
@@ -1791,6 +2749,7 @@ export class PlayContextImpl {
|
|
|
1791
2749
|
cacheKey: entry.request.cacheKey,
|
|
1792
2750
|
receiptKey: entry.request.receiptKey,
|
|
1793
2751
|
}),
|
|
2752
|
+
requestInput: entry.request.input,
|
|
1794
2753
|
}),
|
|
1795
2754
|
})),
|
|
1796
2755
|
);
|
|
@@ -1801,12 +2760,13 @@ export class PlayContextImpl {
|
|
|
1801
2760
|
{
|
|
1802
2761
|
key: receiptKey,
|
|
1803
2762
|
runId: this.currentRunId,
|
|
2763
|
+
leaseId: entry.request.receiptLeaseId ?? null,
|
|
1804
2764
|
output: serializeToolExecuteResult(entry.wrapped),
|
|
1805
2765
|
},
|
|
1806
2766
|
]
|
|
1807
2767
|
: [];
|
|
1808
2768
|
});
|
|
1809
|
-
let completedByKey
|
|
2769
|
+
let completedByKey = new Map<string, RuntimeStepReceipt>();
|
|
1810
2770
|
if (receiptInputs.length > 0) {
|
|
1811
2771
|
try {
|
|
1812
2772
|
completedByKey = await this.completeRuntimeStepReceipts(receiptInputs);
|
|
@@ -1814,10 +2774,15 @@ export class PlayContextImpl {
|
|
|
1814
2774
|
// Completion failed AFTER successful (billed) tool calls: trip the
|
|
1815
2775
|
// breaker so remaining rows stop dispatching new provider calls.
|
|
1816
2776
|
tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
|
|
2777
|
+
await Promise.all(
|
|
2778
|
+
wrappedEntries.map((entry) =>
|
|
2779
|
+
this.rejectToolCall(toolId, entry.request, receiptError, {
|
|
2780
|
+
persistReceiptFailure: false,
|
|
2781
|
+
}),
|
|
2782
|
+
),
|
|
2783
|
+
);
|
|
1817
2784
|
throw receiptError;
|
|
1818
2785
|
}
|
|
1819
|
-
} else {
|
|
1820
|
-
completedByKey = new Map<string, RuntimeStepReceipt>();
|
|
1821
2786
|
}
|
|
1822
2787
|
const results: unknown[] = [];
|
|
1823
2788
|
for (const entry of wrappedEntries) {
|
|
@@ -1841,9 +2806,23 @@ export class PlayContextImpl {
|
|
|
1841
2806
|
completed: RuntimeStepReceipt | null | undefined,
|
|
1842
2807
|
): Promise<unknown> {
|
|
1843
2808
|
const cacheKey = request.cacheKey;
|
|
2809
|
+
const receiptKey = request.receiptKey?.trim() || null;
|
|
2810
|
+
const canPersistReceiptCompletion = Boolean(
|
|
2811
|
+
this.#options.completeRuntimeStepReceipt ||
|
|
2812
|
+
this.#options.completeRuntimeStepReceipts,
|
|
2813
|
+
);
|
|
2814
|
+
if (
|
|
2815
|
+
receiptKey &&
|
|
2816
|
+
canPersistReceiptCompletion &&
|
|
2817
|
+
completed?.status !== 'completed' &&
|
|
2818
|
+
completed?.status !== 'skipped'
|
|
2819
|
+
) {
|
|
2820
|
+
throw new Error(
|
|
2821
|
+
`Durable tool call ${receiptKey} lost receipt ownership before completion.`,
|
|
2822
|
+
);
|
|
2823
|
+
}
|
|
1844
2824
|
const finalWrapped =
|
|
1845
|
-
|
|
1846
|
-
completed.runId !== this.currentRunId
|
|
2825
|
+
completed?.status === 'completed' || completed?.status === 'skipped'
|
|
1847
2826
|
? await this.wrapToolExecutionResult({
|
|
1848
2827
|
toolId,
|
|
1849
2828
|
status:
|
|
@@ -1851,6 +2830,7 @@ export class PlayContextImpl {
|
|
|
1851
2830
|
? 'no_result'
|
|
1852
2831
|
: 'completed',
|
|
1853
2832
|
result: this.runtimeReceiptOutput(completed),
|
|
2833
|
+
requestInput: request.input,
|
|
1854
2834
|
execution: toolExecutionMetadataForOutcome(
|
|
1855
2835
|
completed.runId === this.currentRunId
|
|
1856
2836
|
? {
|
|
@@ -1861,13 +2841,15 @@ export class PlayContextImpl {
|
|
|
1861
2841
|
: {
|
|
1862
2842
|
kind: 'cache',
|
|
1863
2843
|
cacheKey,
|
|
1864
|
-
receiptKey:
|
|
1865
|
-
attachedToReceiptKey:
|
|
2844
|
+
receiptKey: receiptKey ?? '',
|
|
2845
|
+
attachedToReceiptKey: receiptKey,
|
|
1866
2846
|
},
|
|
1867
2847
|
),
|
|
1868
2848
|
})
|
|
1869
2849
|
: wrapped;
|
|
1870
|
-
|
|
2850
|
+
if (request.cacheable !== false) {
|
|
2851
|
+
this.cacheToolResult(toolId, cacheKey, finalWrapped);
|
|
2852
|
+
}
|
|
1871
2853
|
|
|
1872
2854
|
const resolver = this.toolCallResolvers.get(request.callId);
|
|
1873
2855
|
if (resolver) {
|
|
@@ -1905,7 +2887,9 @@ export class PlayContextImpl {
|
|
|
1905
2887
|
{
|
|
1906
2888
|
key: receiptKey,
|
|
1907
2889
|
runId: this.currentRunId,
|
|
2890
|
+
leaseId: request.receiptLeaseId ?? null,
|
|
1908
2891
|
error: message,
|
|
2892
|
+
failureKind: runtimeReceiptFailureKindForError(error),
|
|
1909
2893
|
},
|
|
1910
2894
|
]);
|
|
1911
2895
|
} catch (receiptError) {
|
|
@@ -2189,6 +3173,7 @@ export class PlayContextImpl {
|
|
|
2189
3173
|
this.toOutputRow(item as Record<string, unknown>),
|
|
2190
3174
|
);
|
|
2191
3175
|
itemOriginalIndexes = materializedItems.map((_item, index) => index);
|
|
3176
|
+
const mapStartSeedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
2192
3177
|
if (this.#options.onMapStart) {
|
|
2193
3178
|
const shouldPassRowKey = explicitKeyResolver != null;
|
|
2194
3179
|
// toSerializableCsvAliasedRow (not a plain spread): projected CSV
|
|
@@ -2210,21 +3195,22 @@ export class PlayContextImpl {
|
|
|
2210
3195
|
playName: this.#options.playName,
|
|
2211
3196
|
playId: this.#options.playId,
|
|
2212
3197
|
runId: this.#options.runId,
|
|
3198
|
+
executorToken: this.#options.executorToken,
|
|
2213
3199
|
staticPipeline: this.#options.staticPipeline,
|
|
2214
|
-
forceRefresh:
|
|
2215
|
-
this.#options.cachePolicy?.forceToolRefresh === true,
|
|
3200
|
+
forceRefresh: this.#options.cachePolicy?.forceToolRefresh === true,
|
|
2216
3201
|
},
|
|
2217
3202
|
);
|
|
2218
3203
|
resolvedTableNamespace = normalizeTableNamespace(
|
|
2219
3204
|
mapStartResult.tableNamespace,
|
|
2220
3205
|
);
|
|
3206
|
+
if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
|
|
3207
|
+
throw new RuntimeSheetRowsBlockedError({
|
|
3208
|
+
tableNamespace: resolvedTableNamespace,
|
|
3209
|
+
blockedRows: mapStartResult.blockedRows ?? [],
|
|
3210
|
+
});
|
|
3211
|
+
}
|
|
2221
3212
|
const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
|
|
2222
3213
|
resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
|
|
2223
|
-
const pendingKeys = new Set(
|
|
2224
|
-
mapStartResult.pendingRows.map((row, index) =>
|
|
2225
|
-
persistedRowIdentity(row, index),
|
|
2226
|
-
),
|
|
2227
|
-
);
|
|
2228
3214
|
const pendingRowsByKey = new Map<string, Record<string, unknown>>();
|
|
2229
3215
|
for (
|
|
2230
3216
|
let index = 0;
|
|
@@ -2235,15 +3221,20 @@ export class PlayContextImpl {
|
|
|
2235
3221
|
const rowKey = persistedRowIdentity(row, index);
|
|
2236
3222
|
if (rowKey) pendingRowsByKey.set(rowKey, row);
|
|
2237
3223
|
}
|
|
2238
|
-
const
|
|
3224
|
+
for (const row of mapStartResult.completedRows ?? []) {
|
|
3225
|
+
const rowKey = persistedRowIdentity(row);
|
|
3226
|
+
if (!rowKey) continue;
|
|
3227
|
+
mapStartSeedRowsByKey.set(rowKey, row);
|
|
3228
|
+
}
|
|
3229
|
+
const seededItems = materializedItems
|
|
2239
3230
|
.map((item, index) => ({
|
|
2240
3231
|
row: this.toOutputRow(item as Record<string, unknown>),
|
|
2241
3232
|
index,
|
|
2242
3233
|
}))
|
|
2243
|
-
.filter(({ row, index }) => pendingKeys.has(rowIdentity(row, index)))
|
|
2244
3234
|
.map(({ row, index }) => {
|
|
2245
3235
|
const rowKey = rowIdentity(row, index);
|
|
2246
|
-
const persisted =
|
|
3236
|
+
const persisted =
|
|
3237
|
+
pendingRowsByKey.get(rowKey) ?? mapStartSeedRowsByKey.get(rowKey);
|
|
2247
3238
|
// Persisted rows lose the non-enumerable CSV alias projections on
|
|
2248
3239
|
// the sheet round-trip. Merge the persisted data/meta over the
|
|
2249
3240
|
// in-memory row so key functions and column resolvers keep seeing
|
|
@@ -2253,8 +3244,8 @@ export class PlayContextImpl {
|
|
|
2253
3244
|
index,
|
|
2254
3245
|
};
|
|
2255
3246
|
});
|
|
2256
|
-
itemsToProcess =
|
|
2257
|
-
itemOriginalIndexes =
|
|
3247
|
+
itemsToProcess = seededItems.map((item) => item.row);
|
|
3248
|
+
itemOriginalIndexes = seededItems.map((item) => item.index);
|
|
2258
3249
|
}
|
|
2259
3250
|
|
|
2260
3251
|
const mapScope = this.createMapExecutionScope({
|
|
@@ -2325,6 +3316,7 @@ export class PlayContextImpl {
|
|
|
2325
3316
|
playName: this.#options.playName,
|
|
2326
3317
|
playId: this.#options.playId,
|
|
2327
3318
|
runId: this.#options.runId,
|
|
3319
|
+
executorToken: this.#options.executorToken,
|
|
2328
3320
|
tableNamespace: resolvedTableNamespace,
|
|
2329
3321
|
rows: chunk,
|
|
2330
3322
|
outputFields: datasetColumnNames.filter((field) =>
|
|
@@ -2410,7 +3402,7 @@ export class PlayContextImpl {
|
|
|
2410
3402
|
const directCompletedResults = mapResult.completedRows.map((row) =>
|
|
2411
3403
|
this.toPublicOutputRow(row.data),
|
|
2412
3404
|
);
|
|
2413
|
-
|
|
3405
|
+
let results =
|
|
2414
3406
|
mapResult.failedRows.length === 0 &&
|
|
2415
3407
|
directCompletedResults.length === rawItems.length
|
|
2416
3408
|
? directCompletedResults
|
|
@@ -2463,6 +3455,49 @@ export class PlayContextImpl {
|
|
|
2463
3455
|
}
|
|
2464
3456
|
this.activeMapCellMeta = null;
|
|
2465
3457
|
|
|
3458
|
+
if (this.#options.onMapRowsCompleted) {
|
|
3459
|
+
results = await reconcileNodeRuntimeMapResultsWithPersistedSheet({
|
|
3460
|
+
mapName: normalizedMapNamespace,
|
|
3461
|
+
tableNamespace: resolvedTableNamespace,
|
|
3462
|
+
runId: this.#options.runId,
|
|
3463
|
+
expectedRows: totalInputCount,
|
|
3464
|
+
currentRows: results,
|
|
3465
|
+
failedRowCount: mapResult.failedRows.length,
|
|
3466
|
+
log: (line) => this.log(line),
|
|
3467
|
+
readPersistedRows: async (readInput) => {
|
|
3468
|
+
if (
|
|
3469
|
+
!this.#options.baseUrl ||
|
|
3470
|
+
!this.#options.executorToken ||
|
|
3471
|
+
!this.#options.playName ||
|
|
3472
|
+
!this.#options.runId
|
|
3473
|
+
) {
|
|
3474
|
+
throw new Error(
|
|
3475
|
+
`Runtime sheet finalization mismatch for ctx.dataset("${normalizedMapNamespace}"): ` +
|
|
3476
|
+
'cannot verify terminal persisted rows because the Node runtime context is missing ' +
|
|
3477
|
+
'baseUrl, executorToken, playName, or runId.',
|
|
3478
|
+
);
|
|
3479
|
+
}
|
|
3480
|
+
const result = await readRuntimeSheetDatasetRows(
|
|
3481
|
+
{
|
|
3482
|
+
baseUrl: this.#options.baseUrl,
|
|
3483
|
+
executorToken: this.#options.executorToken,
|
|
3484
|
+
playName: this.#options.playName,
|
|
3485
|
+
userEmail: this.#options.userEmail,
|
|
3486
|
+
runId: this.#options.runId,
|
|
3487
|
+
},
|
|
3488
|
+
{
|
|
3489
|
+
tableNamespace: resolvedTableNamespace,
|
|
3490
|
+
runId: this.#options.runId,
|
|
3491
|
+
rowMode: readInput.rowMode,
|
|
3492
|
+
limit: readInput.limit,
|
|
3493
|
+
offset: readInput.offset,
|
|
3494
|
+
},
|
|
3495
|
+
);
|
|
3496
|
+
return result.rows;
|
|
3497
|
+
},
|
|
3498
|
+
});
|
|
3499
|
+
}
|
|
3500
|
+
|
|
2466
3501
|
return createDeferredPlayDataset({
|
|
2467
3502
|
datasetKind: 'map',
|
|
2468
3503
|
datasetId: createRuntimeDatasetId(
|
|
@@ -2739,7 +3774,7 @@ export class PlayContextImpl {
|
|
|
2739
3774
|
this.lastDatasetStep = datasetStep;
|
|
2740
3775
|
this.activeDatasetStep = null;
|
|
2741
3776
|
this.log(
|
|
2742
|
-
`Map completed: ${results.length + completedRows} results (${results.length}
|
|
3777
|
+
`Map completed: ${results.length + completedRows} results (${results.length} succeeded, 0 failed, ${completedRows} duplicate keys skipped)`,
|
|
2743
3778
|
);
|
|
2744
3779
|
return {
|
|
2745
3780
|
completedRows: results.map((row, index) =>
|
|
@@ -3013,6 +4048,17 @@ export class PlayContextImpl {
|
|
|
3013
4048
|
}
|
|
3014
4049
|
return result.value;
|
|
3015
4050
|
});
|
|
4051
|
+
const accountedRowIndexes = new Set<number>();
|
|
4052
|
+
for (const row of completedRowsToPersist) {
|
|
4053
|
+
if (typeof row.inputIndex === 'number') {
|
|
4054
|
+
accountedRowIndexes.add(row.inputIndex);
|
|
4055
|
+
}
|
|
4056
|
+
}
|
|
4057
|
+
for (const row of failedRowsToPersist) {
|
|
4058
|
+
if (typeof row.inputIndex === 'number') {
|
|
4059
|
+
accountedRowIndexes.add(row.inputIndex);
|
|
4060
|
+
}
|
|
4061
|
+
}
|
|
3016
4062
|
if (this.pendingRowEventBoundaries.length > 0) {
|
|
3017
4063
|
const completedRowKeys = new Set<string>();
|
|
3018
4064
|
for (let index = 0; index < settledResults.length; index += 1) {
|
|
@@ -3025,6 +4071,7 @@ export class PlayContextImpl {
|
|
|
3025
4071
|
if (key) {
|
|
3026
4072
|
completedRowKeys.add(key);
|
|
3027
4073
|
}
|
|
4074
|
+
accountedRowIndexes.add(executionRowIndex(index));
|
|
3028
4075
|
continue;
|
|
3029
4076
|
}
|
|
3030
4077
|
const row = result as Record<string, unknown>;
|
|
@@ -3068,6 +4115,19 @@ export class PlayContextImpl {
|
|
|
3068
4115
|
boundaries: uniqueBoundaries,
|
|
3069
4116
|
});
|
|
3070
4117
|
}
|
|
4118
|
+
if (accountedRowIndexes.size !== items.length) {
|
|
4119
|
+
const expectedRowIndexes = items.map((_item, index) =>
|
|
4120
|
+
executionRowIndex(index),
|
|
4121
|
+
);
|
|
4122
|
+
const missingRowIndexes = expectedRowIndexes.filter(
|
|
4123
|
+
(index) => !accountedRowIndexes.has(index),
|
|
4124
|
+
);
|
|
4125
|
+
throw new Error(
|
|
4126
|
+
`Runtime map execution accounting mismatch for ctx.dataset("${normalizedTableNamespace}"): ` +
|
|
4127
|
+
`expected ${items.length} row(s), accounted for ${accountedRowIndexes.size}; ` +
|
|
4128
|
+
`missing input index(es): ${missingRowIndexes.slice(0, 10).join(', ') || 'unknown'}.`,
|
|
4129
|
+
);
|
|
4130
|
+
}
|
|
3071
4131
|
const results = settledResults.filter(
|
|
3072
4132
|
(result): result is Record<string, unknown> =>
|
|
3073
4133
|
result !== WAITING_ROW && result !== FAILED_ROW,
|
|
@@ -3079,8 +4139,18 @@ export class PlayContextImpl {
|
|
|
3079
4139
|
emitEventType: 'map.completed',
|
|
3080
4140
|
});
|
|
3081
4141
|
}
|
|
4142
|
+
for (const failedRow of failedRowsToPersist.slice(0, 3)) {
|
|
4143
|
+
this.log(
|
|
4144
|
+
`row ${failedMapRowLogLabel(failedRow)} failed: ${failedRow.error ?? 'unknown error'}`,
|
|
4145
|
+
);
|
|
4146
|
+
}
|
|
4147
|
+
if (failedRowsToPersist.length > 3) {
|
|
4148
|
+
this.log(
|
|
4149
|
+
`${failedRowsToPersist.length - 3} additional row failure(s) omitted from map log`,
|
|
4150
|
+
);
|
|
4151
|
+
}
|
|
3082
4152
|
this.log(
|
|
3083
|
-
`Map completed: ${results.length + completedRows} results (${results.length}
|
|
4153
|
+
`Map completed: ${results.length + completedRows} results (${results.length} succeeded, ${failedRowsToPersist.length} failed, ${completedRows} duplicate keys skipped)`,
|
|
3084
4154
|
);
|
|
3085
4155
|
return {
|
|
3086
4156
|
completedRows: [...completedRowsToPersist].sort(
|
|
@@ -3390,6 +4460,13 @@ export class PlayContextImpl {
|
|
|
3390
4460
|
if (!hasPendingWork) {
|
|
3391
4461
|
const status = await raceSettled();
|
|
3392
4462
|
if (status === 'done' && this.toolCallQueue.length === 0) {
|
|
4463
|
+
if (!this.activeDatasetStep) {
|
|
4464
|
+
const childPlaySuspension =
|
|
4465
|
+
this.consumePendingChildPlaySuspension();
|
|
4466
|
+
if (childPlaySuspension) {
|
|
4467
|
+
throw new PlayExecutionSuspendedError(childPlaySuspension);
|
|
4468
|
+
}
|
|
4469
|
+
}
|
|
3393
4470
|
break;
|
|
3394
4471
|
}
|
|
3395
4472
|
|
|
@@ -3486,16 +4563,23 @@ export class PlayContextImpl {
|
|
|
3486
4563
|
toolId,
|
|
3487
4564
|
requestInput: input,
|
|
3488
4565
|
});
|
|
3489
|
-
|
|
4566
|
+
let executionAuthScopeDigest =
|
|
4567
|
+
(await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
|
|
4568
|
+
const cacheableToolResult = !isQueryResultDatasetReadRequest(toolId, input);
|
|
4569
|
+
let durableCacheKey = await this.durableToolCallCacheKey({
|
|
3490
4570
|
toolId,
|
|
3491
4571
|
requestInput: input,
|
|
4572
|
+
executionAuthScopeDigest,
|
|
3492
4573
|
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
3493
4574
|
});
|
|
4575
|
+
const checkpointCacheKeys = [durableCacheKey];
|
|
3494
4576
|
const eventWaitHandler =
|
|
3495
4577
|
(await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
|
|
3496
4578
|
const store = rowContext.getStore();
|
|
3497
4579
|
|
|
3498
|
-
const executeTool = async (
|
|
4580
|
+
const executeTool = async (context?: {
|
|
4581
|
+
leaseId?: string | null;
|
|
4582
|
+
}): Promise<unknown> => {
|
|
3499
4583
|
if (eventWaitHandler) {
|
|
3500
4584
|
return this.executeIntegrationEventWaitTool(
|
|
3501
4585
|
toolId,
|
|
@@ -3518,18 +4602,21 @@ export class PlayContextImpl {
|
|
|
3518
4602
|
|
|
3519
4603
|
if (!store) {
|
|
3520
4604
|
const directCacheKey = durableCacheKey;
|
|
3521
|
-
const cached =
|
|
4605
|
+
const cached = !cacheableToolResult
|
|
3522
4606
|
? null
|
|
3523
|
-
:
|
|
3524
|
-
|
|
3525
|
-
|
|
4607
|
+
: toolCachePolicy.force
|
|
4608
|
+
? null
|
|
4609
|
+
: this.getCachedToolResultCandidate(toolId, checkpointCacheKeys);
|
|
4610
|
+
if (cached) {
|
|
4611
|
+
this.log(`Calling tool: ${toolId} recovered from checkpoint`);
|
|
3526
4612
|
return await this.wrapToolExecutionResult({
|
|
3527
4613
|
toolId,
|
|
3528
|
-
status: cached.result == null ? 'no_result' : 'completed',
|
|
3529
|
-
result: cached.result,
|
|
4614
|
+
status: cached.result.result == null ? 'no_result' : 'completed',
|
|
4615
|
+
result: cached.result.result,
|
|
4616
|
+
requestInput: input,
|
|
3530
4617
|
execution: toolExecutionMetadataForOutcome({
|
|
3531
4618
|
kind: 'checkpoint',
|
|
3532
|
-
cacheKey:
|
|
4619
|
+
cacheKey: cached.cacheKey,
|
|
3533
4620
|
}),
|
|
3534
4621
|
});
|
|
3535
4622
|
}
|
|
@@ -3539,6 +4626,17 @@ export class PlayContextImpl {
|
|
|
3539
4626
|
: `Calling tool: ${toolId}`,
|
|
3540
4627
|
);
|
|
3541
4628
|
const execution = await this.callToolExecutionAPI(toolId, input, {
|
|
4629
|
+
...(cacheableToolResult
|
|
4630
|
+
? {
|
|
4631
|
+
durableCallReceiptKey: directCacheKey,
|
|
4632
|
+
executionAuthScopeDigest,
|
|
4633
|
+
providerIdempotencyKey: this.providerIdempotencyKeyForToolCall({
|
|
4634
|
+
cacheKey: directCacheKey,
|
|
4635
|
+
force: toolCachePolicy.force,
|
|
4636
|
+
leaseId: context?.leaseId ?? null,
|
|
4637
|
+
}),
|
|
4638
|
+
}
|
|
4639
|
+
: {}),
|
|
3542
4640
|
timeoutMs: resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs),
|
|
3543
4641
|
});
|
|
3544
4642
|
const wrapped = await this.wrapToolExecutionResult({
|
|
@@ -3548,19 +4646,22 @@ export class PlayContextImpl {
|
|
|
3548
4646
|
result: execution.result,
|
|
3549
4647
|
metadata: execution.metadata,
|
|
3550
4648
|
meta: execution.meta,
|
|
4649
|
+
requestInput: input,
|
|
3551
4650
|
execution: toolExecutionMetadataForOutcome({
|
|
3552
4651
|
kind: 'live',
|
|
3553
4652
|
cacheKey: directCacheKey,
|
|
3554
4653
|
}),
|
|
3555
4654
|
});
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
4655
|
+
if (cacheableToolResult) {
|
|
4656
|
+
this.checkpoint.completedToolBatches[toolId] = {
|
|
4657
|
+
...(this.checkpoint.completedToolBatches[toolId] ?? {}),
|
|
4658
|
+
[directCacheKey]: {
|
|
4659
|
+
done: true,
|
|
4660
|
+
result: wrapped,
|
|
4661
|
+
},
|
|
4662
|
+
};
|
|
4663
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
4664
|
+
}
|
|
3564
4665
|
return wrapped;
|
|
3565
4666
|
}
|
|
3566
4667
|
|
|
@@ -3581,18 +4682,21 @@ export class PlayContextImpl {
|
|
|
3581
4682
|
}
|
|
3582
4683
|
|
|
3583
4684
|
const toolResultCacheKey = durableCacheKey;
|
|
3584
|
-
const cached =
|
|
4685
|
+
const cached = !cacheableToolResult
|
|
3585
4686
|
? null
|
|
3586
|
-
:
|
|
3587
|
-
|
|
3588
|
-
|
|
4687
|
+
: toolCachePolicy.force
|
|
4688
|
+
? null
|
|
4689
|
+
: this.getCachedToolResultCandidate(toolId, checkpointCacheKeys);
|
|
4690
|
+
if (cached) {
|
|
4691
|
+
this.log(` Row ${rowId} ${toolId}: recovered from checkpoint`);
|
|
3589
4692
|
return await this.wrapToolExecutionResult({
|
|
3590
4693
|
toolId,
|
|
3591
|
-
status: cached.result == null ? 'no_result' : 'completed',
|
|
3592
|
-
result: cached.result,
|
|
4694
|
+
status: cached.result.result == null ? 'no_result' : 'completed',
|
|
4695
|
+
result: cached.result.result,
|
|
4696
|
+
requestInput: input,
|
|
3593
4697
|
execution: toolExecutionMetadataForOutcome({
|
|
3594
4698
|
kind: 'checkpoint',
|
|
3595
|
-
cacheKey:
|
|
4699
|
+
cacheKey: cached.cacheKey,
|
|
3596
4700
|
}),
|
|
3597
4701
|
});
|
|
3598
4702
|
}
|
|
@@ -3624,8 +4728,11 @@ export class PlayContextImpl {
|
|
|
3624
4728
|
this.toolCallQueue.push({
|
|
3625
4729
|
callId,
|
|
3626
4730
|
cacheKey: toolResultCacheKey,
|
|
3627
|
-
|
|
4731
|
+
cacheable: cacheableToolResult,
|
|
4732
|
+
receiptKey: cacheableToolResult ? durableCacheKey : null,
|
|
4733
|
+
executionAuthScopeDigest,
|
|
3628
4734
|
force: toolCachePolicy.force,
|
|
4735
|
+
forceFailedRefresh: toolCachePolicy.forceFailedRefresh,
|
|
3629
4736
|
rowId,
|
|
3630
4737
|
fieldName,
|
|
3631
4738
|
toolId,
|
|
@@ -3641,45 +4748,85 @@ export class PlayContextImpl {
|
|
|
3641
4748
|
});
|
|
3642
4749
|
};
|
|
3643
4750
|
|
|
3644
|
-
if (store) {
|
|
4751
|
+
if (store || !cacheableToolResult) {
|
|
3645
4752
|
return await executeTool();
|
|
3646
4753
|
}
|
|
3647
4754
|
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
|
|
3679
|
-
|
|
3680
|
-
|
|
3681
|
-
|
|
3682
|
-
|
|
4755
|
+
for (
|
|
4756
|
+
let authScopeAttempt = 0;
|
|
4757
|
+
authScopeAttempt < 2;
|
|
4758
|
+
authScopeAttempt += 1
|
|
4759
|
+
) {
|
|
4760
|
+
try {
|
|
4761
|
+
return await this.executeWithRuntimeReceipt(
|
|
4762
|
+
'tool',
|
|
4763
|
+
normalizedKey,
|
|
4764
|
+
this.currentRunId,
|
|
4765
|
+
{
|
|
4766
|
+
receiptKey: durableCacheKey,
|
|
4767
|
+
semanticKey: toolRequestIdentity,
|
|
4768
|
+
force: toolCachePolicy.force,
|
|
4769
|
+
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
4770
|
+
markSkipped: () => {
|
|
4771
|
+
this.log(
|
|
4772
|
+
`ctx.tools.execute(${toolId}): no-op due completed receipt ${toolRequestIdentity} (label: ${normalizedKey})`,
|
|
4773
|
+
);
|
|
4774
|
+
},
|
|
4775
|
+
onClaimedResult: (output, receiptKey) =>
|
|
4776
|
+
markToolExecuteResultExecutionOutcome(output, {
|
|
4777
|
+
kind: 'live',
|
|
4778
|
+
receiptKey,
|
|
4779
|
+
}),
|
|
4780
|
+
onRecovered: (output, _receipt, source) =>
|
|
4781
|
+
markToolExecuteResultExecutionOutcome(
|
|
4782
|
+
output,
|
|
4783
|
+
toolExecutionOutcomeForDurableReceipt({
|
|
4784
|
+
source,
|
|
4785
|
+
receiptKey: durableCacheKey,
|
|
4786
|
+
}),
|
|
4787
|
+
),
|
|
4788
|
+
shouldPersistFailure: (error) =>
|
|
4789
|
+
!(error instanceof ToolExecuteAuthScopeChangedError),
|
|
4790
|
+
execute: ({ leaseId }) => executeTool({ leaseId }),
|
|
4791
|
+
runningReceiptWaitMaxAttempts:
|
|
4792
|
+
resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
4793
|
+
typeof options?.receiptWaitMs === 'number'
|
|
4794
|
+
? { max_wait_ms: options.receiptWaitMs }
|
|
4795
|
+
: input,
|
|
4796
|
+
),
|
|
4797
|
+
},
|
|
4798
|
+
);
|
|
4799
|
+
} catch (error) {
|
|
4800
|
+
if (
|
|
4801
|
+
!(error instanceof ToolExecuteAuthScopeChangedError) ||
|
|
4802
|
+
authScopeAttempt > 0
|
|
4803
|
+
) {
|
|
4804
|
+
throw error;
|
|
4805
|
+
}
|
|
4806
|
+
// Make the bounded auth-scope re-claim observable in run logs instead
|
|
4807
|
+
// of a silent continuation: the credential identity changed after the
|
|
4808
|
+
// receipt key was prepared, so we evict the cached digest, re-resolve,
|
|
4809
|
+
// and re-claim a fresh receipt under the new scope before retrying once.
|
|
4810
|
+
this.log(
|
|
4811
|
+
`ctx.tools.execute(${toolId}): auth_scope_changed_reclaim ` +
|
|
4812
|
+
`(label: ${normalizedKey}); credential identity changed mid-run, ` +
|
|
4813
|
+
`re-resolving auth scope and re-claiming a fresh receipt under the ` +
|
|
4814
|
+
`new scope (attempt ${authScopeAttempt + 1}/2).`,
|
|
4815
|
+
);
|
|
4816
|
+
this.invalidateToolAuthScopeDigest(toolId);
|
|
4817
|
+
executionAuthScopeDigest =
|
|
4818
|
+
(await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
|
|
4819
|
+
durableCacheKey = await this.durableToolCallCacheKey({
|
|
4820
|
+
toolId,
|
|
4821
|
+
requestInput: input,
|
|
4822
|
+
executionAuthScopeDigest,
|
|
4823
|
+
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
4824
|
+
});
|
|
4825
|
+
checkpointCacheKeys[0] = durableCacheKey;
|
|
4826
|
+
}
|
|
4827
|
+
}
|
|
4828
|
+
|
|
4829
|
+
throw new ToolExecuteAuthScopeChangedError();
|
|
3683
4830
|
}
|
|
3684
4831
|
|
|
3685
4832
|
private async executeIntegrationEventWaitTool(
|
|
@@ -3805,163 +4952,349 @@ export class PlayContextImpl {
|
|
|
3805
4952
|
if (!resolvedName.trim()) {
|
|
3806
4953
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
3807
4954
|
}
|
|
4955
|
+
const scheduledChildPlayCall = this.shouldLaunchChildPlaysThroughScheduler()
|
|
4956
|
+
? this.beginScheduledChildPlayCall()
|
|
4957
|
+
: null;
|
|
3808
4958
|
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3826
|
-
// Reserve depth + per-parent + descendant/play-call budget on the parent
|
|
3827
|
-
// and fork the lineage-global snapshot for the child. forkChild owns the
|
|
3828
|
-
// cycle guard, depth cap, per-parent cap, and play/descendant charges, so
|
|
3829
|
-
// a recovered receipt (which skips this body) never consumes budget. The
|
|
3830
|
-
// child run id stays deterministic — it matches the prior post-increment
|
|
3831
|
-
// playCallCount — so receipt keys and replay are stable.
|
|
3832
|
-
const childRunId = `${this.governance.rootRunId}:${resolvedName}:${this.governor.snapshot().playCallCount + 1}`;
|
|
3833
|
-
const childGovernance = this.governor.forkChild({
|
|
4959
|
+
try {
|
|
4960
|
+
if (!this.#options.resolvePlay) {
|
|
4961
|
+
throw new Error(
|
|
4962
|
+
'ctx.runPlay(...) is unavailable because no play resolver was configured.',
|
|
4963
|
+
);
|
|
4964
|
+
}
|
|
4965
|
+
const resolvePlay = this.#options.resolvePlay;
|
|
4966
|
+
const resolvedPlay = await resolvePlay(resolvedName);
|
|
4967
|
+
if (!resolvedPlay) {
|
|
4968
|
+
throw new Error(
|
|
4969
|
+
`Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`,
|
|
4970
|
+
);
|
|
4971
|
+
}
|
|
4972
|
+
const childRevisionFingerprint =
|
|
4973
|
+
resolvedPlayRevisionFingerprint(resolvedPlay);
|
|
4974
|
+
const runPlayRowScope = rowContext.getStore();
|
|
4975
|
+
const runPlaySemanticKey = buildDurableRunPlaySemanticKey({
|
|
3834
4976
|
childPlayName: resolvedName,
|
|
3835
|
-
|
|
4977
|
+
childRevisionFingerprint,
|
|
4978
|
+
input,
|
|
4979
|
+
rowScope: runPlayRowScope
|
|
4980
|
+
? {
|
|
4981
|
+
fieldName: runPlayRowScope.fieldName,
|
|
4982
|
+
rowId: runPlayRowScope.rowId,
|
|
4983
|
+
rowKey: runPlayRowScope.rowKey ?? null,
|
|
4984
|
+
tableNamespace: runPlayRowScope.tableNamespace ?? null,
|
|
4985
|
+
}
|
|
4986
|
+
: null,
|
|
3836
4987
|
});
|
|
3837
|
-
const childPlaySlot = await this.governor.acquireChildPlaySlot();
|
|
3838
|
-
const rowStore = rowContext.getStore();
|
|
3839
|
-
const producer = {
|
|
3840
|
-
kind: 'play' as const,
|
|
3841
|
-
id: normalizedKey,
|
|
3842
|
-
playId: resolvedName,
|
|
3843
|
-
displayName: displayNameFromProducerId(resolvedName),
|
|
3844
|
-
runId: childRunId,
|
|
3845
|
-
};
|
|
3846
4988
|
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
runId: childGovernance.currentRunId,
|
|
3868
|
-
staticPipeline: resolvedPlay.staticPipeline ?? null,
|
|
3869
|
-
checkpoint: this.checkpoint,
|
|
3870
|
-
governance: childGovernance,
|
|
3871
|
-
getRuntimeStepReceipt: undefined,
|
|
3872
|
-
getRuntimeStepReceipts: undefined,
|
|
3873
|
-
claimRuntimeStepReceipt: undefined,
|
|
3874
|
-
claimRuntimeStepReceipts: undefined,
|
|
3875
|
-
completeRuntimeStepReceipt: undefined,
|
|
3876
|
-
completeRuntimeStepReceipts: undefined,
|
|
3877
|
-
failRuntimeStepReceipt: undefined,
|
|
3878
|
-
failRuntimeStepReceipts: undefined,
|
|
3879
|
-
skipRuntimeStepReceipt: undefined,
|
|
3880
|
-
});
|
|
3881
|
-
const childExecution = this.executeResolvedPlay(
|
|
3882
|
-
resolvedPlay,
|
|
3883
|
-
childContext,
|
|
4989
|
+
const executePlayCall = async (): Promise<TOutput> => {
|
|
4990
|
+
// Reserve depth + per-parent + descendant/play-call budget on the parent
|
|
4991
|
+
// and fork the lineage-global snapshot for the child. forkChild owns the
|
|
4992
|
+
// cycle guard, depth cap, per-parent cap, and play/descendant charges, so
|
|
4993
|
+
// a recovered receipt (which skips this body) never consumes budget.
|
|
4994
|
+
//
|
|
4995
|
+
// The child run id is derived through the shared canonical builder so the
|
|
4996
|
+
// in-process runtime and the workers_edge coordinator produce the same
|
|
4997
|
+
// FORMAT (`play/<slug>/run/child-<digest>`) via the same code. The digest
|
|
4998
|
+
// is deterministic across replay because every input (parent lineage,
|
|
4999
|
+
// call key, input) is stable. Per-substrate difference: in-process has no
|
|
5000
|
+
// coordinator idempotency key and no compiled child graphHash, so it uses
|
|
5001
|
+
// the semantic fallback with graphHash omitted; workers_edge feeds the
|
|
5002
|
+
// durable child idempotency key. See child-run-id.ts.
|
|
5003
|
+
const childRunId = buildChildRunId({
|
|
5004
|
+
childPlayName: resolvedName,
|
|
5005
|
+
parentRunId: this.currentRunId,
|
|
5006
|
+
parentPlayName: this.#options.playName,
|
|
5007
|
+
key: normalizedKey,
|
|
5008
|
+
runPlaySemanticKey,
|
|
3884
5009
|
input,
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
const
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
3893
|
-
|
|
3894
|
-
|
|
3895
|
-
|
|
3896
|
-
stage: resolvedName,
|
|
3897
|
-
provider: 'deepline_native',
|
|
3898
|
-
error: null,
|
|
3899
|
-
producer,
|
|
3900
|
-
dataPatch: {},
|
|
3901
|
-
});
|
|
3902
|
-
}
|
|
3903
|
-
const step = {
|
|
3904
|
-
type: 'play_call' as const,
|
|
5010
|
+
graphHash: null,
|
|
5011
|
+
});
|
|
5012
|
+
const childGovernance = this.governor.forkChild({
|
|
5013
|
+
childPlayName: resolvedName,
|
|
5014
|
+
childRunId,
|
|
5015
|
+
});
|
|
5016
|
+
const childPlaySlot = await this.governor.acquireChildSubmitSlot();
|
|
5017
|
+
const rowStore = rowContext.getStore();
|
|
5018
|
+
const producer = {
|
|
5019
|
+
kind: 'play' as const,
|
|
5020
|
+
id: normalizedKey,
|
|
3905
5021
|
playId: resolvedName,
|
|
3906
|
-
|
|
3907
|
-
|
|
5022
|
+
displayName: displayNameFromProducerId(resolvedName),
|
|
5023
|
+
runId: childRunId,
|
|
3908
5024
|
};
|
|
3909
|
-
|
|
3910
|
-
|
|
3911
|
-
|
|
3912
|
-
|
|
3913
|
-
|
|
3914
|
-
|
|
3915
|
-
|
|
3916
|
-
|
|
3917
|
-
|
|
3918
|
-
|
|
3919
|
-
|
|
3920
|
-
|
|
3921
|
-
|
|
3922
|
-
|
|
3923
|
-
|
|
3924
|
-
|
|
3925
|
-
|
|
3926
|
-
|
|
3927
|
-
|
|
3928
|
-
|
|
5025
|
+
|
|
5026
|
+
try {
|
|
5027
|
+
if (rowStore) {
|
|
5028
|
+
this.emitScopedFieldMetaUpdate({
|
|
5029
|
+
rowId: rowStore.rowId,
|
|
5030
|
+
key: rowStore.rowKey ?? null,
|
|
5031
|
+
tableNamespace: rowStore.tableNamespace ?? null,
|
|
5032
|
+
fieldName: rowStore.fieldName,
|
|
5033
|
+
status: 'running',
|
|
5034
|
+
rowStatus: 'running',
|
|
5035
|
+
stage: resolvedName,
|
|
5036
|
+
provider: 'deepline_native',
|
|
5037
|
+
error: null,
|
|
5038
|
+
producer,
|
|
5039
|
+
dataPatch: {},
|
|
5040
|
+
});
|
|
5041
|
+
}
|
|
5042
|
+
if (this.shouldLaunchChildPlaysThroughScheduler()) {
|
|
5043
|
+
const boundaryId = this.childPlayBoundaryId(
|
|
5044
|
+
childGovernance.currentRunId,
|
|
5045
|
+
);
|
|
5046
|
+
const checkpointResult = this.childPlayBoundaryOutput<TOutput>({
|
|
5047
|
+
boundaryId,
|
|
5048
|
+
childRunId: childGovernance.currentRunId,
|
|
5049
|
+
childPlayName: resolvedName,
|
|
5050
|
+
});
|
|
5051
|
+
if (checkpointResult.found) {
|
|
5052
|
+
if (rowStore) {
|
|
5053
|
+
this.emitScopedFieldMetaUpdate({
|
|
5054
|
+
rowId: rowStore.rowId,
|
|
5055
|
+
key: rowStore.rowKey ?? null,
|
|
5056
|
+
tableNamespace: rowStore.tableNamespace ?? null,
|
|
5057
|
+
fieldName: rowStore.fieldName,
|
|
5058
|
+
status: 'completed',
|
|
5059
|
+
rowStatus: 'running',
|
|
5060
|
+
stage: resolvedName,
|
|
5061
|
+
provider: 'deepline_native',
|
|
5062
|
+
error: null,
|
|
5063
|
+
producer,
|
|
5064
|
+
dataPatch: {},
|
|
5065
|
+
});
|
|
5066
|
+
}
|
|
5067
|
+
this.recordPlayCallStep({
|
|
5068
|
+
playId: resolvedName,
|
|
5069
|
+
description: options?.description,
|
|
5070
|
+
});
|
|
5071
|
+
return checkpointResult.output;
|
|
5072
|
+
}
|
|
5073
|
+
|
|
5074
|
+
await this.launchScheduledChildPlay({
|
|
5075
|
+
key: normalizedKey,
|
|
5076
|
+
childRunId: childGovernance.currentRunId,
|
|
5077
|
+
childPlayName: resolvedName,
|
|
5078
|
+
childGovernance,
|
|
5079
|
+
childInput: input,
|
|
5080
|
+
description: options?.description ?? null,
|
|
5081
|
+
});
|
|
5082
|
+
childPlaySlot.release();
|
|
5083
|
+
const terminal = await this.readScheduledChildTerminal<TOutput>({
|
|
5084
|
+
childRunId: childGovernance.currentRunId,
|
|
5085
|
+
childPlayName: resolvedName,
|
|
5086
|
+
});
|
|
5087
|
+
if (terminal?.status === 'completed') {
|
|
5088
|
+
this.checkpoint.resolvedBoundaries = {
|
|
5089
|
+
...(this.checkpoint.resolvedBoundaries ?? {}),
|
|
5090
|
+
[boundaryId]: {
|
|
5091
|
+
kind: 'child_play',
|
|
5092
|
+
childRunId: childGovernance.currentRunId,
|
|
5093
|
+
childPlayName: resolvedName,
|
|
5094
|
+
status: 'completed',
|
|
5095
|
+
output: terminal.output,
|
|
5096
|
+
completedAt: Date.now(),
|
|
5097
|
+
},
|
|
5098
|
+
};
|
|
5099
|
+
if (rowStore) {
|
|
5100
|
+
this.emitScopedFieldMetaUpdate({
|
|
5101
|
+
rowId: rowStore.rowId,
|
|
5102
|
+
key: rowStore.rowKey ?? null,
|
|
5103
|
+
tableNamespace: rowStore.tableNamespace ?? null,
|
|
5104
|
+
fieldName: rowStore.fieldName,
|
|
5105
|
+
status: 'completed',
|
|
5106
|
+
rowStatus: 'running',
|
|
5107
|
+
stage: resolvedName,
|
|
5108
|
+
provider: 'deepline_native',
|
|
5109
|
+
error: null,
|
|
5110
|
+
producer,
|
|
5111
|
+
dataPatch: {},
|
|
5112
|
+
});
|
|
5113
|
+
}
|
|
5114
|
+
this.recordPlayCallStep({
|
|
5115
|
+
playId: resolvedName,
|
|
5116
|
+
description: options?.description,
|
|
5117
|
+
});
|
|
5118
|
+
return terminal.output;
|
|
5119
|
+
}
|
|
5120
|
+
if (terminal) {
|
|
5121
|
+
throw new Error(terminal.error);
|
|
5122
|
+
}
|
|
5123
|
+
|
|
5124
|
+
const suspension = {
|
|
5125
|
+
kind: 'child_play',
|
|
5126
|
+
boundaryId,
|
|
5127
|
+
childRunId: childGovernance.currentRunId,
|
|
5128
|
+
childPlayName: resolvedName,
|
|
5129
|
+
} as const;
|
|
5130
|
+
this.recordPendingChildPlayBoundary(suspension);
|
|
5131
|
+
scheduledChildPlayCall?.release();
|
|
5132
|
+
await this.waitForScheduledChildPlayQuiescence();
|
|
5133
|
+
throw new PlayExecutionSuspendedError(
|
|
5134
|
+
this.consumePendingChildPlaySuspension() ?? suspension,
|
|
5135
|
+
);
|
|
5136
|
+
}
|
|
5137
|
+
const childContext = createPlayContext({
|
|
5138
|
+
...this.#options,
|
|
5139
|
+
playId: resolvedName,
|
|
5140
|
+
playName: resolvedName,
|
|
5141
|
+
runId: childGovernance.currentRunId,
|
|
5142
|
+
executorToken: await this.mintChildExecutorToken({
|
|
5143
|
+
parentRunId: this.currentRunId,
|
|
5144
|
+
parentPlayName: this.#options.playName,
|
|
5145
|
+
childRunId: childGovernance.currentRunId,
|
|
5146
|
+
childPlayName: resolvedName,
|
|
5147
|
+
}),
|
|
5148
|
+
staticPipeline: resolvedPlay.staticPipeline ?? null,
|
|
5149
|
+
checkpoint: this.checkpoint,
|
|
5150
|
+
governance: childGovernance,
|
|
5151
|
+
getRuntimeStepReceipt: undefined,
|
|
5152
|
+
getRuntimeStepReceipts: undefined,
|
|
5153
|
+
claimRuntimeStepReceipt: undefined,
|
|
5154
|
+
claimRuntimeStepReceipts: undefined,
|
|
5155
|
+
completeRuntimeStepReceipt: undefined,
|
|
5156
|
+
completeRuntimeStepReceipts: undefined,
|
|
5157
|
+
failRuntimeStepReceipt: undefined,
|
|
5158
|
+
failRuntimeStepReceipts: undefined,
|
|
5159
|
+
heartbeatRuntimeStepReceipts: undefined,
|
|
5160
|
+
skipRuntimeStepReceipt: undefined,
|
|
3929
5161
|
});
|
|
5162
|
+
const childExecution = this.executeResolvedPlay(
|
|
5163
|
+
resolvedPlay,
|
|
5164
|
+
childContext,
|
|
5165
|
+
input,
|
|
5166
|
+
);
|
|
5167
|
+
await childContext.drainQueuedWork([childExecution]);
|
|
5168
|
+
const result = await childExecution;
|
|
5169
|
+
if (rowStore) {
|
|
5170
|
+
this.emitScopedFieldMetaUpdate({
|
|
5171
|
+
rowId: rowStore.rowId,
|
|
5172
|
+
key: rowStore.rowKey ?? null,
|
|
5173
|
+
tableNamespace: rowStore.tableNamespace ?? null,
|
|
5174
|
+
fieldName: rowStore.fieldName,
|
|
5175
|
+
status: 'completed',
|
|
5176
|
+
rowStatus: 'running',
|
|
5177
|
+
stage: resolvedName,
|
|
5178
|
+
provider: 'deepline_native',
|
|
5179
|
+
error: null,
|
|
5180
|
+
producer,
|
|
5181
|
+
dataPatch: {},
|
|
5182
|
+
});
|
|
5183
|
+
}
|
|
5184
|
+
const step = {
|
|
5185
|
+
type: 'play_call' as const,
|
|
5186
|
+
playId: resolvedName,
|
|
5187
|
+
nestedSteps: childContext.getSteps(),
|
|
5188
|
+
description: normalizeStepDescription(options?.description),
|
|
5189
|
+
};
|
|
5190
|
+
if (this.activeDatasetStep) {
|
|
5191
|
+
this.activeDatasetStep.substeps.push(step);
|
|
5192
|
+
} else {
|
|
5193
|
+
this.steps.push(step);
|
|
5194
|
+
}
|
|
5195
|
+
return result as TOutput;
|
|
5196
|
+
} catch (error) {
|
|
5197
|
+
if (isPlayExecutionSuspendedError(error)) {
|
|
5198
|
+
throw error;
|
|
5199
|
+
}
|
|
5200
|
+
if (rowStore) {
|
|
5201
|
+
this.emitScopedFieldMetaUpdate({
|
|
5202
|
+
rowId: rowStore.rowId,
|
|
5203
|
+
key: rowStore.rowKey ?? null,
|
|
5204
|
+
tableNamespace: rowStore.tableNamespace ?? null,
|
|
5205
|
+
fieldName: rowStore.fieldName,
|
|
5206
|
+
status: 'failed',
|
|
5207
|
+
rowStatus: 'running',
|
|
5208
|
+
stage: resolvedName,
|
|
5209
|
+
provider: 'deepline_native',
|
|
5210
|
+
error: this.formatRuntimeError(error),
|
|
5211
|
+
producer,
|
|
5212
|
+
dataPatch: {},
|
|
5213
|
+
});
|
|
5214
|
+
}
|
|
5215
|
+
throw error;
|
|
5216
|
+
} finally {
|
|
5217
|
+
childPlaySlot.release();
|
|
3930
5218
|
}
|
|
3931
|
-
|
|
3932
|
-
} finally {
|
|
3933
|
-
childPlaySlot.release();
|
|
3934
|
-
}
|
|
3935
|
-
};
|
|
5219
|
+
};
|
|
3936
5220
|
|
|
3937
|
-
|
|
3938
|
-
|
|
5221
|
+
return await this.executeWithRuntimeReceipt<TOutput>(
|
|
5222
|
+
'runPlay',
|
|
5223
|
+
normalizedKey,
|
|
5224
|
+
this.currentRunId,
|
|
5225
|
+
{
|
|
5226
|
+
semanticKey: runPlaySemanticKey,
|
|
5227
|
+
staleAfterSeconds: options?.staleAfterSeconds,
|
|
5228
|
+
markSkipped: () => {
|
|
5229
|
+
this.log(
|
|
5230
|
+
`ctx.runPlay(${normalizedKey}): no-op due completed receipt`,
|
|
5231
|
+
);
|
|
5232
|
+
},
|
|
5233
|
+
repairRunningReceiptForSameRunAfterWaitTimeout: true,
|
|
5234
|
+
runningReceiptWaitMaxAttempts:
|
|
5235
|
+
resolveRuntimeToolReceiptWaitMaxAttempts(input),
|
|
5236
|
+
execute: executePlayCall,
|
|
5237
|
+
},
|
|
5238
|
+
);
|
|
5239
|
+
} finally {
|
|
5240
|
+
scheduledChildPlayCall?.release();
|
|
3939
5241
|
}
|
|
5242
|
+
}
|
|
3940
5243
|
|
|
3941
|
-
|
|
3942
|
-
|
|
3943
|
-
|
|
3944
|
-
|
|
5244
|
+
private async mintChildExecutorToken(input: {
|
|
5245
|
+
parentRunId: string;
|
|
5246
|
+
parentPlayName?: string | null;
|
|
5247
|
+
childRunId: string;
|
|
5248
|
+
childPlayName: string;
|
|
5249
|
+
}): Promise<string | undefined> {
|
|
5250
|
+
if (!this.#options.executorToken || !this.#options.baseUrl) {
|
|
5251
|
+
return this.#options.executorToken;
|
|
5252
|
+
}
|
|
5253
|
+
const parentPlayName = input.parentPlayName?.trim();
|
|
5254
|
+
if (!parentPlayName) {
|
|
5255
|
+
return this.#options.executorToken;
|
|
5256
|
+
}
|
|
5257
|
+
const response = await fetch(
|
|
5258
|
+
`${this.#options.baseUrl.replace(/\/$/, '')}/api/v2/internal/play-runtime/child-executor-token`,
|
|
3945
5259
|
{
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
`ctx.runPlay(${normalizedKey}): no-op due completed receipt`,
|
|
3957
|
-
);
|
|
5260
|
+
method: 'POST',
|
|
5261
|
+
headers: {
|
|
5262
|
+
Authorization: `Bearer ${this.#options.executorToken}`,
|
|
5263
|
+
'Content-Type': 'application/json',
|
|
5264
|
+
...(this.#options.vercelProtectionBypassToken?.trim()
|
|
5265
|
+
? {
|
|
5266
|
+
'x-vercel-protection-bypass':
|
|
5267
|
+
this.#options.vercelProtectionBypassToken.trim(),
|
|
5268
|
+
}
|
|
5269
|
+
: {}),
|
|
3958
5270
|
},
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3962
|
-
|
|
5271
|
+
body: JSON.stringify({
|
|
5272
|
+
parentRunId: input.parentRunId,
|
|
5273
|
+
parentPlayName,
|
|
5274
|
+
childRunId: input.childRunId,
|
|
5275
|
+
childPlayName: input.childPlayName,
|
|
5276
|
+
}),
|
|
3963
5277
|
},
|
|
3964
5278
|
);
|
|
5279
|
+
const body = (await response.json().catch(() => null)) as {
|
|
5280
|
+
executorToken?: unknown;
|
|
5281
|
+
error?: unknown;
|
|
5282
|
+
} | null;
|
|
5283
|
+
if (!response.ok) {
|
|
5284
|
+
const message =
|
|
5285
|
+
typeof body?.error === 'string' && body.error.trim()
|
|
5286
|
+
? body.error.trim()
|
|
5287
|
+
: response.statusText;
|
|
5288
|
+
throw new Error(
|
|
5289
|
+
`ctx.runPlay failed to mint child executor token for "${input.childPlayName}": ${message} (status ${response.status}).`,
|
|
5290
|
+
);
|
|
5291
|
+
}
|
|
5292
|
+
if (typeof body?.executorToken !== 'string' || !body.executorToken.trim()) {
|
|
5293
|
+
throw new Error(
|
|
5294
|
+
`ctx.runPlay failed to mint child executor token for "${input.childPlayName}": response did not include executorToken.`,
|
|
5295
|
+
);
|
|
5296
|
+
}
|
|
5297
|
+
return body.executorToken.trim();
|
|
3965
5298
|
}
|
|
3966
5299
|
|
|
3967
5300
|
/**
|
|
@@ -4347,17 +5680,21 @@ export class PlayContextImpl {
|
|
|
4347
5680
|
await Promise.all(
|
|
4348
5681
|
[...byTool.entries()].map(async ([toolId, requests]) => {
|
|
4349
5682
|
this.log(`Executing tool batch ${toolId}: ${requests.length} calls`);
|
|
5683
|
+
const liveStepResults = new Map<string, unknown>();
|
|
4350
5684
|
|
|
4351
5685
|
const recordToolStep = (stepRequests: ToolCallRequest[]): void => {
|
|
4352
5686
|
if (stepRequests.length === 0) return;
|
|
4353
5687
|
const stepResults: PlayStepRowResult[] = stepRequests.map((req) => {
|
|
4354
|
-
const
|
|
5688
|
+
const hasLiveResult = liveStepResults.has(req.callId);
|
|
5689
|
+
const result = hasLiveResult
|
|
5690
|
+
? liveStepResults.get(req.callId)
|
|
5691
|
+
: this.getCachedToolResult(toolId, req.cacheKey)?.result;
|
|
4355
5692
|
return {
|
|
4356
5693
|
rowId: req.rowId,
|
|
4357
|
-
status:
|
|
4358
|
-
success:
|
|
4359
|
-
value:
|
|
4360
|
-
error:
|
|
5694
|
+
status: result != null ? 'completed' : 'failed',
|
|
5695
|
+
success: result != null,
|
|
5696
|
+
value: result,
|
|
5697
|
+
error: result != null ? null : 'Tool call failed',
|
|
4361
5698
|
};
|
|
4362
5699
|
});
|
|
4363
5700
|
const toolStep = {
|
|
@@ -4377,9 +5714,12 @@ export class PlayContextImpl {
|
|
|
4377
5714
|
const pendingRequests: ToolCallRequest[] = [];
|
|
4378
5715
|
const recoveredRequests: ToolCallRequest[] = [];
|
|
4379
5716
|
for (const req of requests) {
|
|
4380
|
-
const cached =
|
|
4381
|
-
|
|
4382
|
-
|
|
5717
|
+
const cached =
|
|
5718
|
+
req.cacheable === false
|
|
5719
|
+
? undefined
|
|
5720
|
+
: req.force
|
|
5721
|
+
? undefined
|
|
5722
|
+
: this.getCachedToolResult(toolId, req.cacheKey);
|
|
4383
5723
|
if (cached?.done) {
|
|
4384
5724
|
this.log(` Row ${req.rowId} ${toolId}: recovered from checkpoint`);
|
|
4385
5725
|
const resolver = this.toolCallResolvers.get(req.callId);
|
|
@@ -4411,7 +5751,9 @@ export class PlayContextImpl {
|
|
|
4411
5751
|
attachedToReceiptKey: followerReceiptKey,
|
|
4412
5752
|
})
|
|
4413
5753
|
: result;
|
|
4414
|
-
|
|
5754
|
+
if (follower.cacheable !== false) {
|
|
5755
|
+
this.cacheToolResult(toolId, follower.cacheKey, followerResult);
|
|
5756
|
+
}
|
|
4415
5757
|
const resolver = this.toolCallResolvers.get(follower.callId);
|
|
4416
5758
|
if (resolver) {
|
|
4417
5759
|
resolver.resolve(followerResult);
|
|
@@ -4439,7 +5781,11 @@ export class PlayContextImpl {
|
|
|
4439
5781
|
const followers = liveFollowersByOwnerCallId.get(owner.callId) ?? [];
|
|
4440
5782
|
liveFollowersByOwnerCallId.delete(owner.callId);
|
|
4441
5783
|
for (const request of [owner, ...followers]) {
|
|
4442
|
-
await this.rejectToolCall(toolId, request, error
|
|
5784
|
+
await this.rejectToolCall(toolId, request, error, {
|
|
5785
|
+
persistReceiptFailure: !(
|
|
5786
|
+
error instanceof ToolExecuteAuthScopeChangedError
|
|
5787
|
+
),
|
|
5788
|
+
});
|
|
4443
5789
|
}
|
|
4444
5790
|
};
|
|
4445
5791
|
const requestsWithLiveFollowers = (
|
|
@@ -4494,7 +5840,11 @@ export class PlayContextImpl {
|
|
|
4494
5840
|
) {
|
|
4495
5841
|
const requestsByReceiptKey = new Map<
|
|
4496
5842
|
string,
|
|
4497
|
-
{
|
|
5843
|
+
{
|
|
5844
|
+
requests: ToolCallRequest[];
|
|
5845
|
+
forceRefresh: boolean;
|
|
5846
|
+
forceFailedRefresh: boolean;
|
|
5847
|
+
}
|
|
4498
5848
|
>();
|
|
4499
5849
|
const localOnlyRequests: ToolCallRequest[] = [];
|
|
4500
5850
|
for (const request of pendingRequests) {
|
|
@@ -4506,17 +5856,37 @@ export class PlayContextImpl {
|
|
|
4506
5856
|
const group = requestsByReceiptKey.get(receiptKey) ?? {
|
|
4507
5857
|
requests: [],
|
|
4508
5858
|
forceRefresh: false,
|
|
5859
|
+
forceFailedRefresh: false,
|
|
4509
5860
|
};
|
|
4510
5861
|
group.requests.push(request);
|
|
4511
5862
|
group.forceRefresh ||= request.force === true;
|
|
5863
|
+
group.forceFailedRefresh ||= request.forceFailedRefresh === true;
|
|
4512
5864
|
requestsByReceiptKey.set(receiptKey, group);
|
|
4513
5865
|
}
|
|
4514
5866
|
const normalReceiptKeys = [...requestsByReceiptKey]
|
|
4515
|
-
.filter(
|
|
5867
|
+
.filter(
|
|
5868
|
+
([, group]) => !group.forceRefresh && !group.forceFailedRefresh,
|
|
5869
|
+
)
|
|
4516
5870
|
.map(([receiptKey]) => receiptKey);
|
|
4517
5871
|
const forcedReceiptKeys = [...requestsByReceiptKey]
|
|
4518
5872
|
.filter(([, group]) => group.forceRefresh)
|
|
4519
5873
|
.map(([receiptKey]) => receiptKey);
|
|
5874
|
+
const failedRefreshReceiptKeys = [...requestsByReceiptKey]
|
|
5875
|
+
.filter(
|
|
5876
|
+
([, group]) => !group.forceRefresh && group.forceFailedRefresh,
|
|
5877
|
+
)
|
|
5878
|
+
.map(([receiptKey]) => receiptKey);
|
|
5879
|
+
const forcedExisting = forcedReceiptKeys.length
|
|
5880
|
+
? await this.getRuntimeStepReceipts(forcedReceiptKeys)
|
|
5881
|
+
: new Map<string, RuntimeStepReceipt>();
|
|
5882
|
+
const forcedRunningReceiptKeys = new Set(
|
|
5883
|
+
[...forcedExisting]
|
|
5884
|
+
.filter(([, receipt]) => receipt.status === 'running')
|
|
5885
|
+
.map(([receiptKey]) => receiptKey),
|
|
5886
|
+
);
|
|
5887
|
+
const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
|
|
5888
|
+
(receiptKey) => !forcedRunningReceiptKeys.has(receiptKey),
|
|
5889
|
+
);
|
|
4520
5890
|
const claims =
|
|
4521
5891
|
normalReceiptKeys.length > 0
|
|
4522
5892
|
? await this.claimRuntimeStepReceipts(
|
|
@@ -4525,17 +5895,30 @@ export class PlayContextImpl {
|
|
|
4525
5895
|
)
|
|
4526
5896
|
: new Map<string, RuntimeStepReceipt>();
|
|
4527
5897
|
const forcedClaims =
|
|
4528
|
-
|
|
5898
|
+
claimableForcedReceiptKeys.length > 0
|
|
4529
5899
|
? await this.claimRuntimeStepReceipts(
|
|
4530
|
-
|
|
5900
|
+
claimableForcedReceiptKeys,
|
|
4531
5901
|
this.currentRunId,
|
|
5902
|
+
false,
|
|
4532
5903
|
true,
|
|
5904
|
+
)
|
|
5905
|
+
: new Map<string, RuntimeStepReceipt>();
|
|
5906
|
+
const failedRefreshClaims =
|
|
5907
|
+
failedRefreshReceiptKeys.length > 0
|
|
5908
|
+
? await this.claimRuntimeStepReceipts(
|
|
5909
|
+
failedRefreshReceiptKeys,
|
|
5910
|
+
this.currentRunId,
|
|
5911
|
+
false,
|
|
5912
|
+
false,
|
|
4533
5913
|
true,
|
|
4534
5914
|
)
|
|
4535
5915
|
: new Map<string, RuntimeStepReceipt>();
|
|
4536
5916
|
for (const [receiptKey, claim] of forcedClaims) {
|
|
4537
5917
|
claims.set(receiptKey, claim);
|
|
4538
5918
|
}
|
|
5919
|
+
for (const [receiptKey, claim] of failedRefreshClaims) {
|
|
5920
|
+
claims.set(receiptKey, claim);
|
|
5921
|
+
}
|
|
4539
5922
|
const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
|
|
4540
5923
|
const durableRecoveredRequests: ToolCallRequest[] = [];
|
|
4541
5924
|
const resolveRequestsFromReceipt = async (
|
|
@@ -4547,9 +5930,7 @@ export class PlayContextImpl {
|
|
|
4547
5930
|
if (!request) return;
|
|
4548
5931
|
const receiptKey = request.receiptKey?.trim() || null;
|
|
4549
5932
|
if (!receiptKey) {
|
|
4550
|
-
throw new Error(
|
|
4551
|
-
`Durable tool receipt ${source} recovery requires a receipt key.`,
|
|
4552
|
-
);
|
|
5933
|
+
throw new Error(`${source} tool recovery needs receipt key.`);
|
|
4553
5934
|
}
|
|
4554
5935
|
const wrapped = await this.wrapToolExecutionResult({
|
|
4555
5936
|
toolId,
|
|
@@ -4558,6 +5939,7 @@ export class PlayContextImpl {
|
|
|
4558
5939
|
? 'no_result'
|
|
4559
5940
|
: 'completed',
|
|
4560
5941
|
result: this.runtimeReceiptOutput(receipt),
|
|
5942
|
+
requestInput: request.input,
|
|
4561
5943
|
execution: toolExecutionMetadataForOutcome({
|
|
4562
5944
|
kind: source,
|
|
4563
5945
|
cacheKey: request.cacheKey,
|
|
@@ -4602,6 +5984,7 @@ export class PlayContextImpl {
|
|
|
4602
5984
|
const reclaimExistingRunningReceiptAfterWait = async (
|
|
4603
5985
|
receiptKey: string,
|
|
4604
5986
|
requestsForKey: ToolCallRequest[],
|
|
5987
|
+
forceAfterWait = false,
|
|
4605
5988
|
): Promise<void> => {
|
|
4606
5989
|
const waitMaxAttempts =
|
|
4607
5990
|
requestsForKey.length > 0
|
|
@@ -4616,15 +5999,18 @@ export class PlayContextImpl {
|
|
|
4616
5999
|
)
|
|
4617
6000
|
: undefined;
|
|
4618
6001
|
try {
|
|
4619
|
-
await
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
receiptKey,
|
|
4623
|
-
waitMaxAttempts,
|
|
4624
|
-
),
|
|
4625
|
-
'in_flight',
|
|
6002
|
+
const completed = await this.waitForCompletedRuntimeToolReceipt(
|
|
6003
|
+
receiptKey,
|
|
6004
|
+
waitMaxAttempts,
|
|
4626
6005
|
);
|
|
4627
|
-
|
|
6006
|
+
if (!forceAfterWait) {
|
|
6007
|
+
await resolveRequestsFromReceipt(
|
|
6008
|
+
requestsForKey,
|
|
6009
|
+
completed,
|
|
6010
|
+
'in_flight',
|
|
6011
|
+
);
|
|
6012
|
+
return;
|
|
6013
|
+
}
|
|
4628
6014
|
} catch (error) {
|
|
4629
6015
|
if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
|
|
4630
6016
|
for (const request of requestsForKey) {
|
|
@@ -4640,6 +6026,7 @@ export class PlayContextImpl {
|
|
|
4640
6026
|
[receiptKey],
|
|
4641
6027
|
this.currentRunId,
|
|
4642
6028
|
true,
|
|
6029
|
+
forceAfterWait,
|
|
4643
6030
|
)
|
|
4644
6031
|
).get(receiptKey);
|
|
4645
6032
|
if (
|
|
@@ -4670,14 +6057,31 @@ export class PlayContextImpl {
|
|
|
4670
6057
|
) {
|
|
4671
6058
|
const [owner, ...waiters] = requestsForKey;
|
|
4672
6059
|
if (!owner) return;
|
|
6060
|
+
if (!reclaimed.leaseId) {
|
|
6061
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
6062
|
+
receiptKey,
|
|
6063
|
+
runId: this.currentRunId,
|
|
6064
|
+
leaseId: 'missing',
|
|
6065
|
+
});
|
|
6066
|
+
}
|
|
6067
|
+
owner.receiptLeaseId = reclaimed.leaseId ?? null;
|
|
4673
6068
|
if (waiters.length > 0) {
|
|
4674
6069
|
liveFollowersByOwnerCallId.set(owner.callId, waiters);
|
|
4675
6070
|
}
|
|
4676
6071
|
try {
|
|
6072
|
+
await this.assertRuntimeToolReceiptOwnership([owner]);
|
|
4677
6073
|
const execution = await this.callToolExecutionAPI(
|
|
4678
6074
|
toolId,
|
|
4679
6075
|
owner.input,
|
|
4680
6076
|
{
|
|
6077
|
+
durableCallReceiptKey: receiptKey,
|
|
6078
|
+
executionAuthScopeDigest: owner.executionAuthScopeDigest,
|
|
6079
|
+
providerIdempotencyKey:
|
|
6080
|
+
this.providerIdempotencyKeyForToolCall({
|
|
6081
|
+
cacheKey: owner.cacheKey,
|
|
6082
|
+
force: owner.force === true,
|
|
6083
|
+
leaseId: owner.receiptLeaseId,
|
|
6084
|
+
}),
|
|
4681
6085
|
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([owner]),
|
|
4682
6086
|
},
|
|
4683
6087
|
);
|
|
@@ -4689,6 +6093,7 @@ export class PlayContextImpl {
|
|
|
4689
6093
|
execution?.jobId,
|
|
4690
6094
|
execution?.meta,
|
|
4691
6095
|
);
|
|
6096
|
+
liveStepResults.set(owner.callId, result);
|
|
4692
6097
|
resolveLiveFollowers(owner, result);
|
|
4693
6098
|
recordToolStep([owner]);
|
|
4694
6099
|
this.#options.onBatchComplete?.(this.checkpoint);
|
|
@@ -4711,8 +6116,13 @@ export class PlayContextImpl {
|
|
|
4711
6116
|
const requestsForKey = group.requests;
|
|
4712
6117
|
const claim = claims.get(receiptKey);
|
|
4713
6118
|
if (!claim) {
|
|
4714
|
-
|
|
4715
|
-
|
|
6119
|
+
durableExistingRunningHandlers.push(
|
|
6120
|
+
reclaimExistingRunningReceiptAfterWait(
|
|
6121
|
+
receiptKey,
|
|
6122
|
+
requestsForKey,
|
|
6123
|
+
group.forceRefresh,
|
|
6124
|
+
),
|
|
6125
|
+
);
|
|
4716
6126
|
continue;
|
|
4717
6127
|
}
|
|
4718
6128
|
if (claim?.status === 'completed' || claim?.status === 'skipped') {
|
|
@@ -4735,6 +6145,16 @@ export class PlayContextImpl {
|
|
|
4735
6145
|
claim.claimState !== 'existing'
|
|
4736
6146
|
) {
|
|
4737
6147
|
const [owner, ...waiters] = requestsForKey;
|
|
6148
|
+
if (!claim.leaseId) {
|
|
6149
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
6150
|
+
receiptKey,
|
|
6151
|
+
runId: this.currentRunId,
|
|
6152
|
+
leaseId: 'missing',
|
|
6153
|
+
});
|
|
6154
|
+
}
|
|
6155
|
+
if (owner) {
|
|
6156
|
+
owner.receiptLeaseId = claim.leaseId ?? null;
|
|
6157
|
+
}
|
|
4738
6158
|
claimOwnerWithLiveFollowers(owner, waiters);
|
|
4739
6159
|
continue;
|
|
4740
6160
|
}
|
|
@@ -4746,6 +6166,7 @@ export class PlayContextImpl {
|
|
|
4746
6166
|
reclaimExistingRunningReceiptAfterWait(
|
|
4747
6167
|
receiptKey,
|
|
4748
6168
|
requestsForKey,
|
|
6169
|
+
group.forceRefresh,
|
|
4749
6170
|
),
|
|
4750
6171
|
);
|
|
4751
6172
|
continue;
|
|
@@ -4754,6 +6175,7 @@ export class PlayContextImpl {
|
|
|
4754
6175
|
reclaimExistingRunningReceiptAfterWait(
|
|
4755
6176
|
receiptKey,
|
|
4756
6177
|
requestsForKey,
|
|
6178
|
+
group.forceRefresh,
|
|
4757
6179
|
),
|
|
4758
6180
|
);
|
|
4759
6181
|
}
|
|
@@ -4778,18 +6200,13 @@ export class PlayContextImpl {
|
|
|
4778
6200
|
compiledBatches.length > 0
|
|
4779
6201
|
? await this.governor.suggestedParallelism(
|
|
4780
6202
|
compiledBatches[0]!.batchOperation,
|
|
4781
|
-
|
|
6203
|
+
this.governor.policy.pacing
|
|
6204
|
+
.workerToolBatchDefaultParallelism,
|
|
4782
6205
|
)
|
|
4783
|
-
:
|
|
6206
|
+
: this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
4784
6207
|
await executeChunkedRequests({
|
|
4785
6208
|
requests: compiledBatches,
|
|
4786
6209
|
batchSize,
|
|
4787
|
-
// Bounded dispatch wave (postgres_fast parity with workers_edge):
|
|
4788
|
-
// cap the provider calls (counted by batch member size) a single
|
|
4789
|
-
// chunk dispatches before the latch is read again, so a
|
|
4790
|
-
// persistence failure prevents the rest of a large batch group.
|
|
4791
|
-
maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
|
|
4792
|
-
weightOf: (batch) => batch.memberRequests.length,
|
|
4793
6210
|
execute: async (batch) => {
|
|
4794
6211
|
// Circuit breaker: skip dispatching this batch's provider call
|
|
4795
6212
|
// once a persistence failure has occurred in this run.
|
|
@@ -4800,10 +6217,42 @@ export class PlayContextImpl {
|
|
|
4800
6217
|
this.persistenceLatch,
|
|
4801
6218
|
);
|
|
4802
6219
|
}
|
|
6220
|
+
await this.assertRuntimeToolReceiptOwnership(
|
|
6221
|
+
batch.memberRequests,
|
|
6222
|
+
);
|
|
6223
|
+
const receiptKeys = batch.memberRequests.map(
|
|
6224
|
+
(request) => request.cacheKey,
|
|
6225
|
+
);
|
|
6226
|
+
const aggregateReceiptKey = buildDurableToolAggregateReceiptKey(
|
|
6227
|
+
{
|
|
6228
|
+
receiptKeys,
|
|
6229
|
+
prefix: 'batch',
|
|
6230
|
+
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
6231
|
+
orgId: this.#options.orgId,
|
|
6232
|
+
toolId: batch.batchOperation,
|
|
6233
|
+
}),
|
|
6234
|
+
},
|
|
6235
|
+
);
|
|
4803
6236
|
return await this.callToolAPI(
|
|
4804
6237
|
batch.batchOperation,
|
|
4805
6238
|
batch.batchPayload,
|
|
4806
6239
|
{
|
|
6240
|
+
durableCallReceiptKey: aggregateReceiptKey,
|
|
6241
|
+
executionAuthScopeDigest:
|
|
6242
|
+
batch.memberRequests[0]?.executionAuthScopeDigest ?? null,
|
|
6243
|
+
providerIdempotencyKey:
|
|
6244
|
+
buildDurableToolAggregateProviderIdempotencyKey({
|
|
6245
|
+
aggregateReceiptKey,
|
|
6246
|
+
receiptKeys,
|
|
6247
|
+
providerIdempotencyKeys: batch.memberRequests.map(
|
|
6248
|
+
(request) =>
|
|
6249
|
+
this.providerIdempotencyKeyForToolCall({
|
|
6250
|
+
cacheKey: request.cacheKey,
|
|
6251
|
+
force: request.force === true,
|
|
6252
|
+
leaseId: request.receiptLeaseId,
|
|
6253
|
+
}),
|
|
6254
|
+
),
|
|
6255
|
+
}),
|
|
4807
6256
|
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
|
|
4808
6257
|
batch.memberRequests,
|
|
4809
6258
|
),
|
|
@@ -4837,6 +6286,7 @@ export class PlayContextImpl {
|
|
|
4837
6286
|
index += 1
|
|
4838
6287
|
) {
|
|
4839
6288
|
const request = entry.request.memberRequests[index]!;
|
|
6289
|
+
liveStepResults.set(request.callId, resolvedResults[index]);
|
|
4840
6290
|
resolveLiveFollowers(request, resolvedResults[index]);
|
|
4841
6291
|
}
|
|
4842
6292
|
}
|
|
@@ -4848,13 +6298,9 @@ export class PlayContextImpl {
|
|
|
4848
6298
|
},
|
|
4849
6299
|
});
|
|
4850
6300
|
} else {
|
|
4851
|
-
|
|
4852
|
-
|
|
4853
|
-
|
|
4854
|
-
// wave size. Respect a lower provider-pacing suggestion when present.
|
|
4855
|
-
const batchSize = Math.min(
|
|
4856
|
-
await this.governor.suggestedParallelism(toolId, 50),
|
|
4857
|
-
PROVIDER_DISPATCH_WAVE_SIZE,
|
|
6301
|
+
const batchSize = await this.governor.suggestedParallelism(
|
|
6302
|
+
toolId,
|
|
6303
|
+
this.governor.policy.pacing.suggestedMaxParallelism,
|
|
4858
6304
|
);
|
|
4859
6305
|
for (
|
|
4860
6306
|
let start = 0;
|
|
@@ -4877,10 +6323,24 @@ export class PlayContextImpl {
|
|
|
4877
6323
|
return;
|
|
4878
6324
|
}
|
|
4879
6325
|
try {
|
|
6326
|
+
await this.assertRuntimeToolReceiptOwnership([request]);
|
|
4880
6327
|
const execution = await this.callToolExecutionAPI(
|
|
4881
6328
|
toolId,
|
|
4882
6329
|
request.input,
|
|
4883
6330
|
{
|
|
6331
|
+
...(request.receiptKey
|
|
6332
|
+
? {
|
|
6333
|
+
durableCallReceiptKey: request.receiptKey,
|
|
6334
|
+
executionAuthScopeDigest:
|
|
6335
|
+
request.executionAuthScopeDigest,
|
|
6336
|
+
providerIdempotencyKey:
|
|
6337
|
+
this.providerIdempotencyKeyForToolCall({
|
|
6338
|
+
cacheKey: request.receiptKey,
|
|
6339
|
+
force: request.force === true,
|
|
6340
|
+
leaseId: request.receiptLeaseId,
|
|
6341
|
+
}),
|
|
6342
|
+
}
|
|
6343
|
+
: {}),
|
|
4884
6344
|
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
|
|
4885
6345
|
request,
|
|
4886
6346
|
]),
|
|
@@ -4894,6 +6354,7 @@ export class PlayContextImpl {
|
|
|
4894
6354
|
execution?.jobId,
|
|
4895
6355
|
execution?.meta,
|
|
4896
6356
|
);
|
|
6357
|
+
liveStepResults.set(request.callId, result);
|
|
4897
6358
|
resolveLiveFollowers(request, result);
|
|
4898
6359
|
} catch (error) {
|
|
4899
6360
|
await rejectWithLiveFollowers(request, error);
|
|
@@ -4989,7 +6450,7 @@ export class PlayContextImpl {
|
|
|
4989
6450
|
private async callToolAPI(
|
|
4990
6451
|
toolId: string,
|
|
4991
6452
|
input: Record<string, unknown>,
|
|
4992
|
-
options?:
|
|
6453
|
+
options?: ToolExecutionApiOptions,
|
|
4993
6454
|
): Promise<unknown> {
|
|
4994
6455
|
const execution = await this.callToolExecutionAPI(toolId, input, options);
|
|
4995
6456
|
if (execution.toolResponse && 'raw' in execution.toolResponse) {
|
|
@@ -5003,10 +6464,27 @@ export class PlayContextImpl {
|
|
|
5003
6464
|
: execution.result;
|
|
5004
6465
|
}
|
|
5005
6466
|
|
|
6467
|
+
private providerIdempotencyKeyForToolCall(input: {
|
|
6468
|
+
cacheKey: string;
|
|
6469
|
+
force?: boolean;
|
|
6470
|
+
leaseId?: string | null;
|
|
6471
|
+
}): string {
|
|
6472
|
+
const fallbackAttemptId =
|
|
6473
|
+
input.force === true && !input.leaseId
|
|
6474
|
+
? `${this.currentRunId}:${crypto.randomUUID()}`
|
|
6475
|
+
: null;
|
|
6476
|
+
return buildDurableToolProviderIdempotencyKey({
|
|
6477
|
+
receiptKey: input.cacheKey,
|
|
6478
|
+
force: input.force,
|
|
6479
|
+
receiptLeaseId: input.leaseId,
|
|
6480
|
+
fallbackAttemptId,
|
|
6481
|
+
});
|
|
6482
|
+
}
|
|
6483
|
+
|
|
5006
6484
|
private async callToolExecutionAPI(
|
|
5007
6485
|
toolId: string,
|
|
5008
6486
|
input: Record<string, unknown>,
|
|
5009
|
-
options?:
|
|
6487
|
+
options?: ToolExecutionApiOptions,
|
|
5010
6488
|
): Promise<ParsedToolExecuteResponse> {
|
|
5011
6489
|
if (!this.#options.executorToken || !this.#options.baseUrl) {
|
|
5012
6490
|
throw new Error(
|
|
@@ -5045,6 +6523,20 @@ export class PlayContextImpl {
|
|
|
5045
6523
|
|
|
5046
6524
|
while (true) {
|
|
5047
6525
|
let response: Response;
|
|
6526
|
+
const durableCallReceiptKey =
|
|
6527
|
+
options?.durableCallReceiptKey?.trim() || null;
|
|
6528
|
+
const executionAuthScopeDigest =
|
|
6529
|
+
durableCallReceiptKey && options?.executionAuthScopeDigest
|
|
6530
|
+
? options.executionAuthScopeDigest.trim() || null
|
|
6531
|
+
: durableCallReceiptKey
|
|
6532
|
+
? ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ??
|
|
6533
|
+
null)
|
|
6534
|
+
: null;
|
|
6535
|
+
const providerIdempotencyKey =
|
|
6536
|
+
options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
|
|
6537
|
+
const deeplineRequestId = providerIdempotencyKey
|
|
6538
|
+
? `ctx-tool-${stableDigest(providerIdempotencyKey).slice(0, 32)}`
|
|
6539
|
+
: null;
|
|
5048
6540
|
const abortController = timeoutMs ? new AbortController() : null;
|
|
5049
6541
|
const timeoutHandle = timeoutMs
|
|
5050
6542
|
? setTimeout(() => {
|
|
@@ -5064,7 +6556,14 @@ export class PlayContextImpl {
|
|
|
5064
6556
|
Authorization: `Bearer ${this.#options.executorToken}`,
|
|
5065
6557
|
[EXECUTE_RESPONSE_CONTRACT_HEADER]:
|
|
5066
6558
|
V2_EXECUTE_RESPONSE_CONTRACT,
|
|
6559
|
+
[EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
|
|
5067
6560
|
[EXECUTE_TOOL_METADATA_HEADER]: 'true',
|
|
6561
|
+
...(deeplineRequestId
|
|
6562
|
+
? {
|
|
6563
|
+
'x-deepline-request-id': deeplineRequestId,
|
|
6564
|
+
'x-deepline-idempotency-key': providerIdempotencyKey!,
|
|
6565
|
+
}
|
|
6566
|
+
: {}),
|
|
5068
6567
|
...(this.#options.vercelProtectionBypassToken?.trim()
|
|
5069
6568
|
? {
|
|
5070
6569
|
'x-vercel-protection-bypass':
|
|
@@ -5074,7 +6573,44 @@ export class PlayContextImpl {
|
|
|
5074
6573
|
},
|
|
5075
6574
|
body: JSON.stringify({
|
|
5076
6575
|
payload: input,
|
|
5077
|
-
metadata: {
|
|
6576
|
+
metadata: {
|
|
6577
|
+
parent_run_id: this.#options.runId,
|
|
6578
|
+
...(durableCallReceiptKey
|
|
6579
|
+
? {
|
|
6580
|
+
durable_call_receipt_key: durableCallReceiptKey,
|
|
6581
|
+
...(executionAuthScopeDigest
|
|
6582
|
+
? {
|
|
6583
|
+
execution_auth_scope_digest:
|
|
6584
|
+
executionAuthScopeDigest,
|
|
6585
|
+
}
|
|
6586
|
+
: {}),
|
|
6587
|
+
}
|
|
6588
|
+
: {}),
|
|
6589
|
+
...(options?.customerDbDataset
|
|
6590
|
+
? {
|
|
6591
|
+
query_result_dataset: {
|
|
6592
|
+
limit: options.customerDbDataset.limit,
|
|
6593
|
+
offset: options.customerDbDataset.offset,
|
|
6594
|
+
page_size: options.customerDbDataset.pageSize,
|
|
6595
|
+
total_rows: options.customerDbDataset.totalRows,
|
|
6596
|
+
},
|
|
6597
|
+
...(isCustomerDbDatasetTool(toolId)
|
|
6598
|
+
? {
|
|
6599
|
+
customer_db_dataset: {
|
|
6600
|
+
limit: options.customerDbDataset.limit,
|
|
6601
|
+
offset: options.customerDbDataset.offset,
|
|
6602
|
+
page_size: options.customerDbDataset.pageSize,
|
|
6603
|
+
total_rows:
|
|
6604
|
+
options.customerDbDataset.totalRows,
|
|
6605
|
+
},
|
|
6606
|
+
}
|
|
6607
|
+
: {}),
|
|
6608
|
+
}
|
|
6609
|
+
: {}),
|
|
6610
|
+
...(providerIdempotencyKey
|
|
6611
|
+
? { provider_idempotency_key: providerIdempotencyKey }
|
|
6612
|
+
: {}),
|
|
6613
|
+
},
|
|
5078
6614
|
...(this.#options.integrationMode
|
|
5079
6615
|
? { integration_mode: this.#options.integrationMode }
|
|
5080
6616
|
: {}),
|
|
@@ -5118,6 +6654,13 @@ export class PlayContextImpl {
|
|
|
5118
6654
|
|
|
5119
6655
|
if (!response.ok) {
|
|
5120
6656
|
const text = await response.text();
|
|
6657
|
+
const authScopeChanged = parseToolExecuteAuthScopeChangedError({
|
|
6658
|
+
status: response.status,
|
|
6659
|
+
bodyText: text,
|
|
6660
|
+
});
|
|
6661
|
+
if (authScopeChanged) {
|
|
6662
|
+
throw authScopeChanged;
|
|
6663
|
+
}
|
|
5121
6664
|
const httpFailureAttempt = httpFailureAttempts.next({
|
|
5122
6665
|
toolId,
|
|
5123
6666
|
status: response.status,
|