deepline 0.3.44 → 0.3.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1540 -265
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +22 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +16 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +418 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/index.ts +452 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +304 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/testkit.ts +83 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/always-fresh-adapter.ts +50 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/contract.ts +135 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +291 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts +335 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +904 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +522 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/testkit.ts +165 -0
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/install-integrity.json +11 -0
- package/package.json +1 -1
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
compileRequestsWithStrategy,
|
|
31
31
|
executeChunkedRequests,
|
|
32
32
|
} from './batch-runtime';
|
|
33
|
+
import type { AnyBatchOperationStrategy } from './batching-types';
|
|
33
34
|
import {
|
|
34
35
|
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
35
36
|
runtimeLeaseHeartbeatIntervalFromExpiry,
|
|
@@ -132,6 +133,19 @@ import {
|
|
|
132
133
|
type ToolResultExecutionMetadata,
|
|
133
134
|
type ToolResultMetadataInput,
|
|
134
135
|
} from './tool-result';
|
|
136
|
+
import {
|
|
137
|
+
createAlwaysFreshToolCallAdapter,
|
|
138
|
+
createExternallyOwnedToolResultReceipts,
|
|
139
|
+
createNativeBatchToolCallJob,
|
|
140
|
+
createToolCallReceiptCohortJob,
|
|
141
|
+
createRuntimeStepToolResultReceipts,
|
|
142
|
+
createToolCallJob,
|
|
143
|
+
ToolCallReceiptLeaseLostError,
|
|
144
|
+
type ToolCallReceiptCohortStore,
|
|
145
|
+
type ToolCallReceiptRecord,
|
|
146
|
+
type ToolCallInput,
|
|
147
|
+
type ToolResultReceipts,
|
|
148
|
+
} from './tool-call/index';
|
|
135
149
|
import {
|
|
136
150
|
markToolExecuteResultExecutionOutcome,
|
|
137
151
|
toolExecutionMetadataForOutcome,
|
|
@@ -1217,7 +1231,7 @@ function publicToolResponseEnvelope(value: unknown): {
|
|
|
1217
1231
|
* projection change on receipt replay and duplicate every batch for every row.
|
|
1218
1232
|
*/
|
|
1219
1233
|
function publicToolResponseForBatchedItem(
|
|
1220
|
-
execution:
|
|
1234
|
+
execution: ToolExecuteResult,
|
|
1221
1235
|
result: unknown,
|
|
1222
1236
|
forceRawV2: boolean,
|
|
1223
1237
|
): ParsedToolExecuteResponse['toolResponse'] | undefined {
|
|
@@ -1240,19 +1254,14 @@ function publicToolResponseForBatchedItem(
|
|
|
1240
1254
|
* Batch splitters predate raw-v2 and receive the same legacy value they got
|
|
1241
1255
|
* from callToolAPI. Keep that input contract stable; raw-v2 is attached to the
|
|
1242
1256
|
* completed logical call separately by publicToolResponseForBatchedItem.
|
|
1257
|
+
*
|
|
1258
|
+
* Physical map dispatch now crosses the Tool Call Module, which returns the
|
|
1259
|
+
* public ToolExecuteResult rather than leaking its private HTTP parse shape.
|
|
1260
|
+
* This adapter deliberately projects that result back to the splitter's
|
|
1261
|
+
* long-lived legacy input contract at the edge.
|
|
1243
1262
|
*/
|
|
1244
|
-
function legacyResultForBatchSplitter(
|
|
1245
|
-
execution
|
|
1246
|
-
): unknown {
|
|
1247
|
-
if (execution.toolResponse && 'raw' in execution.toolResponse) {
|
|
1248
|
-
return execution.toolResponse.raw;
|
|
1249
|
-
}
|
|
1250
|
-
return execution.result != null &&
|
|
1251
|
-
typeof execution.result === 'object' &&
|
|
1252
|
-
!Array.isArray(execution.result) &&
|
|
1253
|
-
'data' in execution.result
|
|
1254
|
-
? execution.result.data
|
|
1255
|
-
: execution.result;
|
|
1263
|
+
function legacyResultForBatchSplitter(execution: ToolExecuteResult): unknown {
|
|
1264
|
+
return execution.toolResponse.raw;
|
|
1256
1265
|
}
|
|
1257
1266
|
|
|
1258
1267
|
const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
|
|
@@ -4395,6 +4404,131 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4395
4404
|
);
|
|
4396
4405
|
}
|
|
4397
4406
|
|
|
4407
|
+
/**
|
|
4408
|
+
* Maps already own a richer receipt cohort than a single Tool Call: one
|
|
4409
|
+
* physical native batch can hold many row receipts, and the outer map job
|
|
4410
|
+
* publishes each split result atomically. Keep that ownership where it is,
|
|
4411
|
+
* but make the physical provider dispatch cross the same Tool Call boundary
|
|
4412
|
+
* as a direct invocation.
|
|
4413
|
+
*
|
|
4414
|
+
* The external receipt adapter is intentionally local to this migration.
|
|
4415
|
+
* It checks the exact compatibility identities passed by the map, while the
|
|
4416
|
+
* map remains the sole writer for its member receipts, follower delivery,
|
|
4417
|
+
* and completion barrier. In particular, do not turn an always-fresh map
|
|
4418
|
+
* request into a provider-idempotent request here: `apiOptions` is forwarded
|
|
4419
|
+
* byte-for-byte and decides whether the historical wire carried an identity.
|
|
4420
|
+
*/
|
|
4421
|
+
private async executeMappedToolCallThroughModule(input: {
|
|
4422
|
+
request: ToolCallRequest;
|
|
4423
|
+
dispatchToolId: string;
|
|
4424
|
+
dispatchPayload: Record<string, unknown>;
|
|
4425
|
+
/** The physical operation identity already selected by the map adapter. */
|
|
4426
|
+
providerOperationKey?: string;
|
|
4427
|
+
/** Present only when the physical call has a durable result receipt. */
|
|
4428
|
+
resultReceiptKey?: string;
|
|
4429
|
+
/**
|
|
4430
|
+
* The mapped receipt cohort owns the durable lifecycle for cacheable
|
|
4431
|
+
* scalar calls. Native batches and always-fresh calls deliberately leave
|
|
4432
|
+
* this unset and retain their existing outer compatibility adapter.
|
|
4433
|
+
*/
|
|
4434
|
+
resultReceipts?: ToolResultReceipts;
|
|
4435
|
+
apiOptions: ToolExecutionApiOptions;
|
|
4436
|
+
}): Promise<ToolExecuteResult> {
|
|
4437
|
+
const providerActionVersion =
|
|
4438
|
+
(
|
|
4439
|
+
await this.#options.getToolActionCacheVersion?.(input.dispatchToolId)
|
|
4440
|
+
)?.trim() ?? '';
|
|
4441
|
+
const alwaysFresh = !input.apiOptions.durableCallReceiptKey;
|
|
4442
|
+
const dispatcher = {
|
|
4443
|
+
dispatch: async ({
|
|
4444
|
+
providerOperationKey,
|
|
4445
|
+
}: {
|
|
4446
|
+
providerOperationKey: string | null;
|
|
4447
|
+
}) => {
|
|
4448
|
+
// ToolCall is allowed to transport the compatibility identity, never
|
|
4449
|
+
// derive a replacement one. A mismatch would mean a future adapter
|
|
4450
|
+
// accidentally sent a different operation than it recorded.
|
|
4451
|
+
if (
|
|
4452
|
+
!alwaysFresh &&
|
|
4453
|
+
providerOperationKey !== input.providerOperationKey
|
|
4454
|
+
) {
|
|
4455
|
+
throw new Error(
|
|
4456
|
+
'Map Tool Call operation identity changed in transit.',
|
|
4457
|
+
);
|
|
4458
|
+
}
|
|
4459
|
+
const execution = await this.callToolExecutionAPI(
|
|
4460
|
+
input.dispatchToolId,
|
|
4461
|
+
input.dispatchPayload,
|
|
4462
|
+
input.apiOptions,
|
|
4463
|
+
);
|
|
4464
|
+
return await this.wrapToolExecutionResult({
|
|
4465
|
+
toolId: input.dispatchToolId,
|
|
4466
|
+
status: execution.status,
|
|
4467
|
+
jobId: execution.jobId,
|
|
4468
|
+
result: execution.result,
|
|
4469
|
+
toolResponse: execution.toolResponse,
|
|
4470
|
+
metadata: execution.metadata,
|
|
4471
|
+
meta: execution.meta,
|
|
4472
|
+
requestInput: input.dispatchPayload,
|
|
4473
|
+
execution: toolExecutionMetadataForOutcome({
|
|
4474
|
+
kind: 'live',
|
|
4475
|
+
cacheKey: input.request.cacheKey,
|
|
4476
|
+
receiptKey: input.request.receiptKey,
|
|
4477
|
+
}),
|
|
4478
|
+
});
|
|
4479
|
+
},
|
|
4480
|
+
};
|
|
4481
|
+
const call: Omit<ToolCallInput, 'compatibility'> = {
|
|
4482
|
+
scope: {
|
|
4483
|
+
organizationId: this.#options.orgId ?? 'unknown-org',
|
|
4484
|
+
playScope: this.currentGovernance.currentPlayId,
|
|
4485
|
+
},
|
|
4486
|
+
tool: {
|
|
4487
|
+
id: input.dispatchToolId,
|
|
4488
|
+
providerActionVersion,
|
|
4489
|
+
},
|
|
4490
|
+
payload: input.dispatchPayload,
|
|
4491
|
+
authorization: {
|
|
4492
|
+
scopeDigest: input.request.executionAuthScopeDigest ?? null,
|
|
4493
|
+
},
|
|
4494
|
+
output: {
|
|
4495
|
+
intent: 'dataset',
|
|
4496
|
+
format:
|
|
4497
|
+
this.currentToolResponseContract === RAW_V2_TOOL_RESPONSE_CONTRACT
|
|
4498
|
+
? 'raw-v2'
|
|
4499
|
+
: 'v2',
|
|
4500
|
+
revision: this.currentToolResponseReceiptRevision ?? null,
|
|
4501
|
+
},
|
|
4502
|
+
execution: {
|
|
4503
|
+
mode: 'map',
|
|
4504
|
+
ownerRunId: this.currentReceiptOwnerRunId,
|
|
4505
|
+
force: input.request.force === true,
|
|
4506
|
+
},
|
|
4507
|
+
};
|
|
4508
|
+
if (alwaysFresh) {
|
|
4509
|
+
return await createAlwaysFreshToolCallAdapter({ dispatcher }).call(call);
|
|
4510
|
+
}
|
|
4511
|
+
if (!input.resultReceiptKey || !input.providerOperationKey) {
|
|
4512
|
+
throw new Error('Receipt-backed map Tool Call needs durable identities.');
|
|
4513
|
+
}
|
|
4514
|
+
return await createToolCallJob({
|
|
4515
|
+
resultReceipts:
|
|
4516
|
+
input.resultReceipts ??
|
|
4517
|
+
createExternallyOwnedToolResultReceipts({
|
|
4518
|
+
resultReceiptKey: input.resultReceiptKey,
|
|
4519
|
+
ownerRunId: this.currentReceiptOwnerRunId,
|
|
4520
|
+
leaseId: input.request.receiptLeaseId,
|
|
4521
|
+
}),
|
|
4522
|
+
dispatcher,
|
|
4523
|
+
}).call({
|
|
4524
|
+
...call,
|
|
4525
|
+
compatibility: {
|
|
4526
|
+
resultReceiptKey: input.resultReceiptKey,
|
|
4527
|
+
providerOperationKey: input.providerOperationKey,
|
|
4528
|
+
},
|
|
4529
|
+
});
|
|
4530
|
+
}
|
|
4531
|
+
|
|
4398
4532
|
/**
|
|
4399
4533
|
* Bounded per-cell producer trace (ADR 0018).
|
|
4400
4534
|
*
|
|
@@ -6825,7 +6959,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6825
6959
|
staticPipeline: this.currentStaticPipeline ?? null,
|
|
6826
6960
|
});
|
|
6827
6961
|
if (writeResult) {
|
|
6962
|
+
const recoveredStaleKeys = new Set(
|
|
6963
|
+
writeResult.recoveredStaleKeys ?? [],
|
|
6964
|
+
);
|
|
6828
6965
|
for (const key of writeResult.staleDroppedKeys ?? []) {
|
|
6966
|
+
if (recoveredStaleKeys.has(key)) continue;
|
|
6829
6967
|
staleCompletionKeys.add(key);
|
|
6830
6968
|
}
|
|
6831
6969
|
}
|
|
@@ -8979,13 +9117,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8979
9117
|
requestInput: input,
|
|
8980
9118
|
});
|
|
8981
9119
|
const store = rowContext.getStore();
|
|
8982
|
-
|
|
8983
|
-
|
|
8984
|
-
|
|
8985
|
-
|
|
8986
|
-
|
|
8987
|
-
|
|
8988
|
-
|
|
9120
|
+
const logicalCallScope = store
|
|
9121
|
+
? [
|
|
9122
|
+
'row',
|
|
9123
|
+
store.tableNamespace?.trim() || 'map',
|
|
9124
|
+
store.rowKey?.trim() || String(store.rowId),
|
|
9125
|
+
store.fieldName?.trim() || 'field',
|
|
9126
|
+
].join(':')
|
|
9127
|
+
: 'workflow';
|
|
9128
|
+
const callIndexKey = `${this.currentExecutionScope.receipt.namespace}:${logicalCallScope}:${normalizedKey}:${toolRequestIdentity}`;
|
|
9129
|
+
const callIndex = this.toolCallIndexByKey.get(callIndexKey) ?? 0;
|
|
9130
|
+
this.toolCallIndexByKey.set(callIndexKey, callIndex + 1);
|
|
9131
|
+
const logicalCallId = stableDigest(`${callIndexKey}:${callIndex}`);
|
|
8989
9132
|
let executionAuthScopeDigest =
|
|
8990
9133
|
(await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
|
|
8991
9134
|
const eventWaitHandler =
|
|
@@ -9037,6 +9180,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9037
9180
|
|
|
9038
9181
|
if (!store) {
|
|
9039
9182
|
const directCacheKey = durableCacheKey;
|
|
9183
|
+
let initialDirectReceipt: RuntimeStepReceipt | null | undefined;
|
|
9040
9184
|
const cached = !cacheableToolResult
|
|
9041
9185
|
? null
|
|
9042
9186
|
: toolCachePolicy.force
|
|
@@ -9079,6 +9223,35 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9079
9223
|
}
|
|
9080
9224
|
}
|
|
9081
9225
|
}
|
|
9226
|
+
// A completed historical receipt may predate the ToolExecuteResult
|
|
9227
|
+
// envelope. ToolCall deliberately speaks the current full-result ABI;
|
|
9228
|
+
// preserve those byte-compatible stored values at this one legacy
|
|
9229
|
+
// reader seam rather than teaching the deep Module an untyped result.
|
|
9230
|
+
// New direct calls use the real Tool Result Receipt Adapter below.
|
|
9231
|
+
if (cacheableToolResult && !toolCachePolicy.force) {
|
|
9232
|
+
const historicalReceipt = await this.withRuntimeReceiptHydrationTurn(
|
|
9233
|
+
async () => await this.getRuntimeStepReceipt(directCacheKey),
|
|
9234
|
+
);
|
|
9235
|
+
initialDirectReceipt = historicalReceipt;
|
|
9236
|
+
if (
|
|
9237
|
+
historicalReceipt &&
|
|
9238
|
+
(historicalReceipt.status === 'completed' ||
|
|
9239
|
+
historicalReceipt.status === 'skipped') &&
|
|
9240
|
+
!isSerializedToolExecuteResult(historicalReceipt.output) &&
|
|
9241
|
+
!isToolExecuteResult(historicalReceipt.output)
|
|
9242
|
+
) {
|
|
9243
|
+
const historicalOutput =
|
|
9244
|
+
this.runtimeReceiptOutput<unknown>(historicalReceipt);
|
|
9245
|
+
return markToolExecuteResultExecutionOutcome(
|
|
9246
|
+
historicalOutput,
|
|
9247
|
+
toolExecutionOutcomeForDurableReceipt({
|
|
9248
|
+
source: 'cache',
|
|
9249
|
+
receiptKey: directCacheKey,
|
|
9250
|
+
cacheKey: directCacheKey,
|
|
9251
|
+
}),
|
|
9252
|
+
);
|
|
9253
|
+
}
|
|
9254
|
+
}
|
|
9082
9255
|
this.log(
|
|
9083
9256
|
toolCachePolicy.force
|
|
9084
9257
|
? `Calling tool: ${toolId} (force)`
|
|
@@ -9120,71 +9293,182 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9120
9293
|
orgId: this.#options.orgId ?? 'unknown-org',
|
|
9121
9294
|
toolId,
|
|
9122
9295
|
})}${stableDigest(`${this.currentRunId}:${logicalCallId}:always-fresh`)}`;
|
|
9123
|
-
const
|
|
9124
|
-
|
|
9125
|
-
|
|
9126
|
-
|
|
9127
|
-
|
|
9128
|
-
|
|
9129
|
-
|
|
9130
|
-
|
|
9296
|
+
const providerOperationKey = providerIdempotencyKeyBase;
|
|
9297
|
+
const resultReceiptKey = directCacheKey;
|
|
9298
|
+
const toolRetryPolicy = await this.#options
|
|
9299
|
+
.getToolRetryPolicy?.(toolId, input)
|
|
9300
|
+
.catch(() => null);
|
|
9301
|
+
// Direct cacheable calls now give ToolCall the real completed-result
|
|
9302
|
+
// receipt Adapter. The provider-operation fence remains below the
|
|
9303
|
+
// dispatcher: V2 and legacy presentation keys can therefore publish
|
|
9304
|
+
// independent Tool Result Receipts without a second provider effect.
|
|
9305
|
+
// Map members retain their cohort-owned receipt Adapter until their
|
|
9306
|
+
// batched claim/heartbeat protocol moves behind the same Interface.
|
|
9307
|
+
const dispatcher = {
|
|
9308
|
+
dispatch: async ({
|
|
9309
|
+
providerOperationKey: operationKey,
|
|
9310
|
+
}: {
|
|
9311
|
+
providerOperationKey: string | null;
|
|
9312
|
+
}) => {
|
|
9313
|
+
if (
|
|
9314
|
+
cacheableToolResult &&
|
|
9315
|
+
operationKey !== providerOperationKey
|
|
9316
|
+
) {
|
|
9317
|
+
throw new Error(
|
|
9318
|
+
'Direct Tool Call operation identity changed in transit.',
|
|
9319
|
+
);
|
|
9320
|
+
}
|
|
9321
|
+
const execution = await this.executeDirectToolCall({
|
|
9131
9322
|
toolId,
|
|
9132
9323
|
callKey: normalizedKey,
|
|
9133
|
-
|
|
9134
|
-
|
|
9135
|
-
|
|
9136
|
-
|
|
9137
|
-
|
|
9138
|
-
|
|
9139
|
-
|
|
9140
|
-
|
|
9141
|
-
|
|
9142
|
-
|
|
9143
|
-
|
|
9144
|
-
|
|
9145
|
-
|
|
9146
|
-
|
|
9324
|
+
startedAt: directCallStartedAt,
|
|
9325
|
+
input,
|
|
9326
|
+
options: {
|
|
9327
|
+
...(cacheableToolResult
|
|
9328
|
+
? {
|
|
9329
|
+
durableCallReceiptKey: physicalDirectKey,
|
|
9330
|
+
executionAuthScopeDigest,
|
|
9331
|
+
providerIdempotencyReceiptKey: operationKey,
|
|
9332
|
+
providerIdempotencyKey:
|
|
9333
|
+
this.providerIdempotencyKeyForToolCall({
|
|
9334
|
+
cacheKey: providerIdempotencyKeyBase,
|
|
9335
|
+
force: toolCachePolicy.force,
|
|
9336
|
+
leaseId: directReceiptLeaseId,
|
|
9337
|
+
logicalCallId,
|
|
9338
|
+
}),
|
|
9339
|
+
...(directReceiptLeaseId &&
|
|
9340
|
+
(this.#options.heartbeatRuntimeStepReceipts ||
|
|
9341
|
+
this.#options.getRuntimeStepReceipt ||
|
|
9342
|
+
this.#options.getRuntimeStepReceipts)
|
|
9343
|
+
? {
|
|
9344
|
+
beforeProviderCall: async () => {
|
|
9345
|
+
await this.assertRuntimeToolReceiptOwnership([
|
|
9346
|
+
{
|
|
9347
|
+
callId: 'direct',
|
|
9348
|
+
cacheKey: directCacheKey,
|
|
9349
|
+
receiptKey: directCacheKey,
|
|
9350
|
+
receiptLeaseId: directReceiptLeaseId,
|
|
9351
|
+
rowId: -1,
|
|
9352
|
+
toolId,
|
|
9353
|
+
input,
|
|
9354
|
+
},
|
|
9355
|
+
]);
|
|
9356
|
+
},
|
|
9357
|
+
}
|
|
9358
|
+
: {}),
|
|
9359
|
+
}
|
|
9360
|
+
: {}),
|
|
9361
|
+
playNodeScope: buildPlayNodeScope({
|
|
9362
|
+
toolId,
|
|
9363
|
+
callKey: normalizedKey,
|
|
9364
|
+
}),
|
|
9365
|
+
timeoutMs: resolveToolRuntimeTimeoutMs(
|
|
9366
|
+
toolId,
|
|
9367
|
+
options?.timeoutMs,
|
|
9368
|
+
this.currentAuthoringContractEdition,
|
|
9369
|
+
),
|
|
9370
|
+
retainToolSlot: context?.retainExternalCallSlot,
|
|
9371
|
+
},
|
|
9372
|
+
});
|
|
9373
|
+
return await this.wrapToolExecutionResult({
|
|
9147
9374
|
toolId,
|
|
9148
|
-
|
|
9149
|
-
|
|
9150
|
-
|
|
9151
|
-
|
|
9152
|
-
|
|
9153
|
-
|
|
9154
|
-
|
|
9155
|
-
|
|
9156
|
-
|
|
9157
|
-
|
|
9158
|
-
|
|
9159
|
-
|
|
9160
|
-
|
|
9161
|
-
|
|
9162
|
-
|
|
9163
|
-
|
|
9164
|
-
|
|
9165
|
-
|
|
9166
|
-
|
|
9375
|
+
status: execution.status,
|
|
9376
|
+
jobId: execution.jobId,
|
|
9377
|
+
result: execution.result,
|
|
9378
|
+
toolResponse: execution.toolResponse,
|
|
9379
|
+
metadata: execution.metadata,
|
|
9380
|
+
meta: execution.meta,
|
|
9381
|
+
requestInput: input,
|
|
9382
|
+
execution: toolExecutionMetadataForOutcome({
|
|
9383
|
+
kind: 'live',
|
|
9384
|
+
cacheKey: directCacheKey,
|
|
9385
|
+
...(cacheableToolResult
|
|
9386
|
+
? { receiptKey: resultReceiptKey }
|
|
9387
|
+
: {}),
|
|
9388
|
+
}),
|
|
9389
|
+
});
|
|
9390
|
+
},
|
|
9391
|
+
};
|
|
9392
|
+
const toolCall = cacheableToolResult
|
|
9393
|
+
? createToolCallJob({
|
|
9394
|
+
resultReceipts: createRuntimeStepToolResultReceipts({
|
|
9395
|
+
store: this.durableReceiptExecutionStore(),
|
|
9396
|
+
owner: { runId: this.currentReceiptOwnerRunId },
|
|
9397
|
+
...(initialDirectReceipt !== undefined
|
|
9398
|
+
? {
|
|
9399
|
+
initialReceipt: {
|
|
9400
|
+
key: directCacheKey,
|
|
9401
|
+
receipt: initialDirectReceipt,
|
|
9167
9402
|
},
|
|
9168
|
-
|
|
9169
|
-
},
|
|
9170
|
-
|
|
9171
|
-
|
|
9403
|
+
}
|
|
9404
|
+
: {}),
|
|
9405
|
+
withCompletedReceiptHydration: (hydrate) =>
|
|
9406
|
+
this.withRuntimeReceiptHydrationTurn(hydrate),
|
|
9407
|
+
...(toolRetryPolicy?.requiresExecutionFence === true
|
|
9408
|
+
? {
|
|
9409
|
+
executionLock: {
|
|
9410
|
+
required: true,
|
|
9411
|
+
ttlMs: Math.min(
|
|
9412
|
+
600_000,
|
|
9413
|
+
resolveToolRuntimeTimeoutMs(
|
|
9414
|
+
toolId,
|
|
9415
|
+
options?.timeoutMs,
|
|
9416
|
+
this.currentAuthoringContractEdition,
|
|
9417
|
+
) ?? 300_000,
|
|
9418
|
+
),
|
|
9419
|
+
},
|
|
9420
|
+
}
|
|
9421
|
+
: {}),
|
|
9422
|
+
recover: ({ result, resultReceiptKey, source }) =>
|
|
9423
|
+
cloneToolExecuteResultWithExecution(
|
|
9424
|
+
result,
|
|
9425
|
+
toolExecutionMetadataForOutcome(
|
|
9426
|
+
toolExecutionOutcomeForDurableReceipt({
|
|
9427
|
+
source,
|
|
9428
|
+
receiptKey: resultReceiptKey,
|
|
9429
|
+
cacheKey: directCacheKey,
|
|
9430
|
+
}),
|
|
9431
|
+
),
|
|
9432
|
+
),
|
|
9433
|
+
}),
|
|
9434
|
+
dispatcher,
|
|
9435
|
+
})
|
|
9436
|
+
: null;
|
|
9437
|
+
const providerActionVersion =
|
|
9438
|
+
(await this.#options.getToolActionCacheVersion?.(toolId))?.trim() ??
|
|
9439
|
+
'';
|
|
9440
|
+
const call: Omit<ToolCallInput, 'compatibility'> = {
|
|
9441
|
+
scope: {
|
|
9442
|
+
organizationId: this.#options.orgId ?? 'unknown-org',
|
|
9443
|
+
playScope: this.currentGovernance.currentPlayId,
|
|
9172
9444
|
},
|
|
9173
|
-
|
|
9174
|
-
|
|
9175
|
-
|
|
9176
|
-
|
|
9177
|
-
|
|
9178
|
-
|
|
9179
|
-
|
|
9180
|
-
|
|
9181
|
-
|
|
9182
|
-
|
|
9183
|
-
|
|
9184
|
-
|
|
9185
|
-
|
|
9186
|
-
|
|
9187
|
-
|
|
9445
|
+
tool: { id: toolId, providerActionVersion },
|
|
9446
|
+
payload: input,
|
|
9447
|
+
authorization: { scopeDigest: executionAuthScopeDigest },
|
|
9448
|
+
output: {
|
|
9449
|
+
intent: 'dataset',
|
|
9450
|
+
format:
|
|
9451
|
+
this.currentToolResponseContract ===
|
|
9452
|
+
RAW_V2_TOOL_RESPONSE_CONTRACT
|
|
9453
|
+
? 'raw-v2'
|
|
9454
|
+
: 'v2',
|
|
9455
|
+
revision: this.currentToolResponseReceiptRevision ?? null,
|
|
9456
|
+
},
|
|
9457
|
+
execution: {
|
|
9458
|
+
mode: 'direct',
|
|
9459
|
+
ownerRunId: this.currentReceiptOwnerRunId,
|
|
9460
|
+
force: toolCachePolicy.force,
|
|
9461
|
+
},
|
|
9462
|
+
};
|
|
9463
|
+
const wrapped = toolCall
|
|
9464
|
+
? await toolCall.call({
|
|
9465
|
+
...call,
|
|
9466
|
+
compatibility: {
|
|
9467
|
+
resultReceiptKey,
|
|
9468
|
+
providerOperationKey,
|
|
9469
|
+
},
|
|
9470
|
+
})
|
|
9471
|
+
: await createAlwaysFreshToolCallAdapter({ dispatcher }).call(call);
|
|
9188
9472
|
settleDirectCall(toolAttemptOutcomeForResult(wrapped));
|
|
9189
9473
|
if (cacheableToolResult) {
|
|
9190
9474
|
this.cacheToolResult(toolId, directCacheKey, wrapped, 'direct');
|
|
@@ -9284,6 +9568,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9284
9568
|
);
|
|
9285
9569
|
this.enqueueToolCall({
|
|
9286
9570
|
callId,
|
|
9571
|
+
logicalCallId,
|
|
9287
9572
|
cacheKey: toolResultCacheKey,
|
|
9288
9573
|
providerIdempotencyKeyBase,
|
|
9289
9574
|
cacheable: cacheableToolResult,
|
|
@@ -9327,56 +9612,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9327
9612
|
authScopeAttempt += 1
|
|
9328
9613
|
) {
|
|
9329
9614
|
try {
|
|
9330
|
-
|
|
9331
|
-
.getToolRetryPolicy?.(toolId, input)
|
|
9332
|
-
.catch(() => null);
|
|
9333
|
-
return await this.executeWithRuntimeReceipt(
|
|
9334
|
-
'tool',
|
|
9335
|
-
normalizedKey,
|
|
9336
|
-
this.currentRunId,
|
|
9337
|
-
{
|
|
9338
|
-
receiptKey: durableCacheKey,
|
|
9339
|
-
semanticKey: toolRequestIdentity,
|
|
9340
|
-
force: toolCachePolicy.force,
|
|
9341
|
-
// Missing metadata is intentionally lock-free. Only explicitly
|
|
9342
|
-
// dangerous non-idempotent side effects enter the fence path.
|
|
9343
|
-
requiresExecutionLock:
|
|
9344
|
-
toolRetryPolicy?.requiresExecutionFence === true,
|
|
9345
|
-
executionLockTtlMs: Math.min(
|
|
9346
|
-
600_000,
|
|
9347
|
-
resolveToolRuntimeTimeoutMs(
|
|
9348
|
-
toolId,
|
|
9349
|
-
options?.timeoutMs,
|
|
9350
|
-
this.currentAuthoringContractEdition,
|
|
9351
|
-
) ?? 300_000,
|
|
9352
|
-
),
|
|
9353
|
-
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
9354
|
-
onClaimedResult: (output, receiptKey) =>
|
|
9355
|
-
markToolExecuteResultExecutionOutcome(output, {
|
|
9356
|
-
kind: 'live',
|
|
9357
|
-
receiptKey,
|
|
9358
|
-
}),
|
|
9359
|
-
onRecovered: (output, _receipt, source) =>
|
|
9360
|
-
markToolExecuteResultExecutionOutcome(
|
|
9361
|
-
output,
|
|
9362
|
-
toolExecutionOutcomeForDurableReceipt({
|
|
9363
|
-
source,
|
|
9364
|
-
receiptKey: durableCacheKey,
|
|
9365
|
-
}),
|
|
9366
|
-
),
|
|
9367
|
-
shouldPersistFailure: (error) =>
|
|
9368
|
-
!(error instanceof ToolExecuteAuthScopeChangedError),
|
|
9369
|
-
markRunningBeforeExecute: false,
|
|
9370
|
-
execute: ({ leaseId, retainExternalCallSlot }) =>
|
|
9371
|
-
executeTool({ leaseId, retainExternalCallSlot }),
|
|
9372
|
-
runningReceiptWaitMaxAttempts:
|
|
9373
|
-
resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
9374
|
-
options?.receiptWaitMs !== undefined
|
|
9375
|
-
? { max_wait_ms: options.receiptWaitMs }
|
|
9376
|
-
: input,
|
|
9377
|
-
),
|
|
9378
|
-
},
|
|
9379
|
-
);
|
|
9615
|
+
return await executeTool();
|
|
9380
9616
|
} catch (error) {
|
|
9381
9617
|
if (
|
|
9382
9618
|
!(error instanceof ToolExecuteAuthScopeChangedError) ||
|
|
@@ -10450,6 +10686,938 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10450
10686
|
};
|
|
10451
10687
|
}
|
|
10452
10688
|
|
|
10689
|
+
/**
|
|
10690
|
+
* Concrete RuntimeStepReceipt adapter for the map receipt-cohort Module.
|
|
10691
|
+
*
|
|
10692
|
+
* It is intentionally shared by scalar and provider-native map calls. The
|
|
10693
|
+
* two call shapes differ at provider dispatch, not at the member receipt
|
|
10694
|
+
* protocol: both need an atomic claim/reclaim, a single cohort heartbeat,
|
|
10695
|
+
* and lease-fenced terminal fanout. Keeping that at one seam prevents a
|
|
10696
|
+
* native batch from becoming a second, subtly different receipt runtime.
|
|
10697
|
+
*/
|
|
10698
|
+
private createRuntimeStepToolCallReceiptCohortStore(input: {
|
|
10699
|
+
toolId: string;
|
|
10700
|
+
requestForReceipt: ReadonlyMap<string, ToolCallRequest>;
|
|
10701
|
+
}): ToolCallReceiptCohortStore {
|
|
10702
|
+
const asRecord = async (
|
|
10703
|
+
receiptKey: string,
|
|
10704
|
+
receipt: RuntimeStepReceipt | null | undefined,
|
|
10705
|
+
): Promise<ToolCallReceiptRecord | null> => {
|
|
10706
|
+
if (!receipt) return null;
|
|
10707
|
+
const normalized = this.normalizeRuntimeStepReceipt(receiptKey, receipt);
|
|
10708
|
+
if (!normalized) return null;
|
|
10709
|
+
const request = input.requestForReceipt.get(receiptKey);
|
|
10710
|
+
if (!request) {
|
|
10711
|
+
throw new Error(
|
|
10712
|
+
`Unknown Tool Call Receipt Cohort receipt ${receiptKey}.`,
|
|
10713
|
+
);
|
|
10714
|
+
}
|
|
10715
|
+
const terminal =
|
|
10716
|
+
normalized.status === 'completed' || normalized.status === 'skipped';
|
|
10717
|
+
const result = terminal
|
|
10718
|
+
? await this.wrapToolExecutionResult({
|
|
10719
|
+
toolId: input.toolId,
|
|
10720
|
+
status:
|
|
10721
|
+
normalized.output === null || normalized.output === undefined
|
|
10722
|
+
? 'no_result'
|
|
10723
|
+
: 'completed',
|
|
10724
|
+
result: this.runtimeReceiptOutput(normalized),
|
|
10725
|
+
requestInput: request.input,
|
|
10726
|
+
execution: toolExecutionMetadataForOutcome({
|
|
10727
|
+
kind: 'cache',
|
|
10728
|
+
cacheKey: request.cacheKey,
|
|
10729
|
+
receiptKey,
|
|
10730
|
+
attachedToReceiptKey: receiptKey,
|
|
10731
|
+
}),
|
|
10732
|
+
})
|
|
10733
|
+
: undefined;
|
|
10734
|
+
return {
|
|
10735
|
+
resultReceiptKey: receiptKey,
|
|
10736
|
+
status: normalized.status,
|
|
10737
|
+
...(result !== undefined ? { result } : {}),
|
|
10738
|
+
...(normalized.status === 'failed'
|
|
10739
|
+
? {
|
|
10740
|
+
error: runtimeReceiptFailureError(
|
|
10741
|
+
normalized,
|
|
10742
|
+
'Durable tool call failed',
|
|
10743
|
+
this.currentToolErrorSchemaVersion,
|
|
10744
|
+
),
|
|
10745
|
+
}
|
|
10746
|
+
: {}),
|
|
10747
|
+
...(normalized.leaseId &&
|
|
10748
|
+
(normalized.leaseOwnerRunId ?? normalized.runId)?.trim()
|
|
10749
|
+
? {
|
|
10750
|
+
lease: {
|
|
10751
|
+
leaseId: normalized.leaseId,
|
|
10752
|
+
ownerRunId: (normalized.leaseOwnerRunId ??
|
|
10753
|
+
normalized.runId)!.trim(),
|
|
10754
|
+
},
|
|
10755
|
+
}
|
|
10756
|
+
: {}),
|
|
10757
|
+
...(normalized.leaseExpiresAt
|
|
10758
|
+
? { leaseExpiresAt: normalized.leaseExpiresAt }
|
|
10759
|
+
: {}),
|
|
10760
|
+
...(normalized.claimState ? { claimState: normalized.claimState } : {}),
|
|
10761
|
+
};
|
|
10762
|
+
};
|
|
10763
|
+
const recordsFor = async (
|
|
10764
|
+
receiptKeys: string[],
|
|
10765
|
+
receipts: Map<string, RuntimeStepReceipt>,
|
|
10766
|
+
): Promise<ToolCallReceiptRecord[]> =>
|
|
10767
|
+
(
|
|
10768
|
+
await Promise.all(
|
|
10769
|
+
receiptKeys.map(
|
|
10770
|
+
async (receiptKey) =>
|
|
10771
|
+
await asRecord(receiptKey, receipts.get(receiptKey)),
|
|
10772
|
+
),
|
|
10773
|
+
)
|
|
10774
|
+
).filter((record): record is ToolCallReceiptRecord => record !== null);
|
|
10775
|
+
|
|
10776
|
+
return {
|
|
10777
|
+
read: async ({ resultReceiptKeys }) =>
|
|
10778
|
+
await this.withRuntimeReceiptHydrationTurn(
|
|
10779
|
+
async () =>
|
|
10780
|
+
await recordsFor(
|
|
10781
|
+
resultReceiptKeys,
|
|
10782
|
+
await this.getRuntimeStepReceipts(resultReceiptKeys),
|
|
10783
|
+
),
|
|
10784
|
+
),
|
|
10785
|
+
claim: async ({ receipts }) => {
|
|
10786
|
+
const claims = new Map<string, RuntimeStepReceipt>();
|
|
10787
|
+
const groups = new Map<string, Array<(typeof receipts)[number]>>();
|
|
10788
|
+
for (const receipt of receipts) {
|
|
10789
|
+
const groupKey = [
|
|
10790
|
+
receipt.reclaimRunning ? 'reclaim' : 'ordinary',
|
|
10791
|
+
receipt.force ? 'force' : 'cached',
|
|
10792
|
+
receipt.forceFailedRefresh ? 'failed-force' : 'failed-cached',
|
|
10793
|
+
].join(':');
|
|
10794
|
+
const group = groups.get(groupKey) ?? [];
|
|
10795
|
+
group.push(receipt);
|
|
10796
|
+
groups.set(groupKey, group);
|
|
10797
|
+
}
|
|
10798
|
+
for (const group of groups.values()) {
|
|
10799
|
+
const first = group[0]!;
|
|
10800
|
+
const claimed = await this.claimRuntimeStepReceipts(
|
|
10801
|
+
group.map((receipt) => receipt.resultReceiptKey),
|
|
10802
|
+
this.currentReceiptOwnerRunId,
|
|
10803
|
+
first.reclaimRunning,
|
|
10804
|
+
first.force,
|
|
10805
|
+
first.forceFailedRefresh,
|
|
10806
|
+
);
|
|
10807
|
+
for (const [key, receipt] of claimed) claims.set(key, receipt);
|
|
10808
|
+
}
|
|
10809
|
+
return await recordsFor(
|
|
10810
|
+
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
10811
|
+
claims,
|
|
10812
|
+
);
|
|
10813
|
+
},
|
|
10814
|
+
heartbeat: async ({ receipts }) => {
|
|
10815
|
+
if (!this.#options.heartbeatRuntimeStepReceipts) {
|
|
10816
|
+
return await recordsFor(
|
|
10817
|
+
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
10818
|
+
await this.getRuntimeStepReceipts(
|
|
10819
|
+
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
10820
|
+
),
|
|
10821
|
+
);
|
|
10822
|
+
}
|
|
10823
|
+
const response = await this.#options.heartbeatRuntimeStepReceipts({
|
|
10824
|
+
runId: this.currentReceiptOwnerRunId,
|
|
10825
|
+
runAttempt: this.currentRunAttempt,
|
|
10826
|
+
keys: receipts.map((receipt) => receipt.resultReceiptKey),
|
|
10827
|
+
leaseIds: receipts.map((receipt) => receipt.lease.leaseId),
|
|
10828
|
+
});
|
|
10829
|
+
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
10830
|
+
for (let index = 0; index < receipts.length; index += 1) {
|
|
10831
|
+
const request = receipts[index]!;
|
|
10832
|
+
const normalized = this.normalizeRuntimeStepReceipt(
|
|
10833
|
+
request.resultReceiptKey,
|
|
10834
|
+
response[index],
|
|
10835
|
+
);
|
|
10836
|
+
if (normalized) byKey.set(request.resultReceiptKey, normalized);
|
|
10837
|
+
}
|
|
10838
|
+
return await recordsFor(
|
|
10839
|
+
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
10840
|
+
byKey,
|
|
10841
|
+
);
|
|
10842
|
+
},
|
|
10843
|
+
complete: async ({ receipts }) => {
|
|
10844
|
+
const completed = await this.completeRuntimeStepReceipts(
|
|
10845
|
+
receipts.map((receipt) => ({
|
|
10846
|
+
key: receipt.resultReceiptKey,
|
|
10847
|
+
runId: this.currentReceiptOwnerRunId,
|
|
10848
|
+
leaseId: receipt.lease.leaseId,
|
|
10849
|
+
output: serializeToolExecuteResult(receipt.result),
|
|
10850
|
+
})),
|
|
10851
|
+
);
|
|
10852
|
+
return await recordsFor(
|
|
10853
|
+
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
10854
|
+
completed,
|
|
10855
|
+
);
|
|
10856
|
+
},
|
|
10857
|
+
fail: async ({ receipts }) => {
|
|
10858
|
+
const failed = await this.failRuntimeStepReceipts(
|
|
10859
|
+
receipts.map((receipt) => ({
|
|
10860
|
+
key: receipt.resultReceiptKey,
|
|
10861
|
+
runId: this.currentReceiptOwnerRunId,
|
|
10862
|
+
leaseId: receipt.lease.leaseId,
|
|
10863
|
+
error: this.formatRuntimeError(receipt.error),
|
|
10864
|
+
errorPayload: serializeToolExecutionFailure(receipt.error),
|
|
10865
|
+
failureKind: runtimeReceiptFailureKindForError(receipt.error),
|
|
10866
|
+
})),
|
|
10867
|
+
);
|
|
10868
|
+
return await recordsFor(
|
|
10869
|
+
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
10870
|
+
failed,
|
|
10871
|
+
);
|
|
10872
|
+
},
|
|
10873
|
+
};
|
|
10874
|
+
}
|
|
10875
|
+
|
|
10876
|
+
/**
|
|
10877
|
+
* The scalar map path used to duplicate the receipt state machine in
|
|
10878
|
+
* `executeBatchedToolCalls`: one loop claimed/reclaimed/followed rows and a
|
|
10879
|
+
* later loop heartbeated and completed them. That made it too easy for the
|
|
10880
|
+
* Tool Call boundary to be only cosmetic. This is the narrow replacement
|
|
10881
|
+
* slice: cacheable members of a *non-native* map hand their whole receipt
|
|
10882
|
+
* lifecycle to the cohort Module, while this Context remains only the
|
|
10883
|
+
* transport/presentation Adapter.
|
|
10884
|
+
*
|
|
10885
|
+
* Do not widen this to physical native batches or always-fresh calls. A
|
|
10886
|
+
* native batch owns a different aggregate operation receipt, and
|
|
10887
|
+
* always-fresh calls intentionally have no durable member receipt.
|
|
10888
|
+
*/
|
|
10889
|
+
private async executeCacheableNonNativeMapToolCallCohort(input: {
|
|
10890
|
+
toolId: string;
|
|
10891
|
+
requests: ToolCallRequest[];
|
|
10892
|
+
}): Promise<ToolCallRequest[]> {
|
|
10893
|
+
const { toolId, requests } = input;
|
|
10894
|
+
if (requests.length === 0) return [];
|
|
10895
|
+
|
|
10896
|
+
const requestForReceipt = new Map<string, ToolCallRequest>();
|
|
10897
|
+
for (const request of requests) {
|
|
10898
|
+
const receiptKey = request.receiptKey?.trim();
|
|
10899
|
+
if (!receiptKey) {
|
|
10900
|
+
throw new Error(
|
|
10901
|
+
'Cacheable non-native map Tool Call cohort requires receipt keys.',
|
|
10902
|
+
);
|
|
10903
|
+
}
|
|
10904
|
+
requestForReceipt.set(receiptKey, request);
|
|
10905
|
+
}
|
|
10906
|
+
|
|
10907
|
+
const store = this.createRuntimeStepToolCallReceiptCohortStore({
|
|
10908
|
+
toolId,
|
|
10909
|
+
requestForReceipt,
|
|
10910
|
+
});
|
|
10911
|
+
|
|
10912
|
+
const cohort = await createToolCallReceiptCohortJob({ store }).acquire({
|
|
10913
|
+
owner: {
|
|
10914
|
+
runId: this.currentReceiptOwnerRunId,
|
|
10915
|
+
attempt: this.currentRunAttempt,
|
|
10916
|
+
},
|
|
10917
|
+
members: requests.map((request) => ({
|
|
10918
|
+
memberId: request.callId,
|
|
10919
|
+
resultReceiptKey: request.receiptKey!,
|
|
10920
|
+
force: request.force === true,
|
|
10921
|
+
forceFailedRefresh: request.forceFailedRefresh === true,
|
|
10922
|
+
waitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
10923
|
+
typeof request.receiptWaitMs === 'number'
|
|
10924
|
+
? { max_wait_ms: request.receiptWaitMs }
|
|
10925
|
+
: request.input,
|
|
10926
|
+
),
|
|
10927
|
+
})),
|
|
10928
|
+
// Claim immediately so unrelated owned rows can cross provider work
|
|
10929
|
+
// while foreign members hydrate in the bounded lane.
|
|
10930
|
+
eager: true,
|
|
10931
|
+
});
|
|
10932
|
+
|
|
10933
|
+
// Claims carry their actual expiry. Seed the cohort supervisor from that
|
|
10934
|
+
// lease rather than the generic fallback, otherwise a long provider-shaped
|
|
10935
|
+
// queue can let undispatched rows expire before its first renewal.
|
|
10936
|
+
for (const request of requests) {
|
|
10937
|
+
const member = cohort.member(request.callId);
|
|
10938
|
+
if (member.kind !== 'owned') continue;
|
|
10939
|
+
request.receiptLeaseId = member.lease.leaseId;
|
|
10940
|
+
request.receiptLeaseExpiresAt = member.leaseExpiresAt ?? null;
|
|
10941
|
+
}
|
|
10942
|
+
|
|
10943
|
+
const earliestLeaseExpiry = requests.reduce<string | null>(
|
|
10944
|
+
(earliest, request) => {
|
|
10945
|
+
const expiresAt = request.receiptLeaseExpiresAt ?? null;
|
|
10946
|
+
if (!expiresAt) return earliest;
|
|
10947
|
+
if (!earliest) return expiresAt;
|
|
10948
|
+
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
10949
|
+
? expiresAt
|
|
10950
|
+
: earliest;
|
|
10951
|
+
},
|
|
10952
|
+
null,
|
|
10953
|
+
);
|
|
10954
|
+
// One cohort-wide liveness check closes the claim-to-provider window; the
|
|
10955
|
+
// per-call views additionally preserve the historical exact admission
|
|
10956
|
+
// fence immediately before an individual provider request.
|
|
10957
|
+
await cohort.heartbeat();
|
|
10958
|
+
let cohortLeaseLost: unknown | null = null;
|
|
10959
|
+
const heartbeat = cohort.startHeartbeat({
|
|
10960
|
+
intervalMs: Math.max(
|
|
10961
|
+
1,
|
|
10962
|
+
Math.floor(
|
|
10963
|
+
runtimeLeaseHeartbeatIntervalFromExpiry({
|
|
10964
|
+
leaseExpiresAt: earliestLeaseExpiry,
|
|
10965
|
+
fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
10966
|
+
}) / 2,
|
|
10967
|
+
),
|
|
10968
|
+
),
|
|
10969
|
+
onLeaseLost: (error) => {
|
|
10970
|
+
cohortLeaseLost ??= error;
|
|
10971
|
+
},
|
|
10972
|
+
onTransientFailure: (error) => {
|
|
10973
|
+
this.log(
|
|
10974
|
+
`Tool Call Receipt Cohort heartbeat transport failed; retrying: ${
|
|
10975
|
+
error instanceof Error ? error.message : String(error)
|
|
10976
|
+
}`,
|
|
10977
|
+
);
|
|
10978
|
+
},
|
|
10979
|
+
});
|
|
10980
|
+
|
|
10981
|
+
const successful: ToolCallRequest[] = [];
|
|
10982
|
+
const resolve = async (
|
|
10983
|
+
request: ToolCallRequest,
|
|
10984
|
+
result: ToolExecuteResult,
|
|
10985
|
+
source: 'cache' | 'in_flight' | 'owner',
|
|
10986
|
+
): Promise<void> => {
|
|
10987
|
+
const receiptKey = request.receiptKey!;
|
|
10988
|
+
const finalResult = cloneToolExecuteResultWithExecution(
|
|
10989
|
+
result,
|
|
10990
|
+
toolExecutionMetadataForOutcome({
|
|
10991
|
+
kind: source === 'owner' ? 'live' : source,
|
|
10992
|
+
cacheKey: request.cacheKey,
|
|
10993
|
+
receiptKey,
|
|
10994
|
+
...(source === 'owner' ? {} : { attachedToReceiptKey: receiptKey }),
|
|
10995
|
+
}),
|
|
10996
|
+
);
|
|
10997
|
+
this.cacheToolResult(toolId, request.cacheKey, finalResult);
|
|
10998
|
+
const resolver = this.toolCallResolvers.get(request.callId);
|
|
10999
|
+
if (resolver) {
|
|
11000
|
+
resolver.resolve(finalResult);
|
|
11001
|
+
this.toolCallResolvers.delete(request.callId);
|
|
11002
|
+
}
|
|
11003
|
+
this.emitScopedFieldMetaUpdate({
|
|
11004
|
+
rowId: request.rowId,
|
|
11005
|
+
key: request.rowKey ?? null,
|
|
11006
|
+
tableNamespace: request.tableNamespace ?? null,
|
|
11007
|
+
fieldName: request.fieldName,
|
|
11008
|
+
status: source === 'owner' ? 'running' : 'cached',
|
|
11009
|
+
rowStatus: 'running',
|
|
11010
|
+
stage: toolId,
|
|
11011
|
+
provider: null,
|
|
11012
|
+
error: null,
|
|
11013
|
+
...(source === 'owner' ? {} : { reused: true }),
|
|
11014
|
+
dataPatch: {},
|
|
11015
|
+
});
|
|
11016
|
+
successful.push(request);
|
|
11017
|
+
};
|
|
11018
|
+
const reject = async (request: ToolCallRequest, error: unknown) => {
|
|
11019
|
+
await this.rejectToolCall(toolId, request, error, {
|
|
11020
|
+
// ToolCallReceiptCohort is the one failure writer. Re-entering the
|
|
11021
|
+
// legacy writer here would race its member terminal transition.
|
|
11022
|
+
persistReceiptFailure: false,
|
|
11023
|
+
});
|
|
11024
|
+
};
|
|
11025
|
+
|
|
11026
|
+
try {
|
|
11027
|
+
const handled = new Set<string>();
|
|
11028
|
+
const followOrRecover: Promise<void>[] = [];
|
|
11029
|
+
const collectReadyMembers = (): Array<{
|
|
11030
|
+
request: ToolCallRequest;
|
|
11031
|
+
receipts: ToolResultReceipts;
|
|
11032
|
+
}> => {
|
|
11033
|
+
const owned: Array<{
|
|
11034
|
+
request: ToolCallRequest;
|
|
11035
|
+
receipts: ToolResultReceipts;
|
|
11036
|
+
}> = [];
|
|
11037
|
+
for (const request of requests) {
|
|
11038
|
+
if (handled.has(request.callId)) continue;
|
|
11039
|
+
const member = cohort.member(request.callId);
|
|
11040
|
+
if (member.kind === 'recovering') continue;
|
|
11041
|
+
handled.add(request.callId);
|
|
11042
|
+
if (member.kind === 'completed') {
|
|
11043
|
+
followOrRecover.push(
|
|
11044
|
+
resolve(request, member.result, member.source),
|
|
11045
|
+
);
|
|
11046
|
+
} else if (member.kind === 'failed') {
|
|
11047
|
+
followOrRecover.push(reject(request, member.error));
|
|
11048
|
+
} else if (member.kind === 'following') {
|
|
11049
|
+
followOrRecover.push(
|
|
11050
|
+
member.result.then(
|
|
11051
|
+
async (result) => await resolve(request, result, 'in_flight'),
|
|
11052
|
+
async (error: unknown) => await reject(request, error),
|
|
11053
|
+
),
|
|
11054
|
+
);
|
|
11055
|
+
} else {
|
|
11056
|
+
request.receiptLeaseId = member.lease.leaseId;
|
|
11057
|
+
request.receiptLeaseExpiresAt = member.leaseExpiresAt ?? null;
|
|
11058
|
+
owned.push({ request, receipts: member.receipts });
|
|
11059
|
+
}
|
|
11060
|
+
}
|
|
11061
|
+
return owned;
|
|
11062
|
+
};
|
|
11063
|
+
|
|
11064
|
+
const width = Math.min(
|
|
11065
|
+
this.governor.policy.concurrency.toolCalls,
|
|
11066
|
+
Math.max(
|
|
11067
|
+
1,
|
|
11068
|
+
this.fixtureProviderPacingDisabled()
|
|
11069
|
+
? this.governor.policy.concurrency.toolCalls
|
|
11070
|
+
: await this.resourceGovernor.suggestedToolParallelism(
|
|
11071
|
+
toolId,
|
|
11072
|
+
this.governor.policy.concurrency.toolCalls,
|
|
11073
|
+
),
|
|
11074
|
+
),
|
|
11075
|
+
);
|
|
11076
|
+
const dispatchOwned = async (
|
|
11077
|
+
owned: Array<{
|
|
11078
|
+
request: ToolCallRequest;
|
|
11079
|
+
receipts: ToolResultReceipts;
|
|
11080
|
+
}>,
|
|
11081
|
+
) =>
|
|
11082
|
+
await dispatchBoundedSettled(
|
|
11083
|
+
owned,
|
|
11084
|
+
width,
|
|
11085
|
+
async ({ request, receipts }) => {
|
|
11086
|
+
if (this.persistenceLatch.tripped) {
|
|
11087
|
+
const error = new RuntimePersistenceCircuitOpenError(
|
|
11088
|
+
this.persistenceLatch,
|
|
11089
|
+
);
|
|
11090
|
+
await cohort.fail([{ memberId: request.callId, error }]);
|
|
11091
|
+
throw error;
|
|
11092
|
+
}
|
|
11093
|
+
let releaseToolSlot: () => void = () => undefined;
|
|
11094
|
+
try {
|
|
11095
|
+
const result = await this.executeMappedToolCallThroughModule({
|
|
11096
|
+
request,
|
|
11097
|
+
dispatchToolId: toolId,
|
|
11098
|
+
dispatchPayload: request.input,
|
|
11099
|
+
resultReceiptKey: request.receiptKey!,
|
|
11100
|
+
providerOperationKey:
|
|
11101
|
+
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
11102
|
+
resultReceipts: receipts,
|
|
11103
|
+
apiOptions: {
|
|
11104
|
+
beforeProviderCall: () => {
|
|
11105
|
+
if (cohortLeaseLost) throw cohortLeaseLost;
|
|
11106
|
+
heartbeat.assertOwned();
|
|
11107
|
+
},
|
|
11108
|
+
heartbeatReceipt: async () => {
|
|
11109
|
+
try {
|
|
11110
|
+
const outcome = await cohort.heartbeat();
|
|
11111
|
+
if (outcome === 'terminal') {
|
|
11112
|
+
throw new ToolCallReceiptLeaseLostError(
|
|
11113
|
+
request.receiptKey!,
|
|
11114
|
+
);
|
|
11115
|
+
}
|
|
11116
|
+
} catch (error) {
|
|
11117
|
+
if (error instanceof ToolCallReceiptLeaseLostError) {
|
|
11118
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
11119
|
+
receiptKey: request.receiptKey!,
|
|
11120
|
+
runId: this.currentReceiptOwnerRunId,
|
|
11121
|
+
leaseId:
|
|
11122
|
+
request.receiptLeaseId ?? 'missing-cohort-lease',
|
|
11123
|
+
});
|
|
11124
|
+
}
|
|
11125
|
+
throw error;
|
|
11126
|
+
}
|
|
11127
|
+
},
|
|
11128
|
+
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
11129
|
+
durableCallReceiptKey: request.receiptKey,
|
|
11130
|
+
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
11131
|
+
providerIdempotencyReceiptKey:
|
|
11132
|
+
request.providerIdempotencyKeyBase ?? request.receiptKey,
|
|
11133
|
+
providerIdempotencyKey:
|
|
11134
|
+
this.providerIdempotencyKeyForToolCall({
|
|
11135
|
+
cacheKey:
|
|
11136
|
+
request.providerIdempotencyKeyBase ??
|
|
11137
|
+
request.receiptKey!,
|
|
11138
|
+
force: request.force === true,
|
|
11139
|
+
leaseId: request.receiptLeaseId,
|
|
11140
|
+
logicalCallId: request.logicalCallId,
|
|
11141
|
+
}),
|
|
11142
|
+
receiptLeaseExpiresAt: request.receiptLeaseExpiresAt,
|
|
11143
|
+
timeoutMs:
|
|
11144
|
+
typeof request.timeoutMs === 'number' &&
|
|
11145
|
+
request.timeoutMs > 0
|
|
11146
|
+
? request.timeoutMs
|
|
11147
|
+
: undefined,
|
|
11148
|
+
retainToolSlot: (release) => {
|
|
11149
|
+
releaseToolSlot = release;
|
|
11150
|
+
},
|
|
11151
|
+
},
|
|
11152
|
+
});
|
|
11153
|
+
await resolve(request, result, 'owner');
|
|
11154
|
+
} catch (error) {
|
|
11155
|
+
await reject(request, error);
|
|
11156
|
+
} finally {
|
|
11157
|
+
releaseToolSlot();
|
|
11158
|
+
}
|
|
11159
|
+
},
|
|
11160
|
+
);
|
|
11161
|
+
|
|
11162
|
+
// Eager cohort acquisition makes the initial claim available before a
|
|
11163
|
+
// foreign receipt's read/reclaim path completes. This preserves map
|
|
11164
|
+
// overlap: a broken foreign read must not strand an already-owned
|
|
11165
|
+
// provider operation behind it.
|
|
11166
|
+
const initialDispatch = dispatchOwned(collectReadyMembers());
|
|
11167
|
+
const settlement = cohort.settle();
|
|
11168
|
+
const [initialResult, settlementResult] = await Promise.allSettled([
|
|
11169
|
+
initialDispatch,
|
|
11170
|
+
settlement,
|
|
11171
|
+
]);
|
|
11172
|
+
// A recovery can reclaim an expired receipt and turn it into another
|
|
11173
|
+
// owned member. Collect its final state after the initial work settles.
|
|
11174
|
+
const recoveredDispatch = await dispatchOwned(collectReadyMembers());
|
|
11175
|
+
await Promise.allSettled(followOrRecover);
|
|
11176
|
+
// dispatchBoundedSettled intentionally captures provider failures; all
|
|
11177
|
+
// affected row resolvers have already been rejected above.
|
|
11178
|
+
void initialResult;
|
|
11179
|
+
void recoveredDispatch;
|
|
11180
|
+
if (settlementResult.status === 'rejected') {
|
|
11181
|
+
throw settlementResult.reason;
|
|
11182
|
+
}
|
|
11183
|
+
} finally {
|
|
11184
|
+
heartbeat.stop();
|
|
11185
|
+
}
|
|
11186
|
+
return successful;
|
|
11187
|
+
}
|
|
11188
|
+
|
|
11189
|
+
/**
|
|
11190
|
+
* Executes provider-native batches through the Native Batch Tool Call
|
|
11191
|
+
* Module. Unlike the compatibility path below, the Module owns the member
|
|
11192
|
+
* receipt cohort from claim through recovery, heartbeat and terminal split
|
|
11193
|
+
* fanout. This method is only the runtime Adapter: it compiles provider
|
|
11194
|
+
* payloads, invokes transport, and presents already-terminal member values
|
|
11195
|
+
* to map rows.
|
|
11196
|
+
*/
|
|
11197
|
+
private async executeCacheableNativeMapToolCallCohort(input: {
|
|
11198
|
+
toolId: string;
|
|
11199
|
+
requests: ToolCallRequest[];
|
|
11200
|
+
strategy: AnyBatchOperationStrategy;
|
|
11201
|
+
}): Promise<ToolCallRequest[]> {
|
|
11202
|
+
const { toolId, requests, strategy } = input;
|
|
11203
|
+
if (requests.length === 0) return [];
|
|
11204
|
+
|
|
11205
|
+
const requestForReceipt = new Map<string, ToolCallRequest>();
|
|
11206
|
+
const requestByCallId = new Map<string, ToolCallRequest>();
|
|
11207
|
+
for (const request of requests) {
|
|
11208
|
+
const receiptKey = request.receiptKey?.trim();
|
|
11209
|
+
if (!receiptKey) {
|
|
11210
|
+
throw new Error(
|
|
11211
|
+
'Cacheable native map Tool Call cohort requires receipt keys.',
|
|
11212
|
+
);
|
|
11213
|
+
}
|
|
11214
|
+
requestForReceipt.set(receiptKey, request);
|
|
11215
|
+
requestByCallId.set(request.callId, request);
|
|
11216
|
+
}
|
|
11217
|
+
const store = this.createRuntimeStepToolCallReceiptCohortStore({
|
|
11218
|
+
toolId,
|
|
11219
|
+
requestForReceipt,
|
|
11220
|
+
});
|
|
11221
|
+
const compiledBatches = compileRequestsWithStrategy({
|
|
11222
|
+
requests,
|
|
11223
|
+
strategy,
|
|
11224
|
+
getPayload: (request: ToolCallRequest) => request.input,
|
|
11225
|
+
});
|
|
11226
|
+
const batchParallelismCeiling =
|
|
11227
|
+
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
11228
|
+
const batchSize =
|
|
11229
|
+
compiledBatches.length > 0 && !this.fixtureProviderPacingDisabled()
|
|
11230
|
+
? await this.resourceGovernor.suggestedToolParallelism(
|
|
11231
|
+
compiledBatches[0]!.batchOperation,
|
|
11232
|
+
batchParallelismCeiling,
|
|
11233
|
+
)
|
|
11234
|
+
: batchParallelismCeiling;
|
|
11235
|
+
const successful: ToolCallRequest[] = [];
|
|
11236
|
+
|
|
11237
|
+
const deliver = (request: ToolCallRequest, result: ToolExecuteResult) => {
|
|
11238
|
+
if (request.cacheable !== false) {
|
|
11239
|
+
this.cacheToolResult(toolId, request.cacheKey, result);
|
|
11240
|
+
}
|
|
11241
|
+
const resolver = this.toolCallResolvers.get(request.callId);
|
|
11242
|
+
if (resolver) {
|
|
11243
|
+
resolver.resolve(result);
|
|
11244
|
+
this.toolCallResolvers.delete(request.callId);
|
|
11245
|
+
}
|
|
11246
|
+
this.emitScopedFieldMetaUpdate({
|
|
11247
|
+
rowId: request.rowId,
|
|
11248
|
+
key: request.rowKey ?? null,
|
|
11249
|
+
tableNamespace: request.tableNamespace ?? null,
|
|
11250
|
+
fieldName: request.fieldName,
|
|
11251
|
+
status:
|
|
11252
|
+
result._metadata.execution.cached === true ? 'cached' : 'running',
|
|
11253
|
+
rowStatus: 'running',
|
|
11254
|
+
stage: toolId,
|
|
11255
|
+
provider: null,
|
|
11256
|
+
error: null,
|
|
11257
|
+
...(result._metadata.execution.cached === true ? { reused: true } : {}),
|
|
11258
|
+
dataPatch: {},
|
|
11259
|
+
});
|
|
11260
|
+
successful.push(request);
|
|
11261
|
+
};
|
|
11262
|
+
const reject = async (request: ToolCallRequest, error: unknown) => {
|
|
11263
|
+
await this.rejectToolCall(toolId, request, error, {
|
|
11264
|
+
// NativeBatchToolCall owns the durable member failure transition.
|
|
11265
|
+
persistReceiptFailure: false,
|
|
11266
|
+
});
|
|
11267
|
+
};
|
|
11268
|
+
|
|
11269
|
+
await executeChunkedRequests({
|
|
11270
|
+
requests: compiledBatches,
|
|
11271
|
+
batchSize,
|
|
11272
|
+
execute: async (batch) => {
|
|
11273
|
+
if (this.persistenceLatch.tripped) {
|
|
11274
|
+
this.persistenceLatch.preventedCallCount +=
|
|
11275
|
+
batch.memberRequests.length;
|
|
11276
|
+
throw new RuntimePersistenceCircuitOpenError(this.persistenceLatch);
|
|
11277
|
+
}
|
|
11278
|
+
const batchRequestByCallId = new Map(
|
|
11279
|
+
batch.memberRequests.map((request) => [request.callId, request]),
|
|
11280
|
+
);
|
|
11281
|
+
const dispatchDetails = new Map<
|
|
11282
|
+
string,
|
|
11283
|
+
{
|
|
11284
|
+
members: ToolCallRequest[];
|
|
11285
|
+
providerOperationKey: string;
|
|
11286
|
+
receiptLeaseExpiresAt: string | null;
|
|
11287
|
+
timeoutMs: number | undefined;
|
|
11288
|
+
}
|
|
11289
|
+
>();
|
|
11290
|
+
const physicalToolCall = {
|
|
11291
|
+
call: async (
|
|
11292
|
+
physical: Parameters<
|
|
11293
|
+
ReturnType<typeof createToolCallJob>['call']
|
|
11294
|
+
>[0],
|
|
11295
|
+
) => {
|
|
11296
|
+
const compatibility = physical.compatibility;
|
|
11297
|
+
if (!compatibility) {
|
|
11298
|
+
throw new Error(
|
|
11299
|
+
'Native batch physical Tool Call requires compatibility identities.',
|
|
11300
|
+
);
|
|
11301
|
+
}
|
|
11302
|
+
const detail = dispatchDetails.get(compatibility.resultReceiptKey);
|
|
11303
|
+
if (!detail) {
|
|
11304
|
+
throw new Error(
|
|
11305
|
+
'Native batch physical Tool Call lost its dispatch details.',
|
|
11306
|
+
);
|
|
11307
|
+
}
|
|
11308
|
+
return await createToolCallJob({
|
|
11309
|
+
resultReceipts: createExternallyOwnedToolResultReceipts({
|
|
11310
|
+
resultReceiptKey: compatibility.resultReceiptKey,
|
|
11311
|
+
ownerRunId: this.currentReceiptOwnerRunId,
|
|
11312
|
+
}),
|
|
11313
|
+
dispatcher: {
|
|
11314
|
+
dispatch: async ({ call, providerOperationKey }) => {
|
|
11315
|
+
if (providerOperationKey !== detail.providerOperationKey) {
|
|
11316
|
+
throw new Error(
|
|
11317
|
+
'Native batch Tool Call operation identity changed in transit.',
|
|
11318
|
+
);
|
|
11319
|
+
}
|
|
11320
|
+
const execution = await this.callToolExecutionAPI(
|
|
11321
|
+
call.tool.id,
|
|
11322
|
+
call.payload,
|
|
11323
|
+
{
|
|
11324
|
+
durableCallReceiptKey: compatibility.resultReceiptKey,
|
|
11325
|
+
playNodeScope: playNodeScopeForToolCallRequest(
|
|
11326
|
+
detail.members[0]!,
|
|
11327
|
+
call.tool.id,
|
|
11328
|
+
),
|
|
11329
|
+
executionAuthScopeDigest:
|
|
11330
|
+
detail.members[0]?.executionAuthScopeDigest ?? null,
|
|
11331
|
+
providerIdempotencyReceiptKey:
|
|
11332
|
+
detail.providerOperationKey,
|
|
11333
|
+
providerIdempotencyKey:
|
|
11334
|
+
buildDurableToolAggregateProviderIdempotencyKey({
|
|
11335
|
+
aggregateReceiptKey: detail.providerOperationKey,
|
|
11336
|
+
receiptKeys: detail.members.map(
|
|
11337
|
+
(member) =>
|
|
11338
|
+
member.providerIdempotencyKeyBase ??
|
|
11339
|
+
member.receiptKey!,
|
|
11340
|
+
),
|
|
11341
|
+
providerIdempotencyKeys: detail.members.map(
|
|
11342
|
+
(member) =>
|
|
11343
|
+
this.providerIdempotencyKeyForToolCall({
|
|
11344
|
+
cacheKey:
|
|
11345
|
+
member.providerIdempotencyKeyBase ??
|
|
11346
|
+
member.receiptKey!,
|
|
11347
|
+
force: member.force === true,
|
|
11348
|
+
leaseId: member.receiptLeaseId,
|
|
11349
|
+
logicalCallId: member.logicalCallId,
|
|
11350
|
+
}),
|
|
11351
|
+
),
|
|
11352
|
+
}),
|
|
11353
|
+
receiptLeaseExpiresAt: detail.receiptLeaseExpiresAt,
|
|
11354
|
+
timeoutMs: detail.timeoutMs,
|
|
11355
|
+
},
|
|
11356
|
+
);
|
|
11357
|
+
return await this.wrapToolExecutionResult({
|
|
11358
|
+
toolId: call.tool.id,
|
|
11359
|
+
status: execution.status,
|
|
11360
|
+
jobId: execution.jobId,
|
|
11361
|
+
result: execution.result,
|
|
11362
|
+
toolResponse: execution.toolResponse,
|
|
11363
|
+
metadata: execution.metadata,
|
|
11364
|
+
meta: execution.meta,
|
|
11365
|
+
requestInput: call.payload,
|
|
11366
|
+
execution: toolExecutionMetadataForOutcome({
|
|
11367
|
+
kind: 'live',
|
|
11368
|
+
cacheKey: detail.members[0]!.cacheKey,
|
|
11369
|
+
receiptKey: compatibility.resultReceiptKey,
|
|
11370
|
+
}),
|
|
11371
|
+
});
|
|
11372
|
+
},
|
|
11373
|
+
},
|
|
11374
|
+
}).call(physical);
|
|
11375
|
+
},
|
|
11376
|
+
};
|
|
11377
|
+
const job = createNativeBatchToolCallJob({
|
|
11378
|
+
receiptCohort: createToolCallReceiptCohortJob({ store }),
|
|
11379
|
+
physicalToolCall,
|
|
11380
|
+
});
|
|
11381
|
+
const result = await job.execute({
|
|
11382
|
+
owner: {
|
|
11383
|
+
runId: this.currentReceiptOwnerRunId,
|
|
11384
|
+
attempt: this.currentRunAttempt,
|
|
11385
|
+
},
|
|
11386
|
+
members: batch.memberRequests.map((request) => ({
|
|
11387
|
+
memberId: request.callId,
|
|
11388
|
+
forceFailedRefresh: request.forceFailedRefresh === true,
|
|
11389
|
+
waitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
11390
|
+
typeof request.receiptWaitMs === 'number'
|
|
11391
|
+
? { max_wait_ms: request.receiptWaitMs }
|
|
11392
|
+
: request.input,
|
|
11393
|
+
),
|
|
11394
|
+
call: {
|
|
11395
|
+
scope: {
|
|
11396
|
+
organizationId: this.#options.orgId ?? 'unknown-org',
|
|
11397
|
+
playScope: this.currentGovernance.currentPlayId,
|
|
11398
|
+
},
|
|
11399
|
+
tool: { id: toolId, providerActionVersion: '' },
|
|
11400
|
+
payload: request.input,
|
|
11401
|
+
authorization: {
|
|
11402
|
+
scopeDigest: request.executionAuthScopeDigest ?? null,
|
|
11403
|
+
},
|
|
11404
|
+
output: {
|
|
11405
|
+
intent: 'dataset',
|
|
11406
|
+
format:
|
|
11407
|
+
this.currentToolResponseContract ===
|
|
11408
|
+
RAW_V2_TOOL_RESPONSE_CONTRACT
|
|
11409
|
+
? 'raw-v2'
|
|
11410
|
+
: 'v2',
|
|
11411
|
+
revision: this.currentToolResponseReceiptRevision ?? null,
|
|
11412
|
+
},
|
|
11413
|
+
execution: {
|
|
11414
|
+
mode: 'map',
|
|
11415
|
+
ownerRunId: this.currentReceiptOwnerRunId,
|
|
11416
|
+
force: request.force === true,
|
|
11417
|
+
},
|
|
11418
|
+
compatibility: {
|
|
11419
|
+
resultReceiptKey: request.receiptKey!,
|
|
11420
|
+
providerOperationKey:
|
|
11421
|
+
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
11422
|
+
},
|
|
11423
|
+
},
|
|
11424
|
+
})),
|
|
11425
|
+
buildPhysicalCall: ({ members }) => {
|
|
11426
|
+
const owned = members.map(
|
|
11427
|
+
(member) => batchRequestByCallId.get(member.memberId)!,
|
|
11428
|
+
);
|
|
11429
|
+
const receiptKeys = owned.map((request) => request.receiptKey!);
|
|
11430
|
+
const providerReceiptKeys = owned.map(
|
|
11431
|
+
(request) =>
|
|
11432
|
+
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
11433
|
+
);
|
|
11434
|
+
const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({
|
|
11435
|
+
receiptKeys,
|
|
11436
|
+
prefix: 'batch',
|
|
11437
|
+
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
11438
|
+
orgId: this.#options.orgId,
|
|
11439
|
+
toolId: batch.batchOperation,
|
|
11440
|
+
}),
|
|
11441
|
+
});
|
|
11442
|
+
const providerOperationKey = buildDurableToolAggregateReceiptKey({
|
|
11443
|
+
receiptKeys: providerReceiptKeys,
|
|
11444
|
+
prefix: 'batch',
|
|
11445
|
+
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
11446
|
+
orgId: this.#options.orgId,
|
|
11447
|
+
toolId: batch.batchOperation,
|
|
11448
|
+
}),
|
|
11449
|
+
});
|
|
11450
|
+
dispatchDetails.set(aggregateReceiptKey, {
|
|
11451
|
+
members: owned,
|
|
11452
|
+
providerOperationKey,
|
|
11453
|
+
receiptLeaseExpiresAt: owned.reduce<string | null>(
|
|
11454
|
+
(earliest, request) => {
|
|
11455
|
+
const expiresAt = request.receiptLeaseExpiresAt ?? null;
|
|
11456
|
+
if (!expiresAt) return earliest;
|
|
11457
|
+
if (!earliest) return expiresAt;
|
|
11458
|
+
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
11459
|
+
? expiresAt
|
|
11460
|
+
: earliest;
|
|
11461
|
+
},
|
|
11462
|
+
null,
|
|
11463
|
+
),
|
|
11464
|
+
timeoutMs: owned.every(
|
|
11465
|
+
(request) =>
|
|
11466
|
+
typeof request.timeoutMs === 'number' &&
|
|
11467
|
+
Number.isFinite(request.timeoutMs) &&
|
|
11468
|
+
request.timeoutMs > 0,
|
|
11469
|
+
)
|
|
11470
|
+
? Math.max(...owned.map((request) => request.timeoutMs!))
|
|
11471
|
+
: undefined,
|
|
11472
|
+
});
|
|
11473
|
+
return {
|
|
11474
|
+
...members[0]!.call,
|
|
11475
|
+
tool: { id: batch.batchOperation, providerActionVersion: '' },
|
|
11476
|
+
payload: batch.batchPayload,
|
|
11477
|
+
compatibility: {
|
|
11478
|
+
resultReceiptKey: aggregateReceiptKey,
|
|
11479
|
+
providerOperationKey,
|
|
11480
|
+
},
|
|
11481
|
+
};
|
|
11482
|
+
},
|
|
11483
|
+
splitResult: async ({ result: aggregate, members }) => {
|
|
11484
|
+
const split = batch.splitResults(
|
|
11485
|
+
legacyResultForBatchSplitter(aggregate),
|
|
11486
|
+
);
|
|
11487
|
+
return await Promise.all(
|
|
11488
|
+
members.map(async (member, index) => {
|
|
11489
|
+
const request = batchRequestByCallId.get(member.memberId);
|
|
11490
|
+
if (!request) {
|
|
11491
|
+
throw new Error(
|
|
11492
|
+
`Native batch Tool Call lost member ${member.memberId}.`,
|
|
11493
|
+
);
|
|
11494
|
+
}
|
|
11495
|
+
const item = split[index] ?? null;
|
|
11496
|
+
return {
|
|
11497
|
+
memberId: member.memberId,
|
|
11498
|
+
result: await this.wrapToolExecutionResult({
|
|
11499
|
+
toolId,
|
|
11500
|
+
status: aggregate.status,
|
|
11501
|
+
result: item,
|
|
11502
|
+
toolResponse: publicToolResponseForBatchedItem(
|
|
11503
|
+
aggregate,
|
|
11504
|
+
item,
|
|
11505
|
+
this.currentToolResponseContract ===
|
|
11506
|
+
RAW_V2_TOOL_RESPONSE_CONTRACT,
|
|
11507
|
+
),
|
|
11508
|
+
requestInput: request.input,
|
|
11509
|
+
execution: toolExecutionMetadataForOutcome({
|
|
11510
|
+
kind: 'live',
|
|
11511
|
+
cacheKey: request.cacheKey,
|
|
11512
|
+
receiptKey: request.receiptKey,
|
|
11513
|
+
}),
|
|
11514
|
+
}),
|
|
11515
|
+
};
|
|
11516
|
+
}),
|
|
11517
|
+
);
|
|
11518
|
+
},
|
|
11519
|
+
heartbeat: {
|
|
11520
|
+
// Claim replies carry the real expiry. A fixed fallback interval
|
|
11521
|
+
// can be too slow for a short-lived gateway lease, so derive the
|
|
11522
|
+
// one cohort supervisor from the earliest owned member instead.
|
|
11523
|
+
intervalMs: ({ members }) =>
|
|
11524
|
+
Math.max(
|
|
11525
|
+
1,
|
|
11526
|
+
Math.floor(
|
|
11527
|
+
runtimeLeaseHeartbeatIntervalFromExpiry({
|
|
11528
|
+
leaseExpiresAt: members.reduce<string | null>(
|
|
11529
|
+
(earliest, member) => {
|
|
11530
|
+
const expiresAt = member.leaseExpiresAt ?? null;
|
|
11531
|
+
if (!expiresAt) return earliest;
|
|
11532
|
+
if (!earliest) return expiresAt;
|
|
11533
|
+
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
11534
|
+
? expiresAt
|
|
11535
|
+
: earliest;
|
|
11536
|
+
},
|
|
11537
|
+
null,
|
|
11538
|
+
),
|
|
11539
|
+
fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
11540
|
+
}) / 2,
|
|
11541
|
+
),
|
|
11542
|
+
),
|
|
11543
|
+
onTransientFailure: (error) => {
|
|
11544
|
+
this.log(
|
|
11545
|
+
`Native Tool Call Receipt Cohort heartbeat transport failed; retrying: ${
|
|
11546
|
+
error instanceof Error ? error.message : String(error)
|
|
11547
|
+
}`,
|
|
11548
|
+
);
|
|
11549
|
+
},
|
|
11550
|
+
},
|
|
11551
|
+
});
|
|
11552
|
+
return { batch, result };
|
|
11553
|
+
},
|
|
11554
|
+
onChunkComplete: async (entries) => {
|
|
11555
|
+
for (const entry of entries) {
|
|
11556
|
+
if (entry.error !== undefined) {
|
|
11557
|
+
// The cohort may have recovered each member through its
|
|
11558
|
+
// lease-fenced single-receipt fallback, but that only prevents
|
|
11559
|
+
// duplicate provider work. It must not convert a failed bulk
|
|
11560
|
+
// persistence boundary into a successful map result: callers need
|
|
11561
|
+
// the loud failure so this run can be retried from the durable
|
|
11562
|
+
// receipts instead of continuing after an ambiguous bulk write.
|
|
11563
|
+
for (const request of entry.request.memberRequests) {
|
|
11564
|
+
await reject(request, entry.error);
|
|
11565
|
+
}
|
|
11566
|
+
throw entry.error;
|
|
11567
|
+
}
|
|
11568
|
+
if (!entry.result) {
|
|
11569
|
+
// The NativeBatchToolCall already attempted a lease-fenced failure
|
|
11570
|
+
// fanout. Read the actual durable terminal facts so cache hits in
|
|
11571
|
+
// the same cohort still reach their rows and a stale failure never
|
|
11572
|
+
// causes this Context to write a second terminal transition.
|
|
11573
|
+
const durable = await store.read({
|
|
11574
|
+
resultReceiptKeys: entry.request.memberRequests.map(
|
|
11575
|
+
(request) => request.receiptKey!,
|
|
11576
|
+
),
|
|
11577
|
+
});
|
|
11578
|
+
const byReceipt = new Map(
|
|
11579
|
+
durable.map((record) => [record.resultReceiptKey, record]),
|
|
11580
|
+
);
|
|
11581
|
+
for (const request of entry.request.memberRequests) {
|
|
11582
|
+
const record = byReceipt.get(request.receiptKey!);
|
|
11583
|
+
if (
|
|
11584
|
+
record &&
|
|
11585
|
+
(record.status === 'completed' ||
|
|
11586
|
+
record.status === 'skipped') &&
|
|
11587
|
+
record.result
|
|
11588
|
+
) {
|
|
11589
|
+
deliver(request, record.result);
|
|
11590
|
+
} else {
|
|
11591
|
+
await reject(
|
|
11592
|
+
request,
|
|
11593
|
+
record?.error ??
|
|
11594
|
+
entry.error ??
|
|
11595
|
+
new Error(
|
|
11596
|
+
'Native Tool Call did not reach a terminal receipt.',
|
|
11597
|
+
),
|
|
11598
|
+
);
|
|
11599
|
+
}
|
|
11600
|
+
}
|
|
11601
|
+
continue;
|
|
11602
|
+
}
|
|
11603
|
+
for (const request of entry.request.memberRequests) {
|
|
11604
|
+
const result = entry.result.result.get(request.callId);
|
|
11605
|
+
if (!result) {
|
|
11606
|
+
await reject(
|
|
11607
|
+
request,
|
|
11608
|
+
new Error(`Native Tool Call omitted member ${request.callId}.`),
|
|
11609
|
+
);
|
|
11610
|
+
continue;
|
|
11611
|
+
}
|
|
11612
|
+
deliver(request, result);
|
|
11613
|
+
}
|
|
11614
|
+
}
|
|
11615
|
+
},
|
|
11616
|
+
retainResults: false,
|
|
11617
|
+
});
|
|
11618
|
+
return successful;
|
|
11619
|
+
}
|
|
11620
|
+
|
|
10453
11621
|
private async executeBatchedToolCalls(
|
|
10454
11622
|
queuedToolCalls: ToolCallRequest[],
|
|
10455
11623
|
): Promise<void> {
|
|
@@ -10629,6 +11797,78 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10629
11797
|
return timeoutMs > 0 ? timeoutMs : undefined;
|
|
10630
11798
|
};
|
|
10631
11799
|
|
|
11800
|
+
// Cacheable map members cross the real receipt-cohort boundary before
|
|
11801
|
+
// the legacy claim loop. The scalar and native Modules share member
|
|
11802
|
+
// receipt ownership; native batches retain their separate aggregate
|
|
11803
|
+
// physical-operation receipt behind that seam. Always-fresh calls
|
|
11804
|
+
// intentionally have no durable member receipt and stay below.
|
|
11805
|
+
const operationStrategy =
|
|
11806
|
+
this.#options.getBatchOperationStrategy?.(toolId) ?? null;
|
|
11807
|
+
if (
|
|
11808
|
+
operationStrategy &&
|
|
11809
|
+
this.durableMappedToolResultsBackedByReceipts
|
|
11810
|
+
) {
|
|
11811
|
+
const cohortRequests = pendingRequests.filter(
|
|
11812
|
+
(request) =>
|
|
11813
|
+
request.cacheable !== false &&
|
|
11814
|
+
Boolean(request.receiptKey?.trim()),
|
|
11815
|
+
);
|
|
11816
|
+
if (cohortRequests.length > 0) {
|
|
11817
|
+
const cohortCallIds = new Set(
|
|
11818
|
+
cohortRequests.map((request) => request.callId),
|
|
11819
|
+
);
|
|
11820
|
+
pendingRequests.splice(
|
|
11821
|
+
0,
|
|
11822
|
+
pendingRequests.length,
|
|
11823
|
+
...pendingRequests.filter(
|
|
11824
|
+
(request) => !cohortCallIds.has(request.callId),
|
|
11825
|
+
),
|
|
11826
|
+
);
|
|
11827
|
+
const successful =
|
|
11828
|
+
await this.executeCacheableNativeMapToolCallCohort({
|
|
11829
|
+
toolId,
|
|
11830
|
+
requests: cohortRequests,
|
|
11831
|
+
strategy: operationStrategy,
|
|
11832
|
+
});
|
|
11833
|
+
for (const request of successful) {
|
|
11834
|
+
successfulLiveStepCallIds.add(request.callId);
|
|
11835
|
+
}
|
|
11836
|
+
recordToolStep(cohortRequests);
|
|
11837
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
11838
|
+
}
|
|
11839
|
+
} else if (
|
|
11840
|
+
!operationStrategy &&
|
|
11841
|
+
this.durableMappedToolResultsBackedByReceipts
|
|
11842
|
+
) {
|
|
11843
|
+
const cohortRequests = pendingRequests.filter(
|
|
11844
|
+
(request) =>
|
|
11845
|
+
request.cacheable !== false &&
|
|
11846
|
+
Boolean(request.receiptKey?.trim()),
|
|
11847
|
+
);
|
|
11848
|
+
if (cohortRequests.length > 0) {
|
|
11849
|
+
const cohortCallIds = new Set(
|
|
11850
|
+
cohortRequests.map((request) => request.callId),
|
|
11851
|
+
);
|
|
11852
|
+
pendingRequests.splice(
|
|
11853
|
+
0,
|
|
11854
|
+
pendingRequests.length,
|
|
11855
|
+
...pendingRequests.filter(
|
|
11856
|
+
(request) => !cohortCallIds.has(request.callId),
|
|
11857
|
+
),
|
|
11858
|
+
);
|
|
11859
|
+
const successful =
|
|
11860
|
+
await this.executeCacheableNonNativeMapToolCallCohort({
|
|
11861
|
+
toolId,
|
|
11862
|
+
requests: cohortRequests,
|
|
11863
|
+
});
|
|
11864
|
+
for (const request of successful) {
|
|
11865
|
+
successfulLiveStepCallIds.add(request.callId);
|
|
11866
|
+
}
|
|
11867
|
+
recordToolStep(cohortRequests);
|
|
11868
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
11869
|
+
}
|
|
11870
|
+
}
|
|
11871
|
+
|
|
10632
11872
|
const durableExistingRunningWaits: Array<{
|
|
10633
11873
|
receiptKey: string;
|
|
10634
11874
|
requestsForKey: ToolCallRequest[];
|
|
@@ -10932,40 +12172,44 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10932
12172
|
liveFollowersByOwnerCallId.set(owner.callId, waiters);
|
|
10933
12173
|
}
|
|
10934
12174
|
try {
|
|
10935
|
-
const execution =
|
|
10936
|
-
|
|
10937
|
-
|
|
10938
|
-
|
|
10939
|
-
|
|
10940
|
-
|
|
10941
|
-
|
|
10942
|
-
playNodeScope: playNodeScopeForToolCallRequest(owner),
|
|
10943
|
-
executionAuthScopeDigest: owner.executionAuthScopeDigest,
|
|
10944
|
-
providerIdempotencyReceiptKey:
|
|
12175
|
+
const execution =
|
|
12176
|
+
await this.executeMappedToolCallThroughModule({
|
|
12177
|
+
request: owner,
|
|
12178
|
+
dispatchToolId: toolId,
|
|
12179
|
+
dispatchPayload: owner.input,
|
|
12180
|
+
resultReceiptKey: receiptKey,
|
|
12181
|
+
providerOperationKey:
|
|
10945
12182
|
owner.providerIdempotencyKeyBase ?? owner.cacheKey,
|
|
10946
|
-
|
|
10947
|
-
|
|
10948
|
-
|
|
10949
|
-
|
|
10950
|
-
|
|
10951
|
-
|
|
10952
|
-
|
|
10953
|
-
|
|
10954
|
-
|
|
10955
|
-
|
|
10956
|
-
|
|
10957
|
-
|
|
10958
|
-
|
|
10959
|
-
|
|
10960
|
-
|
|
12183
|
+
apiOptions: {
|
|
12184
|
+
beforeProviderCall: () =>
|
|
12185
|
+
this.assertRuntimeToolReceiptOwnership([owner]),
|
|
12186
|
+
durableCallReceiptKey: receiptKey,
|
|
12187
|
+
playNodeScope: playNodeScopeForToolCallRequest(owner),
|
|
12188
|
+
executionAuthScopeDigest:
|
|
12189
|
+
owner.executionAuthScopeDigest,
|
|
12190
|
+
providerIdempotencyReceiptKey:
|
|
12191
|
+
owner.providerIdempotencyKeyBase ?? owner.cacheKey,
|
|
12192
|
+
receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
|
|
12193
|
+
heartbeatReceipt: () =>
|
|
12194
|
+
this.renewRuntimeToolReceiptOwnership([owner]),
|
|
12195
|
+
providerIdempotencyKey:
|
|
12196
|
+
this.providerIdempotencyKeyForToolCall({
|
|
12197
|
+
cacheKey:
|
|
12198
|
+
owner.providerIdempotencyKeyBase ??
|
|
12199
|
+
owner.cacheKey,
|
|
12200
|
+
force: owner.force === true,
|
|
12201
|
+
leaseId: owner.receiptLeaseId,
|
|
12202
|
+
logicalCallId: owner.logicalCallId,
|
|
12203
|
+
}),
|
|
12204
|
+
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
|
|
12205
|
+
owner,
|
|
12206
|
+
]),
|
|
12207
|
+
},
|
|
12208
|
+
});
|
|
10961
12209
|
const result = await this.resolveToolCall(
|
|
10962
12210
|
toolId,
|
|
10963
12211
|
owner,
|
|
10964
|
-
execution
|
|
10965
|
-
execution?.metadata ?? null,
|
|
10966
|
-
execution?.jobId,
|
|
10967
|
-
execution?.meta,
|
|
10968
|
-
execution?.toolResponse,
|
|
12212
|
+
execution,
|
|
10969
12213
|
);
|
|
10970
12214
|
if (result != null) {
|
|
10971
12215
|
successfulLiveStepCallIds.add(owner.callId);
|
|
@@ -11245,8 +12489,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11245
12489
|
await pendingReceiptHeartbeat.confirmOwned();
|
|
11246
12490
|
}
|
|
11247
12491
|
if (pendingRequests.length > 0) {
|
|
11248
|
-
const strategy =
|
|
11249
|
-
this.#options.getBatchOperationStrategy?.(toolId) ?? null;
|
|
12492
|
+
const strategy = operationStrategy;
|
|
11250
12493
|
|
|
11251
12494
|
if (strategy) {
|
|
11252
12495
|
const compiledBatches = compileRequestsWithStrategy({
|
|
@@ -11306,70 +12549,74 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11306
12549
|
});
|
|
11307
12550
|
let releaseToolSlot: () => void = () => undefined;
|
|
11308
12551
|
try {
|
|
11309
|
-
const execution =
|
|
11310
|
-
|
|
11311
|
-
|
|
11312
|
-
|
|
11313
|
-
|
|
11314
|
-
|
|
11315
|
-
|
|
11316
|
-
// is its members' shared scope.
|
|
11317
|
-
playNodeScope: batch.memberRequests[0]
|
|
11318
|
-
? playNodeScopeForToolCallRequest(
|
|
11319
|
-
batch.memberRequests[0],
|
|
11320
|
-
batch.batchOperation,
|
|
11321
|
-
)
|
|
11322
|
-
: null,
|
|
11323
|
-
executionAuthScopeDigest:
|
|
11324
|
-
batch.memberRequests[0]?.executionAuthScopeDigest ??
|
|
11325
|
-
null,
|
|
11326
|
-
providerIdempotencyReceiptKey:
|
|
12552
|
+
const execution =
|
|
12553
|
+
await this.executeMappedToolCallThroughModule({
|
|
12554
|
+
request: batch.memberRequests[0]!,
|
|
12555
|
+
dispatchToolId: batch.batchOperation,
|
|
12556
|
+
dispatchPayload: batch.batchPayload,
|
|
12557
|
+
resultReceiptKey: aggregateReceiptKey,
|
|
12558
|
+
providerOperationKey:
|
|
11327
12559
|
aggregateProviderIdempotencyReceiptKey,
|
|
11328
|
-
|
|
11329
|
-
|
|
11330
|
-
|
|
11331
|
-
|
|
11332
|
-
|
|
11333
|
-
|
|
11334
|
-
|
|
11335
|
-
|
|
11336
|
-
|
|
11337
|
-
|
|
11338
|
-
|
|
11339
|
-
|
|
11340
|
-
|
|
11341
|
-
|
|
11342
|
-
|
|
11343
|
-
|
|
11344
|
-
|
|
11345
|
-
|
|
11346
|
-
|
|
11347
|
-
|
|
11348
|
-
|
|
11349
|
-
|
|
11350
|
-
|
|
11351
|
-
|
|
11352
|
-
|
|
11353
|
-
|
|
11354
|
-
|
|
11355
|
-
|
|
11356
|
-
|
|
11357
|
-
),
|
|
11358
|
-
beforeProviderCall: () =>
|
|
11359
|
-
pendingReceiptHeartbeat
|
|
11360
|
-
? pendingReceiptHeartbeat.assertOwned()
|
|
11361
|
-
: this.assertRuntimeToolReceiptOwnership(
|
|
11362
|
-
batch.memberRequests,
|
|
12560
|
+
apiOptions: {
|
|
12561
|
+
durableCallReceiptKey: aggregateReceiptKey,
|
|
12562
|
+
// A native batch is one physical provider request for
|
|
12563
|
+
// many rows of the same column, so the batch's node scope
|
|
12564
|
+
// is its members' shared scope.
|
|
12565
|
+
playNodeScope: playNodeScopeForToolCallRequest(
|
|
12566
|
+
batch.memberRequests[0]!,
|
|
12567
|
+
batch.batchOperation,
|
|
12568
|
+
),
|
|
12569
|
+
executionAuthScopeDigest:
|
|
12570
|
+
batch.memberRequests[0]?.executionAuthScopeDigest ??
|
|
12571
|
+
null,
|
|
12572
|
+
providerIdempotencyReceiptKey:
|
|
12573
|
+
aggregateProviderIdempotencyReceiptKey,
|
|
12574
|
+
providerIdempotencyKey:
|
|
12575
|
+
buildDurableToolAggregateProviderIdempotencyKey({
|
|
12576
|
+
aggregateReceiptKey:
|
|
12577
|
+
aggregateProviderIdempotencyReceiptKey,
|
|
12578
|
+
receiptKeys: providerIdempotencyReceiptKeys,
|
|
12579
|
+
providerIdempotencyKeys: batch.memberRequests.map(
|
|
12580
|
+
(request) =>
|
|
12581
|
+
this.providerIdempotencyKeyForToolCall({
|
|
12582
|
+
cacheKey:
|
|
12583
|
+
request.providerIdempotencyKeyBase ??
|
|
12584
|
+
request.cacheKey,
|
|
12585
|
+
force: request.force === true,
|
|
12586
|
+
leaseId: request.receiptLeaseId,
|
|
12587
|
+
logicalCallId: request.logicalCallId,
|
|
12588
|
+
}),
|
|
11363
12589
|
),
|
|
11364
|
-
|
|
11365
|
-
|
|
12590
|
+
}),
|
|
12591
|
+
receiptLeaseExpiresAt: batch.memberRequests.reduce<
|
|
12592
|
+
string | null
|
|
12593
|
+
>((earliest, request) => {
|
|
12594
|
+
const expiresAt =
|
|
12595
|
+
request.receiptLeaseExpiresAt ?? null;
|
|
12596
|
+
if (!expiresAt) return earliest;
|
|
12597
|
+
if (!earliest) return expiresAt;
|
|
12598
|
+
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
12599
|
+
? expiresAt
|
|
12600
|
+
: earliest;
|
|
12601
|
+
}, null),
|
|
12602
|
+
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
|
|
11366
12603
|
batch.memberRequests,
|
|
11367
12604
|
),
|
|
11368
|
-
|
|
11369
|
-
|
|
12605
|
+
beforeProviderCall: () =>
|
|
12606
|
+
pendingReceiptHeartbeat
|
|
12607
|
+
? pendingReceiptHeartbeat.assertOwned()
|
|
12608
|
+
: this.assertRuntimeToolReceiptOwnership(
|
|
12609
|
+
batch.memberRequests,
|
|
12610
|
+
),
|
|
12611
|
+
heartbeatReceipt: () =>
|
|
12612
|
+
this.renewRuntimeToolReceiptOwnership(
|
|
12613
|
+
batch.memberRequests,
|
|
12614
|
+
),
|
|
12615
|
+
retainToolSlot: (release) => {
|
|
12616
|
+
releaseToolSlot = release;
|
|
12617
|
+
},
|
|
11370
12618
|
},
|
|
11371
|
-
}
|
|
11372
|
-
);
|
|
12619
|
+
});
|
|
11373
12620
|
return { execution, releaseToolSlot };
|
|
11374
12621
|
} catch (error) {
|
|
11375
12622
|
releaseToolSlot();
|
|
@@ -11492,17 +12739,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11492
12739
|
};
|
|
11493
12740
|
const enqueueCompletion = (
|
|
11494
12741
|
request: ToolCallRequest,
|
|
11495
|
-
execution:
|
|
12742
|
+
execution: ToolExecuteResult,
|
|
11496
12743
|
): Promise<unknown> =>
|
|
11497
12744
|
new Promise((resolve, reject) => {
|
|
11498
12745
|
completionBuffer.push({
|
|
11499
12746
|
request,
|
|
11500
|
-
result: execution
|
|
12747
|
+
result: execution,
|
|
11501
12748
|
status: execution.status,
|
|
11502
|
-
metadata: execution.metadata ?? null,
|
|
11503
|
-
jobId: execution.jobId,
|
|
11504
|
-
meta: execution.meta,
|
|
11505
|
-
toolResponse: execution.toolResponse,
|
|
11506
12749
|
resolve,
|
|
11507
12750
|
reject,
|
|
11508
12751
|
});
|
|
@@ -11548,47 +12791,62 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11548
12791
|
}
|
|
11549
12792
|
let releaseToolSlot: () => void = () => undefined;
|
|
11550
12793
|
try {
|
|
11551
|
-
const execution =
|
|
11552
|
-
|
|
11553
|
-
|
|
11554
|
-
|
|
11555
|
-
|
|
11556
|
-
|
|
11557
|
-
|
|
11558
|
-
|
|
11559
|
-
|
|
11560
|
-
|
|
11561
|
-
|
|
11562
|
-
|
|
11563
|
-
|
|
11564
|
-
|
|
11565
|
-
|
|
11566
|
-
|
|
11567
|
-
|
|
11568
|
-
|
|
11569
|
-
|
|
11570
|
-
cacheKey:
|
|
11571
|
-
request.providerIdempotencyKeyBase ??
|
|
11572
|
-
request.receiptKey,
|
|
11573
|
-
force: request.force === true,
|
|
11574
|
-
leaseId: request.receiptLeaseId,
|
|
11575
|
-
}),
|
|
11576
|
-
receiptLeaseExpiresAt:
|
|
11577
|
-
request.receiptLeaseExpiresAt,
|
|
11578
|
-
heartbeatReceipt: () =>
|
|
11579
|
-
this.renewRuntimeToolReceiptOwnership([
|
|
12794
|
+
const execution =
|
|
12795
|
+
await this.executeMappedToolCallThroughModule({
|
|
12796
|
+
request,
|
|
12797
|
+
dispatchToolId: toolId,
|
|
12798
|
+
dispatchPayload: request.input,
|
|
12799
|
+
// Always-fresh map calls intentionally have no durable
|
|
12800
|
+
// receipt or provider idempotency header. Their cache
|
|
12801
|
+
// key is only a local ToolCall contract identity.
|
|
12802
|
+
resultReceiptKey:
|
|
12803
|
+
request.receiptKey ?? request.cacheKey,
|
|
12804
|
+
providerOperationKey:
|
|
12805
|
+
request.providerIdempotencyKeyBase ??
|
|
12806
|
+
request.receiptKey ??
|
|
12807
|
+
request.cacheKey,
|
|
12808
|
+
apiOptions: {
|
|
12809
|
+
beforeProviderCall: () =>
|
|
12810
|
+
pendingReceiptHeartbeat
|
|
12811
|
+
? pendingReceiptHeartbeat.assertOwned()
|
|
12812
|
+
: this.assertRuntimeToolReceiptOwnership([
|
|
11580
12813
|
request,
|
|
11581
12814
|
]),
|
|
11582
|
-
|
|
11583
|
-
|
|
11584
|
-
|
|
11585
|
-
|
|
11586
|
-
|
|
11587
|
-
|
|
11588
|
-
|
|
12815
|
+
playNodeScope:
|
|
12816
|
+
playNodeScopeForToolCallRequest(request),
|
|
12817
|
+
...(request.receiptKey
|
|
12818
|
+
? {
|
|
12819
|
+
durableCallReceiptKey: request.receiptKey,
|
|
12820
|
+
executionAuthScopeDigest:
|
|
12821
|
+
request.executionAuthScopeDigest,
|
|
12822
|
+
providerIdempotencyReceiptKey:
|
|
12823
|
+
request.providerIdempotencyKeyBase ??
|
|
12824
|
+
request.receiptKey,
|
|
12825
|
+
providerIdempotencyKey:
|
|
12826
|
+
this.providerIdempotencyKeyForToolCall({
|
|
12827
|
+
cacheKey:
|
|
12828
|
+
request.providerIdempotencyKeyBase ??
|
|
12829
|
+
request.receiptKey,
|
|
12830
|
+
force: request.force === true,
|
|
12831
|
+
leaseId: request.receiptLeaseId,
|
|
12832
|
+
logicalCallId: request.logicalCallId,
|
|
12833
|
+
}),
|
|
12834
|
+
receiptLeaseExpiresAt:
|
|
12835
|
+
request.receiptLeaseExpiresAt,
|
|
12836
|
+
heartbeatReceipt: () =>
|
|
12837
|
+
this.renewRuntimeToolReceiptOwnership([
|
|
12838
|
+
request,
|
|
12839
|
+
]),
|
|
12840
|
+
}
|
|
12841
|
+
: {}),
|
|
12842
|
+
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
|
|
12843
|
+
request,
|
|
12844
|
+
]),
|
|
12845
|
+
retainToolSlot: (release) => {
|
|
12846
|
+
releaseToolSlot = release;
|
|
12847
|
+
},
|
|
11589
12848
|
},
|
|
11590
|
-
}
|
|
11591
|
-
);
|
|
12849
|
+
});
|
|
11592
12850
|
await enqueueCompletion(request, execution);
|
|
11593
12851
|
} finally {
|
|
11594
12852
|
releaseToolSlot();
|
|
@@ -11867,6 +13125,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11867
13125
|
const executionAuthScopeDigest = options?.executionAuthScopeDigest
|
|
11868
13126
|
? options.executionAuthScopeDigest.trim() || null
|
|
11869
13127
|
: ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null);
|
|
13128
|
+
const canonicalOperation = {
|
|
13129
|
+
version: 1 as const,
|
|
13130
|
+
tool_id: toolId,
|
|
13131
|
+
action_version:
|
|
13132
|
+
(
|
|
13133
|
+
await this.#options.getToolActionCacheVersion?.(toolId)
|
|
13134
|
+
)?.trim() || 'unversioned',
|
|
13135
|
+
play_scope: this.currentGovernance.currentPlayId,
|
|
13136
|
+
output_intent: 'dataset',
|
|
13137
|
+
};
|
|
11870
13138
|
const providerIdempotencyKey =
|
|
11871
13139
|
options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
|
|
11872
13140
|
const providerIdempotencyReceiptKey =
|
|
@@ -11891,6 +13159,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11891
13159
|
...(requestsDurableInvocationFence
|
|
11892
13160
|
? { invocation_fence_version: 1 }
|
|
11893
13161
|
: {}),
|
|
13162
|
+
// Canonical provider-operation receipts validate this semantic
|
|
13163
|
+
// plan before replay. It is intentionally removed from the
|
|
13164
|
+
// legacy byte-receipt fingerprint so mixed runner artifacts
|
|
13165
|
+
// retain their existing provider-once behavior.
|
|
13166
|
+
...(requestsDurableInvocationFence
|
|
13167
|
+
? { canonical_operation: canonicalOperation }
|
|
13168
|
+
: {}),
|
|
11894
13169
|
// This is deliberately the provider receipt, rather than the
|
|
11895
13170
|
// worker-owned response cache receipt. It keeps a newer
|
|
11896
13171
|
// worker wire-compatible with an already active Vercel app
|