deepline 0.3.284 → 0.3.286
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +1 -2
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/activity-observation.ts +26 -2
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +836 -2234
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +19 -6
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +127 -789
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/job.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +180 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +3 -22
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts +26 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +59 -8
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +122 -18
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +458 -130
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +60 -9
- package/dist/bundling-sources/shared_libs/play-runtime/test-async-batching.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +13 -2
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +181 -326
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +38 -151
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +8 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +10 -1
- package/dist/bundling-sources/shared_libs/plays/enrich-compat-adapter.ts +2 -0
- package/dist/cli/index.js +30579 -34364
- package/dist/cli/index.mjs +30647 -34443
- package/dist/{client-BCXQ_Ftc.d.ts → client-B7ic-kY5.d.ts} +6 -3
- package/dist/{client-B0AjIkgb.d.mts → client-Dzf_KovW.d.mts} +6 -3
- package/dist/{compiler-manifest-B1zo3igW.d.mts → compiler-manifest-9SQtUuKb.d.mts} +1 -1
- package/dist/{compiler-manifest-B1zo3igW.d.ts → compiler-manifest-9SQtUuKb.d.ts} +1 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +34 -4
- package/dist/index.mjs +34 -4
- package/dist/install-integrity.json +4 -4
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +8 -1
- package/dist/plays/run-request.d.mts +2 -2
- package/dist/plays/run-request.d.ts +2 -2
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -160,6 +160,7 @@ import {
|
|
|
160
160
|
attachRelayPublishedToolResultReference,
|
|
161
161
|
attachToolResultListDataset,
|
|
162
162
|
createToolExecuteResult,
|
|
163
|
+
projectNativeBatchToolResponse,
|
|
163
164
|
parseToolExecuteResponse,
|
|
164
165
|
isToolExecuteResult,
|
|
165
166
|
isSerializedToolExecuteResult,
|
|
@@ -187,7 +188,6 @@ import {
|
|
|
187
188
|
createRuntimeStepToolResultReceipts,
|
|
188
189
|
createToolCallJob,
|
|
189
190
|
resolveToolCallResultReceiptPolicy,
|
|
190
|
-
ToolCallReceiptLeaseLostError,
|
|
191
191
|
type ToolCallReceiptCohortStore,
|
|
192
192
|
type ToolCallReceiptCohortTimings,
|
|
193
193
|
type ToolCallReceiptRecord,
|
|
@@ -262,26 +262,16 @@ import {
|
|
|
262
262
|
} from './log-provenance';
|
|
263
263
|
import {
|
|
264
264
|
COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
|
|
265
|
-
DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS,
|
|
266
265
|
createRuntimeReceiptHydrationGate,
|
|
267
266
|
runtimeStepReceiptPayloadBytes,
|
|
268
|
-
runtimeReceiptClaimLeaseIsLive,
|
|
269
|
-
runtimeReceiptWaitBudgetMs,
|
|
270
|
-
runtimeReceiptWaitDelayMs,
|
|
271
|
-
DURABLE_RECEIPT_WAIT_DELAY_MS,
|
|
272
|
-
RuntimeReceiptHydrationTurnTimeoutError,
|
|
273
267
|
RuntimeReceiptLeaseLostError,
|
|
274
|
-
RuntimeReceiptWaitTimeoutError,
|
|
275
268
|
executeWithDurableRuntimeReceipt,
|
|
276
269
|
resolveRuntimeToolReceiptWaitMaxAttempts,
|
|
277
|
-
runtimeReceiptFailureError,
|
|
278
270
|
runtimeReceiptFailureKindForError,
|
|
279
271
|
runtimeReceiptOutput as durableRuntimeReceiptOutput,
|
|
280
272
|
type RuntimeReceiptHydrationTurn,
|
|
281
273
|
type RuntimeReceiptHydrationGateSnapshot,
|
|
282
274
|
type RuntimeReceiptHydrationSource,
|
|
283
|
-
waitForCompletedRuntimeReceipt,
|
|
284
|
-
waitForCompletedRuntimeReceipts,
|
|
285
275
|
type DurableReceiptExecutionStore,
|
|
286
276
|
} from './durable-receipt-execution';
|
|
287
277
|
import {
|
|
@@ -422,6 +412,7 @@ import type {
|
|
|
422
412
|
ContextOptions,
|
|
423
413
|
PlayCallOptions,
|
|
424
414
|
PlayCheckpoint,
|
|
415
|
+
ProviderRetryCursor,
|
|
425
416
|
PlayStep,
|
|
426
417
|
PlayStepRowResult,
|
|
427
418
|
PlayRowUpdate,
|
|
@@ -609,12 +600,6 @@ const DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN =
|
|
|
609
600
|
/\bdeeplinedeveloper\.com\s*\|\s*502\s*:\s*bad gateway\b/i;
|
|
610
601
|
const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100;
|
|
611
602
|
const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25;
|
|
612
|
-
// A newly claimed receipt should normally use its scheduled heartbeat. When a
|
|
613
|
-
// test/SEV policy deliberately gives it only a moment of lease life, however,
|
|
614
|
-
// scheduler setup and the first provider-dispatch turn can consume that entire
|
|
615
|
-
// window before the timer fires. Renew once before fan-out in that narrow case.
|
|
616
|
-
// Production's normal multi-minute lease remains on the zero-extra-query path.
|
|
617
|
-
const IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS = 100;
|
|
618
603
|
// Per-row sanity cap on distinct inline child invocations. `maxPlayCallDepth`
|
|
619
604
|
// (governor) already bounds nesting; this bounds fan-WIDTH from one active row
|
|
620
605
|
// resolver. The dedupe set is row-local, never retained for the full run.
|
|
@@ -1465,6 +1450,18 @@ function publicToolResponseEnvelope(value: unknown): {
|
|
|
1465
1450
|
};
|
|
1466
1451
|
}
|
|
1467
1452
|
|
|
1453
|
+
function nativeBatchResponseMeta(result: unknown): Record<string, unknown> {
|
|
1454
|
+
return {
|
|
1455
|
+
nativeBatchResponseView:
|
|
1456
|
+
result &&
|
|
1457
|
+
typeof result === 'object' &&
|
|
1458
|
+
!Array.isArray(result) &&
|
|
1459
|
+
Object.prototype.hasOwnProperty.call(result, 'data')
|
|
1460
|
+
? 'data'
|
|
1461
|
+
: 'rawV2',
|
|
1462
|
+
};
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1468
1465
|
/**
|
|
1469
1466
|
* A batched provider request returns one envelope for several logical source
|
|
1470
1467
|
* tool calls. Each source call persists only its own item-shaped canonical
|
|
@@ -1826,7 +1823,7 @@ type ToolExecutionApiOptions = {
|
|
|
1826
1823
|
completedResultCache?: boolean;
|
|
1827
1824
|
executionAuthScopeDigest?: string | null;
|
|
1828
1825
|
providerIdempotencyKey?: string | null;
|
|
1829
|
-
/**
|
|
1826
|
+
/** Invocation identities that participate in the canonical operation key. */
|
|
1830
1827
|
providerOperationForceRefreshIdentities?: readonly string[];
|
|
1831
1828
|
/** Presentation-only cache identity paired with a separate execution fence. */
|
|
1832
1829
|
providerIdempotencyReceiptKey?: string | null;
|
|
@@ -1878,6 +1875,12 @@ type ToolExecutionApiOptions = {
|
|
|
1878
1875
|
* cache stores only its terminal materialized output.
|
|
1879
1876
|
*/
|
|
1880
1877
|
uncachedAsyncLifecycleCall?: boolean;
|
|
1878
|
+
/** Exact outgoing delivery candidates; consume a checkpoint cursor once. */
|
|
1879
|
+
legacyProviderRetryIdentities?: readonly {
|
|
1880
|
+
transportReceiptKey: string;
|
|
1881
|
+
providerOperationKey: string;
|
|
1882
|
+
forceRefreshIdentities?: readonly string[];
|
|
1883
|
+
}[];
|
|
1881
1884
|
/**
|
|
1882
1885
|
* Called once this call's provider response body has been fully read and
|
|
1883
1886
|
* parsed, with that instant on the runner's clock. Measurement only: the
|
|
@@ -2594,12 +2597,6 @@ type ScalarPlayAuthoringRuntimeContext = Pick<
|
|
|
2594
2597
|
export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
2595
2598
|
private rowStates = new Map<number, RowState>();
|
|
2596
2599
|
private toolCallQueue: ToolCallRequest[] = [];
|
|
2597
|
-
private pendingRuntimeToolOwnershipAssertions: Array<{
|
|
2598
|
-
targets: Array<{ receiptKey: string; leaseId: string }>;
|
|
2599
|
-
resolve: () => void;
|
|
2600
|
-
reject: (error: unknown) => void;
|
|
2601
|
-
}> = [];
|
|
2602
|
-
private runtimeToolOwnershipAssertionFlushScheduled = false;
|
|
2603
2600
|
private runtimeToolReceiptCompletionsInFlight = new Map<string, number>();
|
|
2604
2601
|
/**
|
|
2605
2602
|
* The one app-side result-egress controller. It serializes receipt gateway
|
|
@@ -3225,23 +3222,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3225
3222
|
: Date.now();
|
|
3226
3223
|
this.checkpoint.durableCallCacheEpochMs = this.durableCallCacheEpochMs;
|
|
3227
3224
|
this.durableMappedToolResultsBackedByReceipts = Boolean(
|
|
3228
|
-
(options.claimRuntimeStepReceipt || options.claimRuntimeStepReceipts) &&
|
|
3229
3225
|
(options.getRuntimeStepReceipt || options.getRuntimeStepReceipts) &&
|
|
3230
3226
|
(options.completeRuntimeStepReceipt ||
|
|
3231
3227
|
options.completeRuntimeStepReceipts),
|
|
3232
3228
|
);
|
|
3233
|
-
this.durableDirectToolResultsBackedByReceipts =
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
);
|
|
3238
|
-
if (this.durableDirectToolResultsBackedByReceipts) {
|
|
3239
|
-
// A resumed durable runner may receive a legacy checkpoint containing
|
|
3240
|
-
// full tool results. Drop that redundant copy immediately and recover
|
|
3241
|
-
// through the receipt store below. Bulk-only receipt contexts retain the
|
|
3242
|
-
// checkpoint because direct tool calls cannot use the bulk claim path.
|
|
3243
|
-
this.checkpoint.completedToolBatches = {};
|
|
3244
|
-
}
|
|
3229
|
+
this.durableDirectToolResultsBackedByReceipts =
|
|
3230
|
+
this.durableMappedToolResultsBackedByReceipts;
|
|
3231
|
+
// Existing checkpoints can be the sole completed cache from an outgoing
|
|
3232
|
+
// context. Receipt-backed calls stop adding payloads, but retain old hits.
|
|
3245
3233
|
// The governance play id keys durable ctx receipts (durableCtxKey), the
|
|
3246
3234
|
// cycle guard, and per-parent child-call counters. It must be the STABLE
|
|
3247
3235
|
// play name — not the per-run workflow id — so receipts written by one
|
|
@@ -4323,7 +4311,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4323
4311
|
key: string,
|
|
4324
4312
|
): Promise<RuntimeStepReceipt | null> {
|
|
4325
4313
|
if (!this.#options.getRuntimeStepReceipt) {
|
|
4326
|
-
return null;
|
|
4314
|
+
if (!this.#options.getRuntimeStepReceipts) return null;
|
|
4315
|
+
return (await this.getRuntimeStepReceipts([key])).get(key) ?? null;
|
|
4327
4316
|
}
|
|
4328
4317
|
const receipt = await this.#options.getRuntimeStepReceipt({ key });
|
|
4329
4318
|
if (receipt && typeof receipt.key === 'string' && receipt.key.trim()) {
|
|
@@ -4465,7 +4454,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4465
4454
|
path: `ctx.step receipt "${key}"`,
|
|
4466
4455
|
});
|
|
4467
4456
|
if (!this.#options.completeRuntimeStepReceipt) {
|
|
4468
|
-
return null;
|
|
4457
|
+
if (!this.#options.completeRuntimeStepReceipts) return null;
|
|
4458
|
+
const completed = await this.completeRuntimeStepReceipts(
|
|
4459
|
+
[
|
|
4460
|
+
{
|
|
4461
|
+
key,
|
|
4462
|
+
runId: this.currentReceiptOwnerRunId,
|
|
4463
|
+
runAttempt: this.currentRunAttempt,
|
|
4464
|
+
...(leaseId ? { leaseId } : {}),
|
|
4465
|
+
output,
|
|
4466
|
+
...(inlineFields ? { inlineFields } : {}),
|
|
4467
|
+
},
|
|
4468
|
+
],
|
|
4469
|
+
undefined,
|
|
4470
|
+
undefined,
|
|
4471
|
+
{ retainLiveResult: true },
|
|
4472
|
+
);
|
|
4473
|
+
return completed.get(key) ?? null;
|
|
4469
4474
|
}
|
|
4470
4475
|
const completed = await this.#options.completeRuntimeStepReceipt({
|
|
4471
4476
|
key,
|
|
@@ -4527,49 +4532,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4527
4532
|
})),
|
|
4528
4533
|
);
|
|
4529
4534
|
if (manifests.length === 0) return;
|
|
4530
|
-
//
|
|
4531
|
-
//
|
|
4532
|
-
// egress wave. Nothing is gained by skipping the ownership handshake
|
|
4533
|
-
// here, and skipping it costs the caller a receipt row it does not own:
|
|
4534
|
-
// a runtime whose claim seam refuses (or that exposes no claim seam at
|
|
4535
|
-
// all) would still publish pointer rows, and an already published
|
|
4536
|
-
// pointer would be rewritten on every later call. Take the row the way
|
|
4537
|
-
// every other scalar receipt writer takes it, and write only what this
|
|
4538
|
-
// run actually owns. The map/cohort publisher keeps the claim-free
|
|
4539
|
-
// batched write, where the pointer genuinely shares the canonical
|
|
4540
|
-
// result's single bounded request.
|
|
4535
|
+
// The pointer is an immutable completed-cache insert, like the result
|
|
4536
|
+
// it names. Publishing it needs no pending lease or read-back.
|
|
4541
4537
|
if (
|
|
4542
4538
|
!(
|
|
4543
|
-
this.#options.
|
|
4544
|
-
this.#options.
|
|
4539
|
+
this.#options.completeRuntimeStepReceipt ||
|
|
4540
|
+
this.#options.completeRuntimeStepReceipts
|
|
4545
4541
|
)
|
|
4546
4542
|
) {
|
|
4547
4543
|
return;
|
|
4548
4544
|
}
|
|
4549
|
-
|
|
4550
|
-
manifests.map((manifest) => manifest.key),
|
|
4551
|
-
{ hydrateOutput: false },
|
|
4552
|
-
);
|
|
4553
|
-
const missing = manifests.filter((manifest) => {
|
|
4554
|
-
const receipt = existing.get(manifest.key);
|
|
4555
|
-
return receipt?.status !== 'completed' && receipt?.status !== 'skipped';
|
|
4556
|
-
});
|
|
4557
|
-
if (missing.length === 0) return;
|
|
4558
|
-
const claims = await this.claimRuntimeStepReceipts(
|
|
4559
|
-
missing.map((manifest) => manifest.key),
|
|
4560
|
-
sourceRunId,
|
|
4561
|
-
false,
|
|
4562
|
-
false,
|
|
4563
|
-
false,
|
|
4564
|
-
false,
|
|
4565
|
-
);
|
|
4566
|
-
const owned = missing.flatMap((manifest) => {
|
|
4567
|
-
const leaseId = claims.get(manifest.key)?.leaseId;
|
|
4568
|
-
return leaseId ? [{ ...manifest, leaseId }] : [];
|
|
4569
|
-
});
|
|
4570
|
-
if (owned.length === 0) return;
|
|
4571
|
-
await this.completeRuntimeStepReceipts(owned, undefined, undefined, {
|
|
4572
|
-
bestEffortKeys: new Set(owned.map((manifest) => manifest.key)),
|
|
4545
|
+
await this.completeRuntimeStepReceipts(manifests, undefined, undefined, {
|
|
4546
|
+
bestEffortKeys: new Set(manifests.map((manifest) => manifest.key)),
|
|
4573
4547
|
onBestEffortFailure: ({ key, error }) => {
|
|
4574
4548
|
this.runtimeLog(
|
|
4575
4549
|
`Tool source-run receipt manifest write skipped for ${key}: ${error instanceof Error ? error.message : String(error)}`,
|
|
@@ -4789,178 +4763,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4789
4763
|
};
|
|
4790
4764
|
}
|
|
4791
4765
|
|
|
4792
|
-
private async assertRuntimeToolReceiptOwnership(
|
|
4793
|
-
requests: ToolCallRequest[],
|
|
4794
|
-
): Promise<void> {
|
|
4795
|
-
const targets = this.runtimeToolReceiptOwnershipTargets(requests);
|
|
4796
|
-
if (targets.length === 0) return;
|
|
4797
|
-
|
|
4798
|
-
return await new Promise<void>((resolve, reject) => {
|
|
4799
|
-
this.pendingRuntimeToolOwnershipAssertions.push({
|
|
4800
|
-
targets,
|
|
4801
|
-
resolve,
|
|
4802
|
-
reject,
|
|
4803
|
-
});
|
|
4804
|
-
if (this.runtimeToolOwnershipAssertionFlushScheduled) return;
|
|
4805
|
-
this.runtimeToolOwnershipAssertionFlushScheduled = true;
|
|
4806
|
-
queueMicrotask(() => void this.flushRuntimeToolOwnershipAssertionBatch());
|
|
4807
|
-
});
|
|
4808
|
-
}
|
|
4809
|
-
|
|
4810
|
-
private runtimeToolReceiptOwnershipTargets(
|
|
4811
|
-
requests: ToolCallRequest[],
|
|
4812
|
-
): Array<{ receiptKey: string; leaseId: string }> {
|
|
4813
|
-
return requests.flatMap((request) => {
|
|
4814
|
-
const receiptKey = request.receiptKey?.trim() || null;
|
|
4815
|
-
if (!receiptKey) return [];
|
|
4816
|
-
const leaseId = request.receiptLeaseId?.trim() || null;
|
|
4817
|
-
if (!leaseId) return [];
|
|
4818
|
-
return [{ receiptKey, leaseId }];
|
|
4819
|
-
});
|
|
4820
|
-
}
|
|
4821
|
-
|
|
4822
|
-
private async renewRuntimeToolReceiptOwnership(
|
|
4823
|
-
requests: ToolCallRequest[],
|
|
4824
|
-
): Promise<void> {
|
|
4825
|
-
const targets = this.runtimeToolReceiptOwnershipTargets(requests);
|
|
4826
|
-
if (targets.length === 0) return;
|
|
4827
|
-
if (!this.#options.heartbeatRuntimeStepReceipts) {
|
|
4828
|
-
return await this.assertRuntimeToolReceiptOwnership(requests);
|
|
4829
|
-
}
|
|
4830
|
-
const uniqueTargets = [
|
|
4831
|
-
...new Map(targets.map((target) => [target.receiptKey, target])).values(),
|
|
4832
|
-
];
|
|
4833
|
-
const receipts = await this.#options.heartbeatRuntimeStepReceipts({
|
|
4834
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4835
|
-
runAttempt: this.currentRunAttempt,
|
|
4836
|
-
keys: uniqueTargets.map((target) => target.receiptKey),
|
|
4837
|
-
leaseIds: uniqueTargets.map((target) => target.leaseId),
|
|
4838
|
-
});
|
|
4839
|
-
this.assertRuntimeToolReceiptHeartbeatResults(uniqueTargets, receipts);
|
|
4840
|
-
}
|
|
4841
|
-
|
|
4842
|
-
private async flushRuntimeToolOwnershipAssertionBatch(): Promise<void> {
|
|
4843
|
-
this.runtimeToolOwnershipAssertionFlushScheduled = false;
|
|
4844
|
-
const assertions = this.pendingRuntimeToolOwnershipAssertions.splice(0);
|
|
4845
|
-
if (assertions.length === 0) return;
|
|
4846
|
-
const targets = assertions.flatMap((assertion) => assertion.targets);
|
|
4847
|
-
|
|
4848
|
-
if (this.#options.heartbeatRuntimeStepReceipts) {
|
|
4849
|
-
try {
|
|
4850
|
-
const uniqueTargets = [
|
|
4851
|
-
...new Map(
|
|
4852
|
-
targets.map((target) => [target.receiptKey, target]),
|
|
4853
|
-
).values(),
|
|
4854
|
-
];
|
|
4855
|
-
const receipts = await this.#options.heartbeatRuntimeStepReceipts({
|
|
4856
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4857
|
-
runAttempt: this.currentRunAttempt,
|
|
4858
|
-
keys: uniqueTargets.map((target) => target.receiptKey),
|
|
4859
|
-
leaseIds: uniqueTargets.map((target) => target.leaseId),
|
|
4860
|
-
});
|
|
4861
|
-
const renewed = new Map<string, RuntimeStepReceipt | null>();
|
|
4862
|
-
for (let index = 0; index < uniqueTargets.length; index += 1) {
|
|
4863
|
-
renewed.set(
|
|
4864
|
-
uniqueTargets[index]!.receiptKey,
|
|
4865
|
-
receipts[index] ?? null,
|
|
4866
|
-
);
|
|
4867
|
-
}
|
|
4868
|
-
for (const assertion of assertions) {
|
|
4869
|
-
const lost = assertion.targets.find((target) => {
|
|
4870
|
-
const receipt = renewed.get(target.receiptKey);
|
|
4871
|
-
return !this.runtimeToolReceiptStillOwned(receipt, target.leaseId);
|
|
4872
|
-
});
|
|
4873
|
-
if (lost) {
|
|
4874
|
-
assertion.reject(
|
|
4875
|
-
new RuntimeReceiptLeaseLostError({
|
|
4876
|
-
receiptKey: lost.receiptKey,
|
|
4877
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4878
|
-
leaseId: lost.leaseId,
|
|
4879
|
-
}),
|
|
4880
|
-
);
|
|
4881
|
-
} else {
|
|
4882
|
-
assertion.resolve();
|
|
4883
|
-
}
|
|
4884
|
-
}
|
|
4885
|
-
} catch (error) {
|
|
4886
|
-
for (const assertion of assertions) assertion.reject(error);
|
|
4887
|
-
}
|
|
4888
|
-
return;
|
|
4889
|
-
}
|
|
4890
|
-
|
|
4891
|
-
if (
|
|
4892
|
-
this.#options.getRuntimeStepReceipt ||
|
|
4893
|
-
this.#options.getRuntimeStepReceipts
|
|
4894
|
-
) {
|
|
4895
|
-
try {
|
|
4896
|
-
const latest = await this.getRuntimeStepReceipts(
|
|
4897
|
-
targets.map((target) => target.receiptKey),
|
|
4898
|
-
);
|
|
4899
|
-
for (const assertion of assertions) {
|
|
4900
|
-
const lost = assertion.targets.find((target) => {
|
|
4901
|
-
const receipt = latest.get(target.receiptKey);
|
|
4902
|
-
return !this.runtimeToolReceiptStillOwned(receipt, target.leaseId);
|
|
4903
|
-
});
|
|
4904
|
-
if (lost) {
|
|
4905
|
-
assertion.reject(
|
|
4906
|
-
new RuntimeReceiptLeaseLostError({
|
|
4907
|
-
receiptKey: lost.receiptKey,
|
|
4908
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4909
|
-
leaseId: lost.leaseId,
|
|
4910
|
-
}),
|
|
4911
|
-
);
|
|
4912
|
-
} else {
|
|
4913
|
-
assertion.resolve();
|
|
4914
|
-
}
|
|
4915
|
-
}
|
|
4916
|
-
} catch (error) {
|
|
4917
|
-
for (const assertion of assertions) assertion.reject(error);
|
|
4918
|
-
}
|
|
4919
|
-
return;
|
|
4920
|
-
}
|
|
4921
|
-
|
|
4922
|
-
const error = new RuntimeReceiptLeaseLostError({
|
|
4923
|
-
receiptKey: targets[0]?.receiptKey ?? 'unknown',
|
|
4924
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4925
|
-
leaseId: targets[0]?.leaseId ?? 'unknown',
|
|
4926
|
-
});
|
|
4927
|
-
for (const assertion of assertions) assertion.reject(error);
|
|
4928
|
-
}
|
|
4929
|
-
|
|
4930
|
-
private assertRuntimeToolReceiptHeartbeatResults(
|
|
4931
|
-
targets: Array<{ receiptKey: string; leaseId: string }>,
|
|
4932
|
-
receipts: Array<RuntimeStepReceipt | null>,
|
|
4933
|
-
): void {
|
|
4934
|
-
for (let index = 0; index < targets.length; index += 1) {
|
|
4935
|
-
const target = targets[index]!;
|
|
4936
|
-
const receipt = this.normalizeRuntimeStepReceipt(
|
|
4937
|
-
target.receiptKey,
|
|
4938
|
-
receipts[index],
|
|
4939
|
-
);
|
|
4940
|
-
if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) {
|
|
4941
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
4942
|
-
receiptKey: target.receiptKey,
|
|
4943
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4944
|
-
leaseId: target.leaseId,
|
|
4945
|
-
});
|
|
4946
|
-
}
|
|
4947
|
-
}
|
|
4948
|
-
}
|
|
4949
|
-
|
|
4950
|
-
private runtimeToolReceiptStillOwned(
|
|
4951
|
-
receipt: RuntimeStepReceipt | null | undefined,
|
|
4952
|
-
leaseId: string,
|
|
4953
|
-
): boolean {
|
|
4954
|
-
if (!receipt) return false;
|
|
4955
|
-
if (receipt.status === 'completed' || receipt.status === 'skipped') {
|
|
4956
|
-
return true;
|
|
4957
|
-
}
|
|
4958
|
-
if (receipt.status !== 'running') return false;
|
|
4959
|
-
if (receipt.leaseId !== leaseId) return false;
|
|
4960
|
-
const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
|
|
4961
|
-
return !ownerRunId || ownerRunId === this.currentReceiptOwnerRunId;
|
|
4962
|
-
}
|
|
4963
|
-
|
|
4964
4766
|
private runtimeToolReceiptStillOwnedByCurrentAttempt(
|
|
4965
4767
|
receipt: RuntimeStepReceipt | null | undefined,
|
|
4966
4768
|
leaseId: string | null | undefined,
|
|
@@ -4980,18 +4782,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4980
4782
|
return ownerAttempt === this.currentRunAttempt;
|
|
4981
4783
|
}
|
|
4982
4784
|
|
|
4983
|
-
private isOwnedClaimedRuntimeReceipt(
|
|
4984
|
-
receipt: RuntimeStepReceipt | null | undefined,
|
|
4985
|
-
): receipt is RuntimeStepReceipt {
|
|
4986
|
-
return (
|
|
4987
|
-
Boolean(receipt) &&
|
|
4988
|
-
receipt?.claimState !== 'existing' &&
|
|
4989
|
-
(receipt?.status === 'queued' ||
|
|
4990
|
-
receipt?.status === 'pending' ||
|
|
4991
|
-
receipt?.status === 'running')
|
|
4992
|
-
);
|
|
4993
|
-
}
|
|
4994
|
-
|
|
4995
4785
|
private async failRuntimeStepReceipt(
|
|
4996
4786
|
key: string,
|
|
4997
4787
|
_runId: string,
|
|
@@ -5359,91 +5149,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
5359
5149
|
return this.normalizeRuntimeStepReceipt(input.key, receipt);
|
|
5360
5150
|
}
|
|
5361
5151
|
|
|
5362
|
-
private async markRuntimeStepReceiptsRunning(
|
|
5363
|
-
receipts: Array<{
|
|
5364
|
-
key: string;
|
|
5365
|
-
runId: string;
|
|
5366
|
-
leaseId?: string | null;
|
|
5367
|
-
}>,
|
|
5368
|
-
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
5369
|
-
const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
|
|
5370
|
-
if (normalizedInputs.length === 0) return new Map();
|
|
5371
|
-
let marked: Array<RuntimeStepReceipt | null>;
|
|
5372
|
-
const markReceipts = this.#options.markRuntimeStepReceiptsRunning;
|
|
5373
|
-
if (markReceipts) {
|
|
5374
|
-
marked = await this.dispatchChunkedRuntimeReceiptRequest(
|
|
5375
|
-
normalizedInputs.map((receipt) => ({
|
|
5376
|
-
...receipt,
|
|
5377
|
-
runId: this.currentReceiptOwnerRunId,
|
|
5378
|
-
runAttempt: this.currentRunAttempt,
|
|
5379
|
-
})),
|
|
5380
|
-
(chunk) => markReceipts({ receipts: chunk }),
|
|
5381
|
-
);
|
|
5382
|
-
} else {
|
|
5383
|
-
marked = await Promise.all(
|
|
5384
|
-
normalizedInputs.map((receipt) =>
|
|
5385
|
-
this.markRuntimeStepReceiptRunning(receipt),
|
|
5386
|
-
),
|
|
5387
|
-
);
|
|
5388
|
-
}
|
|
5389
|
-
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
5390
|
-
for (let index = 0; index < marked.length; index += 1) {
|
|
5391
|
-
const normalized = this.normalizeRuntimeStepReceipt(
|
|
5392
|
-
normalizedInputs[index]?.key ?? '',
|
|
5393
|
-
marked[index],
|
|
5394
|
-
);
|
|
5395
|
-
if (normalized) byKey.set(normalized.key, normalized);
|
|
5396
|
-
}
|
|
5397
|
-
return byKey;
|
|
5398
|
-
}
|
|
5399
|
-
|
|
5400
|
-
private async markRuntimeStepReceiptsQueued(
|
|
5401
|
-
receipts: Array<{
|
|
5402
|
-
key: string;
|
|
5403
|
-
runId: string;
|
|
5404
|
-
leaseId?: string | null;
|
|
5405
|
-
}>,
|
|
5406
|
-
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
5407
|
-
const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
|
|
5408
|
-
if (normalizedInputs.length === 0) return new Map();
|
|
5409
|
-
if (!this.#options.markRuntimeStepReceiptsQueued) {
|
|
5410
|
-
return new Map();
|
|
5411
|
-
}
|
|
5412
|
-
const queued = await this.dispatchChunkedRuntimeReceiptRequest(
|
|
5413
|
-
normalizedInputs.map((receipt) => ({
|
|
5414
|
-
...receipt,
|
|
5415
|
-
runId: this.currentReceiptOwnerRunId,
|
|
5416
|
-
runAttempt: this.currentRunAttempt,
|
|
5417
|
-
})),
|
|
5418
|
-
(chunk) =>
|
|
5419
|
-
this.#options.markRuntimeStepReceiptsQueued!({ receipts: chunk }),
|
|
5420
|
-
);
|
|
5421
|
-
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
5422
|
-
for (let index = 0; index < queued.length; index += 1) {
|
|
5423
|
-
const normalized = this.normalizeRuntimeStepReceipt(
|
|
5424
|
-
normalizedInputs[index]?.key ?? '',
|
|
5425
|
-
queued[index],
|
|
5426
|
-
);
|
|
5427
|
-
if (normalized) byKey.set(normalized.key, normalized);
|
|
5428
|
-
}
|
|
5429
|
-
return byKey;
|
|
5430
|
-
}
|
|
5431
|
-
|
|
5432
|
-
private async parkRuntimeToolReceiptsQueued(
|
|
5433
|
-
requests: ToolCallRequest[],
|
|
5434
|
-
): Promise<void> {
|
|
5435
|
-
const targets = requests.flatMap((request) => {
|
|
5436
|
-
const receiptKey = request.receiptKey?.trim() || null;
|
|
5437
|
-
const leaseId = request.receiptLeaseId?.trim() || null;
|
|
5438
|
-
if (!receiptKey || !leaseId) return [];
|
|
5439
|
-
return [
|
|
5440
|
-
{ key: receiptKey, runId: this.currentReceiptOwnerRunId, leaseId },
|
|
5441
|
-
];
|
|
5442
|
-
});
|
|
5443
|
-
if (targets.length === 0) return;
|
|
5444
|
-
await this.markRuntimeStepReceiptsQueued(targets);
|
|
5445
|
-
}
|
|
5446
|
-
|
|
5447
5152
|
/**
|
|
5448
5153
|
* Enforce the receipt output contract and remember the output's serialized
|
|
5449
5154
|
* size. Idempotent per input: the first caller pays, everything downstream
|
|
@@ -6326,7 +6031,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6326
6031
|
|
|
6327
6032
|
private durableReceiptExecutionStore(): DurableReceiptExecutionStore {
|
|
6328
6033
|
return {
|
|
6329
|
-
enabled:
|
|
6034
|
+
enabled: this.durableDirectToolResultsBackedByReceipts,
|
|
6330
6035
|
get: (receiptKey) => this.getRuntimeStepReceipt(receiptKey),
|
|
6331
6036
|
getMany: (receiptKeys) => this.getRuntimeStepReceipts(receiptKeys),
|
|
6332
6037
|
claim: (
|
|
@@ -6403,35 +6108,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6403
6108
|
return durableRuntimeReceiptOutput<T>(receipt);
|
|
6404
6109
|
}
|
|
6405
6110
|
|
|
6406
|
-
private async waitForCompletedRuntimeToolReceipt(
|
|
6407
|
-
key: string,
|
|
6408
|
-
maxAttempts?: number,
|
|
6409
|
-
): Promise<RuntimeStepReceipt> {
|
|
6410
|
-
return await waitForCompletedRuntimeReceipt({
|
|
6411
|
-
receiptKey: key,
|
|
6412
|
-
store: this.durableReceiptExecutionStore(),
|
|
6413
|
-
maxAttempts,
|
|
6414
|
-
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
6415
|
-
});
|
|
6416
|
-
}
|
|
6417
|
-
|
|
6418
|
-
private async waitForCompletedRuntimeToolReceipts(
|
|
6419
|
-
keys: string[],
|
|
6420
|
-
maxAttempts?: number,
|
|
6421
|
-
): Promise<{
|
|
6422
|
-
completed: Map<string, RuntimeStepReceipt>;
|
|
6423
|
-
failed: Map<string, Error>;
|
|
6424
|
-
timedOut: Set<string>;
|
|
6425
|
-
}> {
|
|
6426
|
-
return await waitForCompletedRuntimeReceipts({
|
|
6427
|
-
receiptKeys: keys,
|
|
6428
|
-
store: this.durableReceiptExecutionStore(),
|
|
6429
|
-
maxAttempts,
|
|
6430
|
-
log: (message) => this.runtimeLog(message),
|
|
6431
|
-
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
6432
|
-
});
|
|
6433
|
-
}
|
|
6434
|
-
|
|
6435
6111
|
private async executeWithRuntimeReceipt<T>(
|
|
6436
6112
|
operation: DurableCtxOperation,
|
|
6437
6113
|
id: string,
|
|
@@ -6525,7 +6201,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6525
6201
|
onClaimedResult: opts.onClaimedResult,
|
|
6526
6202
|
shouldPersistFailure: opts.shouldPersistFailure,
|
|
6527
6203
|
markRunningBeforeExecute: opts.markRunningBeforeExecute,
|
|
6528
|
-
completedCacheOnly:
|
|
6204
|
+
completedCacheOnly: true,
|
|
6529
6205
|
withCompletedReceiptHydration: (hydrate) =>
|
|
6530
6206
|
this.withRuntimeReceiptHydrationTurn(hydrate, {
|
|
6531
6207
|
source: 'durable_operation',
|
|
@@ -6632,7 +6308,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6632
6308
|
*
|
|
6633
6309
|
* The external receipt adapter is intentionally local to this migration.
|
|
6634
6310
|
* It checks the exact compatibility identities passed by the map, while the
|
|
6635
|
-
* map remains the sole writer for its member receipts,
|
|
6311
|
+
* map remains the sole writer for its member receipts, invocation delivery,
|
|
6636
6312
|
* and completion barrier. In particular, do not turn an always-fresh map
|
|
6637
6313
|
* request into a provider-idempotent request here: `apiOptions` is forwarded
|
|
6638
6314
|
* byte-for-byte and decides whether the historical wire carried an identity.
|
|
@@ -6806,19 +6482,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6806
6482
|
});
|
|
6807
6483
|
}
|
|
6808
6484
|
|
|
6809
|
-
/**
|
|
6810
|
-
* A forced call bypasses an earlier run's shared result, so its physical
|
|
6811
|
-
* gateway invocation must bypass that result's replay identity too. Keep
|
|
6812
|
-
* the transport key deterministic for one logical call: a retry of that
|
|
6813
|
-
* call still converges, while a later explicit force gets a fresh identity.
|
|
6814
|
-
*/
|
|
6485
|
+
/** One replay-stable physical delivery identity for each invocation. */
|
|
6815
6486
|
private toolTransportReceiptKey(input: {
|
|
6816
6487
|
toolId: string;
|
|
6817
|
-
resultReceiptKey: string;
|
|
6818
6488
|
providerOperationKey: string;
|
|
6819
|
-
force?: boolean;
|
|
6820
6489
|
}): string {
|
|
6821
|
-
if (input.force !== true) return input.resultReceiptKey;
|
|
6822
6490
|
return `${buildDurableToolReceiptPrefix({
|
|
6823
6491
|
orgId: this.#options.orgId,
|
|
6824
6492
|
toolId: input.toolId,
|
|
@@ -6841,41 +6509,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6841
6509
|
});
|
|
6842
6510
|
}
|
|
6843
6511
|
|
|
6844
|
-
/**
|
|
6845
|
-
|
|
6846
|
-
* execution relay still needs a semantic signal that an explicit force is
|
|
6847
|
-
* a new provider operation. Unsafe-write operations already carry their
|
|
6848
|
-
* forced provider key directly, so only ordinary calls need this attested
|
|
6849
|
-
* canonical-operation salt.
|
|
6850
|
-
*/
|
|
6851
|
-
private providerForceRefreshIdentitiesForToolCall(input: {
|
|
6512
|
+
/** Keep independent misses distinct in the gateway's canonical operation. */
|
|
6513
|
+
private providerInvocationOptionsForToolCall(input: {
|
|
6852
6514
|
operationKey: string;
|
|
6853
|
-
force?: boolean;
|
|
6854
6515
|
requiresExecutionFence?: boolean;
|
|
6855
6516
|
logicalCallId?: string;
|
|
6856
|
-
}): string[] {
|
|
6857
|
-
if (input.force !== true || input.requiresExecutionFence === true) {
|
|
6858
|
-
return [];
|
|
6859
|
-
}
|
|
6860
|
-
return [
|
|
6861
|
-
this.providerIdempotencyKeyForToolCall({
|
|
6862
|
-
operationKey: input.operationKey,
|
|
6863
|
-
force: true,
|
|
6864
|
-
logicalCallId: input.logicalCallId,
|
|
6865
|
-
}),
|
|
6866
|
-
];
|
|
6867
|
-
}
|
|
6868
|
-
|
|
6869
|
-
private providerForceRefreshOptionsForToolCall(input: {
|
|
6870
|
-
operationKey: string;
|
|
6871
6517
|
force?: boolean;
|
|
6872
|
-
requiresExecutionFence?: boolean;
|
|
6873
|
-
logicalCallId?: string;
|
|
6874
6518
|
}): Pick<ToolExecutionApiOptions, 'providerOperationForceRefreshIdentities'> {
|
|
6875
|
-
|
|
6876
|
-
|
|
6877
|
-
|
|
6878
|
-
|
|
6519
|
+
return input.requiresExecutionFence === true
|
|
6520
|
+
? {}
|
|
6521
|
+
: {
|
|
6522
|
+
providerOperationForceRefreshIdentities: [
|
|
6523
|
+
this.providerInvocationKeyForToolCall(input),
|
|
6524
|
+
],
|
|
6525
|
+
};
|
|
6879
6526
|
}
|
|
6880
6527
|
|
|
6881
6528
|
/**
|
|
@@ -7577,15 +7224,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7577
7224
|
private getCachedToolResult(
|
|
7578
7225
|
toolId: string,
|
|
7579
7226
|
rowCacheKey: string,
|
|
7580
|
-
|
|
7227
|
+
_path: 'mapped' | 'direct' = 'mapped',
|
|
7581
7228
|
): ToolBatchResult | undefined {
|
|
7582
|
-
|
|
7583
|
-
|
|
7584
|
-
? this.durableDirectToolResultsBackedByReceipts
|
|
7585
|
-
: this.durableMappedToolResultsBackedByReceipts
|
|
7586
|
-
) {
|
|
7587
|
-
return undefined;
|
|
7588
|
-
}
|
|
7229
|
+
// Receipt-backed calls never add payloads here. Retained outgoing entries
|
|
7230
|
+
// may predate that capability and remain completed cache hits.
|
|
7589
7231
|
return this.checkpoint.completedToolBatches[toolId]?.[rowCacheKey];
|
|
7590
7232
|
}
|
|
7591
7233
|
|
|
@@ -7636,32 +7278,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7636
7278
|
if (!this.checkpoint.completedToolBatches[toolId]) {
|
|
7637
7279
|
this.checkpoint.completedToolBatches[toolId] = {};
|
|
7638
7280
|
}
|
|
7639
|
-
this.checkpoint.completedToolBatches[toolId]![rowCacheKey]
|
|
7281
|
+
this.checkpoint.completedToolBatches[toolId]![rowCacheKey] ??= {
|
|
7640
7282
|
done: true,
|
|
7641
7283
|
result,
|
|
7642
7284
|
};
|
|
7643
7285
|
}
|
|
7644
7286
|
|
|
7645
|
-
private buildToolResultCacheKey(input: {
|
|
7646
|
-
rowId: number;
|
|
7647
|
-
tableNamespace?: string;
|
|
7648
|
-
rowKey?: string;
|
|
7649
|
-
callId?: string;
|
|
7650
|
-
}): string {
|
|
7651
|
-
const scope =
|
|
7652
|
-
this.currentGovernance.currentRunId || this.#options.runId || 'run';
|
|
7653
|
-
if (input.callId?.trim()) {
|
|
7654
|
-
return `${scope}:${input.callId.trim()}`;
|
|
7655
|
-
}
|
|
7656
|
-
if (input.rowKey?.trim()) {
|
|
7657
|
-
return `${scope}:${input.rowKey.trim()}`;
|
|
7658
|
-
}
|
|
7659
|
-
if (input.tableNamespace?.trim()) {
|
|
7660
|
-
return `${scope}:${input.tableNamespace.trim()}:${input.rowId}`;
|
|
7661
|
-
}
|
|
7662
|
-
return `${scope}:direct:${input.rowId}`;
|
|
7663
|
-
}
|
|
7664
|
-
|
|
7665
7287
|
private async resolveToolResultMetadata(
|
|
7666
7288
|
toolId: string,
|
|
7667
7289
|
): Promise<ToolResultMetadataInput> {
|
|
@@ -7753,6 +7375,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7753
7375
|
private materializeToolExecutionResult(
|
|
7754
7376
|
result: ToolExecuteResult,
|
|
7755
7377
|
): ToolExecuteResult {
|
|
7378
|
+
result = projectNativeBatchToolResponse(
|
|
7379
|
+
result,
|
|
7380
|
+
this.currentRuntimeContract.semantics.tools.nativeBatchResponse,
|
|
7381
|
+
);
|
|
7756
7382
|
if (
|
|
7757
7383
|
this.currentToolResponseContract === RAW_V2_TOOL_RESPONSE_CONTRACT ||
|
|
7758
7384
|
!this.hasPinnedToolResponseContract ||
|
|
@@ -8136,40 +7762,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8136
7762
|
`Durable tool call ${receiptKey} completed live after its receipt lease moved; using live result without overwriting the receipt.`,
|
|
8137
7763
|
);
|
|
8138
7764
|
}
|
|
8139
|
-
const finalWrapped =
|
|
8140
|
-
(completed?.status === 'completed' || completed?.status === 'skipped') &&
|
|
8141
|
-
completed.output !== undefined
|
|
8142
|
-
? await this.wrapToolExecutionResult({
|
|
8143
|
-
toolId,
|
|
8144
|
-
status:
|
|
8145
|
-
completed.output === null || completed.output === undefined
|
|
8146
|
-
? 'no_result'
|
|
8147
|
-
: 'completed',
|
|
8148
|
-
result: this.runtimeReceiptOutput(completed),
|
|
8149
|
-
requestInput: request.input,
|
|
8150
|
-
execution: toolExecutionMetadataForOutcome(
|
|
8151
|
-
completed.runId === this.currentReceiptOwnerRunId
|
|
8152
|
-
? {
|
|
8153
|
-
kind: 'live',
|
|
8154
|
-
cacheKey,
|
|
8155
|
-
receiptKey: request.receiptKey,
|
|
8156
|
-
}
|
|
8157
|
-
: {
|
|
8158
|
-
kind: 'cache',
|
|
8159
|
-
cacheKey,
|
|
8160
|
-
receiptKey: receiptKey ?? '',
|
|
8161
|
-
attachedToReceiptKey: receiptKey,
|
|
8162
|
-
},
|
|
8163
|
-
),
|
|
8164
|
-
})
|
|
8165
|
-
: wrapped;
|
|
8166
7765
|
if (request.cacheable !== false) {
|
|
8167
|
-
this.cacheToolResult(toolId, cacheKey,
|
|
7766
|
+
this.cacheToolResult(toolId, cacheKey, wrapped);
|
|
8168
7767
|
}
|
|
8169
7768
|
|
|
8170
7769
|
const resolver = this.toolCallResolvers.get(request.callId);
|
|
8171
7770
|
if (resolver) {
|
|
8172
|
-
resolver.resolve(
|
|
7771
|
+
resolver.resolve(wrapped);
|
|
8173
7772
|
this.toolCallResolvers.delete(request.callId);
|
|
8174
7773
|
}
|
|
8175
7774
|
|
|
@@ -8185,7 +7784,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8185
7784
|
error: null,
|
|
8186
7785
|
dataPatch: {},
|
|
8187
7786
|
});
|
|
8188
|
-
return
|
|
7787
|
+
return wrapped;
|
|
8189
7788
|
}
|
|
8190
7789
|
|
|
8191
7790
|
private async rejectToolCall(
|
|
@@ -9003,6 +8602,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9003
8602
|
Math.max(0, failedCount - staleFailedKeys.size),
|
|
9004
8603
|
progressSupersededOffset: () =>
|
|
9005
8604
|
staleCompletedKeys.size + staleFailedKeys.size,
|
|
8605
|
+
progressSkippedOffset: () => duplicateReuseCount,
|
|
9006
8606
|
progressTotalRows: () => totalInputCount,
|
|
9007
8607
|
rowFeed: execution.rowFeed,
|
|
9008
8608
|
retainedRowsMemoryBudgetBytes:
|
|
@@ -10231,6 +9831,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10231
9831
|
progressCompletedOffset?: number | (() => number);
|
|
10232
9832
|
progressFailedOffset?: number | (() => number);
|
|
10233
9833
|
progressSupersededOffset?: number | (() => number);
|
|
9834
|
+
/** Input duplicates discovered after admission; excludes resolver outcomes. */
|
|
9835
|
+
progressSkippedOffset?: number | (() => number);
|
|
10234
9836
|
progressTotalRows?: number | (() => number);
|
|
10235
9837
|
/**
|
|
10236
9838
|
* When set, `items`, `executionRowKeys` and `executionRowIndexes` grow
|
|
@@ -10476,6 +10078,37 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10476
10078
|
} | null = null;
|
|
10477
10079
|
let lastFrameFlushAt = 0;
|
|
10478
10080
|
let rowsSinceFrameFlush = 0;
|
|
10081
|
+
// Scalars only: resolver admission is visible before the first row's
|
|
10082
|
+
// durable completion, without publishing one event per row or tool call.
|
|
10083
|
+
let startedResolverRows = 0;
|
|
10084
|
+
let activeResolverRows = 0;
|
|
10085
|
+
let tracksResolverActivity = false;
|
|
10086
|
+
// Persistence can move current resolvers between successful, failed and
|
|
10087
|
+
// superseded offsets. They already contribute to startedResolverRows:
|
|
10088
|
+
// only the settled baseline before this pool started belongs in admission.
|
|
10089
|
+
const initiallyStartedRows =
|
|
10090
|
+
liveNumber(runtimeOptions?.progressCompletedOffset, completedRows) +
|
|
10091
|
+
liveNumber(runtimeOptions?.progressFailedOffset, 0) +
|
|
10092
|
+
liveNumber(runtimeOptions?.progressSupersededOffset, 0);
|
|
10093
|
+
const initiallySkippedRows = liveNumber(
|
|
10094
|
+
runtimeOptions?.progressSkippedOffset,
|
|
10095
|
+
0,
|
|
10096
|
+
);
|
|
10097
|
+
const mapActivityCounts = () => {
|
|
10098
|
+
const startedRows =
|
|
10099
|
+
initiallyStartedRows +
|
|
10100
|
+
(liveNumber(
|
|
10101
|
+
runtimeOptions?.progressSkippedOffset,
|
|
10102
|
+
initiallySkippedRows,
|
|
10103
|
+
) -
|
|
10104
|
+
initiallySkippedRows) +
|
|
10105
|
+
startedResolverRows;
|
|
10106
|
+
return {
|
|
10107
|
+
startedRows,
|
|
10108
|
+
activeRows: activeResolverRows,
|
|
10109
|
+
waitingRows: currentTotalRows() - startedRows,
|
|
10110
|
+
};
|
|
10111
|
+
};
|
|
10479
10112
|
const updateMapFrameProgress = (input: {
|
|
10480
10113
|
status?: MapExecutionFrame['status'];
|
|
10481
10114
|
completedRowKey?: string | null;
|
|
@@ -10607,6 +10240,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10607
10240
|
completedRowKeys.size,
|
|
10608
10241
|
failedRows,
|
|
10609
10242
|
totalRows: currentTotalRows(),
|
|
10243
|
+
...(progressEventType === 'map.progress' && tracksResolverActivity
|
|
10244
|
+
? mapActivityCounts()
|
|
10245
|
+
: {}),
|
|
10610
10246
|
supersededRows: liveNumber(
|
|
10611
10247
|
runtimeOptions?.progressSupersededOffset,
|
|
10612
10248
|
existing.supersededRows ?? 0,
|
|
@@ -11254,6 +10890,39 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11254
10890
|
items[idx] = null as unknown as T;
|
|
11255
10891
|
}
|
|
11256
10892
|
: null;
|
|
10893
|
+
let lastMapActivityKey: string | null = null;
|
|
10894
|
+
tracksResolverActivity = true;
|
|
10895
|
+
const publishMapActivity = () => {
|
|
10896
|
+
const counts = mapActivityCounts();
|
|
10897
|
+
const key = `${counts.startedRows}:${counts.activeRows}:${counts.waitingRows}`;
|
|
10898
|
+
if (key === lastMapActivityKey) return;
|
|
10899
|
+
lastMapActivityKey = key;
|
|
10900
|
+
// Admission samples do not rebuild checkpoint key sets. The map's
|
|
10901
|
+
// writer continues to own durable settlement and checkpoint flushing.
|
|
10902
|
+
this.emitExecutionEvent({
|
|
10903
|
+
type: 'map.progress',
|
|
10904
|
+
mapInvocationId: mapScope.mapInvocationId,
|
|
10905
|
+
mapNodeId: mapScope.mapNodeId ?? null,
|
|
10906
|
+
logicalNamespace: mapScope.logicalNamespace,
|
|
10907
|
+
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
10908
|
+
completedRows:
|
|
10909
|
+
liveNumber(runtimeOptions?.progressCompletedOffset, completedRows) +
|
|
10910
|
+
(liveFrameSets?.completedRowKeys.size ?? 0),
|
|
10911
|
+
failedRows:
|
|
10912
|
+
liveNumber(runtimeOptions?.progressFailedOffset, 0) +
|
|
10913
|
+
(liveFrameSets?.failedRowsCount ?? 0),
|
|
10914
|
+
totalRows: currentTotalRows(),
|
|
10915
|
+
supersededRows: liveNumber(runtimeOptions?.progressSupersededOffset, 0),
|
|
10916
|
+
...counts,
|
|
10917
|
+
...this.inlineChildAggregateEventFields(),
|
|
10918
|
+
at: Date.now(),
|
|
10919
|
+
});
|
|
10920
|
+
};
|
|
10921
|
+
const mapActivityTimer = setInterval(
|
|
10922
|
+
publishMapActivity,
|
|
10923
|
+
PROGRESS_HEARTBEAT_INTERVAL_MS,
|
|
10924
|
+
);
|
|
10925
|
+
mapActivityTimer.unref();
|
|
11257
10926
|
const workers = Array.from({ length: workerCount }, async () => {
|
|
11258
10927
|
while (firstRowError == null) {
|
|
11259
10928
|
if (nextRowIndex >= items.length) {
|
|
@@ -11270,7 +10939,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11270
10939
|
releaseDispatchedInputRow?.(idx);
|
|
11271
10940
|
settledResults[idx] = await mapRowFlow.run({ baseRow }, async () => {
|
|
11272
10941
|
this.assertRowAdmissionOpen();
|
|
11273
|
-
|
|
10942
|
+
startedResolverRows += 1;
|
|
10943
|
+
activeResolverRows += 1;
|
|
10944
|
+
try {
|
|
10945
|
+
return await runRow(idx, baseRow);
|
|
10946
|
+
} finally {
|
|
10947
|
+
activeResolverRows -= 1;
|
|
10948
|
+
}
|
|
11274
10949
|
});
|
|
11275
10950
|
} catch (error) {
|
|
11276
10951
|
// RuntimeFlow has already returned the row slot and memory lease.
|
|
@@ -11364,6 +11039,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11364
11039
|
settled = await Promise.allSettled(workers);
|
|
11365
11040
|
}
|
|
11366
11041
|
} finally {
|
|
11042
|
+
clearInterval(mapActivityTimer);
|
|
11043
|
+
publishMapActivity();
|
|
11367
11044
|
datasetStep.items = currentTotalRows();
|
|
11368
11045
|
this.processedRowCount = currentTotalRows();
|
|
11369
11046
|
mapStallObserver?.stopHeartbeat();
|
|
@@ -12930,7 +12607,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12930
12607
|
options.receiptWaitMs,
|
|
12931
12608
|
);
|
|
12932
12609
|
}
|
|
12933
|
-
const executionScope = this.currentExecutionScope;
|
|
12934
12610
|
const normalizedKey = this.normalizeContextKey(key, 'tool');
|
|
12935
12611
|
const toolCachePolicy = this.effectiveToolCallCachePolicy(options);
|
|
12936
12612
|
const toolRequestIdentity = deriveToolRequestIdentity({
|
|
@@ -13028,10 +12704,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13028
12704
|
...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
|
|
13029
12705
|
];
|
|
13030
12706
|
|
|
13031
|
-
const executeTool = async (
|
|
13032
|
-
leaseId?: string | null;
|
|
13033
|
-
retainExternalCallSlot?: (release: () => void) => void;
|
|
13034
|
-
}): Promise<unknown> => {
|
|
12707
|
+
const executeTool = async (): Promise<unknown> => {
|
|
13035
12708
|
if (eventWaitHandler) {
|
|
13036
12709
|
return this.executeIntegrationEventWaitTool(
|
|
13037
12710
|
toolId,
|
|
@@ -13291,7 +12964,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13291
12964
|
});
|
|
13292
12965
|
};
|
|
13293
12966
|
try {
|
|
13294
|
-
const directReceiptLeaseId = context?.leaseId?.trim() || null;
|
|
13295
12967
|
if (!logicalCallId) {
|
|
13296
12968
|
throw new Error(
|
|
13297
12969
|
'Direct tool execution requires a stable logical call identity.',
|
|
@@ -13299,17 +12971,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13299
12971
|
}
|
|
13300
12972
|
const useDirectResultReceipts =
|
|
13301
12973
|
cacheableToolResult && !bypassDirectResultCache;
|
|
13302
|
-
const providerOperationKey = this.
|
|
12974
|
+
const providerOperationKey = this.providerInvocationKeyForToolCall({
|
|
13303
12975
|
operationKey: providerIdempotencyKeyBase,
|
|
13304
|
-
force: cacheableToolResult && toolCachePolicy.force,
|
|
13305
12976
|
logicalCallId,
|
|
12977
|
+
force: toolCachePolicy.force,
|
|
13306
12978
|
});
|
|
13307
12979
|
const physicalDirectKey = useDirectResultReceipts
|
|
13308
12980
|
? this.toolTransportReceiptKey({
|
|
13309
12981
|
toolId,
|
|
13310
|
-
resultReceiptKey: directCacheKey,
|
|
13311
12982
|
providerOperationKey,
|
|
13312
|
-
force: toolCachePolicy.force,
|
|
13313
12983
|
})
|
|
13314
12984
|
: `${buildDurableToolReceiptPrefix({
|
|
13315
12985
|
orgId: this.#options.orgId ?? 'unknown-org',
|
|
@@ -13323,13 +12993,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13323
12993
|
runId: this.currentRunId,
|
|
13324
12994
|
logicalCallId,
|
|
13325
12995
|
force: toolCachePolicy.force,
|
|
13326
|
-
canStoreResult:
|
|
13327
|
-
this.#options.claimRuntimeStepReceipt &&
|
|
13328
|
-
(this.#options.getRuntimeStepReceipt ||
|
|
13329
|
-
this.#options.getRuntimeStepReceipts) &&
|
|
13330
|
-
(this.#options.completeRuntimeStepReceipt ||
|
|
13331
|
-
this.#options.completeRuntimeStepReceipts),
|
|
13332
|
-
),
|
|
12996
|
+
canStoreResult: this.durableDirectToolResultsBackedByReceipts,
|
|
13333
12997
|
});
|
|
13334
12998
|
const toolRetryPolicy = await this.#options
|
|
13335
12999
|
.getToolRetryPolicy?.(toolId, input)
|
|
@@ -13365,28 +13029,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13365
13029
|
...(useDirectResultReceipts
|
|
13366
13030
|
? {
|
|
13367
13031
|
durableCallReceiptKey: physicalDirectKey,
|
|
13032
|
+
legacyProviderRetryIdentities: toolCachePolicy.force
|
|
13033
|
+
? undefined
|
|
13034
|
+
: [
|
|
13035
|
+
{
|
|
13036
|
+
transportReceiptKey: directCacheKey,
|
|
13037
|
+
providerOperationKey:
|
|
13038
|
+
providerIdempotencyKeyBase,
|
|
13039
|
+
},
|
|
13040
|
+
],
|
|
13368
13041
|
completedResultCache: true,
|
|
13369
13042
|
executionAuthScopeDigest,
|
|
13370
|
-
...(directReceiptLeaseId &&
|
|
13371
|
-
(this.#options.heartbeatRuntimeStepReceipts ||
|
|
13372
|
-
this.#options.getRuntimeStepReceipt ||
|
|
13373
|
-
this.#options.getRuntimeStepReceipts)
|
|
13374
|
-
? {
|
|
13375
|
-
beforeProviderCall: async () => {
|
|
13376
|
-
await this.assertRuntimeToolReceiptOwnership([
|
|
13377
|
-
{
|
|
13378
|
-
callId: 'direct',
|
|
13379
|
-
cacheKey: directCacheKey,
|
|
13380
|
-
receiptKey: directCacheKey,
|
|
13381
|
-
receiptLeaseId: directReceiptLeaseId,
|
|
13382
|
-
rowId: -1,
|
|
13383
|
-
toolId,
|
|
13384
|
-
input,
|
|
13385
|
-
},
|
|
13386
|
-
]);
|
|
13387
|
-
},
|
|
13388
|
-
}
|
|
13389
|
-
: {}),
|
|
13390
13043
|
}
|
|
13391
13044
|
: {}),
|
|
13392
13045
|
...(toolRetryPolicy?.requiresExecutionFence === true
|
|
@@ -13401,15 +13054,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13401
13054
|
: physicalDirectKey,
|
|
13402
13055
|
}
|
|
13403
13056
|
: {}),
|
|
13404
|
-
...(
|
|
13405
|
-
|
|
13406
|
-
|
|
13407
|
-
|
|
13408
|
-
|
|
13409
|
-
|
|
13410
|
-
|
|
13411
|
-
})
|
|
13412
|
-
: {}),
|
|
13057
|
+
...this.providerInvocationOptionsForToolCall({
|
|
13058
|
+
operationKey: providerIdempotencyKeyBase,
|
|
13059
|
+
requiresExecutionFence:
|
|
13060
|
+
toolRetryPolicy?.requiresExecutionFence,
|
|
13061
|
+
logicalCallId,
|
|
13062
|
+
force: toolCachePolicy.force,
|
|
13063
|
+
}),
|
|
13413
13064
|
playNodeScope: buildPlayNodeScope({
|
|
13414
13065
|
toolId,
|
|
13415
13066
|
callKey: normalizedKey,
|
|
@@ -13419,7 +13070,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13419
13070
|
options?.timeoutMs,
|
|
13420
13071
|
this.currentAuthoringContractEdition,
|
|
13421
13072
|
),
|
|
13422
|
-
retainToolSlot: context?.retainExternalCallSlot,
|
|
13423
13073
|
},
|
|
13424
13074
|
});
|
|
13425
13075
|
return await this.wrapToolExecutionResult({
|
|
@@ -13558,20 +13208,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13558
13208
|
|
|
13559
13209
|
const rowId = store.rowId;
|
|
13560
13210
|
const fieldName = store.fieldName;
|
|
13561
|
-
|
|
13562
|
-
|
|
13563
|
-
|
|
13564
|
-
store.rowKey?.trim() || String(rowId),
|
|
13565
|
-
normalizedKey,
|
|
13566
|
-
toolId,
|
|
13567
|
-
toolRequestIdentity,
|
|
13568
|
-
].join(':');
|
|
13569
|
-
if (this.toolCallResolvers.has(callId)) {
|
|
13570
|
-
throw new Error(
|
|
13571
|
-
`ctx.tools.execute("${normalizedKey}") was called more than once concurrently for the same row. ` +
|
|
13572
|
-
'Use a unique id for each row-scoped tool call.',
|
|
13573
|
-
);
|
|
13574
|
-
}
|
|
13211
|
+
// Cache identity is shared by matching completed results. Delivery
|
|
13212
|
+
// identity belongs to this invocation, even when callers reuse an id.
|
|
13213
|
+
const callId = logicalCallId;
|
|
13575
13214
|
|
|
13576
13215
|
const toolResultCacheKey = durableCacheKey;
|
|
13577
13216
|
const cached = !cacheableToolResult
|
|
@@ -13597,7 +13236,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13597
13236
|
const rowToolRetryPolicy = await this.#options
|
|
13598
13237
|
.getToolRetryPolicy?.(toolId, input)
|
|
13599
13238
|
.catch(() => null);
|
|
13600
|
-
|
|
13601
13239
|
// ADR 0018: record this leg before it runs. `producer` still names the
|
|
13602
13240
|
// last leg (the usual cascade winner); `producers` keeps every leg, which
|
|
13603
13241
|
// is the only durable record a losing leg ever gets — a failed tool call
|
|
@@ -13641,7 +13279,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13641
13279
|
// The tail of that wall. The dispatcher stamps the request when this
|
|
13642
13280
|
// call's provider body finished parsing; the difference here is what
|
|
13643
13281
|
// receipt completion and hand-back cost after the provider was done. A
|
|
13644
|
-
// cache hit
|
|
13282
|
+
// cache hit never reads a body, so it leaves no stamp and
|
|
13645
13283
|
// record nothing rather than a delivery of zero.
|
|
13646
13284
|
const observeResultDelivery = () => {
|
|
13647
13285
|
const bodyCompletedAtMs = dispatchRequest.providerBodyCompletedAtMs;
|
|
@@ -15389,7 +15027,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
15389
15027
|
*
|
|
15390
15028
|
* Gated on the same stamp `resultDelivery` is gated on — a body this request
|
|
15391
15029
|
* really read — so every delivery sub-phase counts the same calls and their
|
|
15392
|
-
* sum can be checked against the whole. A cache hit
|
|
15030
|
+
* sum can be checked against the whole. A cache hit and a
|
|
15393
15031
|
* member that never reached the provider leave the stamp unset and record
|
|
15394
15032
|
* nothing rather than a wrap of zero.
|
|
15395
15033
|
*/
|
|
@@ -15521,179 +15159,38 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
15521
15159
|
|
|
15522
15160
|
// ——— Batched tool call execution ———
|
|
15523
15161
|
|
|
15524
|
-
|
|
15525
|
-
|
|
15526
|
-
|
|
15527
|
-
|
|
15528
|
-
|
|
15529
|
-
|
|
15530
|
-
|
|
15531
|
-
|
|
15532
|
-
|
|
15533
|
-
|
|
15534
|
-
|
|
15535
|
-
|
|
15536
|
-
|
|
15537
|
-
|
|
15538
|
-
|
|
15539
|
-
|
|
15540
|
-
(
|
|
15541
|
-
const
|
|
15542
|
-
|
|
15543
|
-
if (!earliest) return expiresAt;
|
|
15544
|
-
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
15545
|
-
? expiresAt
|
|
15546
|
-
: earliest;
|
|
15547
|
-
},
|
|
15548
|
-
null,
|
|
15549
|
-
);
|
|
15550
|
-
let leaseLost: unknown | null = null;
|
|
15551
|
-
let ownershipConfirmedUntilMs = Number.isFinite(
|
|
15552
|
-
Date.parse(earliestLeaseExpiry ?? ''),
|
|
15553
|
-
)
|
|
15554
|
-
? Date.parse(earliestLeaseExpiry!)
|
|
15555
|
-
: Date.now() + PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS;
|
|
15556
|
-
const firstOwnedRequest = ownedRequests[0]!;
|
|
15557
|
-
const heartbeatOwnedRequests = async () => {
|
|
15558
|
-
// Receipt claims are acquired in bulk before the provider-shaped
|
|
15559
|
-
// dispatcher admits every call. Renew the undispatched tail as well as
|
|
15560
|
-
// in-flight calls; otherwise a slow provider can let later claims
|
|
15561
|
-
// expire while they are still waiting behind the concurrency gate.
|
|
15562
|
-
const unsettled = ownedRequests.filter((request) =>
|
|
15563
|
-
this.toolCallResolvers.has(request.callId),
|
|
15564
|
-
);
|
|
15565
|
-
if (unsettled.length === 0) return 'terminal' as const;
|
|
15566
|
-
const byReceiptKey = new Map<string, ToolCallRequest[]>();
|
|
15567
|
-
for (const request of unsettled) {
|
|
15568
|
-
const receiptKey = request.receiptKey!;
|
|
15569
|
-
const group = byReceiptKey.get(receiptKey) ?? [];
|
|
15570
|
-
group.push(request);
|
|
15571
|
-
byReceiptKey.set(receiptKey, group);
|
|
15162
|
+
/**
|
|
15163
|
+
* Concrete RuntimeStepReceipt adapter for the map receipt-cohort Module.
|
|
15164
|
+
*
|
|
15165
|
+
* Scalar and provider-native map calls share completed-result cache reads
|
|
15166
|
+
* and atomic successful inserts. A miss has no durable execution owner.
|
|
15167
|
+
*/
|
|
15168
|
+
private createRuntimeStepToolCallReceiptCohortStore(input: {
|
|
15169
|
+
toolId: string;
|
|
15170
|
+
requestForReceipt: ReadonlyMap<string, ToolCallRequest>;
|
|
15171
|
+
/** Each invocation sharing a completed-result key, for manifests and timing. */
|
|
15172
|
+
requestsForReceipt: ReadonlyMap<string, readonly ToolCallRequest[]>;
|
|
15173
|
+
}): ToolCallReceiptCohortStore {
|
|
15174
|
+
const requestsForPublication = (receipt: {
|
|
15175
|
+
resultReceiptKey: string;
|
|
15176
|
+
memberIds?: readonly string[];
|
|
15177
|
+
}): readonly ToolCallRequest[] => {
|
|
15178
|
+
if (!receipt.memberIds) {
|
|
15179
|
+
const request = input.requestForReceipt.get(receipt.resultReceiptKey);
|
|
15180
|
+
return request ? [request] : [];
|
|
15572
15181
|
}
|
|
15573
|
-
|
|
15574
|
-
|
|
15575
|
-
|
|
15576
|
-
|
|
15577
|
-
|
|
15578
|
-
const leaseIds = entries.map(
|
|
15579
|
-
([, requests]) => requests[0]!.receiptLeaseId!,
|
|
15580
|
-
);
|
|
15581
|
-
const receipts = await heartbeatReceipts({
|
|
15582
|
-
runId: this.currentReceiptOwnerRunId,
|
|
15583
|
-
runAttempt: this.currentRunAttempt,
|
|
15584
|
-
keys,
|
|
15585
|
-
leaseIds,
|
|
15586
|
-
});
|
|
15587
|
-
const renewedExpiries: number[] = [];
|
|
15588
|
-
for (let index = 0; index < entries.length; index += 1) {
|
|
15589
|
-
const [receiptKey, requests] = entries[index]!;
|
|
15590
|
-
const leaseId = leaseIds[index]!;
|
|
15591
|
-
// Completion can race this bulk heartbeat response. A settled
|
|
15592
|
-
// receipt group no longer needs ownership and must not turn that
|
|
15593
|
-
// race into a false lease-loss failure or hold the confirmation
|
|
15594
|
-
// horizon at its old expiry.
|
|
15595
|
-
if (
|
|
15596
|
-
this.runtimeToolReceiptCompletionsInFlight.has(receiptKey) ||
|
|
15597
|
-
!requests.some((request) =>
|
|
15598
|
-
this.toolCallResolvers.has(request.callId),
|
|
15599
|
-
)
|
|
15600
|
-
) {
|
|
15601
|
-
continue;
|
|
15602
|
-
}
|
|
15603
|
-
const receipt = receipts[index] ?? null;
|
|
15604
|
-
const renewedExpiryMs = Date.parse(receipt?.leaseExpiresAt ?? '');
|
|
15605
|
-
if (
|
|
15606
|
-
!this.runtimeToolReceiptStillOwned(receipt, leaseId) ||
|
|
15607
|
-
!Number.isFinite(renewedExpiryMs)
|
|
15608
|
-
) {
|
|
15609
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
15610
|
-
receiptKey,
|
|
15611
|
-
runId: this.currentReceiptOwnerRunId,
|
|
15612
|
-
leaseId,
|
|
15613
|
-
});
|
|
15614
|
-
}
|
|
15615
|
-
renewedExpiries.push(renewedExpiryMs);
|
|
15616
|
-
}
|
|
15617
|
-
if (renewedExpiries.length === 0) return 'terminal' as const;
|
|
15618
|
-
ownershipConfirmedUntilMs = Math.min(...renewedExpiries);
|
|
15619
|
-
return 'active' as const;
|
|
15620
|
-
};
|
|
15621
|
-
const perCallHeartbeatIntervalMs = runtimeLeaseHeartbeatIntervalFromExpiry({
|
|
15622
|
-
leaseExpiresAt: earliestLeaseExpiry,
|
|
15623
|
-
fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
15624
|
-
});
|
|
15625
|
-
const supervisor = createRuntimeReceiptHeartbeatSupervisor({
|
|
15626
|
-
// The cohort owns claims that have not reached provider admission yet.
|
|
15627
|
-
// Renew it before the one-call supervisors can win the same short-lease
|
|
15628
|
-
// timer race, so the queued tail remains fenced while the active call
|
|
15629
|
-
// consumes the provider concurrency slot.
|
|
15630
|
-
intervalMs: Math.max(1, Math.floor(perCallHeartbeatIntervalMs / 2)),
|
|
15631
|
-
heartbeat: heartbeatOwnedRequests,
|
|
15632
|
-
isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
|
|
15633
|
-
onLeaseLost: (error) => {
|
|
15634
|
-
leaseLost ??= error;
|
|
15635
|
-
},
|
|
15636
|
-
onTransientFailure: (error) => {
|
|
15637
|
-
this.runtimeLog(
|
|
15638
|
-
`Pending tool receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`,
|
|
15182
|
+
const requests =
|
|
15183
|
+
input.requestsForReceipt.get(receipt.resultReceiptKey) ?? [];
|
|
15184
|
+
return receipt.memberIds.map((memberId) => {
|
|
15185
|
+
const request = requests.find(
|
|
15186
|
+
(candidate) => candidate.callId === memberId,
|
|
15639
15187
|
);
|
|
15640
|
-
|
|
15641
|
-
|
|
15642
|
-
return {
|
|
15643
|
-
stop: () => supervisor.stop(),
|
|
15644
|
-
leaseLost: () => leaseLost,
|
|
15645
|
-
confirmOwned: async () => {
|
|
15646
|
-
// The scheduler receipt gateway returns `running` from an atomic
|
|
15647
|
-
// claim. That claim is already the provider-execution fence, so short
|
|
15648
|
-
// calls only need scheduled renewal if they approach lease expiry.
|
|
15649
|
-
// Other callers may still return queued/pending claims or omit the
|
|
15650
|
-
// explicit backend capability; retain the immediate ownership check
|
|
15651
|
-
// for those receipts.
|
|
15652
|
-
if (claimsEstablishedExecutionFence) {
|
|
15653
|
-
if (
|
|
15654
|
-
ownershipConfirmedUntilMs - Date.now() <=
|
|
15655
|
-
IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS
|
|
15656
|
-
) {
|
|
15657
|
-
const outcome = await heartbeatOwnedRequests();
|
|
15658
|
-
if (outcome === 'active') supervisor.start();
|
|
15659
|
-
return;
|
|
15660
|
-
}
|
|
15661
|
-
supervisor.start();
|
|
15662
|
-
return;
|
|
15188
|
+
if (!request) {
|
|
15189
|
+
throw new Error(`Unknown Tool Call publication member ${memberId}.`);
|
|
15663
15190
|
}
|
|
15664
|
-
|
|
15665
|
-
|
|
15666
|
-
},
|
|
15667
|
-
assertOwned: () => {
|
|
15668
|
-
if (leaseLost) throw leaseLost;
|
|
15669
|
-
if (Date.now() < ownershipConfirmedUntilMs) return;
|
|
15670
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
15671
|
-
receiptKey: firstOwnedRequest.receiptKey!,
|
|
15672
|
-
runId: this.currentReceiptOwnerRunId,
|
|
15673
|
-
leaseId: firstOwnedRequest.receiptLeaseId!,
|
|
15674
|
-
});
|
|
15675
|
-
},
|
|
15191
|
+
return request;
|
|
15192
|
+
});
|
|
15676
15193
|
};
|
|
15677
|
-
}
|
|
15678
|
-
|
|
15679
|
-
/**
|
|
15680
|
-
* Concrete RuntimeStepReceipt adapter for the map receipt-cohort Module.
|
|
15681
|
-
*
|
|
15682
|
-
* Scalar and provider-native map calls share completed-result cache reads
|
|
15683
|
-
* and atomic successful inserts. A miss has no durable execution owner.
|
|
15684
|
-
*/
|
|
15685
|
-
private createRuntimeStepToolCallReceiptCohortStore(input: {
|
|
15686
|
-
toolId: string;
|
|
15687
|
-
requestForReceipt: ReadonlyMap<string, ToolCallRequest>;
|
|
15688
|
-
/**
|
|
15689
|
-
* Every request sharing a receipt key, for delivery timing only. Two rows
|
|
15690
|
-
* asking the same question share one key, and only one of them — the
|
|
15691
|
-
* cohort's owner — executed and carries the wrap stamp `deliveryPublishWait`
|
|
15692
|
-
* is measured from, so the timing lookup cannot use the single-request map
|
|
15693
|
-
* above, which keeps whichever request was seen last.
|
|
15694
|
-
*/
|
|
15695
|
-
requestsForReceipt: ReadonlyMap<string, readonly ToolCallRequest[]>;
|
|
15696
|
-
}): ToolCallReceiptCohortStore {
|
|
15697
15194
|
const asRecord = async (
|
|
15698
15195
|
resultReceiptKey: string,
|
|
15699
15196
|
storedReceiptKey: string,
|
|
@@ -15879,23 +15376,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
15879
15376
|
);
|
|
15880
15377
|
const manifestCompletions =
|
|
15881
15378
|
this.sourceRunToolReceiptManifestCompletionInputs(
|
|
15882
|
-
receipts.flatMap((receipt) =>
|
|
15883
|
-
|
|
15884
|
-
|
|
15885
|
-
|
|
15886
|
-
|
|
15887
|
-
|
|
15888
|
-
|
|
15889
|
-
|
|
15890
|
-
|
|
15891
|
-
executionAuthScopeDigest:
|
|
15892
|
-
request.executionAuthScopeDigest,
|
|
15893
|
-
resultReceiptKey: receipt.resultReceiptKey,
|
|
15894
|
-
providerActionVersion: request.providerActionVersion,
|
|
15895
|
-
},
|
|
15896
|
-
]
|
|
15897
|
-
: [];
|
|
15898
|
-
}),
|
|
15379
|
+
receipts.flatMap((receipt) =>
|
|
15380
|
+
requestsForPublication(receipt).map((request) => ({
|
|
15381
|
+
toolId: input.toolId,
|
|
15382
|
+
logicalCallId: request.logicalCallId,
|
|
15383
|
+
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
15384
|
+
resultReceiptKey: receipt.resultReceiptKey,
|
|
15385
|
+
providerActionVersion: request.providerActionVersion,
|
|
15386
|
+
})),
|
|
15387
|
+
),
|
|
15899
15388
|
);
|
|
15900
15389
|
await this.completeRuntimeStepReceipts(
|
|
15901
15390
|
[...canonicalCompletions, ...manifestCompletions],
|
|
@@ -15905,18 +15394,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
15905
15394
|
// charged to one of the two rather than to neither.
|
|
15906
15395
|
(enqueuedAtMs) => {
|
|
15907
15396
|
for (const receipt of receipts) {
|
|
15908
|
-
const
|
|
15909
|
-
|
|
15910
|
-
|
|
15911
|
-
(
|
|
15912
|
-
|
|
15913
|
-
|
|
15914
|
-
|
|
15915
|
-
|
|
15916
|
-
|
|
15917
|
-
|
|
15918
|
-
),
|
|
15919
|
-
});
|
|
15397
|
+
for (const request of requestsForPublication(receipt)) {
|
|
15398
|
+
const wrappedAtMs = request.deliveryWrappedAtMs;
|
|
15399
|
+
if (wrappedAtMs == null) continue;
|
|
15400
|
+
this.resourceGovernor.observe({
|
|
15401
|
+
deliveryPublishWaitMs: Math.max(
|
|
15402
|
+
0,
|
|
15403
|
+
enqueuedAtMs - wrappedAtMs,
|
|
15404
|
+
),
|
|
15405
|
+
});
|
|
15406
|
+
}
|
|
15920
15407
|
}
|
|
15921
15408
|
},
|
|
15922
15409
|
// The completion writer batches by arrival, so a publication wave
|
|
@@ -16011,57 +15498,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16011
15498
|
: request.input,
|
|
16012
15499
|
),
|
|
16013
15500
|
})),
|
|
16014
|
-
// Claim immediately so unrelated owned rows can cross provider work
|
|
16015
|
-
// while foreign members hydrate in the bounded lane.
|
|
16016
|
-
eager: true,
|
|
16017
|
-
});
|
|
16018
|
-
|
|
16019
|
-
// Claims carry their actual expiry. Seed the cohort supervisor from that
|
|
16020
|
-
// lease rather than the generic fallback, otherwise a long provider-shaped
|
|
16021
|
-
// queue can let undispatched rows expire before its first renewal.
|
|
16022
|
-
for (const request of requests) {
|
|
16023
|
-
const member = cohort.member(request.callId);
|
|
16024
|
-
if (member.kind !== 'owned') continue;
|
|
16025
|
-
request.receiptLeaseId = member.lease.leaseId;
|
|
16026
|
-
request.receiptLeaseExpiresAt = member.leaseExpiresAt ?? null;
|
|
16027
|
-
}
|
|
16028
|
-
|
|
16029
|
-
const earliestLeaseExpiry = requests.reduce<string | null>(
|
|
16030
|
-
(earliest, request) => {
|
|
16031
|
-
const expiresAt = request.receiptLeaseExpiresAt ?? null;
|
|
16032
|
-
if (!expiresAt) return earliest;
|
|
16033
|
-
if (!earliest) return expiresAt;
|
|
16034
|
-
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
16035
|
-
? expiresAt
|
|
16036
|
-
: earliest;
|
|
16037
|
-
},
|
|
16038
|
-
null,
|
|
16039
|
-
);
|
|
16040
|
-
// One cohort-wide liveness check closes the claim-to-provider window; the
|
|
16041
|
-
// per-call views additionally preserve the historical exact admission
|
|
16042
|
-
// fence immediately before an individual provider request.
|
|
16043
|
-
await cohort.heartbeat();
|
|
16044
|
-
let cohortLeaseLost: unknown | null = null;
|
|
16045
|
-
const heartbeat = cohort.startHeartbeat({
|
|
16046
|
-
intervalMs: Math.max(
|
|
16047
|
-
1,
|
|
16048
|
-
Math.floor(
|
|
16049
|
-
runtimeLeaseHeartbeatIntervalFromExpiry({
|
|
16050
|
-
leaseExpiresAt: earliestLeaseExpiry,
|
|
16051
|
-
fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
16052
|
-
}) / 2,
|
|
16053
|
-
),
|
|
16054
|
-
),
|
|
16055
|
-
onLeaseLost: (error) => {
|
|
16056
|
-
cohortLeaseLost ??= error;
|
|
16057
|
-
},
|
|
16058
|
-
onTransientFailure: (error) => {
|
|
16059
|
-
this.runtimeLog(
|
|
16060
|
-
`Tool Call Receipt Cohort heartbeat transport failed; retrying: ${
|
|
16061
|
-
error instanceof Error ? error.message : String(error)
|
|
16062
|
-
}`,
|
|
16063
|
-
);
|
|
16064
|
-
},
|
|
16065
15501
|
});
|
|
16066
15502
|
|
|
16067
15503
|
const successful: ToolCallRequest[] = [];
|
|
@@ -16108,35 +15544,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16108
15544
|
});
|
|
16109
15545
|
};
|
|
16110
15546
|
|
|
16111
|
-
const
|
|
16112
|
-
const
|
|
16113
|
-
const collectReadyMembers = (): Array<{
|
|
15547
|
+
const cachedResults: Promise<void>[] = [];
|
|
15548
|
+
const owned: Array<{
|
|
16114
15549
|
request: ToolCallRequest;
|
|
16115
15550
|
receipts: ToolResultReceipts;
|
|
16116
|
-
}>
|
|
16117
|
-
|
|
16118
|
-
|
|
16119
|
-
|
|
16120
|
-
|
|
16121
|
-
|
|
16122
|
-
|
|
16123
|
-
|
|
16124
|
-
|
|
16125
|
-
|
|
16126
|
-
|
|
16127
|
-
} else if (member.kind === 'following') {
|
|
16128
|
-
followerResults.push(
|
|
16129
|
-
member.result.then(
|
|
16130
|
-
async (result) => await resolve(request, result, 'owner'),
|
|
16131
|
-
async (error: unknown) => await reject(request, error),
|
|
16132
|
-
),
|
|
16133
|
-
);
|
|
16134
|
-
} else {
|
|
16135
|
-
owned.push({ request, receipts: member.receipts });
|
|
16136
|
-
}
|
|
15551
|
+
}> = [];
|
|
15552
|
+
for (const request of requests) {
|
|
15553
|
+
const member = cohort.member(request.callId);
|
|
15554
|
+
if (member.kind === 'completed') {
|
|
15555
|
+
cachedResults.push(resolve(request, member.result, member.source));
|
|
15556
|
+
} else if (member.kind === 'owned') {
|
|
15557
|
+
owned.push({ request, receipts: member.receipts });
|
|
15558
|
+
} else {
|
|
15559
|
+
throw new Error(
|
|
15560
|
+
'A completed-result cache cannot own a pending invocation.',
|
|
15561
|
+
);
|
|
16137
15562
|
}
|
|
16138
|
-
|
|
16139
|
-
};
|
|
15563
|
+
}
|
|
16140
15564
|
|
|
16141
15565
|
const width = Math.min(
|
|
16142
15566
|
this.governor.policy.concurrency.toolCalls,
|
|
@@ -16148,85 +15572,91 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16148
15572
|
),
|
|
16149
15573
|
),
|
|
16150
15574
|
);
|
|
16151
|
-
|
|
16152
|
-
owned
|
|
16153
|
-
|
|
16154
|
-
|
|
16155
|
-
|
|
16156
|
-
|
|
16157
|
-
|
|
16158
|
-
|
|
16159
|
-
|
|
16160
|
-
|
|
16161
|
-
|
|
16162
|
-
|
|
16163
|
-
|
|
16164
|
-
|
|
16165
|
-
|
|
16166
|
-
|
|
16167
|
-
|
|
16168
|
-
|
|
16169
|
-
|
|
16170
|
-
|
|
16171
|
-
|
|
16172
|
-
|
|
16173
|
-
|
|
16174
|
-
|
|
16175
|
-
|
|
16176
|
-
|
|
16177
|
-
|
|
16178
|
-
|
|
15575
|
+
await dispatchBoundedSettled(
|
|
15576
|
+
owned,
|
|
15577
|
+
width,
|
|
15578
|
+
async ({ request, receipts }) => {
|
|
15579
|
+
// Circuit breaker: do not dispatch this provider call once a
|
|
15580
|
+
// persistence failure has occurred in this run. Rejected through the
|
|
15581
|
+
// same path as a provider failure — `dispatchBoundedSettled` only
|
|
15582
|
+
// collects a thrown error, so throwing past `reject` here would
|
|
15583
|
+
// leave this row's tool-call resolver pending for the life of the
|
|
15584
|
+
// map.
|
|
15585
|
+
if (this.persistenceLatch.tripped) {
|
|
15586
|
+
this.persistenceLatch.preventedCallCount += 1;
|
|
15587
|
+
await reject(
|
|
15588
|
+
request,
|
|
15589
|
+
new RuntimePersistenceCircuitOpenError(this.persistenceLatch),
|
|
15590
|
+
);
|
|
15591
|
+
return;
|
|
15592
|
+
}
|
|
15593
|
+
let releaseToolSlot: () => void = () => undefined;
|
|
15594
|
+
try {
|
|
15595
|
+
const providerOperationKey = this.providerInvocationKeyForToolCall({
|
|
15596
|
+
operationKey:
|
|
15597
|
+
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
15598
|
+
logicalCallId: request.logicalCallId,
|
|
15599
|
+
force: request.force,
|
|
15600
|
+
});
|
|
15601
|
+
const result = await this.executeMappedToolCallThroughModule({
|
|
15602
|
+
request,
|
|
15603
|
+
dispatchToolId: toolId,
|
|
15604
|
+
dispatchPayload: request.input,
|
|
15605
|
+
resultReceiptKey:
|
|
15606
|
+
this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
|
|
15607
|
+
providerOperationKey,
|
|
15608
|
+
resultReceipts: receipts,
|
|
15609
|
+
apiOptions: {
|
|
15610
|
+
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
15611
|
+
durableCallReceiptKey: this.toolTransportReceiptKey({
|
|
15612
|
+
toolId,
|
|
15613
|
+
providerOperationKey,
|
|
15614
|
+
}),
|
|
15615
|
+
legacyProviderRetryIdentities: [
|
|
15616
|
+
{
|
|
15617
|
+
transportReceiptKey: request.receiptKey!,
|
|
15618
|
+
providerOperationKey: request.force
|
|
15619
|
+
? providerOperationKey
|
|
15620
|
+
: this.providerOperationKeyForQueuedToolCall(request),
|
|
15621
|
+
...(request.force && request.requiresExecutionFence !== true
|
|
15622
|
+
? { forceRefreshIdentities: [providerOperationKey] }
|
|
15623
|
+
: {}),
|
|
15624
|
+
},
|
|
15625
|
+
],
|
|
15626
|
+
completedResultCache: true,
|
|
15627
|
+
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
15628
|
+
...(request.requiresExecutionFence === true
|
|
15629
|
+
? {
|
|
15630
|
+
providerIdempotencyKey: providerOperationKey,
|
|
15631
|
+
providerIdempotencyReceiptKey: request.receiptKey,
|
|
15632
|
+
}
|
|
15633
|
+
: {}),
|
|
15634
|
+
...this.providerInvocationOptionsForToolCall({
|
|
16179
15635
|
operationKey:
|
|
16180
15636
|
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
16181
|
-
|
|
15637
|
+
requiresExecutionFence: request.requiresExecutionFence,
|
|
16182
15638
|
logicalCallId: request.logicalCallId,
|
|
15639
|
+
force: request.force,
|
|
15640
|
+
}),
|
|
15641
|
+
timeoutMs:
|
|
15642
|
+
typeof request.timeoutMs === 'number' && request.timeoutMs > 0
|
|
15643
|
+
? request.timeoutMs
|
|
15644
|
+
: undefined,
|
|
15645
|
+
retainToolSlot: (release) => {
|
|
15646
|
+
releaseToolSlot = release;
|
|
16183
15647
|
},
|
|
16184
|
-
|
|
16185
|
-
|
|
16186
|
-
|
|
16187
|
-
|
|
16188
|
-
|
|
16189
|
-
|
|
16190
|
-
|
|
16191
|
-
|
|
16192
|
-
|
|
16193
|
-
|
|
16194
|
-
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
16195
|
-
durableCallReceiptKey: this.toolTransportReceiptKey({
|
|
16196
|
-
toolId,
|
|
16197
|
-
resultReceiptKey: request.receiptKey!,
|
|
16198
|
-
providerOperationKey,
|
|
16199
|
-
force: request.force,
|
|
16200
|
-
}),
|
|
16201
|
-
completedResultCache: true,
|
|
16202
|
-
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
16203
|
-
...this.providerForceRefreshOptionsForToolCall({
|
|
16204
|
-
operationKey:
|
|
16205
|
-
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
16206
|
-
force: request.force,
|
|
16207
|
-
requiresExecutionFence: request.requiresExecutionFence,
|
|
16208
|
-
logicalCallId: request.logicalCallId,
|
|
16209
|
-
}),
|
|
16210
|
-
timeoutMs:
|
|
16211
|
-
typeof request.timeoutMs === 'number' && request.timeoutMs > 0
|
|
16212
|
-
? request.timeoutMs
|
|
16213
|
-
: undefined,
|
|
16214
|
-
retainToolSlot: (release) => {
|
|
16215
|
-
releaseToolSlot = release;
|
|
16216
|
-
},
|
|
16217
|
-
},
|
|
16218
|
-
});
|
|
16219
|
-
await resolve(request, result, 'owner');
|
|
16220
|
-
} catch (error) {
|
|
16221
|
-
await reject(request, error);
|
|
16222
|
-
} finally {
|
|
16223
|
-
releaseToolSlot();
|
|
16224
|
-
}
|
|
16225
|
-
},
|
|
16226
|
-
);
|
|
15648
|
+
},
|
|
15649
|
+
});
|
|
15650
|
+
await resolve(request, result, 'owner');
|
|
15651
|
+
} catch (error) {
|
|
15652
|
+
await reject(request, error);
|
|
15653
|
+
} finally {
|
|
15654
|
+
releaseToolSlot();
|
|
15655
|
+
}
|
|
15656
|
+
},
|
|
15657
|
+
);
|
|
16227
15658
|
|
|
16228
|
-
await
|
|
16229
|
-
await Promise.allSettled(followerResults);
|
|
15659
|
+
await Promise.all(cachedResults);
|
|
16230
15660
|
return successful;
|
|
16231
15661
|
}
|
|
16232
15662
|
|
|
@@ -16398,18 +15828,109 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16398
15828
|
'Native batch Tool Call operation identity changed in transit.',
|
|
16399
15829
|
);
|
|
16400
15830
|
}
|
|
15831
|
+
const legacyForceRefreshIdentities = detail.members
|
|
15832
|
+
.filter(
|
|
15833
|
+
(member) =>
|
|
15834
|
+
member.force === true &&
|
|
15835
|
+
member.requiresExecutionFence !== true,
|
|
15836
|
+
)
|
|
15837
|
+
.map((member) =>
|
|
15838
|
+
this.providerInvocationKeyForToolCall({
|
|
15839
|
+
operationKey:
|
|
15840
|
+
this.providerOperationKeyForQueuedToolCall(member),
|
|
15841
|
+
logicalCallId: member.logicalCallId,
|
|
15842
|
+
force: true,
|
|
15843
|
+
}),
|
|
15844
|
+
);
|
|
15845
|
+
const legacyProviderOperationKey =
|
|
15846
|
+
buildDurableToolPayloadProviderIdempotencyKey({
|
|
15847
|
+
orgId: this.#options.orgId,
|
|
15848
|
+
toolId: call.tool.id,
|
|
15849
|
+
executionAuthScopeDigest:
|
|
15850
|
+
detail.members[0]?.executionAuthScopeDigest ?? null,
|
|
15851
|
+
providerActionVersion: call.tool.providerActionVersion,
|
|
15852
|
+
payload: call.payload,
|
|
15853
|
+
forceRefreshIdentities: legacyForceRefreshIdentities,
|
|
15854
|
+
});
|
|
15855
|
+
const legacyFallbackForceRefreshIdentities = detail.members
|
|
15856
|
+
.filter((member) => member.force === true)
|
|
15857
|
+
.map((member) =>
|
|
15858
|
+
this.providerInvocationKeyForToolCall({
|
|
15859
|
+
operationKey:
|
|
15860
|
+
this.providerOperationKeyForQueuedToolCall(member),
|
|
15861
|
+
logicalCallId: member.logicalCallId,
|
|
15862
|
+
force: true,
|
|
15863
|
+
}),
|
|
15864
|
+
);
|
|
15865
|
+
const legacyFallbackProviderOperationKey =
|
|
15866
|
+
buildDurableToolPayloadProviderIdempotencyKey({
|
|
15867
|
+
orgId: this.#options.orgId,
|
|
15868
|
+
toolId: call.tool.id,
|
|
15869
|
+
executionAuthScopeDigest:
|
|
15870
|
+
detail.members[0]?.executionAuthScopeDigest ?? null,
|
|
15871
|
+
providerActionVersion:
|
|
15872
|
+
(
|
|
15873
|
+
await this.#options.getToolActionCacheVersion?.(
|
|
15874
|
+
call.tool.id,
|
|
15875
|
+
)
|
|
15876
|
+
)?.trim() ?? 'unversioned',
|
|
15877
|
+
payload: call.payload,
|
|
15878
|
+
forceRefreshIdentities:
|
|
15879
|
+
legacyFallbackForceRefreshIdentities,
|
|
15880
|
+
});
|
|
16401
15881
|
const execution = await this.callToolExecutionAPI(
|
|
16402
15882
|
call.tool.id,
|
|
16403
15883
|
call.payload,
|
|
16404
15884
|
{
|
|
16405
15885
|
durableCallReceiptKey: this.toolTransportReceiptKey({
|
|
16406
15886
|
toolId: call.tool.id,
|
|
16407
|
-
resultReceiptKey: compatibility.resultReceiptKey,
|
|
16408
15887
|
providerOperationKey: detail.providerOperationKey,
|
|
16409
|
-
force: detail.members.some(
|
|
16410
|
-
(request) => request.force === true,
|
|
16411
|
-
),
|
|
16412
15888
|
}),
|
|
15889
|
+
legacyProviderRetryIdentities: [
|
|
15890
|
+
{
|
|
15891
|
+
transportReceiptKey: detail.members.some(
|
|
15892
|
+
(member) => member.force,
|
|
15893
|
+
)
|
|
15894
|
+
? this.toolTransportReceiptKey({
|
|
15895
|
+
toolId: call.tool.id,
|
|
15896
|
+
providerOperationKey:
|
|
15897
|
+
legacyProviderOperationKey,
|
|
15898
|
+
})
|
|
15899
|
+
: buildDurableToolAggregateReceiptKey({
|
|
15900
|
+
receiptKeys: detail.members.map(
|
|
15901
|
+
(member) =>
|
|
15902
|
+
this.mappedToolResultReceiptPolicy(member)
|
|
15903
|
+
.resultReceiptKey,
|
|
15904
|
+
),
|
|
15905
|
+
prefix: 'batch',
|
|
15906
|
+
aggregateReceiptPrefix:
|
|
15907
|
+
buildDurableToolReceiptPrefix({
|
|
15908
|
+
orgId: this.#options.orgId,
|
|
15909
|
+
toolId: call.tool.id,
|
|
15910
|
+
}),
|
|
15911
|
+
}),
|
|
15912
|
+
providerOperationKey: legacyProviderOperationKey,
|
|
15913
|
+
forceRefreshIdentities: legacyForceRefreshIdentities,
|
|
15914
|
+
},
|
|
15915
|
+
{
|
|
15916
|
+
transportReceiptKey:
|
|
15917
|
+
buildDurableToolAggregateReceiptKey({
|
|
15918
|
+
receiptKeys: detail.members.map(
|
|
15919
|
+
(member) => member.cacheKey,
|
|
15920
|
+
),
|
|
15921
|
+
prefix: 'batch',
|
|
15922
|
+
aggregateReceiptPrefix:
|
|
15923
|
+
buildDurableToolReceiptPrefix({
|
|
15924
|
+
orgId: this.#options.orgId,
|
|
15925
|
+
toolId: call.tool.id,
|
|
15926
|
+
}),
|
|
15927
|
+
}),
|
|
15928
|
+
providerOperationKey:
|
|
15929
|
+
legacyFallbackProviderOperationKey,
|
|
15930
|
+
forceRefreshIdentities:
|
|
15931
|
+
legacyFallbackForceRefreshIdentities,
|
|
15932
|
+
},
|
|
15933
|
+
],
|
|
16413
15934
|
completedResultCache: true,
|
|
16414
15935
|
memoryLedgerRowScopes: detail.members.map((member) => ({
|
|
16415
15936
|
rowKey: member.rowKey,
|
|
@@ -16426,6 +15947,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16426
15947
|
),
|
|
16427
15948
|
executionAuthScopeDigest:
|
|
16428
15949
|
detail.members[0]?.executionAuthScopeDigest ?? null,
|
|
15950
|
+
providerIdempotencyKey: detail.providerOperationKey,
|
|
15951
|
+
providerIdempotencyReceiptKey:
|
|
15952
|
+
compatibility.resultReceiptKey,
|
|
16429
15953
|
...(detail.forceRefreshIdentities.length > 0
|
|
16430
15954
|
? {
|
|
16431
15955
|
providerOperationForceRefreshIdentities:
|
|
@@ -16535,25 +16059,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16535
16059
|
compiled.batchOperation,
|
|
16536
16060
|
)
|
|
16537
16061
|
)?.trim() ?? '';
|
|
16538
|
-
const
|
|
16539
|
-
(
|
|
16540
|
-
this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
|
|
16541
|
-
);
|
|
16542
|
-
const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({
|
|
16543
|
-
receiptKeys,
|
|
16544
|
-
prefix: 'batch',
|
|
16545
|
-
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
16546
|
-
orgId: this.#options.orgId,
|
|
16547
|
-
toolId: compiled.batchOperation,
|
|
16548
|
-
}),
|
|
16549
|
-
});
|
|
16550
|
-
const forceRefreshIdentities = owned.flatMap((request) =>
|
|
16551
|
-
this.providerForceRefreshIdentitiesForToolCall({
|
|
16062
|
+
const forceRefreshIdentities = owned.map((request) =>
|
|
16063
|
+
this.providerInvocationKeyForToolCall({
|
|
16552
16064
|
operationKey:
|
|
16553
16065
|
this.providerOperationKeyForQueuedToolCall(request),
|
|
16554
|
-
force: request.force,
|
|
16555
|
-
requiresExecutionFence: request.requiresExecutionFence,
|
|
16556
16066
|
logicalCallId: request.logicalCallId,
|
|
16067
|
+
force: request.force,
|
|
16557
16068
|
}),
|
|
16558
16069
|
);
|
|
16559
16070
|
const providerOperationKey =
|
|
@@ -16566,6 +16077,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16566
16077
|
payload: compiled.batchPayload,
|
|
16567
16078
|
forceRefreshIdentities,
|
|
16568
16079
|
});
|
|
16080
|
+
const aggregateReceiptKey = this.toolTransportReceiptKey({
|
|
16081
|
+
toolId: compiled.batchOperation,
|
|
16082
|
+
providerOperationKey,
|
|
16083
|
+
});
|
|
16569
16084
|
dispatchDetails.set(aggregateReceiptKey, {
|
|
16570
16085
|
members: owned,
|
|
16571
16086
|
compiled,
|
|
@@ -16659,6 +16174,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16659
16174
|
toolId,
|
|
16660
16175
|
status: aggregate.status,
|
|
16661
16176
|
result: item,
|
|
16177
|
+
meta: nativeBatchResponseMeta(item),
|
|
16662
16178
|
toolResponse: publicToolResponseForBatchedItem(
|
|
16663
16179
|
aggregate,
|
|
16664
16180
|
item,
|
|
@@ -16825,8 +16341,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16825
16341
|
};
|
|
16826
16342
|
|
|
16827
16343
|
const pendingRequests: ToolCallRequest[] = [];
|
|
16828
|
-
let pendingReceiptClaimsEstablishExecutionFence =
|
|
16829
|
-
this.#options.runtimeReceiptClaimsEstablishExecutionFence === true;
|
|
16830
16344
|
const recoveredRequests: ToolCallRequest[] = [];
|
|
16831
16345
|
for (const req of requests) {
|
|
16832
16346
|
const cached =
|
|
@@ -16851,114 +16365,27 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16851
16365
|
}
|
|
16852
16366
|
recordToolStep(recoveredRequests);
|
|
16853
16367
|
|
|
16854
|
-
const
|
|
16855
|
-
|
|
16856
|
-
owner: ToolCallRequest,
|
|
16857
|
-
result: unknown,
|
|
16858
|
-
): void => {
|
|
16859
|
-
const followers = liveFollowersByOwnerCallId.get(owner.callId);
|
|
16860
|
-
if (!followers || followers.length === 0) return;
|
|
16861
|
-
liveFollowersByOwnerCallId.delete(owner.callId);
|
|
16862
|
-
for (const follower of followers) {
|
|
16863
|
-
const followerReceiptKey = follower.receiptKey?.trim() || null;
|
|
16864
|
-
const followerResult = followerReceiptKey
|
|
16865
|
-
? markToolExecuteResultExecutionOutcome(result, {
|
|
16866
|
-
kind: 'in_flight',
|
|
16867
|
-
receiptKey: followerReceiptKey,
|
|
16868
|
-
attachedToReceiptKey: followerReceiptKey,
|
|
16869
|
-
})
|
|
16870
|
-
: result;
|
|
16871
|
-
if (follower.cacheable !== false) {
|
|
16872
|
-
this.cacheToolResult(toolId, follower.cacheKey, followerResult);
|
|
16873
|
-
}
|
|
16874
|
-
const resolver = this.toolCallResolvers.get(follower.callId);
|
|
16875
|
-
if (resolver) {
|
|
16876
|
-
resolver.resolve(followerResult);
|
|
16877
|
-
this.toolCallResolvers.delete(follower.callId);
|
|
16878
|
-
}
|
|
16879
|
-
this.emitScopedFieldMetaUpdate({
|
|
16880
|
-
rowId: follower.rowId,
|
|
16881
|
-
key: follower.rowKey ?? null,
|
|
16882
|
-
tableNamespace: follower.tableNamespace ?? null,
|
|
16883
|
-
fieldName: follower.fieldName,
|
|
16884
|
-
status: 'cached',
|
|
16885
|
-
rowStatus: 'running',
|
|
16886
|
-
stage: toolId,
|
|
16887
|
-
provider: null,
|
|
16888
|
-
error: null,
|
|
16889
|
-
reused: true,
|
|
16890
|
-
dataPatch: {},
|
|
16891
|
-
});
|
|
16892
|
-
}
|
|
16893
|
-
};
|
|
16894
|
-
const rejectWithLiveFollowers = async (
|
|
16895
|
-
owner: ToolCallRequest,
|
|
16368
|
+
const rejectCall = async (
|
|
16369
|
+
request: ToolCallRequest,
|
|
16896
16370
|
error: unknown,
|
|
16897
16371
|
): Promise<void> => {
|
|
16898
|
-
|
|
16899
|
-
|
|
16900
|
-
|
|
16901
|
-
let releaseFailure: unknown = null;
|
|
16902
|
-
const receiptKey = owner.receiptKey?.trim() || null;
|
|
16903
|
-
try {
|
|
16904
|
-
if (receiptKey && this.durableMappedToolResultsBackedByReceipts) {
|
|
16905
|
-
const leaseId = owner.receiptLeaseId?.trim();
|
|
16906
|
-
if (!leaseId) {
|
|
16907
|
-
throw new Error(
|
|
16908
|
-
`Provider wait cannot release receipt ${receiptKey} without its owned lease.`,
|
|
16909
|
-
);
|
|
16910
|
-
}
|
|
16911
|
-
const released = await this.releaseRuntimeStepReceipt(
|
|
16912
|
-
receiptKey,
|
|
16913
|
-
this.currentReceiptOwnerRunId,
|
|
16914
|
-
leaseId,
|
|
16915
|
-
);
|
|
16916
|
-
if (
|
|
16917
|
-
!released ||
|
|
16918
|
-
(released.status !== 'queued' &&
|
|
16919
|
-
released.status !== 'pending')
|
|
16920
|
-
) {
|
|
16921
|
-
throw new Error(
|
|
16922
|
-
`Provider wait could not release owned receipt ${receiptKey}.`,
|
|
16923
|
-
);
|
|
16924
|
-
}
|
|
16925
|
-
}
|
|
16926
|
-
} catch (failure) {
|
|
16927
|
-
releaseFailure = failure;
|
|
16928
|
-
}
|
|
16929
|
-
// A provider deadline is a nonterminal control signal. The row
|
|
16930
|
-
// catch records waiting metadata after the resolver unwinds; no
|
|
16931
|
-
// follower may see or persist a failed receipt in between. If the
|
|
16932
|
-
// lease could not be released, wake every follower with that loud
|
|
16933
|
-
// infrastructure failure rather than stranding their promises.
|
|
16934
|
-
for (const request of [owner, ...followers]) {
|
|
16935
|
-
const resolver = this.toolCallResolvers.get(request.callId);
|
|
16936
|
-
if (resolver) {
|
|
16937
|
-
resolver.reject(releaseFailure ?? error);
|
|
16938
|
-
this.toolCallResolvers.delete(request.callId);
|
|
16939
|
-
}
|
|
16940
|
-
}
|
|
16941
|
-
if (releaseFailure) throw releaseFailure;
|
|
16942
|
-
return;
|
|
16943
|
-
}
|
|
16944
|
-
const shouldPersistReceiptFailure =
|
|
16945
|
-
!(error instanceof ToolExecuteAuthScopeChangedError) &&
|
|
16946
|
-
!this.persistenceLatch.tripped;
|
|
16947
|
-
for (const request of [owner, ...followers]) {
|
|
16948
|
-
await this.rejectToolCall(toolId, request, error, {
|
|
16949
|
-
persistReceiptFailure: shouldPersistReceiptFailure,
|
|
16950
|
-
});
|
|
16951
|
-
}
|
|
16372
|
+
await this.rejectToolCall(toolId, request, error, {
|
|
16373
|
+
persistReceiptFailure: false,
|
|
16374
|
+
});
|
|
16952
16375
|
};
|
|
16953
|
-
const requestsWithLiveFollowers = (
|
|
16954
|
-
owner: ToolCallRequest,
|
|
16955
|
-
): ToolCallRequest[] => [
|
|
16956
|
-
owner,
|
|
16957
|
-
...(liveFollowersByOwnerCallId.get(owner.callId) ?? []),
|
|
16958
|
-
];
|
|
16959
16376
|
const resolveRuntimeTimeoutMsForRequests = (
|
|
16960
16377
|
requests: ToolCallRequest[],
|
|
16961
16378
|
): number | undefined => {
|
|
16379
|
+
if (
|
|
16380
|
+
requests.some(
|
|
16381
|
+
(request) =>
|
|
16382
|
+
typeof request.timeoutMs !== 'number' ||
|
|
16383
|
+
!Number.isFinite(request.timeoutMs) ||
|
|
16384
|
+
request.timeoutMs <= 0,
|
|
16385
|
+
)
|
|
16386
|
+
) {
|
|
16387
|
+
return undefined;
|
|
16388
|
+
}
|
|
16962
16389
|
const timeoutMs = Math.max(
|
|
16963
16390
|
...requests
|
|
16964
16391
|
.map((request) => request.timeoutMs)
|
|
@@ -16972,76 +16399,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16972
16399
|
);
|
|
16973
16400
|
return timeoutMs > 0 ? timeoutMs : undefined;
|
|
16974
16401
|
};
|
|
16975
|
-
|
|
16976
|
-
|
|
16977
|
-
): number | undefined => {
|
|
16978
|
-
let timeoutMs = 0;
|
|
16979
|
-
for (const owner of owners) {
|
|
16980
|
-
if (
|
|
16981
|
-
typeof owner.timeoutMs !== 'number' ||
|
|
16982
|
-
!Number.isFinite(owner.timeoutMs) ||
|
|
16983
|
-
owner.timeoutMs <= 0
|
|
16984
|
-
) {
|
|
16985
|
-
return undefined;
|
|
16986
|
-
}
|
|
16987
|
-
timeoutMs = Math.max(
|
|
16988
|
-
timeoutMs,
|
|
16989
|
-
resolveRuntimeTimeoutMsForRequests(
|
|
16990
|
-
requestsWithLiveFollowers(owner),
|
|
16991
|
-
) ?? 0,
|
|
16992
|
-
);
|
|
16993
|
-
}
|
|
16994
|
-
return timeoutMs > 0 ? timeoutMs : undefined;
|
|
16995
|
-
};
|
|
16996
|
-
|
|
16997
|
-
// Cacheable map members cross the real receipt-cohort boundary before
|
|
16998
|
-
// the legacy claim loop. The scalar and native Modules share member
|
|
16999
|
-
// receipt ownership; native batches retain their separate aggregate
|
|
17000
|
-
// physical-operation receipt behind that seam. Always-fresh calls
|
|
17001
|
-
// intentionally have no durable member receipt and stay below.
|
|
16402
|
+
// The cohort reads completed results and publishes successful misses.
|
|
16403
|
+
// Always-fresh calls have no durable member result and stay below.
|
|
17002
16404
|
const operationStrategy =
|
|
17003
16405
|
this.#options.getBatchOperationStrategy?.(toolId) ?? null;
|
|
17004
|
-
if (
|
|
17005
|
-
operationStrategy &&
|
|
17006
|
-
this.durableMappedToolResultsBackedByReceipts
|
|
17007
|
-
) {
|
|
17008
|
-
const cohortRequests = pendingRequests.filter(
|
|
17009
|
-
(request) =>
|
|
17010
|
-
request.cacheable !== false &&
|
|
17011
|
-
request.requiresExecutionFence !== true &&
|
|
17012
|
-
Boolean(request.receiptKey?.trim()),
|
|
17013
|
-
);
|
|
17014
|
-
if (cohortRequests.length > 0) {
|
|
17015
|
-
const cohortCallIds = new Set(
|
|
17016
|
-
cohortRequests.map((request) => request.callId),
|
|
17017
|
-
);
|
|
17018
|
-
pendingRequests.splice(
|
|
17019
|
-
0,
|
|
17020
|
-
pendingRequests.length,
|
|
17021
|
-
...pendingRequests.filter(
|
|
17022
|
-
(request) => !cohortCallIds.has(request.callId),
|
|
17023
|
-
),
|
|
17024
|
-
);
|
|
17025
|
-
const successful =
|
|
17026
|
-
await this.executeCacheableNativeMapToolCallCohort({
|
|
17027
|
-
toolId,
|
|
17028
|
-
requests: cohortRequests,
|
|
17029
|
-
strategy: operationStrategy,
|
|
17030
|
-
});
|
|
17031
|
-
for (const request of successful) {
|
|
17032
|
-
successfulLiveStepCallIds.add(request.callId);
|
|
17033
|
-
}
|
|
17034
|
-
recordToolStep(cohortRequests);
|
|
17035
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
17036
|
-
}
|
|
17037
|
-
} else if (
|
|
17038
|
-
!operationStrategy &&
|
|
17039
|
-
this.durableMappedToolResultsBackedByReceipts
|
|
17040
|
-
) {
|
|
16406
|
+
if (this.durableMappedToolResultsBackedByReceipts) {
|
|
17041
16407
|
const cohortRequests = pendingRequests.filter(
|
|
17042
16408
|
(request) =>
|
|
17043
16409
|
request.cacheable !== false &&
|
|
17044
|
-
request.requiresExecutionFence !== true &&
|
|
17045
16410
|
Boolean(request.receiptKey?.trim()),
|
|
17046
16411
|
);
|
|
17047
16412
|
if (cohortRequests.length > 0) {
|
|
@@ -17055,1181 +16420,366 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
17055
16420
|
(request) => !cohortCallIds.has(request.callId),
|
|
17056
16421
|
),
|
|
17057
16422
|
);
|
|
17058
|
-
const successful =
|
|
17059
|
-
await this.
|
|
17060
|
-
|
|
17061
|
-
|
|
17062
|
-
|
|
17063
|
-
|
|
16423
|
+
const successful = operationStrategy
|
|
16424
|
+
? await this.executeCacheableNativeMapToolCallCohort({
|
|
16425
|
+
toolId,
|
|
16426
|
+
requests: cohortRequests,
|
|
16427
|
+
strategy: operationStrategy,
|
|
16428
|
+
})
|
|
16429
|
+
: await this.executeCacheableNonNativeMapToolCallCohort({
|
|
16430
|
+
toolId,
|
|
16431
|
+
requests: cohortRequests,
|
|
16432
|
+
});
|
|
16433
|
+
for (const request of successful)
|
|
17064
16434
|
successfulLiveStepCallIds.add(request.callId);
|
|
17065
|
-
}
|
|
17066
16435
|
recordToolStep(cohortRequests);
|
|
17067
16436
|
this.#options.onBatchComplete?.(this.checkpoint);
|
|
17068
16437
|
}
|
|
17069
16438
|
}
|
|
17070
16439
|
|
|
17071
|
-
|
|
17072
|
-
|
|
17073
|
-
|
|
17074
|
-
|
|
17075
|
-
|
|
17076
|
-
|
|
17077
|
-
|
|
17078
|
-
|
|
17079
|
-
const durableExistingRunningState: {
|
|
17080
|
-
settlement: Promise<
|
|
17081
|
-
{ status: 'fulfilled' } | { status: 'rejected'; reason: unknown }
|
|
17082
|
-
> | null;
|
|
17083
|
-
} = { settlement: null };
|
|
17084
|
-
const durableExistingRunningAbort = new AbortController();
|
|
17085
|
-
// The group's size before the turn. The turn body splices and pushes
|
|
17086
|
-
// `pendingRequests`, so the count read here is the only one that says
|
|
17087
|
-
// how many queued calls this claim turn is blocking.
|
|
17088
|
-
const claimTurnLogicalCalls = pendingRequests.length;
|
|
17089
|
-
if (
|
|
17090
|
-
claimTurnLogicalCalls > 0 &&
|
|
17091
|
-
(this.#options.claimRuntimeStepReceipts ||
|
|
17092
|
-
this.#options.claimRuntimeStepReceipt)
|
|
17093
|
-
) {
|
|
17094
|
-
const requestsByReceiptKey = new Map<
|
|
17095
|
-
string,
|
|
17096
|
-
{
|
|
17097
|
-
requests: ToolCallRequest[];
|
|
17098
|
-
forceRefresh: boolean;
|
|
17099
|
-
forceFailedRefresh: boolean;
|
|
17100
|
-
}
|
|
17101
|
-
>();
|
|
17102
|
-
const localOnlyRequests: ToolCallRequest[] = [];
|
|
17103
|
-
for (const request of pendingRequests) {
|
|
17104
|
-
const receiptKey = request.receiptKey?.trim() || null;
|
|
17105
|
-
if (!receiptKey) {
|
|
17106
|
-
localOnlyRequests.push(request);
|
|
17107
|
-
continue;
|
|
17108
|
-
}
|
|
17109
|
-
const group = requestsByReceiptKey.get(receiptKey) ?? {
|
|
17110
|
-
requests: [],
|
|
17111
|
-
forceRefresh: false,
|
|
17112
|
-
forceFailedRefresh: false,
|
|
17113
|
-
};
|
|
17114
|
-
group.requests.push(request);
|
|
17115
|
-
group.forceRefresh ||= request.force === true;
|
|
17116
|
-
group.forceFailedRefresh ||= request.forceFailedRefresh === true;
|
|
17117
|
-
requestsByReceiptKey.set(receiptKey, group);
|
|
17118
|
-
}
|
|
17119
|
-
const normalReceiptKeys = [...requestsByReceiptKey]
|
|
17120
|
-
.filter(
|
|
17121
|
-
([, group]) => !group.forceRefresh && !group.forceFailedRefresh,
|
|
17122
|
-
)
|
|
17123
|
-
.map(([receiptKey]) => receiptKey);
|
|
17124
|
-
const forcedReceiptKeys = [...requestsByReceiptKey]
|
|
17125
|
-
.filter(([, group]) => group.forceRefresh)
|
|
17126
|
-
.map(([receiptKey]) => receiptKey);
|
|
17127
|
-
const failedRefreshReceiptKeys = [...requestsByReceiptKey]
|
|
17128
|
-
.filter(
|
|
17129
|
-
([, group]) => !group.forceRefresh && group.forceFailedRefresh,
|
|
17130
|
-
)
|
|
17131
|
-
.map(([receiptKey]) => receiptKey);
|
|
17132
|
-
const claimStartedAt = Date.now();
|
|
17133
|
-
const forcedExisting = forcedReceiptKeys.length
|
|
17134
|
-
? await this.getRuntimeStepReceipts(forcedReceiptKeys, {
|
|
17135
|
-
hydrateOutput: false,
|
|
17136
|
-
})
|
|
17137
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
17138
|
-
// A forced refresh must never duplicate a provider call owned by a
|
|
17139
|
-
// live lease. It can, however, immediately claim pending/unleased
|
|
17140
|
-
// work and any expired lease. Waiting first for those receipts turns
|
|
17141
|
-
// an already-recoverable interruption into a five-minute stall.
|
|
17142
|
-
const forcedLiveReceiptKeys = new Set(
|
|
17143
|
-
[...forcedExisting]
|
|
17144
|
-
.filter(([, receipt]) => {
|
|
17145
|
-
if (
|
|
17146
|
-
receipt.status !== 'queued' &&
|
|
17147
|
-
receipt.status !== 'pending' &&
|
|
17148
|
-
receipt.status !== 'running'
|
|
17149
|
-
) {
|
|
17150
|
-
return false;
|
|
17151
|
-
}
|
|
17152
|
-
if (!receipt.leaseId?.trim()) return false;
|
|
17153
|
-
const expiresAt = receipt.leaseExpiresAt
|
|
17154
|
-
? Date.parse(receipt.leaseExpiresAt)
|
|
17155
|
-
: Number.NaN;
|
|
17156
|
-
// Keep an unparseable leased receipt conservative. The
|
|
17157
|
-
// fenced claim remains the recovery path after the wait.
|
|
17158
|
-
return !Number.isFinite(expiresAt) || expiresAt > Date.now();
|
|
17159
|
-
})
|
|
17160
|
-
.map(([receiptKey]) => receiptKey),
|
|
17161
|
-
);
|
|
17162
|
-
const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
|
|
17163
|
-
(receiptKey) => !forcedLiveReceiptKeys.has(receiptKey),
|
|
17164
|
-
);
|
|
17165
|
-
const claims =
|
|
17166
|
-
normalReceiptKeys.length > 0
|
|
17167
|
-
? await this.claimRuntimeStepReceipts(
|
|
17168
|
-
normalReceiptKeys,
|
|
17169
|
-
this.currentReceiptOwnerRunId,
|
|
17170
|
-
false,
|
|
17171
|
-
false,
|
|
17172
|
-
false,
|
|
17173
|
-
false,
|
|
17174
|
-
)
|
|
17175
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
17176
|
-
const forcedClaims =
|
|
17177
|
-
claimableForcedReceiptKeys.length > 0
|
|
17178
|
-
? await this.claimRuntimeStepReceipts(
|
|
17179
|
-
claimableForcedReceiptKeys,
|
|
17180
|
-
this.currentReceiptOwnerRunId,
|
|
17181
|
-
false,
|
|
17182
|
-
true,
|
|
17183
|
-
false,
|
|
17184
|
-
)
|
|
17185
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
17186
|
-
const failedRefreshClaims =
|
|
17187
|
-
failedRefreshReceiptKeys.length > 0
|
|
17188
|
-
? await this.claimRuntimeStepReceipts(
|
|
17189
|
-
failedRefreshReceiptKeys,
|
|
17190
|
-
this.currentReceiptOwnerRunId,
|
|
17191
|
-
false,
|
|
17192
|
-
false,
|
|
17193
|
-
true,
|
|
17194
|
-
false,
|
|
17195
|
-
)
|
|
17196
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
17197
|
-
for (const [receiptKey, claim] of forcedClaims) {
|
|
17198
|
-
claims.set(receiptKey, claim);
|
|
17199
|
-
}
|
|
17200
|
-
for (const [receiptKey, claim] of failedRefreshClaims) {
|
|
17201
|
-
claims.set(receiptKey, claim);
|
|
17202
|
-
}
|
|
17203
|
-
const safeFailedReceiptKeys = [...claims]
|
|
17204
|
-
.filter(([, claim]) => this.canRetryWithoutProviderDispatch(claim))
|
|
17205
|
-
.map(([receiptKey]) => receiptKey);
|
|
17206
|
-
if (safeFailedReceiptKeys.length > 0) {
|
|
17207
|
-
const refreshed = await this.claimRuntimeStepReceipts(
|
|
17208
|
-
safeFailedReceiptKeys,
|
|
17209
|
-
this.currentReceiptOwnerRunId,
|
|
17210
|
-
false,
|
|
17211
|
-
false,
|
|
17212
|
-
true,
|
|
17213
|
-
false,
|
|
17214
|
-
);
|
|
17215
|
-
for (const [receiptKey, claim] of refreshed) {
|
|
17216
|
-
claims.set(receiptKey, claim);
|
|
17217
|
-
}
|
|
17218
|
-
}
|
|
17219
|
-
// Claiming is a state transition, not a reason to expand a
|
|
17220
|
-
// body. Fresh receipts contain no result, yet the old path took
|
|
17221
|
-
// the hydration lane before it learned that fact and charged a
|
|
17222
|
-
// multi-MB row prior to every one. Claims now arrive as bounded
|
|
17223
|
-
// manifests; only a terminal winner redeems its known-size body
|
|
17224
|
-
// under a second, byte-priced turn.
|
|
17225
|
-
const terminalClaimKeys = [...claims]
|
|
17226
|
-
.filter(
|
|
17227
|
-
([, claim]) =>
|
|
17228
|
-
(claim.status === 'completed' || claim.status === 'skipped') &&
|
|
17229
|
-
claim.output === undefined,
|
|
17230
|
-
)
|
|
17231
|
-
.map(([receiptKey]) => receiptKey);
|
|
17232
|
-
if (terminalClaimKeys.length > 0) {
|
|
17233
|
-
const hasBoundedClaimManifests = terminalClaimKeys.every(
|
|
17234
|
-
(receiptKey) => {
|
|
17235
|
-
const claim = claims.get(receiptKey);
|
|
17236
|
-
return (
|
|
17237
|
-
Number.isSafeInteger(claim?.outputByteLength) &&
|
|
17238
|
-
(claim?.outputByteLength ?? -1) >= 0
|
|
17239
|
-
);
|
|
17240
|
-
},
|
|
17241
|
-
);
|
|
17242
|
-
const expectedClaimBytes = hasBoundedClaimManifests
|
|
17243
|
-
? terminalClaimKeys.reduce(
|
|
17244
|
-
(total, receiptKey) =>
|
|
17245
|
-
total + (claims.get(receiptKey)?.outputByteLength ?? 0),
|
|
17246
|
-
0,
|
|
17247
|
-
)
|
|
17248
|
-
: undefined;
|
|
17249
|
-
const hydratedClaims = await this.withRuntimeReceiptHydrationTurn(
|
|
17250
|
-
async (hydrationTurn) => {
|
|
17251
|
-
const receipts =
|
|
17252
|
-
await this.getRuntimeStepReceipts(terminalClaimKeys);
|
|
17253
|
-
hydrationTurn.settleBytes(
|
|
17254
|
-
runtimeStepReceiptPayloadBytes(receipts.values()),
|
|
17255
|
-
);
|
|
17256
|
-
return receipts;
|
|
17257
|
-
},
|
|
17258
|
-
{
|
|
17259
|
-
logicalCalls: claimTurnLogicalCalls,
|
|
17260
|
-
receiptKeys: terminalClaimKeys.length,
|
|
17261
|
-
...(expectedClaimBytes !== undefined
|
|
17262
|
-
? { expectedBytes: expectedClaimBytes }
|
|
17263
|
-
: {}),
|
|
17264
|
-
source: 'map_legacy_claim',
|
|
17265
|
-
},
|
|
17266
|
-
);
|
|
17267
|
-
for (const [receiptKey, receipt] of hydratedClaims) {
|
|
17268
|
-
claims.set(receiptKey, receipt);
|
|
17269
|
-
}
|
|
17270
|
-
}
|
|
17271
|
-
// Every get/claim round trip this dispatch group paid before any
|
|
17272
|
-
// provider work could start.
|
|
17273
|
-
this.resourceGovernor.observe({
|
|
17274
|
-
receiptClaimMs: Date.now() - claimStartedAt,
|
|
17275
|
-
});
|
|
17276
|
-
// Claims are compact state transitions. They do not own output
|
|
17277
|
-
// bytes and must not wait behind the hydration lane's global
|
|
17278
|
-
// memory headroom; a terminal manifest above is the only path
|
|
17279
|
-
// that opens a byte-admitted hydration turn.
|
|
17280
|
-
const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
|
|
17281
|
-
const durableRecoveredRequests: ToolCallRequest[] = [];
|
|
17282
|
-
const resolveRequestsFromReceipt = async (
|
|
17283
|
-
requestsForKey: ToolCallRequest[],
|
|
17284
|
-
receipt: RuntimeStepReceipt,
|
|
17285
|
-
source: 'cache' | 'in_flight',
|
|
17286
|
-
): Promise<void> => {
|
|
17287
|
-
const request = requestsForKey[0];
|
|
17288
|
-
if (!request) return;
|
|
17289
|
-
const receiptKey = request.receiptKey?.trim() || null;
|
|
17290
|
-
if (!receiptKey) {
|
|
17291
|
-
throw new Error(`${source} tool recovery needs receipt key.`);
|
|
17292
|
-
}
|
|
17293
|
-
const wrapped = await this.wrapToolExecutionResult({
|
|
17294
|
-
toolId,
|
|
17295
|
-
status:
|
|
17296
|
-
receipt.output === null || receipt.output === undefined
|
|
17297
|
-
? 'no_result'
|
|
17298
|
-
: 'completed',
|
|
17299
|
-
result: this.runtimeReceiptOutput(receipt),
|
|
17300
|
-
requestInput: request.input,
|
|
17301
|
-
execution: toolExecutionMetadataForOutcome({
|
|
17302
|
-
kind: source,
|
|
17303
|
-
cacheKey: request.cacheKey,
|
|
17304
|
-
receiptKey,
|
|
17305
|
-
attachedToReceiptKey: receiptKey,
|
|
17306
|
-
}),
|
|
17307
|
-
});
|
|
17308
|
-
this.cacheToolResult(toolId, request.cacheKey, wrapped);
|
|
17309
|
-
for (const waitingRequest of requestsForKey) {
|
|
17310
|
-
successfulLiveStepCallIds.add(waitingRequest.callId);
|
|
17311
|
-
const resolver = this.toolCallResolvers.get(
|
|
17312
|
-
waitingRequest.callId,
|
|
17313
|
-
);
|
|
17314
|
-
if (resolver) {
|
|
17315
|
-
resolver.resolve(wrapped);
|
|
17316
|
-
this.toolCallResolvers.delete(waitingRequest.callId);
|
|
17317
|
-
}
|
|
17318
|
-
this.emitScopedFieldMetaUpdate({
|
|
17319
|
-
rowId: waitingRequest.rowId,
|
|
17320
|
-
key: waitingRequest.rowKey ?? null,
|
|
17321
|
-
tableNamespace: waitingRequest.tableNamespace ?? null,
|
|
17322
|
-
fieldName: waitingRequest.fieldName,
|
|
17323
|
-
status: 'cached',
|
|
17324
|
-
rowStatus: 'running',
|
|
17325
|
-
stage: toolId,
|
|
17326
|
-
provider: null,
|
|
17327
|
-
error: null,
|
|
17328
|
-
reused: true,
|
|
17329
|
-
dataPatch: {},
|
|
17330
|
-
});
|
|
17331
|
-
}
|
|
17332
|
-
};
|
|
17333
|
-
const claimOwnerWithLiveFollowers = (
|
|
17334
|
-
owner: ToolCallRequest | undefined,
|
|
17335
|
-
followers: ToolCallRequest[],
|
|
17336
|
-
): void => {
|
|
17337
|
-
if (!owner) return;
|
|
17338
|
-
claimedRequests.push(owner);
|
|
17339
|
-
if (followers.length > 0) {
|
|
17340
|
-
liveFollowersByOwnerCallId.set(owner.callId, followers);
|
|
17341
|
-
}
|
|
17342
|
-
};
|
|
17343
|
-
const resolveReceiptWaitMaxAttempts = (
|
|
17344
|
-
requestsForKey: ToolCallRequest[],
|
|
17345
|
-
): number | undefined =>
|
|
17346
|
-
requestsForKey.length > 0
|
|
17347
|
-
? Math.max(
|
|
17348
|
-
...requestsForKey.map((request) =>
|
|
17349
|
-
resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
17350
|
-
typeof request.receiptWaitMs === 'number'
|
|
17351
|
-
? { max_wait_ms: request.receiptWaitMs }
|
|
17352
|
-
: request.input,
|
|
17353
|
-
),
|
|
17354
|
-
),
|
|
17355
|
-
)
|
|
17356
|
-
: undefined;
|
|
17357
|
-
const queueExistingRunningReceiptWait = (
|
|
17358
|
-
receiptKey: string,
|
|
17359
|
-
requestsForKey: ToolCallRequest[],
|
|
17360
|
-
forceAfterWait = false,
|
|
17361
|
-
skipWait = false,
|
|
17362
|
-
): void => {
|
|
17363
|
-
durableExistingRunningWaits.push({
|
|
17364
|
-
receiptKey,
|
|
17365
|
-
requestsForKey,
|
|
17366
|
-
forceAfterWait,
|
|
17367
|
-
waitMaxAttempts: resolveReceiptWaitMaxAttempts(requestsForKey),
|
|
17368
|
-
skipWait,
|
|
16440
|
+
if (pendingRequests.length > 0) {
|
|
16441
|
+
const strategy = operationStrategy;
|
|
16442
|
+
|
|
16443
|
+
if (strategy) {
|
|
16444
|
+
const compiledBatches = compileRequestsWithStrategy({
|
|
16445
|
+
requests: pendingRequests,
|
|
16446
|
+
strategy,
|
|
16447
|
+
getPayload: (request: ToolCallRequest) => request.input,
|
|
17369
16448
|
});
|
|
17370
|
-
|
|
17371
|
-
|
|
17372
|
-
|
|
17373
|
-
|
|
17374
|
-
|
|
17375
|
-
|
|
17376
|
-
|
|
17377
|
-
|
|
17378
|
-
|
|
17379
|
-
|
|
17380
|
-
|
|
17381
|
-
|
|
17382
|
-
|
|
17383
|
-
|
|
17384
|
-
|
|
17385
|
-
|
|
17386
|
-
);
|
|
17387
|
-
return;
|
|
17388
|
-
}
|
|
17389
|
-
} else if (waitResult?.skipWait !== true) {
|
|
17390
|
-
const waitMaxAttempts =
|
|
17391
|
-
resolveReceiptWaitMaxAttempts(requestsForKey);
|
|
17392
|
-
try {
|
|
17393
|
-
const completed = await this.waitForCompletedRuntimeToolReceipt(
|
|
17394
|
-
receiptKey,
|
|
17395
|
-
waitMaxAttempts,
|
|
17396
|
-
);
|
|
17397
|
-
if (!forceAfterWait) {
|
|
17398
|
-
await resolveRequestsFromReceipt(
|
|
17399
|
-
requestsForKey,
|
|
17400
|
-
completed,
|
|
17401
|
-
'in_flight',
|
|
17402
|
-
);
|
|
17403
|
-
return;
|
|
17404
|
-
}
|
|
17405
|
-
} catch (error) {
|
|
17406
|
-
if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
|
|
17407
|
-
for (const request of requestsForKey) {
|
|
17408
|
-
await this.rejectToolCall(toolId, request, error, {
|
|
17409
|
-
persistReceiptFailure: false,
|
|
17410
|
-
});
|
|
17411
|
-
}
|
|
17412
|
-
return;
|
|
17413
|
-
}
|
|
17414
|
-
}
|
|
17415
|
-
}
|
|
17416
|
-
const reclaimed = (
|
|
17417
|
-
await this.claimRuntimeStepReceipts(
|
|
17418
|
-
[receiptKey],
|
|
17419
|
-
this.currentReceiptOwnerRunId,
|
|
17420
|
-
true,
|
|
17421
|
-
forceAfterWait,
|
|
17422
|
-
)
|
|
17423
|
-
).get(receiptKey);
|
|
17424
|
-
if (
|
|
17425
|
-
reclaimed?.status === 'completed' ||
|
|
17426
|
-
reclaimed?.status === 'skipped'
|
|
17427
|
-
) {
|
|
17428
|
-
await resolveRequestsFromReceipt(
|
|
17429
|
-
requestsForKey,
|
|
17430
|
-
reclaimed,
|
|
17431
|
-
'cache',
|
|
17432
|
-
);
|
|
17433
|
-
durableRecoveredRequests.push(...requestsForKey);
|
|
17434
|
-
return;
|
|
17435
|
-
}
|
|
17436
|
-
if (reclaimed?.status === 'failed') {
|
|
17437
|
-
for (const request of requestsForKey) {
|
|
17438
|
-
await this.rejectToolCall(
|
|
17439
|
-
toolId,
|
|
17440
|
-
request,
|
|
17441
|
-
runtimeReceiptFailureError(
|
|
17442
|
-
reclaimed,
|
|
17443
|
-
'Durable tool call failed',
|
|
17444
|
-
this.currentToolErrorSchemaVersion,
|
|
17445
|
-
),
|
|
17446
|
-
);
|
|
17447
|
-
}
|
|
17448
|
-
return;
|
|
17449
|
-
}
|
|
17450
|
-
if (this.isOwnedClaimedRuntimeReceipt(reclaimed)) {
|
|
17451
|
-
const [owner, ...waiters] = requestsForKey;
|
|
17452
|
-
if (!owner) return;
|
|
17453
|
-
if (!reclaimed.leaseId) {
|
|
17454
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
17455
|
-
receiptKey,
|
|
17456
|
-
runId: this.currentReceiptOwnerRunId,
|
|
17457
|
-
leaseId: 'missing',
|
|
17458
|
-
});
|
|
17459
|
-
}
|
|
17460
|
-
owner.receiptLeaseId = reclaimed.leaseId ?? null;
|
|
17461
|
-
owner.receiptLeaseExpiresAt = reclaimed.leaseExpiresAt ?? null;
|
|
17462
|
-
if (waiters.length > 0) {
|
|
17463
|
-
liveFollowersByOwnerCallId.set(owner.callId, waiters);
|
|
17464
|
-
}
|
|
17465
|
-
try {
|
|
17466
|
-
// Circuit breaker: a receipt reclaimed after its owner's lease
|
|
17467
|
-
// lapsed is still a fresh provider call, and the latch may
|
|
17468
|
-
// have tripped while this group waited for that lease.
|
|
16449
|
+
|
|
16450
|
+
const batchParallelismCeiling =
|
|
16451
|
+
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
16452
|
+
const batchSize =
|
|
16453
|
+
compiledBatches.length > 0
|
|
16454
|
+
? await this.resourceGovernor.suggestedToolParallelism(
|
|
16455
|
+
compiledBatches[0]!.batchOperation,
|
|
16456
|
+
batchParallelismCeiling,
|
|
16457
|
+
)
|
|
16458
|
+
: batchParallelismCeiling;
|
|
16459
|
+
await executeChunkedRequests({
|
|
16460
|
+
requests: compiledBatches,
|
|
16461
|
+
batchSize,
|
|
16462
|
+
execute: async (batch) => {
|
|
16463
|
+
// Circuit breaker: skip dispatching this batch's provider call
|
|
16464
|
+
// once a persistence failure has occurred in this run.
|
|
17469
16465
|
if (this.persistenceLatch.tripped) {
|
|
17470
16466
|
this.persistenceLatch.preventedCallCount +=
|
|
17471
|
-
|
|
16467
|
+
batch.memberRequests.length;
|
|
17472
16468
|
throw new RuntimePersistenceCircuitOpenError(
|
|
17473
16469
|
this.persistenceLatch,
|
|
17474
16470
|
);
|
|
17475
16471
|
}
|
|
17476
|
-
const
|
|
17477
|
-
|
|
17478
|
-
|
|
17479
|
-
|
|
17480
|
-
|
|
17481
|
-
|
|
17482
|
-
|
|
17483
|
-
|
|
17484
|
-
apiOptions: {
|
|
17485
|
-
beforeProviderCall: () =>
|
|
17486
|
-
this.assertRuntimeToolReceiptOwnership([owner]),
|
|
17487
|
-
durableCallReceiptKey: receiptKey,
|
|
17488
|
-
playNodeScope: playNodeScopeForToolCallRequest(owner),
|
|
17489
|
-
executionAuthScopeDigest: owner.executionAuthScopeDigest,
|
|
17490
|
-
receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
|
|
17491
|
-
heartbeatReceipt: () =>
|
|
17492
|
-
this.renewRuntimeToolReceiptOwnership([owner]),
|
|
17493
|
-
providerIdempotencyKey:
|
|
17494
|
-
this.providerIdempotencyKeyForToolCall({
|
|
17495
|
-
operationKey:
|
|
17496
|
-
this.providerOperationKeyForQueuedToolCall(owner),
|
|
17497
|
-
force: owner.force === true,
|
|
17498
|
-
logicalCallId: owner.logicalCallId,
|
|
17499
|
-
}),
|
|
17500
|
-
...this.providerForceRefreshOptionsForToolCall({
|
|
17501
|
-
operationKey:
|
|
17502
|
-
this.providerOperationKeyForQueuedToolCall(owner),
|
|
17503
|
-
force: owner.force,
|
|
17504
|
-
requiresExecutionFence: owner.requiresExecutionFence,
|
|
17505
|
-
logicalCallId: owner.logicalCallId,
|
|
17506
|
-
}),
|
|
17507
|
-
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
|
|
17508
|
-
owner,
|
|
17509
|
-
]),
|
|
17510
|
-
},
|
|
17511
|
-
},
|
|
17512
|
-
);
|
|
17513
|
-
const result = await this.resolveToolCall(
|
|
17514
|
-
toolId,
|
|
17515
|
-
owner,
|
|
17516
|
-
execution,
|
|
17517
|
-
);
|
|
17518
|
-
if (result != null) {
|
|
17519
|
-
successfulLiveStepCallIds.add(owner.callId);
|
|
17520
|
-
}
|
|
17521
|
-
resolveLiveFollowers(owner, result);
|
|
17522
|
-
recordToolStep([owner]);
|
|
17523
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
17524
|
-
} catch (error) {
|
|
17525
|
-
await rejectWithLiveFollowers(owner, error);
|
|
17526
|
-
}
|
|
17527
|
-
return;
|
|
17528
|
-
}
|
|
17529
|
-
for (const request of requestsForKey) {
|
|
17530
|
-
await this.rejectToolCall(
|
|
17531
|
-
toolId,
|
|
17532
|
-
request,
|
|
17533
|
-
new RuntimeReceiptWaitTimeoutError(receiptKey),
|
|
17534
|
-
{ persistReceiptFailure: false },
|
|
17535
|
-
);
|
|
17536
|
-
}
|
|
17537
|
-
return;
|
|
17538
|
-
};
|
|
17539
|
-
const reclaimExistingRunningReceipts = async (
|
|
17540
|
-
waits: Array<(typeof durableExistingRunningWaits)[number]>,
|
|
17541
|
-
): Promise<void> => {
|
|
17542
|
-
if (waits.length === 0) return;
|
|
17543
|
-
const settlements = await Promise.allSettled(
|
|
17544
|
-
waits.map((wait) =>
|
|
17545
|
-
reclaimExistingRunningReceiptAfterWait(
|
|
17546
|
-
wait.receiptKey,
|
|
17547
|
-
wait.requestsForKey,
|
|
17548
|
-
wait.forceAfterWait,
|
|
17549
|
-
{ skipWait: true },
|
|
17550
|
-
),
|
|
17551
|
-
),
|
|
17552
|
-
);
|
|
17553
|
-
const rejected = settlements.find(
|
|
17554
|
-
(settlement) => settlement.status === 'rejected',
|
|
17555
|
-
);
|
|
17556
|
-
if (rejected?.status === 'rejected') throw rejected.reason;
|
|
17557
|
-
};
|
|
17558
|
-
const processExistingRunningReceiptWaits =
|
|
17559
|
-
async (): Promise<void> => {
|
|
17560
|
-
if (durableExistingRunningWaits.length === 0) return;
|
|
17561
|
-
const waitSignal = durableExistingRunningAbort.signal;
|
|
17562
|
-
// A claim that came back `existing` behind an expired lease is
|
|
17563
|
-
// not blocked on live work; it is holding a dead run's row.
|
|
17564
|
-
// Reclaim those now so this run executes the call itself, and
|
|
17565
|
-
// poll only the leases that are genuinely still alive.
|
|
17566
|
-
const expiredLeaseWaits = durableExistingRunningWaits.filter(
|
|
17567
|
-
(wait) => wait.skipWait === true,
|
|
17568
|
-
);
|
|
17569
|
-
const pendingWaits = new Map(
|
|
17570
|
-
durableExistingRunningWaits
|
|
17571
|
-
.filter((wait) => wait.skipWait !== true)
|
|
17572
|
-
.map((wait) => [wait.receiptKey, wait]),
|
|
17573
|
-
);
|
|
17574
|
-
const parkedAtMs = Date.now();
|
|
17575
|
-
for (const wait of durableExistingRunningWaits) {
|
|
17576
|
-
this.#runtimeReceiptWaitParking.set(wait.receiptKey, {
|
|
17577
|
-
startedAtMs: parkedAtMs,
|
|
17578
|
-
rows: wait.requestsForKey.length,
|
|
17579
|
-
stage:
|
|
17580
|
-
wait.skipWait === true
|
|
17581
|
-
? 'expired_lease_reclaim'
|
|
17582
|
-
: 'receipt_lease_wait',
|
|
17583
|
-
});
|
|
17584
|
-
}
|
|
17585
|
-
const pollWaits = async (): Promise<void> => {
|
|
17586
|
-
if (pendingWaits.size === 0) return;
|
|
17587
|
-
const waitMaxAttempts =
|
|
17588
|
-
Math.max(
|
|
17589
|
-
...[...pendingWaits.values()].map(
|
|
17590
|
-
(wait) => wait.waitMaxAttempts ?? 0,
|
|
17591
|
-
),
|
|
17592
|
-
) || DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
|
|
17593
|
-
// The backoff cap means the attempt count no longer states a
|
|
17594
|
-
// wall time. This deadline is the budget those attempts were
|
|
17595
|
-
// meant to express; crossing it falls through to the same
|
|
17596
|
-
// reclaim path an expired lease takes.
|
|
17597
|
-
const waitDeadline =
|
|
17598
|
-
Date.now() +
|
|
17599
|
-
runtimeReceiptWaitBudgetMs(
|
|
17600
|
-
waitMaxAttempts,
|
|
17601
|
-
DURABLE_RECEIPT_WAIT_DELAY_MS,
|
|
16472
|
+
const aggregateForceRefreshIdentities =
|
|
16473
|
+
batch.memberRequests.map((request) =>
|
|
16474
|
+
this.providerInvocationKeyForToolCall({
|
|
16475
|
+
operationKey:
|
|
16476
|
+
this.providerOperationKeyForQueuedToolCall(request),
|
|
16477
|
+
logicalCallId: request.logicalCallId,
|
|
16478
|
+
force: request.force,
|
|
16479
|
+
}),
|
|
17602
16480
|
);
|
|
17603
|
-
|
|
17604
|
-
|
|
17605
|
-
|
|
17606
|
-
|
|
17607
|
-
|
|
17608
|
-
.
|
|
16481
|
+
const legacyForceRefreshIdentities = batch.memberRequests
|
|
16482
|
+
.filter((request) => request.force === true)
|
|
16483
|
+
.map((request) =>
|
|
16484
|
+
this.providerInvocationKeyForToolCall({
|
|
16485
|
+
operationKey:
|
|
16486
|
+
this.providerOperationKeyForQueuedToolCall(request),
|
|
16487
|
+
logicalCallId: request.logicalCallId,
|
|
16488
|
+
force: true,
|
|
16489
|
+
}),
|
|
17609
16490
|
);
|
|
17610
|
-
|
|
17611
|
-
|
|
17612
|
-
|
|
17613
|
-
|
|
17614
|
-
|
|
17615
|
-
|
|
17616
|
-
|
|
17617
|
-
|
|
17618
|
-
|
|
17619
|
-
|
|
17620
|
-
|
|
17621
|
-
|
|
17622
|
-
|
|
17623
|
-
|
|
17624
|
-
|
|
17625
|
-
|
|
17626
|
-
|
|
17627
|
-
|
|
17628
|
-
|
|
17629
|
-
|
|
17630
|
-
|
|
17631
|
-
|
|
17632
|
-
|
|
17633
|
-
|
|
17634
|
-
|
|
17635
|
-
|
|
17636
|
-
|
|
17637
|
-
|
|
17638
|
-
|
|
17639
|
-
|
|
17640
|
-
|
|
17641
|
-
|
|
17642
|
-
|
|
17643
|
-
|
|
17644
|
-
|
|
17645
|
-
|
|
17646
|
-
|
|
17647
|
-
[...pendingWaits.keys()],
|
|
17648
|
-
1,
|
|
17649
|
-
);
|
|
17650
|
-
turn.settleBytes(
|
|
17651
|
-
runtimeStepReceiptPayloadBytes(
|
|
17652
|
-
waited.completed.values(),
|
|
17653
|
-
),
|
|
17654
|
-
);
|
|
17655
|
-
if (waitSignal.aborted) return;
|
|
17656
|
-
const settledWaits = [...pendingWaits.values()].filter(
|
|
17657
|
-
(wait) =>
|
|
17658
|
-
waited.completed.has(wait.receiptKey) ||
|
|
17659
|
-
waited.failed.has(wait.receiptKey),
|
|
17660
|
-
);
|
|
17661
|
-
const deliverySettlements = await Promise.allSettled(
|
|
17662
|
-
settledWaits.map(async (wait) => {
|
|
17663
|
-
const failed = waited.failed.get(wait.receiptKey);
|
|
17664
|
-
if (failed) {
|
|
17665
|
-
for (const request of wait.requestsForKey) {
|
|
17666
|
-
await this.rejectToolCall(
|
|
17667
|
-
toolId,
|
|
17668
|
-
request,
|
|
17669
|
-
failed,
|
|
17670
|
-
{
|
|
17671
|
-
persistReceiptFailure: false,
|
|
17672
|
-
},
|
|
17673
|
-
);
|
|
17674
|
-
}
|
|
17675
|
-
return;
|
|
17676
|
-
}
|
|
17677
|
-
await reclaimExistingRunningReceiptAfterWait(
|
|
17678
|
-
wait.receiptKey,
|
|
17679
|
-
wait.requestsForKey,
|
|
17680
|
-
wait.forceAfterWait,
|
|
17681
|
-
{
|
|
17682
|
-
completedReceipt: waited.completed.get(
|
|
17683
|
-
wait.receiptKey,
|
|
16491
|
+
const batchActionVersion =
|
|
16492
|
+
(
|
|
16493
|
+
await this.#options.getToolActionCacheVersion?.(
|
|
16494
|
+
batch.batchOperation,
|
|
16495
|
+
)
|
|
16496
|
+
)?.trim() ?? 'unversioned';
|
|
16497
|
+
const aggregateProviderOperationKey =
|
|
16498
|
+
buildDurableToolPayloadProviderIdempotencyKey({
|
|
16499
|
+
orgId: this.#options.orgId,
|
|
16500
|
+
toolId: batch.batchOperation,
|
|
16501
|
+
executionAuthScopeDigest:
|
|
16502
|
+
batch.memberRequests[0]?.executionAuthScopeDigest ?? null,
|
|
16503
|
+
providerActionVersion: batchActionVersion,
|
|
16504
|
+
payload: batch.batchPayload,
|
|
16505
|
+
forceRefreshIdentities: aggregateForceRefreshIdentities,
|
|
16506
|
+
});
|
|
16507
|
+
const aggregateReceiptKey = this.toolTransportReceiptKey({
|
|
16508
|
+
toolId: batch.batchOperation,
|
|
16509
|
+
providerOperationKey: aggregateProviderOperationKey,
|
|
16510
|
+
});
|
|
16511
|
+
let releaseToolSlot: () => void = () => undefined;
|
|
16512
|
+
try {
|
|
16513
|
+
const execution =
|
|
16514
|
+
await this.executeMappedToolCallThroughModule({
|
|
16515
|
+
request: batch.memberRequests[0]!,
|
|
16516
|
+
dispatchToolId: batch.batchOperation,
|
|
16517
|
+
dispatchPayload: batch.batchPayload,
|
|
16518
|
+
resultReceiptKey: aggregateReceiptKey,
|
|
16519
|
+
providerOperationKey: aggregateProviderOperationKey,
|
|
16520
|
+
apiOptions: {
|
|
16521
|
+
durableCallReceiptKey: aggregateReceiptKey,
|
|
16522
|
+
legacyProviderRetryIdentities: [
|
|
16523
|
+
{
|
|
16524
|
+
transportReceiptKey:
|
|
16525
|
+
buildDurableToolAggregateReceiptKey({
|
|
16526
|
+
receiptKeys: batch.memberRequests.map(
|
|
16527
|
+
(member) => member.cacheKey,
|
|
17684
16528
|
),
|
|
17685
|
-
|
|
17686
|
-
|
|
17687
|
-
|
|
17688
|
-
|
|
17689
|
-
|
|
17690
|
-
|
|
17691
|
-
|
|
17692
|
-
|
|
17693
|
-
|
|
17694
|
-
|
|
17695
|
-
|
|
17696
|
-
|
|
17697
|
-
|
|
17698
|
-
|
|
17699
|
-
|
|
17700
|
-
|
|
17701
|
-
|
|
17702
|
-
|
|
17703
|
-
|
|
17704
|
-
|
|
17705
|
-
|
|
17706
|
-
|
|
17707
|
-
|
|
17708
|
-
|
|
17709
|
-
|
|
17710
|
-
|
|
17711
|
-
|
|
17712
|
-
|
|
17713
|
-
|
|
17714
|
-
|
|
17715
|
-
|
|
17716
|
-
|
|
17717
|
-
|
|
17718
|
-
|
|
17719
|
-
|
|
17720
|
-
|
|
17721
|
-
|
|
17722
|
-
|
|
17723
|
-
|
|
17724
|
-
|
|
17725
|
-
|
|
17726
|
-
(
|
|
17727
|
-
|
|
17728
|
-
|
|
17729
|
-
|
|
17730
|
-
|
|
17731
|
-
|
|
17732
|
-
|
|
17733
|
-
|
|
17734
|
-
|
|
17735
|
-
|
|
17736
|
-
|
|
17737
|
-
|
|
16529
|
+
prefix: 'batch',
|
|
16530
|
+
aggregateReceiptPrefix:
|
|
16531
|
+
buildDurableToolReceiptPrefix({
|
|
16532
|
+
orgId: this.#options.orgId,
|
|
16533
|
+
toolId: batch.batchOperation,
|
|
16534
|
+
}),
|
|
16535
|
+
}),
|
|
16536
|
+
providerOperationKey:
|
|
16537
|
+
buildDurableToolPayloadProviderIdempotencyKey({
|
|
16538
|
+
orgId: this.#options.orgId,
|
|
16539
|
+
toolId: batch.batchOperation,
|
|
16540
|
+
executionAuthScopeDigest:
|
|
16541
|
+
batch.memberRequests[0]
|
|
16542
|
+
?.executionAuthScopeDigest ?? null,
|
|
16543
|
+
providerActionVersion: batchActionVersion,
|
|
16544
|
+
payload: batch.batchPayload,
|
|
16545
|
+
forceRefreshIdentities:
|
|
16546
|
+
legacyForceRefreshIdentities,
|
|
16547
|
+
}),
|
|
16548
|
+
forceRefreshIdentities:
|
|
16549
|
+
legacyForceRefreshIdentities,
|
|
16550
|
+
},
|
|
16551
|
+
],
|
|
16552
|
+
// A native batch is one physical provider request for
|
|
16553
|
+
// many rows of the same column, so the batch's node scope
|
|
16554
|
+
// is its members' shared scope.
|
|
16555
|
+
playNodeScope: playNodeScopeForToolCallRequest(
|
|
16556
|
+
batch.memberRequests[0]!,
|
|
16557
|
+
batch.batchOperation,
|
|
16558
|
+
),
|
|
16559
|
+
executionAuthScopeDigest:
|
|
16560
|
+
batch.memberRequests[0]?.executionAuthScopeDigest ??
|
|
16561
|
+
null,
|
|
16562
|
+
providerIdempotencyKey: aggregateProviderOperationKey,
|
|
16563
|
+
providerOperationForceRefreshIdentities:
|
|
16564
|
+
aggregateForceRefreshIdentities,
|
|
16565
|
+
receiptLeaseExpiresAt: batch.memberRequests.reduce<
|
|
16566
|
+
string | null
|
|
16567
|
+
>((earliest, request) => {
|
|
16568
|
+
const expiresAt =
|
|
16569
|
+
request.receiptLeaseExpiresAt ?? null;
|
|
16570
|
+
if (!expiresAt) return earliest;
|
|
16571
|
+
if (!earliest) return expiresAt;
|
|
16572
|
+
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
16573
|
+
? expiresAt
|
|
16574
|
+
: earliest;
|
|
16575
|
+
}, null),
|
|
16576
|
+
timeoutMs: resolveRuntimeTimeoutMsForRequests(
|
|
16577
|
+
batch.memberRequests,
|
|
16578
|
+
),
|
|
16579
|
+
retainToolSlot: (release) => {
|
|
16580
|
+
releaseToolSlot = release;
|
|
16581
|
+
},
|
|
17738
16582
|
},
|
|
17739
|
-
);
|
|
17740
|
-
|
|
17741
|
-
|
|
17742
|
-
|
|
17743
|
-
|
|
17744
|
-
!(
|
|
17745
|
-
error instanceof RuntimeReceiptHydrationTurnTimeoutError
|
|
17746
|
-
)
|
|
17747
|
-
) {
|
|
17748
|
-
throw error;
|
|
17749
|
-
}
|
|
17750
|
-
break;
|
|
17751
|
-
}
|
|
17752
|
-
}
|
|
17753
|
-
if (waitSignal.aborted) return;
|
|
17754
|
-
await reclaimExistingRunningReceipts([
|
|
17755
|
-
...pendingWaits.values(),
|
|
17756
|
-
]);
|
|
17757
|
-
};
|
|
17758
|
-
try {
|
|
17759
|
-
const lanes = await Promise.allSettled([
|
|
17760
|
-
reclaimExistingRunningReceipts(expiredLeaseWaits),
|
|
17761
|
-
pollWaits(),
|
|
17762
|
-
]);
|
|
17763
|
-
const rejectedLane = lanes.find(
|
|
17764
|
-
(lane) => lane.status === 'rejected',
|
|
17765
|
-
);
|
|
17766
|
-
if (rejectedLane?.status === 'rejected') {
|
|
17767
|
-
throw rejectedLane.reason;
|
|
17768
|
-
}
|
|
17769
|
-
} finally {
|
|
17770
|
-
for (const wait of durableExistingRunningWaits) {
|
|
17771
|
-
this.#runtimeReceiptWaitParking.delete(wait.receiptKey);
|
|
16583
|
+
});
|
|
16584
|
+
return { execution, releaseToolSlot };
|
|
16585
|
+
} catch (error) {
|
|
16586
|
+
releaseToolSlot();
|
|
16587
|
+
throw error;
|
|
17772
16588
|
}
|
|
17773
|
-
}
|
|
17774
|
-
|
|
17775
|
-
|
|
17776
|
-
|
|
17777
|
-
|
|
17778
|
-
|
|
17779
|
-
|
|
17780
|
-
|
|
17781
|
-
receiptKey,
|
|
17782
|
-
requestsForKey,
|
|
17783
|
-
group.forceRefresh,
|
|
17784
|
-
);
|
|
17785
|
-
continue;
|
|
17786
|
-
}
|
|
17787
|
-
if (claim?.status === 'completed' || claim?.status === 'skipped') {
|
|
17788
|
-
await resolveRequestsFromReceipt(requestsForKey, claim, 'cache');
|
|
17789
|
-
durableRecoveredRequests.push(...requestsForKey);
|
|
17790
|
-
continue;
|
|
17791
|
-
}
|
|
17792
|
-
if (claim?.status === 'failed') {
|
|
17793
|
-
for (const request of requestsForKey) {
|
|
17794
|
-
await this.rejectToolCall(
|
|
17795
|
-
toolId,
|
|
17796
|
-
request,
|
|
17797
|
-
runtimeReceiptFailureError(
|
|
17798
|
-
claim,
|
|
17799
|
-
'Durable tool call failed',
|
|
17800
|
-
this.currentToolErrorSchemaVersion,
|
|
17801
|
-
),
|
|
17802
|
-
);
|
|
17803
|
-
}
|
|
17804
|
-
continue;
|
|
17805
|
-
}
|
|
17806
|
-
if (
|
|
17807
|
-
(claim?.status === 'queued' ||
|
|
17808
|
-
claim?.status === 'pending' ||
|
|
17809
|
-
claim?.status === 'running') &&
|
|
17810
|
-
claim.claimState === 'existing'
|
|
17811
|
-
) {
|
|
17812
|
-
queueExistingRunningReceiptWait(
|
|
17813
|
-
receiptKey,
|
|
17814
|
-
requestsForKey,
|
|
17815
|
-
group.forceRefresh,
|
|
17816
|
-
!runtimeReceiptClaimLeaseIsLive(claim),
|
|
17817
|
-
);
|
|
17818
|
-
continue;
|
|
17819
|
-
}
|
|
17820
|
-
if (this.isOwnedClaimedRuntimeReceipt(claim)) {
|
|
17821
|
-
const [owner, ...waiters] = requestsForKey;
|
|
17822
|
-
if (!claim.leaseId) {
|
|
17823
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
17824
|
-
receiptKey,
|
|
17825
|
-
runId: this.currentReceiptOwnerRunId,
|
|
17826
|
-
leaseId: 'missing',
|
|
17827
|
-
});
|
|
17828
|
-
}
|
|
17829
|
-
if (owner) {
|
|
17830
|
-
owner.receiptLeaseId = claim.leaseId ?? null;
|
|
17831
|
-
owner.receiptLeaseExpiresAt = claim.leaseExpiresAt ?? null;
|
|
17832
|
-
const claimLeaseExpiresAtMs = Date.parse(
|
|
17833
|
-
claim.leaseExpiresAt ?? '',
|
|
17834
|
-
);
|
|
17835
|
-
pendingReceiptClaimsEstablishExecutionFence &&=
|
|
17836
|
-
claim.claimState === 'claimed' &&
|
|
17837
|
-
claim.status === 'running' &&
|
|
17838
|
-
Number.isFinite(claimLeaseExpiresAtMs) &&
|
|
17839
|
-
claimLeaseExpiresAtMs > Date.now();
|
|
17840
|
-
}
|
|
17841
|
-
claimOwnerWithLiveFollowers(owner, waiters);
|
|
17842
|
-
continue;
|
|
17843
|
-
}
|
|
17844
|
-
queueExistingRunningReceiptWait(
|
|
17845
|
-
receiptKey,
|
|
17846
|
-
requestsForKey,
|
|
17847
|
-
group.forceRefresh,
|
|
17848
|
-
);
|
|
17849
|
-
}
|
|
17850
|
-
|
|
17851
|
-
pendingRequests.length = 0;
|
|
17852
|
-
pendingRequests.push(...claimedRequests);
|
|
17853
|
-
durableExistingRunningState.settlement =
|
|
17854
|
-
durableExistingRunningWaits.length > 0
|
|
17855
|
-
? processExistingRunningReceiptWaits().then(
|
|
17856
|
-
() => ({ status: 'fulfilled' as const }),
|
|
17857
|
-
(reason: unknown) => ({
|
|
17858
|
-
status: 'rejected' as const,
|
|
17859
|
-
reason,
|
|
17860
|
-
}),
|
|
17861
|
-
)
|
|
17862
|
-
: null;
|
|
17863
|
-
recordToolStep(durableRecoveredRequests);
|
|
17864
|
-
}
|
|
17865
|
-
|
|
17866
|
-
// Claims establish one lease cohort. Revalidate that cohort in one
|
|
17867
|
-
// gateway operation before provider admission, then let the existing
|
|
17868
|
-
// cohort heartbeat keep the undispatched tail alive. Revalidating each
|
|
17869
|
-
// call separately after a tool slot opens fragments the writer into
|
|
17870
|
-
// one-row batches and starves otherwise healthy providers.
|
|
17871
|
-
const pendingReceiptHeartbeat = this.startPendingToolReceiptHeartbeat(
|
|
17872
|
-
pendingRequests,
|
|
17873
|
-
pendingReceiptClaimsEstablishExecutionFence,
|
|
17874
|
-
);
|
|
17875
|
-
let providerFailure: { reason: unknown } | null = null;
|
|
17876
|
-
try {
|
|
17877
|
-
if (pendingReceiptHeartbeat) {
|
|
17878
|
-
await pendingReceiptHeartbeat.confirmOwned();
|
|
17879
|
-
}
|
|
17880
|
-
if (pendingRequests.length > 0) {
|
|
17881
|
-
const strategy = operationStrategy;
|
|
17882
|
-
|
|
17883
|
-
if (strategy) {
|
|
17884
|
-
const compiledBatches = compileRequestsWithStrategy({
|
|
17885
|
-
requests: pendingRequests,
|
|
17886
|
-
strategy,
|
|
17887
|
-
getPayload: (request: ToolCallRequest) => request.input,
|
|
17888
|
-
});
|
|
17889
|
-
|
|
17890
|
-
const batchParallelismCeiling =
|
|
17891
|
-
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
17892
|
-
const batchSize =
|
|
17893
|
-
compiledBatches.length > 0
|
|
17894
|
-
? await this.resourceGovernor.suggestedToolParallelism(
|
|
17895
|
-
compiledBatches[0]!.batchOperation,
|
|
17896
|
-
batchParallelismCeiling,
|
|
17897
|
-
)
|
|
17898
|
-
: batchParallelismCeiling;
|
|
17899
|
-
await executeChunkedRequests({
|
|
17900
|
-
requests: compiledBatches,
|
|
17901
|
-
batchSize,
|
|
17902
|
-
execute: async (batch) => {
|
|
17903
|
-
// Circuit breaker: skip dispatching this batch's provider call
|
|
17904
|
-
// once a persistence failure has occurred in this run.
|
|
17905
|
-
if (this.persistenceLatch.tripped) {
|
|
17906
|
-
this.persistenceLatch.preventedCallCount +=
|
|
17907
|
-
batch.memberRequests.length;
|
|
17908
|
-
throw new RuntimePersistenceCircuitOpenError(
|
|
17909
|
-
this.persistenceLatch,
|
|
16589
|
+
},
|
|
16590
|
+
onChunkComplete: async (chunkResults) => {
|
|
16591
|
+
for (const entry of chunkResults) {
|
|
16592
|
+
if (entry.error !== undefined) {
|
|
16593
|
+
const settlements = await Promise.allSettled(
|
|
16594
|
+
entry.request.memberRequests.map((request) =>
|
|
16595
|
+
rejectCall(request, entry.error),
|
|
16596
|
+
),
|
|
17910
16597
|
);
|
|
17911
|
-
|
|
17912
|
-
|
|
17913
|
-
|
|
17914
|
-
);
|
|
17915
|
-
const aggregateReceiptKey =
|
|
17916
|
-
buildDurableToolAggregateReceiptKey({
|
|
17917
|
-
receiptKeys,
|
|
17918
|
-
prefix: 'batch',
|
|
17919
|
-
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
17920
|
-
orgId: this.#options.orgId,
|
|
17921
|
-
toolId: batch.batchOperation,
|
|
17922
|
-
}),
|
|
17923
|
-
});
|
|
17924
|
-
const aggregateForceRefreshIdentities = batch.memberRequests
|
|
17925
|
-
.filter((request) => request.force === true)
|
|
17926
|
-
.map((request) =>
|
|
17927
|
-
this.providerIdempotencyKeyForToolCall({
|
|
17928
|
-
operationKey:
|
|
17929
|
-
this.providerOperationKeyForQueuedToolCall(request),
|
|
17930
|
-
force: true,
|
|
17931
|
-
logicalCallId: request.logicalCallId,
|
|
17932
|
-
}),
|
|
16598
|
+
const rejected = settlements.find(
|
|
16599
|
+
(settlement): settlement is PromiseRejectedResult =>
|
|
16600
|
+
settlement.status === 'rejected',
|
|
17933
16601
|
);
|
|
17934
|
-
|
|
17935
|
-
|
|
17936
|
-
|
|
17937
|
-
|
|
17938
|
-
executionAuthScopeDigest:
|
|
17939
|
-
batch.memberRequests[0]?.executionAuthScopeDigest ??
|
|
17940
|
-
null,
|
|
17941
|
-
providerActionVersion:
|
|
17942
|
-
(
|
|
17943
|
-
await this.#options.getToolActionCacheVersion?.(
|
|
17944
|
-
batch.batchOperation,
|
|
17945
|
-
)
|
|
17946
|
-
)?.trim() ?? 'unversioned',
|
|
17947
|
-
payload: batch.batchPayload,
|
|
17948
|
-
forceRefreshIdentities: aggregateForceRefreshIdentities,
|
|
17949
|
-
});
|
|
17950
|
-
let releaseToolSlot: () => void = () => undefined;
|
|
16602
|
+
if (rejected) throw rejected.reason;
|
|
16603
|
+
continue;
|
|
16604
|
+
}
|
|
16605
|
+
const batchExecution = entry.result;
|
|
17951
16606
|
try {
|
|
17952
|
-
const
|
|
17953
|
-
|
|
17954
|
-
|
|
17955
|
-
|
|
17956
|
-
|
|
17957
|
-
|
|
17958
|
-
|
|
17959
|
-
|
|
17960
|
-
|
|
17961
|
-
|
|
17962
|
-
|
|
17963
|
-
|
|
17964
|
-
|
|
17965
|
-
|
|
17966
|
-
|
|
17967
|
-
|
|
17968
|
-
|
|
17969
|
-
batch.memberRequests[0]?.executionAuthScopeDigest ??
|
|
17970
|
-
null,
|
|
17971
|
-
providerIdempotencyKey: aggregateProviderOperationKey,
|
|
17972
|
-
providerOperationForceRefreshIdentities:
|
|
17973
|
-
aggregateForceRefreshIdentities,
|
|
17974
|
-
receiptLeaseExpiresAt: batch.memberRequests.reduce<
|
|
17975
|
-
string | null
|
|
17976
|
-
>((earliest, request) => {
|
|
17977
|
-
const expiresAt =
|
|
17978
|
-
request.receiptLeaseExpiresAt ?? null;
|
|
17979
|
-
if (!expiresAt) return earliest;
|
|
17980
|
-
if (!earliest) return expiresAt;
|
|
17981
|
-
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
17982
|
-
? expiresAt
|
|
17983
|
-
: earliest;
|
|
17984
|
-
}, null),
|
|
17985
|
-
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
|
|
17986
|
-
batch.memberRequests,
|
|
16607
|
+
const splitResults =
|
|
16608
|
+
batchExecution != null
|
|
16609
|
+
? entry.request.splitResults(
|
|
16610
|
+
legacyResultForBatchSplitter(
|
|
16611
|
+
batchExecution.execution,
|
|
16612
|
+
),
|
|
16613
|
+
)
|
|
16614
|
+
: entry.request.memberRequests.map(() => null);
|
|
16615
|
+
const resolvedResults =
|
|
16616
|
+
await this.resolveToolCallBatchResults(
|
|
16617
|
+
toolId,
|
|
16618
|
+
entry.request.memberRequests.map((request, index) => ({
|
|
16619
|
+
request,
|
|
16620
|
+
result: splitResults[index] ?? null,
|
|
16621
|
+
status: batchExecution?.execution.status,
|
|
16622
|
+
meta: nativeBatchResponseMeta(
|
|
16623
|
+
splitResults[index] ?? null,
|
|
17987
16624
|
),
|
|
17988
|
-
|
|
17989
|
-
|
|
17990
|
-
?
|
|
17991
|
-
:
|
|
17992
|
-
|
|
16625
|
+
toolResponse:
|
|
16626
|
+
batchExecution == null
|
|
16627
|
+
? undefined
|
|
16628
|
+
: publicToolResponseForBatchedItem(
|
|
16629
|
+
batchExecution.execution,
|
|
16630
|
+
splitResults[index] ?? null,
|
|
16631
|
+
true,
|
|
17993
16632
|
),
|
|
17994
|
-
|
|
17995
|
-
this.renewRuntimeToolReceiptOwnership(
|
|
17996
|
-
batch.memberRequests,
|
|
17997
|
-
),
|
|
17998
|
-
retainToolSlot: (release) => {
|
|
17999
|
-
releaseToolSlot = release;
|
|
18000
|
-
},
|
|
18001
|
-
},
|
|
18002
|
-
});
|
|
18003
|
-
return { execution, releaseToolSlot };
|
|
18004
|
-
} catch (error) {
|
|
18005
|
-
releaseToolSlot();
|
|
18006
|
-
throw error;
|
|
18007
|
-
}
|
|
18008
|
-
},
|
|
18009
|
-
onChunkComplete: async (chunkResults) => {
|
|
18010
|
-
for (const entry of chunkResults) {
|
|
18011
|
-
if (entry.error !== undefined) {
|
|
18012
|
-
const settlements = await Promise.allSettled(
|
|
18013
|
-
entry.request.memberRequests.map((request) =>
|
|
18014
|
-
rejectWithLiveFollowers(request, entry.error),
|
|
18015
|
-
),
|
|
18016
|
-
);
|
|
18017
|
-
const rejected = settlements.find(
|
|
18018
|
-
(settlement): settlement is PromiseRejectedResult =>
|
|
18019
|
-
settlement.status === 'rejected',
|
|
16633
|
+
})),
|
|
18020
16634
|
);
|
|
18021
|
-
if (rejected) throw rejected.reason;
|
|
18022
|
-
continue;
|
|
18023
|
-
}
|
|
18024
|
-
const batchExecution = entry.result;
|
|
18025
|
-
try {
|
|
18026
|
-
const splitResults =
|
|
18027
|
-
batchExecution != null
|
|
18028
|
-
? entry.request.splitResults(
|
|
18029
|
-
legacyResultForBatchSplitter(
|
|
18030
|
-
batchExecution.execution,
|
|
18031
|
-
),
|
|
18032
|
-
)
|
|
18033
|
-
: entry.request.memberRequests.map(() => null);
|
|
18034
|
-
const resolvedResults =
|
|
18035
|
-
await this.resolveToolCallBatchResults(
|
|
18036
|
-
toolId,
|
|
18037
|
-
entry.request.memberRequests.map(
|
|
18038
|
-
(request, index) => ({
|
|
18039
|
-
request,
|
|
18040
|
-
result: splitResults[index] ?? null,
|
|
18041
|
-
status: batchExecution?.execution.status,
|
|
18042
|
-
toolResponse:
|
|
18043
|
-
batchExecution == null
|
|
18044
|
-
? undefined
|
|
18045
|
-
: publicToolResponseForBatchedItem(
|
|
18046
|
-
batchExecution.execution,
|
|
18047
|
-
splitResults[index] ?? null,
|
|
18048
|
-
true,
|
|
18049
|
-
),
|
|
18050
|
-
}),
|
|
18051
|
-
),
|
|
18052
|
-
);
|
|
18053
16635
|
|
|
18054
|
-
|
|
18055
|
-
|
|
18056
|
-
|
|
18057
|
-
|
|
18058
|
-
|
|
18059
|
-
|
|
18060
|
-
|
|
18061
|
-
|
|
18062
|
-
}
|
|
18063
|
-
resolveLiveFollowers(request, resolvedResults[index]);
|
|
16636
|
+
for (
|
|
16637
|
+
let index = 0;
|
|
16638
|
+
index < entry.request.memberRequests.length;
|
|
16639
|
+
index += 1
|
|
16640
|
+
) {
|
|
16641
|
+
const request = entry.request.memberRequests[index]!;
|
|
16642
|
+
if (resolvedResults[index] != null) {
|
|
16643
|
+
successfulLiveStepCallIds.add(request.callId);
|
|
18064
16644
|
}
|
|
18065
|
-
} finally {
|
|
18066
|
-
batchExecution?.releaseToolSlot();
|
|
18067
16645
|
}
|
|
16646
|
+
} finally {
|
|
16647
|
+
batchExecution?.releaseToolSlot();
|
|
18068
16648
|
}
|
|
16649
|
+
}
|
|
18069
16650
|
|
|
18070
|
-
|
|
18071
|
-
|
|
18072
|
-
(entry) => entry.request.memberRequests,
|
|
18073
|
-
),
|
|
18074
|
-
);
|
|
18075
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
18076
|
-
},
|
|
18077
|
-
retainResults: false,
|
|
18078
|
-
});
|
|
18079
|
-
} else {
|
|
18080
|
-
// Bound direct map dispatch separately from ordinary provider
|
|
18081
|
-
// residency. This prevents a blocked first chunk from launching
|
|
18082
|
-
// another chunk ahead of a provider's admitted start rate, while
|
|
18083
|
-
// preserving the wider resident-call capacity used by batched and
|
|
18084
|
-
// high-latency providers.
|
|
18085
|
-
const toolCallConcurrencyCeiling =
|
|
18086
|
-
this.governor.policy.concurrency.toolCalls;
|
|
18087
|
-
const shapedToolParallelism =
|
|
18088
|
-
await this.resourceGovernor.suggestedDirectToolParallelism(
|
|
18089
|
-
toolId,
|
|
18090
|
-
toolCallConcurrencyCeiling,
|
|
16651
|
+
recordToolStep(
|
|
16652
|
+
chunkResults.flatMap((entry) => entry.request.memberRequests),
|
|
18091
16653
|
);
|
|
18092
|
-
|
|
16654
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
16655
|
+
},
|
|
16656
|
+
retainResults: false,
|
|
16657
|
+
});
|
|
16658
|
+
} else {
|
|
16659
|
+
// Bound direct map dispatch separately from ordinary provider
|
|
16660
|
+
// residency. This prevents a blocked first chunk from launching
|
|
16661
|
+
// another chunk ahead of a provider's admitted start rate, while
|
|
16662
|
+
// preserving the wider resident-call capacity used by batched and
|
|
16663
|
+
// high-latency providers.
|
|
16664
|
+
const toolCallConcurrencyCeiling =
|
|
16665
|
+
this.governor.policy.concurrency.toolCalls;
|
|
16666
|
+
const shapedToolParallelism =
|
|
16667
|
+
await this.resourceGovernor.suggestedDirectToolParallelism(
|
|
16668
|
+
toolId,
|
|
18093
16669
|
toolCallConcurrencyCeiling,
|
|
18094
|
-
Math.max(1, shapedToolParallelism),
|
|
18095
16670
|
);
|
|
18096
|
-
|
|
18097
|
-
|
|
18098
|
-
|
|
18099
|
-
|
|
18100
|
-
|
|
18101
|
-
|
|
18102
|
-
|
|
18103
|
-
|
|
18104
|
-
|
|
18105
|
-
|
|
18106
|
-
|
|
18107
|
-
|
|
18108
|
-
|
|
18109
|
-
|
|
18110
|
-
|
|
18111
|
-
|
|
18112
|
-
|
|
18113
|
-
|
|
18114
|
-
|
|
18115
|
-
|
|
18116
|
-
|
|
18117
|
-
|
|
18118
|
-
|
|
18119
|
-
|
|
18120
|
-
|
|
18121
|
-
|
|
18122
|
-
|
|
18123
|
-
|
|
18124
|
-
|
|
18125
|
-
|
|
18126
|
-
|
|
18127
|
-
|
|
18128
|
-
|
|
18129
|
-
|
|
18130
|
-
|
|
18131
|
-
|
|
18132
|
-
|
|
18133
|
-
|
|
18134
|
-
|
|
18135
|
-
|
|
18136
|
-
|
|
18137
|
-
|
|
18138
|
-
|
|
18139
|
-
|
|
18140
|
-
|
|
18141
|
-
|
|
18142
|
-
|
|
18143
|
-
|
|
18144
|
-
|
|
18145
|
-
|
|
18146
|
-
|
|
18147
|
-
|
|
18148
|
-
|
|
18149
|
-
|
|
18150
|
-
),
|
|
18151
|
-
force: request.force,
|
|
18152
|
-
requiresExecutionFence:
|
|
18153
|
-
request.requiresExecutionFence,
|
|
18154
|
-
logicalCallId: request.logicalCallId,
|
|
16671
|
+
const dispatchWidth = Math.min(
|
|
16672
|
+
toolCallConcurrencyCeiling,
|
|
16673
|
+
Math.max(1, shapedToolParallelism),
|
|
16674
|
+
);
|
|
16675
|
+
const dispatchResults = await dispatchBoundedSettled(
|
|
16676
|
+
pendingRequests,
|
|
16677
|
+
dispatchWidth,
|
|
16678
|
+
async (request) => {
|
|
16679
|
+
// Circuit breaker: do not dispatch this provider call once a
|
|
16680
|
+
// persistence failure has occurred in this run.
|
|
16681
|
+
if (this.persistenceLatch.tripped) {
|
|
16682
|
+
this.persistenceLatch.preventedCallCount += 1;
|
|
16683
|
+
throw new RuntimePersistenceCircuitOpenError(
|
|
16684
|
+
this.persistenceLatch,
|
|
16685
|
+
);
|
|
16686
|
+
}
|
|
16687
|
+
const providerOperationKey =
|
|
16688
|
+
this.providerInvocationKeyForToolCall({
|
|
16689
|
+
operationKey:
|
|
16690
|
+
this.providerOperationKeyForQueuedToolCall(request),
|
|
16691
|
+
logicalCallId: request.logicalCallId,
|
|
16692
|
+
force: request.force,
|
|
16693
|
+
});
|
|
16694
|
+
let releaseToolSlot: () => void = () => undefined;
|
|
16695
|
+
try {
|
|
16696
|
+
const execution =
|
|
16697
|
+
await this.executeMappedToolCallThroughModule({
|
|
16698
|
+
request,
|
|
16699
|
+
dispatchToolId: toolId,
|
|
16700
|
+
dispatchPayload: request.input,
|
|
16701
|
+
// Always-fresh map calls intentionally have no
|
|
16702
|
+
// result cache. Their provider-operation identity is
|
|
16703
|
+
// a unique forced semantic request, never a cache key.
|
|
16704
|
+
resultReceiptKey: request.receiptKey ?? request.cacheKey,
|
|
16705
|
+
providerOperationKey,
|
|
16706
|
+
apiOptions: {
|
|
16707
|
+
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
16708
|
+
executionAuthScopeDigest:
|
|
16709
|
+
request.executionAuthScopeDigest,
|
|
16710
|
+
providerIdempotencyKey: providerOperationKey,
|
|
16711
|
+
...this.providerInvocationOptionsForToolCall({
|
|
16712
|
+
operationKey:
|
|
16713
|
+
this.providerOperationKeyForQueuedToolCall(request),
|
|
16714
|
+
requiresExecutionFence:
|
|
16715
|
+
request.requiresExecutionFence,
|
|
16716
|
+
logicalCallId: request.logicalCallId,
|
|
16717
|
+
force: request.force,
|
|
16718
|
+
}),
|
|
16719
|
+
...(request.receiptKey
|
|
16720
|
+
? {
|
|
16721
|
+
durableCallReceiptKey:
|
|
16722
|
+
this.toolTransportReceiptKey({
|
|
16723
|
+
toolId,
|
|
16724
|
+
providerOperationKey,
|
|
18155
16725
|
}),
|
|
18156
|
-
|
|
18157
|
-
|
|
18158
|
-
|
|
18159
|
-
|
|
18160
|
-
|
|
18161
|
-
|
|
18162
|
-
}
|
|
18163
|
-
: {
|
|
18164
|
-
providerIdempotencyKey:
|
|
18165
|
-
this.providerIdempotencyKeyForToolCall({
|
|
18166
|
-
operationKey:
|
|
18167
|
-
this.providerOperationKeyForQueuedToolCall(
|
|
16726
|
+
legacyProviderRetryIdentities: [
|
|
16727
|
+
{
|
|
16728
|
+
transportReceiptKey: request.receiptKey,
|
|
16729
|
+
providerOperationKey: request.force
|
|
16730
|
+
? providerOperationKey
|
|
16731
|
+
: this.providerOperationKeyForQueuedToolCall(
|
|
18168
16732
|
request,
|
|
18169
16733
|
),
|
|
18170
|
-
|
|
18171
|
-
|
|
18172
|
-
|
|
18173
|
-
|
|
18174
|
-
|
|
18175
|
-
|
|
18176
|
-
|
|
18177
|
-
|
|
18178
|
-
|
|
18179
|
-
|
|
16734
|
+
...(request.force &&
|
|
16735
|
+
request.requiresExecutionFence !== true
|
|
16736
|
+
? {
|
|
16737
|
+
forceRefreshIdentities: [
|
|
16738
|
+
providerOperationKey,
|
|
16739
|
+
],
|
|
16740
|
+
}
|
|
16741
|
+
: {}),
|
|
16742
|
+
},
|
|
16743
|
+
],
|
|
16744
|
+
completedResultCache: true,
|
|
16745
|
+
}
|
|
16746
|
+
: {}),
|
|
16747
|
+
timeoutMs: resolveRuntimeTimeoutMsForRequests([
|
|
16748
|
+
request,
|
|
16749
|
+
]),
|
|
16750
|
+
retainToolSlot: (release) => {
|
|
16751
|
+
releaseToolSlot = release;
|
|
18180
16752
|
},
|
|
18181
|
-
}
|
|
18182
|
-
|
|
18183
|
-
|
|
18184
|
-
|
|
18185
|
-
|
|
18186
|
-
|
|
18187
|
-
|
|
18188
|
-
|
|
18189
|
-
|
|
18190
|
-
resolveLiveFollowers(request, result);
|
|
18191
|
-
} finally {
|
|
18192
|
-
releaseToolSlot();
|
|
16753
|
+
},
|
|
16754
|
+
});
|
|
16755
|
+
const result = await this.resolveToolCall(
|
|
16756
|
+
toolId,
|
|
16757
|
+
request,
|
|
16758
|
+
execution,
|
|
16759
|
+
);
|
|
16760
|
+
if (result != null) {
|
|
16761
|
+
successfulLiveStepCallIds.add(request.callId);
|
|
18193
16762
|
}
|
|
18194
|
-
}
|
|
18195
|
-
|
|
18196
|
-
|
|
18197
|
-
|
|
18198
|
-
|
|
18199
|
-
|
|
18200
|
-
|
|
18201
|
-
|
|
18202
|
-
|
|
18203
|
-
)
|
|
18204
|
-
|
|
18205
|
-
|
|
18206
|
-
|
|
18207
|
-
|
|
18208
|
-
|
|
18209
|
-
|
|
18210
|
-
|
|
18211
|
-
|
|
18212
|
-
}
|
|
18213
|
-
} catch (reason) {
|
|
18214
|
-
providerFailure = { reason };
|
|
18215
|
-
durableExistingRunningAbort.abort(reason);
|
|
18216
|
-
} finally {
|
|
18217
|
-
pendingReceiptHeartbeat?.stop();
|
|
18218
|
-
}
|
|
18219
|
-
const pendingReceiptLeaseLost = pendingReceiptHeartbeat?.leaseLost();
|
|
18220
|
-
if (pendingReceiptLeaseLost && providerFailure === null) {
|
|
18221
|
-
providerFailure = { reason: pendingReceiptLeaseLost };
|
|
18222
|
-
durableExistingRunningAbort.abort(pendingReceiptLeaseLost);
|
|
18223
|
-
}
|
|
18224
|
-
const durableExistingRunningSettlement =
|
|
18225
|
-
durableExistingRunningState.settlement;
|
|
18226
|
-
if (durableExistingRunningSettlement) {
|
|
18227
|
-
const settlement = await durableExistingRunningSettlement;
|
|
18228
|
-
if (settlement.status === 'rejected' && providerFailure === null) {
|
|
18229
|
-
providerFailure = { reason: settlement.reason };
|
|
16763
|
+
} finally {
|
|
16764
|
+
releaseToolSlot();
|
|
16765
|
+
}
|
|
16766
|
+
},
|
|
16767
|
+
);
|
|
16768
|
+
const failedEntries = dispatchResults.filter(
|
|
16769
|
+
(entry) => entry.error !== undefined,
|
|
16770
|
+
);
|
|
16771
|
+
const failedSettlements = await Promise.allSettled(
|
|
16772
|
+
failedEntries.map((entry) => rejectCall(entry.job, entry.error)),
|
|
16773
|
+
);
|
|
16774
|
+
const failedRejection = failedSettlements.find(
|
|
16775
|
+
(settlement): settlement is PromiseRejectedResult =>
|
|
16776
|
+
settlement.status === 'rejected',
|
|
16777
|
+
);
|
|
16778
|
+
if (failedRejection) throw failedRejection.reason;
|
|
16779
|
+
recordToolStep(pendingRequests);
|
|
16780
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
18230
16781
|
}
|
|
18231
16782
|
}
|
|
18232
|
-
if (providerFailure) throw providerFailure.reason;
|
|
18233
16783
|
}),
|
|
18234
16784
|
);
|
|
18235
16785
|
const rejected = toolSettlements.find(
|
|
@@ -18323,24 +16873,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18323
16873
|
)(ctx, input);
|
|
18324
16874
|
}
|
|
18325
16875
|
|
|
18326
|
-
private
|
|
16876
|
+
private providerInvocationKeyForToolCall(input: {
|
|
18327
16877
|
operationKey: string;
|
|
18328
|
-
force?: boolean;
|
|
18329
16878
|
logicalCallId?: string;
|
|
16879
|
+
force?: boolean;
|
|
18330
16880
|
}): string {
|
|
18331
|
-
|
|
18332
|
-
|
|
18333
|
-
if (!input.logicalCallId) {
|
|
18334
|
-
throw new Error(
|
|
18335
|
-
'Forced tool execution requires a stable logical call identity.',
|
|
18336
|
-
);
|
|
18337
|
-
}
|
|
18338
|
-
fallbackAttemptId = `${this.currentRunId}:${input.logicalCallId}`;
|
|
16881
|
+
if (!input.logicalCallId) {
|
|
16882
|
+
throw new Error('Tool execution requires a stable invocation identity.');
|
|
18339
16883
|
}
|
|
18340
16884
|
return buildProviderOperationIdempotencyKey({
|
|
18341
16885
|
operationKey: input.operationKey,
|
|
18342
|
-
force:
|
|
18343
|
-
fallbackAttemptId
|
|
16886
|
+
force: true,
|
|
16887
|
+
fallbackAttemptId: `${this.currentRunId}:${input.logicalCallId}${input.force === true ? '' : ':cache-miss'}`,
|
|
18344
16888
|
});
|
|
18345
16889
|
}
|
|
18346
16890
|
|
|
@@ -18986,7 +17530,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18986
17530
|
const ambiguousTransportReplayAllowed =
|
|
18987
17531
|
readOnlyQuery || retrySafeTransientHttp;
|
|
18988
17532
|
const invocationOrgId = this.#options.orgId?.trim() ?? '';
|
|
18989
|
-
|
|
17533
|
+
let durableCallReceiptKey = uncachedAsyncLifecycleCall
|
|
18990
17534
|
? null
|
|
18991
17535
|
: options?.durableCallReceiptKey?.trim() ||
|
|
18992
17536
|
(usesDurableInvocationFence && invocationOrgId
|
|
@@ -19004,9 +17548,57 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19004
17548
|
this.#options.durableBoundaries && durableCallReceiptKey
|
|
19005
17549
|
? stableDigest(`provider-retry:${durableCallReceiptKey}`)
|
|
19006
17550
|
: null;
|
|
19007
|
-
|
|
17551
|
+
let savedProviderRetryCursor = providerRetryCursorKey
|
|
19008
17552
|
? this.checkpoint.providerRetryCursors?.[providerRetryCursorKey]
|
|
19009
17553
|
: undefined;
|
|
17554
|
+
if (providerRetryCursorKey && !savedProviderRetryCursor) {
|
|
17555
|
+
// Native calls can move from the outgoing fallback into the Module.
|
|
17556
|
+
// Probe only exact owner-supplied checkpoint identities; never wait
|
|
17557
|
+
// on another invocation or derive a fence from customer strings.
|
|
17558
|
+
for (const legacy of options?.legacyProviderRetryIdentities ?? []) {
|
|
17559
|
+
const legacyCursorKey = stableDigest(
|
|
17560
|
+
`provider-retry:${legacy.transportReceiptKey}`,
|
|
17561
|
+
);
|
|
17562
|
+
const cursor =
|
|
17563
|
+
this.checkpoint.providerRetryCursors?.[legacyCursorKey];
|
|
17564
|
+
if (!cursor) continue;
|
|
17565
|
+
// Consume synchronously before any await so a sibling cannot
|
|
17566
|
+
// inherit the same outgoing invocation.
|
|
17567
|
+
delete this.checkpoint.providerRetryCursors![legacyCursorKey];
|
|
17568
|
+
savedProviderRetryCursor = {
|
|
17569
|
+
...cursor,
|
|
17570
|
+
legacyProviderOperationKey:
|
|
17571
|
+
cursor.legacyProviderOperationKey ??
|
|
17572
|
+
legacy.providerOperationKey,
|
|
17573
|
+
legacyTransportReceiptKey:
|
|
17574
|
+
cursor.legacyTransportReceiptKey ??
|
|
17575
|
+
legacy.transportReceiptKey,
|
|
17576
|
+
...(legacy.forceRefreshIdentities?.length
|
|
17577
|
+
? {
|
|
17578
|
+
legacyProviderOperationForceRefreshIdentities:
|
|
17579
|
+
cursor.legacyProviderOperationForceRefreshIdentities ??
|
|
17580
|
+
legacy.forceRefreshIdentities,
|
|
17581
|
+
}
|
|
17582
|
+
: {}),
|
|
17583
|
+
};
|
|
17584
|
+
this.checkpoint.providerRetryCursors![providerRetryCursorKey] =
|
|
17585
|
+
savedProviderRetryCursor;
|
|
17586
|
+
break;
|
|
17587
|
+
}
|
|
17588
|
+
}
|
|
17589
|
+
// The checkpoint index stays unique to this invocation. Its wire
|
|
17590
|
+
// identities must still name the prior gateway outcome so that a
|
|
17591
|
+
// qualified 429 retry can be attested after any further suspension.
|
|
17592
|
+
const legacyProviderOperationKey =
|
|
17593
|
+
savedProviderRetryCursor?.legacyProviderOperationKey;
|
|
17594
|
+
if (savedProviderRetryCursor?.legacyTransportReceiptKey) {
|
|
17595
|
+
durableCallReceiptKey =
|
|
17596
|
+
savedProviderRetryCursor.legacyTransportReceiptKey;
|
|
17597
|
+
}
|
|
17598
|
+
const providerOperationForceRefreshIdentities =
|
|
17599
|
+
legacyProviderOperationKey
|
|
17600
|
+
? savedProviderRetryCursor?.legacyProviderOperationForceRefreshIdentities
|
|
17601
|
+
: options?.providerOperationForceRefreshIdentities;
|
|
19010
17602
|
const executionRequestNonce = crypto.randomUUID();
|
|
19011
17603
|
// Transport ambiguity must retain the billing identity. An ordinary
|
|
19012
17604
|
// (unfenced) call advances it only after a definite provider-owned
|
|
@@ -19020,7 +17612,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19020
17612
|
const providerIdempotencyReceiptKey =
|
|
19021
17613
|
options?.providerIdempotencyReceiptKey?.trim() || null;
|
|
19022
17614
|
const providerExecutionFenceKey = requiresExecutionFence
|
|
19023
|
-
?
|
|
17615
|
+
? legacyProviderOperationKey ||
|
|
17616
|
+
options?.providerIdempotencyKey?.trim() ||
|
|
17617
|
+
null
|
|
19024
17618
|
: null;
|
|
19025
17619
|
if (requiresExecutionFence && !providerExecutionFenceKey) {
|
|
19026
17620
|
throw new Error(
|
|
@@ -19029,14 +17623,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19029
17623
|
}
|
|
19030
17624
|
const canonicalOperationActionVersion =
|
|
19031
17625
|
providerExecutionFenceKey ||
|
|
19032
|
-
|
|
17626
|
+
providerOperationForceRefreshIdentities?.length
|
|
19033
17627
|
? (
|
|
19034
17628
|
await this.#options.getToolActionCacheVersion?.(toolId)
|
|
19035
17629
|
)?.trim() || null
|
|
19036
17630
|
: null;
|
|
19037
17631
|
const canonicalOperation =
|
|
17632
|
+
!uncachedAsyncLifecycleCall &&
|
|
19038
17633
|
(providerExecutionFenceKey ||
|
|
19039
|
-
|
|
17634
|
+
providerOperationForceRefreshIdentities?.length) &&
|
|
19040
17635
|
canonicalOperationActionVersion
|
|
19041
17636
|
? {
|
|
19042
17637
|
version: 1,
|
|
@@ -19046,10 +17641,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19046
17641
|
output_intent: options?.completedResultCache
|
|
19047
17642
|
? 'dataset'
|
|
19048
17643
|
: 'value',
|
|
19049
|
-
...(
|
|
17644
|
+
...(providerOperationForceRefreshIdentities?.length
|
|
19050
17645
|
? {
|
|
19051
17646
|
force_refresh_identities:
|
|
19052
|
-
|
|
17647
|
+
providerOperationForceRefreshIdentities,
|
|
19053
17648
|
}
|
|
19054
17649
|
: {}),
|
|
19055
17650
|
}
|
|
@@ -19183,9 +17778,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19183
17778
|
};
|
|
19184
17779
|
const persistProviderRetryCursor = (retryFrom: number) => {
|
|
19185
17780
|
if (!providerRetryCursorKey) return;
|
|
19186
|
-
const cursor = {
|
|
17781
|
+
const cursor: ProviderRetryCursor = {
|
|
17782
|
+
...savedProviderRetryCursor,
|
|
19187
17783
|
invocationAttempt: retryFrom + 1,
|
|
19188
17784
|
retryFromInvocationAttempt: retryFrom,
|
|
17785
|
+
...(legacyProviderOperationKey
|
|
17786
|
+
? {
|
|
17787
|
+
legacyProviderOperationKey,
|
|
17788
|
+
legacyTransportReceiptKey: durableCallReceiptKey!,
|
|
17789
|
+
}
|
|
17790
|
+
: {}),
|
|
19189
17791
|
};
|
|
19190
17792
|
this.checkpoint.providerRetryCursors ??= {};
|
|
19191
17793
|
this.checkpoint.providerRetryCursors[providerRetryCursorKey] =
|