deepline 0.3.283 → 0.3.285
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/activity-observation.ts +26 -2
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +817 -2236
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +19 -6
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +127 -789
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +180 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +3 -22
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts +26 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +50 -8
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +122 -18
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +218 -41
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +20 -2
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +13 -2
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +181 -326
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +38 -151
- package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +8 -0
- package/dist/bundling-sources/shared_libs/security/secret-redaction.ts +89 -1
- package/dist/cli/index.js +28 -3
- package/dist/cli/index.mjs +28 -3
- package/dist/{client-B0AjIkgb.d.mts → client-Cl-zj2oS.d.mts} +4 -0
- package/dist/{client-BCXQ_Ftc.d.ts → client-T2jVnZrq.d.ts} +4 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +25 -1
- package/dist/index.mjs +25 -1
- package/dist/install-integrity.json +2 -2
- package/dist/plays/run-request.d.mts +1 -1
- package/dist/plays/run-request.d.ts +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -187,7 +187,6 @@ import {
|
|
|
187
187
|
createRuntimeStepToolResultReceipts,
|
|
188
188
|
createToolCallJob,
|
|
189
189
|
resolveToolCallResultReceiptPolicy,
|
|
190
|
-
ToolCallReceiptLeaseLostError,
|
|
191
190
|
type ToolCallReceiptCohortStore,
|
|
192
191
|
type ToolCallReceiptCohortTimings,
|
|
193
192
|
type ToolCallReceiptRecord,
|
|
@@ -262,26 +261,16 @@ import {
|
|
|
262
261
|
} from './log-provenance';
|
|
263
262
|
import {
|
|
264
263
|
COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
|
|
265
|
-
DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS,
|
|
266
264
|
createRuntimeReceiptHydrationGate,
|
|
267
265
|
runtimeStepReceiptPayloadBytes,
|
|
268
|
-
runtimeReceiptClaimLeaseIsLive,
|
|
269
|
-
runtimeReceiptWaitBudgetMs,
|
|
270
|
-
runtimeReceiptWaitDelayMs,
|
|
271
|
-
DURABLE_RECEIPT_WAIT_DELAY_MS,
|
|
272
|
-
RuntimeReceiptHydrationTurnTimeoutError,
|
|
273
266
|
RuntimeReceiptLeaseLostError,
|
|
274
|
-
RuntimeReceiptWaitTimeoutError,
|
|
275
267
|
executeWithDurableRuntimeReceipt,
|
|
276
268
|
resolveRuntimeToolReceiptWaitMaxAttempts,
|
|
277
|
-
runtimeReceiptFailureError,
|
|
278
269
|
runtimeReceiptFailureKindForError,
|
|
279
270
|
runtimeReceiptOutput as durableRuntimeReceiptOutput,
|
|
280
271
|
type RuntimeReceiptHydrationTurn,
|
|
281
272
|
type RuntimeReceiptHydrationGateSnapshot,
|
|
282
273
|
type RuntimeReceiptHydrationSource,
|
|
283
|
-
waitForCompletedRuntimeReceipt,
|
|
284
|
-
waitForCompletedRuntimeReceipts,
|
|
285
274
|
type DurableReceiptExecutionStore,
|
|
286
275
|
} from './durable-receipt-execution';
|
|
287
276
|
import {
|
|
@@ -422,6 +411,7 @@ import type {
|
|
|
422
411
|
ContextOptions,
|
|
423
412
|
PlayCallOptions,
|
|
424
413
|
PlayCheckpoint,
|
|
414
|
+
ProviderRetryCursor,
|
|
425
415
|
PlayStep,
|
|
426
416
|
PlayStepRowResult,
|
|
427
417
|
PlayRowUpdate,
|
|
@@ -609,12 +599,6 @@ const DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN =
|
|
|
609
599
|
/\bdeeplinedeveloper\.com\s*\|\s*502\s*:\s*bad gateway\b/i;
|
|
610
600
|
const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100;
|
|
611
601
|
const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25;
|
|
612
|
-
// A newly claimed receipt should normally use its scheduled heartbeat. When a
|
|
613
|
-
// test/SEV policy deliberately gives it only a moment of lease life, however,
|
|
614
|
-
// scheduler setup and the first provider-dispatch turn can consume that entire
|
|
615
|
-
// window before the timer fires. Renew once before fan-out in that narrow case.
|
|
616
|
-
// Production's normal multi-minute lease remains on the zero-extra-query path.
|
|
617
|
-
const IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS = 100;
|
|
618
602
|
// Per-row sanity cap on distinct inline child invocations. `maxPlayCallDepth`
|
|
619
603
|
// (governor) already bounds nesting; this bounds fan-WIDTH from one active row
|
|
620
604
|
// resolver. The dedupe set is row-local, never retained for the full run.
|
|
@@ -1826,7 +1810,7 @@ type ToolExecutionApiOptions = {
|
|
|
1826
1810
|
completedResultCache?: boolean;
|
|
1827
1811
|
executionAuthScopeDigest?: string | null;
|
|
1828
1812
|
providerIdempotencyKey?: string | null;
|
|
1829
|
-
/**
|
|
1813
|
+
/** Invocation identities that participate in the canonical operation key. */
|
|
1830
1814
|
providerOperationForceRefreshIdentities?: readonly string[];
|
|
1831
1815
|
/** Presentation-only cache identity paired with a separate execution fence. */
|
|
1832
1816
|
providerIdempotencyReceiptKey?: string | null;
|
|
@@ -1878,6 +1862,12 @@ type ToolExecutionApiOptions = {
|
|
|
1878
1862
|
* cache stores only its terminal materialized output.
|
|
1879
1863
|
*/
|
|
1880
1864
|
uncachedAsyncLifecycleCall?: boolean;
|
|
1865
|
+
/** Exact outgoing delivery candidates; consume a checkpoint cursor once. */
|
|
1866
|
+
legacyProviderRetryIdentities?: readonly {
|
|
1867
|
+
transportReceiptKey: string;
|
|
1868
|
+
providerOperationKey: string;
|
|
1869
|
+
forceRefreshIdentities?: readonly string[];
|
|
1870
|
+
}[];
|
|
1881
1871
|
/**
|
|
1882
1872
|
* Called once this call's provider response body has been fully read and
|
|
1883
1873
|
* parsed, with that instant on the runner's clock. Measurement only: the
|
|
@@ -2594,12 +2584,6 @@ type ScalarPlayAuthoringRuntimeContext = Pick<
|
|
|
2594
2584
|
export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
2595
2585
|
private rowStates = new Map<number, RowState>();
|
|
2596
2586
|
private toolCallQueue: ToolCallRequest[] = [];
|
|
2597
|
-
private pendingRuntimeToolOwnershipAssertions: Array<{
|
|
2598
|
-
targets: Array<{ receiptKey: string; leaseId: string }>;
|
|
2599
|
-
resolve: () => void;
|
|
2600
|
-
reject: (error: unknown) => void;
|
|
2601
|
-
}> = [];
|
|
2602
|
-
private runtimeToolOwnershipAssertionFlushScheduled = false;
|
|
2603
2587
|
private runtimeToolReceiptCompletionsInFlight = new Map<string, number>();
|
|
2604
2588
|
/**
|
|
2605
2589
|
* The one app-side result-egress controller. It serializes receipt gateway
|
|
@@ -3225,23 +3209,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3225
3209
|
: Date.now();
|
|
3226
3210
|
this.checkpoint.durableCallCacheEpochMs = this.durableCallCacheEpochMs;
|
|
3227
3211
|
this.durableMappedToolResultsBackedByReceipts = Boolean(
|
|
3228
|
-
(options.claimRuntimeStepReceipt || options.claimRuntimeStepReceipts) &&
|
|
3229
3212
|
(options.getRuntimeStepReceipt || options.getRuntimeStepReceipts) &&
|
|
3230
3213
|
(options.completeRuntimeStepReceipt ||
|
|
3231
3214
|
options.completeRuntimeStepReceipts),
|
|
3232
3215
|
);
|
|
3233
|
-
this.durableDirectToolResultsBackedByReceipts =
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
);
|
|
3238
|
-
if (this.durableDirectToolResultsBackedByReceipts) {
|
|
3239
|
-
// A resumed durable runner may receive a legacy checkpoint containing
|
|
3240
|
-
// full tool results. Drop that redundant copy immediately and recover
|
|
3241
|
-
// through the receipt store below. Bulk-only receipt contexts retain the
|
|
3242
|
-
// checkpoint because direct tool calls cannot use the bulk claim path.
|
|
3243
|
-
this.checkpoint.completedToolBatches = {};
|
|
3244
|
-
}
|
|
3216
|
+
this.durableDirectToolResultsBackedByReceipts =
|
|
3217
|
+
this.durableMappedToolResultsBackedByReceipts;
|
|
3218
|
+
// Existing checkpoints can be the sole completed cache from an outgoing
|
|
3219
|
+
// context. Receipt-backed calls stop adding payloads, but retain old hits.
|
|
3245
3220
|
// The governance play id keys durable ctx receipts (durableCtxKey), the
|
|
3246
3221
|
// cycle guard, and per-parent child-call counters. It must be the STABLE
|
|
3247
3222
|
// play name — not the per-run workflow id — so receipts written by one
|
|
@@ -4323,7 +4298,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4323
4298
|
key: string,
|
|
4324
4299
|
): Promise<RuntimeStepReceipt | null> {
|
|
4325
4300
|
if (!this.#options.getRuntimeStepReceipt) {
|
|
4326
|
-
return null;
|
|
4301
|
+
if (!this.#options.getRuntimeStepReceipts) return null;
|
|
4302
|
+
return (await this.getRuntimeStepReceipts([key])).get(key) ?? null;
|
|
4327
4303
|
}
|
|
4328
4304
|
const receipt = await this.#options.getRuntimeStepReceipt({ key });
|
|
4329
4305
|
if (receipt && typeof receipt.key === 'string' && receipt.key.trim()) {
|
|
@@ -4465,7 +4441,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4465
4441
|
path: `ctx.step receipt "${key}"`,
|
|
4466
4442
|
});
|
|
4467
4443
|
if (!this.#options.completeRuntimeStepReceipt) {
|
|
4468
|
-
return null;
|
|
4444
|
+
if (!this.#options.completeRuntimeStepReceipts) return null;
|
|
4445
|
+
const completed = await this.completeRuntimeStepReceipts(
|
|
4446
|
+
[
|
|
4447
|
+
{
|
|
4448
|
+
key,
|
|
4449
|
+
runId: this.currentReceiptOwnerRunId,
|
|
4450
|
+
runAttempt: this.currentRunAttempt,
|
|
4451
|
+
...(leaseId ? { leaseId } : {}),
|
|
4452
|
+
output,
|
|
4453
|
+
...(inlineFields ? { inlineFields } : {}),
|
|
4454
|
+
},
|
|
4455
|
+
],
|
|
4456
|
+
undefined,
|
|
4457
|
+
undefined,
|
|
4458
|
+
{ retainLiveResult: true },
|
|
4459
|
+
);
|
|
4460
|
+
return completed.get(key) ?? null;
|
|
4469
4461
|
}
|
|
4470
4462
|
const completed = await this.#options.completeRuntimeStepReceipt({
|
|
4471
4463
|
key,
|
|
@@ -4527,49 +4519,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4527
4519
|
})),
|
|
4528
4520
|
);
|
|
4529
4521
|
if (manifests.length === 0) return;
|
|
4530
|
-
//
|
|
4531
|
-
//
|
|
4532
|
-
// egress wave. Nothing is gained by skipping the ownership handshake
|
|
4533
|
-
// here, and skipping it costs the caller a receipt row it does not own:
|
|
4534
|
-
// a runtime whose claim seam refuses (or that exposes no claim seam at
|
|
4535
|
-
// all) would still publish pointer rows, and an already published
|
|
4536
|
-
// pointer would be rewritten on every later call. Take the row the way
|
|
4537
|
-
// every other scalar receipt writer takes it, and write only what this
|
|
4538
|
-
// run actually owns. The map/cohort publisher keeps the claim-free
|
|
4539
|
-
// batched write, where the pointer genuinely shares the canonical
|
|
4540
|
-
// result's single bounded request.
|
|
4522
|
+
// The pointer is an immutable completed-cache insert, like the result
|
|
4523
|
+
// it names. Publishing it needs no pending lease or read-back.
|
|
4541
4524
|
if (
|
|
4542
4525
|
!(
|
|
4543
|
-
this.#options.
|
|
4544
|
-
this.#options.
|
|
4526
|
+
this.#options.completeRuntimeStepReceipt ||
|
|
4527
|
+
this.#options.completeRuntimeStepReceipts
|
|
4545
4528
|
)
|
|
4546
4529
|
) {
|
|
4547
4530
|
return;
|
|
4548
4531
|
}
|
|
4549
|
-
|
|
4550
|
-
manifests.map((manifest) => manifest.key),
|
|
4551
|
-
{ hydrateOutput: false },
|
|
4552
|
-
);
|
|
4553
|
-
const missing = manifests.filter((manifest) => {
|
|
4554
|
-
const receipt = existing.get(manifest.key);
|
|
4555
|
-
return receipt?.status !== 'completed' && receipt?.status !== 'skipped';
|
|
4556
|
-
});
|
|
4557
|
-
if (missing.length === 0) return;
|
|
4558
|
-
const claims = await this.claimRuntimeStepReceipts(
|
|
4559
|
-
missing.map((manifest) => manifest.key),
|
|
4560
|
-
sourceRunId,
|
|
4561
|
-
false,
|
|
4562
|
-
false,
|
|
4563
|
-
false,
|
|
4564
|
-
false,
|
|
4565
|
-
);
|
|
4566
|
-
const owned = missing.flatMap((manifest) => {
|
|
4567
|
-
const leaseId = claims.get(manifest.key)?.leaseId;
|
|
4568
|
-
return leaseId ? [{ ...manifest, leaseId }] : [];
|
|
4569
|
-
});
|
|
4570
|
-
if (owned.length === 0) return;
|
|
4571
|
-
await this.completeRuntimeStepReceipts(owned, undefined, undefined, {
|
|
4572
|
-
bestEffortKeys: new Set(owned.map((manifest) => manifest.key)),
|
|
4532
|
+
await this.completeRuntimeStepReceipts(manifests, undefined, undefined, {
|
|
4533
|
+
bestEffortKeys: new Set(manifests.map((manifest) => manifest.key)),
|
|
4573
4534
|
onBestEffortFailure: ({ key, error }) => {
|
|
4574
4535
|
this.runtimeLog(
|
|
4575
4536
|
`Tool source-run receipt manifest write skipped for ${key}: ${error instanceof Error ? error.message : String(error)}`,
|
|
@@ -4789,178 +4750,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4789
4750
|
};
|
|
4790
4751
|
}
|
|
4791
4752
|
|
|
4792
|
-
private async assertRuntimeToolReceiptOwnership(
|
|
4793
|
-
requests: ToolCallRequest[],
|
|
4794
|
-
): Promise<void> {
|
|
4795
|
-
const targets = this.runtimeToolReceiptOwnershipTargets(requests);
|
|
4796
|
-
if (targets.length === 0) return;
|
|
4797
|
-
|
|
4798
|
-
return await new Promise<void>((resolve, reject) => {
|
|
4799
|
-
this.pendingRuntimeToolOwnershipAssertions.push({
|
|
4800
|
-
targets,
|
|
4801
|
-
resolve,
|
|
4802
|
-
reject,
|
|
4803
|
-
});
|
|
4804
|
-
if (this.runtimeToolOwnershipAssertionFlushScheduled) return;
|
|
4805
|
-
this.runtimeToolOwnershipAssertionFlushScheduled = true;
|
|
4806
|
-
queueMicrotask(() => void this.flushRuntimeToolOwnershipAssertionBatch());
|
|
4807
|
-
});
|
|
4808
|
-
}
|
|
4809
|
-
|
|
4810
|
-
private runtimeToolReceiptOwnershipTargets(
|
|
4811
|
-
requests: ToolCallRequest[],
|
|
4812
|
-
): Array<{ receiptKey: string; leaseId: string }> {
|
|
4813
|
-
return requests.flatMap((request) => {
|
|
4814
|
-
const receiptKey = request.receiptKey?.trim() || null;
|
|
4815
|
-
if (!receiptKey) return [];
|
|
4816
|
-
const leaseId = request.receiptLeaseId?.trim() || null;
|
|
4817
|
-
if (!leaseId) return [];
|
|
4818
|
-
return [{ receiptKey, leaseId }];
|
|
4819
|
-
});
|
|
4820
|
-
}
|
|
4821
|
-
|
|
4822
|
-
private async renewRuntimeToolReceiptOwnership(
|
|
4823
|
-
requests: ToolCallRequest[],
|
|
4824
|
-
): Promise<void> {
|
|
4825
|
-
const targets = this.runtimeToolReceiptOwnershipTargets(requests);
|
|
4826
|
-
if (targets.length === 0) return;
|
|
4827
|
-
if (!this.#options.heartbeatRuntimeStepReceipts) {
|
|
4828
|
-
return await this.assertRuntimeToolReceiptOwnership(requests);
|
|
4829
|
-
}
|
|
4830
|
-
const uniqueTargets = [
|
|
4831
|
-
...new Map(targets.map((target) => [target.receiptKey, target])).values(),
|
|
4832
|
-
];
|
|
4833
|
-
const receipts = await this.#options.heartbeatRuntimeStepReceipts({
|
|
4834
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4835
|
-
runAttempt: this.currentRunAttempt,
|
|
4836
|
-
keys: uniqueTargets.map((target) => target.receiptKey),
|
|
4837
|
-
leaseIds: uniqueTargets.map((target) => target.leaseId),
|
|
4838
|
-
});
|
|
4839
|
-
this.assertRuntimeToolReceiptHeartbeatResults(uniqueTargets, receipts);
|
|
4840
|
-
}
|
|
4841
|
-
|
|
4842
|
-
private async flushRuntimeToolOwnershipAssertionBatch(): Promise<void> {
|
|
4843
|
-
this.runtimeToolOwnershipAssertionFlushScheduled = false;
|
|
4844
|
-
const assertions = this.pendingRuntimeToolOwnershipAssertions.splice(0);
|
|
4845
|
-
if (assertions.length === 0) return;
|
|
4846
|
-
const targets = assertions.flatMap((assertion) => assertion.targets);
|
|
4847
|
-
|
|
4848
|
-
if (this.#options.heartbeatRuntimeStepReceipts) {
|
|
4849
|
-
try {
|
|
4850
|
-
const uniqueTargets = [
|
|
4851
|
-
...new Map(
|
|
4852
|
-
targets.map((target) => [target.receiptKey, target]),
|
|
4853
|
-
).values(),
|
|
4854
|
-
];
|
|
4855
|
-
const receipts = await this.#options.heartbeatRuntimeStepReceipts({
|
|
4856
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4857
|
-
runAttempt: this.currentRunAttempt,
|
|
4858
|
-
keys: uniqueTargets.map((target) => target.receiptKey),
|
|
4859
|
-
leaseIds: uniqueTargets.map((target) => target.leaseId),
|
|
4860
|
-
});
|
|
4861
|
-
const renewed = new Map<string, RuntimeStepReceipt | null>();
|
|
4862
|
-
for (let index = 0; index < uniqueTargets.length; index += 1) {
|
|
4863
|
-
renewed.set(
|
|
4864
|
-
uniqueTargets[index]!.receiptKey,
|
|
4865
|
-
receipts[index] ?? null,
|
|
4866
|
-
);
|
|
4867
|
-
}
|
|
4868
|
-
for (const assertion of assertions) {
|
|
4869
|
-
const lost = assertion.targets.find((target) => {
|
|
4870
|
-
const receipt = renewed.get(target.receiptKey);
|
|
4871
|
-
return !this.runtimeToolReceiptStillOwned(receipt, target.leaseId);
|
|
4872
|
-
});
|
|
4873
|
-
if (lost) {
|
|
4874
|
-
assertion.reject(
|
|
4875
|
-
new RuntimeReceiptLeaseLostError({
|
|
4876
|
-
receiptKey: lost.receiptKey,
|
|
4877
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4878
|
-
leaseId: lost.leaseId,
|
|
4879
|
-
}),
|
|
4880
|
-
);
|
|
4881
|
-
} else {
|
|
4882
|
-
assertion.resolve();
|
|
4883
|
-
}
|
|
4884
|
-
}
|
|
4885
|
-
} catch (error) {
|
|
4886
|
-
for (const assertion of assertions) assertion.reject(error);
|
|
4887
|
-
}
|
|
4888
|
-
return;
|
|
4889
|
-
}
|
|
4890
|
-
|
|
4891
|
-
if (
|
|
4892
|
-
this.#options.getRuntimeStepReceipt ||
|
|
4893
|
-
this.#options.getRuntimeStepReceipts
|
|
4894
|
-
) {
|
|
4895
|
-
try {
|
|
4896
|
-
const latest = await this.getRuntimeStepReceipts(
|
|
4897
|
-
targets.map((target) => target.receiptKey),
|
|
4898
|
-
);
|
|
4899
|
-
for (const assertion of assertions) {
|
|
4900
|
-
const lost = assertion.targets.find((target) => {
|
|
4901
|
-
const receipt = latest.get(target.receiptKey);
|
|
4902
|
-
return !this.runtimeToolReceiptStillOwned(receipt, target.leaseId);
|
|
4903
|
-
});
|
|
4904
|
-
if (lost) {
|
|
4905
|
-
assertion.reject(
|
|
4906
|
-
new RuntimeReceiptLeaseLostError({
|
|
4907
|
-
receiptKey: lost.receiptKey,
|
|
4908
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4909
|
-
leaseId: lost.leaseId,
|
|
4910
|
-
}),
|
|
4911
|
-
);
|
|
4912
|
-
} else {
|
|
4913
|
-
assertion.resolve();
|
|
4914
|
-
}
|
|
4915
|
-
}
|
|
4916
|
-
} catch (error) {
|
|
4917
|
-
for (const assertion of assertions) assertion.reject(error);
|
|
4918
|
-
}
|
|
4919
|
-
return;
|
|
4920
|
-
}
|
|
4921
|
-
|
|
4922
|
-
const error = new RuntimeReceiptLeaseLostError({
|
|
4923
|
-
receiptKey: targets[0]?.receiptKey ?? 'unknown',
|
|
4924
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4925
|
-
leaseId: targets[0]?.leaseId ?? 'unknown',
|
|
4926
|
-
});
|
|
4927
|
-
for (const assertion of assertions) assertion.reject(error);
|
|
4928
|
-
}
|
|
4929
|
-
|
|
4930
|
-
private assertRuntimeToolReceiptHeartbeatResults(
|
|
4931
|
-
targets: Array<{ receiptKey: string; leaseId: string }>,
|
|
4932
|
-
receipts: Array<RuntimeStepReceipt | null>,
|
|
4933
|
-
): void {
|
|
4934
|
-
for (let index = 0; index < targets.length; index += 1) {
|
|
4935
|
-
const target = targets[index]!;
|
|
4936
|
-
const receipt = this.normalizeRuntimeStepReceipt(
|
|
4937
|
-
target.receiptKey,
|
|
4938
|
-
receipts[index],
|
|
4939
|
-
);
|
|
4940
|
-
if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) {
|
|
4941
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
4942
|
-
receiptKey: target.receiptKey,
|
|
4943
|
-
runId: this.currentReceiptOwnerRunId,
|
|
4944
|
-
leaseId: target.leaseId,
|
|
4945
|
-
});
|
|
4946
|
-
}
|
|
4947
|
-
}
|
|
4948
|
-
}
|
|
4949
|
-
|
|
4950
|
-
private runtimeToolReceiptStillOwned(
|
|
4951
|
-
receipt: RuntimeStepReceipt | null | undefined,
|
|
4952
|
-
leaseId: string,
|
|
4953
|
-
): boolean {
|
|
4954
|
-
if (!receipt) return false;
|
|
4955
|
-
if (receipt.status === 'completed' || receipt.status === 'skipped') {
|
|
4956
|
-
return true;
|
|
4957
|
-
}
|
|
4958
|
-
if (receipt.status !== 'running') return false;
|
|
4959
|
-
if (receipt.leaseId !== leaseId) return false;
|
|
4960
|
-
const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
|
|
4961
|
-
return !ownerRunId || ownerRunId === this.currentReceiptOwnerRunId;
|
|
4962
|
-
}
|
|
4963
|
-
|
|
4964
4753
|
private runtimeToolReceiptStillOwnedByCurrentAttempt(
|
|
4965
4754
|
receipt: RuntimeStepReceipt | null | undefined,
|
|
4966
4755
|
leaseId: string | null | undefined,
|
|
@@ -4980,18 +4769,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4980
4769
|
return ownerAttempt === this.currentRunAttempt;
|
|
4981
4770
|
}
|
|
4982
4771
|
|
|
4983
|
-
private isOwnedClaimedRuntimeReceipt(
|
|
4984
|
-
receipt: RuntimeStepReceipt | null | undefined,
|
|
4985
|
-
): receipt is RuntimeStepReceipt {
|
|
4986
|
-
return (
|
|
4987
|
-
Boolean(receipt) &&
|
|
4988
|
-
receipt?.claimState !== 'existing' &&
|
|
4989
|
-
(receipt?.status === 'queued' ||
|
|
4990
|
-
receipt?.status === 'pending' ||
|
|
4991
|
-
receipt?.status === 'running')
|
|
4992
|
-
);
|
|
4993
|
-
}
|
|
4994
|
-
|
|
4995
4772
|
private async failRuntimeStepReceipt(
|
|
4996
4773
|
key: string,
|
|
4997
4774
|
_runId: string,
|
|
@@ -5359,91 +5136,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
5359
5136
|
return this.normalizeRuntimeStepReceipt(input.key, receipt);
|
|
5360
5137
|
}
|
|
5361
5138
|
|
|
5362
|
-
private async markRuntimeStepReceiptsRunning(
|
|
5363
|
-
receipts: Array<{
|
|
5364
|
-
key: string;
|
|
5365
|
-
runId: string;
|
|
5366
|
-
leaseId?: string | null;
|
|
5367
|
-
}>,
|
|
5368
|
-
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
5369
|
-
const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
|
|
5370
|
-
if (normalizedInputs.length === 0) return new Map();
|
|
5371
|
-
let marked: Array<RuntimeStepReceipt | null>;
|
|
5372
|
-
const markReceipts = this.#options.markRuntimeStepReceiptsRunning;
|
|
5373
|
-
if (markReceipts) {
|
|
5374
|
-
marked = await this.dispatchChunkedRuntimeReceiptRequest(
|
|
5375
|
-
normalizedInputs.map((receipt) => ({
|
|
5376
|
-
...receipt,
|
|
5377
|
-
runId: this.currentReceiptOwnerRunId,
|
|
5378
|
-
runAttempt: this.currentRunAttempt,
|
|
5379
|
-
})),
|
|
5380
|
-
(chunk) => markReceipts({ receipts: chunk }),
|
|
5381
|
-
);
|
|
5382
|
-
} else {
|
|
5383
|
-
marked = await Promise.all(
|
|
5384
|
-
normalizedInputs.map((receipt) =>
|
|
5385
|
-
this.markRuntimeStepReceiptRunning(receipt),
|
|
5386
|
-
),
|
|
5387
|
-
);
|
|
5388
|
-
}
|
|
5389
|
-
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
5390
|
-
for (let index = 0; index < marked.length; index += 1) {
|
|
5391
|
-
const normalized = this.normalizeRuntimeStepReceipt(
|
|
5392
|
-
normalizedInputs[index]?.key ?? '',
|
|
5393
|
-
marked[index],
|
|
5394
|
-
);
|
|
5395
|
-
if (normalized) byKey.set(normalized.key, normalized);
|
|
5396
|
-
}
|
|
5397
|
-
return byKey;
|
|
5398
|
-
}
|
|
5399
|
-
|
|
5400
|
-
private async markRuntimeStepReceiptsQueued(
|
|
5401
|
-
receipts: Array<{
|
|
5402
|
-
key: string;
|
|
5403
|
-
runId: string;
|
|
5404
|
-
leaseId?: string | null;
|
|
5405
|
-
}>,
|
|
5406
|
-
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
5407
|
-
const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
|
|
5408
|
-
if (normalizedInputs.length === 0) return new Map();
|
|
5409
|
-
if (!this.#options.markRuntimeStepReceiptsQueued) {
|
|
5410
|
-
return new Map();
|
|
5411
|
-
}
|
|
5412
|
-
const queued = await this.dispatchChunkedRuntimeReceiptRequest(
|
|
5413
|
-
normalizedInputs.map((receipt) => ({
|
|
5414
|
-
...receipt,
|
|
5415
|
-
runId: this.currentReceiptOwnerRunId,
|
|
5416
|
-
runAttempt: this.currentRunAttempt,
|
|
5417
|
-
})),
|
|
5418
|
-
(chunk) =>
|
|
5419
|
-
this.#options.markRuntimeStepReceiptsQueued!({ receipts: chunk }),
|
|
5420
|
-
);
|
|
5421
|
-
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
5422
|
-
for (let index = 0; index < queued.length; index += 1) {
|
|
5423
|
-
const normalized = this.normalizeRuntimeStepReceipt(
|
|
5424
|
-
normalizedInputs[index]?.key ?? '',
|
|
5425
|
-
queued[index],
|
|
5426
|
-
);
|
|
5427
|
-
if (normalized) byKey.set(normalized.key, normalized);
|
|
5428
|
-
}
|
|
5429
|
-
return byKey;
|
|
5430
|
-
}
|
|
5431
|
-
|
|
5432
|
-
private async parkRuntimeToolReceiptsQueued(
|
|
5433
|
-
requests: ToolCallRequest[],
|
|
5434
|
-
): Promise<void> {
|
|
5435
|
-
const targets = requests.flatMap((request) => {
|
|
5436
|
-
const receiptKey = request.receiptKey?.trim() || null;
|
|
5437
|
-
const leaseId = request.receiptLeaseId?.trim() || null;
|
|
5438
|
-
if (!receiptKey || !leaseId) return [];
|
|
5439
|
-
return [
|
|
5440
|
-
{ key: receiptKey, runId: this.currentReceiptOwnerRunId, leaseId },
|
|
5441
|
-
];
|
|
5442
|
-
});
|
|
5443
|
-
if (targets.length === 0) return;
|
|
5444
|
-
await this.markRuntimeStepReceiptsQueued(targets);
|
|
5445
|
-
}
|
|
5446
|
-
|
|
5447
5139
|
/**
|
|
5448
5140
|
* Enforce the receipt output contract and remember the output's serialized
|
|
5449
5141
|
* size. Idempotent per input: the first caller pays, everything downstream
|
|
@@ -6326,7 +6018,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6326
6018
|
|
|
6327
6019
|
private durableReceiptExecutionStore(): DurableReceiptExecutionStore {
|
|
6328
6020
|
return {
|
|
6329
|
-
enabled:
|
|
6021
|
+
enabled: this.durableDirectToolResultsBackedByReceipts,
|
|
6330
6022
|
get: (receiptKey) => this.getRuntimeStepReceipt(receiptKey),
|
|
6331
6023
|
getMany: (receiptKeys) => this.getRuntimeStepReceipts(receiptKeys),
|
|
6332
6024
|
claim: (
|
|
@@ -6403,35 +6095,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6403
6095
|
return durableRuntimeReceiptOutput<T>(receipt);
|
|
6404
6096
|
}
|
|
6405
6097
|
|
|
6406
|
-
private async waitForCompletedRuntimeToolReceipt(
|
|
6407
|
-
key: string,
|
|
6408
|
-
maxAttempts?: number,
|
|
6409
|
-
): Promise<RuntimeStepReceipt> {
|
|
6410
|
-
return await waitForCompletedRuntimeReceipt({
|
|
6411
|
-
receiptKey: key,
|
|
6412
|
-
store: this.durableReceiptExecutionStore(),
|
|
6413
|
-
maxAttempts,
|
|
6414
|
-
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
6415
|
-
});
|
|
6416
|
-
}
|
|
6417
|
-
|
|
6418
|
-
private async waitForCompletedRuntimeToolReceipts(
|
|
6419
|
-
keys: string[],
|
|
6420
|
-
maxAttempts?: number,
|
|
6421
|
-
): Promise<{
|
|
6422
|
-
completed: Map<string, RuntimeStepReceipt>;
|
|
6423
|
-
failed: Map<string, Error>;
|
|
6424
|
-
timedOut: Set<string>;
|
|
6425
|
-
}> {
|
|
6426
|
-
return await waitForCompletedRuntimeReceipts({
|
|
6427
|
-
receiptKeys: keys,
|
|
6428
|
-
store: this.durableReceiptExecutionStore(),
|
|
6429
|
-
maxAttempts,
|
|
6430
|
-
log: (message) => this.runtimeLog(message),
|
|
6431
|
-
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
6432
|
-
});
|
|
6433
|
-
}
|
|
6434
|
-
|
|
6435
6098
|
private async executeWithRuntimeReceipt<T>(
|
|
6436
6099
|
operation: DurableCtxOperation,
|
|
6437
6100
|
id: string,
|
|
@@ -6525,7 +6188,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6525
6188
|
onClaimedResult: opts.onClaimedResult,
|
|
6526
6189
|
shouldPersistFailure: opts.shouldPersistFailure,
|
|
6527
6190
|
markRunningBeforeExecute: opts.markRunningBeforeExecute,
|
|
6528
|
-
completedCacheOnly:
|
|
6191
|
+
completedCacheOnly: true,
|
|
6529
6192
|
withCompletedReceiptHydration: (hydrate) =>
|
|
6530
6193
|
this.withRuntimeReceiptHydrationTurn(hydrate, {
|
|
6531
6194
|
source: 'durable_operation',
|
|
@@ -6632,7 +6295,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6632
6295
|
*
|
|
6633
6296
|
* The external receipt adapter is intentionally local to this migration.
|
|
6634
6297
|
* It checks the exact compatibility identities passed by the map, while the
|
|
6635
|
-
* map remains the sole writer for its member receipts,
|
|
6298
|
+
* map remains the sole writer for its member receipts, invocation delivery,
|
|
6636
6299
|
* and completion barrier. In particular, do not turn an always-fresh map
|
|
6637
6300
|
* request into a provider-idempotent request here: `apiOptions` is forwarded
|
|
6638
6301
|
* byte-for-byte and decides whether the historical wire carried an identity.
|
|
@@ -6806,19 +6469,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6806
6469
|
});
|
|
6807
6470
|
}
|
|
6808
6471
|
|
|
6809
|
-
/**
|
|
6810
|
-
* A forced call bypasses an earlier run's shared result, so its physical
|
|
6811
|
-
* gateway invocation must bypass that result's replay identity too. Keep
|
|
6812
|
-
* the transport key deterministic for one logical call: a retry of that
|
|
6813
|
-
* call still converges, while a later explicit force gets a fresh identity.
|
|
6814
|
-
*/
|
|
6472
|
+
/** One replay-stable physical delivery identity for each invocation. */
|
|
6815
6473
|
private toolTransportReceiptKey(input: {
|
|
6816
6474
|
toolId: string;
|
|
6817
|
-
resultReceiptKey: string;
|
|
6818
6475
|
providerOperationKey: string;
|
|
6819
|
-
force?: boolean;
|
|
6820
6476
|
}): string {
|
|
6821
|
-
if (input.force !== true) return input.resultReceiptKey;
|
|
6822
6477
|
return `${buildDurableToolReceiptPrefix({
|
|
6823
6478
|
orgId: this.#options.orgId,
|
|
6824
6479
|
toolId: input.toolId,
|
|
@@ -6841,41 +6496,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6841
6496
|
});
|
|
6842
6497
|
}
|
|
6843
6498
|
|
|
6844
|
-
/**
|
|
6845
|
-
|
|
6846
|
-
* execution relay still needs a semantic signal that an explicit force is
|
|
6847
|
-
* a new provider operation. Unsafe-write operations already carry their
|
|
6848
|
-
* forced provider key directly, so only ordinary calls need this attested
|
|
6849
|
-
* canonical-operation salt.
|
|
6850
|
-
*/
|
|
6851
|
-
private providerForceRefreshIdentitiesForToolCall(input: {
|
|
6499
|
+
/** Keep independent misses distinct in the gateway's canonical operation. */
|
|
6500
|
+
private providerInvocationOptionsForToolCall(input: {
|
|
6852
6501
|
operationKey: string;
|
|
6853
|
-
force?: boolean;
|
|
6854
6502
|
requiresExecutionFence?: boolean;
|
|
6855
6503
|
logicalCallId?: string;
|
|
6856
|
-
}): string[] {
|
|
6857
|
-
if (input.force !== true || input.requiresExecutionFence === true) {
|
|
6858
|
-
return [];
|
|
6859
|
-
}
|
|
6860
|
-
return [
|
|
6861
|
-
this.providerIdempotencyKeyForToolCall({
|
|
6862
|
-
operationKey: input.operationKey,
|
|
6863
|
-
force: true,
|
|
6864
|
-
logicalCallId: input.logicalCallId,
|
|
6865
|
-
}),
|
|
6866
|
-
];
|
|
6867
|
-
}
|
|
6868
|
-
|
|
6869
|
-
private providerForceRefreshOptionsForToolCall(input: {
|
|
6870
|
-
operationKey: string;
|
|
6871
6504
|
force?: boolean;
|
|
6872
|
-
requiresExecutionFence?: boolean;
|
|
6873
|
-
logicalCallId?: string;
|
|
6874
6505
|
}): Pick<ToolExecutionApiOptions, 'providerOperationForceRefreshIdentities'> {
|
|
6875
|
-
|
|
6876
|
-
|
|
6877
|
-
|
|
6878
|
-
|
|
6506
|
+
return input.requiresExecutionFence === true
|
|
6507
|
+
? {}
|
|
6508
|
+
: {
|
|
6509
|
+
providerOperationForceRefreshIdentities: [
|
|
6510
|
+
this.providerInvocationKeyForToolCall(input),
|
|
6511
|
+
],
|
|
6512
|
+
};
|
|
6879
6513
|
}
|
|
6880
6514
|
|
|
6881
6515
|
/**
|
|
@@ -7577,15 +7211,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7577
7211
|
private getCachedToolResult(
|
|
7578
7212
|
toolId: string,
|
|
7579
7213
|
rowCacheKey: string,
|
|
7580
|
-
|
|
7214
|
+
_path: 'mapped' | 'direct' = 'mapped',
|
|
7581
7215
|
): ToolBatchResult | undefined {
|
|
7582
|
-
|
|
7583
|
-
|
|
7584
|
-
? this.durableDirectToolResultsBackedByReceipts
|
|
7585
|
-
: this.durableMappedToolResultsBackedByReceipts
|
|
7586
|
-
) {
|
|
7587
|
-
return undefined;
|
|
7588
|
-
}
|
|
7216
|
+
// Receipt-backed calls never add payloads here. Retained outgoing entries
|
|
7217
|
+
// may predate that capability and remain completed cache hits.
|
|
7589
7218
|
return this.checkpoint.completedToolBatches[toolId]?.[rowCacheKey];
|
|
7590
7219
|
}
|
|
7591
7220
|
|
|
@@ -7636,32 +7265,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7636
7265
|
if (!this.checkpoint.completedToolBatches[toolId]) {
|
|
7637
7266
|
this.checkpoint.completedToolBatches[toolId] = {};
|
|
7638
7267
|
}
|
|
7639
|
-
this.checkpoint.completedToolBatches[toolId]![rowCacheKey]
|
|
7268
|
+
this.checkpoint.completedToolBatches[toolId]![rowCacheKey] ??= {
|
|
7640
7269
|
done: true,
|
|
7641
7270
|
result,
|
|
7642
7271
|
};
|
|
7643
7272
|
}
|
|
7644
7273
|
|
|
7645
|
-
private buildToolResultCacheKey(input: {
|
|
7646
|
-
rowId: number;
|
|
7647
|
-
tableNamespace?: string;
|
|
7648
|
-
rowKey?: string;
|
|
7649
|
-
callId?: string;
|
|
7650
|
-
}): string {
|
|
7651
|
-
const scope =
|
|
7652
|
-
this.currentGovernance.currentRunId || this.#options.runId || 'run';
|
|
7653
|
-
if (input.callId?.trim()) {
|
|
7654
|
-
return `${scope}:${input.callId.trim()}`;
|
|
7655
|
-
}
|
|
7656
|
-
if (input.rowKey?.trim()) {
|
|
7657
|
-
return `${scope}:${input.rowKey.trim()}`;
|
|
7658
|
-
}
|
|
7659
|
-
if (input.tableNamespace?.trim()) {
|
|
7660
|
-
return `${scope}:${input.tableNamespace.trim()}:${input.rowId}`;
|
|
7661
|
-
}
|
|
7662
|
-
return `${scope}:direct:${input.rowId}`;
|
|
7663
|
-
}
|
|
7664
|
-
|
|
7665
7274
|
private async resolveToolResultMetadata(
|
|
7666
7275
|
toolId: string,
|
|
7667
7276
|
): Promise<ToolResultMetadataInput> {
|
|
@@ -8136,40 +7745,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8136
7745
|
`Durable tool call ${receiptKey} completed live after its receipt lease moved; using live result without overwriting the receipt.`,
|
|
8137
7746
|
);
|
|
8138
7747
|
}
|
|
8139
|
-
const finalWrapped =
|
|
8140
|
-
(completed?.status === 'completed' || completed?.status === 'skipped') &&
|
|
8141
|
-
completed.output !== undefined
|
|
8142
|
-
? await this.wrapToolExecutionResult({
|
|
8143
|
-
toolId,
|
|
8144
|
-
status:
|
|
8145
|
-
completed.output === null || completed.output === undefined
|
|
8146
|
-
? 'no_result'
|
|
8147
|
-
: 'completed',
|
|
8148
|
-
result: this.runtimeReceiptOutput(completed),
|
|
8149
|
-
requestInput: request.input,
|
|
8150
|
-
execution: toolExecutionMetadataForOutcome(
|
|
8151
|
-
completed.runId === this.currentReceiptOwnerRunId
|
|
8152
|
-
? {
|
|
8153
|
-
kind: 'live',
|
|
8154
|
-
cacheKey,
|
|
8155
|
-
receiptKey: request.receiptKey,
|
|
8156
|
-
}
|
|
8157
|
-
: {
|
|
8158
|
-
kind: 'cache',
|
|
8159
|
-
cacheKey,
|
|
8160
|
-
receiptKey: receiptKey ?? '',
|
|
8161
|
-
attachedToReceiptKey: receiptKey,
|
|
8162
|
-
},
|
|
8163
|
-
),
|
|
8164
|
-
})
|
|
8165
|
-
: wrapped;
|
|
8166
7748
|
if (request.cacheable !== false) {
|
|
8167
|
-
this.cacheToolResult(toolId, cacheKey,
|
|
7749
|
+
this.cacheToolResult(toolId, cacheKey, wrapped);
|
|
8168
7750
|
}
|
|
8169
7751
|
|
|
8170
7752
|
const resolver = this.toolCallResolvers.get(request.callId);
|
|
8171
7753
|
if (resolver) {
|
|
8172
|
-
resolver.resolve(
|
|
7754
|
+
resolver.resolve(wrapped);
|
|
8173
7755
|
this.toolCallResolvers.delete(request.callId);
|
|
8174
7756
|
}
|
|
8175
7757
|
|
|
@@ -8185,7 +7767,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8185
7767
|
error: null,
|
|
8186
7768
|
dataPatch: {},
|
|
8187
7769
|
});
|
|
8188
|
-
return
|
|
7770
|
+
return wrapped;
|
|
8189
7771
|
}
|
|
8190
7772
|
|
|
8191
7773
|
private async rejectToolCall(
|
|
@@ -9003,6 +8585,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9003
8585
|
Math.max(0, failedCount - staleFailedKeys.size),
|
|
9004
8586
|
progressSupersededOffset: () =>
|
|
9005
8587
|
staleCompletedKeys.size + staleFailedKeys.size,
|
|
8588
|
+
progressSkippedOffset: () => duplicateReuseCount,
|
|
9006
8589
|
progressTotalRows: () => totalInputCount,
|
|
9007
8590
|
rowFeed: execution.rowFeed,
|
|
9008
8591
|
retainedRowsMemoryBudgetBytes:
|
|
@@ -10231,6 +9814,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10231
9814
|
progressCompletedOffset?: number | (() => number);
|
|
10232
9815
|
progressFailedOffset?: number | (() => number);
|
|
10233
9816
|
progressSupersededOffset?: number | (() => number);
|
|
9817
|
+
/** Input duplicates discovered after admission; excludes resolver outcomes. */
|
|
9818
|
+
progressSkippedOffset?: number | (() => number);
|
|
10234
9819
|
progressTotalRows?: number | (() => number);
|
|
10235
9820
|
/**
|
|
10236
9821
|
* When set, `items`, `executionRowKeys` and `executionRowIndexes` grow
|
|
@@ -10476,6 +10061,37 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10476
10061
|
} | null = null;
|
|
10477
10062
|
let lastFrameFlushAt = 0;
|
|
10478
10063
|
let rowsSinceFrameFlush = 0;
|
|
10064
|
+
// Scalars only: resolver admission is visible before the first row's
|
|
10065
|
+
// durable completion, without publishing one event per row or tool call.
|
|
10066
|
+
let startedResolverRows = 0;
|
|
10067
|
+
let activeResolverRows = 0;
|
|
10068
|
+
let tracksResolverActivity = false;
|
|
10069
|
+
// Persistence can move current resolvers between successful, failed and
|
|
10070
|
+
// superseded offsets. They already contribute to startedResolverRows:
|
|
10071
|
+
// only the settled baseline before this pool started belongs in admission.
|
|
10072
|
+
const initiallyStartedRows =
|
|
10073
|
+
liveNumber(runtimeOptions?.progressCompletedOffset, completedRows) +
|
|
10074
|
+
liveNumber(runtimeOptions?.progressFailedOffset, 0) +
|
|
10075
|
+
liveNumber(runtimeOptions?.progressSupersededOffset, 0);
|
|
10076
|
+
const initiallySkippedRows = liveNumber(
|
|
10077
|
+
runtimeOptions?.progressSkippedOffset,
|
|
10078
|
+
0,
|
|
10079
|
+
);
|
|
10080
|
+
const mapActivityCounts = () => {
|
|
10081
|
+
const startedRows =
|
|
10082
|
+
initiallyStartedRows +
|
|
10083
|
+
(liveNumber(
|
|
10084
|
+
runtimeOptions?.progressSkippedOffset,
|
|
10085
|
+
initiallySkippedRows,
|
|
10086
|
+
) -
|
|
10087
|
+
initiallySkippedRows) +
|
|
10088
|
+
startedResolverRows;
|
|
10089
|
+
return {
|
|
10090
|
+
startedRows,
|
|
10091
|
+
activeRows: activeResolverRows,
|
|
10092
|
+
waitingRows: currentTotalRows() - startedRows,
|
|
10093
|
+
};
|
|
10094
|
+
};
|
|
10479
10095
|
const updateMapFrameProgress = (input: {
|
|
10480
10096
|
status?: MapExecutionFrame['status'];
|
|
10481
10097
|
completedRowKey?: string | null;
|
|
@@ -10607,6 +10223,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10607
10223
|
completedRowKeys.size,
|
|
10608
10224
|
failedRows,
|
|
10609
10225
|
totalRows: currentTotalRows(),
|
|
10226
|
+
...(progressEventType === 'map.progress' && tracksResolverActivity
|
|
10227
|
+
? mapActivityCounts()
|
|
10228
|
+
: {}),
|
|
10610
10229
|
supersededRows: liveNumber(
|
|
10611
10230
|
runtimeOptions?.progressSupersededOffset,
|
|
10612
10231
|
existing.supersededRows ?? 0,
|
|
@@ -11254,6 +10873,39 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11254
10873
|
items[idx] = null as unknown as T;
|
|
11255
10874
|
}
|
|
11256
10875
|
: null;
|
|
10876
|
+
let lastMapActivityKey: string | null = null;
|
|
10877
|
+
tracksResolverActivity = true;
|
|
10878
|
+
const publishMapActivity = () => {
|
|
10879
|
+
const counts = mapActivityCounts();
|
|
10880
|
+
const key = `${counts.startedRows}:${counts.activeRows}:${counts.waitingRows}`;
|
|
10881
|
+
if (key === lastMapActivityKey) return;
|
|
10882
|
+
lastMapActivityKey = key;
|
|
10883
|
+
// Admission samples do not rebuild checkpoint key sets. The map's
|
|
10884
|
+
// writer continues to own durable settlement and checkpoint flushing.
|
|
10885
|
+
this.emitExecutionEvent({
|
|
10886
|
+
type: 'map.progress',
|
|
10887
|
+
mapInvocationId: mapScope.mapInvocationId,
|
|
10888
|
+
mapNodeId: mapScope.mapNodeId ?? null,
|
|
10889
|
+
logicalNamespace: mapScope.logicalNamespace,
|
|
10890
|
+
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
10891
|
+
completedRows:
|
|
10892
|
+
liveNumber(runtimeOptions?.progressCompletedOffset, completedRows) +
|
|
10893
|
+
(liveFrameSets?.completedRowKeys.size ?? 0),
|
|
10894
|
+
failedRows:
|
|
10895
|
+
liveNumber(runtimeOptions?.progressFailedOffset, 0) +
|
|
10896
|
+
(liveFrameSets?.failedRowsCount ?? 0),
|
|
10897
|
+
totalRows: currentTotalRows(),
|
|
10898
|
+
supersededRows: liveNumber(runtimeOptions?.progressSupersededOffset, 0),
|
|
10899
|
+
...counts,
|
|
10900
|
+
...this.inlineChildAggregateEventFields(),
|
|
10901
|
+
at: Date.now(),
|
|
10902
|
+
});
|
|
10903
|
+
};
|
|
10904
|
+
const mapActivityTimer = setInterval(
|
|
10905
|
+
publishMapActivity,
|
|
10906
|
+
PROGRESS_HEARTBEAT_INTERVAL_MS,
|
|
10907
|
+
);
|
|
10908
|
+
mapActivityTimer.unref();
|
|
11257
10909
|
const workers = Array.from({ length: workerCount }, async () => {
|
|
11258
10910
|
while (firstRowError == null) {
|
|
11259
10911
|
if (nextRowIndex >= items.length) {
|
|
@@ -11270,7 +10922,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11270
10922
|
releaseDispatchedInputRow?.(idx);
|
|
11271
10923
|
settledResults[idx] = await mapRowFlow.run({ baseRow }, async () => {
|
|
11272
10924
|
this.assertRowAdmissionOpen();
|
|
11273
|
-
|
|
10925
|
+
startedResolverRows += 1;
|
|
10926
|
+
activeResolverRows += 1;
|
|
10927
|
+
try {
|
|
10928
|
+
return await runRow(idx, baseRow);
|
|
10929
|
+
} finally {
|
|
10930
|
+
activeResolverRows -= 1;
|
|
10931
|
+
}
|
|
11274
10932
|
});
|
|
11275
10933
|
} catch (error) {
|
|
11276
10934
|
// RuntimeFlow has already returned the row slot and memory lease.
|
|
@@ -11364,6 +11022,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11364
11022
|
settled = await Promise.allSettled(workers);
|
|
11365
11023
|
}
|
|
11366
11024
|
} finally {
|
|
11025
|
+
clearInterval(mapActivityTimer);
|
|
11026
|
+
publishMapActivity();
|
|
11367
11027
|
datasetStep.items = currentTotalRows();
|
|
11368
11028
|
this.processedRowCount = currentTotalRows();
|
|
11369
11029
|
mapStallObserver?.stopHeartbeat();
|
|
@@ -12930,7 +12590,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12930
12590
|
options.receiptWaitMs,
|
|
12931
12591
|
);
|
|
12932
12592
|
}
|
|
12933
|
-
const executionScope = this.currentExecutionScope;
|
|
12934
12593
|
const normalizedKey = this.normalizeContextKey(key, 'tool');
|
|
12935
12594
|
const toolCachePolicy = this.effectiveToolCallCachePolicy(options);
|
|
12936
12595
|
const toolRequestIdentity = deriveToolRequestIdentity({
|
|
@@ -13028,10 +12687,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13028
12687
|
...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
|
|
13029
12688
|
];
|
|
13030
12689
|
|
|
13031
|
-
const executeTool = async (
|
|
13032
|
-
leaseId?: string | null;
|
|
13033
|
-
retainExternalCallSlot?: (release: () => void) => void;
|
|
13034
|
-
}): Promise<unknown> => {
|
|
12690
|
+
const executeTool = async (): Promise<unknown> => {
|
|
13035
12691
|
if (eventWaitHandler) {
|
|
13036
12692
|
return this.executeIntegrationEventWaitTool(
|
|
13037
12693
|
toolId,
|
|
@@ -13291,7 +12947,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13291
12947
|
});
|
|
13292
12948
|
};
|
|
13293
12949
|
try {
|
|
13294
|
-
const directReceiptLeaseId = context?.leaseId?.trim() || null;
|
|
13295
12950
|
if (!logicalCallId) {
|
|
13296
12951
|
throw new Error(
|
|
13297
12952
|
'Direct tool execution requires a stable logical call identity.',
|
|
@@ -13299,17 +12954,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13299
12954
|
}
|
|
13300
12955
|
const useDirectResultReceipts =
|
|
13301
12956
|
cacheableToolResult && !bypassDirectResultCache;
|
|
13302
|
-
const providerOperationKey = this.
|
|
12957
|
+
const providerOperationKey = this.providerInvocationKeyForToolCall({
|
|
13303
12958
|
operationKey: providerIdempotencyKeyBase,
|
|
13304
|
-
force: cacheableToolResult && toolCachePolicy.force,
|
|
13305
12959
|
logicalCallId,
|
|
12960
|
+
force: toolCachePolicy.force,
|
|
13306
12961
|
});
|
|
13307
12962
|
const physicalDirectKey = useDirectResultReceipts
|
|
13308
12963
|
? this.toolTransportReceiptKey({
|
|
13309
12964
|
toolId,
|
|
13310
|
-
resultReceiptKey: directCacheKey,
|
|
13311
12965
|
providerOperationKey,
|
|
13312
|
-
force: toolCachePolicy.force,
|
|
13313
12966
|
})
|
|
13314
12967
|
: `${buildDurableToolReceiptPrefix({
|
|
13315
12968
|
orgId: this.#options.orgId ?? 'unknown-org',
|
|
@@ -13323,13 +12976,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13323
12976
|
runId: this.currentRunId,
|
|
13324
12977
|
logicalCallId,
|
|
13325
12978
|
force: toolCachePolicy.force,
|
|
13326
|
-
canStoreResult:
|
|
13327
|
-
this.#options.claimRuntimeStepReceipt &&
|
|
13328
|
-
(this.#options.getRuntimeStepReceipt ||
|
|
13329
|
-
this.#options.getRuntimeStepReceipts) &&
|
|
13330
|
-
(this.#options.completeRuntimeStepReceipt ||
|
|
13331
|
-
this.#options.completeRuntimeStepReceipts),
|
|
13332
|
-
),
|
|
12979
|
+
canStoreResult: this.durableDirectToolResultsBackedByReceipts,
|
|
13333
12980
|
});
|
|
13334
12981
|
const toolRetryPolicy = await this.#options
|
|
13335
12982
|
.getToolRetryPolicy?.(toolId, input)
|
|
@@ -13365,28 +13012,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13365
13012
|
...(useDirectResultReceipts
|
|
13366
13013
|
? {
|
|
13367
13014
|
durableCallReceiptKey: physicalDirectKey,
|
|
13015
|
+
legacyProviderRetryIdentities: toolCachePolicy.force
|
|
13016
|
+
? undefined
|
|
13017
|
+
: [
|
|
13018
|
+
{
|
|
13019
|
+
transportReceiptKey: directCacheKey,
|
|
13020
|
+
providerOperationKey:
|
|
13021
|
+
providerIdempotencyKeyBase,
|
|
13022
|
+
},
|
|
13023
|
+
],
|
|
13368
13024
|
completedResultCache: true,
|
|
13369
13025
|
executionAuthScopeDigest,
|
|
13370
|
-
...(directReceiptLeaseId &&
|
|
13371
|
-
(this.#options.heartbeatRuntimeStepReceipts ||
|
|
13372
|
-
this.#options.getRuntimeStepReceipt ||
|
|
13373
|
-
this.#options.getRuntimeStepReceipts)
|
|
13374
|
-
? {
|
|
13375
|
-
beforeProviderCall: async () => {
|
|
13376
|
-
await this.assertRuntimeToolReceiptOwnership([
|
|
13377
|
-
{
|
|
13378
|
-
callId: 'direct',
|
|
13379
|
-
cacheKey: directCacheKey,
|
|
13380
|
-
receiptKey: directCacheKey,
|
|
13381
|
-
receiptLeaseId: directReceiptLeaseId,
|
|
13382
|
-
rowId: -1,
|
|
13383
|
-
toolId,
|
|
13384
|
-
input,
|
|
13385
|
-
},
|
|
13386
|
-
]);
|
|
13387
|
-
},
|
|
13388
|
-
}
|
|
13389
|
-
: {}),
|
|
13390
13026
|
}
|
|
13391
13027
|
: {}),
|
|
13392
13028
|
...(toolRetryPolicy?.requiresExecutionFence === true
|
|
@@ -13401,15 +13037,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13401
13037
|
: physicalDirectKey,
|
|
13402
13038
|
}
|
|
13403
13039
|
: {}),
|
|
13404
|
-
...(
|
|
13405
|
-
|
|
13406
|
-
|
|
13407
|
-
|
|
13408
|
-
|
|
13409
|
-
|
|
13410
|
-
|
|
13411
|
-
})
|
|
13412
|
-
: {}),
|
|
13040
|
+
...this.providerInvocationOptionsForToolCall({
|
|
13041
|
+
operationKey: providerIdempotencyKeyBase,
|
|
13042
|
+
requiresExecutionFence:
|
|
13043
|
+
toolRetryPolicy?.requiresExecutionFence,
|
|
13044
|
+
logicalCallId,
|
|
13045
|
+
force: toolCachePolicy.force,
|
|
13046
|
+
}),
|
|
13413
13047
|
playNodeScope: buildPlayNodeScope({
|
|
13414
13048
|
toolId,
|
|
13415
13049
|
callKey: normalizedKey,
|
|
@@ -13419,7 +13053,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13419
13053
|
options?.timeoutMs,
|
|
13420
13054
|
this.currentAuthoringContractEdition,
|
|
13421
13055
|
),
|
|
13422
|
-
retainToolSlot: context?.retainExternalCallSlot,
|
|
13423
13056
|
},
|
|
13424
13057
|
});
|
|
13425
13058
|
return await this.wrapToolExecutionResult({
|
|
@@ -13558,20 +13191,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13558
13191
|
|
|
13559
13192
|
const rowId = store.rowId;
|
|
13560
13193
|
const fieldName = store.fieldName;
|
|
13561
|
-
|
|
13562
|
-
|
|
13563
|
-
|
|
13564
|
-
store.rowKey?.trim() || String(rowId),
|
|
13565
|
-
normalizedKey,
|
|
13566
|
-
toolId,
|
|
13567
|
-
toolRequestIdentity,
|
|
13568
|
-
].join(':');
|
|
13569
|
-
if (this.toolCallResolvers.has(callId)) {
|
|
13570
|
-
throw new Error(
|
|
13571
|
-
`ctx.tools.execute("${normalizedKey}") was called more than once concurrently for the same row. ` +
|
|
13572
|
-
'Use a unique id for each row-scoped tool call.',
|
|
13573
|
-
);
|
|
13574
|
-
}
|
|
13194
|
+
// Cache identity is shared by matching completed results. Delivery
|
|
13195
|
+
// identity belongs to this invocation, even when callers reuse an id.
|
|
13196
|
+
const callId = logicalCallId;
|
|
13575
13197
|
|
|
13576
13198
|
const toolResultCacheKey = durableCacheKey;
|
|
13577
13199
|
const cached = !cacheableToolResult
|
|
@@ -13597,7 +13219,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13597
13219
|
const rowToolRetryPolicy = await this.#options
|
|
13598
13220
|
.getToolRetryPolicy?.(toolId, input)
|
|
13599
13221
|
.catch(() => null);
|
|
13600
|
-
|
|
13601
13222
|
// ADR 0018: record this leg before it runs. `producer` still names the
|
|
13602
13223
|
// last leg (the usual cascade winner); `producers` keeps every leg, which
|
|
13603
13224
|
// is the only durable record a losing leg ever gets — a failed tool call
|
|
@@ -13641,7 +13262,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13641
13262
|
// The tail of that wall. The dispatcher stamps the request when this
|
|
13642
13263
|
// call's provider body finished parsing; the difference here is what
|
|
13643
13264
|
// receipt completion and hand-back cost after the provider was done. A
|
|
13644
|
-
// cache hit
|
|
13265
|
+
// cache hit never reads a body, so it leaves no stamp and
|
|
13645
13266
|
// record nothing rather than a delivery of zero.
|
|
13646
13267
|
const observeResultDelivery = () => {
|
|
13647
13268
|
const bodyCompletedAtMs = dispatchRequest.providerBodyCompletedAtMs;
|
|
@@ -15389,7 +15010,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
15389
15010
|
*
|
|
15390
15011
|
* Gated on the same stamp `resultDelivery` is gated on — a body this request
|
|
15391
15012
|
* really read — so every delivery sub-phase counts the same calls and their
|
|
15392
|
-
* sum can be checked against the whole. A cache hit
|
|
15013
|
+
* sum can be checked against the whole. A cache hit and a
|
|
15393
15014
|
* member that never reached the provider leave the stamp unset and record
|
|
15394
15015
|
* nothing rather than a wrap of zero.
|
|
15395
15016
|
*/
|
|
@@ -15521,179 +15142,38 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
15521
15142
|
|
|
15522
15143
|
// ——— Batched tool call execution ———
|
|
15523
15144
|
|
|
15524
|
-
|
|
15525
|
-
|
|
15526
|
-
|
|
15527
|
-
|
|
15528
|
-
|
|
15529
|
-
|
|
15530
|
-
|
|
15531
|
-
|
|
15532
|
-
|
|
15533
|
-
|
|
15534
|
-
|
|
15535
|
-
|
|
15536
|
-
|
|
15537
|
-
|
|
15538
|
-
|
|
15539
|
-
|
|
15540
|
-
(
|
|
15541
|
-
const
|
|
15542
|
-
|
|
15543
|
-
if (!earliest) return expiresAt;
|
|
15544
|
-
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
15545
|
-
? expiresAt
|
|
15546
|
-
: earliest;
|
|
15547
|
-
},
|
|
15548
|
-
null,
|
|
15549
|
-
);
|
|
15550
|
-
let leaseLost: unknown | null = null;
|
|
15551
|
-
let ownershipConfirmedUntilMs = Number.isFinite(
|
|
15552
|
-
Date.parse(earliestLeaseExpiry ?? ''),
|
|
15553
|
-
)
|
|
15554
|
-
? Date.parse(earliestLeaseExpiry!)
|
|
15555
|
-
: Date.now() + PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS;
|
|
15556
|
-
const firstOwnedRequest = ownedRequests[0]!;
|
|
15557
|
-
const heartbeatOwnedRequests = async () => {
|
|
15558
|
-
// Receipt claims are acquired in bulk before the provider-shaped
|
|
15559
|
-
// dispatcher admits every call. Renew the undispatched tail as well as
|
|
15560
|
-
// in-flight calls; otherwise a slow provider can let later claims
|
|
15561
|
-
// expire while they are still waiting behind the concurrency gate.
|
|
15562
|
-
const unsettled = ownedRequests.filter((request) =>
|
|
15563
|
-
this.toolCallResolvers.has(request.callId),
|
|
15564
|
-
);
|
|
15565
|
-
if (unsettled.length === 0) return 'terminal' as const;
|
|
15566
|
-
const byReceiptKey = new Map<string, ToolCallRequest[]>();
|
|
15567
|
-
for (const request of unsettled) {
|
|
15568
|
-
const receiptKey = request.receiptKey!;
|
|
15569
|
-
const group = byReceiptKey.get(receiptKey) ?? [];
|
|
15570
|
-
group.push(request);
|
|
15571
|
-
byReceiptKey.set(receiptKey, group);
|
|
15145
|
+
/**
|
|
15146
|
+
* Concrete RuntimeStepReceipt adapter for the map receipt-cohort Module.
|
|
15147
|
+
*
|
|
15148
|
+
* Scalar and provider-native map calls share completed-result cache reads
|
|
15149
|
+
* and atomic successful inserts. A miss has no durable execution owner.
|
|
15150
|
+
*/
|
|
15151
|
+
private createRuntimeStepToolCallReceiptCohortStore(input: {
|
|
15152
|
+
toolId: string;
|
|
15153
|
+
requestForReceipt: ReadonlyMap<string, ToolCallRequest>;
|
|
15154
|
+
/** Each invocation sharing a completed-result key, for manifests and timing. */
|
|
15155
|
+
requestsForReceipt: ReadonlyMap<string, readonly ToolCallRequest[]>;
|
|
15156
|
+
}): ToolCallReceiptCohortStore {
|
|
15157
|
+
const requestsForPublication = (receipt: {
|
|
15158
|
+
resultReceiptKey: string;
|
|
15159
|
+
memberIds?: readonly string[];
|
|
15160
|
+
}): readonly ToolCallRequest[] => {
|
|
15161
|
+
if (!receipt.memberIds) {
|
|
15162
|
+
const request = input.requestForReceipt.get(receipt.resultReceiptKey);
|
|
15163
|
+
return request ? [request] : [];
|
|
15572
15164
|
}
|
|
15573
|
-
|
|
15574
|
-
|
|
15575
|
-
|
|
15576
|
-
|
|
15577
|
-
|
|
15578
|
-
|
|
15579
|
-
(
|
|
15580
|
-
|
|
15581
|
-
const receipts = await heartbeatReceipts({
|
|
15582
|
-
runId: this.currentReceiptOwnerRunId,
|
|
15583
|
-
runAttempt: this.currentRunAttempt,
|
|
15584
|
-
keys,
|
|
15585
|
-
leaseIds,
|
|
15586
|
-
});
|
|
15587
|
-
const renewedExpiries: number[] = [];
|
|
15588
|
-
for (let index = 0; index < entries.length; index += 1) {
|
|
15589
|
-
const [receiptKey, requests] = entries[index]!;
|
|
15590
|
-
const leaseId = leaseIds[index]!;
|
|
15591
|
-
// Completion can race this bulk heartbeat response. A settled
|
|
15592
|
-
// receipt group no longer needs ownership and must not turn that
|
|
15593
|
-
// race into a false lease-loss failure or hold the confirmation
|
|
15594
|
-
// horizon at its old expiry.
|
|
15595
|
-
if (
|
|
15596
|
-
this.runtimeToolReceiptCompletionsInFlight.has(receiptKey) ||
|
|
15597
|
-
!requests.some((request) =>
|
|
15598
|
-
this.toolCallResolvers.has(request.callId),
|
|
15599
|
-
)
|
|
15600
|
-
) {
|
|
15601
|
-
continue;
|
|
15165
|
+
const requests =
|
|
15166
|
+
input.requestsForReceipt.get(receipt.resultReceiptKey) ?? [];
|
|
15167
|
+
return receipt.memberIds.map((memberId) => {
|
|
15168
|
+
const request = requests.find(
|
|
15169
|
+
(candidate) => candidate.callId === memberId,
|
|
15170
|
+
);
|
|
15171
|
+
if (!request) {
|
|
15172
|
+
throw new Error(`Unknown Tool Call publication member ${memberId}.`);
|
|
15602
15173
|
}
|
|
15603
|
-
|
|
15604
|
-
|
|
15605
|
-
if (
|
|
15606
|
-
!this.runtimeToolReceiptStillOwned(receipt, leaseId) ||
|
|
15607
|
-
!Number.isFinite(renewedExpiryMs)
|
|
15608
|
-
) {
|
|
15609
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
15610
|
-
receiptKey,
|
|
15611
|
-
runId: this.currentReceiptOwnerRunId,
|
|
15612
|
-
leaseId,
|
|
15613
|
-
});
|
|
15614
|
-
}
|
|
15615
|
-
renewedExpiries.push(renewedExpiryMs);
|
|
15616
|
-
}
|
|
15617
|
-
if (renewedExpiries.length === 0) return 'terminal' as const;
|
|
15618
|
-
ownershipConfirmedUntilMs = Math.min(...renewedExpiries);
|
|
15619
|
-
return 'active' as const;
|
|
15620
|
-
};
|
|
15621
|
-
const perCallHeartbeatIntervalMs = runtimeLeaseHeartbeatIntervalFromExpiry({
|
|
15622
|
-
leaseExpiresAt: earliestLeaseExpiry,
|
|
15623
|
-
fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
15624
|
-
});
|
|
15625
|
-
const supervisor = createRuntimeReceiptHeartbeatSupervisor({
|
|
15626
|
-
// The cohort owns claims that have not reached provider admission yet.
|
|
15627
|
-
// Renew it before the one-call supervisors can win the same short-lease
|
|
15628
|
-
// timer race, so the queued tail remains fenced while the active call
|
|
15629
|
-
// consumes the provider concurrency slot.
|
|
15630
|
-
intervalMs: Math.max(1, Math.floor(perCallHeartbeatIntervalMs / 2)),
|
|
15631
|
-
heartbeat: heartbeatOwnedRequests,
|
|
15632
|
-
isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
|
|
15633
|
-
onLeaseLost: (error) => {
|
|
15634
|
-
leaseLost ??= error;
|
|
15635
|
-
},
|
|
15636
|
-
onTransientFailure: (error) => {
|
|
15637
|
-
this.runtimeLog(
|
|
15638
|
-
`Pending tool receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`,
|
|
15639
|
-
);
|
|
15640
|
-
},
|
|
15641
|
-
});
|
|
15642
|
-
return {
|
|
15643
|
-
stop: () => supervisor.stop(),
|
|
15644
|
-
leaseLost: () => leaseLost,
|
|
15645
|
-
confirmOwned: async () => {
|
|
15646
|
-
// The scheduler receipt gateway returns `running` from an atomic
|
|
15647
|
-
// claim. That claim is already the provider-execution fence, so short
|
|
15648
|
-
// calls only need scheduled renewal if they approach lease expiry.
|
|
15649
|
-
// Other callers may still return queued/pending claims or omit the
|
|
15650
|
-
// explicit backend capability; retain the immediate ownership check
|
|
15651
|
-
// for those receipts.
|
|
15652
|
-
if (claimsEstablishedExecutionFence) {
|
|
15653
|
-
if (
|
|
15654
|
-
ownershipConfirmedUntilMs - Date.now() <=
|
|
15655
|
-
IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS
|
|
15656
|
-
) {
|
|
15657
|
-
const outcome = await heartbeatOwnedRequests();
|
|
15658
|
-
if (outcome === 'active') supervisor.start();
|
|
15659
|
-
return;
|
|
15660
|
-
}
|
|
15661
|
-
supervisor.start();
|
|
15662
|
-
return;
|
|
15663
|
-
}
|
|
15664
|
-
const outcome = await heartbeatOwnedRequests();
|
|
15665
|
-
if (outcome === 'active') supervisor.start();
|
|
15666
|
-
},
|
|
15667
|
-
assertOwned: () => {
|
|
15668
|
-
if (leaseLost) throw leaseLost;
|
|
15669
|
-
if (Date.now() < ownershipConfirmedUntilMs) return;
|
|
15670
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
15671
|
-
receiptKey: firstOwnedRequest.receiptKey!,
|
|
15672
|
-
runId: this.currentReceiptOwnerRunId,
|
|
15673
|
-
leaseId: firstOwnedRequest.receiptLeaseId!,
|
|
15674
|
-
});
|
|
15675
|
-
},
|
|
15174
|
+
return request;
|
|
15175
|
+
});
|
|
15676
15176
|
};
|
|
15677
|
-
}
|
|
15678
|
-
|
|
15679
|
-
/**
|
|
15680
|
-
* Concrete RuntimeStepReceipt adapter for the map receipt-cohort Module.
|
|
15681
|
-
*
|
|
15682
|
-
* Scalar and provider-native map calls share completed-result cache reads
|
|
15683
|
-
* and atomic successful inserts. A miss has no durable execution owner.
|
|
15684
|
-
*/
|
|
15685
|
-
private createRuntimeStepToolCallReceiptCohortStore(input: {
|
|
15686
|
-
toolId: string;
|
|
15687
|
-
requestForReceipt: ReadonlyMap<string, ToolCallRequest>;
|
|
15688
|
-
/**
|
|
15689
|
-
* Every request sharing a receipt key, for delivery timing only. Two rows
|
|
15690
|
-
* asking the same question share one key, and only one of them — the
|
|
15691
|
-
* cohort's owner — executed and carries the wrap stamp `deliveryPublishWait`
|
|
15692
|
-
* is measured from, so the timing lookup cannot use the single-request map
|
|
15693
|
-
* above, which keeps whichever request was seen last.
|
|
15694
|
-
*/
|
|
15695
|
-
requestsForReceipt: ReadonlyMap<string, readonly ToolCallRequest[]>;
|
|
15696
|
-
}): ToolCallReceiptCohortStore {
|
|
15697
15177
|
const asRecord = async (
|
|
15698
15178
|
resultReceiptKey: string,
|
|
15699
15179
|
storedReceiptKey: string,
|
|
@@ -15879,23 +15359,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
15879
15359
|
);
|
|
15880
15360
|
const manifestCompletions =
|
|
15881
15361
|
this.sourceRunToolReceiptManifestCompletionInputs(
|
|
15882
|
-
receipts.flatMap((receipt) =>
|
|
15883
|
-
|
|
15884
|
-
|
|
15885
|
-
|
|
15886
|
-
|
|
15887
|
-
|
|
15888
|
-
|
|
15889
|
-
|
|
15890
|
-
|
|
15891
|
-
executionAuthScopeDigest:
|
|
15892
|
-
request.executionAuthScopeDigest,
|
|
15893
|
-
resultReceiptKey: receipt.resultReceiptKey,
|
|
15894
|
-
providerActionVersion: request.providerActionVersion,
|
|
15895
|
-
},
|
|
15896
|
-
]
|
|
15897
|
-
: [];
|
|
15898
|
-
}),
|
|
15362
|
+
receipts.flatMap((receipt) =>
|
|
15363
|
+
requestsForPublication(receipt).map((request) => ({
|
|
15364
|
+
toolId: input.toolId,
|
|
15365
|
+
logicalCallId: request.logicalCallId,
|
|
15366
|
+
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
15367
|
+
resultReceiptKey: receipt.resultReceiptKey,
|
|
15368
|
+
providerActionVersion: request.providerActionVersion,
|
|
15369
|
+
})),
|
|
15370
|
+
),
|
|
15899
15371
|
);
|
|
15900
15372
|
await this.completeRuntimeStepReceipts(
|
|
15901
15373
|
[...canonicalCompletions, ...manifestCompletions],
|
|
@@ -15905,18 +15377,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
15905
15377
|
// charged to one of the two rather than to neither.
|
|
15906
15378
|
(enqueuedAtMs) => {
|
|
15907
15379
|
for (const receipt of receipts) {
|
|
15908
|
-
const
|
|
15909
|
-
|
|
15910
|
-
|
|
15911
|
-
(
|
|
15912
|
-
|
|
15913
|
-
|
|
15914
|
-
|
|
15915
|
-
|
|
15916
|
-
|
|
15917
|
-
|
|
15918
|
-
),
|
|
15919
|
-
});
|
|
15380
|
+
for (const request of requestsForPublication(receipt)) {
|
|
15381
|
+
const wrappedAtMs = request.deliveryWrappedAtMs;
|
|
15382
|
+
if (wrappedAtMs == null) continue;
|
|
15383
|
+
this.resourceGovernor.observe({
|
|
15384
|
+
deliveryPublishWaitMs: Math.max(
|
|
15385
|
+
0,
|
|
15386
|
+
enqueuedAtMs - wrappedAtMs,
|
|
15387
|
+
),
|
|
15388
|
+
});
|
|
15389
|
+
}
|
|
15920
15390
|
}
|
|
15921
15391
|
},
|
|
15922
15392
|
// The completion writer batches by arrival, so a publication wave
|
|
@@ -16011,57 +15481,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16011
15481
|
: request.input,
|
|
16012
15482
|
),
|
|
16013
15483
|
})),
|
|
16014
|
-
// Claim immediately so unrelated owned rows can cross provider work
|
|
16015
|
-
// while foreign members hydrate in the bounded lane.
|
|
16016
|
-
eager: true,
|
|
16017
|
-
});
|
|
16018
|
-
|
|
16019
|
-
// Claims carry their actual expiry. Seed the cohort supervisor from that
|
|
16020
|
-
// lease rather than the generic fallback, otherwise a long provider-shaped
|
|
16021
|
-
// queue can let undispatched rows expire before its first renewal.
|
|
16022
|
-
for (const request of requests) {
|
|
16023
|
-
const member = cohort.member(request.callId);
|
|
16024
|
-
if (member.kind !== 'owned') continue;
|
|
16025
|
-
request.receiptLeaseId = member.lease.leaseId;
|
|
16026
|
-
request.receiptLeaseExpiresAt = member.leaseExpiresAt ?? null;
|
|
16027
|
-
}
|
|
16028
|
-
|
|
16029
|
-
const earliestLeaseExpiry = requests.reduce<string | null>(
|
|
16030
|
-
(earliest, request) => {
|
|
16031
|
-
const expiresAt = request.receiptLeaseExpiresAt ?? null;
|
|
16032
|
-
if (!expiresAt) return earliest;
|
|
16033
|
-
if (!earliest) return expiresAt;
|
|
16034
|
-
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
16035
|
-
? expiresAt
|
|
16036
|
-
: earliest;
|
|
16037
|
-
},
|
|
16038
|
-
null,
|
|
16039
|
-
);
|
|
16040
|
-
// One cohort-wide liveness check closes the claim-to-provider window; the
|
|
16041
|
-
// per-call views additionally preserve the historical exact admission
|
|
16042
|
-
// fence immediately before an individual provider request.
|
|
16043
|
-
await cohort.heartbeat();
|
|
16044
|
-
let cohortLeaseLost: unknown | null = null;
|
|
16045
|
-
const heartbeat = cohort.startHeartbeat({
|
|
16046
|
-
intervalMs: Math.max(
|
|
16047
|
-
1,
|
|
16048
|
-
Math.floor(
|
|
16049
|
-
runtimeLeaseHeartbeatIntervalFromExpiry({
|
|
16050
|
-
leaseExpiresAt: earliestLeaseExpiry,
|
|
16051
|
-
fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
16052
|
-
}) / 2,
|
|
16053
|
-
),
|
|
16054
|
-
),
|
|
16055
|
-
onLeaseLost: (error) => {
|
|
16056
|
-
cohortLeaseLost ??= error;
|
|
16057
|
-
},
|
|
16058
|
-
onTransientFailure: (error) => {
|
|
16059
|
-
this.runtimeLog(
|
|
16060
|
-
`Tool Call Receipt Cohort heartbeat transport failed; retrying: ${
|
|
16061
|
-
error instanceof Error ? error.message : String(error)
|
|
16062
|
-
}`,
|
|
16063
|
-
);
|
|
16064
|
-
},
|
|
16065
15484
|
});
|
|
16066
15485
|
|
|
16067
15486
|
const successful: ToolCallRequest[] = [];
|
|
@@ -16108,35 +15527,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16108
15527
|
});
|
|
16109
15528
|
};
|
|
16110
15529
|
|
|
16111
|
-
const
|
|
16112
|
-
const
|
|
16113
|
-
const collectReadyMembers = (): Array<{
|
|
15530
|
+
const cachedResults: Promise<void>[] = [];
|
|
15531
|
+
const owned: Array<{
|
|
16114
15532
|
request: ToolCallRequest;
|
|
16115
15533
|
receipts: ToolResultReceipts;
|
|
16116
|
-
}>
|
|
16117
|
-
|
|
16118
|
-
|
|
16119
|
-
|
|
16120
|
-
|
|
16121
|
-
|
|
16122
|
-
|
|
16123
|
-
|
|
16124
|
-
|
|
16125
|
-
|
|
16126
|
-
|
|
16127
|
-
} else if (member.kind === 'following') {
|
|
16128
|
-
followerResults.push(
|
|
16129
|
-
member.result.then(
|
|
16130
|
-
async (result) => await resolve(request, result, 'owner'),
|
|
16131
|
-
async (error: unknown) => await reject(request, error),
|
|
16132
|
-
),
|
|
16133
|
-
);
|
|
16134
|
-
} else {
|
|
16135
|
-
owned.push({ request, receipts: member.receipts });
|
|
16136
|
-
}
|
|
15534
|
+
}> = [];
|
|
15535
|
+
for (const request of requests) {
|
|
15536
|
+
const member = cohort.member(request.callId);
|
|
15537
|
+
if (member.kind === 'completed') {
|
|
15538
|
+
cachedResults.push(resolve(request, member.result, member.source));
|
|
15539
|
+
} else if (member.kind === 'owned') {
|
|
15540
|
+
owned.push({ request, receipts: member.receipts });
|
|
15541
|
+
} else {
|
|
15542
|
+
throw new Error(
|
|
15543
|
+
'A completed-result cache cannot own a pending invocation.',
|
|
15544
|
+
);
|
|
16137
15545
|
}
|
|
16138
|
-
|
|
16139
|
-
};
|
|
15546
|
+
}
|
|
16140
15547
|
|
|
16141
15548
|
const width = Math.min(
|
|
16142
15549
|
this.governor.policy.concurrency.toolCalls,
|
|
@@ -16148,85 +15555,91 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16148
15555
|
),
|
|
16149
15556
|
),
|
|
16150
15557
|
);
|
|
16151
|
-
|
|
16152
|
-
owned
|
|
16153
|
-
|
|
16154
|
-
|
|
16155
|
-
|
|
16156
|
-
|
|
16157
|
-
|
|
16158
|
-
|
|
16159
|
-
|
|
16160
|
-
|
|
16161
|
-
|
|
16162
|
-
|
|
16163
|
-
|
|
16164
|
-
|
|
16165
|
-
|
|
16166
|
-
|
|
16167
|
-
|
|
16168
|
-
|
|
16169
|
-
|
|
16170
|
-
|
|
16171
|
-
|
|
16172
|
-
|
|
16173
|
-
|
|
16174
|
-
|
|
16175
|
-
|
|
16176
|
-
|
|
16177
|
-
|
|
16178
|
-
|
|
15558
|
+
await dispatchBoundedSettled(
|
|
15559
|
+
owned,
|
|
15560
|
+
width,
|
|
15561
|
+
async ({ request, receipts }) => {
|
|
15562
|
+
// Circuit breaker: do not dispatch this provider call once a
|
|
15563
|
+
// persistence failure has occurred in this run. Rejected through the
|
|
15564
|
+
// same path as a provider failure — `dispatchBoundedSettled` only
|
|
15565
|
+
// collects a thrown error, so throwing past `reject` here would
|
|
15566
|
+
// leave this row's tool-call resolver pending for the life of the
|
|
15567
|
+
// map.
|
|
15568
|
+
if (this.persistenceLatch.tripped) {
|
|
15569
|
+
this.persistenceLatch.preventedCallCount += 1;
|
|
15570
|
+
await reject(
|
|
15571
|
+
request,
|
|
15572
|
+
new RuntimePersistenceCircuitOpenError(this.persistenceLatch),
|
|
15573
|
+
);
|
|
15574
|
+
return;
|
|
15575
|
+
}
|
|
15576
|
+
let releaseToolSlot: () => void = () => undefined;
|
|
15577
|
+
try {
|
|
15578
|
+
const providerOperationKey = this.providerInvocationKeyForToolCall({
|
|
15579
|
+
operationKey:
|
|
15580
|
+
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
15581
|
+
logicalCallId: request.logicalCallId,
|
|
15582
|
+
force: request.force,
|
|
15583
|
+
});
|
|
15584
|
+
const result = await this.executeMappedToolCallThroughModule({
|
|
15585
|
+
request,
|
|
15586
|
+
dispatchToolId: toolId,
|
|
15587
|
+
dispatchPayload: request.input,
|
|
15588
|
+
resultReceiptKey:
|
|
15589
|
+
this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
|
|
15590
|
+
providerOperationKey,
|
|
15591
|
+
resultReceipts: receipts,
|
|
15592
|
+
apiOptions: {
|
|
15593
|
+
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
15594
|
+
durableCallReceiptKey: this.toolTransportReceiptKey({
|
|
15595
|
+
toolId,
|
|
15596
|
+
providerOperationKey,
|
|
15597
|
+
}),
|
|
15598
|
+
legacyProviderRetryIdentities: [
|
|
15599
|
+
{
|
|
15600
|
+
transportReceiptKey: request.receiptKey!,
|
|
15601
|
+
providerOperationKey: request.force
|
|
15602
|
+
? providerOperationKey
|
|
15603
|
+
: this.providerOperationKeyForQueuedToolCall(request),
|
|
15604
|
+
...(request.force && request.requiresExecutionFence !== true
|
|
15605
|
+
? { forceRefreshIdentities: [providerOperationKey] }
|
|
15606
|
+
: {}),
|
|
15607
|
+
},
|
|
15608
|
+
],
|
|
15609
|
+
completedResultCache: true,
|
|
15610
|
+
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
15611
|
+
...(request.requiresExecutionFence === true
|
|
15612
|
+
? {
|
|
15613
|
+
providerIdempotencyKey: providerOperationKey,
|
|
15614
|
+
providerIdempotencyReceiptKey: request.receiptKey,
|
|
15615
|
+
}
|
|
15616
|
+
: {}),
|
|
15617
|
+
...this.providerInvocationOptionsForToolCall({
|
|
16179
15618
|
operationKey:
|
|
16180
15619
|
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
16181
|
-
|
|
15620
|
+
requiresExecutionFence: request.requiresExecutionFence,
|
|
16182
15621
|
logicalCallId: request.logicalCallId,
|
|
15622
|
+
force: request.force,
|
|
15623
|
+
}),
|
|
15624
|
+
timeoutMs:
|
|
15625
|
+
typeof request.timeoutMs === 'number' && request.timeoutMs > 0
|
|
15626
|
+
? request.timeoutMs
|
|
15627
|
+
: undefined,
|
|
15628
|
+
retainToolSlot: (release) => {
|
|
15629
|
+
releaseToolSlot = release;
|
|
16183
15630
|
},
|
|
16184
|
-
|
|
16185
|
-
|
|
16186
|
-
|
|
16187
|
-
|
|
16188
|
-
|
|
16189
|
-
|
|
16190
|
-
|
|
16191
|
-
|
|
16192
|
-
|
|
16193
|
-
|
|
16194
|
-
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
16195
|
-
durableCallReceiptKey: this.toolTransportReceiptKey({
|
|
16196
|
-
toolId,
|
|
16197
|
-
resultReceiptKey: request.receiptKey!,
|
|
16198
|
-
providerOperationKey,
|
|
16199
|
-
force: request.force,
|
|
16200
|
-
}),
|
|
16201
|
-
completedResultCache: true,
|
|
16202
|
-
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
16203
|
-
...this.providerForceRefreshOptionsForToolCall({
|
|
16204
|
-
operationKey:
|
|
16205
|
-
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
16206
|
-
force: request.force,
|
|
16207
|
-
requiresExecutionFence: request.requiresExecutionFence,
|
|
16208
|
-
logicalCallId: request.logicalCallId,
|
|
16209
|
-
}),
|
|
16210
|
-
timeoutMs:
|
|
16211
|
-
typeof request.timeoutMs === 'number' && request.timeoutMs > 0
|
|
16212
|
-
? request.timeoutMs
|
|
16213
|
-
: undefined,
|
|
16214
|
-
retainToolSlot: (release) => {
|
|
16215
|
-
releaseToolSlot = release;
|
|
16216
|
-
},
|
|
16217
|
-
},
|
|
16218
|
-
});
|
|
16219
|
-
await resolve(request, result, 'owner');
|
|
16220
|
-
} catch (error) {
|
|
16221
|
-
await reject(request, error);
|
|
16222
|
-
} finally {
|
|
16223
|
-
releaseToolSlot();
|
|
16224
|
-
}
|
|
16225
|
-
},
|
|
16226
|
-
);
|
|
15631
|
+
},
|
|
15632
|
+
});
|
|
15633
|
+
await resolve(request, result, 'owner');
|
|
15634
|
+
} catch (error) {
|
|
15635
|
+
await reject(request, error);
|
|
15636
|
+
} finally {
|
|
15637
|
+
releaseToolSlot();
|
|
15638
|
+
}
|
|
15639
|
+
},
|
|
15640
|
+
);
|
|
16227
15641
|
|
|
16228
|
-
await
|
|
16229
|
-
await Promise.allSettled(followerResults);
|
|
15642
|
+
await Promise.all(cachedResults);
|
|
16230
15643
|
return successful;
|
|
16231
15644
|
}
|
|
16232
15645
|
|
|
@@ -16398,18 +15811,109 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16398
15811
|
'Native batch Tool Call operation identity changed in transit.',
|
|
16399
15812
|
);
|
|
16400
15813
|
}
|
|
15814
|
+
const legacyForceRefreshIdentities = detail.members
|
|
15815
|
+
.filter(
|
|
15816
|
+
(member) =>
|
|
15817
|
+
member.force === true &&
|
|
15818
|
+
member.requiresExecutionFence !== true,
|
|
15819
|
+
)
|
|
15820
|
+
.map((member) =>
|
|
15821
|
+
this.providerInvocationKeyForToolCall({
|
|
15822
|
+
operationKey:
|
|
15823
|
+
this.providerOperationKeyForQueuedToolCall(member),
|
|
15824
|
+
logicalCallId: member.logicalCallId,
|
|
15825
|
+
force: true,
|
|
15826
|
+
}),
|
|
15827
|
+
);
|
|
15828
|
+
const legacyProviderOperationKey =
|
|
15829
|
+
buildDurableToolPayloadProviderIdempotencyKey({
|
|
15830
|
+
orgId: this.#options.orgId,
|
|
15831
|
+
toolId: call.tool.id,
|
|
15832
|
+
executionAuthScopeDigest:
|
|
15833
|
+
detail.members[0]?.executionAuthScopeDigest ?? null,
|
|
15834
|
+
providerActionVersion: call.tool.providerActionVersion,
|
|
15835
|
+
payload: call.payload,
|
|
15836
|
+
forceRefreshIdentities: legacyForceRefreshIdentities,
|
|
15837
|
+
});
|
|
15838
|
+
const legacyFallbackForceRefreshIdentities = detail.members
|
|
15839
|
+
.filter((member) => member.force === true)
|
|
15840
|
+
.map((member) =>
|
|
15841
|
+
this.providerInvocationKeyForToolCall({
|
|
15842
|
+
operationKey:
|
|
15843
|
+
this.providerOperationKeyForQueuedToolCall(member),
|
|
15844
|
+
logicalCallId: member.logicalCallId,
|
|
15845
|
+
force: true,
|
|
15846
|
+
}),
|
|
15847
|
+
);
|
|
15848
|
+
const legacyFallbackProviderOperationKey =
|
|
15849
|
+
buildDurableToolPayloadProviderIdempotencyKey({
|
|
15850
|
+
orgId: this.#options.orgId,
|
|
15851
|
+
toolId: call.tool.id,
|
|
15852
|
+
executionAuthScopeDigest:
|
|
15853
|
+
detail.members[0]?.executionAuthScopeDigest ?? null,
|
|
15854
|
+
providerActionVersion:
|
|
15855
|
+
(
|
|
15856
|
+
await this.#options.getToolActionCacheVersion?.(
|
|
15857
|
+
call.tool.id,
|
|
15858
|
+
)
|
|
15859
|
+
)?.trim() ?? 'unversioned',
|
|
15860
|
+
payload: call.payload,
|
|
15861
|
+
forceRefreshIdentities:
|
|
15862
|
+
legacyFallbackForceRefreshIdentities,
|
|
15863
|
+
});
|
|
16401
15864
|
const execution = await this.callToolExecutionAPI(
|
|
16402
15865
|
call.tool.id,
|
|
16403
15866
|
call.payload,
|
|
16404
15867
|
{
|
|
16405
15868
|
durableCallReceiptKey: this.toolTransportReceiptKey({
|
|
16406
15869
|
toolId: call.tool.id,
|
|
16407
|
-
resultReceiptKey: compatibility.resultReceiptKey,
|
|
16408
15870
|
providerOperationKey: detail.providerOperationKey,
|
|
16409
|
-
force: detail.members.some(
|
|
16410
|
-
(request) => request.force === true,
|
|
16411
|
-
),
|
|
16412
15871
|
}),
|
|
15872
|
+
legacyProviderRetryIdentities: [
|
|
15873
|
+
{
|
|
15874
|
+
transportReceiptKey: detail.members.some(
|
|
15875
|
+
(member) => member.force,
|
|
15876
|
+
)
|
|
15877
|
+
? this.toolTransportReceiptKey({
|
|
15878
|
+
toolId: call.tool.id,
|
|
15879
|
+
providerOperationKey:
|
|
15880
|
+
legacyProviderOperationKey,
|
|
15881
|
+
})
|
|
15882
|
+
: buildDurableToolAggregateReceiptKey({
|
|
15883
|
+
receiptKeys: detail.members.map(
|
|
15884
|
+
(member) =>
|
|
15885
|
+
this.mappedToolResultReceiptPolicy(member)
|
|
15886
|
+
.resultReceiptKey,
|
|
15887
|
+
),
|
|
15888
|
+
prefix: 'batch',
|
|
15889
|
+
aggregateReceiptPrefix:
|
|
15890
|
+
buildDurableToolReceiptPrefix({
|
|
15891
|
+
orgId: this.#options.orgId,
|
|
15892
|
+
toolId: call.tool.id,
|
|
15893
|
+
}),
|
|
15894
|
+
}),
|
|
15895
|
+
providerOperationKey: legacyProviderOperationKey,
|
|
15896
|
+
forceRefreshIdentities: legacyForceRefreshIdentities,
|
|
15897
|
+
},
|
|
15898
|
+
{
|
|
15899
|
+
transportReceiptKey:
|
|
15900
|
+
buildDurableToolAggregateReceiptKey({
|
|
15901
|
+
receiptKeys: detail.members.map(
|
|
15902
|
+
(member) => member.cacheKey,
|
|
15903
|
+
),
|
|
15904
|
+
prefix: 'batch',
|
|
15905
|
+
aggregateReceiptPrefix:
|
|
15906
|
+
buildDurableToolReceiptPrefix({
|
|
15907
|
+
orgId: this.#options.orgId,
|
|
15908
|
+
toolId: call.tool.id,
|
|
15909
|
+
}),
|
|
15910
|
+
}),
|
|
15911
|
+
providerOperationKey:
|
|
15912
|
+
legacyFallbackProviderOperationKey,
|
|
15913
|
+
forceRefreshIdentities:
|
|
15914
|
+
legacyFallbackForceRefreshIdentities,
|
|
15915
|
+
},
|
|
15916
|
+
],
|
|
16413
15917
|
completedResultCache: true,
|
|
16414
15918
|
memoryLedgerRowScopes: detail.members.map((member) => ({
|
|
16415
15919
|
rowKey: member.rowKey,
|
|
@@ -16426,6 +15930,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16426
15930
|
),
|
|
16427
15931
|
executionAuthScopeDigest:
|
|
16428
15932
|
detail.members[0]?.executionAuthScopeDigest ?? null,
|
|
15933
|
+
providerIdempotencyKey: detail.providerOperationKey,
|
|
15934
|
+
providerIdempotencyReceiptKey:
|
|
15935
|
+
compatibility.resultReceiptKey,
|
|
16429
15936
|
...(detail.forceRefreshIdentities.length > 0
|
|
16430
15937
|
? {
|
|
16431
15938
|
providerOperationForceRefreshIdentities:
|
|
@@ -16535,25 +16042,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16535
16042
|
compiled.batchOperation,
|
|
16536
16043
|
)
|
|
16537
16044
|
)?.trim() ?? '';
|
|
16538
|
-
const
|
|
16539
|
-
(
|
|
16540
|
-
this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
|
|
16541
|
-
);
|
|
16542
|
-
const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({
|
|
16543
|
-
receiptKeys,
|
|
16544
|
-
prefix: 'batch',
|
|
16545
|
-
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
16546
|
-
orgId: this.#options.orgId,
|
|
16547
|
-
toolId: compiled.batchOperation,
|
|
16548
|
-
}),
|
|
16549
|
-
});
|
|
16550
|
-
const forceRefreshIdentities = owned.flatMap((request) =>
|
|
16551
|
-
this.providerForceRefreshIdentitiesForToolCall({
|
|
16045
|
+
const forceRefreshIdentities = owned.map((request) =>
|
|
16046
|
+
this.providerInvocationKeyForToolCall({
|
|
16552
16047
|
operationKey:
|
|
16553
16048
|
this.providerOperationKeyForQueuedToolCall(request),
|
|
16554
|
-
force: request.force,
|
|
16555
|
-
requiresExecutionFence: request.requiresExecutionFence,
|
|
16556
16049
|
logicalCallId: request.logicalCallId,
|
|
16050
|
+
force: request.force,
|
|
16557
16051
|
}),
|
|
16558
16052
|
);
|
|
16559
16053
|
const providerOperationKey =
|
|
@@ -16566,6 +16060,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16566
16060
|
payload: compiled.batchPayload,
|
|
16567
16061
|
forceRefreshIdentities,
|
|
16568
16062
|
});
|
|
16063
|
+
const aggregateReceiptKey = this.toolTransportReceiptKey({
|
|
16064
|
+
toolId: compiled.batchOperation,
|
|
16065
|
+
providerOperationKey,
|
|
16066
|
+
});
|
|
16569
16067
|
dispatchDetails.set(aggregateReceiptKey, {
|
|
16570
16068
|
members: owned,
|
|
16571
16069
|
compiled,
|
|
@@ -16825,8 +16323,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16825
16323
|
};
|
|
16826
16324
|
|
|
16827
16325
|
const pendingRequests: ToolCallRequest[] = [];
|
|
16828
|
-
let pendingReceiptClaimsEstablishExecutionFence =
|
|
16829
|
-
this.#options.runtimeReceiptClaimsEstablishExecutionFence === true;
|
|
16830
16326
|
const recoveredRequests: ToolCallRequest[] = [];
|
|
16831
16327
|
for (const req of requests) {
|
|
16832
16328
|
const cached =
|
|
@@ -16851,114 +16347,27 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16851
16347
|
}
|
|
16852
16348
|
recordToolStep(recoveredRequests);
|
|
16853
16349
|
|
|
16854
|
-
const
|
|
16855
|
-
|
|
16856
|
-
owner: ToolCallRequest,
|
|
16857
|
-
result: unknown,
|
|
16858
|
-
): void => {
|
|
16859
|
-
const followers = liveFollowersByOwnerCallId.get(owner.callId);
|
|
16860
|
-
if (!followers || followers.length === 0) return;
|
|
16861
|
-
liveFollowersByOwnerCallId.delete(owner.callId);
|
|
16862
|
-
for (const follower of followers) {
|
|
16863
|
-
const followerReceiptKey = follower.receiptKey?.trim() || null;
|
|
16864
|
-
const followerResult = followerReceiptKey
|
|
16865
|
-
? markToolExecuteResultExecutionOutcome(result, {
|
|
16866
|
-
kind: 'in_flight',
|
|
16867
|
-
receiptKey: followerReceiptKey,
|
|
16868
|
-
attachedToReceiptKey: followerReceiptKey,
|
|
16869
|
-
})
|
|
16870
|
-
: result;
|
|
16871
|
-
if (follower.cacheable !== false) {
|
|
16872
|
-
this.cacheToolResult(toolId, follower.cacheKey, followerResult);
|
|
16873
|
-
}
|
|
16874
|
-
const resolver = this.toolCallResolvers.get(follower.callId);
|
|
16875
|
-
if (resolver) {
|
|
16876
|
-
resolver.resolve(followerResult);
|
|
16877
|
-
this.toolCallResolvers.delete(follower.callId);
|
|
16878
|
-
}
|
|
16879
|
-
this.emitScopedFieldMetaUpdate({
|
|
16880
|
-
rowId: follower.rowId,
|
|
16881
|
-
key: follower.rowKey ?? null,
|
|
16882
|
-
tableNamespace: follower.tableNamespace ?? null,
|
|
16883
|
-
fieldName: follower.fieldName,
|
|
16884
|
-
status: 'cached',
|
|
16885
|
-
rowStatus: 'running',
|
|
16886
|
-
stage: toolId,
|
|
16887
|
-
provider: null,
|
|
16888
|
-
error: null,
|
|
16889
|
-
reused: true,
|
|
16890
|
-
dataPatch: {},
|
|
16891
|
-
});
|
|
16892
|
-
}
|
|
16893
|
-
};
|
|
16894
|
-
const rejectWithLiveFollowers = async (
|
|
16895
|
-
owner: ToolCallRequest,
|
|
16350
|
+
const rejectCall = async (
|
|
16351
|
+
request: ToolCallRequest,
|
|
16896
16352
|
error: unknown,
|
|
16897
16353
|
): Promise<void> => {
|
|
16898
|
-
|
|
16899
|
-
|
|
16900
|
-
|
|
16901
|
-
let releaseFailure: unknown = null;
|
|
16902
|
-
const receiptKey = owner.receiptKey?.trim() || null;
|
|
16903
|
-
try {
|
|
16904
|
-
if (receiptKey && this.durableMappedToolResultsBackedByReceipts) {
|
|
16905
|
-
const leaseId = owner.receiptLeaseId?.trim();
|
|
16906
|
-
if (!leaseId) {
|
|
16907
|
-
throw new Error(
|
|
16908
|
-
`Provider wait cannot release receipt ${receiptKey} without its owned lease.`,
|
|
16909
|
-
);
|
|
16910
|
-
}
|
|
16911
|
-
const released = await this.releaseRuntimeStepReceipt(
|
|
16912
|
-
receiptKey,
|
|
16913
|
-
this.currentReceiptOwnerRunId,
|
|
16914
|
-
leaseId,
|
|
16915
|
-
);
|
|
16916
|
-
if (
|
|
16917
|
-
!released ||
|
|
16918
|
-
(released.status !== 'queued' &&
|
|
16919
|
-
released.status !== 'pending')
|
|
16920
|
-
) {
|
|
16921
|
-
throw new Error(
|
|
16922
|
-
`Provider wait could not release owned receipt ${receiptKey}.`,
|
|
16923
|
-
);
|
|
16924
|
-
}
|
|
16925
|
-
}
|
|
16926
|
-
} catch (failure) {
|
|
16927
|
-
releaseFailure = failure;
|
|
16928
|
-
}
|
|
16929
|
-
// A provider deadline is a nonterminal control signal. The row
|
|
16930
|
-
// catch records waiting metadata after the resolver unwinds; no
|
|
16931
|
-
// follower may see or persist a failed receipt in between. If the
|
|
16932
|
-
// lease could not be released, wake every follower with that loud
|
|
16933
|
-
// infrastructure failure rather than stranding their promises.
|
|
16934
|
-
for (const request of [owner, ...followers]) {
|
|
16935
|
-
const resolver = this.toolCallResolvers.get(request.callId);
|
|
16936
|
-
if (resolver) {
|
|
16937
|
-
resolver.reject(releaseFailure ?? error);
|
|
16938
|
-
this.toolCallResolvers.delete(request.callId);
|
|
16939
|
-
}
|
|
16940
|
-
}
|
|
16941
|
-
if (releaseFailure) throw releaseFailure;
|
|
16942
|
-
return;
|
|
16943
|
-
}
|
|
16944
|
-
const shouldPersistReceiptFailure =
|
|
16945
|
-
!(error instanceof ToolExecuteAuthScopeChangedError) &&
|
|
16946
|
-
!this.persistenceLatch.tripped;
|
|
16947
|
-
for (const request of [owner, ...followers]) {
|
|
16948
|
-
await this.rejectToolCall(toolId, request, error, {
|
|
16949
|
-
persistReceiptFailure: shouldPersistReceiptFailure,
|
|
16950
|
-
});
|
|
16951
|
-
}
|
|
16354
|
+
await this.rejectToolCall(toolId, request, error, {
|
|
16355
|
+
persistReceiptFailure: false,
|
|
16356
|
+
});
|
|
16952
16357
|
};
|
|
16953
|
-
const requestsWithLiveFollowers = (
|
|
16954
|
-
owner: ToolCallRequest,
|
|
16955
|
-
): ToolCallRequest[] => [
|
|
16956
|
-
owner,
|
|
16957
|
-
...(liveFollowersByOwnerCallId.get(owner.callId) ?? []),
|
|
16958
|
-
];
|
|
16959
16358
|
const resolveRuntimeTimeoutMsForRequests = (
|
|
16960
16359
|
requests: ToolCallRequest[],
|
|
16961
16360
|
): number | undefined => {
|
|
16361
|
+
if (
|
|
16362
|
+
requests.some(
|
|
16363
|
+
(request) =>
|
|
16364
|
+
typeof request.timeoutMs !== 'number' ||
|
|
16365
|
+
!Number.isFinite(request.timeoutMs) ||
|
|
16366
|
+
request.timeoutMs <= 0,
|
|
16367
|
+
)
|
|
16368
|
+
) {
|
|
16369
|
+
return undefined;
|
|
16370
|
+
}
|
|
16962
16371
|
const timeoutMs = Math.max(
|
|
16963
16372
|
...requests
|
|
16964
16373
|
.map((request) => request.timeoutMs)
|
|
@@ -16972,76 +16381,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16972
16381
|
);
|
|
16973
16382
|
return timeoutMs > 0 ? timeoutMs : undefined;
|
|
16974
16383
|
};
|
|
16975
|
-
|
|
16976
|
-
|
|
16977
|
-
): number | undefined => {
|
|
16978
|
-
let timeoutMs = 0;
|
|
16979
|
-
for (const owner of owners) {
|
|
16980
|
-
if (
|
|
16981
|
-
typeof owner.timeoutMs !== 'number' ||
|
|
16982
|
-
!Number.isFinite(owner.timeoutMs) ||
|
|
16983
|
-
owner.timeoutMs <= 0
|
|
16984
|
-
) {
|
|
16985
|
-
return undefined;
|
|
16986
|
-
}
|
|
16987
|
-
timeoutMs = Math.max(
|
|
16988
|
-
timeoutMs,
|
|
16989
|
-
resolveRuntimeTimeoutMsForRequests(
|
|
16990
|
-
requestsWithLiveFollowers(owner),
|
|
16991
|
-
) ?? 0,
|
|
16992
|
-
);
|
|
16993
|
-
}
|
|
16994
|
-
return timeoutMs > 0 ? timeoutMs : undefined;
|
|
16995
|
-
};
|
|
16996
|
-
|
|
16997
|
-
// Cacheable map members cross the real receipt-cohort boundary before
|
|
16998
|
-
// the legacy claim loop. The scalar and native Modules share member
|
|
16999
|
-
// receipt ownership; native batches retain their separate aggregate
|
|
17000
|
-
// physical-operation receipt behind that seam. Always-fresh calls
|
|
17001
|
-
// intentionally have no durable member receipt and stay below.
|
|
16384
|
+
// The cohort reads completed results and publishes successful misses.
|
|
16385
|
+
// Always-fresh calls have no durable member result and stay below.
|
|
17002
16386
|
const operationStrategy =
|
|
17003
16387
|
this.#options.getBatchOperationStrategy?.(toolId) ?? null;
|
|
17004
|
-
if (
|
|
17005
|
-
operationStrategy &&
|
|
17006
|
-
this.durableMappedToolResultsBackedByReceipts
|
|
17007
|
-
) {
|
|
17008
|
-
const cohortRequests = pendingRequests.filter(
|
|
17009
|
-
(request) =>
|
|
17010
|
-
request.cacheable !== false &&
|
|
17011
|
-
request.requiresExecutionFence !== true &&
|
|
17012
|
-
Boolean(request.receiptKey?.trim()),
|
|
17013
|
-
);
|
|
17014
|
-
if (cohortRequests.length > 0) {
|
|
17015
|
-
const cohortCallIds = new Set(
|
|
17016
|
-
cohortRequests.map((request) => request.callId),
|
|
17017
|
-
);
|
|
17018
|
-
pendingRequests.splice(
|
|
17019
|
-
0,
|
|
17020
|
-
pendingRequests.length,
|
|
17021
|
-
...pendingRequests.filter(
|
|
17022
|
-
(request) => !cohortCallIds.has(request.callId),
|
|
17023
|
-
),
|
|
17024
|
-
);
|
|
17025
|
-
const successful =
|
|
17026
|
-
await this.executeCacheableNativeMapToolCallCohort({
|
|
17027
|
-
toolId,
|
|
17028
|
-
requests: cohortRequests,
|
|
17029
|
-
strategy: operationStrategy,
|
|
17030
|
-
});
|
|
17031
|
-
for (const request of successful) {
|
|
17032
|
-
successfulLiveStepCallIds.add(request.callId);
|
|
17033
|
-
}
|
|
17034
|
-
recordToolStep(cohortRequests);
|
|
17035
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
17036
|
-
}
|
|
17037
|
-
} else if (
|
|
17038
|
-
!operationStrategy &&
|
|
17039
|
-
this.durableMappedToolResultsBackedByReceipts
|
|
17040
|
-
) {
|
|
16388
|
+
if (this.durableMappedToolResultsBackedByReceipts) {
|
|
17041
16389
|
const cohortRequests = pendingRequests.filter(
|
|
17042
16390
|
(request) =>
|
|
17043
16391
|
request.cacheable !== false &&
|
|
17044
|
-
request.requiresExecutionFence !== true &&
|
|
17045
16392
|
Boolean(request.receiptKey?.trim()),
|
|
17046
16393
|
);
|
|
17047
16394
|
if (cohortRequests.length > 0) {
|
|
@@ -17055,1181 +16402,363 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
17055
16402
|
(request) => !cohortCallIds.has(request.callId),
|
|
17056
16403
|
),
|
|
17057
16404
|
);
|
|
17058
|
-
const successful =
|
|
17059
|
-
await this.
|
|
17060
|
-
|
|
17061
|
-
|
|
17062
|
-
|
|
17063
|
-
|
|
16405
|
+
const successful = operationStrategy
|
|
16406
|
+
? await this.executeCacheableNativeMapToolCallCohort({
|
|
16407
|
+
toolId,
|
|
16408
|
+
requests: cohortRequests,
|
|
16409
|
+
strategy: operationStrategy,
|
|
16410
|
+
})
|
|
16411
|
+
: await this.executeCacheableNonNativeMapToolCallCohort({
|
|
16412
|
+
toolId,
|
|
16413
|
+
requests: cohortRequests,
|
|
16414
|
+
});
|
|
16415
|
+
for (const request of successful)
|
|
17064
16416
|
successfulLiveStepCallIds.add(request.callId);
|
|
17065
|
-
}
|
|
17066
16417
|
recordToolStep(cohortRequests);
|
|
17067
16418
|
this.#options.onBatchComplete?.(this.checkpoint);
|
|
17068
16419
|
}
|
|
17069
16420
|
}
|
|
17070
16421
|
|
|
17071
|
-
|
|
17072
|
-
|
|
17073
|
-
|
|
17074
|
-
|
|
17075
|
-
|
|
17076
|
-
|
|
17077
|
-
|
|
17078
|
-
|
|
17079
|
-
const durableExistingRunningState: {
|
|
17080
|
-
settlement: Promise<
|
|
17081
|
-
{ status: 'fulfilled' } | { status: 'rejected'; reason: unknown }
|
|
17082
|
-
> | null;
|
|
17083
|
-
} = { settlement: null };
|
|
17084
|
-
const durableExistingRunningAbort = new AbortController();
|
|
17085
|
-
// The group's size before the turn. The turn body splices and pushes
|
|
17086
|
-
// `pendingRequests`, so the count read here is the only one that says
|
|
17087
|
-
// how many queued calls this claim turn is blocking.
|
|
17088
|
-
const claimTurnLogicalCalls = pendingRequests.length;
|
|
17089
|
-
if (
|
|
17090
|
-
claimTurnLogicalCalls > 0 &&
|
|
17091
|
-
(this.#options.claimRuntimeStepReceipts ||
|
|
17092
|
-
this.#options.claimRuntimeStepReceipt)
|
|
17093
|
-
) {
|
|
17094
|
-
const requestsByReceiptKey = new Map<
|
|
17095
|
-
string,
|
|
17096
|
-
{
|
|
17097
|
-
requests: ToolCallRequest[];
|
|
17098
|
-
forceRefresh: boolean;
|
|
17099
|
-
forceFailedRefresh: boolean;
|
|
17100
|
-
}
|
|
17101
|
-
>();
|
|
17102
|
-
const localOnlyRequests: ToolCallRequest[] = [];
|
|
17103
|
-
for (const request of pendingRequests) {
|
|
17104
|
-
const receiptKey = request.receiptKey?.trim() || null;
|
|
17105
|
-
if (!receiptKey) {
|
|
17106
|
-
localOnlyRequests.push(request);
|
|
17107
|
-
continue;
|
|
17108
|
-
}
|
|
17109
|
-
const group = requestsByReceiptKey.get(receiptKey) ?? {
|
|
17110
|
-
requests: [],
|
|
17111
|
-
forceRefresh: false,
|
|
17112
|
-
forceFailedRefresh: false,
|
|
17113
|
-
};
|
|
17114
|
-
group.requests.push(request);
|
|
17115
|
-
group.forceRefresh ||= request.force === true;
|
|
17116
|
-
group.forceFailedRefresh ||= request.forceFailedRefresh === true;
|
|
17117
|
-
requestsByReceiptKey.set(receiptKey, group);
|
|
17118
|
-
}
|
|
17119
|
-
const normalReceiptKeys = [...requestsByReceiptKey]
|
|
17120
|
-
.filter(
|
|
17121
|
-
([, group]) => !group.forceRefresh && !group.forceFailedRefresh,
|
|
17122
|
-
)
|
|
17123
|
-
.map(([receiptKey]) => receiptKey);
|
|
17124
|
-
const forcedReceiptKeys = [...requestsByReceiptKey]
|
|
17125
|
-
.filter(([, group]) => group.forceRefresh)
|
|
17126
|
-
.map(([receiptKey]) => receiptKey);
|
|
17127
|
-
const failedRefreshReceiptKeys = [...requestsByReceiptKey]
|
|
17128
|
-
.filter(
|
|
17129
|
-
([, group]) => !group.forceRefresh && group.forceFailedRefresh,
|
|
17130
|
-
)
|
|
17131
|
-
.map(([receiptKey]) => receiptKey);
|
|
17132
|
-
const claimStartedAt = Date.now();
|
|
17133
|
-
const forcedExisting = forcedReceiptKeys.length
|
|
17134
|
-
? await this.getRuntimeStepReceipts(forcedReceiptKeys, {
|
|
17135
|
-
hydrateOutput: false,
|
|
17136
|
-
})
|
|
17137
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
17138
|
-
// A forced refresh must never duplicate a provider call owned by a
|
|
17139
|
-
// live lease. It can, however, immediately claim pending/unleased
|
|
17140
|
-
// work and any expired lease. Waiting first for those receipts turns
|
|
17141
|
-
// an already-recoverable interruption into a five-minute stall.
|
|
17142
|
-
const forcedLiveReceiptKeys = new Set(
|
|
17143
|
-
[...forcedExisting]
|
|
17144
|
-
.filter(([, receipt]) => {
|
|
17145
|
-
if (
|
|
17146
|
-
receipt.status !== 'queued' &&
|
|
17147
|
-
receipt.status !== 'pending' &&
|
|
17148
|
-
receipt.status !== 'running'
|
|
17149
|
-
) {
|
|
17150
|
-
return false;
|
|
17151
|
-
}
|
|
17152
|
-
if (!receipt.leaseId?.trim()) return false;
|
|
17153
|
-
const expiresAt = receipt.leaseExpiresAt
|
|
17154
|
-
? Date.parse(receipt.leaseExpiresAt)
|
|
17155
|
-
: Number.NaN;
|
|
17156
|
-
// Keep an unparseable leased receipt conservative. The
|
|
17157
|
-
// fenced claim remains the recovery path after the wait.
|
|
17158
|
-
return !Number.isFinite(expiresAt) || expiresAt > Date.now();
|
|
17159
|
-
})
|
|
17160
|
-
.map(([receiptKey]) => receiptKey),
|
|
17161
|
-
);
|
|
17162
|
-
const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
|
|
17163
|
-
(receiptKey) => !forcedLiveReceiptKeys.has(receiptKey),
|
|
17164
|
-
);
|
|
17165
|
-
const claims =
|
|
17166
|
-
normalReceiptKeys.length > 0
|
|
17167
|
-
? await this.claimRuntimeStepReceipts(
|
|
17168
|
-
normalReceiptKeys,
|
|
17169
|
-
this.currentReceiptOwnerRunId,
|
|
17170
|
-
false,
|
|
17171
|
-
false,
|
|
17172
|
-
false,
|
|
17173
|
-
false,
|
|
17174
|
-
)
|
|
17175
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
17176
|
-
const forcedClaims =
|
|
17177
|
-
claimableForcedReceiptKeys.length > 0
|
|
17178
|
-
? await this.claimRuntimeStepReceipts(
|
|
17179
|
-
claimableForcedReceiptKeys,
|
|
17180
|
-
this.currentReceiptOwnerRunId,
|
|
17181
|
-
false,
|
|
17182
|
-
true,
|
|
17183
|
-
false,
|
|
17184
|
-
)
|
|
17185
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
17186
|
-
const failedRefreshClaims =
|
|
17187
|
-
failedRefreshReceiptKeys.length > 0
|
|
17188
|
-
? await this.claimRuntimeStepReceipts(
|
|
17189
|
-
failedRefreshReceiptKeys,
|
|
17190
|
-
this.currentReceiptOwnerRunId,
|
|
17191
|
-
false,
|
|
17192
|
-
false,
|
|
17193
|
-
true,
|
|
17194
|
-
false,
|
|
17195
|
-
)
|
|
17196
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
17197
|
-
for (const [receiptKey, claim] of forcedClaims) {
|
|
17198
|
-
claims.set(receiptKey, claim);
|
|
17199
|
-
}
|
|
17200
|
-
for (const [receiptKey, claim] of failedRefreshClaims) {
|
|
17201
|
-
claims.set(receiptKey, claim);
|
|
17202
|
-
}
|
|
17203
|
-
const safeFailedReceiptKeys = [...claims]
|
|
17204
|
-
.filter(([, claim]) => this.canRetryWithoutProviderDispatch(claim))
|
|
17205
|
-
.map(([receiptKey]) => receiptKey);
|
|
17206
|
-
if (safeFailedReceiptKeys.length > 0) {
|
|
17207
|
-
const refreshed = await this.claimRuntimeStepReceipts(
|
|
17208
|
-
safeFailedReceiptKeys,
|
|
17209
|
-
this.currentReceiptOwnerRunId,
|
|
17210
|
-
false,
|
|
17211
|
-
false,
|
|
17212
|
-
true,
|
|
17213
|
-
false,
|
|
17214
|
-
);
|
|
17215
|
-
for (const [receiptKey, claim] of refreshed) {
|
|
17216
|
-
claims.set(receiptKey, claim);
|
|
17217
|
-
}
|
|
17218
|
-
}
|
|
17219
|
-
// Claiming is a state transition, not a reason to expand a
|
|
17220
|
-
// body. Fresh receipts contain no result, yet the old path took
|
|
17221
|
-
// the hydration lane before it learned that fact and charged a
|
|
17222
|
-
// multi-MB row prior to every one. Claims now arrive as bounded
|
|
17223
|
-
// manifests; only a terminal winner redeems its known-size body
|
|
17224
|
-
// under a second, byte-priced turn.
|
|
17225
|
-
const terminalClaimKeys = [...claims]
|
|
17226
|
-
.filter(
|
|
17227
|
-
([, claim]) =>
|
|
17228
|
-
(claim.status === 'completed' || claim.status === 'skipped') &&
|
|
17229
|
-
claim.output === undefined,
|
|
17230
|
-
)
|
|
17231
|
-
.map(([receiptKey]) => receiptKey);
|
|
17232
|
-
if (terminalClaimKeys.length > 0) {
|
|
17233
|
-
const hasBoundedClaimManifests = terminalClaimKeys.every(
|
|
17234
|
-
(receiptKey) => {
|
|
17235
|
-
const claim = claims.get(receiptKey);
|
|
17236
|
-
return (
|
|
17237
|
-
Number.isSafeInteger(claim?.outputByteLength) &&
|
|
17238
|
-
(claim?.outputByteLength ?? -1) >= 0
|
|
17239
|
-
);
|
|
17240
|
-
},
|
|
17241
|
-
);
|
|
17242
|
-
const expectedClaimBytes = hasBoundedClaimManifests
|
|
17243
|
-
? terminalClaimKeys.reduce(
|
|
17244
|
-
(total, receiptKey) =>
|
|
17245
|
-
total + (claims.get(receiptKey)?.outputByteLength ?? 0),
|
|
17246
|
-
0,
|
|
17247
|
-
)
|
|
17248
|
-
: undefined;
|
|
17249
|
-
const hydratedClaims = await this.withRuntimeReceiptHydrationTurn(
|
|
17250
|
-
async (hydrationTurn) => {
|
|
17251
|
-
const receipts =
|
|
17252
|
-
await this.getRuntimeStepReceipts(terminalClaimKeys);
|
|
17253
|
-
hydrationTurn.settleBytes(
|
|
17254
|
-
runtimeStepReceiptPayloadBytes(receipts.values()),
|
|
17255
|
-
);
|
|
17256
|
-
return receipts;
|
|
17257
|
-
},
|
|
17258
|
-
{
|
|
17259
|
-
logicalCalls: claimTurnLogicalCalls,
|
|
17260
|
-
receiptKeys: terminalClaimKeys.length,
|
|
17261
|
-
...(expectedClaimBytes !== undefined
|
|
17262
|
-
? { expectedBytes: expectedClaimBytes }
|
|
17263
|
-
: {}),
|
|
17264
|
-
source: 'map_legacy_claim',
|
|
17265
|
-
},
|
|
17266
|
-
);
|
|
17267
|
-
for (const [receiptKey, receipt] of hydratedClaims) {
|
|
17268
|
-
claims.set(receiptKey, receipt);
|
|
17269
|
-
}
|
|
17270
|
-
}
|
|
17271
|
-
// Every get/claim round trip this dispatch group paid before any
|
|
17272
|
-
// provider work could start.
|
|
17273
|
-
this.resourceGovernor.observe({
|
|
17274
|
-
receiptClaimMs: Date.now() - claimStartedAt,
|
|
17275
|
-
});
|
|
17276
|
-
// Claims are compact state transitions. They do not own output
|
|
17277
|
-
// bytes and must not wait behind the hydration lane's global
|
|
17278
|
-
// memory headroom; a terminal manifest above is the only path
|
|
17279
|
-
// that opens a byte-admitted hydration turn.
|
|
17280
|
-
const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
|
|
17281
|
-
const durableRecoveredRequests: ToolCallRequest[] = [];
|
|
17282
|
-
const resolveRequestsFromReceipt = async (
|
|
17283
|
-
requestsForKey: ToolCallRequest[],
|
|
17284
|
-
receipt: RuntimeStepReceipt,
|
|
17285
|
-
source: 'cache' | 'in_flight',
|
|
17286
|
-
): Promise<void> => {
|
|
17287
|
-
const request = requestsForKey[0];
|
|
17288
|
-
if (!request) return;
|
|
17289
|
-
const receiptKey = request.receiptKey?.trim() || null;
|
|
17290
|
-
if (!receiptKey) {
|
|
17291
|
-
throw new Error(`${source} tool recovery needs receipt key.`);
|
|
17292
|
-
}
|
|
17293
|
-
const wrapped = await this.wrapToolExecutionResult({
|
|
17294
|
-
toolId,
|
|
17295
|
-
status:
|
|
17296
|
-
receipt.output === null || receipt.output === undefined
|
|
17297
|
-
? 'no_result'
|
|
17298
|
-
: 'completed',
|
|
17299
|
-
result: this.runtimeReceiptOutput(receipt),
|
|
17300
|
-
requestInput: request.input,
|
|
17301
|
-
execution: toolExecutionMetadataForOutcome({
|
|
17302
|
-
kind: source,
|
|
17303
|
-
cacheKey: request.cacheKey,
|
|
17304
|
-
receiptKey,
|
|
17305
|
-
attachedToReceiptKey: receiptKey,
|
|
17306
|
-
}),
|
|
17307
|
-
});
|
|
17308
|
-
this.cacheToolResult(toolId, request.cacheKey, wrapped);
|
|
17309
|
-
for (const waitingRequest of requestsForKey) {
|
|
17310
|
-
successfulLiveStepCallIds.add(waitingRequest.callId);
|
|
17311
|
-
const resolver = this.toolCallResolvers.get(
|
|
17312
|
-
waitingRequest.callId,
|
|
17313
|
-
);
|
|
17314
|
-
if (resolver) {
|
|
17315
|
-
resolver.resolve(wrapped);
|
|
17316
|
-
this.toolCallResolvers.delete(waitingRequest.callId);
|
|
17317
|
-
}
|
|
17318
|
-
this.emitScopedFieldMetaUpdate({
|
|
17319
|
-
rowId: waitingRequest.rowId,
|
|
17320
|
-
key: waitingRequest.rowKey ?? null,
|
|
17321
|
-
tableNamespace: waitingRequest.tableNamespace ?? null,
|
|
17322
|
-
fieldName: waitingRequest.fieldName,
|
|
17323
|
-
status: 'cached',
|
|
17324
|
-
rowStatus: 'running',
|
|
17325
|
-
stage: toolId,
|
|
17326
|
-
provider: null,
|
|
17327
|
-
error: null,
|
|
17328
|
-
reused: true,
|
|
17329
|
-
dataPatch: {},
|
|
17330
|
-
});
|
|
17331
|
-
}
|
|
17332
|
-
};
|
|
17333
|
-
const claimOwnerWithLiveFollowers = (
|
|
17334
|
-
owner: ToolCallRequest | undefined,
|
|
17335
|
-
followers: ToolCallRequest[],
|
|
17336
|
-
): void => {
|
|
17337
|
-
if (!owner) return;
|
|
17338
|
-
claimedRequests.push(owner);
|
|
17339
|
-
if (followers.length > 0) {
|
|
17340
|
-
liveFollowersByOwnerCallId.set(owner.callId, followers);
|
|
17341
|
-
}
|
|
17342
|
-
};
|
|
17343
|
-
const resolveReceiptWaitMaxAttempts = (
|
|
17344
|
-
requestsForKey: ToolCallRequest[],
|
|
17345
|
-
): number | undefined =>
|
|
17346
|
-
requestsForKey.length > 0
|
|
17347
|
-
? Math.max(
|
|
17348
|
-
...requestsForKey.map((request) =>
|
|
17349
|
-
resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
17350
|
-
typeof request.receiptWaitMs === 'number'
|
|
17351
|
-
? { max_wait_ms: request.receiptWaitMs }
|
|
17352
|
-
: request.input,
|
|
17353
|
-
),
|
|
17354
|
-
),
|
|
17355
|
-
)
|
|
17356
|
-
: undefined;
|
|
17357
|
-
const queueExistingRunningReceiptWait = (
|
|
17358
|
-
receiptKey: string,
|
|
17359
|
-
requestsForKey: ToolCallRequest[],
|
|
17360
|
-
forceAfterWait = false,
|
|
17361
|
-
skipWait = false,
|
|
17362
|
-
): void => {
|
|
17363
|
-
durableExistingRunningWaits.push({
|
|
17364
|
-
receiptKey,
|
|
17365
|
-
requestsForKey,
|
|
17366
|
-
forceAfterWait,
|
|
17367
|
-
waitMaxAttempts: resolveReceiptWaitMaxAttempts(requestsForKey),
|
|
17368
|
-
skipWait,
|
|
16422
|
+
if (pendingRequests.length > 0) {
|
|
16423
|
+
const strategy = operationStrategy;
|
|
16424
|
+
|
|
16425
|
+
if (strategy) {
|
|
16426
|
+
const compiledBatches = compileRequestsWithStrategy({
|
|
16427
|
+
requests: pendingRequests,
|
|
16428
|
+
strategy,
|
|
16429
|
+
getPayload: (request: ToolCallRequest) => request.input,
|
|
17369
16430
|
});
|
|
17370
|
-
|
|
17371
|
-
|
|
17372
|
-
|
|
17373
|
-
|
|
17374
|
-
|
|
17375
|
-
|
|
17376
|
-
|
|
17377
|
-
|
|
17378
|
-
|
|
17379
|
-
|
|
17380
|
-
|
|
17381
|
-
|
|
17382
|
-
|
|
17383
|
-
|
|
17384
|
-
|
|
17385
|
-
|
|
17386
|
-
);
|
|
17387
|
-
return;
|
|
17388
|
-
}
|
|
17389
|
-
} else if (waitResult?.skipWait !== true) {
|
|
17390
|
-
const waitMaxAttempts =
|
|
17391
|
-
resolveReceiptWaitMaxAttempts(requestsForKey);
|
|
17392
|
-
try {
|
|
17393
|
-
const completed = await this.waitForCompletedRuntimeToolReceipt(
|
|
17394
|
-
receiptKey,
|
|
17395
|
-
waitMaxAttempts,
|
|
17396
|
-
);
|
|
17397
|
-
if (!forceAfterWait) {
|
|
17398
|
-
await resolveRequestsFromReceipt(
|
|
17399
|
-
requestsForKey,
|
|
17400
|
-
completed,
|
|
17401
|
-
'in_flight',
|
|
17402
|
-
);
|
|
17403
|
-
return;
|
|
17404
|
-
}
|
|
17405
|
-
} catch (error) {
|
|
17406
|
-
if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
|
|
17407
|
-
for (const request of requestsForKey) {
|
|
17408
|
-
await this.rejectToolCall(toolId, request, error, {
|
|
17409
|
-
persistReceiptFailure: false,
|
|
17410
|
-
});
|
|
17411
|
-
}
|
|
17412
|
-
return;
|
|
17413
|
-
}
|
|
17414
|
-
}
|
|
17415
|
-
}
|
|
17416
|
-
const reclaimed = (
|
|
17417
|
-
await this.claimRuntimeStepReceipts(
|
|
17418
|
-
[receiptKey],
|
|
17419
|
-
this.currentReceiptOwnerRunId,
|
|
17420
|
-
true,
|
|
17421
|
-
forceAfterWait,
|
|
17422
|
-
)
|
|
17423
|
-
).get(receiptKey);
|
|
17424
|
-
if (
|
|
17425
|
-
reclaimed?.status === 'completed' ||
|
|
17426
|
-
reclaimed?.status === 'skipped'
|
|
17427
|
-
) {
|
|
17428
|
-
await resolveRequestsFromReceipt(
|
|
17429
|
-
requestsForKey,
|
|
17430
|
-
reclaimed,
|
|
17431
|
-
'cache',
|
|
17432
|
-
);
|
|
17433
|
-
durableRecoveredRequests.push(...requestsForKey);
|
|
17434
|
-
return;
|
|
17435
|
-
}
|
|
17436
|
-
if (reclaimed?.status === 'failed') {
|
|
17437
|
-
for (const request of requestsForKey) {
|
|
17438
|
-
await this.rejectToolCall(
|
|
17439
|
-
toolId,
|
|
17440
|
-
request,
|
|
17441
|
-
runtimeReceiptFailureError(
|
|
17442
|
-
reclaimed,
|
|
17443
|
-
'Durable tool call failed',
|
|
17444
|
-
this.currentToolErrorSchemaVersion,
|
|
17445
|
-
),
|
|
17446
|
-
);
|
|
17447
|
-
}
|
|
17448
|
-
return;
|
|
17449
|
-
}
|
|
17450
|
-
if (this.isOwnedClaimedRuntimeReceipt(reclaimed)) {
|
|
17451
|
-
const [owner, ...waiters] = requestsForKey;
|
|
17452
|
-
if (!owner) return;
|
|
17453
|
-
if (!reclaimed.leaseId) {
|
|
17454
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
17455
|
-
receiptKey,
|
|
17456
|
-
runId: this.currentReceiptOwnerRunId,
|
|
17457
|
-
leaseId: 'missing',
|
|
17458
|
-
});
|
|
17459
|
-
}
|
|
17460
|
-
owner.receiptLeaseId = reclaimed.leaseId ?? null;
|
|
17461
|
-
owner.receiptLeaseExpiresAt = reclaimed.leaseExpiresAt ?? null;
|
|
17462
|
-
if (waiters.length > 0) {
|
|
17463
|
-
liveFollowersByOwnerCallId.set(owner.callId, waiters);
|
|
17464
|
-
}
|
|
17465
|
-
try {
|
|
17466
|
-
// Circuit breaker: a receipt reclaimed after its owner's lease
|
|
17467
|
-
// lapsed is still a fresh provider call, and the latch may
|
|
17468
|
-
// have tripped while this group waited for that lease.
|
|
16431
|
+
|
|
16432
|
+
const batchParallelismCeiling =
|
|
16433
|
+
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
16434
|
+
const batchSize =
|
|
16435
|
+
compiledBatches.length > 0
|
|
16436
|
+
? await this.resourceGovernor.suggestedToolParallelism(
|
|
16437
|
+
compiledBatches[0]!.batchOperation,
|
|
16438
|
+
batchParallelismCeiling,
|
|
16439
|
+
)
|
|
16440
|
+
: batchParallelismCeiling;
|
|
16441
|
+
await executeChunkedRequests({
|
|
16442
|
+
requests: compiledBatches,
|
|
16443
|
+
batchSize,
|
|
16444
|
+
execute: async (batch) => {
|
|
16445
|
+
// Circuit breaker: skip dispatching this batch's provider call
|
|
16446
|
+
// once a persistence failure has occurred in this run.
|
|
17469
16447
|
if (this.persistenceLatch.tripped) {
|
|
17470
16448
|
this.persistenceLatch.preventedCallCount +=
|
|
17471
|
-
|
|
16449
|
+
batch.memberRequests.length;
|
|
17472
16450
|
throw new RuntimePersistenceCircuitOpenError(
|
|
17473
16451
|
this.persistenceLatch,
|
|
17474
16452
|
);
|
|
17475
16453
|
}
|
|
17476
|
-
const
|
|
17477
|
-
|
|
17478
|
-
|
|
17479
|
-
|
|
17480
|
-
|
|
17481
|
-
|
|
17482
|
-
|
|
17483
|
-
|
|
17484
|
-
apiOptions: {
|
|
17485
|
-
beforeProviderCall: () =>
|
|
17486
|
-
this.assertRuntimeToolReceiptOwnership([owner]),
|
|
17487
|
-
durableCallReceiptKey: receiptKey,
|
|
17488
|
-
playNodeScope: playNodeScopeForToolCallRequest(owner),
|
|
17489
|
-
executionAuthScopeDigest: owner.executionAuthScopeDigest,
|
|
17490
|
-
receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
|
|
17491
|
-
heartbeatReceipt: () =>
|
|
17492
|
-
this.renewRuntimeToolReceiptOwnership([owner]),
|
|
17493
|
-
providerIdempotencyKey:
|
|
17494
|
-
this.providerIdempotencyKeyForToolCall({
|
|
17495
|
-
operationKey:
|
|
17496
|
-
this.providerOperationKeyForQueuedToolCall(owner),
|
|
17497
|
-
force: owner.force === true,
|
|
17498
|
-
logicalCallId: owner.logicalCallId,
|
|
17499
|
-
}),
|
|
17500
|
-
...this.providerForceRefreshOptionsForToolCall({
|
|
17501
|
-
operationKey:
|
|
17502
|
-
this.providerOperationKeyForQueuedToolCall(owner),
|
|
17503
|
-
force: owner.force,
|
|
17504
|
-
requiresExecutionFence: owner.requiresExecutionFence,
|
|
17505
|
-
logicalCallId: owner.logicalCallId,
|
|
17506
|
-
}),
|
|
17507
|
-
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
|
|
17508
|
-
owner,
|
|
17509
|
-
]),
|
|
17510
|
-
},
|
|
17511
|
-
},
|
|
17512
|
-
);
|
|
17513
|
-
const result = await this.resolveToolCall(
|
|
17514
|
-
toolId,
|
|
17515
|
-
owner,
|
|
17516
|
-
execution,
|
|
17517
|
-
);
|
|
17518
|
-
if (result != null) {
|
|
17519
|
-
successfulLiveStepCallIds.add(owner.callId);
|
|
17520
|
-
}
|
|
17521
|
-
resolveLiveFollowers(owner, result);
|
|
17522
|
-
recordToolStep([owner]);
|
|
17523
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
17524
|
-
} catch (error) {
|
|
17525
|
-
await rejectWithLiveFollowers(owner, error);
|
|
17526
|
-
}
|
|
17527
|
-
return;
|
|
17528
|
-
}
|
|
17529
|
-
for (const request of requestsForKey) {
|
|
17530
|
-
await this.rejectToolCall(
|
|
17531
|
-
toolId,
|
|
17532
|
-
request,
|
|
17533
|
-
new RuntimeReceiptWaitTimeoutError(receiptKey),
|
|
17534
|
-
{ persistReceiptFailure: false },
|
|
17535
|
-
);
|
|
17536
|
-
}
|
|
17537
|
-
return;
|
|
17538
|
-
};
|
|
17539
|
-
const reclaimExistingRunningReceipts = async (
|
|
17540
|
-
waits: Array<(typeof durableExistingRunningWaits)[number]>,
|
|
17541
|
-
): Promise<void> => {
|
|
17542
|
-
if (waits.length === 0) return;
|
|
17543
|
-
const settlements = await Promise.allSettled(
|
|
17544
|
-
waits.map((wait) =>
|
|
17545
|
-
reclaimExistingRunningReceiptAfterWait(
|
|
17546
|
-
wait.receiptKey,
|
|
17547
|
-
wait.requestsForKey,
|
|
17548
|
-
wait.forceAfterWait,
|
|
17549
|
-
{ skipWait: true },
|
|
17550
|
-
),
|
|
17551
|
-
),
|
|
17552
|
-
);
|
|
17553
|
-
const rejected = settlements.find(
|
|
17554
|
-
(settlement) => settlement.status === 'rejected',
|
|
17555
|
-
);
|
|
17556
|
-
if (rejected?.status === 'rejected') throw rejected.reason;
|
|
17557
|
-
};
|
|
17558
|
-
const processExistingRunningReceiptWaits =
|
|
17559
|
-
async (): Promise<void> => {
|
|
17560
|
-
if (durableExistingRunningWaits.length === 0) return;
|
|
17561
|
-
const waitSignal = durableExistingRunningAbort.signal;
|
|
17562
|
-
// A claim that came back `existing` behind an expired lease is
|
|
17563
|
-
// not blocked on live work; it is holding a dead run's row.
|
|
17564
|
-
// Reclaim those now so this run executes the call itself, and
|
|
17565
|
-
// poll only the leases that are genuinely still alive.
|
|
17566
|
-
const expiredLeaseWaits = durableExistingRunningWaits.filter(
|
|
17567
|
-
(wait) => wait.skipWait === true,
|
|
17568
|
-
);
|
|
17569
|
-
const pendingWaits = new Map(
|
|
17570
|
-
durableExistingRunningWaits
|
|
17571
|
-
.filter((wait) => wait.skipWait !== true)
|
|
17572
|
-
.map((wait) => [wait.receiptKey, wait]),
|
|
17573
|
-
);
|
|
17574
|
-
const parkedAtMs = Date.now();
|
|
17575
|
-
for (const wait of durableExistingRunningWaits) {
|
|
17576
|
-
this.#runtimeReceiptWaitParking.set(wait.receiptKey, {
|
|
17577
|
-
startedAtMs: parkedAtMs,
|
|
17578
|
-
rows: wait.requestsForKey.length,
|
|
17579
|
-
stage:
|
|
17580
|
-
wait.skipWait === true
|
|
17581
|
-
? 'expired_lease_reclaim'
|
|
17582
|
-
: 'receipt_lease_wait',
|
|
17583
|
-
});
|
|
17584
|
-
}
|
|
17585
|
-
const pollWaits = async (): Promise<void> => {
|
|
17586
|
-
if (pendingWaits.size === 0) return;
|
|
17587
|
-
const waitMaxAttempts =
|
|
17588
|
-
Math.max(
|
|
17589
|
-
...[...pendingWaits.values()].map(
|
|
17590
|
-
(wait) => wait.waitMaxAttempts ?? 0,
|
|
17591
|
-
),
|
|
17592
|
-
) || DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
|
|
17593
|
-
// The backoff cap means the attempt count no longer states a
|
|
17594
|
-
// wall time. This deadline is the budget those attempts were
|
|
17595
|
-
// meant to express; crossing it falls through to the same
|
|
17596
|
-
// reclaim path an expired lease takes.
|
|
17597
|
-
const waitDeadline =
|
|
17598
|
-
Date.now() +
|
|
17599
|
-
runtimeReceiptWaitBudgetMs(
|
|
17600
|
-
waitMaxAttempts,
|
|
17601
|
-
DURABLE_RECEIPT_WAIT_DELAY_MS,
|
|
16454
|
+
const aggregateForceRefreshIdentities =
|
|
16455
|
+
batch.memberRequests.map((request) =>
|
|
16456
|
+
this.providerInvocationKeyForToolCall({
|
|
16457
|
+
operationKey:
|
|
16458
|
+
this.providerOperationKeyForQueuedToolCall(request),
|
|
16459
|
+
logicalCallId: request.logicalCallId,
|
|
16460
|
+
force: request.force,
|
|
16461
|
+
}),
|
|
17602
16462
|
);
|
|
17603
|
-
|
|
17604
|
-
|
|
17605
|
-
|
|
17606
|
-
|
|
17607
|
-
|
|
17608
|
-
.
|
|
16463
|
+
const legacyForceRefreshIdentities = batch.memberRequests
|
|
16464
|
+
.filter((request) => request.force === true)
|
|
16465
|
+
.map((request) =>
|
|
16466
|
+
this.providerInvocationKeyForToolCall({
|
|
16467
|
+
operationKey:
|
|
16468
|
+
this.providerOperationKeyForQueuedToolCall(request),
|
|
16469
|
+
logicalCallId: request.logicalCallId,
|
|
16470
|
+
force: true,
|
|
16471
|
+
}),
|
|
17609
16472
|
);
|
|
17610
|
-
|
|
17611
|
-
|
|
17612
|
-
|
|
17613
|
-
|
|
17614
|
-
|
|
17615
|
-
|
|
17616
|
-
|
|
17617
|
-
|
|
17618
|
-
|
|
17619
|
-
|
|
17620
|
-
|
|
17621
|
-
|
|
17622
|
-
|
|
17623
|
-
|
|
17624
|
-
|
|
17625
|
-
|
|
17626
|
-
|
|
17627
|
-
|
|
17628
|
-
|
|
17629
|
-
|
|
17630
|
-
|
|
17631
|
-
|
|
17632
|
-
|
|
17633
|
-
|
|
17634
|
-
|
|
17635
|
-
|
|
17636
|
-
|
|
17637
|
-
|
|
17638
|
-
|
|
17639
|
-
|
|
17640
|
-
|
|
17641
|
-
|
|
17642
|
-
|
|
17643
|
-
|
|
17644
|
-
|
|
17645
|
-
|
|
17646
|
-
|
|
17647
|
-
[...pendingWaits.keys()],
|
|
17648
|
-
1,
|
|
17649
|
-
);
|
|
17650
|
-
turn.settleBytes(
|
|
17651
|
-
runtimeStepReceiptPayloadBytes(
|
|
17652
|
-
waited.completed.values(),
|
|
17653
|
-
),
|
|
17654
|
-
);
|
|
17655
|
-
if (waitSignal.aborted) return;
|
|
17656
|
-
const settledWaits = [...pendingWaits.values()].filter(
|
|
17657
|
-
(wait) =>
|
|
17658
|
-
waited.completed.has(wait.receiptKey) ||
|
|
17659
|
-
waited.failed.has(wait.receiptKey),
|
|
17660
|
-
);
|
|
17661
|
-
const deliverySettlements = await Promise.allSettled(
|
|
17662
|
-
settledWaits.map(async (wait) => {
|
|
17663
|
-
const failed = waited.failed.get(wait.receiptKey);
|
|
17664
|
-
if (failed) {
|
|
17665
|
-
for (const request of wait.requestsForKey) {
|
|
17666
|
-
await this.rejectToolCall(
|
|
17667
|
-
toolId,
|
|
17668
|
-
request,
|
|
17669
|
-
failed,
|
|
17670
|
-
{
|
|
17671
|
-
persistReceiptFailure: false,
|
|
17672
|
-
},
|
|
17673
|
-
);
|
|
17674
|
-
}
|
|
17675
|
-
return;
|
|
17676
|
-
}
|
|
17677
|
-
await reclaimExistingRunningReceiptAfterWait(
|
|
17678
|
-
wait.receiptKey,
|
|
17679
|
-
wait.requestsForKey,
|
|
17680
|
-
wait.forceAfterWait,
|
|
17681
|
-
{
|
|
17682
|
-
completedReceipt: waited.completed.get(
|
|
17683
|
-
wait.receiptKey,
|
|
16473
|
+
const batchActionVersion =
|
|
16474
|
+
(
|
|
16475
|
+
await this.#options.getToolActionCacheVersion?.(
|
|
16476
|
+
batch.batchOperation,
|
|
16477
|
+
)
|
|
16478
|
+
)?.trim() ?? 'unversioned';
|
|
16479
|
+
const aggregateProviderOperationKey =
|
|
16480
|
+
buildDurableToolPayloadProviderIdempotencyKey({
|
|
16481
|
+
orgId: this.#options.orgId,
|
|
16482
|
+
toolId: batch.batchOperation,
|
|
16483
|
+
executionAuthScopeDigest:
|
|
16484
|
+
batch.memberRequests[0]?.executionAuthScopeDigest ?? null,
|
|
16485
|
+
providerActionVersion: batchActionVersion,
|
|
16486
|
+
payload: batch.batchPayload,
|
|
16487
|
+
forceRefreshIdentities: aggregateForceRefreshIdentities,
|
|
16488
|
+
});
|
|
16489
|
+
const aggregateReceiptKey = this.toolTransportReceiptKey({
|
|
16490
|
+
toolId: batch.batchOperation,
|
|
16491
|
+
providerOperationKey: aggregateProviderOperationKey,
|
|
16492
|
+
});
|
|
16493
|
+
let releaseToolSlot: () => void = () => undefined;
|
|
16494
|
+
try {
|
|
16495
|
+
const execution =
|
|
16496
|
+
await this.executeMappedToolCallThroughModule({
|
|
16497
|
+
request: batch.memberRequests[0]!,
|
|
16498
|
+
dispatchToolId: batch.batchOperation,
|
|
16499
|
+
dispatchPayload: batch.batchPayload,
|
|
16500
|
+
resultReceiptKey: aggregateReceiptKey,
|
|
16501
|
+
providerOperationKey: aggregateProviderOperationKey,
|
|
16502
|
+
apiOptions: {
|
|
16503
|
+
durableCallReceiptKey: aggregateReceiptKey,
|
|
16504
|
+
legacyProviderRetryIdentities: [
|
|
16505
|
+
{
|
|
16506
|
+
transportReceiptKey:
|
|
16507
|
+
buildDurableToolAggregateReceiptKey({
|
|
16508
|
+
receiptKeys: batch.memberRequests.map(
|
|
16509
|
+
(member) => member.cacheKey,
|
|
17684
16510
|
),
|
|
17685
|
-
|
|
17686
|
-
|
|
17687
|
-
|
|
17688
|
-
|
|
17689
|
-
|
|
17690
|
-
|
|
17691
|
-
|
|
17692
|
-
|
|
17693
|
-
|
|
17694
|
-
|
|
17695
|
-
|
|
17696
|
-
|
|
17697
|
-
|
|
17698
|
-
|
|
17699
|
-
|
|
17700
|
-
|
|
17701
|
-
|
|
17702
|
-
|
|
17703
|
-
|
|
17704
|
-
|
|
17705
|
-
|
|
17706
|
-
|
|
17707
|
-
|
|
17708
|
-
|
|
17709
|
-
|
|
17710
|
-
|
|
17711
|
-
|
|
17712
|
-
|
|
17713
|
-
|
|
17714
|
-
|
|
17715
|
-
|
|
17716
|
-
|
|
17717
|
-
|
|
17718
|
-
|
|
17719
|
-
|
|
17720
|
-
|
|
17721
|
-
|
|
17722
|
-
|
|
17723
|
-
|
|
17724
|
-
|
|
17725
|
-
|
|
17726
|
-
(
|
|
17727
|
-
|
|
17728
|
-
|
|
17729
|
-
|
|
17730
|
-
|
|
17731
|
-
|
|
17732
|
-
|
|
17733
|
-
|
|
17734
|
-
|
|
17735
|
-
|
|
17736
|
-
|
|
17737
|
-
|
|
16511
|
+
prefix: 'batch',
|
|
16512
|
+
aggregateReceiptPrefix:
|
|
16513
|
+
buildDurableToolReceiptPrefix({
|
|
16514
|
+
orgId: this.#options.orgId,
|
|
16515
|
+
toolId: batch.batchOperation,
|
|
16516
|
+
}),
|
|
16517
|
+
}),
|
|
16518
|
+
providerOperationKey:
|
|
16519
|
+
buildDurableToolPayloadProviderIdempotencyKey({
|
|
16520
|
+
orgId: this.#options.orgId,
|
|
16521
|
+
toolId: batch.batchOperation,
|
|
16522
|
+
executionAuthScopeDigest:
|
|
16523
|
+
batch.memberRequests[0]
|
|
16524
|
+
?.executionAuthScopeDigest ?? null,
|
|
16525
|
+
providerActionVersion: batchActionVersion,
|
|
16526
|
+
payload: batch.batchPayload,
|
|
16527
|
+
forceRefreshIdentities:
|
|
16528
|
+
legacyForceRefreshIdentities,
|
|
16529
|
+
}),
|
|
16530
|
+
forceRefreshIdentities:
|
|
16531
|
+
legacyForceRefreshIdentities,
|
|
16532
|
+
},
|
|
16533
|
+
],
|
|
16534
|
+
// A native batch is one physical provider request for
|
|
16535
|
+
// many rows of the same column, so the batch's node scope
|
|
16536
|
+
// is its members' shared scope.
|
|
16537
|
+
playNodeScope: playNodeScopeForToolCallRequest(
|
|
16538
|
+
batch.memberRequests[0]!,
|
|
16539
|
+
batch.batchOperation,
|
|
16540
|
+
),
|
|
16541
|
+
executionAuthScopeDigest:
|
|
16542
|
+
batch.memberRequests[0]?.executionAuthScopeDigest ??
|
|
16543
|
+
null,
|
|
16544
|
+
providerIdempotencyKey: aggregateProviderOperationKey,
|
|
16545
|
+
providerOperationForceRefreshIdentities:
|
|
16546
|
+
aggregateForceRefreshIdentities,
|
|
16547
|
+
receiptLeaseExpiresAt: batch.memberRequests.reduce<
|
|
16548
|
+
string | null
|
|
16549
|
+
>((earliest, request) => {
|
|
16550
|
+
const expiresAt =
|
|
16551
|
+
request.receiptLeaseExpiresAt ?? null;
|
|
16552
|
+
if (!expiresAt) return earliest;
|
|
16553
|
+
if (!earliest) return expiresAt;
|
|
16554
|
+
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
16555
|
+
? expiresAt
|
|
16556
|
+
: earliest;
|
|
16557
|
+
}, null),
|
|
16558
|
+
timeoutMs: resolveRuntimeTimeoutMsForRequests(
|
|
16559
|
+
batch.memberRequests,
|
|
16560
|
+
),
|
|
16561
|
+
retainToolSlot: (release) => {
|
|
16562
|
+
releaseToolSlot = release;
|
|
16563
|
+
},
|
|
17738
16564
|
},
|
|
17739
|
-
);
|
|
17740
|
-
|
|
17741
|
-
|
|
17742
|
-
|
|
17743
|
-
|
|
17744
|
-
!(
|
|
17745
|
-
error instanceof RuntimeReceiptHydrationTurnTimeoutError
|
|
17746
|
-
)
|
|
17747
|
-
) {
|
|
17748
|
-
throw error;
|
|
17749
|
-
}
|
|
17750
|
-
break;
|
|
17751
|
-
}
|
|
17752
|
-
}
|
|
17753
|
-
if (waitSignal.aborted) return;
|
|
17754
|
-
await reclaimExistingRunningReceipts([
|
|
17755
|
-
...pendingWaits.values(),
|
|
17756
|
-
]);
|
|
17757
|
-
};
|
|
17758
|
-
try {
|
|
17759
|
-
const lanes = await Promise.allSettled([
|
|
17760
|
-
reclaimExistingRunningReceipts(expiredLeaseWaits),
|
|
17761
|
-
pollWaits(),
|
|
17762
|
-
]);
|
|
17763
|
-
const rejectedLane = lanes.find(
|
|
17764
|
-
(lane) => lane.status === 'rejected',
|
|
17765
|
-
);
|
|
17766
|
-
if (rejectedLane?.status === 'rejected') {
|
|
17767
|
-
throw rejectedLane.reason;
|
|
17768
|
-
}
|
|
17769
|
-
} finally {
|
|
17770
|
-
for (const wait of durableExistingRunningWaits) {
|
|
17771
|
-
this.#runtimeReceiptWaitParking.delete(wait.receiptKey);
|
|
16565
|
+
});
|
|
16566
|
+
return { execution, releaseToolSlot };
|
|
16567
|
+
} catch (error) {
|
|
16568
|
+
releaseToolSlot();
|
|
16569
|
+
throw error;
|
|
17772
16570
|
}
|
|
17773
|
-
}
|
|
17774
|
-
|
|
17775
|
-
|
|
17776
|
-
|
|
17777
|
-
|
|
17778
|
-
|
|
17779
|
-
|
|
17780
|
-
|
|
17781
|
-
receiptKey,
|
|
17782
|
-
requestsForKey,
|
|
17783
|
-
group.forceRefresh,
|
|
17784
|
-
);
|
|
17785
|
-
continue;
|
|
17786
|
-
}
|
|
17787
|
-
if (claim?.status === 'completed' || claim?.status === 'skipped') {
|
|
17788
|
-
await resolveRequestsFromReceipt(requestsForKey, claim, 'cache');
|
|
17789
|
-
durableRecoveredRequests.push(...requestsForKey);
|
|
17790
|
-
continue;
|
|
17791
|
-
}
|
|
17792
|
-
if (claim?.status === 'failed') {
|
|
17793
|
-
for (const request of requestsForKey) {
|
|
17794
|
-
await this.rejectToolCall(
|
|
17795
|
-
toolId,
|
|
17796
|
-
request,
|
|
17797
|
-
runtimeReceiptFailureError(
|
|
17798
|
-
claim,
|
|
17799
|
-
'Durable tool call failed',
|
|
17800
|
-
this.currentToolErrorSchemaVersion,
|
|
17801
|
-
),
|
|
17802
|
-
);
|
|
17803
|
-
}
|
|
17804
|
-
continue;
|
|
17805
|
-
}
|
|
17806
|
-
if (
|
|
17807
|
-
(claim?.status === 'queued' ||
|
|
17808
|
-
claim?.status === 'pending' ||
|
|
17809
|
-
claim?.status === 'running') &&
|
|
17810
|
-
claim.claimState === 'existing'
|
|
17811
|
-
) {
|
|
17812
|
-
queueExistingRunningReceiptWait(
|
|
17813
|
-
receiptKey,
|
|
17814
|
-
requestsForKey,
|
|
17815
|
-
group.forceRefresh,
|
|
17816
|
-
!runtimeReceiptClaimLeaseIsLive(claim),
|
|
17817
|
-
);
|
|
17818
|
-
continue;
|
|
17819
|
-
}
|
|
17820
|
-
if (this.isOwnedClaimedRuntimeReceipt(claim)) {
|
|
17821
|
-
const [owner, ...waiters] = requestsForKey;
|
|
17822
|
-
if (!claim.leaseId) {
|
|
17823
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
17824
|
-
receiptKey,
|
|
17825
|
-
runId: this.currentReceiptOwnerRunId,
|
|
17826
|
-
leaseId: 'missing',
|
|
17827
|
-
});
|
|
17828
|
-
}
|
|
17829
|
-
if (owner) {
|
|
17830
|
-
owner.receiptLeaseId = claim.leaseId ?? null;
|
|
17831
|
-
owner.receiptLeaseExpiresAt = claim.leaseExpiresAt ?? null;
|
|
17832
|
-
const claimLeaseExpiresAtMs = Date.parse(
|
|
17833
|
-
claim.leaseExpiresAt ?? '',
|
|
17834
|
-
);
|
|
17835
|
-
pendingReceiptClaimsEstablishExecutionFence &&=
|
|
17836
|
-
claim.claimState === 'claimed' &&
|
|
17837
|
-
claim.status === 'running' &&
|
|
17838
|
-
Number.isFinite(claimLeaseExpiresAtMs) &&
|
|
17839
|
-
claimLeaseExpiresAtMs > Date.now();
|
|
17840
|
-
}
|
|
17841
|
-
claimOwnerWithLiveFollowers(owner, waiters);
|
|
17842
|
-
continue;
|
|
17843
|
-
}
|
|
17844
|
-
queueExistingRunningReceiptWait(
|
|
17845
|
-
receiptKey,
|
|
17846
|
-
requestsForKey,
|
|
17847
|
-
group.forceRefresh,
|
|
17848
|
-
);
|
|
17849
|
-
}
|
|
17850
|
-
|
|
17851
|
-
pendingRequests.length = 0;
|
|
17852
|
-
pendingRequests.push(...claimedRequests);
|
|
17853
|
-
durableExistingRunningState.settlement =
|
|
17854
|
-
durableExistingRunningWaits.length > 0
|
|
17855
|
-
? processExistingRunningReceiptWaits().then(
|
|
17856
|
-
() => ({ status: 'fulfilled' as const }),
|
|
17857
|
-
(reason: unknown) => ({
|
|
17858
|
-
status: 'rejected' as const,
|
|
17859
|
-
reason,
|
|
17860
|
-
}),
|
|
17861
|
-
)
|
|
17862
|
-
: null;
|
|
17863
|
-
recordToolStep(durableRecoveredRequests);
|
|
17864
|
-
}
|
|
17865
|
-
|
|
17866
|
-
// Claims establish one lease cohort. Revalidate that cohort in one
|
|
17867
|
-
// gateway operation before provider admission, then let the existing
|
|
17868
|
-
// cohort heartbeat keep the undispatched tail alive. Revalidating each
|
|
17869
|
-
// call separately after a tool slot opens fragments the writer into
|
|
17870
|
-
// one-row batches and starves otherwise healthy providers.
|
|
17871
|
-
const pendingReceiptHeartbeat = this.startPendingToolReceiptHeartbeat(
|
|
17872
|
-
pendingRequests,
|
|
17873
|
-
pendingReceiptClaimsEstablishExecutionFence,
|
|
17874
|
-
);
|
|
17875
|
-
let providerFailure: { reason: unknown } | null = null;
|
|
17876
|
-
try {
|
|
17877
|
-
if (pendingReceiptHeartbeat) {
|
|
17878
|
-
await pendingReceiptHeartbeat.confirmOwned();
|
|
17879
|
-
}
|
|
17880
|
-
if (pendingRequests.length > 0) {
|
|
17881
|
-
const strategy = operationStrategy;
|
|
17882
|
-
|
|
17883
|
-
if (strategy) {
|
|
17884
|
-
const compiledBatches = compileRequestsWithStrategy({
|
|
17885
|
-
requests: pendingRequests,
|
|
17886
|
-
strategy,
|
|
17887
|
-
getPayload: (request: ToolCallRequest) => request.input,
|
|
17888
|
-
});
|
|
17889
|
-
|
|
17890
|
-
const batchParallelismCeiling =
|
|
17891
|
-
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
17892
|
-
const batchSize =
|
|
17893
|
-
compiledBatches.length > 0
|
|
17894
|
-
? await this.resourceGovernor.suggestedToolParallelism(
|
|
17895
|
-
compiledBatches[0]!.batchOperation,
|
|
17896
|
-
batchParallelismCeiling,
|
|
17897
|
-
)
|
|
17898
|
-
: batchParallelismCeiling;
|
|
17899
|
-
await executeChunkedRequests({
|
|
17900
|
-
requests: compiledBatches,
|
|
17901
|
-
batchSize,
|
|
17902
|
-
execute: async (batch) => {
|
|
17903
|
-
// Circuit breaker: skip dispatching this batch's provider call
|
|
17904
|
-
// once a persistence failure has occurred in this run.
|
|
17905
|
-
if (this.persistenceLatch.tripped) {
|
|
17906
|
-
this.persistenceLatch.preventedCallCount +=
|
|
17907
|
-
batch.memberRequests.length;
|
|
17908
|
-
throw new RuntimePersistenceCircuitOpenError(
|
|
17909
|
-
this.persistenceLatch,
|
|
16571
|
+
},
|
|
16572
|
+
onChunkComplete: async (chunkResults) => {
|
|
16573
|
+
for (const entry of chunkResults) {
|
|
16574
|
+
if (entry.error !== undefined) {
|
|
16575
|
+
const settlements = await Promise.allSettled(
|
|
16576
|
+
entry.request.memberRequests.map((request) =>
|
|
16577
|
+
rejectCall(request, entry.error),
|
|
16578
|
+
),
|
|
17910
16579
|
);
|
|
17911
|
-
|
|
17912
|
-
|
|
17913
|
-
|
|
17914
|
-
);
|
|
17915
|
-
const aggregateReceiptKey =
|
|
17916
|
-
buildDurableToolAggregateReceiptKey({
|
|
17917
|
-
receiptKeys,
|
|
17918
|
-
prefix: 'batch',
|
|
17919
|
-
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
17920
|
-
orgId: this.#options.orgId,
|
|
17921
|
-
toolId: batch.batchOperation,
|
|
17922
|
-
}),
|
|
17923
|
-
});
|
|
17924
|
-
const aggregateForceRefreshIdentities = batch.memberRequests
|
|
17925
|
-
.filter((request) => request.force === true)
|
|
17926
|
-
.map((request) =>
|
|
17927
|
-
this.providerIdempotencyKeyForToolCall({
|
|
17928
|
-
operationKey:
|
|
17929
|
-
this.providerOperationKeyForQueuedToolCall(request),
|
|
17930
|
-
force: true,
|
|
17931
|
-
logicalCallId: request.logicalCallId,
|
|
17932
|
-
}),
|
|
16580
|
+
const rejected = settlements.find(
|
|
16581
|
+
(settlement): settlement is PromiseRejectedResult =>
|
|
16582
|
+
settlement.status === 'rejected',
|
|
17933
16583
|
);
|
|
17934
|
-
|
|
17935
|
-
|
|
17936
|
-
|
|
17937
|
-
|
|
17938
|
-
executionAuthScopeDigest:
|
|
17939
|
-
batch.memberRequests[0]?.executionAuthScopeDigest ??
|
|
17940
|
-
null,
|
|
17941
|
-
providerActionVersion:
|
|
17942
|
-
(
|
|
17943
|
-
await this.#options.getToolActionCacheVersion?.(
|
|
17944
|
-
batch.batchOperation,
|
|
17945
|
-
)
|
|
17946
|
-
)?.trim() ?? 'unversioned',
|
|
17947
|
-
payload: batch.batchPayload,
|
|
17948
|
-
forceRefreshIdentities: aggregateForceRefreshIdentities,
|
|
17949
|
-
});
|
|
17950
|
-
let releaseToolSlot: () => void = () => undefined;
|
|
16584
|
+
if (rejected) throw rejected.reason;
|
|
16585
|
+
continue;
|
|
16586
|
+
}
|
|
16587
|
+
const batchExecution = entry.result;
|
|
17951
16588
|
try {
|
|
17952
|
-
const
|
|
17953
|
-
|
|
17954
|
-
|
|
17955
|
-
|
|
17956
|
-
|
|
17957
|
-
resultReceiptKey: aggregateReceiptKey,
|
|
17958
|
-
providerOperationKey: aggregateProviderOperationKey,
|
|
17959
|
-
apiOptions: {
|
|
17960
|
-
durableCallReceiptKey: aggregateReceiptKey,
|
|
17961
|
-
// A native batch is one physical provider request for
|
|
17962
|
-
// many rows of the same column, so the batch's node scope
|
|
17963
|
-
// is its members' shared scope.
|
|
17964
|
-
playNodeScope: playNodeScopeForToolCallRequest(
|
|
17965
|
-
batch.memberRequests[0]!,
|
|
17966
|
-
batch.batchOperation,
|
|
17967
|
-
),
|
|
17968
|
-
executionAuthScopeDigest:
|
|
17969
|
-
batch.memberRequests[0]?.executionAuthScopeDigest ??
|
|
17970
|
-
null,
|
|
17971
|
-
providerIdempotencyKey: aggregateProviderOperationKey,
|
|
17972
|
-
providerOperationForceRefreshIdentities:
|
|
17973
|
-
aggregateForceRefreshIdentities,
|
|
17974
|
-
receiptLeaseExpiresAt: batch.memberRequests.reduce<
|
|
17975
|
-
string | null
|
|
17976
|
-
>((earliest, request) => {
|
|
17977
|
-
const expiresAt =
|
|
17978
|
-
request.receiptLeaseExpiresAt ?? null;
|
|
17979
|
-
if (!expiresAt) return earliest;
|
|
17980
|
-
if (!earliest) return expiresAt;
|
|
17981
|
-
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
17982
|
-
? expiresAt
|
|
17983
|
-
: earliest;
|
|
17984
|
-
}, null),
|
|
17985
|
-
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
|
|
17986
|
-
batch.memberRequests,
|
|
17987
|
-
),
|
|
17988
|
-
beforeProviderCall: () =>
|
|
17989
|
-
pendingReceiptHeartbeat
|
|
17990
|
-
? pendingReceiptHeartbeat.assertOwned()
|
|
17991
|
-
: this.assertRuntimeToolReceiptOwnership(
|
|
17992
|
-
batch.memberRequests,
|
|
17993
|
-
),
|
|
17994
|
-
heartbeatReceipt: () =>
|
|
17995
|
-
this.renewRuntimeToolReceiptOwnership(
|
|
17996
|
-
batch.memberRequests,
|
|
16589
|
+
const splitResults =
|
|
16590
|
+
batchExecution != null
|
|
16591
|
+
? entry.request.splitResults(
|
|
16592
|
+
legacyResultForBatchSplitter(
|
|
16593
|
+
batchExecution.execution,
|
|
17997
16594
|
),
|
|
17998
|
-
|
|
17999
|
-
|
|
18000
|
-
|
|
18001
|
-
|
|
18002
|
-
|
|
18003
|
-
|
|
18004
|
-
|
|
18005
|
-
|
|
18006
|
-
|
|
18007
|
-
|
|
18008
|
-
|
|
18009
|
-
|
|
18010
|
-
|
|
18011
|
-
|
|
18012
|
-
|
|
18013
|
-
|
|
18014
|
-
|
|
18015
|
-
),
|
|
18016
|
-
);
|
|
18017
|
-
const rejected = settlements.find(
|
|
18018
|
-
(settlement): settlement is PromiseRejectedResult =>
|
|
18019
|
-
settlement.status === 'rejected',
|
|
16595
|
+
)
|
|
16596
|
+
: entry.request.memberRequests.map(() => null);
|
|
16597
|
+
const resolvedResults =
|
|
16598
|
+
await this.resolveToolCallBatchResults(
|
|
16599
|
+
toolId,
|
|
16600
|
+
entry.request.memberRequests.map((request, index) => ({
|
|
16601
|
+
request,
|
|
16602
|
+
result: splitResults[index] ?? null,
|
|
16603
|
+
status: batchExecution?.execution.status,
|
|
16604
|
+
toolResponse:
|
|
16605
|
+
batchExecution == null
|
|
16606
|
+
? undefined
|
|
16607
|
+
: publicToolResponseForBatchedItem(
|
|
16608
|
+
batchExecution.execution,
|
|
16609
|
+
splitResults[index] ?? null,
|
|
16610
|
+
true,
|
|
16611
|
+
),
|
|
16612
|
+
})),
|
|
18020
16613
|
);
|
|
18021
|
-
if (rejected) throw rejected.reason;
|
|
18022
|
-
continue;
|
|
18023
|
-
}
|
|
18024
|
-
const batchExecution = entry.result;
|
|
18025
|
-
try {
|
|
18026
|
-
const splitResults =
|
|
18027
|
-
batchExecution != null
|
|
18028
|
-
? entry.request.splitResults(
|
|
18029
|
-
legacyResultForBatchSplitter(
|
|
18030
|
-
batchExecution.execution,
|
|
18031
|
-
),
|
|
18032
|
-
)
|
|
18033
|
-
: entry.request.memberRequests.map(() => null);
|
|
18034
|
-
const resolvedResults =
|
|
18035
|
-
await this.resolveToolCallBatchResults(
|
|
18036
|
-
toolId,
|
|
18037
|
-
entry.request.memberRequests.map(
|
|
18038
|
-
(request, index) => ({
|
|
18039
|
-
request,
|
|
18040
|
-
result: splitResults[index] ?? null,
|
|
18041
|
-
status: batchExecution?.execution.status,
|
|
18042
|
-
toolResponse:
|
|
18043
|
-
batchExecution == null
|
|
18044
|
-
? undefined
|
|
18045
|
-
: publicToolResponseForBatchedItem(
|
|
18046
|
-
batchExecution.execution,
|
|
18047
|
-
splitResults[index] ?? null,
|
|
18048
|
-
true,
|
|
18049
|
-
),
|
|
18050
|
-
}),
|
|
18051
|
-
),
|
|
18052
|
-
);
|
|
18053
16614
|
|
|
18054
|
-
|
|
18055
|
-
|
|
18056
|
-
|
|
18057
|
-
|
|
18058
|
-
|
|
18059
|
-
|
|
18060
|
-
|
|
18061
|
-
|
|
18062
|
-
}
|
|
18063
|
-
resolveLiveFollowers(request, resolvedResults[index]);
|
|
16615
|
+
for (
|
|
16616
|
+
let index = 0;
|
|
16617
|
+
index < entry.request.memberRequests.length;
|
|
16618
|
+
index += 1
|
|
16619
|
+
) {
|
|
16620
|
+
const request = entry.request.memberRequests[index]!;
|
|
16621
|
+
if (resolvedResults[index] != null) {
|
|
16622
|
+
successfulLiveStepCallIds.add(request.callId);
|
|
18064
16623
|
}
|
|
18065
|
-
} finally {
|
|
18066
|
-
batchExecution?.releaseToolSlot();
|
|
18067
16624
|
}
|
|
16625
|
+
} finally {
|
|
16626
|
+
batchExecution?.releaseToolSlot();
|
|
18068
16627
|
}
|
|
16628
|
+
}
|
|
18069
16629
|
|
|
18070
|
-
|
|
18071
|
-
|
|
18072
|
-
(entry) => entry.request.memberRequests,
|
|
18073
|
-
),
|
|
18074
|
-
);
|
|
18075
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
18076
|
-
},
|
|
18077
|
-
retainResults: false,
|
|
18078
|
-
});
|
|
18079
|
-
} else {
|
|
18080
|
-
// Bound direct map dispatch separately from ordinary provider
|
|
18081
|
-
// residency. This prevents a blocked first chunk from launching
|
|
18082
|
-
// another chunk ahead of a provider's admitted start rate, while
|
|
18083
|
-
// preserving the wider resident-call capacity used by batched and
|
|
18084
|
-
// high-latency providers.
|
|
18085
|
-
const toolCallConcurrencyCeiling =
|
|
18086
|
-
this.governor.policy.concurrency.toolCalls;
|
|
18087
|
-
const shapedToolParallelism =
|
|
18088
|
-
await this.resourceGovernor.suggestedDirectToolParallelism(
|
|
18089
|
-
toolId,
|
|
18090
|
-
toolCallConcurrencyCeiling,
|
|
16630
|
+
recordToolStep(
|
|
16631
|
+
chunkResults.flatMap((entry) => entry.request.memberRequests),
|
|
18091
16632
|
);
|
|
18092
|
-
|
|
16633
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
16634
|
+
},
|
|
16635
|
+
retainResults: false,
|
|
16636
|
+
});
|
|
16637
|
+
} else {
|
|
16638
|
+
// Bound direct map dispatch separately from ordinary provider
|
|
16639
|
+
// residency. This prevents a blocked first chunk from launching
|
|
16640
|
+
// another chunk ahead of a provider's admitted start rate, while
|
|
16641
|
+
// preserving the wider resident-call capacity used by batched and
|
|
16642
|
+
// high-latency providers.
|
|
16643
|
+
const toolCallConcurrencyCeiling =
|
|
16644
|
+
this.governor.policy.concurrency.toolCalls;
|
|
16645
|
+
const shapedToolParallelism =
|
|
16646
|
+
await this.resourceGovernor.suggestedDirectToolParallelism(
|
|
16647
|
+
toolId,
|
|
18093
16648
|
toolCallConcurrencyCeiling,
|
|
18094
|
-
Math.max(1, shapedToolParallelism),
|
|
18095
16649
|
);
|
|
18096
|
-
|
|
18097
|
-
|
|
18098
|
-
|
|
18099
|
-
|
|
18100
|
-
|
|
18101
|
-
|
|
18102
|
-
|
|
18103
|
-
|
|
18104
|
-
|
|
18105
|
-
|
|
18106
|
-
|
|
18107
|
-
|
|
18108
|
-
|
|
18109
|
-
|
|
18110
|
-
|
|
18111
|
-
|
|
18112
|
-
|
|
18113
|
-
|
|
18114
|
-
|
|
18115
|
-
|
|
18116
|
-
|
|
18117
|
-
|
|
18118
|
-
|
|
18119
|
-
|
|
18120
|
-
|
|
18121
|
-
|
|
18122
|
-
|
|
18123
|
-
|
|
18124
|
-
|
|
18125
|
-
|
|
18126
|
-
|
|
18127
|
-
|
|
18128
|
-
|
|
18129
|
-
|
|
18130
|
-
|
|
18131
|
-
|
|
18132
|
-
|
|
18133
|
-
|
|
18134
|
-
|
|
18135
|
-
|
|
18136
|
-
|
|
18137
|
-
|
|
18138
|
-
|
|
18139
|
-
|
|
18140
|
-
|
|
18141
|
-
|
|
18142
|
-
|
|
18143
|
-
|
|
18144
|
-
|
|
18145
|
-
|
|
18146
|
-
|
|
18147
|
-
|
|
18148
|
-
|
|
18149
|
-
|
|
18150
|
-
),
|
|
18151
|
-
force: request.force,
|
|
18152
|
-
requiresExecutionFence:
|
|
18153
|
-
request.requiresExecutionFence,
|
|
18154
|
-
logicalCallId: request.logicalCallId,
|
|
16650
|
+
const dispatchWidth = Math.min(
|
|
16651
|
+
toolCallConcurrencyCeiling,
|
|
16652
|
+
Math.max(1, shapedToolParallelism),
|
|
16653
|
+
);
|
|
16654
|
+
const dispatchResults = await dispatchBoundedSettled(
|
|
16655
|
+
pendingRequests,
|
|
16656
|
+
dispatchWidth,
|
|
16657
|
+
async (request) => {
|
|
16658
|
+
// Circuit breaker: do not dispatch this provider call once a
|
|
16659
|
+
// persistence failure has occurred in this run.
|
|
16660
|
+
if (this.persistenceLatch.tripped) {
|
|
16661
|
+
this.persistenceLatch.preventedCallCount += 1;
|
|
16662
|
+
throw new RuntimePersistenceCircuitOpenError(
|
|
16663
|
+
this.persistenceLatch,
|
|
16664
|
+
);
|
|
16665
|
+
}
|
|
16666
|
+
const providerOperationKey =
|
|
16667
|
+
this.providerInvocationKeyForToolCall({
|
|
16668
|
+
operationKey:
|
|
16669
|
+
this.providerOperationKeyForQueuedToolCall(request),
|
|
16670
|
+
logicalCallId: request.logicalCallId,
|
|
16671
|
+
force: request.force,
|
|
16672
|
+
});
|
|
16673
|
+
let releaseToolSlot: () => void = () => undefined;
|
|
16674
|
+
try {
|
|
16675
|
+
const execution =
|
|
16676
|
+
await this.executeMappedToolCallThroughModule({
|
|
16677
|
+
request,
|
|
16678
|
+
dispatchToolId: toolId,
|
|
16679
|
+
dispatchPayload: request.input,
|
|
16680
|
+
// Always-fresh map calls intentionally have no
|
|
16681
|
+
// result cache. Their provider-operation identity is
|
|
16682
|
+
// a unique forced semantic request, never a cache key.
|
|
16683
|
+
resultReceiptKey: request.receiptKey ?? request.cacheKey,
|
|
16684
|
+
providerOperationKey,
|
|
16685
|
+
apiOptions: {
|
|
16686
|
+
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
16687
|
+
executionAuthScopeDigest:
|
|
16688
|
+
request.executionAuthScopeDigest,
|
|
16689
|
+
providerIdempotencyKey: providerOperationKey,
|
|
16690
|
+
...this.providerInvocationOptionsForToolCall({
|
|
16691
|
+
operationKey:
|
|
16692
|
+
this.providerOperationKeyForQueuedToolCall(request),
|
|
16693
|
+
requiresExecutionFence:
|
|
16694
|
+
request.requiresExecutionFence,
|
|
16695
|
+
logicalCallId: request.logicalCallId,
|
|
16696
|
+
force: request.force,
|
|
16697
|
+
}),
|
|
16698
|
+
...(request.receiptKey
|
|
16699
|
+
? {
|
|
16700
|
+
durableCallReceiptKey:
|
|
16701
|
+
this.toolTransportReceiptKey({
|
|
16702
|
+
toolId,
|
|
16703
|
+
providerOperationKey,
|
|
18155
16704
|
}),
|
|
18156
|
-
|
|
18157
|
-
|
|
18158
|
-
|
|
18159
|
-
|
|
18160
|
-
|
|
18161
|
-
|
|
18162
|
-
}
|
|
18163
|
-
: {
|
|
18164
|
-
providerIdempotencyKey:
|
|
18165
|
-
this.providerIdempotencyKeyForToolCall({
|
|
18166
|
-
operationKey:
|
|
18167
|
-
this.providerOperationKeyForQueuedToolCall(
|
|
16705
|
+
legacyProviderRetryIdentities: [
|
|
16706
|
+
{
|
|
16707
|
+
transportReceiptKey: request.receiptKey,
|
|
16708
|
+
providerOperationKey: request.force
|
|
16709
|
+
? providerOperationKey
|
|
16710
|
+
: this.providerOperationKeyForQueuedToolCall(
|
|
18168
16711
|
request,
|
|
18169
16712
|
),
|
|
18170
|
-
|
|
18171
|
-
|
|
18172
|
-
|
|
18173
|
-
|
|
18174
|
-
|
|
18175
|
-
|
|
18176
|
-
|
|
18177
|
-
|
|
18178
|
-
|
|
18179
|
-
|
|
16713
|
+
...(request.force &&
|
|
16714
|
+
request.requiresExecutionFence !== true
|
|
16715
|
+
? {
|
|
16716
|
+
forceRefreshIdentities: [
|
|
16717
|
+
providerOperationKey,
|
|
16718
|
+
],
|
|
16719
|
+
}
|
|
16720
|
+
: {}),
|
|
16721
|
+
},
|
|
16722
|
+
],
|
|
16723
|
+
completedResultCache: true,
|
|
16724
|
+
}
|
|
16725
|
+
: {}),
|
|
16726
|
+
timeoutMs: resolveRuntimeTimeoutMsForRequests([
|
|
16727
|
+
request,
|
|
16728
|
+
]),
|
|
16729
|
+
retainToolSlot: (release) => {
|
|
16730
|
+
releaseToolSlot = release;
|
|
18180
16731
|
},
|
|
18181
|
-
}
|
|
18182
|
-
|
|
18183
|
-
|
|
18184
|
-
|
|
18185
|
-
|
|
18186
|
-
|
|
18187
|
-
|
|
18188
|
-
|
|
18189
|
-
|
|
18190
|
-
resolveLiveFollowers(request, result);
|
|
18191
|
-
} finally {
|
|
18192
|
-
releaseToolSlot();
|
|
16732
|
+
},
|
|
16733
|
+
});
|
|
16734
|
+
const result = await this.resolveToolCall(
|
|
16735
|
+
toolId,
|
|
16736
|
+
request,
|
|
16737
|
+
execution,
|
|
16738
|
+
);
|
|
16739
|
+
if (result != null) {
|
|
16740
|
+
successfulLiveStepCallIds.add(request.callId);
|
|
18193
16741
|
}
|
|
18194
|
-
}
|
|
18195
|
-
|
|
18196
|
-
|
|
18197
|
-
|
|
18198
|
-
|
|
18199
|
-
|
|
18200
|
-
|
|
18201
|
-
|
|
18202
|
-
|
|
18203
|
-
)
|
|
18204
|
-
|
|
18205
|
-
|
|
18206
|
-
|
|
18207
|
-
|
|
18208
|
-
|
|
18209
|
-
|
|
18210
|
-
|
|
18211
|
-
|
|
18212
|
-
}
|
|
18213
|
-
} catch (reason) {
|
|
18214
|
-
providerFailure = { reason };
|
|
18215
|
-
durableExistingRunningAbort.abort(reason);
|
|
18216
|
-
} finally {
|
|
18217
|
-
pendingReceiptHeartbeat?.stop();
|
|
18218
|
-
}
|
|
18219
|
-
const pendingReceiptLeaseLost = pendingReceiptHeartbeat?.leaseLost();
|
|
18220
|
-
if (pendingReceiptLeaseLost && providerFailure === null) {
|
|
18221
|
-
providerFailure = { reason: pendingReceiptLeaseLost };
|
|
18222
|
-
durableExistingRunningAbort.abort(pendingReceiptLeaseLost);
|
|
18223
|
-
}
|
|
18224
|
-
const durableExistingRunningSettlement =
|
|
18225
|
-
durableExistingRunningState.settlement;
|
|
18226
|
-
if (durableExistingRunningSettlement) {
|
|
18227
|
-
const settlement = await durableExistingRunningSettlement;
|
|
18228
|
-
if (settlement.status === 'rejected' && providerFailure === null) {
|
|
18229
|
-
providerFailure = { reason: settlement.reason };
|
|
16742
|
+
} finally {
|
|
16743
|
+
releaseToolSlot();
|
|
16744
|
+
}
|
|
16745
|
+
},
|
|
16746
|
+
);
|
|
16747
|
+
const failedEntries = dispatchResults.filter(
|
|
16748
|
+
(entry) => entry.error !== undefined,
|
|
16749
|
+
);
|
|
16750
|
+
const failedSettlements = await Promise.allSettled(
|
|
16751
|
+
failedEntries.map((entry) => rejectCall(entry.job, entry.error)),
|
|
16752
|
+
);
|
|
16753
|
+
const failedRejection = failedSettlements.find(
|
|
16754
|
+
(settlement): settlement is PromiseRejectedResult =>
|
|
16755
|
+
settlement.status === 'rejected',
|
|
16756
|
+
);
|
|
16757
|
+
if (failedRejection) throw failedRejection.reason;
|
|
16758
|
+
recordToolStep(pendingRequests);
|
|
16759
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
18230
16760
|
}
|
|
18231
16761
|
}
|
|
18232
|
-
if (providerFailure) throw providerFailure.reason;
|
|
18233
16762
|
}),
|
|
18234
16763
|
);
|
|
18235
16764
|
const rejected = toolSettlements.find(
|
|
@@ -18323,24 +16852,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18323
16852
|
)(ctx, input);
|
|
18324
16853
|
}
|
|
18325
16854
|
|
|
18326
|
-
private
|
|
16855
|
+
private providerInvocationKeyForToolCall(input: {
|
|
18327
16856
|
operationKey: string;
|
|
18328
|
-
force?: boolean;
|
|
18329
16857
|
logicalCallId?: string;
|
|
16858
|
+
force?: boolean;
|
|
18330
16859
|
}): string {
|
|
18331
|
-
|
|
18332
|
-
|
|
18333
|
-
if (!input.logicalCallId) {
|
|
18334
|
-
throw new Error(
|
|
18335
|
-
'Forced tool execution requires a stable logical call identity.',
|
|
18336
|
-
);
|
|
18337
|
-
}
|
|
18338
|
-
fallbackAttemptId = `${this.currentRunId}:${input.logicalCallId}`;
|
|
16860
|
+
if (!input.logicalCallId) {
|
|
16861
|
+
throw new Error('Tool execution requires a stable invocation identity.');
|
|
18339
16862
|
}
|
|
18340
16863
|
return buildProviderOperationIdempotencyKey({
|
|
18341
16864
|
operationKey: input.operationKey,
|
|
18342
|
-
force:
|
|
18343
|
-
fallbackAttemptId
|
|
16865
|
+
force: true,
|
|
16866
|
+
fallbackAttemptId: `${this.currentRunId}:${input.logicalCallId}${input.force === true ? '' : ':cache-miss'}`,
|
|
18344
16867
|
});
|
|
18345
16868
|
}
|
|
18346
16869
|
|
|
@@ -18986,7 +17509,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18986
17509
|
const ambiguousTransportReplayAllowed =
|
|
18987
17510
|
readOnlyQuery || retrySafeTransientHttp;
|
|
18988
17511
|
const invocationOrgId = this.#options.orgId?.trim() ?? '';
|
|
18989
|
-
|
|
17512
|
+
let durableCallReceiptKey = uncachedAsyncLifecycleCall
|
|
18990
17513
|
? null
|
|
18991
17514
|
: options?.durableCallReceiptKey?.trim() ||
|
|
18992
17515
|
(usesDurableInvocationFence && invocationOrgId
|
|
@@ -19004,9 +17527,57 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19004
17527
|
this.#options.durableBoundaries && durableCallReceiptKey
|
|
19005
17528
|
? stableDigest(`provider-retry:${durableCallReceiptKey}`)
|
|
19006
17529
|
: null;
|
|
19007
|
-
|
|
17530
|
+
let savedProviderRetryCursor = providerRetryCursorKey
|
|
19008
17531
|
? this.checkpoint.providerRetryCursors?.[providerRetryCursorKey]
|
|
19009
17532
|
: undefined;
|
|
17533
|
+
if (providerRetryCursorKey && !savedProviderRetryCursor) {
|
|
17534
|
+
// Native calls can move from the outgoing fallback into the Module.
|
|
17535
|
+
// Probe only exact owner-supplied checkpoint identities; never wait
|
|
17536
|
+
// on another invocation or derive a fence from customer strings.
|
|
17537
|
+
for (const legacy of options?.legacyProviderRetryIdentities ?? []) {
|
|
17538
|
+
const legacyCursorKey = stableDigest(
|
|
17539
|
+
`provider-retry:${legacy.transportReceiptKey}`,
|
|
17540
|
+
);
|
|
17541
|
+
const cursor =
|
|
17542
|
+
this.checkpoint.providerRetryCursors?.[legacyCursorKey];
|
|
17543
|
+
if (!cursor) continue;
|
|
17544
|
+
// Consume synchronously before any await so a sibling cannot
|
|
17545
|
+
// inherit the same outgoing invocation.
|
|
17546
|
+
delete this.checkpoint.providerRetryCursors![legacyCursorKey];
|
|
17547
|
+
savedProviderRetryCursor = {
|
|
17548
|
+
...cursor,
|
|
17549
|
+
legacyProviderOperationKey:
|
|
17550
|
+
cursor.legacyProviderOperationKey ??
|
|
17551
|
+
legacy.providerOperationKey,
|
|
17552
|
+
legacyTransportReceiptKey:
|
|
17553
|
+
cursor.legacyTransportReceiptKey ??
|
|
17554
|
+
legacy.transportReceiptKey,
|
|
17555
|
+
...(legacy.forceRefreshIdentities?.length
|
|
17556
|
+
? {
|
|
17557
|
+
legacyProviderOperationForceRefreshIdentities:
|
|
17558
|
+
cursor.legacyProviderOperationForceRefreshIdentities ??
|
|
17559
|
+
legacy.forceRefreshIdentities,
|
|
17560
|
+
}
|
|
17561
|
+
: {}),
|
|
17562
|
+
};
|
|
17563
|
+
this.checkpoint.providerRetryCursors![providerRetryCursorKey] =
|
|
17564
|
+
savedProviderRetryCursor;
|
|
17565
|
+
break;
|
|
17566
|
+
}
|
|
17567
|
+
}
|
|
17568
|
+
// The checkpoint index stays unique to this invocation. Its wire
|
|
17569
|
+
// identities must still name the prior gateway outcome so that a
|
|
17570
|
+
// qualified 429 retry can be attested after any further suspension.
|
|
17571
|
+
const legacyProviderOperationKey =
|
|
17572
|
+
savedProviderRetryCursor?.legacyProviderOperationKey;
|
|
17573
|
+
if (savedProviderRetryCursor?.legacyTransportReceiptKey) {
|
|
17574
|
+
durableCallReceiptKey =
|
|
17575
|
+
savedProviderRetryCursor.legacyTransportReceiptKey;
|
|
17576
|
+
}
|
|
17577
|
+
const providerOperationForceRefreshIdentities =
|
|
17578
|
+
legacyProviderOperationKey
|
|
17579
|
+
? savedProviderRetryCursor?.legacyProviderOperationForceRefreshIdentities
|
|
17580
|
+
: options?.providerOperationForceRefreshIdentities;
|
|
19010
17581
|
const executionRequestNonce = crypto.randomUUID();
|
|
19011
17582
|
// Transport ambiguity must retain the billing identity. An ordinary
|
|
19012
17583
|
// (unfenced) call advances it only after a definite provider-owned
|
|
@@ -19020,7 +17591,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19020
17591
|
const providerIdempotencyReceiptKey =
|
|
19021
17592
|
options?.providerIdempotencyReceiptKey?.trim() || null;
|
|
19022
17593
|
const providerExecutionFenceKey = requiresExecutionFence
|
|
19023
|
-
?
|
|
17594
|
+
? legacyProviderOperationKey ||
|
|
17595
|
+
options?.providerIdempotencyKey?.trim() ||
|
|
17596
|
+
null
|
|
19024
17597
|
: null;
|
|
19025
17598
|
if (requiresExecutionFence && !providerExecutionFenceKey) {
|
|
19026
17599
|
throw new Error(
|
|
@@ -19029,14 +17602,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19029
17602
|
}
|
|
19030
17603
|
const canonicalOperationActionVersion =
|
|
19031
17604
|
providerExecutionFenceKey ||
|
|
19032
|
-
|
|
17605
|
+
providerOperationForceRefreshIdentities?.length
|
|
19033
17606
|
? (
|
|
19034
17607
|
await this.#options.getToolActionCacheVersion?.(toolId)
|
|
19035
17608
|
)?.trim() || null
|
|
19036
17609
|
: null;
|
|
19037
17610
|
const canonicalOperation =
|
|
17611
|
+
!uncachedAsyncLifecycleCall &&
|
|
19038
17612
|
(providerExecutionFenceKey ||
|
|
19039
|
-
|
|
17613
|
+
providerOperationForceRefreshIdentities?.length) &&
|
|
19040
17614
|
canonicalOperationActionVersion
|
|
19041
17615
|
? {
|
|
19042
17616
|
version: 1,
|
|
@@ -19046,10 +17620,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19046
17620
|
output_intent: options?.completedResultCache
|
|
19047
17621
|
? 'dataset'
|
|
19048
17622
|
: 'value',
|
|
19049
|
-
...(
|
|
17623
|
+
...(providerOperationForceRefreshIdentities?.length
|
|
19050
17624
|
? {
|
|
19051
17625
|
force_refresh_identities:
|
|
19052
|
-
|
|
17626
|
+
providerOperationForceRefreshIdentities,
|
|
19053
17627
|
}
|
|
19054
17628
|
: {}),
|
|
19055
17629
|
}
|
|
@@ -19183,9 +17757,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19183
17757
|
};
|
|
19184
17758
|
const persistProviderRetryCursor = (retryFrom: number) => {
|
|
19185
17759
|
if (!providerRetryCursorKey) return;
|
|
19186
|
-
const cursor = {
|
|
17760
|
+
const cursor: ProviderRetryCursor = {
|
|
17761
|
+
...savedProviderRetryCursor,
|
|
19187
17762
|
invocationAttempt: retryFrom + 1,
|
|
19188
17763
|
retryFromInvocationAttempt: retryFrom,
|
|
17764
|
+
...(legacyProviderOperationKey
|
|
17765
|
+
? {
|
|
17766
|
+
legacyProviderOperationKey,
|
|
17767
|
+
legacyTransportReceiptKey: durableCallReceiptKey!,
|
|
17768
|
+
}
|
|
17769
|
+
: {}),
|
|
19189
17770
|
};
|
|
19190
17771
|
this.checkpoint.providerRetryCursors ??= {};
|
|
19191
17772
|
this.checkpoint.providerRetryCursors[providerRetryCursorKey] =
|