deepline 0.3.283 → 0.3.285

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/shared_libs/play-runtime/activity-observation.ts +26 -2
  3. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +7 -0
  4. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +817 -2236
  5. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +19 -6
  6. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +127 -789
  7. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +180 -14
  8. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +1 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +3 -22
  10. package/dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts +26 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +50 -8
  12. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +122 -18
  13. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +218 -41
  14. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +20 -2
  15. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +13 -2
  16. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +181 -326
  17. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +38 -151
  18. package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +8 -0
  19. package/dist/bundling-sources/shared_libs/security/secret-redaction.ts +89 -1
  20. package/dist/cli/index.js +28 -3
  21. package/dist/cli/index.mjs +28 -3
  22. package/dist/{client-B0AjIkgb.d.mts → client-Cl-zj2oS.d.mts} +4 -0
  23. package/dist/{client-BCXQ_Ftc.d.ts → client-T2jVnZrq.d.ts} +4 -0
  24. package/dist/index.d.mts +2 -2
  25. package/dist/index.d.ts +2 -2
  26. package/dist/index.js +25 -1
  27. package/dist/index.mjs +25 -1
  28. package/dist/install-integrity.json +2 -2
  29. package/dist/plays/run-request.d.mts +1 -1
  30. package/dist/plays/run-request.d.ts +1 -1
  31. package/dist/release.d.mts +1 -1
  32. package/dist/release.d.ts +1 -1
  33. package/dist/release.js +1 -1
  34. package/dist/release.mjs +1 -1
  35. package/package.json +1 -1
@@ -187,7 +187,6 @@ import {
187
187
  createRuntimeStepToolResultReceipts,
188
188
  createToolCallJob,
189
189
  resolveToolCallResultReceiptPolicy,
190
- ToolCallReceiptLeaseLostError,
191
190
  type ToolCallReceiptCohortStore,
192
191
  type ToolCallReceiptCohortTimings,
193
192
  type ToolCallReceiptRecord,
@@ -262,26 +261,16 @@ import {
262
261
  } from './log-provenance';
263
262
  import {
264
263
  COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
265
- DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS,
266
264
  createRuntimeReceiptHydrationGate,
267
265
  runtimeStepReceiptPayloadBytes,
268
- runtimeReceiptClaimLeaseIsLive,
269
- runtimeReceiptWaitBudgetMs,
270
- runtimeReceiptWaitDelayMs,
271
- DURABLE_RECEIPT_WAIT_DELAY_MS,
272
- RuntimeReceiptHydrationTurnTimeoutError,
273
266
  RuntimeReceiptLeaseLostError,
274
- RuntimeReceiptWaitTimeoutError,
275
267
  executeWithDurableRuntimeReceipt,
276
268
  resolveRuntimeToolReceiptWaitMaxAttempts,
277
- runtimeReceiptFailureError,
278
269
  runtimeReceiptFailureKindForError,
279
270
  runtimeReceiptOutput as durableRuntimeReceiptOutput,
280
271
  type RuntimeReceiptHydrationTurn,
281
272
  type RuntimeReceiptHydrationGateSnapshot,
282
273
  type RuntimeReceiptHydrationSource,
283
- waitForCompletedRuntimeReceipt,
284
- waitForCompletedRuntimeReceipts,
285
274
  type DurableReceiptExecutionStore,
286
275
  } from './durable-receipt-execution';
287
276
  import {
@@ -422,6 +411,7 @@ import type {
422
411
  ContextOptions,
423
412
  PlayCallOptions,
424
413
  PlayCheckpoint,
414
+ ProviderRetryCursor,
425
415
  PlayStep,
426
416
  PlayStepRowResult,
427
417
  PlayRowUpdate,
@@ -609,12 +599,6 @@ const DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN =
609
599
  /\bdeeplinedeveloper\.com\s*\|\s*502\s*:\s*bad gateway\b/i;
610
600
  const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100;
611
601
  const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25;
612
- // A newly claimed receipt should normally use its scheduled heartbeat. When a
613
- // test/SEV policy deliberately gives it only a moment of lease life, however,
614
- // scheduler setup and the first provider-dispatch turn can consume that entire
615
- // window before the timer fires. Renew once before fan-out in that narrow case.
616
- // Production's normal multi-minute lease remains on the zero-extra-query path.
617
- const IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS = 100;
618
602
  // Per-row sanity cap on distinct inline child invocations. `maxPlayCallDepth`
619
603
  // (governor) already bounds nesting; this bounds fan-WIDTH from one active row
620
604
  // resolver. The dedupe set is row-local, never retained for the full run.
@@ -1826,7 +1810,7 @@ type ToolExecutionApiOptions = {
1826
1810
  completedResultCache?: boolean;
1827
1811
  executionAuthScopeDigest?: string | null;
1828
1812
  providerIdempotencyKey?: string | null;
1829
- /** Exact forced-member identities that participate in a native batch key. */
1813
+ /** Invocation identities that participate in the canonical operation key. */
1830
1814
  providerOperationForceRefreshIdentities?: readonly string[];
1831
1815
  /** Presentation-only cache identity paired with a separate execution fence. */
1832
1816
  providerIdempotencyReceiptKey?: string | null;
@@ -1878,6 +1862,12 @@ type ToolExecutionApiOptions = {
1878
1862
  * cache stores only its terminal materialized output.
1879
1863
  */
1880
1864
  uncachedAsyncLifecycleCall?: boolean;
1865
+ /** Exact outgoing delivery candidates; consume a checkpoint cursor once. */
1866
+ legacyProviderRetryIdentities?: readonly {
1867
+ transportReceiptKey: string;
1868
+ providerOperationKey: string;
1869
+ forceRefreshIdentities?: readonly string[];
1870
+ }[];
1881
1871
  /**
1882
1872
  * Called once this call's provider response body has been fully read and
1883
1873
  * parsed, with that instant on the runner's clock. Measurement only: the
@@ -2594,12 +2584,6 @@ type ScalarPlayAuthoringRuntimeContext = Pick<
2594
2584
  export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2595
2585
  private rowStates = new Map<number, RowState>();
2596
2586
  private toolCallQueue: ToolCallRequest[] = [];
2597
- private pendingRuntimeToolOwnershipAssertions: Array<{
2598
- targets: Array<{ receiptKey: string; leaseId: string }>;
2599
- resolve: () => void;
2600
- reject: (error: unknown) => void;
2601
- }> = [];
2602
- private runtimeToolOwnershipAssertionFlushScheduled = false;
2603
2587
  private runtimeToolReceiptCompletionsInFlight = new Map<string, number>();
2604
2588
  /**
2605
2589
  * The one app-side result-egress controller. It serializes receipt gateway
@@ -3225,23 +3209,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
3225
3209
  : Date.now();
3226
3210
  this.checkpoint.durableCallCacheEpochMs = this.durableCallCacheEpochMs;
3227
3211
  this.durableMappedToolResultsBackedByReceipts = Boolean(
3228
- (options.claimRuntimeStepReceipt || options.claimRuntimeStepReceipts) &&
3229
3212
  (options.getRuntimeStepReceipt || options.getRuntimeStepReceipts) &&
3230
3213
  (options.completeRuntimeStepReceipt ||
3231
3214
  options.completeRuntimeStepReceipts),
3232
3215
  );
3233
- this.durableDirectToolResultsBackedByReceipts = Boolean(
3234
- options.claimRuntimeStepReceipt &&
3235
- options.getRuntimeStepReceipt &&
3236
- options.completeRuntimeStepReceipt,
3237
- );
3238
- if (this.durableDirectToolResultsBackedByReceipts) {
3239
- // A resumed durable runner may receive a legacy checkpoint containing
3240
- // full tool results. Drop that redundant copy immediately and recover
3241
- // through the receipt store below. Bulk-only receipt contexts retain the
3242
- // checkpoint because direct tool calls cannot use the bulk claim path.
3243
- this.checkpoint.completedToolBatches = {};
3244
- }
3216
+ this.durableDirectToolResultsBackedByReceipts =
3217
+ this.durableMappedToolResultsBackedByReceipts;
3218
+ // Existing checkpoints can be the sole completed cache from an outgoing
3219
+ // context. Receipt-backed calls stop adding payloads, but retain old hits.
3245
3220
  // The governance play id keys durable ctx receipts (durableCtxKey), the
3246
3221
  // cycle guard, and per-parent child-call counters. It must be the STABLE
3247
3222
  // play name — not the per-run workflow id — so receipts written by one
@@ -4323,7 +4298,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4323
4298
  key: string,
4324
4299
  ): Promise<RuntimeStepReceipt | null> {
4325
4300
  if (!this.#options.getRuntimeStepReceipt) {
4326
- return null;
4301
+ if (!this.#options.getRuntimeStepReceipts) return null;
4302
+ return (await this.getRuntimeStepReceipts([key])).get(key) ?? null;
4327
4303
  }
4328
4304
  const receipt = await this.#options.getRuntimeStepReceipt({ key });
4329
4305
  if (receipt && typeof receipt.key === 'string' && receipt.key.trim()) {
@@ -4465,7 +4441,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4465
4441
  path: `ctx.step receipt "${key}"`,
4466
4442
  });
4467
4443
  if (!this.#options.completeRuntimeStepReceipt) {
4468
- return null;
4444
+ if (!this.#options.completeRuntimeStepReceipts) return null;
4445
+ const completed = await this.completeRuntimeStepReceipts(
4446
+ [
4447
+ {
4448
+ key,
4449
+ runId: this.currentReceiptOwnerRunId,
4450
+ runAttempt: this.currentRunAttempt,
4451
+ ...(leaseId ? { leaseId } : {}),
4452
+ output,
4453
+ ...(inlineFields ? { inlineFields } : {}),
4454
+ },
4455
+ ],
4456
+ undefined,
4457
+ undefined,
4458
+ { retainLiveResult: true },
4459
+ );
4460
+ return completed.get(key) ?? null;
4469
4461
  }
4470
4462
  const completed = await this.#options.completeRuntimeStepReceipt({
4471
4463
  key,
@@ -4527,49 +4519,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4527
4519
  })),
4528
4520
  );
4529
4521
  if (manifests.length === 0) return;
4530
- // A scalar call publishes its pointer on its own, after the canonical
4531
- // result has already committed, so it never rides the cohort's shared
4532
- // egress wave. Nothing is gained by skipping the ownership handshake
4533
- // here, and skipping it costs the caller a receipt row it does not own:
4534
- // a runtime whose claim seam refuses (or that exposes no claim seam at
4535
- // all) would still publish pointer rows, and an already published
4536
- // pointer would be rewritten on every later call. Take the row the way
4537
- // every other scalar receipt writer takes it, and write only what this
4538
- // run actually owns. The map/cohort publisher keeps the claim-free
4539
- // batched write, where the pointer genuinely shares the canonical
4540
- // result's single bounded request.
4522
+ // The pointer is an immutable completed-cache insert, like the result
4523
+ // it names. Publishing it needs no pending lease or read-back.
4541
4524
  if (
4542
4525
  !(
4543
- this.#options.claimRuntimeStepReceipt ||
4544
- this.#options.claimRuntimeStepReceipts
4526
+ this.#options.completeRuntimeStepReceipt ||
4527
+ this.#options.completeRuntimeStepReceipts
4545
4528
  )
4546
4529
  ) {
4547
4530
  return;
4548
4531
  }
4549
- const existing = await this.getRuntimeStepReceipts(
4550
- manifests.map((manifest) => manifest.key),
4551
- { hydrateOutput: false },
4552
- );
4553
- const missing = manifests.filter((manifest) => {
4554
- const receipt = existing.get(manifest.key);
4555
- return receipt?.status !== 'completed' && receipt?.status !== 'skipped';
4556
- });
4557
- if (missing.length === 0) return;
4558
- const claims = await this.claimRuntimeStepReceipts(
4559
- missing.map((manifest) => manifest.key),
4560
- sourceRunId,
4561
- false,
4562
- false,
4563
- false,
4564
- false,
4565
- );
4566
- const owned = missing.flatMap((manifest) => {
4567
- const leaseId = claims.get(manifest.key)?.leaseId;
4568
- return leaseId ? [{ ...manifest, leaseId }] : [];
4569
- });
4570
- if (owned.length === 0) return;
4571
- await this.completeRuntimeStepReceipts(owned, undefined, undefined, {
4572
- bestEffortKeys: new Set(owned.map((manifest) => manifest.key)),
4532
+ await this.completeRuntimeStepReceipts(manifests, undefined, undefined, {
4533
+ bestEffortKeys: new Set(manifests.map((manifest) => manifest.key)),
4573
4534
  onBestEffortFailure: ({ key, error }) => {
4574
4535
  this.runtimeLog(
4575
4536
  `Tool source-run receipt manifest write skipped for ${key}: ${error instanceof Error ? error.message : String(error)}`,
@@ -4789,178 +4750,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4789
4750
  };
4790
4751
  }
4791
4752
 
4792
- private async assertRuntimeToolReceiptOwnership(
4793
- requests: ToolCallRequest[],
4794
- ): Promise<void> {
4795
- const targets = this.runtimeToolReceiptOwnershipTargets(requests);
4796
- if (targets.length === 0) return;
4797
-
4798
- return await new Promise<void>((resolve, reject) => {
4799
- this.pendingRuntimeToolOwnershipAssertions.push({
4800
- targets,
4801
- resolve,
4802
- reject,
4803
- });
4804
- if (this.runtimeToolOwnershipAssertionFlushScheduled) return;
4805
- this.runtimeToolOwnershipAssertionFlushScheduled = true;
4806
- queueMicrotask(() => void this.flushRuntimeToolOwnershipAssertionBatch());
4807
- });
4808
- }
4809
-
4810
- private runtimeToolReceiptOwnershipTargets(
4811
- requests: ToolCallRequest[],
4812
- ): Array<{ receiptKey: string; leaseId: string }> {
4813
- return requests.flatMap((request) => {
4814
- const receiptKey = request.receiptKey?.trim() || null;
4815
- if (!receiptKey) return [];
4816
- const leaseId = request.receiptLeaseId?.trim() || null;
4817
- if (!leaseId) return [];
4818
- return [{ receiptKey, leaseId }];
4819
- });
4820
- }
4821
-
4822
- private async renewRuntimeToolReceiptOwnership(
4823
- requests: ToolCallRequest[],
4824
- ): Promise<void> {
4825
- const targets = this.runtimeToolReceiptOwnershipTargets(requests);
4826
- if (targets.length === 0) return;
4827
- if (!this.#options.heartbeatRuntimeStepReceipts) {
4828
- return await this.assertRuntimeToolReceiptOwnership(requests);
4829
- }
4830
- const uniqueTargets = [
4831
- ...new Map(targets.map((target) => [target.receiptKey, target])).values(),
4832
- ];
4833
- const receipts = await this.#options.heartbeatRuntimeStepReceipts({
4834
- runId: this.currentReceiptOwnerRunId,
4835
- runAttempt: this.currentRunAttempt,
4836
- keys: uniqueTargets.map((target) => target.receiptKey),
4837
- leaseIds: uniqueTargets.map((target) => target.leaseId),
4838
- });
4839
- this.assertRuntimeToolReceiptHeartbeatResults(uniqueTargets, receipts);
4840
- }
4841
-
4842
- private async flushRuntimeToolOwnershipAssertionBatch(): Promise<void> {
4843
- this.runtimeToolOwnershipAssertionFlushScheduled = false;
4844
- const assertions = this.pendingRuntimeToolOwnershipAssertions.splice(0);
4845
- if (assertions.length === 0) return;
4846
- const targets = assertions.flatMap((assertion) => assertion.targets);
4847
-
4848
- if (this.#options.heartbeatRuntimeStepReceipts) {
4849
- try {
4850
- const uniqueTargets = [
4851
- ...new Map(
4852
- targets.map((target) => [target.receiptKey, target]),
4853
- ).values(),
4854
- ];
4855
- const receipts = await this.#options.heartbeatRuntimeStepReceipts({
4856
- runId: this.currentReceiptOwnerRunId,
4857
- runAttempt: this.currentRunAttempt,
4858
- keys: uniqueTargets.map((target) => target.receiptKey),
4859
- leaseIds: uniqueTargets.map((target) => target.leaseId),
4860
- });
4861
- const renewed = new Map<string, RuntimeStepReceipt | null>();
4862
- for (let index = 0; index < uniqueTargets.length; index += 1) {
4863
- renewed.set(
4864
- uniqueTargets[index]!.receiptKey,
4865
- receipts[index] ?? null,
4866
- );
4867
- }
4868
- for (const assertion of assertions) {
4869
- const lost = assertion.targets.find((target) => {
4870
- const receipt = renewed.get(target.receiptKey);
4871
- return !this.runtimeToolReceiptStillOwned(receipt, target.leaseId);
4872
- });
4873
- if (lost) {
4874
- assertion.reject(
4875
- new RuntimeReceiptLeaseLostError({
4876
- receiptKey: lost.receiptKey,
4877
- runId: this.currentReceiptOwnerRunId,
4878
- leaseId: lost.leaseId,
4879
- }),
4880
- );
4881
- } else {
4882
- assertion.resolve();
4883
- }
4884
- }
4885
- } catch (error) {
4886
- for (const assertion of assertions) assertion.reject(error);
4887
- }
4888
- return;
4889
- }
4890
-
4891
- if (
4892
- this.#options.getRuntimeStepReceipt ||
4893
- this.#options.getRuntimeStepReceipts
4894
- ) {
4895
- try {
4896
- const latest = await this.getRuntimeStepReceipts(
4897
- targets.map((target) => target.receiptKey),
4898
- );
4899
- for (const assertion of assertions) {
4900
- const lost = assertion.targets.find((target) => {
4901
- const receipt = latest.get(target.receiptKey);
4902
- return !this.runtimeToolReceiptStillOwned(receipt, target.leaseId);
4903
- });
4904
- if (lost) {
4905
- assertion.reject(
4906
- new RuntimeReceiptLeaseLostError({
4907
- receiptKey: lost.receiptKey,
4908
- runId: this.currentReceiptOwnerRunId,
4909
- leaseId: lost.leaseId,
4910
- }),
4911
- );
4912
- } else {
4913
- assertion.resolve();
4914
- }
4915
- }
4916
- } catch (error) {
4917
- for (const assertion of assertions) assertion.reject(error);
4918
- }
4919
- return;
4920
- }
4921
-
4922
- const error = new RuntimeReceiptLeaseLostError({
4923
- receiptKey: targets[0]?.receiptKey ?? 'unknown',
4924
- runId: this.currentReceiptOwnerRunId,
4925
- leaseId: targets[0]?.leaseId ?? 'unknown',
4926
- });
4927
- for (const assertion of assertions) assertion.reject(error);
4928
- }
4929
-
4930
- private assertRuntimeToolReceiptHeartbeatResults(
4931
- targets: Array<{ receiptKey: string; leaseId: string }>,
4932
- receipts: Array<RuntimeStepReceipt | null>,
4933
- ): void {
4934
- for (let index = 0; index < targets.length; index += 1) {
4935
- const target = targets[index]!;
4936
- const receipt = this.normalizeRuntimeStepReceipt(
4937
- target.receiptKey,
4938
- receipts[index],
4939
- );
4940
- if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) {
4941
- throw new RuntimeReceiptLeaseLostError({
4942
- receiptKey: target.receiptKey,
4943
- runId: this.currentReceiptOwnerRunId,
4944
- leaseId: target.leaseId,
4945
- });
4946
- }
4947
- }
4948
- }
4949
-
4950
- private runtimeToolReceiptStillOwned(
4951
- receipt: RuntimeStepReceipt | null | undefined,
4952
- leaseId: string,
4953
- ): boolean {
4954
- if (!receipt) return false;
4955
- if (receipt.status === 'completed' || receipt.status === 'skipped') {
4956
- return true;
4957
- }
4958
- if (receipt.status !== 'running') return false;
4959
- if (receipt.leaseId !== leaseId) return false;
4960
- const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
4961
- return !ownerRunId || ownerRunId === this.currentReceiptOwnerRunId;
4962
- }
4963
-
4964
4753
  private runtimeToolReceiptStillOwnedByCurrentAttempt(
4965
4754
  receipt: RuntimeStepReceipt | null | undefined,
4966
4755
  leaseId: string | null | undefined,
@@ -4980,18 +4769,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4980
4769
  return ownerAttempt === this.currentRunAttempt;
4981
4770
  }
4982
4771
 
4983
- private isOwnedClaimedRuntimeReceipt(
4984
- receipt: RuntimeStepReceipt | null | undefined,
4985
- ): receipt is RuntimeStepReceipt {
4986
- return (
4987
- Boolean(receipt) &&
4988
- receipt?.claimState !== 'existing' &&
4989
- (receipt?.status === 'queued' ||
4990
- receipt?.status === 'pending' ||
4991
- receipt?.status === 'running')
4992
- );
4993
- }
4994
-
4995
4772
  private async failRuntimeStepReceipt(
4996
4773
  key: string,
4997
4774
  _runId: string,
@@ -5359,91 +5136,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
5359
5136
  return this.normalizeRuntimeStepReceipt(input.key, receipt);
5360
5137
  }
5361
5138
 
5362
- private async markRuntimeStepReceiptsRunning(
5363
- receipts: Array<{
5364
- key: string;
5365
- runId: string;
5366
- leaseId?: string | null;
5367
- }>,
5368
- ): Promise<Map<string, RuntimeStepReceipt>> {
5369
- const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
5370
- if (normalizedInputs.length === 0) return new Map();
5371
- let marked: Array<RuntimeStepReceipt | null>;
5372
- const markReceipts = this.#options.markRuntimeStepReceiptsRunning;
5373
- if (markReceipts) {
5374
- marked = await this.dispatchChunkedRuntimeReceiptRequest(
5375
- normalizedInputs.map((receipt) => ({
5376
- ...receipt,
5377
- runId: this.currentReceiptOwnerRunId,
5378
- runAttempt: this.currentRunAttempt,
5379
- })),
5380
- (chunk) => markReceipts({ receipts: chunk }),
5381
- );
5382
- } else {
5383
- marked = await Promise.all(
5384
- normalizedInputs.map((receipt) =>
5385
- this.markRuntimeStepReceiptRunning(receipt),
5386
- ),
5387
- );
5388
- }
5389
- const byKey = new Map<string, RuntimeStepReceipt>();
5390
- for (let index = 0; index < marked.length; index += 1) {
5391
- const normalized = this.normalizeRuntimeStepReceipt(
5392
- normalizedInputs[index]?.key ?? '',
5393
- marked[index],
5394
- );
5395
- if (normalized) byKey.set(normalized.key, normalized);
5396
- }
5397
- return byKey;
5398
- }
5399
-
5400
- private async markRuntimeStepReceiptsQueued(
5401
- receipts: Array<{
5402
- key: string;
5403
- runId: string;
5404
- leaseId?: string | null;
5405
- }>,
5406
- ): Promise<Map<string, RuntimeStepReceipt>> {
5407
- const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
5408
- if (normalizedInputs.length === 0) return new Map();
5409
- if (!this.#options.markRuntimeStepReceiptsQueued) {
5410
- return new Map();
5411
- }
5412
- const queued = await this.dispatchChunkedRuntimeReceiptRequest(
5413
- normalizedInputs.map((receipt) => ({
5414
- ...receipt,
5415
- runId: this.currentReceiptOwnerRunId,
5416
- runAttempt: this.currentRunAttempt,
5417
- })),
5418
- (chunk) =>
5419
- this.#options.markRuntimeStepReceiptsQueued!({ receipts: chunk }),
5420
- );
5421
- const byKey = new Map<string, RuntimeStepReceipt>();
5422
- for (let index = 0; index < queued.length; index += 1) {
5423
- const normalized = this.normalizeRuntimeStepReceipt(
5424
- normalizedInputs[index]?.key ?? '',
5425
- queued[index],
5426
- );
5427
- if (normalized) byKey.set(normalized.key, normalized);
5428
- }
5429
- return byKey;
5430
- }
5431
-
5432
- private async parkRuntimeToolReceiptsQueued(
5433
- requests: ToolCallRequest[],
5434
- ): Promise<void> {
5435
- const targets = requests.flatMap((request) => {
5436
- const receiptKey = request.receiptKey?.trim() || null;
5437
- const leaseId = request.receiptLeaseId?.trim() || null;
5438
- if (!receiptKey || !leaseId) return [];
5439
- return [
5440
- { key: receiptKey, runId: this.currentReceiptOwnerRunId, leaseId },
5441
- ];
5442
- });
5443
- if (targets.length === 0) return;
5444
- await this.markRuntimeStepReceiptsQueued(targets);
5445
- }
5446
-
5447
5139
  /**
5448
5140
  * Enforce the receipt output contract and remember the output's serialized
5449
5141
  * size. Idempotent per input: the first caller pays, everything downstream
@@ -6326,7 +6018,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6326
6018
 
6327
6019
  private durableReceiptExecutionStore(): DurableReceiptExecutionStore {
6328
6020
  return {
6329
- enabled: Boolean(this.#options.claimRuntimeStepReceipt),
6021
+ enabled: this.durableDirectToolResultsBackedByReceipts,
6330
6022
  get: (receiptKey) => this.getRuntimeStepReceipt(receiptKey),
6331
6023
  getMany: (receiptKeys) => this.getRuntimeStepReceipts(receiptKeys),
6332
6024
  claim: (
@@ -6403,35 +6095,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6403
6095
  return durableRuntimeReceiptOutput<T>(receipt);
6404
6096
  }
6405
6097
 
6406
- private async waitForCompletedRuntimeToolReceipt(
6407
- key: string,
6408
- maxAttempts?: number,
6409
- ): Promise<RuntimeStepReceipt> {
6410
- return await waitForCompletedRuntimeReceipt({
6411
- receiptKey: key,
6412
- store: this.durableReceiptExecutionStore(),
6413
- maxAttempts,
6414
- toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
6415
- });
6416
- }
6417
-
6418
- private async waitForCompletedRuntimeToolReceipts(
6419
- keys: string[],
6420
- maxAttempts?: number,
6421
- ): Promise<{
6422
- completed: Map<string, RuntimeStepReceipt>;
6423
- failed: Map<string, Error>;
6424
- timedOut: Set<string>;
6425
- }> {
6426
- return await waitForCompletedRuntimeReceipts({
6427
- receiptKeys: keys,
6428
- store: this.durableReceiptExecutionStore(),
6429
- maxAttempts,
6430
- log: (message) => this.runtimeLog(message),
6431
- toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
6432
- });
6433
- }
6434
-
6435
6098
  private async executeWithRuntimeReceipt<T>(
6436
6099
  operation: DurableCtxOperation,
6437
6100
  id: string,
@@ -6525,7 +6188,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6525
6188
  onClaimedResult: opts.onClaimedResult,
6526
6189
  shouldPersistFailure: opts.shouldPersistFailure,
6527
6190
  markRunningBeforeExecute: opts.markRunningBeforeExecute,
6528
- completedCacheOnly: operation === 'tool',
6191
+ completedCacheOnly: true,
6529
6192
  withCompletedReceiptHydration: (hydrate) =>
6530
6193
  this.withRuntimeReceiptHydrationTurn(hydrate, {
6531
6194
  source: 'durable_operation',
@@ -6632,7 +6295,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6632
6295
  *
6633
6296
  * The external receipt adapter is intentionally local to this migration.
6634
6297
  * It checks the exact compatibility identities passed by the map, while the
6635
- * map remains the sole writer for its member receipts, follower delivery,
6298
+ * map remains the sole writer for its member receipts, invocation delivery,
6636
6299
  * and completion barrier. In particular, do not turn an always-fresh map
6637
6300
  * request into a provider-idempotent request here: `apiOptions` is forwarded
6638
6301
  * byte-for-byte and decides whether the historical wire carried an identity.
@@ -6806,19 +6469,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6806
6469
  });
6807
6470
  }
6808
6471
 
6809
- /**
6810
- * A forced call bypasses an earlier run's shared result, so its physical
6811
- * gateway invocation must bypass that result's replay identity too. Keep
6812
- * the transport key deterministic for one logical call: a retry of that
6813
- * call still converges, while a later explicit force gets a fresh identity.
6814
- */
6472
+ /** One replay-stable physical delivery identity for each invocation. */
6815
6473
  private toolTransportReceiptKey(input: {
6816
6474
  toolId: string;
6817
- resultReceiptKey: string;
6818
6475
  providerOperationKey: string;
6819
- force?: boolean;
6820
6476
  }): string {
6821
- if (input.force !== true) return input.resultReceiptKey;
6822
6477
  return `${buildDurableToolReceiptPrefix({
6823
6478
  orgId: this.#options.orgId,
6824
6479
  toolId: input.toolId,
@@ -6841,41 +6496,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6841
6496
  });
6842
6497
  }
6843
6498
 
6844
- /**
6845
- * Ordinary operations do not carry provider-idempotency metadata, but the
6846
- * execution relay still needs a semantic signal that an explicit force is
6847
- * a new provider operation. Unsafe-write operations already carry their
6848
- * forced provider key directly, so only ordinary calls need this attested
6849
- * canonical-operation salt.
6850
- */
6851
- private providerForceRefreshIdentitiesForToolCall(input: {
6499
+ /** Keep independent misses distinct in the gateway's canonical operation. */
6500
+ private providerInvocationOptionsForToolCall(input: {
6852
6501
  operationKey: string;
6853
- force?: boolean;
6854
6502
  requiresExecutionFence?: boolean;
6855
6503
  logicalCallId?: string;
6856
- }): string[] {
6857
- if (input.force !== true || input.requiresExecutionFence === true) {
6858
- return [];
6859
- }
6860
- return [
6861
- this.providerIdempotencyKeyForToolCall({
6862
- operationKey: input.operationKey,
6863
- force: true,
6864
- logicalCallId: input.logicalCallId,
6865
- }),
6866
- ];
6867
- }
6868
-
6869
- private providerForceRefreshOptionsForToolCall(input: {
6870
- operationKey: string;
6871
6504
  force?: boolean;
6872
- requiresExecutionFence?: boolean;
6873
- logicalCallId?: string;
6874
6505
  }): Pick<ToolExecutionApiOptions, 'providerOperationForceRefreshIdentities'> {
6875
- const identities = this.providerForceRefreshIdentitiesForToolCall(input);
6876
- return identities.length > 0
6877
- ? { providerOperationForceRefreshIdentities: identities }
6878
- : {};
6506
+ return input.requiresExecutionFence === true
6507
+ ? {}
6508
+ : {
6509
+ providerOperationForceRefreshIdentities: [
6510
+ this.providerInvocationKeyForToolCall(input),
6511
+ ],
6512
+ };
6879
6513
  }
6880
6514
 
6881
6515
  /**
@@ -7577,15 +7211,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7577
7211
  private getCachedToolResult(
7578
7212
  toolId: string,
7579
7213
  rowCacheKey: string,
7580
- path: 'mapped' | 'direct' = 'mapped',
7214
+ _path: 'mapped' | 'direct' = 'mapped',
7581
7215
  ): ToolBatchResult | undefined {
7582
- if (
7583
- path === 'direct'
7584
- ? this.durableDirectToolResultsBackedByReceipts
7585
- : this.durableMappedToolResultsBackedByReceipts
7586
- ) {
7587
- return undefined;
7588
- }
7216
+ // Receipt-backed calls never add payloads here. Retained outgoing entries
7217
+ // may predate that capability and remain completed cache hits.
7589
7218
  return this.checkpoint.completedToolBatches[toolId]?.[rowCacheKey];
7590
7219
  }
7591
7220
 
@@ -7636,32 +7265,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7636
7265
  if (!this.checkpoint.completedToolBatches[toolId]) {
7637
7266
  this.checkpoint.completedToolBatches[toolId] = {};
7638
7267
  }
7639
- this.checkpoint.completedToolBatches[toolId]![rowCacheKey] = {
7268
+ this.checkpoint.completedToolBatches[toolId]![rowCacheKey] ??= {
7640
7269
  done: true,
7641
7270
  result,
7642
7271
  };
7643
7272
  }
7644
7273
 
7645
- private buildToolResultCacheKey(input: {
7646
- rowId: number;
7647
- tableNamespace?: string;
7648
- rowKey?: string;
7649
- callId?: string;
7650
- }): string {
7651
- const scope =
7652
- this.currentGovernance.currentRunId || this.#options.runId || 'run';
7653
- if (input.callId?.trim()) {
7654
- return `${scope}:${input.callId.trim()}`;
7655
- }
7656
- if (input.rowKey?.trim()) {
7657
- return `${scope}:${input.rowKey.trim()}`;
7658
- }
7659
- if (input.tableNamespace?.trim()) {
7660
- return `${scope}:${input.tableNamespace.trim()}:${input.rowId}`;
7661
- }
7662
- return `${scope}:direct:${input.rowId}`;
7663
- }
7664
-
7665
7274
  private async resolveToolResultMetadata(
7666
7275
  toolId: string,
7667
7276
  ): Promise<ToolResultMetadataInput> {
@@ -8136,40 +7745,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8136
7745
  `Durable tool call ${receiptKey} completed live after its receipt lease moved; using live result without overwriting the receipt.`,
8137
7746
  );
8138
7747
  }
8139
- const finalWrapped =
8140
- (completed?.status === 'completed' || completed?.status === 'skipped') &&
8141
- completed.output !== undefined
8142
- ? await this.wrapToolExecutionResult({
8143
- toolId,
8144
- status:
8145
- completed.output === null || completed.output === undefined
8146
- ? 'no_result'
8147
- : 'completed',
8148
- result: this.runtimeReceiptOutput(completed),
8149
- requestInput: request.input,
8150
- execution: toolExecutionMetadataForOutcome(
8151
- completed.runId === this.currentReceiptOwnerRunId
8152
- ? {
8153
- kind: 'live',
8154
- cacheKey,
8155
- receiptKey: request.receiptKey,
8156
- }
8157
- : {
8158
- kind: 'cache',
8159
- cacheKey,
8160
- receiptKey: receiptKey ?? '',
8161
- attachedToReceiptKey: receiptKey,
8162
- },
8163
- ),
8164
- })
8165
- : wrapped;
8166
7748
  if (request.cacheable !== false) {
8167
- this.cacheToolResult(toolId, cacheKey, finalWrapped);
7749
+ this.cacheToolResult(toolId, cacheKey, wrapped);
8168
7750
  }
8169
7751
 
8170
7752
  const resolver = this.toolCallResolvers.get(request.callId);
8171
7753
  if (resolver) {
8172
- resolver.resolve(finalWrapped);
7754
+ resolver.resolve(wrapped);
8173
7755
  this.toolCallResolvers.delete(request.callId);
8174
7756
  }
8175
7757
 
@@ -8185,7 +7767,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8185
7767
  error: null,
8186
7768
  dataPatch: {},
8187
7769
  });
8188
- return finalWrapped;
7770
+ return wrapped;
8189
7771
  }
8190
7772
 
8191
7773
  private async rejectToolCall(
@@ -9003,6 +8585,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9003
8585
  Math.max(0, failedCount - staleFailedKeys.size),
9004
8586
  progressSupersededOffset: () =>
9005
8587
  staleCompletedKeys.size + staleFailedKeys.size,
8588
+ progressSkippedOffset: () => duplicateReuseCount,
9006
8589
  progressTotalRows: () => totalInputCount,
9007
8590
  rowFeed: execution.rowFeed,
9008
8591
  retainedRowsMemoryBudgetBytes:
@@ -10231,6 +9814,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10231
9814
  progressCompletedOffset?: number | (() => number);
10232
9815
  progressFailedOffset?: number | (() => number);
10233
9816
  progressSupersededOffset?: number | (() => number);
9817
+ /** Input duplicates discovered after admission; excludes resolver outcomes. */
9818
+ progressSkippedOffset?: number | (() => number);
10234
9819
  progressTotalRows?: number | (() => number);
10235
9820
  /**
10236
9821
  * When set, `items`, `executionRowKeys` and `executionRowIndexes` grow
@@ -10476,6 +10061,37 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10476
10061
  } | null = null;
10477
10062
  let lastFrameFlushAt = 0;
10478
10063
  let rowsSinceFrameFlush = 0;
10064
+ // Scalars only: resolver admission is visible before the first row's
10065
+ // durable completion, without publishing one event per row or tool call.
10066
+ let startedResolverRows = 0;
10067
+ let activeResolverRows = 0;
10068
+ let tracksResolverActivity = false;
10069
+ // Persistence can move current resolvers between successful, failed and
10070
+ // superseded offsets. They already contribute to startedResolverRows:
10071
+ // only the settled baseline before this pool started belongs in admission.
10072
+ const initiallyStartedRows =
10073
+ liveNumber(runtimeOptions?.progressCompletedOffset, completedRows) +
10074
+ liveNumber(runtimeOptions?.progressFailedOffset, 0) +
10075
+ liveNumber(runtimeOptions?.progressSupersededOffset, 0);
10076
+ const initiallySkippedRows = liveNumber(
10077
+ runtimeOptions?.progressSkippedOffset,
10078
+ 0,
10079
+ );
10080
+ const mapActivityCounts = () => {
10081
+ const startedRows =
10082
+ initiallyStartedRows +
10083
+ (liveNumber(
10084
+ runtimeOptions?.progressSkippedOffset,
10085
+ initiallySkippedRows,
10086
+ ) -
10087
+ initiallySkippedRows) +
10088
+ startedResolverRows;
10089
+ return {
10090
+ startedRows,
10091
+ activeRows: activeResolverRows,
10092
+ waitingRows: currentTotalRows() - startedRows,
10093
+ };
10094
+ };
10479
10095
  const updateMapFrameProgress = (input: {
10480
10096
  status?: MapExecutionFrame['status'];
10481
10097
  completedRowKey?: string | null;
@@ -10607,6 +10223,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10607
10223
  completedRowKeys.size,
10608
10224
  failedRows,
10609
10225
  totalRows: currentTotalRows(),
10226
+ ...(progressEventType === 'map.progress' && tracksResolverActivity
10227
+ ? mapActivityCounts()
10228
+ : {}),
10610
10229
  supersededRows: liveNumber(
10611
10230
  runtimeOptions?.progressSupersededOffset,
10612
10231
  existing.supersededRows ?? 0,
@@ -11254,6 +10873,39 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11254
10873
  items[idx] = null as unknown as T;
11255
10874
  }
11256
10875
  : null;
10876
+ let lastMapActivityKey: string | null = null;
10877
+ tracksResolverActivity = true;
10878
+ const publishMapActivity = () => {
10879
+ const counts = mapActivityCounts();
10880
+ const key = `${counts.startedRows}:${counts.activeRows}:${counts.waitingRows}`;
10881
+ if (key === lastMapActivityKey) return;
10882
+ lastMapActivityKey = key;
10883
+ // Admission samples do not rebuild checkpoint key sets. The map's
10884
+ // writer continues to own durable settlement and checkpoint flushing.
10885
+ this.emitExecutionEvent({
10886
+ type: 'map.progress',
10887
+ mapInvocationId: mapScope.mapInvocationId,
10888
+ mapNodeId: mapScope.mapNodeId ?? null,
10889
+ logicalNamespace: mapScope.logicalNamespace,
10890
+ artifactTableNamespace: mapScope.artifactTableNamespace,
10891
+ completedRows:
10892
+ liveNumber(runtimeOptions?.progressCompletedOffset, completedRows) +
10893
+ (liveFrameSets?.completedRowKeys.size ?? 0),
10894
+ failedRows:
10895
+ liveNumber(runtimeOptions?.progressFailedOffset, 0) +
10896
+ (liveFrameSets?.failedRowsCount ?? 0),
10897
+ totalRows: currentTotalRows(),
10898
+ supersededRows: liveNumber(runtimeOptions?.progressSupersededOffset, 0),
10899
+ ...counts,
10900
+ ...this.inlineChildAggregateEventFields(),
10901
+ at: Date.now(),
10902
+ });
10903
+ };
10904
+ const mapActivityTimer = setInterval(
10905
+ publishMapActivity,
10906
+ PROGRESS_HEARTBEAT_INTERVAL_MS,
10907
+ );
10908
+ mapActivityTimer.unref();
11257
10909
  const workers = Array.from({ length: workerCount }, async () => {
11258
10910
  while (firstRowError == null) {
11259
10911
  if (nextRowIndex >= items.length) {
@@ -11270,7 +10922,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11270
10922
  releaseDispatchedInputRow?.(idx);
11271
10923
  settledResults[idx] = await mapRowFlow.run({ baseRow }, async () => {
11272
10924
  this.assertRowAdmissionOpen();
11273
- return await runRow(idx, baseRow);
10925
+ startedResolverRows += 1;
10926
+ activeResolverRows += 1;
10927
+ try {
10928
+ return await runRow(idx, baseRow);
10929
+ } finally {
10930
+ activeResolverRows -= 1;
10931
+ }
11274
10932
  });
11275
10933
  } catch (error) {
11276
10934
  // RuntimeFlow has already returned the row slot and memory lease.
@@ -11364,6 +11022,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11364
11022
  settled = await Promise.allSettled(workers);
11365
11023
  }
11366
11024
  } finally {
11025
+ clearInterval(mapActivityTimer);
11026
+ publishMapActivity();
11367
11027
  datasetStep.items = currentTotalRows();
11368
11028
  this.processedRowCount = currentTotalRows();
11369
11029
  mapStallObserver?.stopHeartbeat();
@@ -12930,7 +12590,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
12930
12590
  options.receiptWaitMs,
12931
12591
  );
12932
12592
  }
12933
- const executionScope = this.currentExecutionScope;
12934
12593
  const normalizedKey = this.normalizeContextKey(key, 'tool');
12935
12594
  const toolCachePolicy = this.effectiveToolCallCachePolicy(options);
12936
12595
  const toolRequestIdentity = deriveToolRequestIdentity({
@@ -13028,10 +12687,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13028
12687
  ...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
13029
12688
  ];
13030
12689
 
13031
- const executeTool = async (context?: {
13032
- leaseId?: string | null;
13033
- retainExternalCallSlot?: (release: () => void) => void;
13034
- }): Promise<unknown> => {
12690
+ const executeTool = async (): Promise<unknown> => {
13035
12691
  if (eventWaitHandler) {
13036
12692
  return this.executeIntegrationEventWaitTool(
13037
12693
  toolId,
@@ -13291,7 +12947,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13291
12947
  });
13292
12948
  };
13293
12949
  try {
13294
- const directReceiptLeaseId = context?.leaseId?.trim() || null;
13295
12950
  if (!logicalCallId) {
13296
12951
  throw new Error(
13297
12952
  'Direct tool execution requires a stable logical call identity.',
@@ -13299,17 +12954,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13299
12954
  }
13300
12955
  const useDirectResultReceipts =
13301
12956
  cacheableToolResult && !bypassDirectResultCache;
13302
- const providerOperationKey = this.providerIdempotencyKeyForToolCall({
12957
+ const providerOperationKey = this.providerInvocationKeyForToolCall({
13303
12958
  operationKey: providerIdempotencyKeyBase,
13304
- force: cacheableToolResult && toolCachePolicy.force,
13305
12959
  logicalCallId,
12960
+ force: toolCachePolicy.force,
13306
12961
  });
13307
12962
  const physicalDirectKey = useDirectResultReceipts
13308
12963
  ? this.toolTransportReceiptKey({
13309
12964
  toolId,
13310
- resultReceiptKey: directCacheKey,
13311
12965
  providerOperationKey,
13312
- force: toolCachePolicy.force,
13313
12966
  })
13314
12967
  : `${buildDurableToolReceiptPrefix({
13315
12968
  orgId: this.#options.orgId ?? 'unknown-org',
@@ -13323,13 +12976,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13323
12976
  runId: this.currentRunId,
13324
12977
  logicalCallId,
13325
12978
  force: toolCachePolicy.force,
13326
- canStoreResult: Boolean(
13327
- this.#options.claimRuntimeStepReceipt &&
13328
- (this.#options.getRuntimeStepReceipt ||
13329
- this.#options.getRuntimeStepReceipts) &&
13330
- (this.#options.completeRuntimeStepReceipt ||
13331
- this.#options.completeRuntimeStepReceipts),
13332
- ),
12979
+ canStoreResult: this.durableDirectToolResultsBackedByReceipts,
13333
12980
  });
13334
12981
  const toolRetryPolicy = await this.#options
13335
12982
  .getToolRetryPolicy?.(toolId, input)
@@ -13365,28 +13012,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13365
13012
  ...(useDirectResultReceipts
13366
13013
  ? {
13367
13014
  durableCallReceiptKey: physicalDirectKey,
13015
+ legacyProviderRetryIdentities: toolCachePolicy.force
13016
+ ? undefined
13017
+ : [
13018
+ {
13019
+ transportReceiptKey: directCacheKey,
13020
+ providerOperationKey:
13021
+ providerIdempotencyKeyBase,
13022
+ },
13023
+ ],
13368
13024
  completedResultCache: true,
13369
13025
  executionAuthScopeDigest,
13370
- ...(directReceiptLeaseId &&
13371
- (this.#options.heartbeatRuntimeStepReceipts ||
13372
- this.#options.getRuntimeStepReceipt ||
13373
- this.#options.getRuntimeStepReceipts)
13374
- ? {
13375
- beforeProviderCall: async () => {
13376
- await this.assertRuntimeToolReceiptOwnership([
13377
- {
13378
- callId: 'direct',
13379
- cacheKey: directCacheKey,
13380
- receiptKey: directCacheKey,
13381
- receiptLeaseId: directReceiptLeaseId,
13382
- rowId: -1,
13383
- toolId,
13384
- input,
13385
- },
13386
- ]);
13387
- },
13388
- }
13389
- : {}),
13390
13026
  }
13391
13027
  : {}),
13392
13028
  ...(toolRetryPolicy?.requiresExecutionFence === true
@@ -13401,15 +13037,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13401
13037
  : physicalDirectKey,
13402
13038
  }
13403
13039
  : {}),
13404
- ...(useDirectResultReceipts && operationKey
13405
- ? this.providerForceRefreshOptionsForToolCall({
13406
- operationKey: providerIdempotencyKeyBase,
13407
- force: toolCachePolicy.force,
13408
- requiresExecutionFence:
13409
- toolRetryPolicy?.requiresExecutionFence,
13410
- logicalCallId,
13411
- })
13412
- : {}),
13040
+ ...this.providerInvocationOptionsForToolCall({
13041
+ operationKey: providerIdempotencyKeyBase,
13042
+ requiresExecutionFence:
13043
+ toolRetryPolicy?.requiresExecutionFence,
13044
+ logicalCallId,
13045
+ force: toolCachePolicy.force,
13046
+ }),
13413
13047
  playNodeScope: buildPlayNodeScope({
13414
13048
  toolId,
13415
13049
  callKey: normalizedKey,
@@ -13419,7 +13053,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13419
13053
  options?.timeoutMs,
13420
13054
  this.currentAuthoringContractEdition,
13421
13055
  ),
13422
- retainToolSlot: context?.retainExternalCallSlot,
13423
13056
  },
13424
13057
  });
13425
13058
  return await this.wrapToolExecutionResult({
@@ -13558,20 +13191,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13558
13191
 
13559
13192
  const rowId = store.rowId;
13560
13193
  const fieldName = store.fieldName;
13561
- const callId = [
13562
- executionScope.receipt.namespace,
13563
- store.tableNamespace?.trim() || 'map',
13564
- store.rowKey?.trim() || String(rowId),
13565
- normalizedKey,
13566
- toolId,
13567
- toolRequestIdentity,
13568
- ].join(':');
13569
- if (this.toolCallResolvers.has(callId)) {
13570
- throw new Error(
13571
- `ctx.tools.execute("${normalizedKey}") was called more than once concurrently for the same row. ` +
13572
- 'Use a unique id for each row-scoped tool call.',
13573
- );
13574
- }
13194
+ // Cache identity is shared by matching completed results. Delivery
13195
+ // identity belongs to this invocation, even when callers reuse an id.
13196
+ const callId = logicalCallId;
13575
13197
 
13576
13198
  const toolResultCacheKey = durableCacheKey;
13577
13199
  const cached = !cacheableToolResult
@@ -13597,7 +13219,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13597
13219
  const rowToolRetryPolicy = await this.#options
13598
13220
  .getToolRetryPolicy?.(toolId, input)
13599
13221
  .catch(() => null);
13600
-
13601
13222
  // ADR 0018: record this leg before it runs. `producer` still names the
13602
13223
  // last leg (the usual cascade winner); `producers` keeps every leg, which
13603
13224
  // is the only durable record a losing leg ever gets — a failed tool call
@@ -13641,7 +13262,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13641
13262
  // The tail of that wall. The dispatcher stamps the request when this
13642
13263
  // call's provider body finished parsing; the difference here is what
13643
13264
  // receipt completion and hand-back cost after the provider was done. A
13644
- // cache hit and a follower never read a body, so they leave no stamp and
13265
+ // cache hit never reads a body, so it leaves no stamp and
13645
13266
  // record nothing rather than a delivery of zero.
13646
13267
  const observeResultDelivery = () => {
13647
13268
  const bodyCompletedAtMs = dispatchRequest.providerBodyCompletedAtMs;
@@ -15389,7 +15010,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
15389
15010
  *
15390
15011
  * Gated on the same stamp `resultDelivery` is gated on — a body this request
15391
15012
  * really read — so every delivery sub-phase counts the same calls and their
15392
- * sum can be checked against the whole. A cache hit, a follower, and a
15013
+ * sum can be checked against the whole. A cache hit and a
15393
15014
  * member that never reached the provider leave the stamp unset and record
15394
15015
  * nothing rather than a wrap of zero.
15395
15016
  */
@@ -15521,179 +15142,38 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
15521
15142
 
15522
15143
  // ——— Batched tool call execution ———
15523
15144
 
15524
- private startPendingToolReceiptHeartbeat(
15525
- requests: ToolCallRequest[],
15526
- claimsEstablishedExecutionFence: boolean,
15527
- ): {
15528
- stop: () => void;
15529
- leaseLost: () => unknown | null;
15530
- confirmOwned: () => Promise<void>;
15531
- assertOwned: () => void;
15532
- } | null {
15533
- const heartbeatReceipts = this.#options.heartbeatRuntimeStepReceipts;
15534
- const ownedRequests = requests.filter(
15535
- (request) => request.receiptKey && request.receiptLeaseId,
15536
- );
15537
- if (!heartbeatReceipts || ownedRequests.length === 0) return null;
15538
-
15539
- const earliestLeaseExpiry = ownedRequests.reduce<string | null>(
15540
- (earliest, request) => {
15541
- const expiresAt = request.receiptLeaseExpiresAt ?? null;
15542
- if (!expiresAt) return earliest;
15543
- if (!earliest) return expiresAt;
15544
- return Date.parse(expiresAt) < Date.parse(earliest)
15545
- ? expiresAt
15546
- : earliest;
15547
- },
15548
- null,
15549
- );
15550
- let leaseLost: unknown | null = null;
15551
- let ownershipConfirmedUntilMs = Number.isFinite(
15552
- Date.parse(earliestLeaseExpiry ?? ''),
15553
- )
15554
- ? Date.parse(earliestLeaseExpiry!)
15555
- : Date.now() + PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS;
15556
- const firstOwnedRequest = ownedRequests[0]!;
15557
- const heartbeatOwnedRequests = async () => {
15558
- // Receipt claims are acquired in bulk before the provider-shaped
15559
- // dispatcher admits every call. Renew the undispatched tail as well as
15560
- // in-flight calls; otherwise a slow provider can let later claims
15561
- // expire while they are still waiting behind the concurrency gate.
15562
- const unsettled = ownedRequests.filter((request) =>
15563
- this.toolCallResolvers.has(request.callId),
15564
- );
15565
- if (unsettled.length === 0) return 'terminal' as const;
15566
- const byReceiptKey = new Map<string, ToolCallRequest[]>();
15567
- for (const request of unsettled) {
15568
- const receiptKey = request.receiptKey!;
15569
- const group = byReceiptKey.get(receiptKey) ?? [];
15570
- group.push(request);
15571
- byReceiptKey.set(receiptKey, group);
15145
+ /**
15146
+ * Concrete RuntimeStepReceipt adapter for the map receipt-cohort Module.
15147
+ *
15148
+ * Scalar and provider-native map calls share completed-result cache reads
15149
+ * and atomic successful inserts. A miss has no durable execution owner.
15150
+ */
15151
+ private createRuntimeStepToolCallReceiptCohortStore(input: {
15152
+ toolId: string;
15153
+ requestForReceipt: ReadonlyMap<string, ToolCallRequest>;
15154
+ /** Each invocation sharing a completed-result key, for manifests and timing. */
15155
+ requestsForReceipt: ReadonlyMap<string, readonly ToolCallRequest[]>;
15156
+ }): ToolCallReceiptCohortStore {
15157
+ const requestsForPublication = (receipt: {
15158
+ resultReceiptKey: string;
15159
+ memberIds?: readonly string[];
15160
+ }): readonly ToolCallRequest[] => {
15161
+ if (!receipt.memberIds) {
15162
+ const request = input.requestForReceipt.get(receipt.resultReceiptKey);
15163
+ return request ? [request] : [];
15572
15164
  }
15573
- // One content-addressed receipt can have many same-run followers. The
15574
- // store renews unique receipt rows, not request positions, so dedupe by
15575
- // key while preserving the matching per-row lease id.
15576
- const entries = [...byReceiptKey];
15577
- const keys = entries.map(([receiptKey]) => receiptKey);
15578
- const leaseIds = entries.map(
15579
- ([, requests]) => requests[0]!.receiptLeaseId!,
15580
- );
15581
- const receipts = await heartbeatReceipts({
15582
- runId: this.currentReceiptOwnerRunId,
15583
- runAttempt: this.currentRunAttempt,
15584
- keys,
15585
- leaseIds,
15586
- });
15587
- const renewedExpiries: number[] = [];
15588
- for (let index = 0; index < entries.length; index += 1) {
15589
- const [receiptKey, requests] = entries[index]!;
15590
- const leaseId = leaseIds[index]!;
15591
- // Completion can race this bulk heartbeat response. A settled
15592
- // receipt group no longer needs ownership and must not turn that
15593
- // race into a false lease-loss failure or hold the confirmation
15594
- // horizon at its old expiry.
15595
- if (
15596
- this.runtimeToolReceiptCompletionsInFlight.has(receiptKey) ||
15597
- !requests.some((request) =>
15598
- this.toolCallResolvers.has(request.callId),
15599
- )
15600
- ) {
15601
- continue;
15165
+ const requests =
15166
+ input.requestsForReceipt.get(receipt.resultReceiptKey) ?? [];
15167
+ return receipt.memberIds.map((memberId) => {
15168
+ const request = requests.find(
15169
+ (candidate) => candidate.callId === memberId,
15170
+ );
15171
+ if (!request) {
15172
+ throw new Error(`Unknown Tool Call publication member ${memberId}.`);
15602
15173
  }
15603
- const receipt = receipts[index] ?? null;
15604
- const renewedExpiryMs = Date.parse(receipt?.leaseExpiresAt ?? '');
15605
- if (
15606
- !this.runtimeToolReceiptStillOwned(receipt, leaseId) ||
15607
- !Number.isFinite(renewedExpiryMs)
15608
- ) {
15609
- throw new RuntimeReceiptLeaseLostError({
15610
- receiptKey,
15611
- runId: this.currentReceiptOwnerRunId,
15612
- leaseId,
15613
- });
15614
- }
15615
- renewedExpiries.push(renewedExpiryMs);
15616
- }
15617
- if (renewedExpiries.length === 0) return 'terminal' as const;
15618
- ownershipConfirmedUntilMs = Math.min(...renewedExpiries);
15619
- return 'active' as const;
15620
- };
15621
- const perCallHeartbeatIntervalMs = runtimeLeaseHeartbeatIntervalFromExpiry({
15622
- leaseExpiresAt: earliestLeaseExpiry,
15623
- fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
15624
- });
15625
- const supervisor = createRuntimeReceiptHeartbeatSupervisor({
15626
- // The cohort owns claims that have not reached provider admission yet.
15627
- // Renew it before the one-call supervisors can win the same short-lease
15628
- // timer race, so the queued tail remains fenced while the active call
15629
- // consumes the provider concurrency slot.
15630
- intervalMs: Math.max(1, Math.floor(perCallHeartbeatIntervalMs / 2)),
15631
- heartbeat: heartbeatOwnedRequests,
15632
- isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
15633
- onLeaseLost: (error) => {
15634
- leaseLost ??= error;
15635
- },
15636
- onTransientFailure: (error) => {
15637
- this.runtimeLog(
15638
- `Pending tool receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`,
15639
- );
15640
- },
15641
- });
15642
- return {
15643
- stop: () => supervisor.stop(),
15644
- leaseLost: () => leaseLost,
15645
- confirmOwned: async () => {
15646
- // The scheduler receipt gateway returns `running` from an atomic
15647
- // claim. That claim is already the provider-execution fence, so short
15648
- // calls only need scheduled renewal if they approach lease expiry.
15649
- // Other callers may still return queued/pending claims or omit the
15650
- // explicit backend capability; retain the immediate ownership check
15651
- // for those receipts.
15652
- if (claimsEstablishedExecutionFence) {
15653
- if (
15654
- ownershipConfirmedUntilMs - Date.now() <=
15655
- IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS
15656
- ) {
15657
- const outcome = await heartbeatOwnedRequests();
15658
- if (outcome === 'active') supervisor.start();
15659
- return;
15660
- }
15661
- supervisor.start();
15662
- return;
15663
- }
15664
- const outcome = await heartbeatOwnedRequests();
15665
- if (outcome === 'active') supervisor.start();
15666
- },
15667
- assertOwned: () => {
15668
- if (leaseLost) throw leaseLost;
15669
- if (Date.now() < ownershipConfirmedUntilMs) return;
15670
- throw new RuntimeReceiptLeaseLostError({
15671
- receiptKey: firstOwnedRequest.receiptKey!,
15672
- runId: this.currentReceiptOwnerRunId,
15673
- leaseId: firstOwnedRequest.receiptLeaseId!,
15674
- });
15675
- },
15174
+ return request;
15175
+ });
15676
15176
  };
15677
- }
15678
-
15679
- /**
15680
- * Concrete RuntimeStepReceipt adapter for the map receipt-cohort Module.
15681
- *
15682
- * Scalar and provider-native map calls share completed-result cache reads
15683
- * and atomic successful inserts. A miss has no durable execution owner.
15684
- */
15685
- private createRuntimeStepToolCallReceiptCohortStore(input: {
15686
- toolId: string;
15687
- requestForReceipt: ReadonlyMap<string, ToolCallRequest>;
15688
- /**
15689
- * Every request sharing a receipt key, for delivery timing only. Two rows
15690
- * asking the same question share one key, and only one of them — the
15691
- * cohort's owner — executed and carries the wrap stamp `deliveryPublishWait`
15692
- * is measured from, so the timing lookup cannot use the single-request map
15693
- * above, which keeps whichever request was seen last.
15694
- */
15695
- requestsForReceipt: ReadonlyMap<string, readonly ToolCallRequest[]>;
15696
- }): ToolCallReceiptCohortStore {
15697
15177
  const asRecord = async (
15698
15178
  resultReceiptKey: string,
15699
15179
  storedReceiptKey: string,
@@ -15879,23 +15359,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
15879
15359
  );
15880
15360
  const manifestCompletions =
15881
15361
  this.sourceRunToolReceiptManifestCompletionInputs(
15882
- receipts.flatMap((receipt) => {
15883
- const request = input.requestForReceipt.get(
15884
- receipt.resultReceiptKey,
15885
- );
15886
- return request
15887
- ? [
15888
- {
15889
- toolId: input.toolId,
15890
- logicalCallId: request.logicalCallId,
15891
- executionAuthScopeDigest:
15892
- request.executionAuthScopeDigest,
15893
- resultReceiptKey: receipt.resultReceiptKey,
15894
- providerActionVersion: request.providerActionVersion,
15895
- },
15896
- ]
15897
- : [];
15898
- }),
15362
+ receipts.flatMap((receipt) =>
15363
+ requestsForPublication(receipt).map((request) => ({
15364
+ toolId: input.toolId,
15365
+ logicalCallId: request.logicalCallId,
15366
+ executionAuthScopeDigest: request.executionAuthScopeDigest,
15367
+ resultReceiptKey: receipt.resultReceiptKey,
15368
+ providerActionVersion: request.providerActionVersion,
15369
+ })),
15370
+ ),
15899
15371
  );
15900
15372
  await this.completeRuntimeStepReceipts(
15901
15373
  [...canonicalCompletions, ...manifestCompletions],
@@ -15905,18 +15377,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
15905
15377
  // charged to one of the two rather than to neither.
15906
15378
  (enqueuedAtMs) => {
15907
15379
  for (const receipt of receipts) {
15908
- const wrappedAtMs = (
15909
- input.requestsForReceipt.get(receipt.resultReceiptKey) ?? []
15910
- ).find(
15911
- (request) => request.deliveryWrappedAtMs != null,
15912
- )?.deliveryWrappedAtMs;
15913
- if (wrappedAtMs == null) continue;
15914
- this.resourceGovernor.observe({
15915
- deliveryPublishWaitMs: Math.max(
15916
- 0,
15917
- enqueuedAtMs - wrappedAtMs,
15918
- ),
15919
- });
15380
+ for (const request of requestsForPublication(receipt)) {
15381
+ const wrappedAtMs = request.deliveryWrappedAtMs;
15382
+ if (wrappedAtMs == null) continue;
15383
+ this.resourceGovernor.observe({
15384
+ deliveryPublishWaitMs: Math.max(
15385
+ 0,
15386
+ enqueuedAtMs - wrappedAtMs,
15387
+ ),
15388
+ });
15389
+ }
15920
15390
  }
15921
15391
  },
15922
15392
  // The completion writer batches by arrival, so a publication wave
@@ -16011,57 +15481,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16011
15481
  : request.input,
16012
15482
  ),
16013
15483
  })),
16014
- // Claim immediately so unrelated owned rows can cross provider work
16015
- // while foreign members hydrate in the bounded lane.
16016
- eager: true,
16017
- });
16018
-
16019
- // Claims carry their actual expiry. Seed the cohort supervisor from that
16020
- // lease rather than the generic fallback, otherwise a long provider-shaped
16021
- // queue can let undispatched rows expire before its first renewal.
16022
- for (const request of requests) {
16023
- const member = cohort.member(request.callId);
16024
- if (member.kind !== 'owned') continue;
16025
- request.receiptLeaseId = member.lease.leaseId;
16026
- request.receiptLeaseExpiresAt = member.leaseExpiresAt ?? null;
16027
- }
16028
-
16029
- const earliestLeaseExpiry = requests.reduce<string | null>(
16030
- (earliest, request) => {
16031
- const expiresAt = request.receiptLeaseExpiresAt ?? null;
16032
- if (!expiresAt) return earliest;
16033
- if (!earliest) return expiresAt;
16034
- return Date.parse(expiresAt) < Date.parse(earliest)
16035
- ? expiresAt
16036
- : earliest;
16037
- },
16038
- null,
16039
- );
16040
- // One cohort-wide liveness check closes the claim-to-provider window; the
16041
- // per-call views additionally preserve the historical exact admission
16042
- // fence immediately before an individual provider request.
16043
- await cohort.heartbeat();
16044
- let cohortLeaseLost: unknown | null = null;
16045
- const heartbeat = cohort.startHeartbeat({
16046
- intervalMs: Math.max(
16047
- 1,
16048
- Math.floor(
16049
- runtimeLeaseHeartbeatIntervalFromExpiry({
16050
- leaseExpiresAt: earliestLeaseExpiry,
16051
- fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
16052
- }) / 2,
16053
- ),
16054
- ),
16055
- onLeaseLost: (error) => {
16056
- cohortLeaseLost ??= error;
16057
- },
16058
- onTransientFailure: (error) => {
16059
- this.runtimeLog(
16060
- `Tool Call Receipt Cohort heartbeat transport failed; retrying: ${
16061
- error instanceof Error ? error.message : String(error)
16062
- }`,
16063
- );
16064
- },
16065
15484
  });
16066
15485
 
16067
15486
  const successful: ToolCallRequest[] = [];
@@ -16108,35 +15527,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16108
15527
  });
16109
15528
  };
16110
15529
 
16111
- const handled = new Set<string>();
16112
- const followerResults: Promise<void>[] = [];
16113
- const collectReadyMembers = (): Array<{
15530
+ const cachedResults: Promise<void>[] = [];
15531
+ const owned: Array<{
16114
15532
  request: ToolCallRequest;
16115
15533
  receipts: ToolResultReceipts;
16116
- }> => {
16117
- const owned: Array<{
16118
- request: ToolCallRequest;
16119
- receipts: ToolResultReceipts;
16120
- }> = [];
16121
- for (const request of requests) {
16122
- if (handled.has(request.callId)) continue;
16123
- const member = cohort.member(request.callId);
16124
- handled.add(request.callId);
16125
- if (member.kind === 'completed') {
16126
- followerResults.push(resolve(request, member.result, member.source));
16127
- } else if (member.kind === 'following') {
16128
- followerResults.push(
16129
- member.result.then(
16130
- async (result) => await resolve(request, result, 'owner'),
16131
- async (error: unknown) => await reject(request, error),
16132
- ),
16133
- );
16134
- } else {
16135
- owned.push({ request, receipts: member.receipts });
16136
- }
15534
+ }> = [];
15535
+ for (const request of requests) {
15536
+ const member = cohort.member(request.callId);
15537
+ if (member.kind === 'completed') {
15538
+ cachedResults.push(resolve(request, member.result, member.source));
15539
+ } else if (member.kind === 'owned') {
15540
+ owned.push({ request, receipts: member.receipts });
15541
+ } else {
15542
+ throw new Error(
15543
+ 'A completed-result cache cannot own a pending invocation.',
15544
+ );
16137
15545
  }
16138
- return owned;
16139
- };
15546
+ }
16140
15547
 
16141
15548
  const width = Math.min(
16142
15549
  this.governor.policy.concurrency.toolCalls,
@@ -16148,85 +15555,91 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16148
15555
  ),
16149
15556
  ),
16150
15557
  );
16151
- const dispatchOwned = async (
16152
- owned: Array<{
16153
- request: ToolCallRequest;
16154
- receipts: ToolResultReceipts;
16155
- }>,
16156
- ) =>
16157
- await dispatchBoundedSettled(
16158
- owned,
16159
- width,
16160
- async ({ request, receipts }) => {
16161
- // Circuit breaker: do not dispatch this provider call once a
16162
- // persistence failure has occurred in this run. Rejected through the
16163
- // same path as a provider failure — `dispatchBoundedSettled` only
16164
- // collects a thrown error, so throwing past `reject` here would
16165
- // leave this row's tool-call resolver pending for the life of the
16166
- // map.
16167
- if (this.persistenceLatch.tripped) {
16168
- this.persistenceLatch.preventedCallCount += 1;
16169
- await reject(
16170
- request,
16171
- new RuntimePersistenceCircuitOpenError(this.persistenceLatch),
16172
- );
16173
- return;
16174
- }
16175
- let releaseToolSlot: () => void = () => undefined;
16176
- try {
16177
- const providerOperationKey = this.providerIdempotencyKeyForToolCall(
16178
- {
15558
+ await dispatchBoundedSettled(
15559
+ owned,
15560
+ width,
15561
+ async ({ request, receipts }) => {
15562
+ // Circuit breaker: do not dispatch this provider call once a
15563
+ // persistence failure has occurred in this run. Rejected through the
15564
+ // same path as a provider failure — `dispatchBoundedSettled` only
15565
+ // collects a thrown error, so throwing past `reject` here would
15566
+ // leave this row's tool-call resolver pending for the life of the
15567
+ // map.
15568
+ if (this.persistenceLatch.tripped) {
15569
+ this.persistenceLatch.preventedCallCount += 1;
15570
+ await reject(
15571
+ request,
15572
+ new RuntimePersistenceCircuitOpenError(this.persistenceLatch),
15573
+ );
15574
+ return;
15575
+ }
15576
+ let releaseToolSlot: () => void = () => undefined;
15577
+ try {
15578
+ const providerOperationKey = this.providerInvocationKeyForToolCall({
15579
+ operationKey:
15580
+ request.providerIdempotencyKeyBase ?? request.receiptKey!,
15581
+ logicalCallId: request.logicalCallId,
15582
+ force: request.force,
15583
+ });
15584
+ const result = await this.executeMappedToolCallThroughModule({
15585
+ request,
15586
+ dispatchToolId: toolId,
15587
+ dispatchPayload: request.input,
15588
+ resultReceiptKey:
15589
+ this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
15590
+ providerOperationKey,
15591
+ resultReceipts: receipts,
15592
+ apiOptions: {
15593
+ playNodeScope: playNodeScopeForToolCallRequest(request),
15594
+ durableCallReceiptKey: this.toolTransportReceiptKey({
15595
+ toolId,
15596
+ providerOperationKey,
15597
+ }),
15598
+ legacyProviderRetryIdentities: [
15599
+ {
15600
+ transportReceiptKey: request.receiptKey!,
15601
+ providerOperationKey: request.force
15602
+ ? providerOperationKey
15603
+ : this.providerOperationKeyForQueuedToolCall(request),
15604
+ ...(request.force && request.requiresExecutionFence !== true
15605
+ ? { forceRefreshIdentities: [providerOperationKey] }
15606
+ : {}),
15607
+ },
15608
+ ],
15609
+ completedResultCache: true,
15610
+ executionAuthScopeDigest: request.executionAuthScopeDigest,
15611
+ ...(request.requiresExecutionFence === true
15612
+ ? {
15613
+ providerIdempotencyKey: providerOperationKey,
15614
+ providerIdempotencyReceiptKey: request.receiptKey,
15615
+ }
15616
+ : {}),
15617
+ ...this.providerInvocationOptionsForToolCall({
16179
15618
  operationKey:
16180
15619
  request.providerIdempotencyKeyBase ?? request.receiptKey!,
16181
- force: request.force === true,
15620
+ requiresExecutionFence: request.requiresExecutionFence,
16182
15621
  logicalCallId: request.logicalCallId,
15622
+ force: request.force,
15623
+ }),
15624
+ timeoutMs:
15625
+ typeof request.timeoutMs === 'number' && request.timeoutMs > 0
15626
+ ? request.timeoutMs
15627
+ : undefined,
15628
+ retainToolSlot: (release) => {
15629
+ releaseToolSlot = release;
16183
15630
  },
16184
- );
16185
- const result = await this.executeMappedToolCallThroughModule({
16186
- request,
16187
- dispatchToolId: toolId,
16188
- dispatchPayload: request.input,
16189
- resultReceiptKey:
16190
- this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
16191
- providerOperationKey,
16192
- resultReceipts: receipts,
16193
- apiOptions: {
16194
- playNodeScope: playNodeScopeForToolCallRequest(request),
16195
- durableCallReceiptKey: this.toolTransportReceiptKey({
16196
- toolId,
16197
- resultReceiptKey: request.receiptKey!,
16198
- providerOperationKey,
16199
- force: request.force,
16200
- }),
16201
- completedResultCache: true,
16202
- executionAuthScopeDigest: request.executionAuthScopeDigest,
16203
- ...this.providerForceRefreshOptionsForToolCall({
16204
- operationKey:
16205
- request.providerIdempotencyKeyBase ?? request.receiptKey!,
16206
- force: request.force,
16207
- requiresExecutionFence: request.requiresExecutionFence,
16208
- logicalCallId: request.logicalCallId,
16209
- }),
16210
- timeoutMs:
16211
- typeof request.timeoutMs === 'number' && request.timeoutMs > 0
16212
- ? request.timeoutMs
16213
- : undefined,
16214
- retainToolSlot: (release) => {
16215
- releaseToolSlot = release;
16216
- },
16217
- },
16218
- });
16219
- await resolve(request, result, 'owner');
16220
- } catch (error) {
16221
- await reject(request, error);
16222
- } finally {
16223
- releaseToolSlot();
16224
- }
16225
- },
16226
- );
15631
+ },
15632
+ });
15633
+ await resolve(request, result, 'owner');
15634
+ } catch (error) {
15635
+ await reject(request, error);
15636
+ } finally {
15637
+ releaseToolSlot();
15638
+ }
15639
+ },
15640
+ );
16227
15641
 
16228
- await dispatchOwned(collectReadyMembers());
16229
- await Promise.allSettled(followerResults);
15642
+ await Promise.all(cachedResults);
16230
15643
  return successful;
16231
15644
  }
16232
15645
 
@@ -16398,18 +15811,109 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16398
15811
  'Native batch Tool Call operation identity changed in transit.',
16399
15812
  );
16400
15813
  }
15814
+ const legacyForceRefreshIdentities = detail.members
15815
+ .filter(
15816
+ (member) =>
15817
+ member.force === true &&
15818
+ member.requiresExecutionFence !== true,
15819
+ )
15820
+ .map((member) =>
15821
+ this.providerInvocationKeyForToolCall({
15822
+ operationKey:
15823
+ this.providerOperationKeyForQueuedToolCall(member),
15824
+ logicalCallId: member.logicalCallId,
15825
+ force: true,
15826
+ }),
15827
+ );
15828
+ const legacyProviderOperationKey =
15829
+ buildDurableToolPayloadProviderIdempotencyKey({
15830
+ orgId: this.#options.orgId,
15831
+ toolId: call.tool.id,
15832
+ executionAuthScopeDigest:
15833
+ detail.members[0]?.executionAuthScopeDigest ?? null,
15834
+ providerActionVersion: call.tool.providerActionVersion,
15835
+ payload: call.payload,
15836
+ forceRefreshIdentities: legacyForceRefreshIdentities,
15837
+ });
15838
+ const legacyFallbackForceRefreshIdentities = detail.members
15839
+ .filter((member) => member.force === true)
15840
+ .map((member) =>
15841
+ this.providerInvocationKeyForToolCall({
15842
+ operationKey:
15843
+ this.providerOperationKeyForQueuedToolCall(member),
15844
+ logicalCallId: member.logicalCallId,
15845
+ force: true,
15846
+ }),
15847
+ );
15848
+ const legacyFallbackProviderOperationKey =
15849
+ buildDurableToolPayloadProviderIdempotencyKey({
15850
+ orgId: this.#options.orgId,
15851
+ toolId: call.tool.id,
15852
+ executionAuthScopeDigest:
15853
+ detail.members[0]?.executionAuthScopeDigest ?? null,
15854
+ providerActionVersion:
15855
+ (
15856
+ await this.#options.getToolActionCacheVersion?.(
15857
+ call.tool.id,
15858
+ )
15859
+ )?.trim() ?? 'unversioned',
15860
+ payload: call.payload,
15861
+ forceRefreshIdentities:
15862
+ legacyFallbackForceRefreshIdentities,
15863
+ });
16401
15864
  const execution = await this.callToolExecutionAPI(
16402
15865
  call.tool.id,
16403
15866
  call.payload,
16404
15867
  {
16405
15868
  durableCallReceiptKey: this.toolTransportReceiptKey({
16406
15869
  toolId: call.tool.id,
16407
- resultReceiptKey: compatibility.resultReceiptKey,
16408
15870
  providerOperationKey: detail.providerOperationKey,
16409
- force: detail.members.some(
16410
- (request) => request.force === true,
16411
- ),
16412
15871
  }),
15872
+ legacyProviderRetryIdentities: [
15873
+ {
15874
+ transportReceiptKey: detail.members.some(
15875
+ (member) => member.force,
15876
+ )
15877
+ ? this.toolTransportReceiptKey({
15878
+ toolId: call.tool.id,
15879
+ providerOperationKey:
15880
+ legacyProviderOperationKey,
15881
+ })
15882
+ : buildDurableToolAggregateReceiptKey({
15883
+ receiptKeys: detail.members.map(
15884
+ (member) =>
15885
+ this.mappedToolResultReceiptPolicy(member)
15886
+ .resultReceiptKey,
15887
+ ),
15888
+ prefix: 'batch',
15889
+ aggregateReceiptPrefix:
15890
+ buildDurableToolReceiptPrefix({
15891
+ orgId: this.#options.orgId,
15892
+ toolId: call.tool.id,
15893
+ }),
15894
+ }),
15895
+ providerOperationKey: legacyProviderOperationKey,
15896
+ forceRefreshIdentities: legacyForceRefreshIdentities,
15897
+ },
15898
+ {
15899
+ transportReceiptKey:
15900
+ buildDurableToolAggregateReceiptKey({
15901
+ receiptKeys: detail.members.map(
15902
+ (member) => member.cacheKey,
15903
+ ),
15904
+ prefix: 'batch',
15905
+ aggregateReceiptPrefix:
15906
+ buildDurableToolReceiptPrefix({
15907
+ orgId: this.#options.orgId,
15908
+ toolId: call.tool.id,
15909
+ }),
15910
+ }),
15911
+ providerOperationKey:
15912
+ legacyFallbackProviderOperationKey,
15913
+ forceRefreshIdentities:
15914
+ legacyFallbackForceRefreshIdentities,
15915
+ },
15916
+ ],
16413
15917
  completedResultCache: true,
16414
15918
  memoryLedgerRowScopes: detail.members.map((member) => ({
16415
15919
  rowKey: member.rowKey,
@@ -16426,6 +15930,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16426
15930
  ),
16427
15931
  executionAuthScopeDigest:
16428
15932
  detail.members[0]?.executionAuthScopeDigest ?? null,
15933
+ providerIdempotencyKey: detail.providerOperationKey,
15934
+ providerIdempotencyReceiptKey:
15935
+ compatibility.resultReceiptKey,
16429
15936
  ...(detail.forceRefreshIdentities.length > 0
16430
15937
  ? {
16431
15938
  providerOperationForceRefreshIdentities:
@@ -16535,25 +16042,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16535
16042
  compiled.batchOperation,
16536
16043
  )
16537
16044
  )?.trim() ?? '';
16538
- const receiptKeys = owned.map(
16539
- (request) =>
16540
- this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
16541
- );
16542
- const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({
16543
- receiptKeys,
16544
- prefix: 'batch',
16545
- aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
16546
- orgId: this.#options.orgId,
16547
- toolId: compiled.batchOperation,
16548
- }),
16549
- });
16550
- const forceRefreshIdentities = owned.flatMap((request) =>
16551
- this.providerForceRefreshIdentitiesForToolCall({
16045
+ const forceRefreshIdentities = owned.map((request) =>
16046
+ this.providerInvocationKeyForToolCall({
16552
16047
  operationKey:
16553
16048
  this.providerOperationKeyForQueuedToolCall(request),
16554
- force: request.force,
16555
- requiresExecutionFence: request.requiresExecutionFence,
16556
16049
  logicalCallId: request.logicalCallId,
16050
+ force: request.force,
16557
16051
  }),
16558
16052
  );
16559
16053
  const providerOperationKey =
@@ -16566,6 +16060,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16566
16060
  payload: compiled.batchPayload,
16567
16061
  forceRefreshIdentities,
16568
16062
  });
16063
+ const aggregateReceiptKey = this.toolTransportReceiptKey({
16064
+ toolId: compiled.batchOperation,
16065
+ providerOperationKey,
16066
+ });
16569
16067
  dispatchDetails.set(aggregateReceiptKey, {
16570
16068
  members: owned,
16571
16069
  compiled,
@@ -16825,8 +16323,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16825
16323
  };
16826
16324
 
16827
16325
  const pendingRequests: ToolCallRequest[] = [];
16828
- let pendingReceiptClaimsEstablishExecutionFence =
16829
- this.#options.runtimeReceiptClaimsEstablishExecutionFence === true;
16830
16326
  const recoveredRequests: ToolCallRequest[] = [];
16831
16327
  for (const req of requests) {
16832
16328
  const cached =
@@ -16851,114 +16347,27 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16851
16347
  }
16852
16348
  recordToolStep(recoveredRequests);
16853
16349
 
16854
- const liveFollowersByOwnerCallId = new Map<string, ToolCallRequest[]>();
16855
- const resolveLiveFollowers = (
16856
- owner: ToolCallRequest,
16857
- result: unknown,
16858
- ): void => {
16859
- const followers = liveFollowersByOwnerCallId.get(owner.callId);
16860
- if (!followers || followers.length === 0) return;
16861
- liveFollowersByOwnerCallId.delete(owner.callId);
16862
- for (const follower of followers) {
16863
- const followerReceiptKey = follower.receiptKey?.trim() || null;
16864
- const followerResult = followerReceiptKey
16865
- ? markToolExecuteResultExecutionOutcome(result, {
16866
- kind: 'in_flight',
16867
- receiptKey: followerReceiptKey,
16868
- attachedToReceiptKey: followerReceiptKey,
16869
- })
16870
- : result;
16871
- if (follower.cacheable !== false) {
16872
- this.cacheToolResult(toolId, follower.cacheKey, followerResult);
16873
- }
16874
- const resolver = this.toolCallResolvers.get(follower.callId);
16875
- if (resolver) {
16876
- resolver.resolve(followerResult);
16877
- this.toolCallResolvers.delete(follower.callId);
16878
- }
16879
- this.emitScopedFieldMetaUpdate({
16880
- rowId: follower.rowId,
16881
- key: follower.rowKey ?? null,
16882
- tableNamespace: follower.tableNamespace ?? null,
16883
- fieldName: follower.fieldName,
16884
- status: 'cached',
16885
- rowStatus: 'running',
16886
- stage: toolId,
16887
- provider: null,
16888
- error: null,
16889
- reused: true,
16890
- dataPatch: {},
16891
- });
16892
- }
16893
- };
16894
- const rejectWithLiveFollowers = async (
16895
- owner: ToolCallRequest,
16350
+ const rejectCall = async (
16351
+ request: ToolCallRequest,
16896
16352
  error: unknown,
16897
16353
  ): Promise<void> => {
16898
- const followers = liveFollowersByOwnerCallId.get(owner.callId) ?? [];
16899
- liveFollowersByOwnerCallId.delete(owner.callId);
16900
- if (error instanceof PlayRowProviderBackpressureError) {
16901
- let releaseFailure: unknown = null;
16902
- const receiptKey = owner.receiptKey?.trim() || null;
16903
- try {
16904
- if (receiptKey && this.durableMappedToolResultsBackedByReceipts) {
16905
- const leaseId = owner.receiptLeaseId?.trim();
16906
- if (!leaseId) {
16907
- throw new Error(
16908
- `Provider wait cannot release receipt ${receiptKey} without its owned lease.`,
16909
- );
16910
- }
16911
- const released = await this.releaseRuntimeStepReceipt(
16912
- receiptKey,
16913
- this.currentReceiptOwnerRunId,
16914
- leaseId,
16915
- );
16916
- if (
16917
- !released ||
16918
- (released.status !== 'queued' &&
16919
- released.status !== 'pending')
16920
- ) {
16921
- throw new Error(
16922
- `Provider wait could not release owned receipt ${receiptKey}.`,
16923
- );
16924
- }
16925
- }
16926
- } catch (failure) {
16927
- releaseFailure = failure;
16928
- }
16929
- // A provider deadline is a nonterminal control signal. The row
16930
- // catch records waiting metadata after the resolver unwinds; no
16931
- // follower may see or persist a failed receipt in between. If the
16932
- // lease could not be released, wake every follower with that loud
16933
- // infrastructure failure rather than stranding their promises.
16934
- for (const request of [owner, ...followers]) {
16935
- const resolver = this.toolCallResolvers.get(request.callId);
16936
- if (resolver) {
16937
- resolver.reject(releaseFailure ?? error);
16938
- this.toolCallResolvers.delete(request.callId);
16939
- }
16940
- }
16941
- if (releaseFailure) throw releaseFailure;
16942
- return;
16943
- }
16944
- const shouldPersistReceiptFailure =
16945
- !(error instanceof ToolExecuteAuthScopeChangedError) &&
16946
- !this.persistenceLatch.tripped;
16947
- for (const request of [owner, ...followers]) {
16948
- await this.rejectToolCall(toolId, request, error, {
16949
- persistReceiptFailure: shouldPersistReceiptFailure,
16950
- });
16951
- }
16354
+ await this.rejectToolCall(toolId, request, error, {
16355
+ persistReceiptFailure: false,
16356
+ });
16952
16357
  };
16953
- const requestsWithLiveFollowers = (
16954
- owner: ToolCallRequest,
16955
- ): ToolCallRequest[] => [
16956
- owner,
16957
- ...(liveFollowersByOwnerCallId.get(owner.callId) ?? []),
16958
- ];
16959
16358
  const resolveRuntimeTimeoutMsForRequests = (
16960
16359
  requests: ToolCallRequest[],
16961
16360
  ): number | undefined => {
16361
+ if (
16362
+ requests.some(
16363
+ (request) =>
16364
+ typeof request.timeoutMs !== 'number' ||
16365
+ !Number.isFinite(request.timeoutMs) ||
16366
+ request.timeoutMs <= 0,
16367
+ )
16368
+ ) {
16369
+ return undefined;
16370
+ }
16962
16371
  const timeoutMs = Math.max(
16963
16372
  ...requests
16964
16373
  .map((request) => request.timeoutMs)
@@ -16972,76 +16381,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16972
16381
  );
16973
16382
  return timeoutMs > 0 ? timeoutMs : undefined;
16974
16383
  };
16975
- const resolveRuntimeTimeoutMsForClaimedOwners = (
16976
- owners: ToolCallRequest[],
16977
- ): number | undefined => {
16978
- let timeoutMs = 0;
16979
- for (const owner of owners) {
16980
- if (
16981
- typeof owner.timeoutMs !== 'number' ||
16982
- !Number.isFinite(owner.timeoutMs) ||
16983
- owner.timeoutMs <= 0
16984
- ) {
16985
- return undefined;
16986
- }
16987
- timeoutMs = Math.max(
16988
- timeoutMs,
16989
- resolveRuntimeTimeoutMsForRequests(
16990
- requestsWithLiveFollowers(owner),
16991
- ) ?? 0,
16992
- );
16993
- }
16994
- return timeoutMs > 0 ? timeoutMs : undefined;
16995
- };
16996
-
16997
- // Cacheable map members cross the real receipt-cohort boundary before
16998
- // the legacy claim loop. The scalar and native Modules share member
16999
- // receipt ownership; native batches retain their separate aggregate
17000
- // physical-operation receipt behind that seam. Always-fresh calls
17001
- // intentionally have no durable member receipt and stay below.
16384
+ // The cohort reads completed results and publishes successful misses.
16385
+ // Always-fresh calls have no durable member result and stay below.
17002
16386
  const operationStrategy =
17003
16387
  this.#options.getBatchOperationStrategy?.(toolId) ?? null;
17004
- if (
17005
- operationStrategy &&
17006
- this.durableMappedToolResultsBackedByReceipts
17007
- ) {
17008
- const cohortRequests = pendingRequests.filter(
17009
- (request) =>
17010
- request.cacheable !== false &&
17011
- request.requiresExecutionFence !== true &&
17012
- Boolean(request.receiptKey?.trim()),
17013
- );
17014
- if (cohortRequests.length > 0) {
17015
- const cohortCallIds = new Set(
17016
- cohortRequests.map((request) => request.callId),
17017
- );
17018
- pendingRequests.splice(
17019
- 0,
17020
- pendingRequests.length,
17021
- ...pendingRequests.filter(
17022
- (request) => !cohortCallIds.has(request.callId),
17023
- ),
17024
- );
17025
- const successful =
17026
- await this.executeCacheableNativeMapToolCallCohort({
17027
- toolId,
17028
- requests: cohortRequests,
17029
- strategy: operationStrategy,
17030
- });
17031
- for (const request of successful) {
17032
- successfulLiveStepCallIds.add(request.callId);
17033
- }
17034
- recordToolStep(cohortRequests);
17035
- this.#options.onBatchComplete?.(this.checkpoint);
17036
- }
17037
- } else if (
17038
- !operationStrategy &&
17039
- this.durableMappedToolResultsBackedByReceipts
17040
- ) {
16388
+ if (this.durableMappedToolResultsBackedByReceipts) {
17041
16389
  const cohortRequests = pendingRequests.filter(
17042
16390
  (request) =>
17043
16391
  request.cacheable !== false &&
17044
- request.requiresExecutionFence !== true &&
17045
16392
  Boolean(request.receiptKey?.trim()),
17046
16393
  );
17047
16394
  if (cohortRequests.length > 0) {
@@ -17055,1181 +16402,363 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
17055
16402
  (request) => !cohortCallIds.has(request.callId),
17056
16403
  ),
17057
16404
  );
17058
- const successful =
17059
- await this.executeCacheableNonNativeMapToolCallCohort({
17060
- toolId,
17061
- requests: cohortRequests,
17062
- });
17063
- for (const request of successful) {
16405
+ const successful = operationStrategy
16406
+ ? await this.executeCacheableNativeMapToolCallCohort({
16407
+ toolId,
16408
+ requests: cohortRequests,
16409
+ strategy: operationStrategy,
16410
+ })
16411
+ : await this.executeCacheableNonNativeMapToolCallCohort({
16412
+ toolId,
16413
+ requests: cohortRequests,
16414
+ });
16415
+ for (const request of successful)
17064
16416
  successfulLiveStepCallIds.add(request.callId);
17065
- }
17066
16417
  recordToolStep(cohortRequests);
17067
16418
  this.#options.onBatchComplete?.(this.checkpoint);
17068
16419
  }
17069
16420
  }
17070
16421
 
17071
- const durableExistingRunningWaits: Array<{
17072
- receiptKey: string;
17073
- requestsForKey: ToolCallRequest[];
17074
- forceAfterWait: boolean;
17075
- waitMaxAttempts?: number;
17076
- /** Reclaim without polling: the prior owner's lease already expired. */
17077
- skipWait?: boolean;
17078
- }> = [];
17079
- const durableExistingRunningState: {
17080
- settlement: Promise<
17081
- { status: 'fulfilled' } | { status: 'rejected'; reason: unknown }
17082
- > | null;
17083
- } = { settlement: null };
17084
- const durableExistingRunningAbort = new AbortController();
17085
- // The group's size before the turn. The turn body splices and pushes
17086
- // `pendingRequests`, so the count read here is the only one that says
17087
- // how many queued calls this claim turn is blocking.
17088
- const claimTurnLogicalCalls = pendingRequests.length;
17089
- if (
17090
- claimTurnLogicalCalls > 0 &&
17091
- (this.#options.claimRuntimeStepReceipts ||
17092
- this.#options.claimRuntimeStepReceipt)
17093
- ) {
17094
- const requestsByReceiptKey = new Map<
17095
- string,
17096
- {
17097
- requests: ToolCallRequest[];
17098
- forceRefresh: boolean;
17099
- forceFailedRefresh: boolean;
17100
- }
17101
- >();
17102
- const localOnlyRequests: ToolCallRequest[] = [];
17103
- for (const request of pendingRequests) {
17104
- const receiptKey = request.receiptKey?.trim() || null;
17105
- if (!receiptKey) {
17106
- localOnlyRequests.push(request);
17107
- continue;
17108
- }
17109
- const group = requestsByReceiptKey.get(receiptKey) ?? {
17110
- requests: [],
17111
- forceRefresh: false,
17112
- forceFailedRefresh: false,
17113
- };
17114
- group.requests.push(request);
17115
- group.forceRefresh ||= request.force === true;
17116
- group.forceFailedRefresh ||= request.forceFailedRefresh === true;
17117
- requestsByReceiptKey.set(receiptKey, group);
17118
- }
17119
- const normalReceiptKeys = [...requestsByReceiptKey]
17120
- .filter(
17121
- ([, group]) => !group.forceRefresh && !group.forceFailedRefresh,
17122
- )
17123
- .map(([receiptKey]) => receiptKey);
17124
- const forcedReceiptKeys = [...requestsByReceiptKey]
17125
- .filter(([, group]) => group.forceRefresh)
17126
- .map(([receiptKey]) => receiptKey);
17127
- const failedRefreshReceiptKeys = [...requestsByReceiptKey]
17128
- .filter(
17129
- ([, group]) => !group.forceRefresh && group.forceFailedRefresh,
17130
- )
17131
- .map(([receiptKey]) => receiptKey);
17132
- const claimStartedAt = Date.now();
17133
- const forcedExisting = forcedReceiptKeys.length
17134
- ? await this.getRuntimeStepReceipts(forcedReceiptKeys, {
17135
- hydrateOutput: false,
17136
- })
17137
- : new Map<string, RuntimeStepReceipt>();
17138
- // A forced refresh must never duplicate a provider call owned by a
17139
- // live lease. It can, however, immediately claim pending/unleased
17140
- // work and any expired lease. Waiting first for those receipts turns
17141
- // an already-recoverable interruption into a five-minute stall.
17142
- const forcedLiveReceiptKeys = new Set(
17143
- [...forcedExisting]
17144
- .filter(([, receipt]) => {
17145
- if (
17146
- receipt.status !== 'queued' &&
17147
- receipt.status !== 'pending' &&
17148
- receipt.status !== 'running'
17149
- ) {
17150
- return false;
17151
- }
17152
- if (!receipt.leaseId?.trim()) return false;
17153
- const expiresAt = receipt.leaseExpiresAt
17154
- ? Date.parse(receipt.leaseExpiresAt)
17155
- : Number.NaN;
17156
- // Keep an unparseable leased receipt conservative. The
17157
- // fenced claim remains the recovery path after the wait.
17158
- return !Number.isFinite(expiresAt) || expiresAt > Date.now();
17159
- })
17160
- .map(([receiptKey]) => receiptKey),
17161
- );
17162
- const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
17163
- (receiptKey) => !forcedLiveReceiptKeys.has(receiptKey),
17164
- );
17165
- const claims =
17166
- normalReceiptKeys.length > 0
17167
- ? await this.claimRuntimeStepReceipts(
17168
- normalReceiptKeys,
17169
- this.currentReceiptOwnerRunId,
17170
- false,
17171
- false,
17172
- false,
17173
- false,
17174
- )
17175
- : new Map<string, RuntimeStepReceipt>();
17176
- const forcedClaims =
17177
- claimableForcedReceiptKeys.length > 0
17178
- ? await this.claimRuntimeStepReceipts(
17179
- claimableForcedReceiptKeys,
17180
- this.currentReceiptOwnerRunId,
17181
- false,
17182
- true,
17183
- false,
17184
- )
17185
- : new Map<string, RuntimeStepReceipt>();
17186
- const failedRefreshClaims =
17187
- failedRefreshReceiptKeys.length > 0
17188
- ? await this.claimRuntimeStepReceipts(
17189
- failedRefreshReceiptKeys,
17190
- this.currentReceiptOwnerRunId,
17191
- false,
17192
- false,
17193
- true,
17194
- false,
17195
- )
17196
- : new Map<string, RuntimeStepReceipt>();
17197
- for (const [receiptKey, claim] of forcedClaims) {
17198
- claims.set(receiptKey, claim);
17199
- }
17200
- for (const [receiptKey, claim] of failedRefreshClaims) {
17201
- claims.set(receiptKey, claim);
17202
- }
17203
- const safeFailedReceiptKeys = [...claims]
17204
- .filter(([, claim]) => this.canRetryWithoutProviderDispatch(claim))
17205
- .map(([receiptKey]) => receiptKey);
17206
- if (safeFailedReceiptKeys.length > 0) {
17207
- const refreshed = await this.claimRuntimeStepReceipts(
17208
- safeFailedReceiptKeys,
17209
- this.currentReceiptOwnerRunId,
17210
- false,
17211
- false,
17212
- true,
17213
- false,
17214
- );
17215
- for (const [receiptKey, claim] of refreshed) {
17216
- claims.set(receiptKey, claim);
17217
- }
17218
- }
17219
- // Claiming is a state transition, not a reason to expand a
17220
- // body. Fresh receipts contain no result, yet the old path took
17221
- // the hydration lane before it learned that fact and charged a
17222
- // multi-MB row prior to every one. Claims now arrive as bounded
17223
- // manifests; only a terminal winner redeems its known-size body
17224
- // under a second, byte-priced turn.
17225
- const terminalClaimKeys = [...claims]
17226
- .filter(
17227
- ([, claim]) =>
17228
- (claim.status === 'completed' || claim.status === 'skipped') &&
17229
- claim.output === undefined,
17230
- )
17231
- .map(([receiptKey]) => receiptKey);
17232
- if (terminalClaimKeys.length > 0) {
17233
- const hasBoundedClaimManifests = terminalClaimKeys.every(
17234
- (receiptKey) => {
17235
- const claim = claims.get(receiptKey);
17236
- return (
17237
- Number.isSafeInteger(claim?.outputByteLength) &&
17238
- (claim?.outputByteLength ?? -1) >= 0
17239
- );
17240
- },
17241
- );
17242
- const expectedClaimBytes = hasBoundedClaimManifests
17243
- ? terminalClaimKeys.reduce(
17244
- (total, receiptKey) =>
17245
- total + (claims.get(receiptKey)?.outputByteLength ?? 0),
17246
- 0,
17247
- )
17248
- : undefined;
17249
- const hydratedClaims = await this.withRuntimeReceiptHydrationTurn(
17250
- async (hydrationTurn) => {
17251
- const receipts =
17252
- await this.getRuntimeStepReceipts(terminalClaimKeys);
17253
- hydrationTurn.settleBytes(
17254
- runtimeStepReceiptPayloadBytes(receipts.values()),
17255
- );
17256
- return receipts;
17257
- },
17258
- {
17259
- logicalCalls: claimTurnLogicalCalls,
17260
- receiptKeys: terminalClaimKeys.length,
17261
- ...(expectedClaimBytes !== undefined
17262
- ? { expectedBytes: expectedClaimBytes }
17263
- : {}),
17264
- source: 'map_legacy_claim',
17265
- },
17266
- );
17267
- for (const [receiptKey, receipt] of hydratedClaims) {
17268
- claims.set(receiptKey, receipt);
17269
- }
17270
- }
17271
- // Every get/claim round trip this dispatch group paid before any
17272
- // provider work could start.
17273
- this.resourceGovernor.observe({
17274
- receiptClaimMs: Date.now() - claimStartedAt,
17275
- });
17276
- // Claims are compact state transitions. They do not own output
17277
- // bytes and must not wait behind the hydration lane's global
17278
- // memory headroom; a terminal manifest above is the only path
17279
- // that opens a byte-admitted hydration turn.
17280
- const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
17281
- const durableRecoveredRequests: ToolCallRequest[] = [];
17282
- const resolveRequestsFromReceipt = async (
17283
- requestsForKey: ToolCallRequest[],
17284
- receipt: RuntimeStepReceipt,
17285
- source: 'cache' | 'in_flight',
17286
- ): Promise<void> => {
17287
- const request = requestsForKey[0];
17288
- if (!request) return;
17289
- const receiptKey = request.receiptKey?.trim() || null;
17290
- if (!receiptKey) {
17291
- throw new Error(`${source} tool recovery needs receipt key.`);
17292
- }
17293
- const wrapped = await this.wrapToolExecutionResult({
17294
- toolId,
17295
- status:
17296
- receipt.output === null || receipt.output === undefined
17297
- ? 'no_result'
17298
- : 'completed',
17299
- result: this.runtimeReceiptOutput(receipt),
17300
- requestInput: request.input,
17301
- execution: toolExecutionMetadataForOutcome({
17302
- kind: source,
17303
- cacheKey: request.cacheKey,
17304
- receiptKey,
17305
- attachedToReceiptKey: receiptKey,
17306
- }),
17307
- });
17308
- this.cacheToolResult(toolId, request.cacheKey, wrapped);
17309
- for (const waitingRequest of requestsForKey) {
17310
- successfulLiveStepCallIds.add(waitingRequest.callId);
17311
- const resolver = this.toolCallResolvers.get(
17312
- waitingRequest.callId,
17313
- );
17314
- if (resolver) {
17315
- resolver.resolve(wrapped);
17316
- this.toolCallResolvers.delete(waitingRequest.callId);
17317
- }
17318
- this.emitScopedFieldMetaUpdate({
17319
- rowId: waitingRequest.rowId,
17320
- key: waitingRequest.rowKey ?? null,
17321
- tableNamespace: waitingRequest.tableNamespace ?? null,
17322
- fieldName: waitingRequest.fieldName,
17323
- status: 'cached',
17324
- rowStatus: 'running',
17325
- stage: toolId,
17326
- provider: null,
17327
- error: null,
17328
- reused: true,
17329
- dataPatch: {},
17330
- });
17331
- }
17332
- };
17333
- const claimOwnerWithLiveFollowers = (
17334
- owner: ToolCallRequest | undefined,
17335
- followers: ToolCallRequest[],
17336
- ): void => {
17337
- if (!owner) return;
17338
- claimedRequests.push(owner);
17339
- if (followers.length > 0) {
17340
- liveFollowersByOwnerCallId.set(owner.callId, followers);
17341
- }
17342
- };
17343
- const resolveReceiptWaitMaxAttempts = (
17344
- requestsForKey: ToolCallRequest[],
17345
- ): number | undefined =>
17346
- requestsForKey.length > 0
17347
- ? Math.max(
17348
- ...requestsForKey.map((request) =>
17349
- resolveRuntimeToolReceiptWaitMaxAttempts(
17350
- typeof request.receiptWaitMs === 'number'
17351
- ? { max_wait_ms: request.receiptWaitMs }
17352
- : request.input,
17353
- ),
17354
- ),
17355
- )
17356
- : undefined;
17357
- const queueExistingRunningReceiptWait = (
17358
- receiptKey: string,
17359
- requestsForKey: ToolCallRequest[],
17360
- forceAfterWait = false,
17361
- skipWait = false,
17362
- ): void => {
17363
- durableExistingRunningWaits.push({
17364
- receiptKey,
17365
- requestsForKey,
17366
- forceAfterWait,
17367
- waitMaxAttempts: resolveReceiptWaitMaxAttempts(requestsForKey),
17368
- skipWait,
16422
+ if (pendingRequests.length > 0) {
16423
+ const strategy = operationStrategy;
16424
+
16425
+ if (strategy) {
16426
+ const compiledBatches = compileRequestsWithStrategy({
16427
+ requests: pendingRequests,
16428
+ strategy,
16429
+ getPayload: (request: ToolCallRequest) => request.input,
17369
16430
  });
17370
- };
17371
- const reclaimExistingRunningReceiptAfterWait = async (
17372
- receiptKey: string,
17373
- requestsForKey: ToolCallRequest[],
17374
- forceAfterWait = false,
17375
- waitResult?: {
17376
- completedReceipt?: RuntimeStepReceipt;
17377
- skipWait?: boolean;
17378
- },
17379
- ): Promise<void> => {
17380
- if (waitResult?.completedReceipt) {
17381
- if (!forceAfterWait) {
17382
- await resolveRequestsFromReceipt(
17383
- requestsForKey,
17384
- waitResult.completedReceipt,
17385
- 'in_flight',
17386
- );
17387
- return;
17388
- }
17389
- } else if (waitResult?.skipWait !== true) {
17390
- const waitMaxAttempts =
17391
- resolveReceiptWaitMaxAttempts(requestsForKey);
17392
- try {
17393
- const completed = await this.waitForCompletedRuntimeToolReceipt(
17394
- receiptKey,
17395
- waitMaxAttempts,
17396
- );
17397
- if (!forceAfterWait) {
17398
- await resolveRequestsFromReceipt(
17399
- requestsForKey,
17400
- completed,
17401
- 'in_flight',
17402
- );
17403
- return;
17404
- }
17405
- } catch (error) {
17406
- if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
17407
- for (const request of requestsForKey) {
17408
- await this.rejectToolCall(toolId, request, error, {
17409
- persistReceiptFailure: false,
17410
- });
17411
- }
17412
- return;
17413
- }
17414
- }
17415
- }
17416
- const reclaimed = (
17417
- await this.claimRuntimeStepReceipts(
17418
- [receiptKey],
17419
- this.currentReceiptOwnerRunId,
17420
- true,
17421
- forceAfterWait,
17422
- )
17423
- ).get(receiptKey);
17424
- if (
17425
- reclaimed?.status === 'completed' ||
17426
- reclaimed?.status === 'skipped'
17427
- ) {
17428
- await resolveRequestsFromReceipt(
17429
- requestsForKey,
17430
- reclaimed,
17431
- 'cache',
17432
- );
17433
- durableRecoveredRequests.push(...requestsForKey);
17434
- return;
17435
- }
17436
- if (reclaimed?.status === 'failed') {
17437
- for (const request of requestsForKey) {
17438
- await this.rejectToolCall(
17439
- toolId,
17440
- request,
17441
- runtimeReceiptFailureError(
17442
- reclaimed,
17443
- 'Durable tool call failed',
17444
- this.currentToolErrorSchemaVersion,
17445
- ),
17446
- );
17447
- }
17448
- return;
17449
- }
17450
- if (this.isOwnedClaimedRuntimeReceipt(reclaimed)) {
17451
- const [owner, ...waiters] = requestsForKey;
17452
- if (!owner) return;
17453
- if (!reclaimed.leaseId) {
17454
- throw new RuntimeReceiptLeaseLostError({
17455
- receiptKey,
17456
- runId: this.currentReceiptOwnerRunId,
17457
- leaseId: 'missing',
17458
- });
17459
- }
17460
- owner.receiptLeaseId = reclaimed.leaseId ?? null;
17461
- owner.receiptLeaseExpiresAt = reclaimed.leaseExpiresAt ?? null;
17462
- if (waiters.length > 0) {
17463
- liveFollowersByOwnerCallId.set(owner.callId, waiters);
17464
- }
17465
- try {
17466
- // Circuit breaker: a receipt reclaimed after its owner's lease
17467
- // lapsed is still a fresh provider call, and the latch may
17468
- // have tripped while this group waited for that lease.
16431
+
16432
+ const batchParallelismCeiling =
16433
+ this.governor.policy.pacing.workerToolBatchDefaultParallelism;
16434
+ const batchSize =
16435
+ compiledBatches.length > 0
16436
+ ? await this.resourceGovernor.suggestedToolParallelism(
16437
+ compiledBatches[0]!.batchOperation,
16438
+ batchParallelismCeiling,
16439
+ )
16440
+ : batchParallelismCeiling;
16441
+ await executeChunkedRequests({
16442
+ requests: compiledBatches,
16443
+ batchSize,
16444
+ execute: async (batch) => {
16445
+ // Circuit breaker: skip dispatching this batch's provider call
16446
+ // once a persistence failure has occurred in this run.
17469
16447
  if (this.persistenceLatch.tripped) {
17470
16448
  this.persistenceLatch.preventedCallCount +=
17471
- requestsForKey.length;
16449
+ batch.memberRequests.length;
17472
16450
  throw new RuntimePersistenceCircuitOpenError(
17473
16451
  this.persistenceLatch,
17474
16452
  );
17475
16453
  }
17476
- const execution = await this.executeMappedToolCallThroughModule(
17477
- {
17478
- request: owner,
17479
- dispatchToolId: toolId,
17480
- dispatchPayload: owner.input,
17481
- resultReceiptKey: receiptKey,
17482
- providerOperationKey:
17483
- this.providerOperationKeyForQueuedToolCall(owner),
17484
- apiOptions: {
17485
- beforeProviderCall: () =>
17486
- this.assertRuntimeToolReceiptOwnership([owner]),
17487
- durableCallReceiptKey: receiptKey,
17488
- playNodeScope: playNodeScopeForToolCallRequest(owner),
17489
- executionAuthScopeDigest: owner.executionAuthScopeDigest,
17490
- receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
17491
- heartbeatReceipt: () =>
17492
- this.renewRuntimeToolReceiptOwnership([owner]),
17493
- providerIdempotencyKey:
17494
- this.providerIdempotencyKeyForToolCall({
17495
- operationKey:
17496
- this.providerOperationKeyForQueuedToolCall(owner),
17497
- force: owner.force === true,
17498
- logicalCallId: owner.logicalCallId,
17499
- }),
17500
- ...this.providerForceRefreshOptionsForToolCall({
17501
- operationKey:
17502
- this.providerOperationKeyForQueuedToolCall(owner),
17503
- force: owner.force,
17504
- requiresExecutionFence: owner.requiresExecutionFence,
17505
- logicalCallId: owner.logicalCallId,
17506
- }),
17507
- timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
17508
- owner,
17509
- ]),
17510
- },
17511
- },
17512
- );
17513
- const result = await this.resolveToolCall(
17514
- toolId,
17515
- owner,
17516
- execution,
17517
- );
17518
- if (result != null) {
17519
- successfulLiveStepCallIds.add(owner.callId);
17520
- }
17521
- resolveLiveFollowers(owner, result);
17522
- recordToolStep([owner]);
17523
- this.#options.onBatchComplete?.(this.checkpoint);
17524
- } catch (error) {
17525
- await rejectWithLiveFollowers(owner, error);
17526
- }
17527
- return;
17528
- }
17529
- for (const request of requestsForKey) {
17530
- await this.rejectToolCall(
17531
- toolId,
17532
- request,
17533
- new RuntimeReceiptWaitTimeoutError(receiptKey),
17534
- { persistReceiptFailure: false },
17535
- );
17536
- }
17537
- return;
17538
- };
17539
- const reclaimExistingRunningReceipts = async (
17540
- waits: Array<(typeof durableExistingRunningWaits)[number]>,
17541
- ): Promise<void> => {
17542
- if (waits.length === 0) return;
17543
- const settlements = await Promise.allSettled(
17544
- waits.map((wait) =>
17545
- reclaimExistingRunningReceiptAfterWait(
17546
- wait.receiptKey,
17547
- wait.requestsForKey,
17548
- wait.forceAfterWait,
17549
- { skipWait: true },
17550
- ),
17551
- ),
17552
- );
17553
- const rejected = settlements.find(
17554
- (settlement) => settlement.status === 'rejected',
17555
- );
17556
- if (rejected?.status === 'rejected') throw rejected.reason;
17557
- };
17558
- const processExistingRunningReceiptWaits =
17559
- async (): Promise<void> => {
17560
- if (durableExistingRunningWaits.length === 0) return;
17561
- const waitSignal = durableExistingRunningAbort.signal;
17562
- // A claim that came back `existing` behind an expired lease is
17563
- // not blocked on live work; it is holding a dead run's row.
17564
- // Reclaim those now so this run executes the call itself, and
17565
- // poll only the leases that are genuinely still alive.
17566
- const expiredLeaseWaits = durableExistingRunningWaits.filter(
17567
- (wait) => wait.skipWait === true,
17568
- );
17569
- const pendingWaits = new Map(
17570
- durableExistingRunningWaits
17571
- .filter((wait) => wait.skipWait !== true)
17572
- .map((wait) => [wait.receiptKey, wait]),
17573
- );
17574
- const parkedAtMs = Date.now();
17575
- for (const wait of durableExistingRunningWaits) {
17576
- this.#runtimeReceiptWaitParking.set(wait.receiptKey, {
17577
- startedAtMs: parkedAtMs,
17578
- rows: wait.requestsForKey.length,
17579
- stage:
17580
- wait.skipWait === true
17581
- ? 'expired_lease_reclaim'
17582
- : 'receipt_lease_wait',
17583
- });
17584
- }
17585
- const pollWaits = async (): Promise<void> => {
17586
- if (pendingWaits.size === 0) return;
17587
- const waitMaxAttempts =
17588
- Math.max(
17589
- ...[...pendingWaits.values()].map(
17590
- (wait) => wait.waitMaxAttempts ?? 0,
17591
- ),
17592
- ) || DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
17593
- // The backoff cap means the attempt count no longer states a
17594
- // wall time. This deadline is the budget those attempts were
17595
- // meant to express; crossing it falls through to the same
17596
- // reclaim path an expired lease takes.
17597
- const waitDeadline =
17598
- Date.now() +
17599
- runtimeReceiptWaitBudgetMs(
17600
- waitMaxAttempts,
17601
- DURABLE_RECEIPT_WAIT_DELAY_MS,
16454
+ const aggregateForceRefreshIdentities =
16455
+ batch.memberRequests.map((request) =>
16456
+ this.providerInvocationKeyForToolCall({
16457
+ operationKey:
16458
+ this.providerOperationKeyForQueuedToolCall(request),
16459
+ logicalCallId: request.logicalCallId,
16460
+ force: request.force,
16461
+ }),
17602
16462
  );
17603
- if (runtimeReceiptReadTraceEnabled) {
17604
- this.runtimeLog(
17605
- `[runtime-receipt-wait] phase=start requested=${pendingWaits.size} ` +
17606
- `request_digests=${[...pendingWaits.keys()]
17607
- .map(runtimeReceiptKeyDigest)
17608
- .join(',')} max_attempts=${waitMaxAttempts}`,
16463
+ const legacyForceRefreshIdentities = batch.memberRequests
16464
+ .filter((request) => request.force === true)
16465
+ .map((request) =>
16466
+ this.providerInvocationKeyForToolCall({
16467
+ operationKey:
16468
+ this.providerOperationKeyForQueuedToolCall(request),
16469
+ logicalCallId: request.logicalCallId,
16470
+ force: true,
16471
+ }),
17609
16472
  );
17610
- }
17611
- for (
17612
- let attempt = 0;
17613
- attempt < waitMaxAttempts && pendingWaits.size > 0;
17614
- attempt += 1
17615
- ) {
17616
- if (waitSignal.aborted) return;
17617
- if (attempt > 0) {
17618
- if (Date.now() >= waitDeadline) break;
17619
- const delayMs = Math.min(
17620
- runtimeReceiptWaitDelayMs(attempt),
17621
- Math.max(0, waitDeadline - Date.now()),
17622
- );
17623
- await new Promise<void>((resolve) => {
17624
- const finish = () => {
17625
- clearTimeout(timeout);
17626
- waitSignal.removeEventListener('abort', finish);
17627
- resolve();
17628
- };
17629
- const timeout = setTimeout(finish, delayMs);
17630
- waitSignal.addEventListener('abort', finish, {
17631
- once: true,
17632
- });
17633
- });
17634
- if (waitSignal.aborted) return;
17635
- }
17636
- // The long backoff lives outside the hydration lane. Only a
17637
- // bounded receipt read and delivery hold the lane, so stale
17638
- // ownership cannot block unrelated cache misses/provider
17639
- // work while terminal payloads still hydrate one group at a
17640
- // time.
17641
- try {
17642
- await this.withRuntimeReceiptHydrationTurn(
17643
- async (turn) => {
17644
- if (waitSignal.aborted) return;
17645
- const waited =
17646
- await this.waitForCompletedRuntimeToolReceipts(
17647
- [...pendingWaits.keys()],
17648
- 1,
17649
- );
17650
- turn.settleBytes(
17651
- runtimeStepReceiptPayloadBytes(
17652
- waited.completed.values(),
17653
- ),
17654
- );
17655
- if (waitSignal.aborted) return;
17656
- const settledWaits = [...pendingWaits.values()].filter(
17657
- (wait) =>
17658
- waited.completed.has(wait.receiptKey) ||
17659
- waited.failed.has(wait.receiptKey),
17660
- );
17661
- const deliverySettlements = await Promise.allSettled(
17662
- settledWaits.map(async (wait) => {
17663
- const failed = waited.failed.get(wait.receiptKey);
17664
- if (failed) {
17665
- for (const request of wait.requestsForKey) {
17666
- await this.rejectToolCall(
17667
- toolId,
17668
- request,
17669
- failed,
17670
- {
17671
- persistReceiptFailure: false,
17672
- },
17673
- );
17674
- }
17675
- return;
17676
- }
17677
- await reclaimExistingRunningReceiptAfterWait(
17678
- wait.receiptKey,
17679
- wait.requestsForKey,
17680
- wait.forceAfterWait,
17681
- {
17682
- completedReceipt: waited.completed.get(
17683
- wait.receiptKey,
16473
+ const batchActionVersion =
16474
+ (
16475
+ await this.#options.getToolActionCacheVersion?.(
16476
+ batch.batchOperation,
16477
+ )
16478
+ )?.trim() ?? 'unversioned';
16479
+ const aggregateProviderOperationKey =
16480
+ buildDurableToolPayloadProviderIdempotencyKey({
16481
+ orgId: this.#options.orgId,
16482
+ toolId: batch.batchOperation,
16483
+ executionAuthScopeDigest:
16484
+ batch.memberRequests[0]?.executionAuthScopeDigest ?? null,
16485
+ providerActionVersion: batchActionVersion,
16486
+ payload: batch.batchPayload,
16487
+ forceRefreshIdentities: aggregateForceRefreshIdentities,
16488
+ });
16489
+ const aggregateReceiptKey = this.toolTransportReceiptKey({
16490
+ toolId: batch.batchOperation,
16491
+ providerOperationKey: aggregateProviderOperationKey,
16492
+ });
16493
+ let releaseToolSlot: () => void = () => undefined;
16494
+ try {
16495
+ const execution =
16496
+ await this.executeMappedToolCallThroughModule({
16497
+ request: batch.memberRequests[0]!,
16498
+ dispatchToolId: batch.batchOperation,
16499
+ dispatchPayload: batch.batchPayload,
16500
+ resultReceiptKey: aggregateReceiptKey,
16501
+ providerOperationKey: aggregateProviderOperationKey,
16502
+ apiOptions: {
16503
+ durableCallReceiptKey: aggregateReceiptKey,
16504
+ legacyProviderRetryIdentities: [
16505
+ {
16506
+ transportReceiptKey:
16507
+ buildDurableToolAggregateReceiptKey({
16508
+ receiptKeys: batch.memberRequests.map(
16509
+ (member) => member.cacheKey,
17684
16510
  ),
17685
- skipWait: true,
17686
- },
17687
- );
17688
- }),
17689
- );
17690
- for (
17691
- let index = 0;
17692
- index < settledWaits.length;
17693
- index += 1
17694
- ) {
17695
- if (
17696
- deliverySettlements[index]?.status === 'fulfilled'
17697
- ) {
17698
- const settledKey = settledWaits[index]!.receiptKey;
17699
- pendingWaits.delete(settledKey);
17700
- this.#runtimeReceiptWaitParking.delete(settledKey);
17701
- }
17702
- }
17703
- if (runtimeReceiptReadTraceEnabled) {
17704
- const rejected = deliverySettlements.flatMap(
17705
- (settlement, index) => {
17706
- if (settlement.status !== 'rejected') return [];
17707
- const receiptKey =
17708
- settledWaits[index]?.receiptKey;
17709
- return receiptKey
17710
- ? [
17711
- `${runtimeReceiptKeyDigest(receiptKey)}:${
17712
- settlement.reason instanceof Error
17713
- ? settlement.reason.name
17714
- : 'non_error'
17715
- }`,
17716
- ]
17717
- : ['unknown'];
17718
- },
17719
- );
17720
- this.runtimeLog(
17721
- `[runtime-receipt-wait] phase=delivery settled=${deliverySettlements.length} ` +
17722
- `rejected=${rejected.length} rejected_digests=${rejected.join(',')}`,
17723
- );
17724
- }
17725
- const rejectedDelivery = deliverySettlements.find(
17726
- (settlement) => settlement.status === 'rejected',
17727
- );
17728
- if (rejectedDelivery?.status === 'rejected') {
17729
- throw rejectedDelivery.reason;
17730
- }
17731
- },
17732
- {
17733
- receiptKeys: pendingWaits.size,
17734
- source: 'map_receipt_wait',
17735
- // The lane wait is part of this loop's budget, not
17736
- // extra time on top of it.
17737
- timeoutMs: Math.max(0, waitDeadline - Date.now()),
16511
+ prefix: 'batch',
16512
+ aggregateReceiptPrefix:
16513
+ buildDurableToolReceiptPrefix({
16514
+ orgId: this.#options.orgId,
16515
+ toolId: batch.batchOperation,
16516
+ }),
16517
+ }),
16518
+ providerOperationKey:
16519
+ buildDurableToolPayloadProviderIdempotencyKey({
16520
+ orgId: this.#options.orgId,
16521
+ toolId: batch.batchOperation,
16522
+ executionAuthScopeDigest:
16523
+ batch.memberRequests[0]
16524
+ ?.executionAuthScopeDigest ?? null,
16525
+ providerActionVersion: batchActionVersion,
16526
+ payload: batch.batchPayload,
16527
+ forceRefreshIdentities:
16528
+ legacyForceRefreshIdentities,
16529
+ }),
16530
+ forceRefreshIdentities:
16531
+ legacyForceRefreshIdentities,
16532
+ },
16533
+ ],
16534
+ // A native batch is one physical provider request for
16535
+ // many rows of the same column, so the batch's node scope
16536
+ // is its members' shared scope.
16537
+ playNodeScope: playNodeScopeForToolCallRequest(
16538
+ batch.memberRequests[0]!,
16539
+ batch.batchOperation,
16540
+ ),
16541
+ executionAuthScopeDigest:
16542
+ batch.memberRequests[0]?.executionAuthScopeDigest ??
16543
+ null,
16544
+ providerIdempotencyKey: aggregateProviderOperationKey,
16545
+ providerOperationForceRefreshIdentities:
16546
+ aggregateForceRefreshIdentities,
16547
+ receiptLeaseExpiresAt: batch.memberRequests.reduce<
16548
+ string | null
16549
+ >((earliest, request) => {
16550
+ const expiresAt =
16551
+ request.receiptLeaseExpiresAt ?? null;
16552
+ if (!expiresAt) return earliest;
16553
+ if (!earliest) return expiresAt;
16554
+ return Date.parse(expiresAt) < Date.parse(earliest)
16555
+ ? expiresAt
16556
+ : earliest;
16557
+ }, null),
16558
+ timeoutMs: resolveRuntimeTimeoutMsForRequests(
16559
+ batch.memberRequests,
16560
+ ),
16561
+ retainToolSlot: (release) => {
16562
+ releaseToolSlot = release;
16563
+ },
17738
16564
  },
17739
- );
17740
- } catch (error) {
17741
- // The budget is spent, not the run: fall through to the
17742
- // same reclaim an expired lease takes.
17743
- if (
17744
- !(
17745
- error instanceof RuntimeReceiptHydrationTurnTimeoutError
17746
- )
17747
- ) {
17748
- throw error;
17749
- }
17750
- break;
17751
- }
17752
- }
17753
- if (waitSignal.aborted) return;
17754
- await reclaimExistingRunningReceipts([
17755
- ...pendingWaits.values(),
17756
- ]);
17757
- };
17758
- try {
17759
- const lanes = await Promise.allSettled([
17760
- reclaimExistingRunningReceipts(expiredLeaseWaits),
17761
- pollWaits(),
17762
- ]);
17763
- const rejectedLane = lanes.find(
17764
- (lane) => lane.status === 'rejected',
17765
- );
17766
- if (rejectedLane?.status === 'rejected') {
17767
- throw rejectedLane.reason;
17768
- }
17769
- } finally {
17770
- for (const wait of durableExistingRunningWaits) {
17771
- this.#runtimeReceiptWaitParking.delete(wait.receiptKey);
16565
+ });
16566
+ return { execution, releaseToolSlot };
16567
+ } catch (error) {
16568
+ releaseToolSlot();
16569
+ throw error;
17772
16570
  }
17773
- }
17774
- };
17775
-
17776
- for (const [receiptKey, group] of requestsByReceiptKey) {
17777
- const requestsForKey = group.requests;
17778
- const claim = claims.get(receiptKey);
17779
- if (!claim) {
17780
- queueExistingRunningReceiptWait(
17781
- receiptKey,
17782
- requestsForKey,
17783
- group.forceRefresh,
17784
- );
17785
- continue;
17786
- }
17787
- if (claim?.status === 'completed' || claim?.status === 'skipped') {
17788
- await resolveRequestsFromReceipt(requestsForKey, claim, 'cache');
17789
- durableRecoveredRequests.push(...requestsForKey);
17790
- continue;
17791
- }
17792
- if (claim?.status === 'failed') {
17793
- for (const request of requestsForKey) {
17794
- await this.rejectToolCall(
17795
- toolId,
17796
- request,
17797
- runtimeReceiptFailureError(
17798
- claim,
17799
- 'Durable tool call failed',
17800
- this.currentToolErrorSchemaVersion,
17801
- ),
17802
- );
17803
- }
17804
- continue;
17805
- }
17806
- if (
17807
- (claim?.status === 'queued' ||
17808
- claim?.status === 'pending' ||
17809
- claim?.status === 'running') &&
17810
- claim.claimState === 'existing'
17811
- ) {
17812
- queueExistingRunningReceiptWait(
17813
- receiptKey,
17814
- requestsForKey,
17815
- group.forceRefresh,
17816
- !runtimeReceiptClaimLeaseIsLive(claim),
17817
- );
17818
- continue;
17819
- }
17820
- if (this.isOwnedClaimedRuntimeReceipt(claim)) {
17821
- const [owner, ...waiters] = requestsForKey;
17822
- if (!claim.leaseId) {
17823
- throw new RuntimeReceiptLeaseLostError({
17824
- receiptKey,
17825
- runId: this.currentReceiptOwnerRunId,
17826
- leaseId: 'missing',
17827
- });
17828
- }
17829
- if (owner) {
17830
- owner.receiptLeaseId = claim.leaseId ?? null;
17831
- owner.receiptLeaseExpiresAt = claim.leaseExpiresAt ?? null;
17832
- const claimLeaseExpiresAtMs = Date.parse(
17833
- claim.leaseExpiresAt ?? '',
17834
- );
17835
- pendingReceiptClaimsEstablishExecutionFence &&=
17836
- claim.claimState === 'claimed' &&
17837
- claim.status === 'running' &&
17838
- Number.isFinite(claimLeaseExpiresAtMs) &&
17839
- claimLeaseExpiresAtMs > Date.now();
17840
- }
17841
- claimOwnerWithLiveFollowers(owner, waiters);
17842
- continue;
17843
- }
17844
- queueExistingRunningReceiptWait(
17845
- receiptKey,
17846
- requestsForKey,
17847
- group.forceRefresh,
17848
- );
17849
- }
17850
-
17851
- pendingRequests.length = 0;
17852
- pendingRequests.push(...claimedRequests);
17853
- durableExistingRunningState.settlement =
17854
- durableExistingRunningWaits.length > 0
17855
- ? processExistingRunningReceiptWaits().then(
17856
- () => ({ status: 'fulfilled' as const }),
17857
- (reason: unknown) => ({
17858
- status: 'rejected' as const,
17859
- reason,
17860
- }),
17861
- )
17862
- : null;
17863
- recordToolStep(durableRecoveredRequests);
17864
- }
17865
-
17866
- // Claims establish one lease cohort. Revalidate that cohort in one
17867
- // gateway operation before provider admission, then let the existing
17868
- // cohort heartbeat keep the undispatched tail alive. Revalidating each
17869
- // call separately after a tool slot opens fragments the writer into
17870
- // one-row batches and starves otherwise healthy providers.
17871
- const pendingReceiptHeartbeat = this.startPendingToolReceiptHeartbeat(
17872
- pendingRequests,
17873
- pendingReceiptClaimsEstablishExecutionFence,
17874
- );
17875
- let providerFailure: { reason: unknown } | null = null;
17876
- try {
17877
- if (pendingReceiptHeartbeat) {
17878
- await pendingReceiptHeartbeat.confirmOwned();
17879
- }
17880
- if (pendingRequests.length > 0) {
17881
- const strategy = operationStrategy;
17882
-
17883
- if (strategy) {
17884
- const compiledBatches = compileRequestsWithStrategy({
17885
- requests: pendingRequests,
17886
- strategy,
17887
- getPayload: (request: ToolCallRequest) => request.input,
17888
- });
17889
-
17890
- const batchParallelismCeiling =
17891
- this.governor.policy.pacing.workerToolBatchDefaultParallelism;
17892
- const batchSize =
17893
- compiledBatches.length > 0
17894
- ? await this.resourceGovernor.suggestedToolParallelism(
17895
- compiledBatches[0]!.batchOperation,
17896
- batchParallelismCeiling,
17897
- )
17898
- : batchParallelismCeiling;
17899
- await executeChunkedRequests({
17900
- requests: compiledBatches,
17901
- batchSize,
17902
- execute: async (batch) => {
17903
- // Circuit breaker: skip dispatching this batch's provider call
17904
- // once a persistence failure has occurred in this run.
17905
- if (this.persistenceLatch.tripped) {
17906
- this.persistenceLatch.preventedCallCount +=
17907
- batch.memberRequests.length;
17908
- throw new RuntimePersistenceCircuitOpenError(
17909
- this.persistenceLatch,
16571
+ },
16572
+ onChunkComplete: async (chunkResults) => {
16573
+ for (const entry of chunkResults) {
16574
+ if (entry.error !== undefined) {
16575
+ const settlements = await Promise.allSettled(
16576
+ entry.request.memberRequests.map((request) =>
16577
+ rejectCall(request, entry.error),
16578
+ ),
17910
16579
  );
17911
- }
17912
- const receiptKeys = batch.memberRequests.map(
17913
- (request) => request.cacheKey,
17914
- );
17915
- const aggregateReceiptKey =
17916
- buildDurableToolAggregateReceiptKey({
17917
- receiptKeys,
17918
- prefix: 'batch',
17919
- aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
17920
- orgId: this.#options.orgId,
17921
- toolId: batch.batchOperation,
17922
- }),
17923
- });
17924
- const aggregateForceRefreshIdentities = batch.memberRequests
17925
- .filter((request) => request.force === true)
17926
- .map((request) =>
17927
- this.providerIdempotencyKeyForToolCall({
17928
- operationKey:
17929
- this.providerOperationKeyForQueuedToolCall(request),
17930
- force: true,
17931
- logicalCallId: request.logicalCallId,
17932
- }),
16580
+ const rejected = settlements.find(
16581
+ (settlement): settlement is PromiseRejectedResult =>
16582
+ settlement.status === 'rejected',
17933
16583
  );
17934
- const aggregateProviderOperationKey =
17935
- buildDurableToolPayloadProviderIdempotencyKey({
17936
- orgId: this.#options.orgId,
17937
- toolId: batch.batchOperation,
17938
- executionAuthScopeDigest:
17939
- batch.memberRequests[0]?.executionAuthScopeDigest ??
17940
- null,
17941
- providerActionVersion:
17942
- (
17943
- await this.#options.getToolActionCacheVersion?.(
17944
- batch.batchOperation,
17945
- )
17946
- )?.trim() ?? 'unversioned',
17947
- payload: batch.batchPayload,
17948
- forceRefreshIdentities: aggregateForceRefreshIdentities,
17949
- });
17950
- let releaseToolSlot: () => void = () => undefined;
16584
+ if (rejected) throw rejected.reason;
16585
+ continue;
16586
+ }
16587
+ const batchExecution = entry.result;
17951
16588
  try {
17952
- const execution =
17953
- await this.executeMappedToolCallThroughModule({
17954
- request: batch.memberRequests[0]!,
17955
- dispatchToolId: batch.batchOperation,
17956
- dispatchPayload: batch.batchPayload,
17957
- resultReceiptKey: aggregateReceiptKey,
17958
- providerOperationKey: aggregateProviderOperationKey,
17959
- apiOptions: {
17960
- durableCallReceiptKey: aggregateReceiptKey,
17961
- // A native batch is one physical provider request for
17962
- // many rows of the same column, so the batch's node scope
17963
- // is its members' shared scope.
17964
- playNodeScope: playNodeScopeForToolCallRequest(
17965
- batch.memberRequests[0]!,
17966
- batch.batchOperation,
17967
- ),
17968
- executionAuthScopeDigest:
17969
- batch.memberRequests[0]?.executionAuthScopeDigest ??
17970
- null,
17971
- providerIdempotencyKey: aggregateProviderOperationKey,
17972
- providerOperationForceRefreshIdentities:
17973
- aggregateForceRefreshIdentities,
17974
- receiptLeaseExpiresAt: batch.memberRequests.reduce<
17975
- string | null
17976
- >((earliest, request) => {
17977
- const expiresAt =
17978
- request.receiptLeaseExpiresAt ?? null;
17979
- if (!expiresAt) return earliest;
17980
- if (!earliest) return expiresAt;
17981
- return Date.parse(expiresAt) < Date.parse(earliest)
17982
- ? expiresAt
17983
- : earliest;
17984
- }, null),
17985
- timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
17986
- batch.memberRequests,
17987
- ),
17988
- beforeProviderCall: () =>
17989
- pendingReceiptHeartbeat
17990
- ? pendingReceiptHeartbeat.assertOwned()
17991
- : this.assertRuntimeToolReceiptOwnership(
17992
- batch.memberRequests,
17993
- ),
17994
- heartbeatReceipt: () =>
17995
- this.renewRuntimeToolReceiptOwnership(
17996
- batch.memberRequests,
16589
+ const splitResults =
16590
+ batchExecution != null
16591
+ ? entry.request.splitResults(
16592
+ legacyResultForBatchSplitter(
16593
+ batchExecution.execution,
17997
16594
  ),
17998
- retainToolSlot: (release) => {
17999
- releaseToolSlot = release;
18000
- },
18001
- },
18002
- });
18003
- return { execution, releaseToolSlot };
18004
- } catch (error) {
18005
- releaseToolSlot();
18006
- throw error;
18007
- }
18008
- },
18009
- onChunkComplete: async (chunkResults) => {
18010
- for (const entry of chunkResults) {
18011
- if (entry.error !== undefined) {
18012
- const settlements = await Promise.allSettled(
18013
- entry.request.memberRequests.map((request) =>
18014
- rejectWithLiveFollowers(request, entry.error),
18015
- ),
18016
- );
18017
- const rejected = settlements.find(
18018
- (settlement): settlement is PromiseRejectedResult =>
18019
- settlement.status === 'rejected',
16595
+ )
16596
+ : entry.request.memberRequests.map(() => null);
16597
+ const resolvedResults =
16598
+ await this.resolveToolCallBatchResults(
16599
+ toolId,
16600
+ entry.request.memberRequests.map((request, index) => ({
16601
+ request,
16602
+ result: splitResults[index] ?? null,
16603
+ status: batchExecution?.execution.status,
16604
+ toolResponse:
16605
+ batchExecution == null
16606
+ ? undefined
16607
+ : publicToolResponseForBatchedItem(
16608
+ batchExecution.execution,
16609
+ splitResults[index] ?? null,
16610
+ true,
16611
+ ),
16612
+ })),
18020
16613
  );
18021
- if (rejected) throw rejected.reason;
18022
- continue;
18023
- }
18024
- const batchExecution = entry.result;
18025
- try {
18026
- const splitResults =
18027
- batchExecution != null
18028
- ? entry.request.splitResults(
18029
- legacyResultForBatchSplitter(
18030
- batchExecution.execution,
18031
- ),
18032
- )
18033
- : entry.request.memberRequests.map(() => null);
18034
- const resolvedResults =
18035
- await this.resolveToolCallBatchResults(
18036
- toolId,
18037
- entry.request.memberRequests.map(
18038
- (request, index) => ({
18039
- request,
18040
- result: splitResults[index] ?? null,
18041
- status: batchExecution?.execution.status,
18042
- toolResponse:
18043
- batchExecution == null
18044
- ? undefined
18045
- : publicToolResponseForBatchedItem(
18046
- batchExecution.execution,
18047
- splitResults[index] ?? null,
18048
- true,
18049
- ),
18050
- }),
18051
- ),
18052
- );
18053
16614
 
18054
- for (
18055
- let index = 0;
18056
- index < entry.request.memberRequests.length;
18057
- index += 1
18058
- ) {
18059
- const request = entry.request.memberRequests[index]!;
18060
- if (resolvedResults[index] != null) {
18061
- successfulLiveStepCallIds.add(request.callId);
18062
- }
18063
- resolveLiveFollowers(request, resolvedResults[index]);
16615
+ for (
16616
+ let index = 0;
16617
+ index < entry.request.memberRequests.length;
16618
+ index += 1
16619
+ ) {
16620
+ const request = entry.request.memberRequests[index]!;
16621
+ if (resolvedResults[index] != null) {
16622
+ successfulLiveStepCallIds.add(request.callId);
18064
16623
  }
18065
- } finally {
18066
- batchExecution?.releaseToolSlot();
18067
16624
  }
16625
+ } finally {
16626
+ batchExecution?.releaseToolSlot();
18068
16627
  }
16628
+ }
18069
16629
 
18070
- recordToolStep(
18071
- chunkResults.flatMap(
18072
- (entry) => entry.request.memberRequests,
18073
- ),
18074
- );
18075
- this.#options.onBatchComplete?.(this.checkpoint);
18076
- },
18077
- retainResults: false,
18078
- });
18079
- } else {
18080
- // Bound direct map dispatch separately from ordinary provider
18081
- // residency. This prevents a blocked first chunk from launching
18082
- // another chunk ahead of a provider's admitted start rate, while
18083
- // preserving the wider resident-call capacity used by batched and
18084
- // high-latency providers.
18085
- const toolCallConcurrencyCeiling =
18086
- this.governor.policy.concurrency.toolCalls;
18087
- const shapedToolParallelism =
18088
- await this.resourceGovernor.suggestedDirectToolParallelism(
18089
- toolId,
18090
- toolCallConcurrencyCeiling,
16630
+ recordToolStep(
16631
+ chunkResults.flatMap((entry) => entry.request.memberRequests),
18091
16632
  );
18092
- const dispatchWidth = Math.min(
16633
+ this.#options.onBatchComplete?.(this.checkpoint);
16634
+ },
16635
+ retainResults: false,
16636
+ });
16637
+ } else {
16638
+ // Bound direct map dispatch separately from ordinary provider
16639
+ // residency. This prevents a blocked first chunk from launching
16640
+ // another chunk ahead of a provider's admitted start rate, while
16641
+ // preserving the wider resident-call capacity used by batched and
16642
+ // high-latency providers.
16643
+ const toolCallConcurrencyCeiling =
16644
+ this.governor.policy.concurrency.toolCalls;
16645
+ const shapedToolParallelism =
16646
+ await this.resourceGovernor.suggestedDirectToolParallelism(
16647
+ toolId,
18093
16648
  toolCallConcurrencyCeiling,
18094
- Math.max(1, shapedToolParallelism),
18095
16649
  );
18096
- const dispatchResults = await dispatchBoundedSettled(
18097
- pendingRequests,
18098
- dispatchWidth,
18099
- async (request) => {
18100
- // Circuit breaker: do not dispatch this provider call once a
18101
- // persistence failure has occurred in this run.
18102
- if (this.persistenceLatch.tripped) {
18103
- this.persistenceLatch.preventedCallCount += 1;
18104
- throw new RuntimePersistenceCircuitOpenError(
18105
- this.persistenceLatch,
18106
- );
18107
- }
18108
- let releaseToolSlot: () => void = () => undefined;
18109
- try {
18110
- const execution =
18111
- await this.executeMappedToolCallThroughModule({
18112
- request,
18113
- dispatchToolId: toolId,
18114
- dispatchPayload: request.input,
18115
- // Always-fresh map calls intentionally have no
18116
- // result cache. Their provider-operation identity is
18117
- // a unique forced semantic request, never a cache key.
18118
- resultReceiptKey:
18119
- request.receiptKey ?? request.cacheKey,
18120
- providerOperationKey:
18121
- this.providerOperationKeyForQueuedToolCall(request),
18122
- apiOptions: {
18123
- beforeProviderCall: () =>
18124
- pendingReceiptHeartbeat
18125
- ? pendingReceiptHeartbeat.assertOwned()
18126
- : this.assertRuntimeToolReceiptOwnership([
18127
- request,
18128
- ]),
18129
- playNodeScope:
18130
- playNodeScopeForToolCallRequest(request),
18131
- ...(request.receiptKey
18132
- ? {
18133
- durableCallReceiptKey: request.receiptKey,
18134
- completedResultCache: true,
18135
- executionAuthScopeDigest:
18136
- request.executionAuthScopeDigest,
18137
- providerIdempotencyKey:
18138
- this.providerIdempotencyKeyForToolCall({
18139
- operationKey:
18140
- this.providerOperationKeyForQueuedToolCall(
18141
- request,
18142
- ),
18143
- force: request.force === true,
18144
- logicalCallId: request.logicalCallId,
18145
- }),
18146
- ...this.providerForceRefreshOptionsForToolCall({
18147
- operationKey:
18148
- this.providerOperationKeyForQueuedToolCall(
18149
- request,
18150
- ),
18151
- force: request.force,
18152
- requiresExecutionFence:
18153
- request.requiresExecutionFence,
18154
- logicalCallId: request.logicalCallId,
16650
+ const dispatchWidth = Math.min(
16651
+ toolCallConcurrencyCeiling,
16652
+ Math.max(1, shapedToolParallelism),
16653
+ );
16654
+ const dispatchResults = await dispatchBoundedSettled(
16655
+ pendingRequests,
16656
+ dispatchWidth,
16657
+ async (request) => {
16658
+ // Circuit breaker: do not dispatch this provider call once a
16659
+ // persistence failure has occurred in this run.
16660
+ if (this.persistenceLatch.tripped) {
16661
+ this.persistenceLatch.preventedCallCount += 1;
16662
+ throw new RuntimePersistenceCircuitOpenError(
16663
+ this.persistenceLatch,
16664
+ );
16665
+ }
16666
+ const providerOperationKey =
16667
+ this.providerInvocationKeyForToolCall({
16668
+ operationKey:
16669
+ this.providerOperationKeyForQueuedToolCall(request),
16670
+ logicalCallId: request.logicalCallId,
16671
+ force: request.force,
16672
+ });
16673
+ let releaseToolSlot: () => void = () => undefined;
16674
+ try {
16675
+ const execution =
16676
+ await this.executeMappedToolCallThroughModule({
16677
+ request,
16678
+ dispatchToolId: toolId,
16679
+ dispatchPayload: request.input,
16680
+ // Always-fresh map calls intentionally have no
16681
+ // result cache. Their provider-operation identity is
16682
+ // a unique forced semantic request, never a cache key.
16683
+ resultReceiptKey: request.receiptKey ?? request.cacheKey,
16684
+ providerOperationKey,
16685
+ apiOptions: {
16686
+ playNodeScope: playNodeScopeForToolCallRequest(request),
16687
+ executionAuthScopeDigest:
16688
+ request.executionAuthScopeDigest,
16689
+ providerIdempotencyKey: providerOperationKey,
16690
+ ...this.providerInvocationOptionsForToolCall({
16691
+ operationKey:
16692
+ this.providerOperationKeyForQueuedToolCall(request),
16693
+ requiresExecutionFence:
16694
+ request.requiresExecutionFence,
16695
+ logicalCallId: request.logicalCallId,
16696
+ force: request.force,
16697
+ }),
16698
+ ...(request.receiptKey
16699
+ ? {
16700
+ durableCallReceiptKey:
16701
+ this.toolTransportReceiptKey({
16702
+ toolId,
16703
+ providerOperationKey,
18155
16704
  }),
18156
- receiptLeaseExpiresAt:
18157
- request.receiptLeaseExpiresAt,
18158
- heartbeatReceipt: () =>
18159
- this.renewRuntimeToolReceiptOwnership([
18160
- request,
18161
- ]),
18162
- }
18163
- : {
18164
- providerIdempotencyKey:
18165
- this.providerIdempotencyKeyForToolCall({
18166
- operationKey:
18167
- this.providerOperationKeyForQueuedToolCall(
16705
+ legacyProviderRetryIdentities: [
16706
+ {
16707
+ transportReceiptKey: request.receiptKey,
16708
+ providerOperationKey: request.force
16709
+ ? providerOperationKey
16710
+ : this.providerOperationKeyForQueuedToolCall(
18168
16711
  request,
18169
16712
  ),
18170
- force: true,
18171
- logicalCallId: request.logicalCallId,
18172
- }),
18173
- }),
18174
- timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
18175
- request,
18176
- ]),
18177
- retainToolSlot: (release) => {
18178
- releaseToolSlot = release;
18179
- },
16713
+ ...(request.force &&
16714
+ request.requiresExecutionFence !== true
16715
+ ? {
16716
+ forceRefreshIdentities: [
16717
+ providerOperationKey,
16718
+ ],
16719
+ }
16720
+ : {}),
16721
+ },
16722
+ ],
16723
+ completedResultCache: true,
16724
+ }
16725
+ : {}),
16726
+ timeoutMs: resolveRuntimeTimeoutMsForRequests([
16727
+ request,
16728
+ ]),
16729
+ retainToolSlot: (release) => {
16730
+ releaseToolSlot = release;
18180
16731
  },
18181
- });
18182
- const result = await this.resolveToolCall(
18183
- toolId,
18184
- request,
18185
- execution,
18186
- );
18187
- if (result != null) {
18188
- successfulLiveStepCallIds.add(request.callId);
18189
- }
18190
- resolveLiveFollowers(request, result);
18191
- } finally {
18192
- releaseToolSlot();
16732
+ },
16733
+ });
16734
+ const result = await this.resolveToolCall(
16735
+ toolId,
16736
+ request,
16737
+ execution,
16738
+ );
16739
+ if (result != null) {
16740
+ successfulLiveStepCallIds.add(request.callId);
18193
16741
  }
18194
- },
18195
- );
18196
- const failedEntries = dispatchResults.filter(
18197
- (entry) => entry.error !== undefined,
18198
- );
18199
- const failedSettlements = await Promise.allSettled(
18200
- failedEntries.map((entry) =>
18201
- rejectWithLiveFollowers(entry.job, entry.error),
18202
- ),
18203
- );
18204
- const failedRejection = failedSettlements.find(
18205
- (settlement): settlement is PromiseRejectedResult =>
18206
- settlement.status === 'rejected',
18207
- );
18208
- if (failedRejection) throw failedRejection.reason;
18209
- recordToolStep(pendingRequests);
18210
- this.#options.onBatchComplete?.(this.checkpoint);
18211
- }
18212
- }
18213
- } catch (reason) {
18214
- providerFailure = { reason };
18215
- durableExistingRunningAbort.abort(reason);
18216
- } finally {
18217
- pendingReceiptHeartbeat?.stop();
18218
- }
18219
- const pendingReceiptLeaseLost = pendingReceiptHeartbeat?.leaseLost();
18220
- if (pendingReceiptLeaseLost && providerFailure === null) {
18221
- providerFailure = { reason: pendingReceiptLeaseLost };
18222
- durableExistingRunningAbort.abort(pendingReceiptLeaseLost);
18223
- }
18224
- const durableExistingRunningSettlement =
18225
- durableExistingRunningState.settlement;
18226
- if (durableExistingRunningSettlement) {
18227
- const settlement = await durableExistingRunningSettlement;
18228
- if (settlement.status === 'rejected' && providerFailure === null) {
18229
- providerFailure = { reason: settlement.reason };
16742
+ } finally {
16743
+ releaseToolSlot();
16744
+ }
16745
+ },
16746
+ );
16747
+ const failedEntries = dispatchResults.filter(
16748
+ (entry) => entry.error !== undefined,
16749
+ );
16750
+ const failedSettlements = await Promise.allSettled(
16751
+ failedEntries.map((entry) => rejectCall(entry.job, entry.error)),
16752
+ );
16753
+ const failedRejection = failedSettlements.find(
16754
+ (settlement): settlement is PromiseRejectedResult =>
16755
+ settlement.status === 'rejected',
16756
+ );
16757
+ if (failedRejection) throw failedRejection.reason;
16758
+ recordToolStep(pendingRequests);
16759
+ this.#options.onBatchComplete?.(this.checkpoint);
18230
16760
  }
18231
16761
  }
18232
- if (providerFailure) throw providerFailure.reason;
18233
16762
  }),
18234
16763
  );
18235
16764
  const rejected = toolSettlements.find(
@@ -18323,24 +16852,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
18323
16852
  )(ctx, input);
18324
16853
  }
18325
16854
 
18326
- private providerIdempotencyKeyForToolCall(input: {
16855
+ private providerInvocationKeyForToolCall(input: {
18327
16856
  operationKey: string;
18328
- force?: boolean;
18329
16857
  logicalCallId?: string;
16858
+ force?: boolean;
18330
16859
  }): string {
18331
- let fallbackAttemptId: string | null = null;
18332
- if (input.force === true) {
18333
- if (!input.logicalCallId) {
18334
- throw new Error(
18335
- 'Forced tool execution requires a stable logical call identity.',
18336
- );
18337
- }
18338
- fallbackAttemptId = `${this.currentRunId}:${input.logicalCallId}`;
16860
+ if (!input.logicalCallId) {
16861
+ throw new Error('Tool execution requires a stable invocation identity.');
18339
16862
  }
18340
16863
  return buildProviderOperationIdempotencyKey({
18341
16864
  operationKey: input.operationKey,
18342
- force: input.force,
18343
- fallbackAttemptId,
16865
+ force: true,
16866
+ fallbackAttemptId: `${this.currentRunId}:${input.logicalCallId}${input.force === true ? '' : ':cache-miss'}`,
18344
16867
  });
18345
16868
  }
18346
16869
 
@@ -18986,7 +17509,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
18986
17509
  const ambiguousTransportReplayAllowed =
18987
17510
  readOnlyQuery || retrySafeTransientHttp;
18988
17511
  const invocationOrgId = this.#options.orgId?.trim() ?? '';
18989
- const durableCallReceiptKey = uncachedAsyncLifecycleCall
17512
+ let durableCallReceiptKey = uncachedAsyncLifecycleCall
18990
17513
  ? null
18991
17514
  : options?.durableCallReceiptKey?.trim() ||
18992
17515
  (usesDurableInvocationFence && invocationOrgId
@@ -19004,9 +17527,57 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19004
17527
  this.#options.durableBoundaries && durableCallReceiptKey
19005
17528
  ? stableDigest(`provider-retry:${durableCallReceiptKey}`)
19006
17529
  : null;
19007
- const savedProviderRetryCursor = providerRetryCursorKey
17530
+ let savedProviderRetryCursor = providerRetryCursorKey
19008
17531
  ? this.checkpoint.providerRetryCursors?.[providerRetryCursorKey]
19009
17532
  : undefined;
17533
+ if (providerRetryCursorKey && !savedProviderRetryCursor) {
17534
+ // Native calls can move from the outgoing fallback into the Module.
17535
+ // Probe only exact owner-supplied checkpoint identities; never wait
17536
+ // on another invocation or derive a fence from customer strings.
17537
+ for (const legacy of options?.legacyProviderRetryIdentities ?? []) {
17538
+ const legacyCursorKey = stableDigest(
17539
+ `provider-retry:${legacy.transportReceiptKey}`,
17540
+ );
17541
+ const cursor =
17542
+ this.checkpoint.providerRetryCursors?.[legacyCursorKey];
17543
+ if (!cursor) continue;
17544
+ // Consume synchronously before any await so a sibling cannot
17545
+ // inherit the same outgoing invocation.
17546
+ delete this.checkpoint.providerRetryCursors![legacyCursorKey];
17547
+ savedProviderRetryCursor = {
17548
+ ...cursor,
17549
+ legacyProviderOperationKey:
17550
+ cursor.legacyProviderOperationKey ??
17551
+ legacy.providerOperationKey,
17552
+ legacyTransportReceiptKey:
17553
+ cursor.legacyTransportReceiptKey ??
17554
+ legacy.transportReceiptKey,
17555
+ ...(legacy.forceRefreshIdentities?.length
17556
+ ? {
17557
+ legacyProviderOperationForceRefreshIdentities:
17558
+ cursor.legacyProviderOperationForceRefreshIdentities ??
17559
+ legacy.forceRefreshIdentities,
17560
+ }
17561
+ : {}),
17562
+ };
17563
+ this.checkpoint.providerRetryCursors![providerRetryCursorKey] =
17564
+ savedProviderRetryCursor;
17565
+ break;
17566
+ }
17567
+ }
17568
+ // The checkpoint index stays unique to this invocation. Its wire
17569
+ // identities must still name the prior gateway outcome so that a
17570
+ // qualified 429 retry can be attested after any further suspension.
17571
+ const legacyProviderOperationKey =
17572
+ savedProviderRetryCursor?.legacyProviderOperationKey;
17573
+ if (savedProviderRetryCursor?.legacyTransportReceiptKey) {
17574
+ durableCallReceiptKey =
17575
+ savedProviderRetryCursor.legacyTransportReceiptKey;
17576
+ }
17577
+ const providerOperationForceRefreshIdentities =
17578
+ legacyProviderOperationKey
17579
+ ? savedProviderRetryCursor?.legacyProviderOperationForceRefreshIdentities
17580
+ : options?.providerOperationForceRefreshIdentities;
19010
17581
  const executionRequestNonce = crypto.randomUUID();
19011
17582
  // Transport ambiguity must retain the billing identity. An ordinary
19012
17583
  // (unfenced) call advances it only after a definite provider-owned
@@ -19020,7 +17591,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19020
17591
  const providerIdempotencyReceiptKey =
19021
17592
  options?.providerIdempotencyReceiptKey?.trim() || null;
19022
17593
  const providerExecutionFenceKey = requiresExecutionFence
19023
- ? options?.providerIdempotencyKey?.trim() || null
17594
+ ? legacyProviderOperationKey ||
17595
+ options?.providerIdempotencyKey?.trim() ||
17596
+ null
19024
17597
  : null;
19025
17598
  if (requiresExecutionFence && !providerExecutionFenceKey) {
19026
17599
  throw new Error(
@@ -19029,14 +17602,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19029
17602
  }
19030
17603
  const canonicalOperationActionVersion =
19031
17604
  providerExecutionFenceKey ||
19032
- options?.providerOperationForceRefreshIdentities?.length
17605
+ providerOperationForceRefreshIdentities?.length
19033
17606
  ? (
19034
17607
  await this.#options.getToolActionCacheVersion?.(toolId)
19035
17608
  )?.trim() || null
19036
17609
  : null;
19037
17610
  const canonicalOperation =
17611
+ !uncachedAsyncLifecycleCall &&
19038
17612
  (providerExecutionFenceKey ||
19039
- options?.providerOperationForceRefreshIdentities?.length) &&
17613
+ providerOperationForceRefreshIdentities?.length) &&
19040
17614
  canonicalOperationActionVersion
19041
17615
  ? {
19042
17616
  version: 1,
@@ -19046,10 +17620,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19046
17620
  output_intent: options?.completedResultCache
19047
17621
  ? 'dataset'
19048
17622
  : 'value',
19049
- ...(options?.providerOperationForceRefreshIdentities?.length
17623
+ ...(providerOperationForceRefreshIdentities?.length
19050
17624
  ? {
19051
17625
  force_refresh_identities:
19052
- options.providerOperationForceRefreshIdentities,
17626
+ providerOperationForceRefreshIdentities,
19053
17627
  }
19054
17628
  : {}),
19055
17629
  }
@@ -19183,9 +17757,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19183
17757
  };
19184
17758
  const persistProviderRetryCursor = (retryFrom: number) => {
19185
17759
  if (!providerRetryCursorKey) return;
19186
- const cursor = {
17760
+ const cursor: ProviderRetryCursor = {
17761
+ ...savedProviderRetryCursor,
19187
17762
  invocationAttempt: retryFrom + 1,
19188
17763
  retryFromInvocationAttempt: retryFrom,
17764
+ ...(legacyProviderOperationKey
17765
+ ? {
17766
+ legacyProviderOperationKey,
17767
+ legacyTransportReceiptKey: durableCallReceiptKey!,
17768
+ }
17769
+ : {}),
19189
17770
  };
19190
17771
  this.checkpoint.providerRetryCursors ??= {};
19191
17772
  this.checkpoint.providerRetryCursors[providerRetryCursorKey] =