deepline 0.3.284 → 0.3.286

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +1 -2
  2. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  3. package/dist/bundling-sources/shared_libs/play-runtime/activity-observation.ts +26 -2
  4. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +7 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +836 -2234
  6. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +19 -6
  7. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +127 -789
  8. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/job.ts +18 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +16 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +180 -14
  11. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +1 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +3 -22
  13. package/dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts +26 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +59 -8
  15. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +122 -18
  16. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +458 -130
  17. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +60 -9
  18. package/dist/bundling-sources/shared_libs/play-runtime/test-async-batching.ts +4 -1
  19. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +13 -2
  20. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +181 -326
  21. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +38 -151
  22. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +68 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +8 -0
  24. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +10 -1
  25. package/dist/bundling-sources/shared_libs/plays/enrich-compat-adapter.ts +2 -0
  26. package/dist/cli/index.js +30579 -34364
  27. package/dist/cli/index.mjs +30647 -34443
  28. package/dist/{client-BCXQ_Ftc.d.ts → client-B7ic-kY5.d.ts} +6 -3
  29. package/dist/{client-B0AjIkgb.d.mts → client-Dzf_KovW.d.mts} +6 -3
  30. package/dist/{compiler-manifest-B1zo3igW.d.mts → compiler-manifest-9SQtUuKb.d.mts} +1 -1
  31. package/dist/{compiler-manifest-B1zo3igW.d.ts → compiler-manifest-9SQtUuKb.d.ts} +1 -1
  32. package/dist/index.d.mts +4 -4
  33. package/dist/index.d.ts +4 -4
  34. package/dist/index.js +34 -4
  35. package/dist/index.mjs +34 -4
  36. package/dist/install-integrity.json +4 -4
  37. package/dist/plays/bundle-play-file.d.mts +2 -2
  38. package/dist/plays/bundle-play-file.d.ts +2 -2
  39. package/dist/plays/bundle-play-file.mjs +8 -1
  40. package/dist/plays/run-request.d.mts +2 -2
  41. package/dist/plays/run-request.d.ts +2 -2
  42. package/dist/release.d.mts +1 -1
  43. package/dist/release.d.ts +1 -1
  44. package/dist/release.js +1 -1
  45. package/dist/release.mjs +1 -1
  46. package/package.json +1 -1
@@ -160,6 +160,7 @@ import {
160
160
  attachRelayPublishedToolResultReference,
161
161
  attachToolResultListDataset,
162
162
  createToolExecuteResult,
163
+ projectNativeBatchToolResponse,
163
164
  parseToolExecuteResponse,
164
165
  isToolExecuteResult,
165
166
  isSerializedToolExecuteResult,
@@ -187,7 +188,6 @@ import {
187
188
  createRuntimeStepToolResultReceipts,
188
189
  createToolCallJob,
189
190
  resolveToolCallResultReceiptPolicy,
190
- ToolCallReceiptLeaseLostError,
191
191
  type ToolCallReceiptCohortStore,
192
192
  type ToolCallReceiptCohortTimings,
193
193
  type ToolCallReceiptRecord,
@@ -262,26 +262,16 @@ import {
262
262
  } from './log-provenance';
263
263
  import {
264
264
  COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
265
- DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS,
266
265
  createRuntimeReceiptHydrationGate,
267
266
  runtimeStepReceiptPayloadBytes,
268
- runtimeReceiptClaimLeaseIsLive,
269
- runtimeReceiptWaitBudgetMs,
270
- runtimeReceiptWaitDelayMs,
271
- DURABLE_RECEIPT_WAIT_DELAY_MS,
272
- RuntimeReceiptHydrationTurnTimeoutError,
273
267
  RuntimeReceiptLeaseLostError,
274
- RuntimeReceiptWaitTimeoutError,
275
268
  executeWithDurableRuntimeReceipt,
276
269
  resolveRuntimeToolReceiptWaitMaxAttempts,
277
- runtimeReceiptFailureError,
278
270
  runtimeReceiptFailureKindForError,
279
271
  runtimeReceiptOutput as durableRuntimeReceiptOutput,
280
272
  type RuntimeReceiptHydrationTurn,
281
273
  type RuntimeReceiptHydrationGateSnapshot,
282
274
  type RuntimeReceiptHydrationSource,
283
- waitForCompletedRuntimeReceipt,
284
- waitForCompletedRuntimeReceipts,
285
275
  type DurableReceiptExecutionStore,
286
276
  } from './durable-receipt-execution';
287
277
  import {
@@ -422,6 +412,7 @@ import type {
422
412
  ContextOptions,
423
413
  PlayCallOptions,
424
414
  PlayCheckpoint,
415
+ ProviderRetryCursor,
425
416
  PlayStep,
426
417
  PlayStepRowResult,
427
418
  PlayRowUpdate,
@@ -609,12 +600,6 @@ const DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN =
609
600
  /\bdeeplinedeveloper\.com\s*\|\s*502\s*:\s*bad gateway\b/i;
610
601
  const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100;
611
602
  const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25;
612
- // A newly claimed receipt should normally use its scheduled heartbeat. When a
613
- // test/SEV policy deliberately gives it only a moment of lease life, however,
614
- // scheduler setup and the first provider-dispatch turn can consume that entire
615
- // window before the timer fires. Renew once before fan-out in that narrow case.
616
- // Production's normal multi-minute lease remains on the zero-extra-query path.
617
- const IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS = 100;
618
603
  // Per-row sanity cap on distinct inline child invocations. `maxPlayCallDepth`
619
604
  // (governor) already bounds nesting; this bounds fan-WIDTH from one active row
620
605
  // resolver. The dedupe set is row-local, never retained for the full run.
@@ -1465,6 +1450,18 @@ function publicToolResponseEnvelope(value: unknown): {
1465
1450
  };
1466
1451
  }
1467
1452
 
1453
+ function nativeBatchResponseMeta(result: unknown): Record<string, unknown> {
1454
+ return {
1455
+ nativeBatchResponseView:
1456
+ result &&
1457
+ typeof result === 'object' &&
1458
+ !Array.isArray(result) &&
1459
+ Object.prototype.hasOwnProperty.call(result, 'data')
1460
+ ? 'data'
1461
+ : 'rawV2',
1462
+ };
1463
+ }
1464
+
1468
1465
  /**
1469
1466
  * A batched provider request returns one envelope for several logical source
1470
1467
  * tool calls. Each source call persists only its own item-shaped canonical
@@ -1826,7 +1823,7 @@ type ToolExecutionApiOptions = {
1826
1823
  completedResultCache?: boolean;
1827
1824
  executionAuthScopeDigest?: string | null;
1828
1825
  providerIdempotencyKey?: string | null;
1829
- /** Exact forced-member identities that participate in a native batch key. */
1826
+ /** Invocation identities that participate in the canonical operation key. */
1830
1827
  providerOperationForceRefreshIdentities?: readonly string[];
1831
1828
  /** Presentation-only cache identity paired with a separate execution fence. */
1832
1829
  providerIdempotencyReceiptKey?: string | null;
@@ -1878,6 +1875,12 @@ type ToolExecutionApiOptions = {
1878
1875
  * cache stores only its terminal materialized output.
1879
1876
  */
1880
1877
  uncachedAsyncLifecycleCall?: boolean;
1878
+ /** Exact outgoing delivery candidates; consume a checkpoint cursor once. */
1879
+ legacyProviderRetryIdentities?: readonly {
1880
+ transportReceiptKey: string;
1881
+ providerOperationKey: string;
1882
+ forceRefreshIdentities?: readonly string[];
1883
+ }[];
1881
1884
  /**
1882
1885
  * Called once this call's provider response body has been fully read and
1883
1886
  * parsed, with that instant on the runner's clock. Measurement only: the
@@ -2594,12 +2597,6 @@ type ScalarPlayAuthoringRuntimeContext = Pick<
2594
2597
  export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2595
2598
  private rowStates = new Map<number, RowState>();
2596
2599
  private toolCallQueue: ToolCallRequest[] = [];
2597
- private pendingRuntimeToolOwnershipAssertions: Array<{
2598
- targets: Array<{ receiptKey: string; leaseId: string }>;
2599
- resolve: () => void;
2600
- reject: (error: unknown) => void;
2601
- }> = [];
2602
- private runtimeToolOwnershipAssertionFlushScheduled = false;
2603
2600
  private runtimeToolReceiptCompletionsInFlight = new Map<string, number>();
2604
2601
  /**
2605
2602
  * The one app-side result-egress controller. It serializes receipt gateway
@@ -3225,23 +3222,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
3225
3222
  : Date.now();
3226
3223
  this.checkpoint.durableCallCacheEpochMs = this.durableCallCacheEpochMs;
3227
3224
  this.durableMappedToolResultsBackedByReceipts = Boolean(
3228
- (options.claimRuntimeStepReceipt || options.claimRuntimeStepReceipts) &&
3229
3225
  (options.getRuntimeStepReceipt || options.getRuntimeStepReceipts) &&
3230
3226
  (options.completeRuntimeStepReceipt ||
3231
3227
  options.completeRuntimeStepReceipts),
3232
3228
  );
3233
- this.durableDirectToolResultsBackedByReceipts = Boolean(
3234
- options.claimRuntimeStepReceipt &&
3235
- options.getRuntimeStepReceipt &&
3236
- options.completeRuntimeStepReceipt,
3237
- );
3238
- if (this.durableDirectToolResultsBackedByReceipts) {
3239
- // A resumed durable runner may receive a legacy checkpoint containing
3240
- // full tool results. Drop that redundant copy immediately and recover
3241
- // through the receipt store below. Bulk-only receipt contexts retain the
3242
- // checkpoint because direct tool calls cannot use the bulk claim path.
3243
- this.checkpoint.completedToolBatches = {};
3244
- }
3229
+ this.durableDirectToolResultsBackedByReceipts =
3230
+ this.durableMappedToolResultsBackedByReceipts;
3231
+ // Existing checkpoints can be the sole completed cache from an outgoing
3232
+ // context. Receipt-backed calls stop adding payloads, but retain old hits.
3245
3233
  // The governance play id keys durable ctx receipts (durableCtxKey), the
3246
3234
  // cycle guard, and per-parent child-call counters. It must be the STABLE
3247
3235
  // play name — not the per-run workflow id — so receipts written by one
@@ -4323,7 +4311,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4323
4311
  key: string,
4324
4312
  ): Promise<RuntimeStepReceipt | null> {
4325
4313
  if (!this.#options.getRuntimeStepReceipt) {
4326
- return null;
4314
+ if (!this.#options.getRuntimeStepReceipts) return null;
4315
+ return (await this.getRuntimeStepReceipts([key])).get(key) ?? null;
4327
4316
  }
4328
4317
  const receipt = await this.#options.getRuntimeStepReceipt({ key });
4329
4318
  if (receipt && typeof receipt.key === 'string' && receipt.key.trim()) {
@@ -4465,7 +4454,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4465
4454
  path: `ctx.step receipt "${key}"`,
4466
4455
  });
4467
4456
  if (!this.#options.completeRuntimeStepReceipt) {
4468
- return null;
4457
+ if (!this.#options.completeRuntimeStepReceipts) return null;
4458
+ const completed = await this.completeRuntimeStepReceipts(
4459
+ [
4460
+ {
4461
+ key,
4462
+ runId: this.currentReceiptOwnerRunId,
4463
+ runAttempt: this.currentRunAttempt,
4464
+ ...(leaseId ? { leaseId } : {}),
4465
+ output,
4466
+ ...(inlineFields ? { inlineFields } : {}),
4467
+ },
4468
+ ],
4469
+ undefined,
4470
+ undefined,
4471
+ { retainLiveResult: true },
4472
+ );
4473
+ return completed.get(key) ?? null;
4469
4474
  }
4470
4475
  const completed = await this.#options.completeRuntimeStepReceipt({
4471
4476
  key,
@@ -4527,49 +4532,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4527
4532
  })),
4528
4533
  );
4529
4534
  if (manifests.length === 0) return;
4530
- // A scalar call publishes its pointer on its own, after the canonical
4531
- // result has already committed, so it never rides the cohort's shared
4532
- // egress wave. Nothing is gained by skipping the ownership handshake
4533
- // here, and skipping it costs the caller a receipt row it does not own:
4534
- // a runtime whose claim seam refuses (or that exposes no claim seam at
4535
- // all) would still publish pointer rows, and an already published
4536
- // pointer would be rewritten on every later call. Take the row the way
4537
- // every other scalar receipt writer takes it, and write only what this
4538
- // run actually owns. The map/cohort publisher keeps the claim-free
4539
- // batched write, where the pointer genuinely shares the canonical
4540
- // result's single bounded request.
4535
+ // The pointer is an immutable completed-cache insert, like the result
4536
+ // it names. Publishing it needs no pending lease or read-back.
4541
4537
  if (
4542
4538
  !(
4543
- this.#options.claimRuntimeStepReceipt ||
4544
- this.#options.claimRuntimeStepReceipts
4539
+ this.#options.completeRuntimeStepReceipt ||
4540
+ this.#options.completeRuntimeStepReceipts
4545
4541
  )
4546
4542
  ) {
4547
4543
  return;
4548
4544
  }
4549
- const existing = await this.getRuntimeStepReceipts(
4550
- manifests.map((manifest) => manifest.key),
4551
- { hydrateOutput: false },
4552
- );
4553
- const missing = manifests.filter((manifest) => {
4554
- const receipt = existing.get(manifest.key);
4555
- return receipt?.status !== 'completed' && receipt?.status !== 'skipped';
4556
- });
4557
- if (missing.length === 0) return;
4558
- const claims = await this.claimRuntimeStepReceipts(
4559
- missing.map((manifest) => manifest.key),
4560
- sourceRunId,
4561
- false,
4562
- false,
4563
- false,
4564
- false,
4565
- );
4566
- const owned = missing.flatMap((manifest) => {
4567
- const leaseId = claims.get(manifest.key)?.leaseId;
4568
- return leaseId ? [{ ...manifest, leaseId }] : [];
4569
- });
4570
- if (owned.length === 0) return;
4571
- await this.completeRuntimeStepReceipts(owned, undefined, undefined, {
4572
- bestEffortKeys: new Set(owned.map((manifest) => manifest.key)),
4545
+ await this.completeRuntimeStepReceipts(manifests, undefined, undefined, {
4546
+ bestEffortKeys: new Set(manifests.map((manifest) => manifest.key)),
4573
4547
  onBestEffortFailure: ({ key, error }) => {
4574
4548
  this.runtimeLog(
4575
4549
  `Tool source-run receipt manifest write skipped for ${key}: ${error instanceof Error ? error.message : String(error)}`,
@@ -4789,178 +4763,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4789
4763
  };
4790
4764
  }
4791
4765
 
4792
- private async assertRuntimeToolReceiptOwnership(
4793
- requests: ToolCallRequest[],
4794
- ): Promise<void> {
4795
- const targets = this.runtimeToolReceiptOwnershipTargets(requests);
4796
- if (targets.length === 0) return;
4797
-
4798
- return await new Promise<void>((resolve, reject) => {
4799
- this.pendingRuntimeToolOwnershipAssertions.push({
4800
- targets,
4801
- resolve,
4802
- reject,
4803
- });
4804
- if (this.runtimeToolOwnershipAssertionFlushScheduled) return;
4805
- this.runtimeToolOwnershipAssertionFlushScheduled = true;
4806
- queueMicrotask(() => void this.flushRuntimeToolOwnershipAssertionBatch());
4807
- });
4808
- }
4809
-
4810
- private runtimeToolReceiptOwnershipTargets(
4811
- requests: ToolCallRequest[],
4812
- ): Array<{ receiptKey: string; leaseId: string }> {
4813
- return requests.flatMap((request) => {
4814
- const receiptKey = request.receiptKey?.trim() || null;
4815
- if (!receiptKey) return [];
4816
- const leaseId = request.receiptLeaseId?.trim() || null;
4817
- if (!leaseId) return [];
4818
- return [{ receiptKey, leaseId }];
4819
- });
4820
- }
4821
-
4822
- private async renewRuntimeToolReceiptOwnership(
4823
- requests: ToolCallRequest[],
4824
- ): Promise<void> {
4825
- const targets = this.runtimeToolReceiptOwnershipTargets(requests);
4826
- if (targets.length === 0) return;
4827
- if (!this.#options.heartbeatRuntimeStepReceipts) {
4828
- return await this.assertRuntimeToolReceiptOwnership(requests);
4829
- }
4830
- const uniqueTargets = [
4831
- ...new Map(targets.map((target) => [target.receiptKey, target])).values(),
4832
- ];
4833
- const receipts = await this.#options.heartbeatRuntimeStepReceipts({
4834
- runId: this.currentReceiptOwnerRunId,
4835
- runAttempt: this.currentRunAttempt,
4836
- keys: uniqueTargets.map((target) => target.receiptKey),
4837
- leaseIds: uniqueTargets.map((target) => target.leaseId),
4838
- });
4839
- this.assertRuntimeToolReceiptHeartbeatResults(uniqueTargets, receipts);
4840
- }
4841
-
4842
- private async flushRuntimeToolOwnershipAssertionBatch(): Promise<void> {
4843
- this.runtimeToolOwnershipAssertionFlushScheduled = false;
4844
- const assertions = this.pendingRuntimeToolOwnershipAssertions.splice(0);
4845
- if (assertions.length === 0) return;
4846
- const targets = assertions.flatMap((assertion) => assertion.targets);
4847
-
4848
- if (this.#options.heartbeatRuntimeStepReceipts) {
4849
- try {
4850
- const uniqueTargets = [
4851
- ...new Map(
4852
- targets.map((target) => [target.receiptKey, target]),
4853
- ).values(),
4854
- ];
4855
- const receipts = await this.#options.heartbeatRuntimeStepReceipts({
4856
- runId: this.currentReceiptOwnerRunId,
4857
- runAttempt: this.currentRunAttempt,
4858
- keys: uniqueTargets.map((target) => target.receiptKey),
4859
- leaseIds: uniqueTargets.map((target) => target.leaseId),
4860
- });
4861
- const renewed = new Map<string, RuntimeStepReceipt | null>();
4862
- for (let index = 0; index < uniqueTargets.length; index += 1) {
4863
- renewed.set(
4864
- uniqueTargets[index]!.receiptKey,
4865
- receipts[index] ?? null,
4866
- );
4867
- }
4868
- for (const assertion of assertions) {
4869
- const lost = assertion.targets.find((target) => {
4870
- const receipt = renewed.get(target.receiptKey);
4871
- return !this.runtimeToolReceiptStillOwned(receipt, target.leaseId);
4872
- });
4873
- if (lost) {
4874
- assertion.reject(
4875
- new RuntimeReceiptLeaseLostError({
4876
- receiptKey: lost.receiptKey,
4877
- runId: this.currentReceiptOwnerRunId,
4878
- leaseId: lost.leaseId,
4879
- }),
4880
- );
4881
- } else {
4882
- assertion.resolve();
4883
- }
4884
- }
4885
- } catch (error) {
4886
- for (const assertion of assertions) assertion.reject(error);
4887
- }
4888
- return;
4889
- }
4890
-
4891
- if (
4892
- this.#options.getRuntimeStepReceipt ||
4893
- this.#options.getRuntimeStepReceipts
4894
- ) {
4895
- try {
4896
- const latest = await this.getRuntimeStepReceipts(
4897
- targets.map((target) => target.receiptKey),
4898
- );
4899
- for (const assertion of assertions) {
4900
- const lost = assertion.targets.find((target) => {
4901
- const receipt = latest.get(target.receiptKey);
4902
- return !this.runtimeToolReceiptStillOwned(receipt, target.leaseId);
4903
- });
4904
- if (lost) {
4905
- assertion.reject(
4906
- new RuntimeReceiptLeaseLostError({
4907
- receiptKey: lost.receiptKey,
4908
- runId: this.currentReceiptOwnerRunId,
4909
- leaseId: lost.leaseId,
4910
- }),
4911
- );
4912
- } else {
4913
- assertion.resolve();
4914
- }
4915
- }
4916
- } catch (error) {
4917
- for (const assertion of assertions) assertion.reject(error);
4918
- }
4919
- return;
4920
- }
4921
-
4922
- const error = new RuntimeReceiptLeaseLostError({
4923
- receiptKey: targets[0]?.receiptKey ?? 'unknown',
4924
- runId: this.currentReceiptOwnerRunId,
4925
- leaseId: targets[0]?.leaseId ?? 'unknown',
4926
- });
4927
- for (const assertion of assertions) assertion.reject(error);
4928
- }
4929
-
4930
- private assertRuntimeToolReceiptHeartbeatResults(
4931
- targets: Array<{ receiptKey: string; leaseId: string }>,
4932
- receipts: Array<RuntimeStepReceipt | null>,
4933
- ): void {
4934
- for (let index = 0; index < targets.length; index += 1) {
4935
- const target = targets[index]!;
4936
- const receipt = this.normalizeRuntimeStepReceipt(
4937
- target.receiptKey,
4938
- receipts[index],
4939
- );
4940
- if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) {
4941
- throw new RuntimeReceiptLeaseLostError({
4942
- receiptKey: target.receiptKey,
4943
- runId: this.currentReceiptOwnerRunId,
4944
- leaseId: target.leaseId,
4945
- });
4946
- }
4947
- }
4948
- }
4949
-
4950
- private runtimeToolReceiptStillOwned(
4951
- receipt: RuntimeStepReceipt | null | undefined,
4952
- leaseId: string,
4953
- ): boolean {
4954
- if (!receipt) return false;
4955
- if (receipt.status === 'completed' || receipt.status === 'skipped') {
4956
- return true;
4957
- }
4958
- if (receipt.status !== 'running') return false;
4959
- if (receipt.leaseId !== leaseId) return false;
4960
- const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
4961
- return !ownerRunId || ownerRunId === this.currentReceiptOwnerRunId;
4962
- }
4963
-
4964
4766
  private runtimeToolReceiptStillOwnedByCurrentAttempt(
4965
4767
  receipt: RuntimeStepReceipt | null | undefined,
4966
4768
  leaseId: string | null | undefined,
@@ -4980,18 +4782,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4980
4782
  return ownerAttempt === this.currentRunAttempt;
4981
4783
  }
4982
4784
 
4983
- private isOwnedClaimedRuntimeReceipt(
4984
- receipt: RuntimeStepReceipt | null | undefined,
4985
- ): receipt is RuntimeStepReceipt {
4986
- return (
4987
- Boolean(receipt) &&
4988
- receipt?.claimState !== 'existing' &&
4989
- (receipt?.status === 'queued' ||
4990
- receipt?.status === 'pending' ||
4991
- receipt?.status === 'running')
4992
- );
4993
- }
4994
-
4995
4785
  private async failRuntimeStepReceipt(
4996
4786
  key: string,
4997
4787
  _runId: string,
@@ -5359,91 +5149,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
5359
5149
  return this.normalizeRuntimeStepReceipt(input.key, receipt);
5360
5150
  }
5361
5151
 
5362
- private async markRuntimeStepReceiptsRunning(
5363
- receipts: Array<{
5364
- key: string;
5365
- runId: string;
5366
- leaseId?: string | null;
5367
- }>,
5368
- ): Promise<Map<string, RuntimeStepReceipt>> {
5369
- const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
5370
- if (normalizedInputs.length === 0) return new Map();
5371
- let marked: Array<RuntimeStepReceipt | null>;
5372
- const markReceipts = this.#options.markRuntimeStepReceiptsRunning;
5373
- if (markReceipts) {
5374
- marked = await this.dispatchChunkedRuntimeReceiptRequest(
5375
- normalizedInputs.map((receipt) => ({
5376
- ...receipt,
5377
- runId: this.currentReceiptOwnerRunId,
5378
- runAttempt: this.currentRunAttempt,
5379
- })),
5380
- (chunk) => markReceipts({ receipts: chunk }),
5381
- );
5382
- } else {
5383
- marked = await Promise.all(
5384
- normalizedInputs.map((receipt) =>
5385
- this.markRuntimeStepReceiptRunning(receipt),
5386
- ),
5387
- );
5388
- }
5389
- const byKey = new Map<string, RuntimeStepReceipt>();
5390
- for (let index = 0; index < marked.length; index += 1) {
5391
- const normalized = this.normalizeRuntimeStepReceipt(
5392
- normalizedInputs[index]?.key ?? '',
5393
- marked[index],
5394
- );
5395
- if (normalized) byKey.set(normalized.key, normalized);
5396
- }
5397
- return byKey;
5398
- }
5399
-
5400
- private async markRuntimeStepReceiptsQueued(
5401
- receipts: Array<{
5402
- key: string;
5403
- runId: string;
5404
- leaseId?: string | null;
5405
- }>,
5406
- ): Promise<Map<string, RuntimeStepReceipt>> {
5407
- const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
5408
- if (normalizedInputs.length === 0) return new Map();
5409
- if (!this.#options.markRuntimeStepReceiptsQueued) {
5410
- return new Map();
5411
- }
5412
- const queued = await this.dispatchChunkedRuntimeReceiptRequest(
5413
- normalizedInputs.map((receipt) => ({
5414
- ...receipt,
5415
- runId: this.currentReceiptOwnerRunId,
5416
- runAttempt: this.currentRunAttempt,
5417
- })),
5418
- (chunk) =>
5419
- this.#options.markRuntimeStepReceiptsQueued!({ receipts: chunk }),
5420
- );
5421
- const byKey = new Map<string, RuntimeStepReceipt>();
5422
- for (let index = 0; index < queued.length; index += 1) {
5423
- const normalized = this.normalizeRuntimeStepReceipt(
5424
- normalizedInputs[index]?.key ?? '',
5425
- queued[index],
5426
- );
5427
- if (normalized) byKey.set(normalized.key, normalized);
5428
- }
5429
- return byKey;
5430
- }
5431
-
5432
- private async parkRuntimeToolReceiptsQueued(
5433
- requests: ToolCallRequest[],
5434
- ): Promise<void> {
5435
- const targets = requests.flatMap((request) => {
5436
- const receiptKey = request.receiptKey?.trim() || null;
5437
- const leaseId = request.receiptLeaseId?.trim() || null;
5438
- if (!receiptKey || !leaseId) return [];
5439
- return [
5440
- { key: receiptKey, runId: this.currentReceiptOwnerRunId, leaseId },
5441
- ];
5442
- });
5443
- if (targets.length === 0) return;
5444
- await this.markRuntimeStepReceiptsQueued(targets);
5445
- }
5446
-
5447
5152
  /**
5448
5153
  * Enforce the receipt output contract and remember the output's serialized
5449
5154
  * size. Idempotent per input: the first caller pays, everything downstream
@@ -6326,7 +6031,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6326
6031
 
6327
6032
  private durableReceiptExecutionStore(): DurableReceiptExecutionStore {
6328
6033
  return {
6329
- enabled: Boolean(this.#options.claimRuntimeStepReceipt),
6034
+ enabled: this.durableDirectToolResultsBackedByReceipts,
6330
6035
  get: (receiptKey) => this.getRuntimeStepReceipt(receiptKey),
6331
6036
  getMany: (receiptKeys) => this.getRuntimeStepReceipts(receiptKeys),
6332
6037
  claim: (
@@ -6403,35 +6108,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6403
6108
  return durableRuntimeReceiptOutput<T>(receipt);
6404
6109
  }
6405
6110
 
6406
- private async waitForCompletedRuntimeToolReceipt(
6407
- key: string,
6408
- maxAttempts?: number,
6409
- ): Promise<RuntimeStepReceipt> {
6410
- return await waitForCompletedRuntimeReceipt({
6411
- receiptKey: key,
6412
- store: this.durableReceiptExecutionStore(),
6413
- maxAttempts,
6414
- toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
6415
- });
6416
- }
6417
-
6418
- private async waitForCompletedRuntimeToolReceipts(
6419
- keys: string[],
6420
- maxAttempts?: number,
6421
- ): Promise<{
6422
- completed: Map<string, RuntimeStepReceipt>;
6423
- failed: Map<string, Error>;
6424
- timedOut: Set<string>;
6425
- }> {
6426
- return await waitForCompletedRuntimeReceipts({
6427
- receiptKeys: keys,
6428
- store: this.durableReceiptExecutionStore(),
6429
- maxAttempts,
6430
- log: (message) => this.runtimeLog(message),
6431
- toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
6432
- });
6433
- }
6434
-
6435
6111
  private async executeWithRuntimeReceipt<T>(
6436
6112
  operation: DurableCtxOperation,
6437
6113
  id: string,
@@ -6525,7 +6201,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6525
6201
  onClaimedResult: opts.onClaimedResult,
6526
6202
  shouldPersistFailure: opts.shouldPersistFailure,
6527
6203
  markRunningBeforeExecute: opts.markRunningBeforeExecute,
6528
- completedCacheOnly: operation === 'tool',
6204
+ completedCacheOnly: true,
6529
6205
  withCompletedReceiptHydration: (hydrate) =>
6530
6206
  this.withRuntimeReceiptHydrationTurn(hydrate, {
6531
6207
  source: 'durable_operation',
@@ -6632,7 +6308,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6632
6308
  *
6633
6309
  * The external receipt adapter is intentionally local to this migration.
6634
6310
  * It checks the exact compatibility identities passed by the map, while the
6635
- * map remains the sole writer for its member receipts, follower delivery,
6311
+ * map remains the sole writer for its member receipts, invocation delivery,
6636
6312
  * and completion barrier. In particular, do not turn an always-fresh map
6637
6313
  * request into a provider-idempotent request here: `apiOptions` is forwarded
6638
6314
  * byte-for-byte and decides whether the historical wire carried an identity.
@@ -6806,19 +6482,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6806
6482
  });
6807
6483
  }
6808
6484
 
6809
- /**
6810
- * A forced call bypasses an earlier run's shared result, so its physical
6811
- * gateway invocation must bypass that result's replay identity too. Keep
6812
- * the transport key deterministic for one logical call: a retry of that
6813
- * call still converges, while a later explicit force gets a fresh identity.
6814
- */
6485
+ /** One replay-stable physical delivery identity for each invocation. */
6815
6486
  private toolTransportReceiptKey(input: {
6816
6487
  toolId: string;
6817
- resultReceiptKey: string;
6818
6488
  providerOperationKey: string;
6819
- force?: boolean;
6820
6489
  }): string {
6821
- if (input.force !== true) return input.resultReceiptKey;
6822
6490
  return `${buildDurableToolReceiptPrefix({
6823
6491
  orgId: this.#options.orgId,
6824
6492
  toolId: input.toolId,
@@ -6841,41 +6509,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6841
6509
  });
6842
6510
  }
6843
6511
 
6844
- /**
6845
- * Ordinary operations do not carry provider-idempotency metadata, but the
6846
- * execution relay still needs a semantic signal that an explicit force is
6847
- * a new provider operation. Unsafe-write operations already carry their
6848
- * forced provider key directly, so only ordinary calls need this attested
6849
- * canonical-operation salt.
6850
- */
6851
- private providerForceRefreshIdentitiesForToolCall(input: {
6512
+ /** Keep independent misses distinct in the gateway's canonical operation. */
6513
+ private providerInvocationOptionsForToolCall(input: {
6852
6514
  operationKey: string;
6853
- force?: boolean;
6854
6515
  requiresExecutionFence?: boolean;
6855
6516
  logicalCallId?: string;
6856
- }): string[] {
6857
- if (input.force !== true || input.requiresExecutionFence === true) {
6858
- return [];
6859
- }
6860
- return [
6861
- this.providerIdempotencyKeyForToolCall({
6862
- operationKey: input.operationKey,
6863
- force: true,
6864
- logicalCallId: input.logicalCallId,
6865
- }),
6866
- ];
6867
- }
6868
-
6869
- private providerForceRefreshOptionsForToolCall(input: {
6870
- operationKey: string;
6871
6517
  force?: boolean;
6872
- requiresExecutionFence?: boolean;
6873
- logicalCallId?: string;
6874
6518
  }): Pick<ToolExecutionApiOptions, 'providerOperationForceRefreshIdentities'> {
6875
- const identities = this.providerForceRefreshIdentitiesForToolCall(input);
6876
- return identities.length > 0
6877
- ? { providerOperationForceRefreshIdentities: identities }
6878
- : {};
6519
+ return input.requiresExecutionFence === true
6520
+ ? {}
6521
+ : {
6522
+ providerOperationForceRefreshIdentities: [
6523
+ this.providerInvocationKeyForToolCall(input),
6524
+ ],
6525
+ };
6879
6526
  }
6880
6527
 
6881
6528
  /**
@@ -7577,15 +7224,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7577
7224
  private getCachedToolResult(
7578
7225
  toolId: string,
7579
7226
  rowCacheKey: string,
7580
- path: 'mapped' | 'direct' = 'mapped',
7227
+ _path: 'mapped' | 'direct' = 'mapped',
7581
7228
  ): ToolBatchResult | undefined {
7582
- if (
7583
- path === 'direct'
7584
- ? this.durableDirectToolResultsBackedByReceipts
7585
- : this.durableMappedToolResultsBackedByReceipts
7586
- ) {
7587
- return undefined;
7588
- }
7229
+ // Receipt-backed calls never add payloads here. Retained outgoing entries
7230
+ // may predate that capability and remain completed cache hits.
7589
7231
  return this.checkpoint.completedToolBatches[toolId]?.[rowCacheKey];
7590
7232
  }
7591
7233
 
@@ -7636,32 +7278,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7636
7278
  if (!this.checkpoint.completedToolBatches[toolId]) {
7637
7279
  this.checkpoint.completedToolBatches[toolId] = {};
7638
7280
  }
7639
- this.checkpoint.completedToolBatches[toolId]![rowCacheKey] = {
7281
+ this.checkpoint.completedToolBatches[toolId]![rowCacheKey] ??= {
7640
7282
  done: true,
7641
7283
  result,
7642
7284
  };
7643
7285
  }
7644
7286
 
7645
- private buildToolResultCacheKey(input: {
7646
- rowId: number;
7647
- tableNamespace?: string;
7648
- rowKey?: string;
7649
- callId?: string;
7650
- }): string {
7651
- const scope =
7652
- this.currentGovernance.currentRunId || this.#options.runId || 'run';
7653
- if (input.callId?.trim()) {
7654
- return `${scope}:${input.callId.trim()}`;
7655
- }
7656
- if (input.rowKey?.trim()) {
7657
- return `${scope}:${input.rowKey.trim()}`;
7658
- }
7659
- if (input.tableNamespace?.trim()) {
7660
- return `${scope}:${input.tableNamespace.trim()}:${input.rowId}`;
7661
- }
7662
- return `${scope}:direct:${input.rowId}`;
7663
- }
7664
-
7665
7287
  private async resolveToolResultMetadata(
7666
7288
  toolId: string,
7667
7289
  ): Promise<ToolResultMetadataInput> {
@@ -7753,6 +7375,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7753
7375
  private materializeToolExecutionResult(
7754
7376
  result: ToolExecuteResult,
7755
7377
  ): ToolExecuteResult {
7378
+ result = projectNativeBatchToolResponse(
7379
+ result,
7380
+ this.currentRuntimeContract.semantics.tools.nativeBatchResponse,
7381
+ );
7756
7382
  if (
7757
7383
  this.currentToolResponseContract === RAW_V2_TOOL_RESPONSE_CONTRACT ||
7758
7384
  !this.hasPinnedToolResponseContract ||
@@ -8136,40 +7762,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8136
7762
  `Durable tool call ${receiptKey} completed live after its receipt lease moved; using live result without overwriting the receipt.`,
8137
7763
  );
8138
7764
  }
8139
- const finalWrapped =
8140
- (completed?.status === 'completed' || completed?.status === 'skipped') &&
8141
- completed.output !== undefined
8142
- ? await this.wrapToolExecutionResult({
8143
- toolId,
8144
- status:
8145
- completed.output === null || completed.output === undefined
8146
- ? 'no_result'
8147
- : 'completed',
8148
- result: this.runtimeReceiptOutput(completed),
8149
- requestInput: request.input,
8150
- execution: toolExecutionMetadataForOutcome(
8151
- completed.runId === this.currentReceiptOwnerRunId
8152
- ? {
8153
- kind: 'live',
8154
- cacheKey,
8155
- receiptKey: request.receiptKey,
8156
- }
8157
- : {
8158
- kind: 'cache',
8159
- cacheKey,
8160
- receiptKey: receiptKey ?? '',
8161
- attachedToReceiptKey: receiptKey,
8162
- },
8163
- ),
8164
- })
8165
- : wrapped;
8166
7765
  if (request.cacheable !== false) {
8167
- this.cacheToolResult(toolId, cacheKey, finalWrapped);
7766
+ this.cacheToolResult(toolId, cacheKey, wrapped);
8168
7767
  }
8169
7768
 
8170
7769
  const resolver = this.toolCallResolvers.get(request.callId);
8171
7770
  if (resolver) {
8172
- resolver.resolve(finalWrapped);
7771
+ resolver.resolve(wrapped);
8173
7772
  this.toolCallResolvers.delete(request.callId);
8174
7773
  }
8175
7774
 
@@ -8185,7 +7784,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8185
7784
  error: null,
8186
7785
  dataPatch: {},
8187
7786
  });
8188
- return finalWrapped;
7787
+ return wrapped;
8189
7788
  }
8190
7789
 
8191
7790
  private async rejectToolCall(
@@ -9003,6 +8602,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9003
8602
  Math.max(0, failedCount - staleFailedKeys.size),
9004
8603
  progressSupersededOffset: () =>
9005
8604
  staleCompletedKeys.size + staleFailedKeys.size,
8605
+ progressSkippedOffset: () => duplicateReuseCount,
9006
8606
  progressTotalRows: () => totalInputCount,
9007
8607
  rowFeed: execution.rowFeed,
9008
8608
  retainedRowsMemoryBudgetBytes:
@@ -10231,6 +9831,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10231
9831
  progressCompletedOffset?: number | (() => number);
10232
9832
  progressFailedOffset?: number | (() => number);
10233
9833
  progressSupersededOffset?: number | (() => number);
9834
+ /** Input duplicates discovered after admission; excludes resolver outcomes. */
9835
+ progressSkippedOffset?: number | (() => number);
10234
9836
  progressTotalRows?: number | (() => number);
10235
9837
  /**
10236
9838
  * When set, `items`, `executionRowKeys` and `executionRowIndexes` grow
@@ -10476,6 +10078,37 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10476
10078
  } | null = null;
10477
10079
  let lastFrameFlushAt = 0;
10478
10080
  let rowsSinceFrameFlush = 0;
10081
+ // Scalars only: resolver admission is visible before the first row's
10082
+ // durable completion, without publishing one event per row or tool call.
10083
+ let startedResolverRows = 0;
10084
+ let activeResolverRows = 0;
10085
+ let tracksResolverActivity = false;
10086
+ // Persistence can move current resolvers between successful, failed and
10087
+ // superseded offsets. They already contribute to startedResolverRows:
10088
+ // only the settled baseline before this pool started belongs in admission.
10089
+ const initiallyStartedRows =
10090
+ liveNumber(runtimeOptions?.progressCompletedOffset, completedRows) +
10091
+ liveNumber(runtimeOptions?.progressFailedOffset, 0) +
10092
+ liveNumber(runtimeOptions?.progressSupersededOffset, 0);
10093
+ const initiallySkippedRows = liveNumber(
10094
+ runtimeOptions?.progressSkippedOffset,
10095
+ 0,
10096
+ );
10097
+ const mapActivityCounts = () => {
10098
+ const startedRows =
10099
+ initiallyStartedRows +
10100
+ (liveNumber(
10101
+ runtimeOptions?.progressSkippedOffset,
10102
+ initiallySkippedRows,
10103
+ ) -
10104
+ initiallySkippedRows) +
10105
+ startedResolverRows;
10106
+ return {
10107
+ startedRows,
10108
+ activeRows: activeResolverRows,
10109
+ waitingRows: currentTotalRows() - startedRows,
10110
+ };
10111
+ };
10479
10112
  const updateMapFrameProgress = (input: {
10480
10113
  status?: MapExecutionFrame['status'];
10481
10114
  completedRowKey?: string | null;
@@ -10607,6 +10240,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10607
10240
  completedRowKeys.size,
10608
10241
  failedRows,
10609
10242
  totalRows: currentTotalRows(),
10243
+ ...(progressEventType === 'map.progress' && tracksResolverActivity
10244
+ ? mapActivityCounts()
10245
+ : {}),
10610
10246
  supersededRows: liveNumber(
10611
10247
  runtimeOptions?.progressSupersededOffset,
10612
10248
  existing.supersededRows ?? 0,
@@ -11254,6 +10890,39 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11254
10890
  items[idx] = null as unknown as T;
11255
10891
  }
11256
10892
  : null;
10893
+ let lastMapActivityKey: string | null = null;
10894
+ tracksResolverActivity = true;
10895
+ const publishMapActivity = () => {
10896
+ const counts = mapActivityCounts();
10897
+ const key = `${counts.startedRows}:${counts.activeRows}:${counts.waitingRows}`;
10898
+ if (key === lastMapActivityKey) return;
10899
+ lastMapActivityKey = key;
10900
+ // Admission samples do not rebuild checkpoint key sets. The map's
10901
+ // writer continues to own durable settlement and checkpoint flushing.
10902
+ this.emitExecutionEvent({
10903
+ type: 'map.progress',
10904
+ mapInvocationId: mapScope.mapInvocationId,
10905
+ mapNodeId: mapScope.mapNodeId ?? null,
10906
+ logicalNamespace: mapScope.logicalNamespace,
10907
+ artifactTableNamespace: mapScope.artifactTableNamespace,
10908
+ completedRows:
10909
+ liveNumber(runtimeOptions?.progressCompletedOffset, completedRows) +
10910
+ (liveFrameSets?.completedRowKeys.size ?? 0),
10911
+ failedRows:
10912
+ liveNumber(runtimeOptions?.progressFailedOffset, 0) +
10913
+ (liveFrameSets?.failedRowsCount ?? 0),
10914
+ totalRows: currentTotalRows(),
10915
+ supersededRows: liveNumber(runtimeOptions?.progressSupersededOffset, 0),
10916
+ ...counts,
10917
+ ...this.inlineChildAggregateEventFields(),
10918
+ at: Date.now(),
10919
+ });
10920
+ };
10921
+ const mapActivityTimer = setInterval(
10922
+ publishMapActivity,
10923
+ PROGRESS_HEARTBEAT_INTERVAL_MS,
10924
+ );
10925
+ mapActivityTimer.unref();
11257
10926
  const workers = Array.from({ length: workerCount }, async () => {
11258
10927
  while (firstRowError == null) {
11259
10928
  if (nextRowIndex >= items.length) {
@@ -11270,7 +10939,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11270
10939
  releaseDispatchedInputRow?.(idx);
11271
10940
  settledResults[idx] = await mapRowFlow.run({ baseRow }, async () => {
11272
10941
  this.assertRowAdmissionOpen();
11273
- return await runRow(idx, baseRow);
10942
+ startedResolverRows += 1;
10943
+ activeResolverRows += 1;
10944
+ try {
10945
+ return await runRow(idx, baseRow);
10946
+ } finally {
10947
+ activeResolverRows -= 1;
10948
+ }
11274
10949
  });
11275
10950
  } catch (error) {
11276
10951
  // RuntimeFlow has already returned the row slot and memory lease.
@@ -11364,6 +11039,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11364
11039
  settled = await Promise.allSettled(workers);
11365
11040
  }
11366
11041
  } finally {
11042
+ clearInterval(mapActivityTimer);
11043
+ publishMapActivity();
11367
11044
  datasetStep.items = currentTotalRows();
11368
11045
  this.processedRowCount = currentTotalRows();
11369
11046
  mapStallObserver?.stopHeartbeat();
@@ -12930,7 +12607,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
12930
12607
  options.receiptWaitMs,
12931
12608
  );
12932
12609
  }
12933
- const executionScope = this.currentExecutionScope;
12934
12610
  const normalizedKey = this.normalizeContextKey(key, 'tool');
12935
12611
  const toolCachePolicy = this.effectiveToolCallCachePolicy(options);
12936
12612
  const toolRequestIdentity = deriveToolRequestIdentity({
@@ -13028,10 +12704,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13028
12704
  ...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
13029
12705
  ];
13030
12706
 
13031
- const executeTool = async (context?: {
13032
- leaseId?: string | null;
13033
- retainExternalCallSlot?: (release: () => void) => void;
13034
- }): Promise<unknown> => {
12707
+ const executeTool = async (): Promise<unknown> => {
13035
12708
  if (eventWaitHandler) {
13036
12709
  return this.executeIntegrationEventWaitTool(
13037
12710
  toolId,
@@ -13291,7 +12964,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13291
12964
  });
13292
12965
  };
13293
12966
  try {
13294
- const directReceiptLeaseId = context?.leaseId?.trim() || null;
13295
12967
  if (!logicalCallId) {
13296
12968
  throw new Error(
13297
12969
  'Direct tool execution requires a stable logical call identity.',
@@ -13299,17 +12971,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13299
12971
  }
13300
12972
  const useDirectResultReceipts =
13301
12973
  cacheableToolResult && !bypassDirectResultCache;
13302
- const providerOperationKey = this.providerIdempotencyKeyForToolCall({
12974
+ const providerOperationKey = this.providerInvocationKeyForToolCall({
13303
12975
  operationKey: providerIdempotencyKeyBase,
13304
- force: cacheableToolResult && toolCachePolicy.force,
13305
12976
  logicalCallId,
12977
+ force: toolCachePolicy.force,
13306
12978
  });
13307
12979
  const physicalDirectKey = useDirectResultReceipts
13308
12980
  ? this.toolTransportReceiptKey({
13309
12981
  toolId,
13310
- resultReceiptKey: directCacheKey,
13311
12982
  providerOperationKey,
13312
- force: toolCachePolicy.force,
13313
12983
  })
13314
12984
  : `${buildDurableToolReceiptPrefix({
13315
12985
  orgId: this.#options.orgId ?? 'unknown-org',
@@ -13323,13 +12993,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13323
12993
  runId: this.currentRunId,
13324
12994
  logicalCallId,
13325
12995
  force: toolCachePolicy.force,
13326
- canStoreResult: Boolean(
13327
- this.#options.claimRuntimeStepReceipt &&
13328
- (this.#options.getRuntimeStepReceipt ||
13329
- this.#options.getRuntimeStepReceipts) &&
13330
- (this.#options.completeRuntimeStepReceipt ||
13331
- this.#options.completeRuntimeStepReceipts),
13332
- ),
12996
+ canStoreResult: this.durableDirectToolResultsBackedByReceipts,
13333
12997
  });
13334
12998
  const toolRetryPolicy = await this.#options
13335
12999
  .getToolRetryPolicy?.(toolId, input)
@@ -13365,28 +13029,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13365
13029
  ...(useDirectResultReceipts
13366
13030
  ? {
13367
13031
  durableCallReceiptKey: physicalDirectKey,
13032
+ legacyProviderRetryIdentities: toolCachePolicy.force
13033
+ ? undefined
13034
+ : [
13035
+ {
13036
+ transportReceiptKey: directCacheKey,
13037
+ providerOperationKey:
13038
+ providerIdempotencyKeyBase,
13039
+ },
13040
+ ],
13368
13041
  completedResultCache: true,
13369
13042
  executionAuthScopeDigest,
13370
- ...(directReceiptLeaseId &&
13371
- (this.#options.heartbeatRuntimeStepReceipts ||
13372
- this.#options.getRuntimeStepReceipt ||
13373
- this.#options.getRuntimeStepReceipts)
13374
- ? {
13375
- beforeProviderCall: async () => {
13376
- await this.assertRuntimeToolReceiptOwnership([
13377
- {
13378
- callId: 'direct',
13379
- cacheKey: directCacheKey,
13380
- receiptKey: directCacheKey,
13381
- receiptLeaseId: directReceiptLeaseId,
13382
- rowId: -1,
13383
- toolId,
13384
- input,
13385
- },
13386
- ]);
13387
- },
13388
- }
13389
- : {}),
13390
13043
  }
13391
13044
  : {}),
13392
13045
  ...(toolRetryPolicy?.requiresExecutionFence === true
@@ -13401,15 +13054,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13401
13054
  : physicalDirectKey,
13402
13055
  }
13403
13056
  : {}),
13404
- ...(useDirectResultReceipts && operationKey
13405
- ? this.providerForceRefreshOptionsForToolCall({
13406
- operationKey: providerIdempotencyKeyBase,
13407
- force: toolCachePolicy.force,
13408
- requiresExecutionFence:
13409
- toolRetryPolicy?.requiresExecutionFence,
13410
- logicalCallId,
13411
- })
13412
- : {}),
13057
+ ...this.providerInvocationOptionsForToolCall({
13058
+ operationKey: providerIdempotencyKeyBase,
13059
+ requiresExecutionFence:
13060
+ toolRetryPolicy?.requiresExecutionFence,
13061
+ logicalCallId,
13062
+ force: toolCachePolicy.force,
13063
+ }),
13413
13064
  playNodeScope: buildPlayNodeScope({
13414
13065
  toolId,
13415
13066
  callKey: normalizedKey,
@@ -13419,7 +13070,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13419
13070
  options?.timeoutMs,
13420
13071
  this.currentAuthoringContractEdition,
13421
13072
  ),
13422
- retainToolSlot: context?.retainExternalCallSlot,
13423
13073
  },
13424
13074
  });
13425
13075
  return await this.wrapToolExecutionResult({
@@ -13558,20 +13208,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13558
13208
 
13559
13209
  const rowId = store.rowId;
13560
13210
  const fieldName = store.fieldName;
13561
- const callId = [
13562
- executionScope.receipt.namespace,
13563
- store.tableNamespace?.trim() || 'map',
13564
- store.rowKey?.trim() || String(rowId),
13565
- normalizedKey,
13566
- toolId,
13567
- toolRequestIdentity,
13568
- ].join(':');
13569
- if (this.toolCallResolvers.has(callId)) {
13570
- throw new Error(
13571
- `ctx.tools.execute("${normalizedKey}") was called more than once concurrently for the same row. ` +
13572
- 'Use a unique id for each row-scoped tool call.',
13573
- );
13574
- }
13211
+ // Cache identity is shared by matching completed results. Delivery
13212
+ // identity belongs to this invocation, even when callers reuse an id.
13213
+ const callId = logicalCallId;
13575
13214
 
13576
13215
  const toolResultCacheKey = durableCacheKey;
13577
13216
  const cached = !cacheableToolResult
@@ -13597,7 +13236,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13597
13236
  const rowToolRetryPolicy = await this.#options
13598
13237
  .getToolRetryPolicy?.(toolId, input)
13599
13238
  .catch(() => null);
13600
-
13601
13239
  // ADR 0018: record this leg before it runs. `producer` still names the
13602
13240
  // last leg (the usual cascade winner); `producers` keeps every leg, which
13603
13241
  // is the only durable record a losing leg ever gets — a failed tool call
@@ -13641,7 +13279,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13641
13279
  // The tail of that wall. The dispatcher stamps the request when this
13642
13280
  // call's provider body finished parsing; the difference here is what
13643
13281
  // receipt completion and hand-back cost after the provider was done. A
13644
- // cache hit and a follower never read a body, so they leave no stamp and
13282
+ // cache hit never reads a body, so it leaves no stamp and
13645
13283
  // record nothing rather than a delivery of zero.
13646
13284
  const observeResultDelivery = () => {
13647
13285
  const bodyCompletedAtMs = dispatchRequest.providerBodyCompletedAtMs;
@@ -15389,7 +15027,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
15389
15027
  *
15390
15028
  * Gated on the same stamp `resultDelivery` is gated on — a body this request
15391
15029
  * really read — so every delivery sub-phase counts the same calls and their
15392
- * sum can be checked against the whole. A cache hit, a follower, and a
15030
+ * sum can be checked against the whole. A cache hit and a
15393
15031
  * member that never reached the provider leave the stamp unset and record
15394
15032
  * nothing rather than a wrap of zero.
15395
15033
  */
@@ -15521,179 +15159,38 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
15521
15159
 
15522
15160
  // ——— Batched tool call execution ———
15523
15161
 
15524
- private startPendingToolReceiptHeartbeat(
15525
- requests: ToolCallRequest[],
15526
- claimsEstablishedExecutionFence: boolean,
15527
- ): {
15528
- stop: () => void;
15529
- leaseLost: () => unknown | null;
15530
- confirmOwned: () => Promise<void>;
15531
- assertOwned: () => void;
15532
- } | null {
15533
- const heartbeatReceipts = this.#options.heartbeatRuntimeStepReceipts;
15534
- const ownedRequests = requests.filter(
15535
- (request) => request.receiptKey && request.receiptLeaseId,
15536
- );
15537
- if (!heartbeatReceipts || ownedRequests.length === 0) return null;
15538
-
15539
- const earliestLeaseExpiry = ownedRequests.reduce<string | null>(
15540
- (earliest, request) => {
15541
- const expiresAt = request.receiptLeaseExpiresAt ?? null;
15542
- if (!expiresAt) return earliest;
15543
- if (!earliest) return expiresAt;
15544
- return Date.parse(expiresAt) < Date.parse(earliest)
15545
- ? expiresAt
15546
- : earliest;
15547
- },
15548
- null,
15549
- );
15550
- let leaseLost: unknown | null = null;
15551
- let ownershipConfirmedUntilMs = Number.isFinite(
15552
- Date.parse(earliestLeaseExpiry ?? ''),
15553
- )
15554
- ? Date.parse(earliestLeaseExpiry!)
15555
- : Date.now() + PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS;
15556
- const firstOwnedRequest = ownedRequests[0]!;
15557
- const heartbeatOwnedRequests = async () => {
15558
- // Receipt claims are acquired in bulk before the provider-shaped
15559
- // dispatcher admits every call. Renew the undispatched tail as well as
15560
- // in-flight calls; otherwise a slow provider can let later claims
15561
- // expire while they are still waiting behind the concurrency gate.
15562
- const unsettled = ownedRequests.filter((request) =>
15563
- this.toolCallResolvers.has(request.callId),
15564
- );
15565
- if (unsettled.length === 0) return 'terminal' as const;
15566
- const byReceiptKey = new Map<string, ToolCallRequest[]>();
15567
- for (const request of unsettled) {
15568
- const receiptKey = request.receiptKey!;
15569
- const group = byReceiptKey.get(receiptKey) ?? [];
15570
- group.push(request);
15571
- byReceiptKey.set(receiptKey, group);
15162
+ /**
15163
+ * Concrete RuntimeStepReceipt adapter for the map receipt-cohort Module.
15164
+ *
15165
+ * Scalar and provider-native map calls share completed-result cache reads
15166
+ * and atomic successful inserts. A miss has no durable execution owner.
15167
+ */
15168
+ private createRuntimeStepToolCallReceiptCohortStore(input: {
15169
+ toolId: string;
15170
+ requestForReceipt: ReadonlyMap<string, ToolCallRequest>;
15171
+ /** Each invocation sharing a completed-result key, for manifests and timing. */
15172
+ requestsForReceipt: ReadonlyMap<string, readonly ToolCallRequest[]>;
15173
+ }): ToolCallReceiptCohortStore {
15174
+ const requestsForPublication = (receipt: {
15175
+ resultReceiptKey: string;
15176
+ memberIds?: readonly string[];
15177
+ }): readonly ToolCallRequest[] => {
15178
+ if (!receipt.memberIds) {
15179
+ const request = input.requestForReceipt.get(receipt.resultReceiptKey);
15180
+ return request ? [request] : [];
15572
15181
  }
15573
- // One content-addressed receipt can have many same-run followers. The
15574
- // store renews unique receipt rows, not request positions, so dedupe by
15575
- // key while preserving the matching per-row lease id.
15576
- const entries = [...byReceiptKey];
15577
- const keys = entries.map(([receiptKey]) => receiptKey);
15578
- const leaseIds = entries.map(
15579
- ([, requests]) => requests[0]!.receiptLeaseId!,
15580
- );
15581
- const receipts = await heartbeatReceipts({
15582
- runId: this.currentReceiptOwnerRunId,
15583
- runAttempt: this.currentRunAttempt,
15584
- keys,
15585
- leaseIds,
15586
- });
15587
- const renewedExpiries: number[] = [];
15588
- for (let index = 0; index < entries.length; index += 1) {
15589
- const [receiptKey, requests] = entries[index]!;
15590
- const leaseId = leaseIds[index]!;
15591
- // Completion can race this bulk heartbeat response. A settled
15592
- // receipt group no longer needs ownership and must not turn that
15593
- // race into a false lease-loss failure or hold the confirmation
15594
- // horizon at its old expiry.
15595
- if (
15596
- this.runtimeToolReceiptCompletionsInFlight.has(receiptKey) ||
15597
- !requests.some((request) =>
15598
- this.toolCallResolvers.has(request.callId),
15599
- )
15600
- ) {
15601
- continue;
15602
- }
15603
- const receipt = receipts[index] ?? null;
15604
- const renewedExpiryMs = Date.parse(receipt?.leaseExpiresAt ?? '');
15605
- if (
15606
- !this.runtimeToolReceiptStillOwned(receipt, leaseId) ||
15607
- !Number.isFinite(renewedExpiryMs)
15608
- ) {
15609
- throw new RuntimeReceiptLeaseLostError({
15610
- receiptKey,
15611
- runId: this.currentReceiptOwnerRunId,
15612
- leaseId,
15613
- });
15614
- }
15615
- renewedExpiries.push(renewedExpiryMs);
15616
- }
15617
- if (renewedExpiries.length === 0) return 'terminal' as const;
15618
- ownershipConfirmedUntilMs = Math.min(...renewedExpiries);
15619
- return 'active' as const;
15620
- };
15621
- const perCallHeartbeatIntervalMs = runtimeLeaseHeartbeatIntervalFromExpiry({
15622
- leaseExpiresAt: earliestLeaseExpiry,
15623
- fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
15624
- });
15625
- const supervisor = createRuntimeReceiptHeartbeatSupervisor({
15626
- // The cohort owns claims that have not reached provider admission yet.
15627
- // Renew it before the one-call supervisors can win the same short-lease
15628
- // timer race, so the queued tail remains fenced while the active call
15629
- // consumes the provider concurrency slot.
15630
- intervalMs: Math.max(1, Math.floor(perCallHeartbeatIntervalMs / 2)),
15631
- heartbeat: heartbeatOwnedRequests,
15632
- isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
15633
- onLeaseLost: (error) => {
15634
- leaseLost ??= error;
15635
- },
15636
- onTransientFailure: (error) => {
15637
- this.runtimeLog(
15638
- `Pending tool receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`,
15182
+ const requests =
15183
+ input.requestsForReceipt.get(receipt.resultReceiptKey) ?? [];
15184
+ return receipt.memberIds.map((memberId) => {
15185
+ const request = requests.find(
15186
+ (candidate) => candidate.callId === memberId,
15639
15187
  );
15640
- },
15641
- });
15642
- return {
15643
- stop: () => supervisor.stop(),
15644
- leaseLost: () => leaseLost,
15645
- confirmOwned: async () => {
15646
- // The scheduler receipt gateway returns `running` from an atomic
15647
- // claim. That claim is already the provider-execution fence, so short
15648
- // calls only need scheduled renewal if they approach lease expiry.
15649
- // Other callers may still return queued/pending claims or omit the
15650
- // explicit backend capability; retain the immediate ownership check
15651
- // for those receipts.
15652
- if (claimsEstablishedExecutionFence) {
15653
- if (
15654
- ownershipConfirmedUntilMs - Date.now() <=
15655
- IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS
15656
- ) {
15657
- const outcome = await heartbeatOwnedRequests();
15658
- if (outcome === 'active') supervisor.start();
15659
- return;
15660
- }
15661
- supervisor.start();
15662
- return;
15188
+ if (!request) {
15189
+ throw new Error(`Unknown Tool Call publication member ${memberId}.`);
15663
15190
  }
15664
- const outcome = await heartbeatOwnedRequests();
15665
- if (outcome === 'active') supervisor.start();
15666
- },
15667
- assertOwned: () => {
15668
- if (leaseLost) throw leaseLost;
15669
- if (Date.now() < ownershipConfirmedUntilMs) return;
15670
- throw new RuntimeReceiptLeaseLostError({
15671
- receiptKey: firstOwnedRequest.receiptKey!,
15672
- runId: this.currentReceiptOwnerRunId,
15673
- leaseId: firstOwnedRequest.receiptLeaseId!,
15674
- });
15675
- },
15191
+ return request;
15192
+ });
15676
15193
  };
15677
- }
15678
-
15679
- /**
15680
- * Concrete RuntimeStepReceipt adapter for the map receipt-cohort Module.
15681
- *
15682
- * Scalar and provider-native map calls share completed-result cache reads
15683
- * and atomic successful inserts. A miss has no durable execution owner.
15684
- */
15685
- private createRuntimeStepToolCallReceiptCohortStore(input: {
15686
- toolId: string;
15687
- requestForReceipt: ReadonlyMap<string, ToolCallRequest>;
15688
- /**
15689
- * Every request sharing a receipt key, for delivery timing only. Two rows
15690
- * asking the same question share one key, and only one of them — the
15691
- * cohort's owner — executed and carries the wrap stamp `deliveryPublishWait`
15692
- * is measured from, so the timing lookup cannot use the single-request map
15693
- * above, which keeps whichever request was seen last.
15694
- */
15695
- requestsForReceipt: ReadonlyMap<string, readonly ToolCallRequest[]>;
15696
- }): ToolCallReceiptCohortStore {
15697
15194
  const asRecord = async (
15698
15195
  resultReceiptKey: string,
15699
15196
  storedReceiptKey: string,
@@ -15879,23 +15376,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
15879
15376
  );
15880
15377
  const manifestCompletions =
15881
15378
  this.sourceRunToolReceiptManifestCompletionInputs(
15882
- receipts.flatMap((receipt) => {
15883
- const request = input.requestForReceipt.get(
15884
- receipt.resultReceiptKey,
15885
- );
15886
- return request
15887
- ? [
15888
- {
15889
- toolId: input.toolId,
15890
- logicalCallId: request.logicalCallId,
15891
- executionAuthScopeDigest:
15892
- request.executionAuthScopeDigest,
15893
- resultReceiptKey: receipt.resultReceiptKey,
15894
- providerActionVersion: request.providerActionVersion,
15895
- },
15896
- ]
15897
- : [];
15898
- }),
15379
+ receipts.flatMap((receipt) =>
15380
+ requestsForPublication(receipt).map((request) => ({
15381
+ toolId: input.toolId,
15382
+ logicalCallId: request.logicalCallId,
15383
+ executionAuthScopeDigest: request.executionAuthScopeDigest,
15384
+ resultReceiptKey: receipt.resultReceiptKey,
15385
+ providerActionVersion: request.providerActionVersion,
15386
+ })),
15387
+ ),
15899
15388
  );
15900
15389
  await this.completeRuntimeStepReceipts(
15901
15390
  [...canonicalCompletions, ...manifestCompletions],
@@ -15905,18 +15394,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
15905
15394
  // charged to one of the two rather than to neither.
15906
15395
  (enqueuedAtMs) => {
15907
15396
  for (const receipt of receipts) {
15908
- const wrappedAtMs = (
15909
- input.requestsForReceipt.get(receipt.resultReceiptKey) ?? []
15910
- ).find(
15911
- (request) => request.deliveryWrappedAtMs != null,
15912
- )?.deliveryWrappedAtMs;
15913
- if (wrappedAtMs == null) continue;
15914
- this.resourceGovernor.observe({
15915
- deliveryPublishWaitMs: Math.max(
15916
- 0,
15917
- enqueuedAtMs - wrappedAtMs,
15918
- ),
15919
- });
15397
+ for (const request of requestsForPublication(receipt)) {
15398
+ const wrappedAtMs = request.deliveryWrappedAtMs;
15399
+ if (wrappedAtMs == null) continue;
15400
+ this.resourceGovernor.observe({
15401
+ deliveryPublishWaitMs: Math.max(
15402
+ 0,
15403
+ enqueuedAtMs - wrappedAtMs,
15404
+ ),
15405
+ });
15406
+ }
15920
15407
  }
15921
15408
  },
15922
15409
  // The completion writer batches by arrival, so a publication wave
@@ -16011,57 +15498,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16011
15498
  : request.input,
16012
15499
  ),
16013
15500
  })),
16014
- // Claim immediately so unrelated owned rows can cross provider work
16015
- // while foreign members hydrate in the bounded lane.
16016
- eager: true,
16017
- });
16018
-
16019
- // Claims carry their actual expiry. Seed the cohort supervisor from that
16020
- // lease rather than the generic fallback, otherwise a long provider-shaped
16021
- // queue can let undispatched rows expire before its first renewal.
16022
- for (const request of requests) {
16023
- const member = cohort.member(request.callId);
16024
- if (member.kind !== 'owned') continue;
16025
- request.receiptLeaseId = member.lease.leaseId;
16026
- request.receiptLeaseExpiresAt = member.leaseExpiresAt ?? null;
16027
- }
16028
-
16029
- const earliestLeaseExpiry = requests.reduce<string | null>(
16030
- (earliest, request) => {
16031
- const expiresAt = request.receiptLeaseExpiresAt ?? null;
16032
- if (!expiresAt) return earliest;
16033
- if (!earliest) return expiresAt;
16034
- return Date.parse(expiresAt) < Date.parse(earliest)
16035
- ? expiresAt
16036
- : earliest;
16037
- },
16038
- null,
16039
- );
16040
- // One cohort-wide liveness check closes the claim-to-provider window; the
16041
- // per-call views additionally preserve the historical exact admission
16042
- // fence immediately before an individual provider request.
16043
- await cohort.heartbeat();
16044
- let cohortLeaseLost: unknown | null = null;
16045
- const heartbeat = cohort.startHeartbeat({
16046
- intervalMs: Math.max(
16047
- 1,
16048
- Math.floor(
16049
- runtimeLeaseHeartbeatIntervalFromExpiry({
16050
- leaseExpiresAt: earliestLeaseExpiry,
16051
- fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
16052
- }) / 2,
16053
- ),
16054
- ),
16055
- onLeaseLost: (error) => {
16056
- cohortLeaseLost ??= error;
16057
- },
16058
- onTransientFailure: (error) => {
16059
- this.runtimeLog(
16060
- `Tool Call Receipt Cohort heartbeat transport failed; retrying: ${
16061
- error instanceof Error ? error.message : String(error)
16062
- }`,
16063
- );
16064
- },
16065
15501
  });
16066
15502
 
16067
15503
  const successful: ToolCallRequest[] = [];
@@ -16108,35 +15544,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16108
15544
  });
16109
15545
  };
16110
15546
 
16111
- const handled = new Set<string>();
16112
- const followerResults: Promise<void>[] = [];
16113
- const collectReadyMembers = (): Array<{
15547
+ const cachedResults: Promise<void>[] = [];
15548
+ const owned: Array<{
16114
15549
  request: ToolCallRequest;
16115
15550
  receipts: ToolResultReceipts;
16116
- }> => {
16117
- const owned: Array<{
16118
- request: ToolCallRequest;
16119
- receipts: ToolResultReceipts;
16120
- }> = [];
16121
- for (const request of requests) {
16122
- if (handled.has(request.callId)) continue;
16123
- const member = cohort.member(request.callId);
16124
- handled.add(request.callId);
16125
- if (member.kind === 'completed') {
16126
- followerResults.push(resolve(request, member.result, member.source));
16127
- } else if (member.kind === 'following') {
16128
- followerResults.push(
16129
- member.result.then(
16130
- async (result) => await resolve(request, result, 'owner'),
16131
- async (error: unknown) => await reject(request, error),
16132
- ),
16133
- );
16134
- } else {
16135
- owned.push({ request, receipts: member.receipts });
16136
- }
15551
+ }> = [];
15552
+ for (const request of requests) {
15553
+ const member = cohort.member(request.callId);
15554
+ if (member.kind === 'completed') {
15555
+ cachedResults.push(resolve(request, member.result, member.source));
15556
+ } else if (member.kind === 'owned') {
15557
+ owned.push({ request, receipts: member.receipts });
15558
+ } else {
15559
+ throw new Error(
15560
+ 'A completed-result cache cannot own a pending invocation.',
15561
+ );
16137
15562
  }
16138
- return owned;
16139
- };
15563
+ }
16140
15564
 
16141
15565
  const width = Math.min(
16142
15566
  this.governor.policy.concurrency.toolCalls,
@@ -16148,85 +15572,91 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16148
15572
  ),
16149
15573
  ),
16150
15574
  );
16151
- const dispatchOwned = async (
16152
- owned: Array<{
16153
- request: ToolCallRequest;
16154
- receipts: ToolResultReceipts;
16155
- }>,
16156
- ) =>
16157
- await dispatchBoundedSettled(
16158
- owned,
16159
- width,
16160
- async ({ request, receipts }) => {
16161
- // Circuit breaker: do not dispatch this provider call once a
16162
- // persistence failure has occurred in this run. Rejected through the
16163
- // same path as a provider failure — `dispatchBoundedSettled` only
16164
- // collects a thrown error, so throwing past `reject` here would
16165
- // leave this row's tool-call resolver pending for the life of the
16166
- // map.
16167
- if (this.persistenceLatch.tripped) {
16168
- this.persistenceLatch.preventedCallCount += 1;
16169
- await reject(
16170
- request,
16171
- new RuntimePersistenceCircuitOpenError(this.persistenceLatch),
16172
- );
16173
- return;
16174
- }
16175
- let releaseToolSlot: () => void = () => undefined;
16176
- try {
16177
- const providerOperationKey = this.providerIdempotencyKeyForToolCall(
16178
- {
15575
+ await dispatchBoundedSettled(
15576
+ owned,
15577
+ width,
15578
+ async ({ request, receipts }) => {
15579
+ // Circuit breaker: do not dispatch this provider call once a
15580
+ // persistence failure has occurred in this run. Rejected through the
15581
+ // same path as a provider failure — `dispatchBoundedSettled` only
15582
+ // collects a thrown error, so throwing past `reject` here would
15583
+ // leave this row's tool-call resolver pending for the life of the
15584
+ // map.
15585
+ if (this.persistenceLatch.tripped) {
15586
+ this.persistenceLatch.preventedCallCount += 1;
15587
+ await reject(
15588
+ request,
15589
+ new RuntimePersistenceCircuitOpenError(this.persistenceLatch),
15590
+ );
15591
+ return;
15592
+ }
15593
+ let releaseToolSlot: () => void = () => undefined;
15594
+ try {
15595
+ const providerOperationKey = this.providerInvocationKeyForToolCall({
15596
+ operationKey:
15597
+ request.providerIdempotencyKeyBase ?? request.receiptKey!,
15598
+ logicalCallId: request.logicalCallId,
15599
+ force: request.force,
15600
+ });
15601
+ const result = await this.executeMappedToolCallThroughModule({
15602
+ request,
15603
+ dispatchToolId: toolId,
15604
+ dispatchPayload: request.input,
15605
+ resultReceiptKey:
15606
+ this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
15607
+ providerOperationKey,
15608
+ resultReceipts: receipts,
15609
+ apiOptions: {
15610
+ playNodeScope: playNodeScopeForToolCallRequest(request),
15611
+ durableCallReceiptKey: this.toolTransportReceiptKey({
15612
+ toolId,
15613
+ providerOperationKey,
15614
+ }),
15615
+ legacyProviderRetryIdentities: [
15616
+ {
15617
+ transportReceiptKey: request.receiptKey!,
15618
+ providerOperationKey: request.force
15619
+ ? providerOperationKey
15620
+ : this.providerOperationKeyForQueuedToolCall(request),
15621
+ ...(request.force && request.requiresExecutionFence !== true
15622
+ ? { forceRefreshIdentities: [providerOperationKey] }
15623
+ : {}),
15624
+ },
15625
+ ],
15626
+ completedResultCache: true,
15627
+ executionAuthScopeDigest: request.executionAuthScopeDigest,
15628
+ ...(request.requiresExecutionFence === true
15629
+ ? {
15630
+ providerIdempotencyKey: providerOperationKey,
15631
+ providerIdempotencyReceiptKey: request.receiptKey,
15632
+ }
15633
+ : {}),
15634
+ ...this.providerInvocationOptionsForToolCall({
16179
15635
  operationKey:
16180
15636
  request.providerIdempotencyKeyBase ?? request.receiptKey!,
16181
- force: request.force === true,
15637
+ requiresExecutionFence: request.requiresExecutionFence,
16182
15638
  logicalCallId: request.logicalCallId,
15639
+ force: request.force,
15640
+ }),
15641
+ timeoutMs:
15642
+ typeof request.timeoutMs === 'number' && request.timeoutMs > 0
15643
+ ? request.timeoutMs
15644
+ : undefined,
15645
+ retainToolSlot: (release) => {
15646
+ releaseToolSlot = release;
16183
15647
  },
16184
- );
16185
- const result = await this.executeMappedToolCallThroughModule({
16186
- request,
16187
- dispatchToolId: toolId,
16188
- dispatchPayload: request.input,
16189
- resultReceiptKey:
16190
- this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
16191
- providerOperationKey,
16192
- resultReceipts: receipts,
16193
- apiOptions: {
16194
- playNodeScope: playNodeScopeForToolCallRequest(request),
16195
- durableCallReceiptKey: this.toolTransportReceiptKey({
16196
- toolId,
16197
- resultReceiptKey: request.receiptKey!,
16198
- providerOperationKey,
16199
- force: request.force,
16200
- }),
16201
- completedResultCache: true,
16202
- executionAuthScopeDigest: request.executionAuthScopeDigest,
16203
- ...this.providerForceRefreshOptionsForToolCall({
16204
- operationKey:
16205
- request.providerIdempotencyKeyBase ?? request.receiptKey!,
16206
- force: request.force,
16207
- requiresExecutionFence: request.requiresExecutionFence,
16208
- logicalCallId: request.logicalCallId,
16209
- }),
16210
- timeoutMs:
16211
- typeof request.timeoutMs === 'number' && request.timeoutMs > 0
16212
- ? request.timeoutMs
16213
- : undefined,
16214
- retainToolSlot: (release) => {
16215
- releaseToolSlot = release;
16216
- },
16217
- },
16218
- });
16219
- await resolve(request, result, 'owner');
16220
- } catch (error) {
16221
- await reject(request, error);
16222
- } finally {
16223
- releaseToolSlot();
16224
- }
16225
- },
16226
- );
15648
+ },
15649
+ });
15650
+ await resolve(request, result, 'owner');
15651
+ } catch (error) {
15652
+ await reject(request, error);
15653
+ } finally {
15654
+ releaseToolSlot();
15655
+ }
15656
+ },
15657
+ );
16227
15658
 
16228
- await dispatchOwned(collectReadyMembers());
16229
- await Promise.allSettled(followerResults);
15659
+ await Promise.all(cachedResults);
16230
15660
  return successful;
16231
15661
  }
16232
15662
 
@@ -16398,18 +15828,109 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16398
15828
  'Native batch Tool Call operation identity changed in transit.',
16399
15829
  );
16400
15830
  }
15831
+ const legacyForceRefreshIdentities = detail.members
15832
+ .filter(
15833
+ (member) =>
15834
+ member.force === true &&
15835
+ member.requiresExecutionFence !== true,
15836
+ )
15837
+ .map((member) =>
15838
+ this.providerInvocationKeyForToolCall({
15839
+ operationKey:
15840
+ this.providerOperationKeyForQueuedToolCall(member),
15841
+ logicalCallId: member.logicalCallId,
15842
+ force: true,
15843
+ }),
15844
+ );
15845
+ const legacyProviderOperationKey =
15846
+ buildDurableToolPayloadProviderIdempotencyKey({
15847
+ orgId: this.#options.orgId,
15848
+ toolId: call.tool.id,
15849
+ executionAuthScopeDigest:
15850
+ detail.members[0]?.executionAuthScopeDigest ?? null,
15851
+ providerActionVersion: call.tool.providerActionVersion,
15852
+ payload: call.payload,
15853
+ forceRefreshIdentities: legacyForceRefreshIdentities,
15854
+ });
15855
+ const legacyFallbackForceRefreshIdentities = detail.members
15856
+ .filter((member) => member.force === true)
15857
+ .map((member) =>
15858
+ this.providerInvocationKeyForToolCall({
15859
+ operationKey:
15860
+ this.providerOperationKeyForQueuedToolCall(member),
15861
+ logicalCallId: member.logicalCallId,
15862
+ force: true,
15863
+ }),
15864
+ );
15865
+ const legacyFallbackProviderOperationKey =
15866
+ buildDurableToolPayloadProviderIdempotencyKey({
15867
+ orgId: this.#options.orgId,
15868
+ toolId: call.tool.id,
15869
+ executionAuthScopeDigest:
15870
+ detail.members[0]?.executionAuthScopeDigest ?? null,
15871
+ providerActionVersion:
15872
+ (
15873
+ await this.#options.getToolActionCacheVersion?.(
15874
+ call.tool.id,
15875
+ )
15876
+ )?.trim() ?? 'unversioned',
15877
+ payload: call.payload,
15878
+ forceRefreshIdentities:
15879
+ legacyFallbackForceRefreshIdentities,
15880
+ });
16401
15881
  const execution = await this.callToolExecutionAPI(
16402
15882
  call.tool.id,
16403
15883
  call.payload,
16404
15884
  {
16405
15885
  durableCallReceiptKey: this.toolTransportReceiptKey({
16406
15886
  toolId: call.tool.id,
16407
- resultReceiptKey: compatibility.resultReceiptKey,
16408
15887
  providerOperationKey: detail.providerOperationKey,
16409
- force: detail.members.some(
16410
- (request) => request.force === true,
16411
- ),
16412
15888
  }),
15889
+ legacyProviderRetryIdentities: [
15890
+ {
15891
+ transportReceiptKey: detail.members.some(
15892
+ (member) => member.force,
15893
+ )
15894
+ ? this.toolTransportReceiptKey({
15895
+ toolId: call.tool.id,
15896
+ providerOperationKey:
15897
+ legacyProviderOperationKey,
15898
+ })
15899
+ : buildDurableToolAggregateReceiptKey({
15900
+ receiptKeys: detail.members.map(
15901
+ (member) =>
15902
+ this.mappedToolResultReceiptPolicy(member)
15903
+ .resultReceiptKey,
15904
+ ),
15905
+ prefix: 'batch',
15906
+ aggregateReceiptPrefix:
15907
+ buildDurableToolReceiptPrefix({
15908
+ orgId: this.#options.orgId,
15909
+ toolId: call.tool.id,
15910
+ }),
15911
+ }),
15912
+ providerOperationKey: legacyProviderOperationKey,
15913
+ forceRefreshIdentities: legacyForceRefreshIdentities,
15914
+ },
15915
+ {
15916
+ transportReceiptKey:
15917
+ buildDurableToolAggregateReceiptKey({
15918
+ receiptKeys: detail.members.map(
15919
+ (member) => member.cacheKey,
15920
+ ),
15921
+ prefix: 'batch',
15922
+ aggregateReceiptPrefix:
15923
+ buildDurableToolReceiptPrefix({
15924
+ orgId: this.#options.orgId,
15925
+ toolId: call.tool.id,
15926
+ }),
15927
+ }),
15928
+ providerOperationKey:
15929
+ legacyFallbackProviderOperationKey,
15930
+ forceRefreshIdentities:
15931
+ legacyFallbackForceRefreshIdentities,
15932
+ },
15933
+ ],
16413
15934
  completedResultCache: true,
16414
15935
  memoryLedgerRowScopes: detail.members.map((member) => ({
16415
15936
  rowKey: member.rowKey,
@@ -16426,6 +15947,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16426
15947
  ),
16427
15948
  executionAuthScopeDigest:
16428
15949
  detail.members[0]?.executionAuthScopeDigest ?? null,
15950
+ providerIdempotencyKey: detail.providerOperationKey,
15951
+ providerIdempotencyReceiptKey:
15952
+ compatibility.resultReceiptKey,
16429
15953
  ...(detail.forceRefreshIdentities.length > 0
16430
15954
  ? {
16431
15955
  providerOperationForceRefreshIdentities:
@@ -16535,25 +16059,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16535
16059
  compiled.batchOperation,
16536
16060
  )
16537
16061
  )?.trim() ?? '';
16538
- const receiptKeys = owned.map(
16539
- (request) =>
16540
- this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
16541
- );
16542
- const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({
16543
- receiptKeys,
16544
- prefix: 'batch',
16545
- aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
16546
- orgId: this.#options.orgId,
16547
- toolId: compiled.batchOperation,
16548
- }),
16549
- });
16550
- const forceRefreshIdentities = owned.flatMap((request) =>
16551
- this.providerForceRefreshIdentitiesForToolCall({
16062
+ const forceRefreshIdentities = owned.map((request) =>
16063
+ this.providerInvocationKeyForToolCall({
16552
16064
  operationKey:
16553
16065
  this.providerOperationKeyForQueuedToolCall(request),
16554
- force: request.force,
16555
- requiresExecutionFence: request.requiresExecutionFence,
16556
16066
  logicalCallId: request.logicalCallId,
16067
+ force: request.force,
16557
16068
  }),
16558
16069
  );
16559
16070
  const providerOperationKey =
@@ -16566,6 +16077,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16566
16077
  payload: compiled.batchPayload,
16567
16078
  forceRefreshIdentities,
16568
16079
  });
16080
+ const aggregateReceiptKey = this.toolTransportReceiptKey({
16081
+ toolId: compiled.batchOperation,
16082
+ providerOperationKey,
16083
+ });
16569
16084
  dispatchDetails.set(aggregateReceiptKey, {
16570
16085
  members: owned,
16571
16086
  compiled,
@@ -16659,6 +16174,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16659
16174
  toolId,
16660
16175
  status: aggregate.status,
16661
16176
  result: item,
16177
+ meta: nativeBatchResponseMeta(item),
16662
16178
  toolResponse: publicToolResponseForBatchedItem(
16663
16179
  aggregate,
16664
16180
  item,
@@ -16825,8 +16341,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16825
16341
  };
16826
16342
 
16827
16343
  const pendingRequests: ToolCallRequest[] = [];
16828
- let pendingReceiptClaimsEstablishExecutionFence =
16829
- this.#options.runtimeReceiptClaimsEstablishExecutionFence === true;
16830
16344
  const recoveredRequests: ToolCallRequest[] = [];
16831
16345
  for (const req of requests) {
16832
16346
  const cached =
@@ -16851,114 +16365,27 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16851
16365
  }
16852
16366
  recordToolStep(recoveredRequests);
16853
16367
 
16854
- const liveFollowersByOwnerCallId = new Map<string, ToolCallRequest[]>();
16855
- const resolveLiveFollowers = (
16856
- owner: ToolCallRequest,
16857
- result: unknown,
16858
- ): void => {
16859
- const followers = liveFollowersByOwnerCallId.get(owner.callId);
16860
- if (!followers || followers.length === 0) return;
16861
- liveFollowersByOwnerCallId.delete(owner.callId);
16862
- for (const follower of followers) {
16863
- const followerReceiptKey = follower.receiptKey?.trim() || null;
16864
- const followerResult = followerReceiptKey
16865
- ? markToolExecuteResultExecutionOutcome(result, {
16866
- kind: 'in_flight',
16867
- receiptKey: followerReceiptKey,
16868
- attachedToReceiptKey: followerReceiptKey,
16869
- })
16870
- : result;
16871
- if (follower.cacheable !== false) {
16872
- this.cacheToolResult(toolId, follower.cacheKey, followerResult);
16873
- }
16874
- const resolver = this.toolCallResolvers.get(follower.callId);
16875
- if (resolver) {
16876
- resolver.resolve(followerResult);
16877
- this.toolCallResolvers.delete(follower.callId);
16878
- }
16879
- this.emitScopedFieldMetaUpdate({
16880
- rowId: follower.rowId,
16881
- key: follower.rowKey ?? null,
16882
- tableNamespace: follower.tableNamespace ?? null,
16883
- fieldName: follower.fieldName,
16884
- status: 'cached',
16885
- rowStatus: 'running',
16886
- stage: toolId,
16887
- provider: null,
16888
- error: null,
16889
- reused: true,
16890
- dataPatch: {},
16891
- });
16892
- }
16893
- };
16894
- const rejectWithLiveFollowers = async (
16895
- owner: ToolCallRequest,
16368
+ const rejectCall = async (
16369
+ request: ToolCallRequest,
16896
16370
  error: unknown,
16897
16371
  ): Promise<void> => {
16898
- const followers = liveFollowersByOwnerCallId.get(owner.callId) ?? [];
16899
- liveFollowersByOwnerCallId.delete(owner.callId);
16900
- if (error instanceof PlayRowProviderBackpressureError) {
16901
- let releaseFailure: unknown = null;
16902
- const receiptKey = owner.receiptKey?.trim() || null;
16903
- try {
16904
- if (receiptKey && this.durableMappedToolResultsBackedByReceipts) {
16905
- const leaseId = owner.receiptLeaseId?.trim();
16906
- if (!leaseId) {
16907
- throw new Error(
16908
- `Provider wait cannot release receipt ${receiptKey} without its owned lease.`,
16909
- );
16910
- }
16911
- const released = await this.releaseRuntimeStepReceipt(
16912
- receiptKey,
16913
- this.currentReceiptOwnerRunId,
16914
- leaseId,
16915
- );
16916
- if (
16917
- !released ||
16918
- (released.status !== 'queued' &&
16919
- released.status !== 'pending')
16920
- ) {
16921
- throw new Error(
16922
- `Provider wait could not release owned receipt ${receiptKey}.`,
16923
- );
16924
- }
16925
- }
16926
- } catch (failure) {
16927
- releaseFailure = failure;
16928
- }
16929
- // A provider deadline is a nonterminal control signal. The row
16930
- // catch records waiting metadata after the resolver unwinds; no
16931
- // follower may see or persist a failed receipt in between. If the
16932
- // lease could not be released, wake every follower with that loud
16933
- // infrastructure failure rather than stranding their promises.
16934
- for (const request of [owner, ...followers]) {
16935
- const resolver = this.toolCallResolvers.get(request.callId);
16936
- if (resolver) {
16937
- resolver.reject(releaseFailure ?? error);
16938
- this.toolCallResolvers.delete(request.callId);
16939
- }
16940
- }
16941
- if (releaseFailure) throw releaseFailure;
16942
- return;
16943
- }
16944
- const shouldPersistReceiptFailure =
16945
- !(error instanceof ToolExecuteAuthScopeChangedError) &&
16946
- !this.persistenceLatch.tripped;
16947
- for (const request of [owner, ...followers]) {
16948
- await this.rejectToolCall(toolId, request, error, {
16949
- persistReceiptFailure: shouldPersistReceiptFailure,
16950
- });
16951
- }
16372
+ await this.rejectToolCall(toolId, request, error, {
16373
+ persistReceiptFailure: false,
16374
+ });
16952
16375
  };
16953
- const requestsWithLiveFollowers = (
16954
- owner: ToolCallRequest,
16955
- ): ToolCallRequest[] => [
16956
- owner,
16957
- ...(liveFollowersByOwnerCallId.get(owner.callId) ?? []),
16958
- ];
16959
16376
  const resolveRuntimeTimeoutMsForRequests = (
16960
16377
  requests: ToolCallRequest[],
16961
16378
  ): number | undefined => {
16379
+ if (
16380
+ requests.some(
16381
+ (request) =>
16382
+ typeof request.timeoutMs !== 'number' ||
16383
+ !Number.isFinite(request.timeoutMs) ||
16384
+ request.timeoutMs <= 0,
16385
+ )
16386
+ ) {
16387
+ return undefined;
16388
+ }
16962
16389
  const timeoutMs = Math.max(
16963
16390
  ...requests
16964
16391
  .map((request) => request.timeoutMs)
@@ -16972,76 +16399,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
16972
16399
  );
16973
16400
  return timeoutMs > 0 ? timeoutMs : undefined;
16974
16401
  };
16975
- const resolveRuntimeTimeoutMsForClaimedOwners = (
16976
- owners: ToolCallRequest[],
16977
- ): number | undefined => {
16978
- let timeoutMs = 0;
16979
- for (const owner of owners) {
16980
- if (
16981
- typeof owner.timeoutMs !== 'number' ||
16982
- !Number.isFinite(owner.timeoutMs) ||
16983
- owner.timeoutMs <= 0
16984
- ) {
16985
- return undefined;
16986
- }
16987
- timeoutMs = Math.max(
16988
- timeoutMs,
16989
- resolveRuntimeTimeoutMsForRequests(
16990
- requestsWithLiveFollowers(owner),
16991
- ) ?? 0,
16992
- );
16993
- }
16994
- return timeoutMs > 0 ? timeoutMs : undefined;
16995
- };
16996
-
16997
- // Cacheable map members cross the real receipt-cohort boundary before
16998
- // the legacy claim loop. The scalar and native Modules share member
16999
- // receipt ownership; native batches retain their separate aggregate
17000
- // physical-operation receipt behind that seam. Always-fresh calls
17001
- // intentionally have no durable member receipt and stay below.
16402
+ // The cohort reads completed results and publishes successful misses.
16403
+ // Always-fresh calls have no durable member result and stay below.
17002
16404
  const operationStrategy =
17003
16405
  this.#options.getBatchOperationStrategy?.(toolId) ?? null;
17004
- if (
17005
- operationStrategy &&
17006
- this.durableMappedToolResultsBackedByReceipts
17007
- ) {
17008
- const cohortRequests = pendingRequests.filter(
17009
- (request) =>
17010
- request.cacheable !== false &&
17011
- request.requiresExecutionFence !== true &&
17012
- Boolean(request.receiptKey?.trim()),
17013
- );
17014
- if (cohortRequests.length > 0) {
17015
- const cohortCallIds = new Set(
17016
- cohortRequests.map((request) => request.callId),
17017
- );
17018
- pendingRequests.splice(
17019
- 0,
17020
- pendingRequests.length,
17021
- ...pendingRequests.filter(
17022
- (request) => !cohortCallIds.has(request.callId),
17023
- ),
17024
- );
17025
- const successful =
17026
- await this.executeCacheableNativeMapToolCallCohort({
17027
- toolId,
17028
- requests: cohortRequests,
17029
- strategy: operationStrategy,
17030
- });
17031
- for (const request of successful) {
17032
- successfulLiveStepCallIds.add(request.callId);
17033
- }
17034
- recordToolStep(cohortRequests);
17035
- this.#options.onBatchComplete?.(this.checkpoint);
17036
- }
17037
- } else if (
17038
- !operationStrategy &&
17039
- this.durableMappedToolResultsBackedByReceipts
17040
- ) {
16406
+ if (this.durableMappedToolResultsBackedByReceipts) {
17041
16407
  const cohortRequests = pendingRequests.filter(
17042
16408
  (request) =>
17043
16409
  request.cacheable !== false &&
17044
- request.requiresExecutionFence !== true &&
17045
16410
  Boolean(request.receiptKey?.trim()),
17046
16411
  );
17047
16412
  if (cohortRequests.length > 0) {
@@ -17055,1181 +16420,366 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
17055
16420
  (request) => !cohortCallIds.has(request.callId),
17056
16421
  ),
17057
16422
  );
17058
- const successful =
17059
- await this.executeCacheableNonNativeMapToolCallCohort({
17060
- toolId,
17061
- requests: cohortRequests,
17062
- });
17063
- for (const request of successful) {
16423
+ const successful = operationStrategy
16424
+ ? await this.executeCacheableNativeMapToolCallCohort({
16425
+ toolId,
16426
+ requests: cohortRequests,
16427
+ strategy: operationStrategy,
16428
+ })
16429
+ : await this.executeCacheableNonNativeMapToolCallCohort({
16430
+ toolId,
16431
+ requests: cohortRequests,
16432
+ });
16433
+ for (const request of successful)
17064
16434
  successfulLiveStepCallIds.add(request.callId);
17065
- }
17066
16435
  recordToolStep(cohortRequests);
17067
16436
  this.#options.onBatchComplete?.(this.checkpoint);
17068
16437
  }
17069
16438
  }
17070
16439
 
17071
- const durableExistingRunningWaits: Array<{
17072
- receiptKey: string;
17073
- requestsForKey: ToolCallRequest[];
17074
- forceAfterWait: boolean;
17075
- waitMaxAttempts?: number;
17076
- /** Reclaim without polling: the prior owner's lease already expired. */
17077
- skipWait?: boolean;
17078
- }> = [];
17079
- const durableExistingRunningState: {
17080
- settlement: Promise<
17081
- { status: 'fulfilled' } | { status: 'rejected'; reason: unknown }
17082
- > | null;
17083
- } = { settlement: null };
17084
- const durableExistingRunningAbort = new AbortController();
17085
- // The group's size before the turn. The turn body splices and pushes
17086
- // `pendingRequests`, so the count read here is the only one that says
17087
- // how many queued calls this claim turn is blocking.
17088
- const claimTurnLogicalCalls = pendingRequests.length;
17089
- if (
17090
- claimTurnLogicalCalls > 0 &&
17091
- (this.#options.claimRuntimeStepReceipts ||
17092
- this.#options.claimRuntimeStepReceipt)
17093
- ) {
17094
- const requestsByReceiptKey = new Map<
17095
- string,
17096
- {
17097
- requests: ToolCallRequest[];
17098
- forceRefresh: boolean;
17099
- forceFailedRefresh: boolean;
17100
- }
17101
- >();
17102
- const localOnlyRequests: ToolCallRequest[] = [];
17103
- for (const request of pendingRequests) {
17104
- const receiptKey = request.receiptKey?.trim() || null;
17105
- if (!receiptKey) {
17106
- localOnlyRequests.push(request);
17107
- continue;
17108
- }
17109
- const group = requestsByReceiptKey.get(receiptKey) ?? {
17110
- requests: [],
17111
- forceRefresh: false,
17112
- forceFailedRefresh: false,
17113
- };
17114
- group.requests.push(request);
17115
- group.forceRefresh ||= request.force === true;
17116
- group.forceFailedRefresh ||= request.forceFailedRefresh === true;
17117
- requestsByReceiptKey.set(receiptKey, group);
17118
- }
17119
- const normalReceiptKeys = [...requestsByReceiptKey]
17120
- .filter(
17121
- ([, group]) => !group.forceRefresh && !group.forceFailedRefresh,
17122
- )
17123
- .map(([receiptKey]) => receiptKey);
17124
- const forcedReceiptKeys = [...requestsByReceiptKey]
17125
- .filter(([, group]) => group.forceRefresh)
17126
- .map(([receiptKey]) => receiptKey);
17127
- const failedRefreshReceiptKeys = [...requestsByReceiptKey]
17128
- .filter(
17129
- ([, group]) => !group.forceRefresh && group.forceFailedRefresh,
17130
- )
17131
- .map(([receiptKey]) => receiptKey);
17132
- const claimStartedAt = Date.now();
17133
- const forcedExisting = forcedReceiptKeys.length
17134
- ? await this.getRuntimeStepReceipts(forcedReceiptKeys, {
17135
- hydrateOutput: false,
17136
- })
17137
- : new Map<string, RuntimeStepReceipt>();
17138
- // A forced refresh must never duplicate a provider call owned by a
17139
- // live lease. It can, however, immediately claim pending/unleased
17140
- // work and any expired lease. Waiting first for those receipts turns
17141
- // an already-recoverable interruption into a five-minute stall.
17142
- const forcedLiveReceiptKeys = new Set(
17143
- [...forcedExisting]
17144
- .filter(([, receipt]) => {
17145
- if (
17146
- receipt.status !== 'queued' &&
17147
- receipt.status !== 'pending' &&
17148
- receipt.status !== 'running'
17149
- ) {
17150
- return false;
17151
- }
17152
- if (!receipt.leaseId?.trim()) return false;
17153
- const expiresAt = receipt.leaseExpiresAt
17154
- ? Date.parse(receipt.leaseExpiresAt)
17155
- : Number.NaN;
17156
- // Keep an unparseable leased receipt conservative. The
17157
- // fenced claim remains the recovery path after the wait.
17158
- return !Number.isFinite(expiresAt) || expiresAt > Date.now();
17159
- })
17160
- .map(([receiptKey]) => receiptKey),
17161
- );
17162
- const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
17163
- (receiptKey) => !forcedLiveReceiptKeys.has(receiptKey),
17164
- );
17165
- const claims =
17166
- normalReceiptKeys.length > 0
17167
- ? await this.claimRuntimeStepReceipts(
17168
- normalReceiptKeys,
17169
- this.currentReceiptOwnerRunId,
17170
- false,
17171
- false,
17172
- false,
17173
- false,
17174
- )
17175
- : new Map<string, RuntimeStepReceipt>();
17176
- const forcedClaims =
17177
- claimableForcedReceiptKeys.length > 0
17178
- ? await this.claimRuntimeStepReceipts(
17179
- claimableForcedReceiptKeys,
17180
- this.currentReceiptOwnerRunId,
17181
- false,
17182
- true,
17183
- false,
17184
- )
17185
- : new Map<string, RuntimeStepReceipt>();
17186
- const failedRefreshClaims =
17187
- failedRefreshReceiptKeys.length > 0
17188
- ? await this.claimRuntimeStepReceipts(
17189
- failedRefreshReceiptKeys,
17190
- this.currentReceiptOwnerRunId,
17191
- false,
17192
- false,
17193
- true,
17194
- false,
17195
- )
17196
- : new Map<string, RuntimeStepReceipt>();
17197
- for (const [receiptKey, claim] of forcedClaims) {
17198
- claims.set(receiptKey, claim);
17199
- }
17200
- for (const [receiptKey, claim] of failedRefreshClaims) {
17201
- claims.set(receiptKey, claim);
17202
- }
17203
- const safeFailedReceiptKeys = [...claims]
17204
- .filter(([, claim]) => this.canRetryWithoutProviderDispatch(claim))
17205
- .map(([receiptKey]) => receiptKey);
17206
- if (safeFailedReceiptKeys.length > 0) {
17207
- const refreshed = await this.claimRuntimeStepReceipts(
17208
- safeFailedReceiptKeys,
17209
- this.currentReceiptOwnerRunId,
17210
- false,
17211
- false,
17212
- true,
17213
- false,
17214
- );
17215
- for (const [receiptKey, claim] of refreshed) {
17216
- claims.set(receiptKey, claim);
17217
- }
17218
- }
17219
- // Claiming is a state transition, not a reason to expand a
17220
- // body. Fresh receipts contain no result, yet the old path took
17221
- // the hydration lane before it learned that fact and charged a
17222
- // multi-MB row prior to every one. Claims now arrive as bounded
17223
- // manifests; only a terminal winner redeems its known-size body
17224
- // under a second, byte-priced turn.
17225
- const terminalClaimKeys = [...claims]
17226
- .filter(
17227
- ([, claim]) =>
17228
- (claim.status === 'completed' || claim.status === 'skipped') &&
17229
- claim.output === undefined,
17230
- )
17231
- .map(([receiptKey]) => receiptKey);
17232
- if (terminalClaimKeys.length > 0) {
17233
- const hasBoundedClaimManifests = terminalClaimKeys.every(
17234
- (receiptKey) => {
17235
- const claim = claims.get(receiptKey);
17236
- return (
17237
- Number.isSafeInteger(claim?.outputByteLength) &&
17238
- (claim?.outputByteLength ?? -1) >= 0
17239
- );
17240
- },
17241
- );
17242
- const expectedClaimBytes = hasBoundedClaimManifests
17243
- ? terminalClaimKeys.reduce(
17244
- (total, receiptKey) =>
17245
- total + (claims.get(receiptKey)?.outputByteLength ?? 0),
17246
- 0,
17247
- )
17248
- : undefined;
17249
- const hydratedClaims = await this.withRuntimeReceiptHydrationTurn(
17250
- async (hydrationTurn) => {
17251
- const receipts =
17252
- await this.getRuntimeStepReceipts(terminalClaimKeys);
17253
- hydrationTurn.settleBytes(
17254
- runtimeStepReceiptPayloadBytes(receipts.values()),
17255
- );
17256
- return receipts;
17257
- },
17258
- {
17259
- logicalCalls: claimTurnLogicalCalls,
17260
- receiptKeys: terminalClaimKeys.length,
17261
- ...(expectedClaimBytes !== undefined
17262
- ? { expectedBytes: expectedClaimBytes }
17263
- : {}),
17264
- source: 'map_legacy_claim',
17265
- },
17266
- );
17267
- for (const [receiptKey, receipt] of hydratedClaims) {
17268
- claims.set(receiptKey, receipt);
17269
- }
17270
- }
17271
- // Every get/claim round trip this dispatch group paid before any
17272
- // provider work could start.
17273
- this.resourceGovernor.observe({
17274
- receiptClaimMs: Date.now() - claimStartedAt,
17275
- });
17276
- // Claims are compact state transitions. They do not own output
17277
- // bytes and must not wait behind the hydration lane's global
17278
- // memory headroom; a terminal manifest above is the only path
17279
- // that opens a byte-admitted hydration turn.
17280
- const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
17281
- const durableRecoveredRequests: ToolCallRequest[] = [];
17282
- const resolveRequestsFromReceipt = async (
17283
- requestsForKey: ToolCallRequest[],
17284
- receipt: RuntimeStepReceipt,
17285
- source: 'cache' | 'in_flight',
17286
- ): Promise<void> => {
17287
- const request = requestsForKey[0];
17288
- if (!request) return;
17289
- const receiptKey = request.receiptKey?.trim() || null;
17290
- if (!receiptKey) {
17291
- throw new Error(`${source} tool recovery needs receipt key.`);
17292
- }
17293
- const wrapped = await this.wrapToolExecutionResult({
17294
- toolId,
17295
- status:
17296
- receipt.output === null || receipt.output === undefined
17297
- ? 'no_result'
17298
- : 'completed',
17299
- result: this.runtimeReceiptOutput(receipt),
17300
- requestInput: request.input,
17301
- execution: toolExecutionMetadataForOutcome({
17302
- kind: source,
17303
- cacheKey: request.cacheKey,
17304
- receiptKey,
17305
- attachedToReceiptKey: receiptKey,
17306
- }),
17307
- });
17308
- this.cacheToolResult(toolId, request.cacheKey, wrapped);
17309
- for (const waitingRequest of requestsForKey) {
17310
- successfulLiveStepCallIds.add(waitingRequest.callId);
17311
- const resolver = this.toolCallResolvers.get(
17312
- waitingRequest.callId,
17313
- );
17314
- if (resolver) {
17315
- resolver.resolve(wrapped);
17316
- this.toolCallResolvers.delete(waitingRequest.callId);
17317
- }
17318
- this.emitScopedFieldMetaUpdate({
17319
- rowId: waitingRequest.rowId,
17320
- key: waitingRequest.rowKey ?? null,
17321
- tableNamespace: waitingRequest.tableNamespace ?? null,
17322
- fieldName: waitingRequest.fieldName,
17323
- status: 'cached',
17324
- rowStatus: 'running',
17325
- stage: toolId,
17326
- provider: null,
17327
- error: null,
17328
- reused: true,
17329
- dataPatch: {},
17330
- });
17331
- }
17332
- };
17333
- const claimOwnerWithLiveFollowers = (
17334
- owner: ToolCallRequest | undefined,
17335
- followers: ToolCallRequest[],
17336
- ): void => {
17337
- if (!owner) return;
17338
- claimedRequests.push(owner);
17339
- if (followers.length > 0) {
17340
- liveFollowersByOwnerCallId.set(owner.callId, followers);
17341
- }
17342
- };
17343
- const resolveReceiptWaitMaxAttempts = (
17344
- requestsForKey: ToolCallRequest[],
17345
- ): number | undefined =>
17346
- requestsForKey.length > 0
17347
- ? Math.max(
17348
- ...requestsForKey.map((request) =>
17349
- resolveRuntimeToolReceiptWaitMaxAttempts(
17350
- typeof request.receiptWaitMs === 'number'
17351
- ? { max_wait_ms: request.receiptWaitMs }
17352
- : request.input,
17353
- ),
17354
- ),
17355
- )
17356
- : undefined;
17357
- const queueExistingRunningReceiptWait = (
17358
- receiptKey: string,
17359
- requestsForKey: ToolCallRequest[],
17360
- forceAfterWait = false,
17361
- skipWait = false,
17362
- ): void => {
17363
- durableExistingRunningWaits.push({
17364
- receiptKey,
17365
- requestsForKey,
17366
- forceAfterWait,
17367
- waitMaxAttempts: resolveReceiptWaitMaxAttempts(requestsForKey),
17368
- skipWait,
16440
+ if (pendingRequests.length > 0) {
16441
+ const strategy = operationStrategy;
16442
+
16443
+ if (strategy) {
16444
+ const compiledBatches = compileRequestsWithStrategy({
16445
+ requests: pendingRequests,
16446
+ strategy,
16447
+ getPayload: (request: ToolCallRequest) => request.input,
17369
16448
  });
17370
- };
17371
- const reclaimExistingRunningReceiptAfterWait = async (
17372
- receiptKey: string,
17373
- requestsForKey: ToolCallRequest[],
17374
- forceAfterWait = false,
17375
- waitResult?: {
17376
- completedReceipt?: RuntimeStepReceipt;
17377
- skipWait?: boolean;
17378
- },
17379
- ): Promise<void> => {
17380
- if (waitResult?.completedReceipt) {
17381
- if (!forceAfterWait) {
17382
- await resolveRequestsFromReceipt(
17383
- requestsForKey,
17384
- waitResult.completedReceipt,
17385
- 'in_flight',
17386
- );
17387
- return;
17388
- }
17389
- } else if (waitResult?.skipWait !== true) {
17390
- const waitMaxAttempts =
17391
- resolveReceiptWaitMaxAttempts(requestsForKey);
17392
- try {
17393
- const completed = await this.waitForCompletedRuntimeToolReceipt(
17394
- receiptKey,
17395
- waitMaxAttempts,
17396
- );
17397
- if (!forceAfterWait) {
17398
- await resolveRequestsFromReceipt(
17399
- requestsForKey,
17400
- completed,
17401
- 'in_flight',
17402
- );
17403
- return;
17404
- }
17405
- } catch (error) {
17406
- if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
17407
- for (const request of requestsForKey) {
17408
- await this.rejectToolCall(toolId, request, error, {
17409
- persistReceiptFailure: false,
17410
- });
17411
- }
17412
- return;
17413
- }
17414
- }
17415
- }
17416
- const reclaimed = (
17417
- await this.claimRuntimeStepReceipts(
17418
- [receiptKey],
17419
- this.currentReceiptOwnerRunId,
17420
- true,
17421
- forceAfterWait,
17422
- )
17423
- ).get(receiptKey);
17424
- if (
17425
- reclaimed?.status === 'completed' ||
17426
- reclaimed?.status === 'skipped'
17427
- ) {
17428
- await resolveRequestsFromReceipt(
17429
- requestsForKey,
17430
- reclaimed,
17431
- 'cache',
17432
- );
17433
- durableRecoveredRequests.push(...requestsForKey);
17434
- return;
17435
- }
17436
- if (reclaimed?.status === 'failed') {
17437
- for (const request of requestsForKey) {
17438
- await this.rejectToolCall(
17439
- toolId,
17440
- request,
17441
- runtimeReceiptFailureError(
17442
- reclaimed,
17443
- 'Durable tool call failed',
17444
- this.currentToolErrorSchemaVersion,
17445
- ),
17446
- );
17447
- }
17448
- return;
17449
- }
17450
- if (this.isOwnedClaimedRuntimeReceipt(reclaimed)) {
17451
- const [owner, ...waiters] = requestsForKey;
17452
- if (!owner) return;
17453
- if (!reclaimed.leaseId) {
17454
- throw new RuntimeReceiptLeaseLostError({
17455
- receiptKey,
17456
- runId: this.currentReceiptOwnerRunId,
17457
- leaseId: 'missing',
17458
- });
17459
- }
17460
- owner.receiptLeaseId = reclaimed.leaseId ?? null;
17461
- owner.receiptLeaseExpiresAt = reclaimed.leaseExpiresAt ?? null;
17462
- if (waiters.length > 0) {
17463
- liveFollowersByOwnerCallId.set(owner.callId, waiters);
17464
- }
17465
- try {
17466
- // Circuit breaker: a receipt reclaimed after its owner's lease
17467
- // lapsed is still a fresh provider call, and the latch may
17468
- // have tripped while this group waited for that lease.
16449
+
16450
+ const batchParallelismCeiling =
16451
+ this.governor.policy.pacing.workerToolBatchDefaultParallelism;
16452
+ const batchSize =
16453
+ compiledBatches.length > 0
16454
+ ? await this.resourceGovernor.suggestedToolParallelism(
16455
+ compiledBatches[0]!.batchOperation,
16456
+ batchParallelismCeiling,
16457
+ )
16458
+ : batchParallelismCeiling;
16459
+ await executeChunkedRequests({
16460
+ requests: compiledBatches,
16461
+ batchSize,
16462
+ execute: async (batch) => {
16463
+ // Circuit breaker: skip dispatching this batch's provider call
16464
+ // once a persistence failure has occurred in this run.
17469
16465
  if (this.persistenceLatch.tripped) {
17470
16466
  this.persistenceLatch.preventedCallCount +=
17471
- requestsForKey.length;
16467
+ batch.memberRequests.length;
17472
16468
  throw new RuntimePersistenceCircuitOpenError(
17473
16469
  this.persistenceLatch,
17474
16470
  );
17475
16471
  }
17476
- const execution = await this.executeMappedToolCallThroughModule(
17477
- {
17478
- request: owner,
17479
- dispatchToolId: toolId,
17480
- dispatchPayload: owner.input,
17481
- resultReceiptKey: receiptKey,
17482
- providerOperationKey:
17483
- this.providerOperationKeyForQueuedToolCall(owner),
17484
- apiOptions: {
17485
- beforeProviderCall: () =>
17486
- this.assertRuntimeToolReceiptOwnership([owner]),
17487
- durableCallReceiptKey: receiptKey,
17488
- playNodeScope: playNodeScopeForToolCallRequest(owner),
17489
- executionAuthScopeDigest: owner.executionAuthScopeDigest,
17490
- receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
17491
- heartbeatReceipt: () =>
17492
- this.renewRuntimeToolReceiptOwnership([owner]),
17493
- providerIdempotencyKey:
17494
- this.providerIdempotencyKeyForToolCall({
17495
- operationKey:
17496
- this.providerOperationKeyForQueuedToolCall(owner),
17497
- force: owner.force === true,
17498
- logicalCallId: owner.logicalCallId,
17499
- }),
17500
- ...this.providerForceRefreshOptionsForToolCall({
17501
- operationKey:
17502
- this.providerOperationKeyForQueuedToolCall(owner),
17503
- force: owner.force,
17504
- requiresExecutionFence: owner.requiresExecutionFence,
17505
- logicalCallId: owner.logicalCallId,
17506
- }),
17507
- timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
17508
- owner,
17509
- ]),
17510
- },
17511
- },
17512
- );
17513
- const result = await this.resolveToolCall(
17514
- toolId,
17515
- owner,
17516
- execution,
17517
- );
17518
- if (result != null) {
17519
- successfulLiveStepCallIds.add(owner.callId);
17520
- }
17521
- resolveLiveFollowers(owner, result);
17522
- recordToolStep([owner]);
17523
- this.#options.onBatchComplete?.(this.checkpoint);
17524
- } catch (error) {
17525
- await rejectWithLiveFollowers(owner, error);
17526
- }
17527
- return;
17528
- }
17529
- for (const request of requestsForKey) {
17530
- await this.rejectToolCall(
17531
- toolId,
17532
- request,
17533
- new RuntimeReceiptWaitTimeoutError(receiptKey),
17534
- { persistReceiptFailure: false },
17535
- );
17536
- }
17537
- return;
17538
- };
17539
- const reclaimExistingRunningReceipts = async (
17540
- waits: Array<(typeof durableExistingRunningWaits)[number]>,
17541
- ): Promise<void> => {
17542
- if (waits.length === 0) return;
17543
- const settlements = await Promise.allSettled(
17544
- waits.map((wait) =>
17545
- reclaimExistingRunningReceiptAfterWait(
17546
- wait.receiptKey,
17547
- wait.requestsForKey,
17548
- wait.forceAfterWait,
17549
- { skipWait: true },
17550
- ),
17551
- ),
17552
- );
17553
- const rejected = settlements.find(
17554
- (settlement) => settlement.status === 'rejected',
17555
- );
17556
- if (rejected?.status === 'rejected') throw rejected.reason;
17557
- };
17558
- const processExistingRunningReceiptWaits =
17559
- async (): Promise<void> => {
17560
- if (durableExistingRunningWaits.length === 0) return;
17561
- const waitSignal = durableExistingRunningAbort.signal;
17562
- // A claim that came back `existing` behind an expired lease is
17563
- // not blocked on live work; it is holding a dead run's row.
17564
- // Reclaim those now so this run executes the call itself, and
17565
- // poll only the leases that are genuinely still alive.
17566
- const expiredLeaseWaits = durableExistingRunningWaits.filter(
17567
- (wait) => wait.skipWait === true,
17568
- );
17569
- const pendingWaits = new Map(
17570
- durableExistingRunningWaits
17571
- .filter((wait) => wait.skipWait !== true)
17572
- .map((wait) => [wait.receiptKey, wait]),
17573
- );
17574
- const parkedAtMs = Date.now();
17575
- for (const wait of durableExistingRunningWaits) {
17576
- this.#runtimeReceiptWaitParking.set(wait.receiptKey, {
17577
- startedAtMs: parkedAtMs,
17578
- rows: wait.requestsForKey.length,
17579
- stage:
17580
- wait.skipWait === true
17581
- ? 'expired_lease_reclaim'
17582
- : 'receipt_lease_wait',
17583
- });
17584
- }
17585
- const pollWaits = async (): Promise<void> => {
17586
- if (pendingWaits.size === 0) return;
17587
- const waitMaxAttempts =
17588
- Math.max(
17589
- ...[...pendingWaits.values()].map(
17590
- (wait) => wait.waitMaxAttempts ?? 0,
17591
- ),
17592
- ) || DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
17593
- // The backoff cap means the attempt count no longer states a
17594
- // wall time. This deadline is the budget those attempts were
17595
- // meant to express; crossing it falls through to the same
17596
- // reclaim path an expired lease takes.
17597
- const waitDeadline =
17598
- Date.now() +
17599
- runtimeReceiptWaitBudgetMs(
17600
- waitMaxAttempts,
17601
- DURABLE_RECEIPT_WAIT_DELAY_MS,
16472
+ const aggregateForceRefreshIdentities =
16473
+ batch.memberRequests.map((request) =>
16474
+ this.providerInvocationKeyForToolCall({
16475
+ operationKey:
16476
+ this.providerOperationKeyForQueuedToolCall(request),
16477
+ logicalCallId: request.logicalCallId,
16478
+ force: request.force,
16479
+ }),
17602
16480
  );
17603
- if (runtimeReceiptReadTraceEnabled) {
17604
- this.runtimeLog(
17605
- `[runtime-receipt-wait] phase=start requested=${pendingWaits.size} ` +
17606
- `request_digests=${[...pendingWaits.keys()]
17607
- .map(runtimeReceiptKeyDigest)
17608
- .join(',')} max_attempts=${waitMaxAttempts}`,
16481
+ const legacyForceRefreshIdentities = batch.memberRequests
16482
+ .filter((request) => request.force === true)
16483
+ .map((request) =>
16484
+ this.providerInvocationKeyForToolCall({
16485
+ operationKey:
16486
+ this.providerOperationKeyForQueuedToolCall(request),
16487
+ logicalCallId: request.logicalCallId,
16488
+ force: true,
16489
+ }),
17609
16490
  );
17610
- }
17611
- for (
17612
- let attempt = 0;
17613
- attempt < waitMaxAttempts && pendingWaits.size > 0;
17614
- attempt += 1
17615
- ) {
17616
- if (waitSignal.aborted) return;
17617
- if (attempt > 0) {
17618
- if (Date.now() >= waitDeadline) break;
17619
- const delayMs = Math.min(
17620
- runtimeReceiptWaitDelayMs(attempt),
17621
- Math.max(0, waitDeadline - Date.now()),
17622
- );
17623
- await new Promise<void>((resolve) => {
17624
- const finish = () => {
17625
- clearTimeout(timeout);
17626
- waitSignal.removeEventListener('abort', finish);
17627
- resolve();
17628
- };
17629
- const timeout = setTimeout(finish, delayMs);
17630
- waitSignal.addEventListener('abort', finish, {
17631
- once: true,
17632
- });
17633
- });
17634
- if (waitSignal.aborted) return;
17635
- }
17636
- // The long backoff lives outside the hydration lane. Only a
17637
- // bounded receipt read and delivery hold the lane, so stale
17638
- // ownership cannot block unrelated cache misses/provider
17639
- // work while terminal payloads still hydrate one group at a
17640
- // time.
17641
- try {
17642
- await this.withRuntimeReceiptHydrationTurn(
17643
- async (turn) => {
17644
- if (waitSignal.aborted) return;
17645
- const waited =
17646
- await this.waitForCompletedRuntimeToolReceipts(
17647
- [...pendingWaits.keys()],
17648
- 1,
17649
- );
17650
- turn.settleBytes(
17651
- runtimeStepReceiptPayloadBytes(
17652
- waited.completed.values(),
17653
- ),
17654
- );
17655
- if (waitSignal.aborted) return;
17656
- const settledWaits = [...pendingWaits.values()].filter(
17657
- (wait) =>
17658
- waited.completed.has(wait.receiptKey) ||
17659
- waited.failed.has(wait.receiptKey),
17660
- );
17661
- const deliverySettlements = await Promise.allSettled(
17662
- settledWaits.map(async (wait) => {
17663
- const failed = waited.failed.get(wait.receiptKey);
17664
- if (failed) {
17665
- for (const request of wait.requestsForKey) {
17666
- await this.rejectToolCall(
17667
- toolId,
17668
- request,
17669
- failed,
17670
- {
17671
- persistReceiptFailure: false,
17672
- },
17673
- );
17674
- }
17675
- return;
17676
- }
17677
- await reclaimExistingRunningReceiptAfterWait(
17678
- wait.receiptKey,
17679
- wait.requestsForKey,
17680
- wait.forceAfterWait,
17681
- {
17682
- completedReceipt: waited.completed.get(
17683
- wait.receiptKey,
16491
+ const batchActionVersion =
16492
+ (
16493
+ await this.#options.getToolActionCacheVersion?.(
16494
+ batch.batchOperation,
16495
+ )
16496
+ )?.trim() ?? 'unversioned';
16497
+ const aggregateProviderOperationKey =
16498
+ buildDurableToolPayloadProviderIdempotencyKey({
16499
+ orgId: this.#options.orgId,
16500
+ toolId: batch.batchOperation,
16501
+ executionAuthScopeDigest:
16502
+ batch.memberRequests[0]?.executionAuthScopeDigest ?? null,
16503
+ providerActionVersion: batchActionVersion,
16504
+ payload: batch.batchPayload,
16505
+ forceRefreshIdentities: aggregateForceRefreshIdentities,
16506
+ });
16507
+ const aggregateReceiptKey = this.toolTransportReceiptKey({
16508
+ toolId: batch.batchOperation,
16509
+ providerOperationKey: aggregateProviderOperationKey,
16510
+ });
16511
+ let releaseToolSlot: () => void = () => undefined;
16512
+ try {
16513
+ const execution =
16514
+ await this.executeMappedToolCallThroughModule({
16515
+ request: batch.memberRequests[0]!,
16516
+ dispatchToolId: batch.batchOperation,
16517
+ dispatchPayload: batch.batchPayload,
16518
+ resultReceiptKey: aggregateReceiptKey,
16519
+ providerOperationKey: aggregateProviderOperationKey,
16520
+ apiOptions: {
16521
+ durableCallReceiptKey: aggregateReceiptKey,
16522
+ legacyProviderRetryIdentities: [
16523
+ {
16524
+ transportReceiptKey:
16525
+ buildDurableToolAggregateReceiptKey({
16526
+ receiptKeys: batch.memberRequests.map(
16527
+ (member) => member.cacheKey,
17684
16528
  ),
17685
- skipWait: true,
17686
- },
17687
- );
17688
- }),
17689
- );
17690
- for (
17691
- let index = 0;
17692
- index < settledWaits.length;
17693
- index += 1
17694
- ) {
17695
- if (
17696
- deliverySettlements[index]?.status === 'fulfilled'
17697
- ) {
17698
- const settledKey = settledWaits[index]!.receiptKey;
17699
- pendingWaits.delete(settledKey);
17700
- this.#runtimeReceiptWaitParking.delete(settledKey);
17701
- }
17702
- }
17703
- if (runtimeReceiptReadTraceEnabled) {
17704
- const rejected = deliverySettlements.flatMap(
17705
- (settlement, index) => {
17706
- if (settlement.status !== 'rejected') return [];
17707
- const receiptKey =
17708
- settledWaits[index]?.receiptKey;
17709
- return receiptKey
17710
- ? [
17711
- `${runtimeReceiptKeyDigest(receiptKey)}:${
17712
- settlement.reason instanceof Error
17713
- ? settlement.reason.name
17714
- : 'non_error'
17715
- }`,
17716
- ]
17717
- : ['unknown'];
17718
- },
17719
- );
17720
- this.runtimeLog(
17721
- `[runtime-receipt-wait] phase=delivery settled=${deliverySettlements.length} ` +
17722
- `rejected=${rejected.length} rejected_digests=${rejected.join(',')}`,
17723
- );
17724
- }
17725
- const rejectedDelivery = deliverySettlements.find(
17726
- (settlement) => settlement.status === 'rejected',
17727
- );
17728
- if (rejectedDelivery?.status === 'rejected') {
17729
- throw rejectedDelivery.reason;
17730
- }
17731
- },
17732
- {
17733
- receiptKeys: pendingWaits.size,
17734
- source: 'map_receipt_wait',
17735
- // The lane wait is part of this loop's budget, not
17736
- // extra time on top of it.
17737
- timeoutMs: Math.max(0, waitDeadline - Date.now()),
16529
+ prefix: 'batch',
16530
+ aggregateReceiptPrefix:
16531
+ buildDurableToolReceiptPrefix({
16532
+ orgId: this.#options.orgId,
16533
+ toolId: batch.batchOperation,
16534
+ }),
16535
+ }),
16536
+ providerOperationKey:
16537
+ buildDurableToolPayloadProviderIdempotencyKey({
16538
+ orgId: this.#options.orgId,
16539
+ toolId: batch.batchOperation,
16540
+ executionAuthScopeDigest:
16541
+ batch.memberRequests[0]
16542
+ ?.executionAuthScopeDigest ?? null,
16543
+ providerActionVersion: batchActionVersion,
16544
+ payload: batch.batchPayload,
16545
+ forceRefreshIdentities:
16546
+ legacyForceRefreshIdentities,
16547
+ }),
16548
+ forceRefreshIdentities:
16549
+ legacyForceRefreshIdentities,
16550
+ },
16551
+ ],
16552
+ // A native batch is one physical provider request for
16553
+ // many rows of the same column, so the batch's node scope
16554
+ // is its members' shared scope.
16555
+ playNodeScope: playNodeScopeForToolCallRequest(
16556
+ batch.memberRequests[0]!,
16557
+ batch.batchOperation,
16558
+ ),
16559
+ executionAuthScopeDigest:
16560
+ batch.memberRequests[0]?.executionAuthScopeDigest ??
16561
+ null,
16562
+ providerIdempotencyKey: aggregateProviderOperationKey,
16563
+ providerOperationForceRefreshIdentities:
16564
+ aggregateForceRefreshIdentities,
16565
+ receiptLeaseExpiresAt: batch.memberRequests.reduce<
16566
+ string | null
16567
+ >((earliest, request) => {
16568
+ const expiresAt =
16569
+ request.receiptLeaseExpiresAt ?? null;
16570
+ if (!expiresAt) return earliest;
16571
+ if (!earliest) return expiresAt;
16572
+ return Date.parse(expiresAt) < Date.parse(earliest)
16573
+ ? expiresAt
16574
+ : earliest;
16575
+ }, null),
16576
+ timeoutMs: resolveRuntimeTimeoutMsForRequests(
16577
+ batch.memberRequests,
16578
+ ),
16579
+ retainToolSlot: (release) => {
16580
+ releaseToolSlot = release;
16581
+ },
17738
16582
  },
17739
- );
17740
- } catch (error) {
17741
- // The budget is spent, not the run: fall through to the
17742
- // same reclaim an expired lease takes.
17743
- if (
17744
- !(
17745
- error instanceof RuntimeReceiptHydrationTurnTimeoutError
17746
- )
17747
- ) {
17748
- throw error;
17749
- }
17750
- break;
17751
- }
17752
- }
17753
- if (waitSignal.aborted) return;
17754
- await reclaimExistingRunningReceipts([
17755
- ...pendingWaits.values(),
17756
- ]);
17757
- };
17758
- try {
17759
- const lanes = await Promise.allSettled([
17760
- reclaimExistingRunningReceipts(expiredLeaseWaits),
17761
- pollWaits(),
17762
- ]);
17763
- const rejectedLane = lanes.find(
17764
- (lane) => lane.status === 'rejected',
17765
- );
17766
- if (rejectedLane?.status === 'rejected') {
17767
- throw rejectedLane.reason;
17768
- }
17769
- } finally {
17770
- for (const wait of durableExistingRunningWaits) {
17771
- this.#runtimeReceiptWaitParking.delete(wait.receiptKey);
16583
+ });
16584
+ return { execution, releaseToolSlot };
16585
+ } catch (error) {
16586
+ releaseToolSlot();
16587
+ throw error;
17772
16588
  }
17773
- }
17774
- };
17775
-
17776
- for (const [receiptKey, group] of requestsByReceiptKey) {
17777
- const requestsForKey = group.requests;
17778
- const claim = claims.get(receiptKey);
17779
- if (!claim) {
17780
- queueExistingRunningReceiptWait(
17781
- receiptKey,
17782
- requestsForKey,
17783
- group.forceRefresh,
17784
- );
17785
- continue;
17786
- }
17787
- if (claim?.status === 'completed' || claim?.status === 'skipped') {
17788
- await resolveRequestsFromReceipt(requestsForKey, claim, 'cache');
17789
- durableRecoveredRequests.push(...requestsForKey);
17790
- continue;
17791
- }
17792
- if (claim?.status === 'failed') {
17793
- for (const request of requestsForKey) {
17794
- await this.rejectToolCall(
17795
- toolId,
17796
- request,
17797
- runtimeReceiptFailureError(
17798
- claim,
17799
- 'Durable tool call failed',
17800
- this.currentToolErrorSchemaVersion,
17801
- ),
17802
- );
17803
- }
17804
- continue;
17805
- }
17806
- if (
17807
- (claim?.status === 'queued' ||
17808
- claim?.status === 'pending' ||
17809
- claim?.status === 'running') &&
17810
- claim.claimState === 'existing'
17811
- ) {
17812
- queueExistingRunningReceiptWait(
17813
- receiptKey,
17814
- requestsForKey,
17815
- group.forceRefresh,
17816
- !runtimeReceiptClaimLeaseIsLive(claim),
17817
- );
17818
- continue;
17819
- }
17820
- if (this.isOwnedClaimedRuntimeReceipt(claim)) {
17821
- const [owner, ...waiters] = requestsForKey;
17822
- if (!claim.leaseId) {
17823
- throw new RuntimeReceiptLeaseLostError({
17824
- receiptKey,
17825
- runId: this.currentReceiptOwnerRunId,
17826
- leaseId: 'missing',
17827
- });
17828
- }
17829
- if (owner) {
17830
- owner.receiptLeaseId = claim.leaseId ?? null;
17831
- owner.receiptLeaseExpiresAt = claim.leaseExpiresAt ?? null;
17832
- const claimLeaseExpiresAtMs = Date.parse(
17833
- claim.leaseExpiresAt ?? '',
17834
- );
17835
- pendingReceiptClaimsEstablishExecutionFence &&=
17836
- claim.claimState === 'claimed' &&
17837
- claim.status === 'running' &&
17838
- Number.isFinite(claimLeaseExpiresAtMs) &&
17839
- claimLeaseExpiresAtMs > Date.now();
17840
- }
17841
- claimOwnerWithLiveFollowers(owner, waiters);
17842
- continue;
17843
- }
17844
- queueExistingRunningReceiptWait(
17845
- receiptKey,
17846
- requestsForKey,
17847
- group.forceRefresh,
17848
- );
17849
- }
17850
-
17851
- pendingRequests.length = 0;
17852
- pendingRequests.push(...claimedRequests);
17853
- durableExistingRunningState.settlement =
17854
- durableExistingRunningWaits.length > 0
17855
- ? processExistingRunningReceiptWaits().then(
17856
- () => ({ status: 'fulfilled' as const }),
17857
- (reason: unknown) => ({
17858
- status: 'rejected' as const,
17859
- reason,
17860
- }),
17861
- )
17862
- : null;
17863
- recordToolStep(durableRecoveredRequests);
17864
- }
17865
-
17866
- // Claims establish one lease cohort. Revalidate that cohort in one
17867
- // gateway operation before provider admission, then let the existing
17868
- // cohort heartbeat keep the undispatched tail alive. Revalidating each
17869
- // call separately after a tool slot opens fragments the writer into
17870
- // one-row batches and starves otherwise healthy providers.
17871
- const pendingReceiptHeartbeat = this.startPendingToolReceiptHeartbeat(
17872
- pendingRequests,
17873
- pendingReceiptClaimsEstablishExecutionFence,
17874
- );
17875
- let providerFailure: { reason: unknown } | null = null;
17876
- try {
17877
- if (pendingReceiptHeartbeat) {
17878
- await pendingReceiptHeartbeat.confirmOwned();
17879
- }
17880
- if (pendingRequests.length > 0) {
17881
- const strategy = operationStrategy;
17882
-
17883
- if (strategy) {
17884
- const compiledBatches = compileRequestsWithStrategy({
17885
- requests: pendingRequests,
17886
- strategy,
17887
- getPayload: (request: ToolCallRequest) => request.input,
17888
- });
17889
-
17890
- const batchParallelismCeiling =
17891
- this.governor.policy.pacing.workerToolBatchDefaultParallelism;
17892
- const batchSize =
17893
- compiledBatches.length > 0
17894
- ? await this.resourceGovernor.suggestedToolParallelism(
17895
- compiledBatches[0]!.batchOperation,
17896
- batchParallelismCeiling,
17897
- )
17898
- : batchParallelismCeiling;
17899
- await executeChunkedRequests({
17900
- requests: compiledBatches,
17901
- batchSize,
17902
- execute: async (batch) => {
17903
- // Circuit breaker: skip dispatching this batch's provider call
17904
- // once a persistence failure has occurred in this run.
17905
- if (this.persistenceLatch.tripped) {
17906
- this.persistenceLatch.preventedCallCount +=
17907
- batch.memberRequests.length;
17908
- throw new RuntimePersistenceCircuitOpenError(
17909
- this.persistenceLatch,
16589
+ },
16590
+ onChunkComplete: async (chunkResults) => {
16591
+ for (const entry of chunkResults) {
16592
+ if (entry.error !== undefined) {
16593
+ const settlements = await Promise.allSettled(
16594
+ entry.request.memberRequests.map((request) =>
16595
+ rejectCall(request, entry.error),
16596
+ ),
17910
16597
  );
17911
- }
17912
- const receiptKeys = batch.memberRequests.map(
17913
- (request) => request.cacheKey,
17914
- );
17915
- const aggregateReceiptKey =
17916
- buildDurableToolAggregateReceiptKey({
17917
- receiptKeys,
17918
- prefix: 'batch',
17919
- aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
17920
- orgId: this.#options.orgId,
17921
- toolId: batch.batchOperation,
17922
- }),
17923
- });
17924
- const aggregateForceRefreshIdentities = batch.memberRequests
17925
- .filter((request) => request.force === true)
17926
- .map((request) =>
17927
- this.providerIdempotencyKeyForToolCall({
17928
- operationKey:
17929
- this.providerOperationKeyForQueuedToolCall(request),
17930
- force: true,
17931
- logicalCallId: request.logicalCallId,
17932
- }),
16598
+ const rejected = settlements.find(
16599
+ (settlement): settlement is PromiseRejectedResult =>
16600
+ settlement.status === 'rejected',
17933
16601
  );
17934
- const aggregateProviderOperationKey =
17935
- buildDurableToolPayloadProviderIdempotencyKey({
17936
- orgId: this.#options.orgId,
17937
- toolId: batch.batchOperation,
17938
- executionAuthScopeDigest:
17939
- batch.memberRequests[0]?.executionAuthScopeDigest ??
17940
- null,
17941
- providerActionVersion:
17942
- (
17943
- await this.#options.getToolActionCacheVersion?.(
17944
- batch.batchOperation,
17945
- )
17946
- )?.trim() ?? 'unversioned',
17947
- payload: batch.batchPayload,
17948
- forceRefreshIdentities: aggregateForceRefreshIdentities,
17949
- });
17950
- let releaseToolSlot: () => void = () => undefined;
16602
+ if (rejected) throw rejected.reason;
16603
+ continue;
16604
+ }
16605
+ const batchExecution = entry.result;
17951
16606
  try {
17952
- const execution =
17953
- await this.executeMappedToolCallThroughModule({
17954
- request: batch.memberRequests[0]!,
17955
- dispatchToolId: batch.batchOperation,
17956
- dispatchPayload: batch.batchPayload,
17957
- resultReceiptKey: aggregateReceiptKey,
17958
- providerOperationKey: aggregateProviderOperationKey,
17959
- apiOptions: {
17960
- durableCallReceiptKey: aggregateReceiptKey,
17961
- // A native batch is one physical provider request for
17962
- // many rows of the same column, so the batch's node scope
17963
- // is its members' shared scope.
17964
- playNodeScope: playNodeScopeForToolCallRequest(
17965
- batch.memberRequests[0]!,
17966
- batch.batchOperation,
17967
- ),
17968
- executionAuthScopeDigest:
17969
- batch.memberRequests[0]?.executionAuthScopeDigest ??
17970
- null,
17971
- providerIdempotencyKey: aggregateProviderOperationKey,
17972
- providerOperationForceRefreshIdentities:
17973
- aggregateForceRefreshIdentities,
17974
- receiptLeaseExpiresAt: batch.memberRequests.reduce<
17975
- string | null
17976
- >((earliest, request) => {
17977
- const expiresAt =
17978
- request.receiptLeaseExpiresAt ?? null;
17979
- if (!expiresAt) return earliest;
17980
- if (!earliest) return expiresAt;
17981
- return Date.parse(expiresAt) < Date.parse(earliest)
17982
- ? expiresAt
17983
- : earliest;
17984
- }, null),
17985
- timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
17986
- batch.memberRequests,
16607
+ const splitResults =
16608
+ batchExecution != null
16609
+ ? entry.request.splitResults(
16610
+ legacyResultForBatchSplitter(
16611
+ batchExecution.execution,
16612
+ ),
16613
+ )
16614
+ : entry.request.memberRequests.map(() => null);
16615
+ const resolvedResults =
16616
+ await this.resolveToolCallBatchResults(
16617
+ toolId,
16618
+ entry.request.memberRequests.map((request, index) => ({
16619
+ request,
16620
+ result: splitResults[index] ?? null,
16621
+ status: batchExecution?.execution.status,
16622
+ meta: nativeBatchResponseMeta(
16623
+ splitResults[index] ?? null,
17987
16624
  ),
17988
- beforeProviderCall: () =>
17989
- pendingReceiptHeartbeat
17990
- ? pendingReceiptHeartbeat.assertOwned()
17991
- : this.assertRuntimeToolReceiptOwnership(
17992
- batch.memberRequests,
16625
+ toolResponse:
16626
+ batchExecution == null
16627
+ ? undefined
16628
+ : publicToolResponseForBatchedItem(
16629
+ batchExecution.execution,
16630
+ splitResults[index] ?? null,
16631
+ true,
17993
16632
  ),
17994
- heartbeatReceipt: () =>
17995
- this.renewRuntimeToolReceiptOwnership(
17996
- batch.memberRequests,
17997
- ),
17998
- retainToolSlot: (release) => {
17999
- releaseToolSlot = release;
18000
- },
18001
- },
18002
- });
18003
- return { execution, releaseToolSlot };
18004
- } catch (error) {
18005
- releaseToolSlot();
18006
- throw error;
18007
- }
18008
- },
18009
- onChunkComplete: async (chunkResults) => {
18010
- for (const entry of chunkResults) {
18011
- if (entry.error !== undefined) {
18012
- const settlements = await Promise.allSettled(
18013
- entry.request.memberRequests.map((request) =>
18014
- rejectWithLiveFollowers(request, entry.error),
18015
- ),
18016
- );
18017
- const rejected = settlements.find(
18018
- (settlement): settlement is PromiseRejectedResult =>
18019
- settlement.status === 'rejected',
16633
+ })),
18020
16634
  );
18021
- if (rejected) throw rejected.reason;
18022
- continue;
18023
- }
18024
- const batchExecution = entry.result;
18025
- try {
18026
- const splitResults =
18027
- batchExecution != null
18028
- ? entry.request.splitResults(
18029
- legacyResultForBatchSplitter(
18030
- batchExecution.execution,
18031
- ),
18032
- )
18033
- : entry.request.memberRequests.map(() => null);
18034
- const resolvedResults =
18035
- await this.resolveToolCallBatchResults(
18036
- toolId,
18037
- entry.request.memberRequests.map(
18038
- (request, index) => ({
18039
- request,
18040
- result: splitResults[index] ?? null,
18041
- status: batchExecution?.execution.status,
18042
- toolResponse:
18043
- batchExecution == null
18044
- ? undefined
18045
- : publicToolResponseForBatchedItem(
18046
- batchExecution.execution,
18047
- splitResults[index] ?? null,
18048
- true,
18049
- ),
18050
- }),
18051
- ),
18052
- );
18053
16635
 
18054
- for (
18055
- let index = 0;
18056
- index < entry.request.memberRequests.length;
18057
- index += 1
18058
- ) {
18059
- const request = entry.request.memberRequests[index]!;
18060
- if (resolvedResults[index] != null) {
18061
- successfulLiveStepCallIds.add(request.callId);
18062
- }
18063
- resolveLiveFollowers(request, resolvedResults[index]);
16636
+ for (
16637
+ let index = 0;
16638
+ index < entry.request.memberRequests.length;
16639
+ index += 1
16640
+ ) {
16641
+ const request = entry.request.memberRequests[index]!;
16642
+ if (resolvedResults[index] != null) {
16643
+ successfulLiveStepCallIds.add(request.callId);
18064
16644
  }
18065
- } finally {
18066
- batchExecution?.releaseToolSlot();
18067
16645
  }
16646
+ } finally {
16647
+ batchExecution?.releaseToolSlot();
18068
16648
  }
16649
+ }
18069
16650
 
18070
- recordToolStep(
18071
- chunkResults.flatMap(
18072
- (entry) => entry.request.memberRequests,
18073
- ),
18074
- );
18075
- this.#options.onBatchComplete?.(this.checkpoint);
18076
- },
18077
- retainResults: false,
18078
- });
18079
- } else {
18080
- // Bound direct map dispatch separately from ordinary provider
18081
- // residency. This prevents a blocked first chunk from launching
18082
- // another chunk ahead of a provider's admitted start rate, while
18083
- // preserving the wider resident-call capacity used by batched and
18084
- // high-latency providers.
18085
- const toolCallConcurrencyCeiling =
18086
- this.governor.policy.concurrency.toolCalls;
18087
- const shapedToolParallelism =
18088
- await this.resourceGovernor.suggestedDirectToolParallelism(
18089
- toolId,
18090
- toolCallConcurrencyCeiling,
16651
+ recordToolStep(
16652
+ chunkResults.flatMap((entry) => entry.request.memberRequests),
18091
16653
  );
18092
- const dispatchWidth = Math.min(
16654
+ this.#options.onBatchComplete?.(this.checkpoint);
16655
+ },
16656
+ retainResults: false,
16657
+ });
16658
+ } else {
16659
+ // Bound direct map dispatch separately from ordinary provider
16660
+ // residency. This prevents a blocked first chunk from launching
16661
+ // another chunk ahead of a provider's admitted start rate, while
16662
+ // preserving the wider resident-call capacity used by batched and
16663
+ // high-latency providers.
16664
+ const toolCallConcurrencyCeiling =
16665
+ this.governor.policy.concurrency.toolCalls;
16666
+ const shapedToolParallelism =
16667
+ await this.resourceGovernor.suggestedDirectToolParallelism(
16668
+ toolId,
18093
16669
  toolCallConcurrencyCeiling,
18094
- Math.max(1, shapedToolParallelism),
18095
16670
  );
18096
- const dispatchResults = await dispatchBoundedSettled(
18097
- pendingRequests,
18098
- dispatchWidth,
18099
- async (request) => {
18100
- // Circuit breaker: do not dispatch this provider call once a
18101
- // persistence failure has occurred in this run.
18102
- if (this.persistenceLatch.tripped) {
18103
- this.persistenceLatch.preventedCallCount += 1;
18104
- throw new RuntimePersistenceCircuitOpenError(
18105
- this.persistenceLatch,
18106
- );
18107
- }
18108
- let releaseToolSlot: () => void = () => undefined;
18109
- try {
18110
- const execution =
18111
- await this.executeMappedToolCallThroughModule({
18112
- request,
18113
- dispatchToolId: toolId,
18114
- dispatchPayload: request.input,
18115
- // Always-fresh map calls intentionally have no
18116
- // result cache. Their provider-operation identity is
18117
- // a unique forced semantic request, never a cache key.
18118
- resultReceiptKey:
18119
- request.receiptKey ?? request.cacheKey,
18120
- providerOperationKey:
18121
- this.providerOperationKeyForQueuedToolCall(request),
18122
- apiOptions: {
18123
- beforeProviderCall: () =>
18124
- pendingReceiptHeartbeat
18125
- ? pendingReceiptHeartbeat.assertOwned()
18126
- : this.assertRuntimeToolReceiptOwnership([
18127
- request,
18128
- ]),
18129
- playNodeScope:
18130
- playNodeScopeForToolCallRequest(request),
18131
- ...(request.receiptKey
18132
- ? {
18133
- durableCallReceiptKey: request.receiptKey,
18134
- completedResultCache: true,
18135
- executionAuthScopeDigest:
18136
- request.executionAuthScopeDigest,
18137
- providerIdempotencyKey:
18138
- this.providerIdempotencyKeyForToolCall({
18139
- operationKey:
18140
- this.providerOperationKeyForQueuedToolCall(
18141
- request,
18142
- ),
18143
- force: request.force === true,
18144
- logicalCallId: request.logicalCallId,
18145
- }),
18146
- ...this.providerForceRefreshOptionsForToolCall({
18147
- operationKey:
18148
- this.providerOperationKeyForQueuedToolCall(
18149
- request,
18150
- ),
18151
- force: request.force,
18152
- requiresExecutionFence:
18153
- request.requiresExecutionFence,
18154
- logicalCallId: request.logicalCallId,
16671
+ const dispatchWidth = Math.min(
16672
+ toolCallConcurrencyCeiling,
16673
+ Math.max(1, shapedToolParallelism),
16674
+ );
16675
+ const dispatchResults = await dispatchBoundedSettled(
16676
+ pendingRequests,
16677
+ dispatchWidth,
16678
+ async (request) => {
16679
+ // Circuit breaker: do not dispatch this provider call once a
16680
+ // persistence failure has occurred in this run.
16681
+ if (this.persistenceLatch.tripped) {
16682
+ this.persistenceLatch.preventedCallCount += 1;
16683
+ throw new RuntimePersistenceCircuitOpenError(
16684
+ this.persistenceLatch,
16685
+ );
16686
+ }
16687
+ const providerOperationKey =
16688
+ this.providerInvocationKeyForToolCall({
16689
+ operationKey:
16690
+ this.providerOperationKeyForQueuedToolCall(request),
16691
+ logicalCallId: request.logicalCallId,
16692
+ force: request.force,
16693
+ });
16694
+ let releaseToolSlot: () => void = () => undefined;
16695
+ try {
16696
+ const execution =
16697
+ await this.executeMappedToolCallThroughModule({
16698
+ request,
16699
+ dispatchToolId: toolId,
16700
+ dispatchPayload: request.input,
16701
+ // Always-fresh map calls intentionally have no
16702
+ // result cache. Their provider-operation identity is
16703
+ // a unique forced semantic request, never a cache key.
16704
+ resultReceiptKey: request.receiptKey ?? request.cacheKey,
16705
+ providerOperationKey,
16706
+ apiOptions: {
16707
+ playNodeScope: playNodeScopeForToolCallRequest(request),
16708
+ executionAuthScopeDigest:
16709
+ request.executionAuthScopeDigest,
16710
+ providerIdempotencyKey: providerOperationKey,
16711
+ ...this.providerInvocationOptionsForToolCall({
16712
+ operationKey:
16713
+ this.providerOperationKeyForQueuedToolCall(request),
16714
+ requiresExecutionFence:
16715
+ request.requiresExecutionFence,
16716
+ logicalCallId: request.logicalCallId,
16717
+ force: request.force,
16718
+ }),
16719
+ ...(request.receiptKey
16720
+ ? {
16721
+ durableCallReceiptKey:
16722
+ this.toolTransportReceiptKey({
16723
+ toolId,
16724
+ providerOperationKey,
18155
16725
  }),
18156
- receiptLeaseExpiresAt:
18157
- request.receiptLeaseExpiresAt,
18158
- heartbeatReceipt: () =>
18159
- this.renewRuntimeToolReceiptOwnership([
18160
- request,
18161
- ]),
18162
- }
18163
- : {
18164
- providerIdempotencyKey:
18165
- this.providerIdempotencyKeyForToolCall({
18166
- operationKey:
18167
- this.providerOperationKeyForQueuedToolCall(
16726
+ legacyProviderRetryIdentities: [
16727
+ {
16728
+ transportReceiptKey: request.receiptKey,
16729
+ providerOperationKey: request.force
16730
+ ? providerOperationKey
16731
+ : this.providerOperationKeyForQueuedToolCall(
18168
16732
  request,
18169
16733
  ),
18170
- force: true,
18171
- logicalCallId: request.logicalCallId,
18172
- }),
18173
- }),
18174
- timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
18175
- request,
18176
- ]),
18177
- retainToolSlot: (release) => {
18178
- releaseToolSlot = release;
18179
- },
16734
+ ...(request.force &&
16735
+ request.requiresExecutionFence !== true
16736
+ ? {
16737
+ forceRefreshIdentities: [
16738
+ providerOperationKey,
16739
+ ],
16740
+ }
16741
+ : {}),
16742
+ },
16743
+ ],
16744
+ completedResultCache: true,
16745
+ }
16746
+ : {}),
16747
+ timeoutMs: resolveRuntimeTimeoutMsForRequests([
16748
+ request,
16749
+ ]),
16750
+ retainToolSlot: (release) => {
16751
+ releaseToolSlot = release;
18180
16752
  },
18181
- });
18182
- const result = await this.resolveToolCall(
18183
- toolId,
18184
- request,
18185
- execution,
18186
- );
18187
- if (result != null) {
18188
- successfulLiveStepCallIds.add(request.callId);
18189
- }
18190
- resolveLiveFollowers(request, result);
18191
- } finally {
18192
- releaseToolSlot();
16753
+ },
16754
+ });
16755
+ const result = await this.resolveToolCall(
16756
+ toolId,
16757
+ request,
16758
+ execution,
16759
+ );
16760
+ if (result != null) {
16761
+ successfulLiveStepCallIds.add(request.callId);
18193
16762
  }
18194
- },
18195
- );
18196
- const failedEntries = dispatchResults.filter(
18197
- (entry) => entry.error !== undefined,
18198
- );
18199
- const failedSettlements = await Promise.allSettled(
18200
- failedEntries.map((entry) =>
18201
- rejectWithLiveFollowers(entry.job, entry.error),
18202
- ),
18203
- );
18204
- const failedRejection = failedSettlements.find(
18205
- (settlement): settlement is PromiseRejectedResult =>
18206
- settlement.status === 'rejected',
18207
- );
18208
- if (failedRejection) throw failedRejection.reason;
18209
- recordToolStep(pendingRequests);
18210
- this.#options.onBatchComplete?.(this.checkpoint);
18211
- }
18212
- }
18213
- } catch (reason) {
18214
- providerFailure = { reason };
18215
- durableExistingRunningAbort.abort(reason);
18216
- } finally {
18217
- pendingReceiptHeartbeat?.stop();
18218
- }
18219
- const pendingReceiptLeaseLost = pendingReceiptHeartbeat?.leaseLost();
18220
- if (pendingReceiptLeaseLost && providerFailure === null) {
18221
- providerFailure = { reason: pendingReceiptLeaseLost };
18222
- durableExistingRunningAbort.abort(pendingReceiptLeaseLost);
18223
- }
18224
- const durableExistingRunningSettlement =
18225
- durableExistingRunningState.settlement;
18226
- if (durableExistingRunningSettlement) {
18227
- const settlement = await durableExistingRunningSettlement;
18228
- if (settlement.status === 'rejected' && providerFailure === null) {
18229
- providerFailure = { reason: settlement.reason };
16763
+ } finally {
16764
+ releaseToolSlot();
16765
+ }
16766
+ },
16767
+ );
16768
+ const failedEntries = dispatchResults.filter(
16769
+ (entry) => entry.error !== undefined,
16770
+ );
16771
+ const failedSettlements = await Promise.allSettled(
16772
+ failedEntries.map((entry) => rejectCall(entry.job, entry.error)),
16773
+ );
16774
+ const failedRejection = failedSettlements.find(
16775
+ (settlement): settlement is PromiseRejectedResult =>
16776
+ settlement.status === 'rejected',
16777
+ );
16778
+ if (failedRejection) throw failedRejection.reason;
16779
+ recordToolStep(pendingRequests);
16780
+ this.#options.onBatchComplete?.(this.checkpoint);
18230
16781
  }
18231
16782
  }
18232
- if (providerFailure) throw providerFailure.reason;
18233
16783
  }),
18234
16784
  );
18235
16785
  const rejected = toolSettlements.find(
@@ -18323,24 +16873,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
18323
16873
  )(ctx, input);
18324
16874
  }
18325
16875
 
18326
- private providerIdempotencyKeyForToolCall(input: {
16876
+ private providerInvocationKeyForToolCall(input: {
18327
16877
  operationKey: string;
18328
- force?: boolean;
18329
16878
  logicalCallId?: string;
16879
+ force?: boolean;
18330
16880
  }): string {
18331
- let fallbackAttemptId: string | null = null;
18332
- if (input.force === true) {
18333
- if (!input.logicalCallId) {
18334
- throw new Error(
18335
- 'Forced tool execution requires a stable logical call identity.',
18336
- );
18337
- }
18338
- fallbackAttemptId = `${this.currentRunId}:${input.logicalCallId}`;
16881
+ if (!input.logicalCallId) {
16882
+ throw new Error('Tool execution requires a stable invocation identity.');
18339
16883
  }
18340
16884
  return buildProviderOperationIdempotencyKey({
18341
16885
  operationKey: input.operationKey,
18342
- force: input.force,
18343
- fallbackAttemptId,
16886
+ force: true,
16887
+ fallbackAttemptId: `${this.currentRunId}:${input.logicalCallId}${input.force === true ? '' : ':cache-miss'}`,
18344
16888
  });
18345
16889
  }
18346
16890
 
@@ -18986,7 +17530,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
18986
17530
  const ambiguousTransportReplayAllowed =
18987
17531
  readOnlyQuery || retrySafeTransientHttp;
18988
17532
  const invocationOrgId = this.#options.orgId?.trim() ?? '';
18989
- const durableCallReceiptKey = uncachedAsyncLifecycleCall
17533
+ let durableCallReceiptKey = uncachedAsyncLifecycleCall
18990
17534
  ? null
18991
17535
  : options?.durableCallReceiptKey?.trim() ||
18992
17536
  (usesDurableInvocationFence && invocationOrgId
@@ -19004,9 +17548,57 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19004
17548
  this.#options.durableBoundaries && durableCallReceiptKey
19005
17549
  ? stableDigest(`provider-retry:${durableCallReceiptKey}`)
19006
17550
  : null;
19007
- const savedProviderRetryCursor = providerRetryCursorKey
17551
+ let savedProviderRetryCursor = providerRetryCursorKey
19008
17552
  ? this.checkpoint.providerRetryCursors?.[providerRetryCursorKey]
19009
17553
  : undefined;
17554
+ if (providerRetryCursorKey && !savedProviderRetryCursor) {
17555
+ // Native calls can move from the outgoing fallback into the Module.
17556
+ // Probe only exact owner-supplied checkpoint identities; never wait
17557
+ // on another invocation or derive a fence from customer strings.
17558
+ for (const legacy of options?.legacyProviderRetryIdentities ?? []) {
17559
+ const legacyCursorKey = stableDigest(
17560
+ `provider-retry:${legacy.transportReceiptKey}`,
17561
+ );
17562
+ const cursor =
17563
+ this.checkpoint.providerRetryCursors?.[legacyCursorKey];
17564
+ if (!cursor) continue;
17565
+ // Consume synchronously before any await so a sibling cannot
17566
+ // inherit the same outgoing invocation.
17567
+ delete this.checkpoint.providerRetryCursors![legacyCursorKey];
17568
+ savedProviderRetryCursor = {
17569
+ ...cursor,
17570
+ legacyProviderOperationKey:
17571
+ cursor.legacyProviderOperationKey ??
17572
+ legacy.providerOperationKey,
17573
+ legacyTransportReceiptKey:
17574
+ cursor.legacyTransportReceiptKey ??
17575
+ legacy.transportReceiptKey,
17576
+ ...(legacy.forceRefreshIdentities?.length
17577
+ ? {
17578
+ legacyProviderOperationForceRefreshIdentities:
17579
+ cursor.legacyProviderOperationForceRefreshIdentities ??
17580
+ legacy.forceRefreshIdentities,
17581
+ }
17582
+ : {}),
17583
+ };
17584
+ this.checkpoint.providerRetryCursors![providerRetryCursorKey] =
17585
+ savedProviderRetryCursor;
17586
+ break;
17587
+ }
17588
+ }
17589
+ // The checkpoint index stays unique to this invocation. Its wire
17590
+ // identities must still name the prior gateway outcome so that a
17591
+ // qualified 429 retry can be attested after any further suspension.
17592
+ const legacyProviderOperationKey =
17593
+ savedProviderRetryCursor?.legacyProviderOperationKey;
17594
+ if (savedProviderRetryCursor?.legacyTransportReceiptKey) {
17595
+ durableCallReceiptKey =
17596
+ savedProviderRetryCursor.legacyTransportReceiptKey;
17597
+ }
17598
+ const providerOperationForceRefreshIdentities =
17599
+ legacyProviderOperationKey
17600
+ ? savedProviderRetryCursor?.legacyProviderOperationForceRefreshIdentities
17601
+ : options?.providerOperationForceRefreshIdentities;
19010
17602
  const executionRequestNonce = crypto.randomUUID();
19011
17603
  // Transport ambiguity must retain the billing identity. An ordinary
19012
17604
  // (unfenced) call advances it only after a definite provider-owned
@@ -19020,7 +17612,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19020
17612
  const providerIdempotencyReceiptKey =
19021
17613
  options?.providerIdempotencyReceiptKey?.trim() || null;
19022
17614
  const providerExecutionFenceKey = requiresExecutionFence
19023
- ? options?.providerIdempotencyKey?.trim() || null
17615
+ ? legacyProviderOperationKey ||
17616
+ options?.providerIdempotencyKey?.trim() ||
17617
+ null
19024
17618
  : null;
19025
17619
  if (requiresExecutionFence && !providerExecutionFenceKey) {
19026
17620
  throw new Error(
@@ -19029,14 +17623,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19029
17623
  }
19030
17624
  const canonicalOperationActionVersion =
19031
17625
  providerExecutionFenceKey ||
19032
- options?.providerOperationForceRefreshIdentities?.length
17626
+ providerOperationForceRefreshIdentities?.length
19033
17627
  ? (
19034
17628
  await this.#options.getToolActionCacheVersion?.(toolId)
19035
17629
  )?.trim() || null
19036
17630
  : null;
19037
17631
  const canonicalOperation =
17632
+ !uncachedAsyncLifecycleCall &&
19038
17633
  (providerExecutionFenceKey ||
19039
- options?.providerOperationForceRefreshIdentities?.length) &&
17634
+ providerOperationForceRefreshIdentities?.length) &&
19040
17635
  canonicalOperationActionVersion
19041
17636
  ? {
19042
17637
  version: 1,
@@ -19046,10 +17641,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19046
17641
  output_intent: options?.completedResultCache
19047
17642
  ? 'dataset'
19048
17643
  : 'value',
19049
- ...(options?.providerOperationForceRefreshIdentities?.length
17644
+ ...(providerOperationForceRefreshIdentities?.length
19050
17645
  ? {
19051
17646
  force_refresh_identities:
19052
- options.providerOperationForceRefreshIdentities,
17647
+ providerOperationForceRefreshIdentities,
19053
17648
  }
19054
17649
  : {}),
19055
17650
  }
@@ -19183,9 +17778,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19183
17778
  };
19184
17779
  const persistProviderRetryCursor = (retryFrom: number) => {
19185
17780
  if (!providerRetryCursorKey) return;
19186
- const cursor = {
17781
+ const cursor: ProviderRetryCursor = {
17782
+ ...savedProviderRetryCursor,
19187
17783
  invocationAttempt: retryFrom + 1,
19188
17784
  retryFromInvocationAttempt: retryFrom,
17785
+ ...(legacyProviderOperationKey
17786
+ ? {
17787
+ legacyProviderOperationKey,
17788
+ legacyTransportReceiptKey: durableCallReceiptKey!,
17789
+ }
17790
+ : {}),
19189
17791
  };
19190
17792
  this.checkpoint.providerRetryCursors ??= {};
19191
17793
  this.checkpoint.providerRetryCursors[providerRetryCursorKey] =