deepline 0.2.55 → 0.2.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +14 -0
  2. package/dist/bundling-sources/sdk/src/http.ts +19 -1
  3. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  4. package/dist/bundling-sources/sdk/src/stream-reconnect.ts +6 -0
  5. package/dist/bundling-sources/sdk/src/types.ts +7 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +146 -12
  7. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +6 -3
  8. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1631 -748
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +20 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +81 -52
  11. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +146 -6
  12. package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +56 -22
  13. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +14 -11
  14. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +21 -2
  15. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +81 -21
  16. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +7 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +21 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +27 -2
  20. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +49 -10
  21. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +96 -3
  22. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +341 -67
  23. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +10 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +8 -1
  25. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +17 -2
  26. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +4 -1
  27. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +17 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +16 -2
  29. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +25 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +2 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +80 -1
  33. package/dist/bundling-sources/shared_libs/plays/docflow.ts +113 -14
  34. package/dist/bundling-sources/shared_libs/plays/play-exports.ts +53 -4
  35. package/dist/bundling-sources/shared_libs/security/safe-fetch.ts +9 -0
  36. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +85 -48
  37. package/dist/cli/index.js +429 -54
  38. package/dist/cli/index.mjs +409 -28
  39. package/dist/{compiler-manifest-Bl8kmLx9.d.mts → compiler-manifest-TgaC4DeD.d.mts} +13 -0
  40. package/dist/{compiler-manifest-Bl8kmLx9.d.ts → compiler-manifest-TgaC4DeD.d.ts} +13 -0
  41. package/dist/index.d.mts +21 -3
  42. package/dist/index.d.ts +21 -3
  43. package/dist/index.js +29 -2
  44. package/dist/index.mjs +29 -2
  45. package/dist/install-integrity.json +2 -2
  46. package/dist/plays/bundle-play-file.d.mts +2 -2
  47. package/dist/plays/bundle-play-file.d.ts +2 -2
  48. package/dist/plays/bundle-play-file.mjs +78 -18
  49. package/package.json +1 -1
@@ -46,7 +46,10 @@ import type { PlayQueueHint } from './governor/rate-state-backend';
46
46
  import type { MapRowOutcome } from './durability-store';
47
47
  import { stringifyPostgresJson } from './postgres-json';
48
48
  import { RUNTIME_ROW_OUTPUT_NOT_JSON_SERIALIZABLE } from './runtime-sheet-row-transition';
49
- import { RuntimeSheetRowWriter } from './runtime-sheet-row-writer';
49
+ import {
50
+ RuntimeSheetRowWriter,
51
+ type RuntimeSheetRowWriterDiagnostics,
52
+ } from './runtime-sheet-row-writer';
50
53
  import type { WorkReceiptFailureKind } from './work-receipts';
51
54
  import {
52
55
  completedMapRowOutcome,
@@ -161,6 +164,8 @@ import {
161
164
  type PlayAuthoringRuntimeContext,
162
165
  } from '../plays/authoring-contract';
163
166
  import {
167
+ DURABLE_RECEIPT_WAIT_DELAY_MS,
168
+ DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS,
164
169
  RuntimeReceiptLeaseLostError,
165
170
  RuntimeReceiptWaitTimeoutError,
166
171
  executeWithDurableRuntimeReceipt,
@@ -331,6 +336,8 @@ const rowContext = new AsyncLocalStorage<{
331
336
  tableNamespace?: string;
332
337
  rowKey?: string;
333
338
  mapScope?: MapExecutionScope;
339
+ mapStallObserver?: RuntimeMapStallObserver;
340
+ mapStallResolverToken?: number;
334
341
  /**
335
342
  * Per-resolver inline-child fan-width guard. This stays in the async row
336
343
  * scope, so it becomes collectible as soon as the resolver settles.
@@ -366,6 +373,7 @@ const MAP_PERSIST_CHUNK_BYTES = 8 * 1024 * 1024;
366
373
  const MAP_ROW_WRITER_BUFFER_BYTES = 16 * 1024 * 1024;
367
374
  const MAP_ROW_WRITER_MAX_FLUSH_MS = 100;
368
375
  const MAP_FRAME_FLUSH_INTERVAL_MS = 250;
376
+ const MAP_STALL_LOG_INTERVAL_MS = 60_000;
369
377
  // Tool scheduling lanes wait at most this long for same-lane row continuations.
370
378
  // The window is fixed from the first item and never resets on later arrivals.
371
379
  const TOOL_BATCH_COALESCE_WINDOW_MS = 5;
@@ -389,6 +397,9 @@ const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
389
397
  const DEFAULT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000 + 30_000;
390
398
  const FETCH_TRANSPORT_MAX_ATTEMPTS = 3;
391
399
  const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
400
+ const CTX_FETCH_HEADERS_TIMEOUT_MS = 30_000;
401
+ const CTX_FETCH_BODY_TIMEOUT_MS = 30_000;
402
+ const CTX_FETCH_TOTAL_TIMEOUT_MS = 120_000;
392
403
  // cloudflared returns this branded HTML when its connection to the local app
393
404
  // resets. It is a transport failure before the app can reach a provider, not a
394
405
  // provider 502; require both the branded page and our development tunnel host.
@@ -427,6 +438,388 @@ class ToolExecuteResponseBodyTimeoutError extends Error {
427
438
  }
428
439
  }
429
440
 
441
+ export class CtxFetchTimeoutError extends Error {
442
+ readonly code = 'CTX_FETCH_TIMEOUT';
443
+
444
+ constructor(
445
+ readonly phase: 'headers' | 'body' | 'total',
446
+ readonly timeoutMs: number,
447
+ ) {
448
+ super(
449
+ `[CTX_FETCH_TIMEOUT phase=${phase} timeout_ms=${timeoutMs}] ` +
450
+ `ctx.fetch exceeded its ${phase} deadline after ${timeoutMs}ms.`,
451
+ );
452
+ this.name = 'CtxFetchTimeoutError';
453
+ }
454
+ }
455
+
456
+ type CtxFetchDeadline = {
457
+ bodyMs: number;
458
+ headersMs: number;
459
+ startedAt: number;
460
+ totalMs: number;
461
+ };
462
+
463
+ function positiveTimeoutMs(
464
+ value: number | undefined,
465
+ fallback: number,
466
+ ): number {
467
+ return value === undefined || !Number.isFinite(value) || value <= 0
468
+ ? fallback
469
+ : Math.max(1, Math.floor(value));
470
+ }
471
+
472
+ function remainingCtxFetchMs(deadline: CtxFetchDeadline): number {
473
+ return Math.max(0, deadline.totalMs - (Date.now() - deadline.startedAt));
474
+ }
475
+
476
+ async function runCtxFetchPhase<T>(input: {
477
+ callerSignal?: AbortSignal | null;
478
+ controller: AbortController;
479
+ deadline: CtxFetchDeadline;
480
+ phase: 'headers' | 'body';
481
+ run: () => Promise<T>;
482
+ }): Promise<T> {
483
+ const remainingMs = remainingCtxFetchMs(input.deadline);
484
+ if (remainingMs <= 0) {
485
+ throw new CtxFetchTimeoutError('total', input.deadline.totalMs);
486
+ }
487
+ if (input.callerSignal?.aborted) {
488
+ throw (
489
+ input.callerSignal.reason ?? new DOMException('Aborted', 'AbortError')
490
+ );
491
+ }
492
+ const phaseLimitMs =
493
+ input.phase === 'headers'
494
+ ? input.deadline.headersMs
495
+ : input.deadline.bodyMs;
496
+ const timeoutMs = Math.min(phaseLimitMs, remainingMs);
497
+ const timeoutPhase = timeoutMs >= remainingMs ? 'total' : input.phase;
498
+ let timeout: ReturnType<typeof setTimeout> | null = null;
499
+ let onCallerAbort: (() => void) | null = null;
500
+ const phaseTimeout = new Promise<never>((_resolve, reject) => {
501
+ timeout = setTimeout(() => {
502
+ const error = new CtxFetchTimeoutError(
503
+ timeoutPhase,
504
+ timeoutPhase === 'total' ? input.deadline.totalMs : phaseLimitMs,
505
+ );
506
+ reject(error);
507
+ input.controller.abort(error);
508
+ }, timeoutMs);
509
+ });
510
+ const callerAbort = input.callerSignal
511
+ ? new Promise<never>((_resolve, reject) => {
512
+ onCallerAbort = () => {
513
+ const reason =
514
+ input.callerSignal?.reason ??
515
+ new DOMException('Aborted', 'AbortError');
516
+ reject(reason);
517
+ input.controller.abort(reason);
518
+ };
519
+ input.callerSignal!.addEventListener('abort', onCallerAbort, {
520
+ once: true,
521
+ });
522
+ })
523
+ : null;
524
+ try {
525
+ return await Promise.race([
526
+ input.run(),
527
+ phaseTimeout,
528
+ ...(callerAbort ? [callerAbort] : []),
529
+ ]);
530
+ } finally {
531
+ if (timeout) clearTimeout(timeout);
532
+ if (onCallerAbort) {
533
+ input.callerSignal?.removeEventListener('abort', onCallerAbort);
534
+ }
535
+ }
536
+ }
537
+
538
+ async function sleepWithinCtxFetchDeadline(input: {
539
+ callerSignal?: AbortSignal | null;
540
+ deadline: CtxFetchDeadline;
541
+ delayMs: number;
542
+ }): Promise<void> {
543
+ const controller = new AbortController();
544
+ await runCtxFetchPhase({
545
+ callerSignal: input.callerSignal,
546
+ controller,
547
+ deadline: input.deadline,
548
+ phase: 'headers',
549
+ run: () =>
550
+ new Promise<void>((resolve) => {
551
+ setTimeout(
552
+ resolve,
553
+ Math.min(input.delayMs, remainingCtxFetchMs(input.deadline)),
554
+ );
555
+ }),
556
+ });
557
+ }
558
+
559
+ async function readCtxFetchBody(input: {
560
+ callerSignal?: AbortSignal | null;
561
+ controller: AbortController;
562
+ deadline: CtxFetchDeadline;
563
+ response: Response;
564
+ }): Promise<string> {
565
+ if (!input.response.body) return '';
566
+ const reader = input.response.body.getReader();
567
+ const decoder = new TextDecoder();
568
+ try {
569
+ return await runCtxFetchPhase({
570
+ callerSignal: input.callerSignal,
571
+ controller: input.controller,
572
+ deadline: input.deadline,
573
+ phase: 'body',
574
+ run: async () => {
575
+ let body = '';
576
+ for (;;) {
577
+ const chunk = await reader.read();
578
+ if (chunk.done) return body + decoder.decode();
579
+ body += decoder.decode(chunk.value, { stream: true });
580
+ }
581
+ },
582
+ });
583
+ } catch (error) {
584
+ // Cancellation is socket hygiene, not part of the timeout contract. A
585
+ // hostile/custom stream may never resolve its cancel hook; awaiting it
586
+ // here would recreate the exact forever-pending ctx.fetch this deadline
587
+ // is meant to eliminate.
588
+ void reader.cancel(error).catch(() => {});
589
+ throw error;
590
+ } finally {
591
+ reader.releaseLock();
592
+ }
593
+ }
594
+
595
+ class RuntimeMapStallObserver {
596
+ readonly #resolvers = new Map<
597
+ number,
598
+ {
599
+ phase: 'resolver' | 'fetch';
600
+ phaseStartedAt: number;
601
+ startedAt: number;
602
+ }
603
+ >();
604
+ readonly #terminalAdmissionStartedAt = new Map<number, number>();
605
+ readonly #terminalCommitStartedAt = new Map<number, number>();
606
+ readonly #getWriterDiagnostics: () => RuntimeSheetRowWriterDiagnostics | null;
607
+ readonly #log: (line: string) => void;
608
+ readonly #mapName: string;
609
+ readonly #pageOffset: number;
610
+ readonly #intervalMs: number;
611
+ #timer: ReturnType<typeof setTimeout> | null = null;
612
+ #lastTerminalCommitAt = Date.now();
613
+ #terminalCommitted = 0;
614
+ #token = 0;
615
+
616
+ constructor(input: {
617
+ getWriterDiagnostics: () => RuntimeSheetRowWriterDiagnostics | null;
618
+ log: (line: string) => void;
619
+ mapName: string;
620
+ pageOffset: number;
621
+ intervalMs?: number;
622
+ }) {
623
+ this.#getWriterDiagnostics = input.getWriterDiagnostics;
624
+ this.#log = input.log;
625
+ this.#mapName = input.mapName;
626
+ this.#pageOffset = input.pageOffset;
627
+ this.#intervalMs = positiveTimeoutMs(
628
+ input.intervalMs,
629
+ MAP_STALL_LOG_INTERVAL_MS,
630
+ );
631
+ }
632
+
633
+ resolverStarted(): { finish: () => void; token: number } {
634
+ const token = ++this.#token;
635
+ const now = Date.now();
636
+ this.#resolvers.set(token, {
637
+ phase: 'resolver',
638
+ phaseStartedAt: now,
639
+ startedAt: now,
640
+ });
641
+ this.#schedule();
642
+ return {
643
+ token,
644
+ finish: () => {
645
+ this.#resolvers.delete(token);
646
+ this.#cancelIfIdle();
647
+ },
648
+ };
649
+ }
650
+
651
+ fetchStarted(resolverToken: number | undefined): () => void {
652
+ if (resolverToken === undefined) return () => {};
653
+ const resolver = this.#resolvers.get(resolverToken);
654
+ if (!resolver) return () => {};
655
+ resolver.phase = 'fetch';
656
+ resolver.phaseStartedAt = Date.now();
657
+ return () => {
658
+ const current = this.#resolvers.get(resolverToken);
659
+ if (!current) return;
660
+ current.phase = 'resolver';
661
+ current.phaseStartedAt = Date.now();
662
+ };
663
+ }
664
+
665
+ terminalQueued(): {
666
+ admitted: () => void;
667
+ committed: () => void;
668
+ failed: () => void;
669
+ } {
670
+ const token = ++this.#token;
671
+ const now = Date.now();
672
+ this.#terminalAdmissionStartedAt.set(token, now);
673
+ this.#schedule();
674
+ return {
675
+ admitted: () => {
676
+ this.#terminalAdmissionStartedAt.delete(token);
677
+ this.#terminalCommitStartedAt.set(token, Date.now());
678
+ },
679
+ committed: () => {
680
+ this.#terminalAdmissionStartedAt.delete(token);
681
+ if (this.#terminalCommitStartedAt.delete(token)) {
682
+ this.#terminalCommitted += 1;
683
+ this.#lastTerminalCommitAt = Date.now();
684
+ }
685
+ this.#cancelIfIdle();
686
+ },
687
+ failed: () => {
688
+ this.#terminalAdmissionStartedAt.delete(token);
689
+ this.#terminalCommitStartedAt.delete(token);
690
+ this.#cancelIfIdle();
691
+ },
692
+ };
693
+ }
694
+
695
+ stop(): void {
696
+ if (this.#timer) clearTimeout(this.#timer);
697
+ this.#timer = null;
698
+ }
699
+
700
+ #schedule(): void {
701
+ if (this.#timer) return;
702
+ this.#timer = setTimeout(() => {
703
+ this.#timer = null;
704
+ this.#emitIfStalled();
705
+ if (this.#hasUnresolved()) this.#schedule();
706
+ }, this.#intervalMs);
707
+ if (
708
+ typeof this.#timer === 'object' &&
709
+ 'unref' in this.#timer &&
710
+ typeof this.#timer.unref === 'function'
711
+ ) {
712
+ this.#timer.unref();
713
+ }
714
+ }
715
+
716
+ #cancelIfIdle(): void {
717
+ if (!this.#hasUnresolved()) this.stop();
718
+ }
719
+
720
+ #hasUnresolved(): boolean {
721
+ return (
722
+ this.#resolvers.size > 0 ||
723
+ this.#terminalAdmissionStartedAt.size > 0 ||
724
+ this.#terminalCommitStartedAt.size > 0
725
+ );
726
+ }
727
+
728
+ #emitIfStalled(): void {
729
+ const now = Date.now();
730
+ if (now - this.#lastTerminalCommitAt < this.#intervalMs) return;
731
+ const writer = this.#getWriterDiagnostics();
732
+ const unresolved =
733
+ this.#resolvers.size +
734
+ this.#terminalAdmissionStartedAt.size +
735
+ this.#terminalCommitStartedAt.size +
736
+ (writer?.activeRows ?? 0) +
737
+ (writer?.queuedTerminalRows ?? 0) +
738
+ (writer?.blockedTerminalRows ?? 0);
739
+ if (unresolved === 0) return;
740
+ const oldest = (...sources: Array<Iterable<number>>): number | null => {
741
+ let value: number | null = null;
742
+ for (const source of sources) {
743
+ for (const candidate of source) {
744
+ value = value === null ? candidate : Math.min(value, candidate);
745
+ }
746
+ }
747
+ return value;
748
+ };
749
+ const oldestUnsettledAt = oldest(
750
+ [...this.#resolvers.values()].map((resolver) => resolver.startedAt),
751
+ this.#terminalAdmissionStartedAt.values(),
752
+ this.#terminalCommitStartedAt.values(),
753
+ writer?.activeStartedAt === null || writer?.activeStartedAt === undefined
754
+ ? []
755
+ : [writer.activeStartedAt],
756
+ );
757
+ const phaseCandidates: Array<{
758
+ at: number;
759
+ phase: 'fetch' | 'resolver' | 'writer_admission' | 'writer_commit';
760
+ }> = [
761
+ ...[...this.#resolvers.values()].map((resolver) => ({
762
+ at: resolver.startedAt,
763
+ phase: resolver.phase,
764
+ })),
765
+ ...[...this.#terminalAdmissionStartedAt.values()].map((at) => ({
766
+ at,
767
+ phase: 'writer_admission' as const,
768
+ })),
769
+ ...[...this.#terminalCommitStartedAt.values()].map((at) => ({
770
+ at,
771
+ phase: 'writer_commit' as const,
772
+ })),
773
+ ];
774
+ if (
775
+ writer?.activeKind === 'terminal' &&
776
+ writer.activeRows > 0 &&
777
+ writer.activeStartedAt !== null
778
+ ) {
779
+ phaseCandidates.push({
780
+ at: writer.activeStartedAt,
781
+ phase: 'writer_commit',
782
+ });
783
+ }
784
+ const oldestPhase = phaseCandidates.reduce<
785
+ (typeof phaseCandidates)[number] | null
786
+ >(
787
+ (current, candidate) =>
788
+ current === null || candidate.at < current.at ? candidate : current,
789
+ null,
790
+ )?.phase;
791
+ const fetchResolvers = [...this.#resolvers.values()].filter(
792
+ (resolver) => resolver.phase === 'fetch',
793
+ );
794
+ this.#log(
795
+ `[runtime.map_stall] ${JSON.stringify({
796
+ map: this.#mapName,
797
+ page_offset: this.#pageOffset,
798
+ resolver_active: this.#resolvers.size,
799
+ fetch_active: fetchResolvers.length,
800
+ oldest_fetch_ms: Math.max(
801
+ 0,
802
+ now -
803
+ (oldest(
804
+ fetchResolvers.map((resolver) => resolver.phaseStartedAt),
805
+ ) ?? now),
806
+ ),
807
+ terminal_admission_waiting: this.#terminalAdmissionStartedAt.size,
808
+ terminal_commit_waiting: this.#terminalCommitStartedAt.size,
809
+ terminal_write_active:
810
+ writer?.activeKind === 'terminal' ? writer.activeRows : 0,
811
+ terminal_queued:
812
+ (writer?.queuedTerminalRows ?? 0) +
813
+ (writer?.blockedTerminalRows ?? 0),
814
+ terminal_committed: this.#terminalCommitted,
815
+ oldest_unsettled_ms: Math.max(0, now - (oldestUnsettledAt ?? now)),
816
+ oldest_phase: oldestPhase ?? 'resolver',
817
+ last_terminal_commit_ms: Math.max(0, now - this.#lastTerminalCommitAt),
818
+ })}`,
819
+ );
820
+ }
821
+ }
822
+
430
823
  async function readToolExecuteResponseBody<T>(input: {
431
824
  toolId: string;
432
825
  abortController: AbortController | null;
@@ -841,6 +1234,12 @@ type ToolExecutionApiOptions = {
841
1234
  beforeProviderCall?: () => Promise<void> | void;
842
1235
  heartbeatReceipt?: () => Promise<void> | void;
843
1236
  parkProviderCall?: () => Promise<void> | void;
1237
+ /**
1238
+ * Transfer the logical tool permit to the caller so it can remain held until
1239
+ * the decoded response is durably settled. The release callback is
1240
+ * idempotent. Callers must release it in a finally block.
1241
+ */
1242
+ retainToolSlot?: (release: () => void) => void;
844
1243
  /**
845
1244
  * Explicit graph-node attribution for this provider call (ADR 0018). Carried
846
1245
  * to the execute route, which stamps it onto the usage event so per-node
@@ -936,6 +1335,10 @@ function failedMapRowLogLabel(row: PersistableMapRow): string {
936
1335
  type FieldMapRunResult = {
937
1336
  completedRows: PersistableMapRow[];
938
1337
  failedRows: PersistableMapRow[];
1338
+ completedRowCount: number;
1339
+ failedRowCount: number;
1340
+ previewRows: Record<string, unknown>[];
1341
+ retainedRowsComplete: boolean;
939
1342
  };
940
1343
 
941
1344
  type RuntimeMapRowPersistence = {
@@ -946,6 +1349,7 @@ type RuntimeMapRowPersistence = {
946
1349
  isPersisted: (row: PersistableMapRow) => boolean;
947
1350
  checkpoint: (updates: PlayRowUpdate[]) => Promise<void>;
948
1351
  flush: () => Promise<void>;
1352
+ diagnostics: () => RuntimeSheetRowWriterDiagnostics;
949
1353
  };
950
1354
 
951
1355
  function createRuntimeMapRowPersistence(
@@ -1050,6 +1454,7 @@ function createRuntimeMapRowPersistence(
1050
1454
  flush: async () => {
1051
1455
  await writer.finish();
1052
1456
  },
1457
+ diagnostics: () => writer.diagnostics(),
1053
1458
  };
1054
1459
  }
1055
1460
 
@@ -1324,8 +1729,9 @@ class RuntimeDatasetBuilder<T extends Record<string, unknown>> {
1324
1729
 
1325
1730
  const WAITING_ROW = Symbol('deepline.waiting_row');
1326
1731
  const FAILED_ROW = Symbol('deepline.failed_row');
1732
+ const COMPLETED_ROW = Symbol('deepline.completed_row');
1327
1733
  type MapRowExecutionResult =
1328
- | Record<string, unknown>
1734
+ | typeof COMPLETED_ROW
1329
1735
  | typeof WAITING_ROW
1330
1736
  | typeof FAILED_ROW;
1331
1737
 
@@ -1446,6 +1852,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
1446
1852
  reject: (error: unknown) => void;
1447
1853
  }> = [];
1448
1854
  private runtimeToolOwnershipAssertionFlushScheduled = false;
1855
+ private runtimeToolReceiptCompletionsInFlight = new Map<string, number>();
1449
1856
  /**
1450
1857
  * Direct durable boundaries (including ctx.fetch) arrive independently from
1451
1858
  * concurrent map rows. Hold one microtask's worth so they can use the bulk
@@ -1465,6 +1872,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
1465
1872
  }>;
1466
1873
  }
1467
1874
  >();
1875
+ /**
1876
+ * Completed durable receipt payloads can expand many times beyond their wire
1877
+ * size during JSON parsing. Scheduling groups may overlap for provider
1878
+ * throughput, but only one group at a time may hydrate and deliver cached
1879
+ * results. Pending claims leave this section before provider execution, so
1880
+ * cache misses retain normal cross-group concurrency.
1881
+ */
1882
+ private runtimeReceiptHydrationTurn: Promise<void> = Promise.resolve();
1468
1883
  /**
1469
1884
  * Fixed, non-resetting coalescing deadlines for ready scheduling lanes. A
1470
1885
  * deadline starts when the first request enters an empty batch bucket. New
@@ -1850,6 +2265,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
1850
2265
  options.getToolProvider,
1851
2266
  ),
1852
2267
  resolveRateScope: options.getToolRateScope,
2268
+ maxConcurrentExternalCalls: options.maxConcurrentExternalCalls,
2269
+ maxConcurrentRows: options.maxConcurrentRows,
1853
2270
  // A root run keeps depth 0 (its first child play is depth 1) but still
1854
2271
  // seeds its own play id into the ancestry/currentPlayId so the cycle guard
1855
2272
  // catches a child re-invoking the root. createDefaultGovernanceSnapshot
@@ -3077,6 +3494,22 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
3077
3494
  return byKey;
3078
3495
  }
3079
3496
 
3497
+ private async withRuntimeReceiptHydrationTurn<T>(
3498
+ hydrate: () => Promise<T>,
3499
+ ): Promise<T> {
3500
+ const precedingTurn = this.runtimeReceiptHydrationTurn;
3501
+ let releaseTurn!: () => void;
3502
+ this.runtimeReceiptHydrationTurn = new Promise<void>((resolve) => {
3503
+ releaseTurn = resolve;
3504
+ });
3505
+ await precedingTurn;
3506
+ try {
3507
+ return await hydrate();
3508
+ } finally {
3509
+ releaseTurn();
3510
+ }
3511
+ }
3512
+
3080
3513
  private async markRuntimeStepReceiptRunning(input: {
3081
3514
  key: string;
3082
3515
  runId: string;
@@ -3587,7 +4020,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
3587
4020
  markRunningBeforeExecute?: boolean;
3588
4021
  requiresExecutionLock?: boolean;
3589
4022
  executionLockTtlMs?: number;
3590
- execute: (context: { leaseId: string | null }) => Promise<T>;
4023
+ execute: (context: {
4024
+ leaseId: string | null;
4025
+ retainExternalCallSlot: (release: () => void) => void;
4026
+ }) => Promise<T>;
3591
4027
  },
3592
4028
  ): Promise<T> {
3593
4029
  const stalePolicy = resolveDurableCallCachePolicy(
@@ -3609,32 +4045,52 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
3609
4045
  staleAfterSeconds: stalePolicy.staleAfterSeconds,
3610
4046
  cacheEpochMs: this.durableCallCacheEpochMs,
3611
4047
  });
3612
- return await executeWithDurableRuntimeReceipt<T>({
3613
- operation,
3614
- id,
3615
- runId: this.currentReceiptOwnerRunId,
3616
- receiptKey,
3617
- store: this.durableReceiptExecutionStore(),
3618
- force: opts.force === true || stalePolicy.forceRefresh,
3619
- repairRunningReceiptForSameRun: opts.repairRunningReceiptForSameRun,
3620
- repairRunningReceiptForSameRunAfterWaitTimeout:
3621
- opts.repairRunningReceiptForSameRunAfterWaitTimeout,
3622
- runningReceiptWaitMaxAttempts: opts.runningReceiptWaitMaxAttempts,
3623
- runningReceiptWaitDelayMs: opts.runningReceiptWaitDelayMs,
3624
- reclaimRunning: opts.reclaimRunning,
3625
- markSkipped: opts.markSkipped,
3626
- onRecovered: opts.onRecovered,
3627
- onClaimedResult: opts.onClaimedResult,
3628
- shouldPersistFailure: opts.shouldPersistFailure,
3629
- markRunningBeforeExecute: opts.markRunningBeforeExecute,
3630
- completedCacheOnly: operation === 'tool',
3631
- requiresExecutionLock: opts.requiresExecutionLock,
3632
- executionLockTtlMs: opts.executionLockTtlMs,
3633
- toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
3634
- formatError: (error) => this.formatRuntimeError(error),
3635
- log: (message) => this.log(message),
3636
- execute: opts.execute,
3637
- });
4048
+ let releaseExternalCallSlot: (() => void) | null = null;
4049
+ try {
4050
+ return await executeWithDurableRuntimeReceipt<T>({
4051
+ operation,
4052
+ id,
4053
+ runId: this.currentReceiptOwnerRunId,
4054
+ receiptKey,
4055
+ store: this.durableReceiptExecutionStore(),
4056
+ force: opts.force === true || stalePolicy.forceRefresh,
4057
+ repairRunningReceiptForSameRun: opts.repairRunningReceiptForSameRun,
4058
+ repairRunningReceiptForSameRunAfterWaitTimeout:
4059
+ opts.repairRunningReceiptForSameRunAfterWaitTimeout,
4060
+ runningReceiptWaitMaxAttempts: opts.runningReceiptWaitMaxAttempts,
4061
+ runningReceiptWaitDelayMs: opts.runningReceiptWaitDelayMs,
4062
+ reclaimRunning: opts.reclaimRunning,
4063
+ markSkipped: opts.markSkipped,
4064
+ onRecovered: opts.onRecovered,
4065
+ onClaimedResult: opts.onClaimedResult,
4066
+ shouldPersistFailure: opts.shouldPersistFailure,
4067
+ markRunningBeforeExecute: opts.markRunningBeforeExecute,
4068
+ completedCacheOnly: operation === 'tool',
4069
+ withCompletedReceiptHydration: (hydrate) =>
4070
+ this.withRuntimeReceiptHydrationTurn(hydrate),
4071
+ requiresExecutionLock: opts.requiresExecutionLock,
4072
+ executionLockTtlMs: opts.executionLockTtlMs,
4073
+ toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
4074
+ formatError: (error) => this.formatRuntimeError(error),
4075
+ log: (message) => this.log(message),
4076
+ execute: ({ leaseId }) =>
4077
+ opts.execute({
4078
+ leaseId,
4079
+ retainExternalCallSlot: (release) => {
4080
+ if (releaseExternalCallSlot) {
4081
+ release();
4082
+ throw new Error(
4083
+ `ctx.${operation}(${id}) attempted to retain more than one external-call slot.`,
4084
+ );
4085
+ }
4086
+ releaseExternalCallSlot = release;
4087
+ },
4088
+ }),
4089
+ });
4090
+ } finally {
4091
+ const release = releaseExternalCallSlot as (() => void) | null;
4092
+ release?.();
4093
+ }
3638
4094
  }
3639
4095
 
3640
4096
  private get currentRunId(): string {
@@ -4669,32 +5125,39 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4669
5125
  }),
4670
5126
  requestInput: request.input,
4671
5127
  });
4672
- let completed: RuntimeStepReceipt | null | undefined = null;
4673
- if (receiptKey) {
4674
- try {
4675
- completed = (
4676
- await this.completeRuntimeStepReceipts([
4677
- {
4678
- key: receiptKey,
4679
- runId: this.currentReceiptOwnerRunId,
4680
- leaseId: request.receiptLeaseId ?? null,
4681
- output: serializeToolExecuteResult(wrapped),
4682
- },
4683
- ])
4684
- ).get(receiptKey);
4685
- } catch (receiptError) {
4686
- // Completion failed AFTER a successful (billed) tool call: trip the
4687
- // breaker so remaining rows stop dispatching new provider calls.
4688
- tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
4689
- throw receiptError;
5128
+ const releaseCompletionHold = this.holdRuntimeToolReceiptCompletions(
5129
+ receiptKey ? [receiptKey] : [],
5130
+ );
5131
+ try {
5132
+ let completed: RuntimeStepReceipt | null | undefined = null;
5133
+ if (receiptKey) {
5134
+ try {
5135
+ completed = (
5136
+ await this.completeRuntimeStepReceipts([
5137
+ {
5138
+ key: receiptKey,
5139
+ runId: this.currentReceiptOwnerRunId,
5140
+ leaseId: request.receiptLeaseId ?? null,
5141
+ output: serializeToolExecuteResult(wrapped),
5142
+ },
5143
+ ])
5144
+ ).get(receiptKey);
5145
+ } catch (receiptError) {
5146
+ // Completion failed AFTER a successful (billed) tool call: trip the
5147
+ // breaker so remaining rows stop dispatching new provider calls.
5148
+ tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
5149
+ throw receiptError;
5150
+ }
4690
5151
  }
5152
+ return await this.finalizeResolvedToolCall(
5153
+ toolId,
5154
+ request,
5155
+ wrapped,
5156
+ completed,
5157
+ );
5158
+ } finally {
5159
+ releaseCompletionHold();
4691
5160
  }
4692
- return await this.finalizeResolvedToolCall(
4693
- toolId,
4694
- request,
4695
- wrapped,
4696
- completed,
4697
- );
4698
5161
  }
4699
5162
 
4700
5163
  private async resolveToolCallBatchResults(
@@ -4739,37 +5202,70 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4739
5202
  ]
4740
5203
  : [];
4741
5204
  });
4742
- let completedByKey = new Map<string, RuntimeStepReceipt>();
4743
- if (receiptInputs.length > 0) {
4744
- try {
4745
- completedByKey = await this.completeRuntimeStepReceipts(receiptInputs);
4746
- } catch (receiptError) {
4747
- // Completion failed AFTER successful (billed) tool calls: trip the
4748
- // breaker so remaining rows stop dispatching new provider calls.
4749
- tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
4750
- await Promise.all(
4751
- wrappedEntries.map((entry) =>
4752
- this.rejectToolCall(toolId, entry.request, receiptError, {
4753
- persistReceiptFailure: false,
4754
- }),
5205
+ const releaseCompletionHold = this.holdRuntimeToolReceiptCompletions(
5206
+ receiptInputs.map((receipt) => receipt.key),
5207
+ );
5208
+ try {
5209
+ let completedByKey = new Map<string, RuntimeStepReceipt>();
5210
+ if (receiptInputs.length > 0) {
5211
+ try {
5212
+ completedByKey =
5213
+ await this.completeRuntimeStepReceipts(receiptInputs);
5214
+ } catch (receiptError) {
5215
+ // Completion failed AFTER successful (billed) tool calls: trip the
5216
+ // breaker so remaining rows stop dispatching new provider calls.
5217
+ tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
5218
+ await Promise.all(
5219
+ wrappedEntries.map((entry) =>
5220
+ this.rejectToolCall(toolId, entry.request, receiptError, {
5221
+ persistReceiptFailure: false,
5222
+ }),
5223
+ ),
5224
+ );
5225
+ throw receiptError;
5226
+ }
5227
+ }
5228
+ const results: unknown[] = [];
5229
+ for (const entry of wrappedEntries) {
5230
+ const receiptKey = entry.request.receiptKey?.trim() || null;
5231
+ results.push(
5232
+ await this.finalizeResolvedToolCall(
5233
+ toolId,
5234
+ entry.request,
5235
+ entry.wrapped,
5236
+ receiptKey ? completedByKey.get(receiptKey) : null,
4755
5237
  ),
4756
5238
  );
4757
- throw receiptError;
4758
5239
  }
5240
+ return results;
5241
+ } finally {
5242
+ releaseCompletionHold();
4759
5243
  }
4760
- const results: unknown[] = [];
4761
- for (const entry of wrappedEntries) {
4762
- const receiptKey = entry.request.receiptKey?.trim() || null;
4763
- results.push(
4764
- await this.finalizeResolvedToolCall(
4765
- toolId,
4766
- entry.request,
4767
- entry.wrapped,
4768
- receiptKey ? completedByKey.get(receiptKey) : null,
4769
- ),
5244
+ }
5245
+
5246
+ private holdRuntimeToolReceiptCompletions(receiptKeys: string[]): () => void {
5247
+ const uniqueKeys = [
5248
+ ...new Set(receiptKeys.map((key) => key.trim())),
5249
+ ].filter(Boolean);
5250
+ for (const key of uniqueKeys) {
5251
+ this.runtimeToolReceiptCompletionsInFlight.set(
5252
+ key,
5253
+ (this.runtimeToolReceiptCompletionsInFlight.get(key) ?? 0) + 1,
4770
5254
  );
4771
5255
  }
4772
- return results;
5256
+ let released = false;
5257
+ return () => {
5258
+ if (released) return;
5259
+ released = true;
5260
+ for (const key of uniqueKeys) {
5261
+ const count = this.runtimeToolReceiptCompletionsInFlight.get(key) ?? 0;
5262
+ if (count <= 1) {
5263
+ this.runtimeToolReceiptCompletionsInFlight.delete(key);
5264
+ } else {
5265
+ this.runtimeToolReceiptCompletionsInFlight.set(key, count - 1);
5266
+ }
5267
+ }
5268
+ };
4773
5269
  }
4774
5270
 
4775
5271
  private async finalizeResolvedToolCall(
@@ -5514,6 +6010,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
5514
6010
  activeRowsMemoryBudgetBytes:
5515
6011
  mapMemoryLimits.activeRowsBudgetBytes,
5516
6012
  incrementalPersistence,
6013
+ releasePersistedRowsOnMemoryPressure: true,
5517
6014
  },
5518
6015
  );
5519
6016
  } catch (error) {
@@ -5560,9 +6057,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
5560
6057
  this.activeMapCellMeta = null;
5561
6058
  this.activeMapCheckpointUpdates = null;
5562
6059
 
6060
+ for (const materializedRow of mapResult.previewRows) {
6061
+ if (previewRows.length < 5) previewRows.push(materializedRow);
6062
+ }
6063
+ if (!mapResult.retainedRowsComplete) {
6064
+ disableImmediateMaterializedCache(
6065
+ 'the current execution page exceeded the in-memory result budget',
6066
+ );
6067
+ }
5563
6068
  for (const row of mapResult.completedRows) {
5564
6069
  const materializedRow = this.toMaterializedOutputRow(row.data);
5565
- if (previewRows.length < 5) previewRows.push(materializedRow);
5566
6070
  if (
5567
6071
  immediateMaterializedCacheEnabled &&
5568
6072
  immediateMaterializedRows.length < materializedResultRowLimit
@@ -5607,8 +6111,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
5607
6111
  }
5608
6112
  }
5609
6113
  executedCount += rowsToExecute.length;
5610
- successfulCount += mapResult.completedRows.length;
5611
- failedCount += mapResult.failedRows.length;
6114
+ successfulCount += mapResult.completedRowCount;
6115
+ failedCount += mapResult.failedRowCount;
5612
6116
  processedCount += rowsToExecute.length;
5613
6117
  page.rows.length = 0;
5614
6118
  }
@@ -6464,6 +6968,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6464
6968
  progressFailedOffset?: number;
6465
6969
  progressTotalRows?: number;
6466
6970
  incrementalPersistence?: RuntimeMapRowPersistence | null;
6971
+ /**
6972
+ * Runtime Sheet-backed maps may discard their optional in-memory result
6973
+ * cache once it reaches the resident-byte budget. The durable sheet is
6974
+ * already authoritative; retaining the same completed payloads until a
6975
+ * source page ends must never turn a successful large map into an OOM.
6976
+ */
6977
+ releasePersistedRowsOnMemoryPressure?: boolean;
6467
6978
  },
6468
6979
  ): Promise<FieldMapRunResult> {
6469
6980
  const fieldEntries = Object.entries(definition);
@@ -6508,12 +7019,41 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6508
7019
  const failFastOnRowError = runtimeOptions?.onRowError === 'fail';
6509
7020
  const completedRowsToPersist: PersistableMapRow[] = [];
6510
7021
  const failedRowsToPersist: PersistableMapRow[] = [];
7022
+ const previewRows: Record<string, unknown>[] = [];
7023
+ let completedRowCount = 0;
7024
+ let failedRowCount = 0;
7025
+ let retainedRowsComplete = true;
6511
7026
  const incrementalPersistence =
6512
7027
  runtimeOptions?.incrementalPersistence ?? null;
7028
+ let mapStallObserver: RuntimeMapStallObserver | null = null;
6513
7029
  const retainedRowsMemoryTracker = createRuntimeMapRetainedRowsTracker({
6514
7030
  mapName: normalizedTableNamespace,
6515
7031
  budgetBytes: runtimeOptions?.retainedRowsMemoryBudgetBytes,
6516
7032
  });
7033
+ const retainPersistedRow = (
7034
+ row: PersistableMapRow,
7035
+ target: PersistableMapRow[],
7036
+ ): void => {
7037
+ if (!retainedRowsComplete) return;
7038
+ try {
7039
+ retainedRowsMemoryTracker.track(row);
7040
+ target.push(row);
7041
+ } catch (error) {
7042
+ if (
7043
+ !runtimeOptions?.releasePersistedRowsOnMemoryPressure ||
7044
+ !incrementalPersistence ||
7045
+ !(error instanceof RuntimeMapMemoryLimitError)
7046
+ ) {
7047
+ throw error;
7048
+ }
7049
+ completedRowsToPersist.length = 0;
7050
+ failedRowsToPersist.length = 0;
7051
+ retainedRowsComplete = false;
7052
+ this.log(
7053
+ `Dataset ${normalizedTableNamespace} released its in-memory result cache after ${completedRowCount + failedRowCount} row(s); using the durable Runtime Sheet.`,
7054
+ );
7055
+ }
7056
+ };
6517
7057
  const enqueueIncrementalPersist = (
6518
7058
  row: PersistableMapRow,
6519
7059
  onCommitted: () => void,
@@ -6523,11 +7063,24 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6523
7063
  return Promise.resolve();
6524
7064
  }
6525
7065
  const settlement = incrementalPersistence.persistRows([row]);
6526
- void settlement.committed.then(onCommitted).catch(() => {
6527
- // The terminal map barrier awaits the writer and surfaces the same
6528
- // latched failure. This continuation only publishes durable progress.
6529
- });
6530
- return settlement.admitted;
7066
+ const stall = mapStallObserver?.terminalQueued();
7067
+ void settlement.committed
7068
+ .then(() => {
7069
+ stall?.committed();
7070
+ onCommitted();
7071
+ })
7072
+ .catch(() => {
7073
+ stall?.failed();
7074
+ // The terminal map barrier awaits the writer and surfaces the same
7075
+ // latched failure. This continuation only publishes durable progress.
7076
+ });
7077
+ return settlement.admitted.then(
7078
+ () => stall?.admitted(),
7079
+ (error) => {
7080
+ stall?.failed();
7081
+ throw error;
7082
+ },
7083
+ );
6531
7084
  };
6532
7085
 
6533
7086
  if (completedRows > 0 || pendingRows !== totalRows) {
@@ -6727,6 +7280,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6727
7280
  }),
6728
7281
  ),
6729
7282
  failedRows: [],
7283
+ completedRowCount: results.length,
7284
+ failedRowCount: 0,
7285
+ previewRows: results.slice(0, 5),
7286
+ retainedRowsComplete: true,
6730
7287
  };
6731
7288
  }
6732
7289
 
@@ -6771,6 +7328,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6771
7328
  this.resourceGovernor.observe({
6772
7329
  memoryBytes: rowAdmission.estimatedResidentBytes,
6773
7330
  });
7331
+ mapStallObserver = new RuntimeMapStallObserver({
7332
+ getWriterDiagnostics: () => incrementalPersistence?.diagnostics() ?? null,
7333
+ log: (line) => this.log(line),
7334
+ mapName: normalizedTableNamespace,
7335
+ pageOffset:
7336
+ runtimeOptions?.executionRowIndexes?.[0] ??
7337
+ runtimeOptions?.progressCompletedOffset ??
7338
+ 0,
7339
+ intervalMs: this.#options.runtimeMapStallLogIntervalMs,
7340
+ });
6774
7341
  const settledResults = new Array<MapRowExecutionResult | undefined>(
6775
7342
  items.length,
6776
7343
  );
@@ -6803,6 +7370,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6803
7370
  const globalRowSlot = await this.resourceGovernor.acquireRow({
6804
7371
  estimatedBytes: runtimeMapJsonByteLength(baseRow),
6805
7372
  });
7373
+ const resolverObservation = mapStallObserver?.resolverStarted();
7374
+ let resolverObservationActive = resolverObservation !== undefined;
7375
+ const finishResolver = () => {
7376
+ if (!resolverObservationActive) return;
7377
+ resolverObservationActive = false;
7378
+ resolverObservation?.finish();
7379
+ };
6806
7380
  try {
6807
7381
  for (const [fieldName, resolver] of fieldEntries) {
6808
7382
  activeFieldName = fieldName;
@@ -6834,6 +7408,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6834
7408
  tableNamespace: normalizedTableNamespace,
6835
7409
  rowKey,
6836
7410
  mapScope,
7411
+ mapStallObserver: mapStallObserver ?? undefined,
7412
+ mapStallResolverToken: resolverObservation?.token,
6837
7413
  },
6838
7414
  async () =>
6839
7415
  await this.resolveMapFieldValue(
@@ -6860,7 +7436,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6860
7436
 
6861
7437
  value = null;
6862
7438
  computedFields[fieldName] = value;
6863
- this.rowStates.get(idx)?.results.set(fieldName, value);
6864
7439
  const formattedError = this.formatRuntimeError(error);
6865
7440
  this.emitScopedFieldMetaUpdate({
6866
7441
  rowId: idx,
@@ -6888,8 +7463,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6888
7463
  : {}),
6889
7464
  error: formattedError,
6890
7465
  });
6891
- retainedRowsMemoryTracker.track(failedRow);
6892
- failedRowsToPersist.push(failedRow);
7466
+ // This row has not entered incremental persistence because the
7467
+ // fail-fast error is about to unwind the resolver. Retain this
7468
+ // one terminal fact even if the optional completed-row cache was
7469
+ // already released, so the outer failure barrier can persist it.
7470
+ if (retainedRowsComplete) {
7471
+ retainPersistedRow(failedRow, failedRowsToPersist);
7472
+ }
7473
+ if (!retainedRowsComplete) {
7474
+ failedRowsToPersist.push(failedRow);
7475
+ }
6893
7476
  throw error;
6894
7477
  }
6895
7478
  const failedData = this.toPersistedOutputRow(
@@ -6903,8 +7486,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6903
7486
  ...(cellMetaPatch ? { cellMetaPatch } : {}),
6904
7487
  error: formattedError,
6905
7488
  });
6906
- retainedRowsMemoryTracker.track(failedRow);
6907
- failedRowsToPersist.push(failedRow);
7489
+ failedRowCount += 1;
7490
+ retainPersistedRow(failedRow, failedRowsToPersist);
6908
7491
  this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
6909
7492
  rowId: idx,
6910
7493
  status: 'failed',
@@ -6913,6 +7496,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6913
7496
  error: formattedError,
6914
7497
  dataPatch: {},
6915
7498
  });
7499
+ finishResolver();
6916
7500
  await enqueueIncrementalPersist(failedRow, () => {
6917
7501
  this.clearActiveMapCheckpointUpdate(
6918
7502
  rowKey,
@@ -6923,7 +7507,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6923
7507
  return FAILED_ROW;
6924
7508
  }
6925
7509
  computedFields[fieldName] = cellValue;
6926
- this.rowStates.get(idx)?.results.set(fieldName, cellValue);
6927
7510
  const currentCellMeta =
6928
7511
  this.activeMapCellMeta?.get(rowKey)?.[fieldName];
6929
7512
  const currentCellRecord =
@@ -6975,6 +7558,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6975
7558
  });
6976
7559
  }
6977
7560
 
7561
+ finishResolver();
6978
7562
  const merged = cloneCsvAliasedRow(baseRow, computedFields);
6979
7563
  activeFieldName = null;
6980
7564
  this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
@@ -6985,7 +7569,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6985
7569
  error: null,
6986
7570
  dataPatch: {},
6987
7571
  });
6988
- const publicRow = this.toPublicOutputRow(merged);
6989
7572
  const completedRow: PersistableMapRow = completedMapRowOutcome({
6990
7573
  key: rowKey,
6991
7574
  inputIndex: rowIndex,
@@ -6994,13 +7577,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6994
7577
  ? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
6995
7578
  : {}),
6996
7579
  });
6997
- retainedRowsMemoryTracker.track(completedRow);
6998
- completedRowsToPersist.push(completedRow);
7580
+ completedRowCount += 1;
7581
+ if (previewRows.length < 5) {
7582
+ previewRows.push(this.toMaterializedOutputRow(completedRow.data));
7583
+ }
7584
+ retainPersistedRow(completedRow, completedRowsToPersist);
6999
7585
  await enqueueIncrementalPersist(completedRow, () => {
7000
7586
  this.clearActiveMapCheckpointUpdate(rowKey, normalizedTableNamespace);
7001
7587
  updateMapFrameProgress({ completedRowKey: rowKey });
7002
7588
  });
7003
- return publicRow;
7589
+ return COMPLETED_ROW;
7004
7590
  } catch (error) {
7005
7591
  if (isPlayRowExecutionSuspendedError(error)) {
7006
7592
  this.pendingRowEventBoundaries.push(error.boundary);
@@ -7043,6 +7629,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7043
7629
  });
7044
7630
  throw error;
7045
7631
  } finally {
7632
+ finishResolver();
7046
7633
  globalRowSlot.release();
7047
7634
  }
7048
7635
  };
@@ -7104,16 +7691,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7104
7691
  const rawItem = this.toOutputRow(
7105
7692
  items[index] as Record<string, unknown>,
7106
7693
  );
7694
+ const key = executionRowKey(rawItem, index);
7107
7695
  if (result === WAITING_ROW) {
7108
- const key = executionRowKey(rawItem, index);
7109
7696
  if (key) {
7110
7697
  completedRowKeys.add(key);
7111
7698
  }
7112
7699
  accountedRowIndexes.add(executionRowIndex(index));
7113
7700
  continue;
7114
7701
  }
7115
- const row = result as Record<string, unknown>;
7116
- const key = executionRowKey(row, index);
7117
7702
  if (key) completedRowKeys.add(key);
7118
7703
  }
7119
7704
  this.setMapFrame({
@@ -7163,6 +7748,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7163
7748
  boundaries: uniqueBoundaries,
7164
7749
  });
7165
7750
  }
7751
+ for (let index = 0; index < rowResults.length; index += 1) {
7752
+ if (rowResults[index] !== WAITING_ROW) {
7753
+ accountedRowIndexes.add(executionRowIndex(index));
7754
+ }
7755
+ }
7166
7756
  if (accountedRowIndexes.size !== items.length) {
7167
7757
  const expectedRowIndexes = items.map((_item, index) =>
7168
7758
  executionRowIndex(index),
@@ -7176,10 +7766,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7176
7766
  `missing input index(es): ${missingRowIndexes.slice(0, 10).join(', ') || 'unknown'}.`,
7177
7767
  );
7178
7768
  }
7179
- const results = rowResults.filter(
7180
- (result): result is Record<string, unknown> =>
7181
- result !== WAITING_ROW && result !== FAILED_ROW,
7182
- );
7769
+ const succeededRows = rowResults.filter(
7770
+ (result) => result === COMPLETED_ROW,
7771
+ ).length;
7183
7772
  // A row is acknowledged only after its terminal sheet batch commits. The
7184
7773
  // incremental promises publish their frame updates from the commit
7185
7774
  // continuation, so this barrier also makes the terminal map event derive
@@ -7203,13 +7792,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7203
7792
  );
7204
7793
  }
7205
7794
  this.log(
7206
- `Map completed: ${results.length + completedRows} results (${results.length} succeeded, ${failedRowsToPersist.length} failed, ${completedRows} duplicate keys skipped)`,
7795
+ `Map completed: ${succeededRows + completedRows} results (${succeededRows} succeeded, ${failedRowCount} failed, ${completedRows} duplicate keys skipped)`,
7207
7796
  );
7208
7797
  return {
7209
7798
  completedRows: [...completedRowsToPersist].sort(
7210
7799
  comparePersistableMapRowsByInputIndex,
7211
7800
  ),
7212
7801
  failedRows: failedRowsToPersist,
7802
+ completedRowCount,
7803
+ failedRowCount,
7804
+ previewRows,
7805
+ retainedRowsComplete,
7213
7806
  };
7214
7807
  }
7215
7808
 
@@ -7719,6 +8312,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7719
8312
  }
7720
8313
  }
7721
8314
 
8315
+ private async rejectUnresolvedToolCalls(
8316
+ requests: readonly ToolCallRequest[],
8317
+ error: unknown,
8318
+ ): Promise<void> {
8319
+ await Promise.all(
8320
+ requests.map(async (request) => {
8321
+ if (!this.toolCallResolvers.has(request.callId)) return;
8322
+ await this.rejectToolCall(request.toolId, request, error, {
8323
+ persistReceiptFailure: false,
8324
+ });
8325
+ }),
8326
+ );
8327
+ }
8328
+
7722
8329
  private takeDispatchableToolCalls(
7723
8330
  nowMs: number,
7724
8331
  blockedLaneKeys: ReadonlySet<string>,
@@ -7886,9 +8493,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7886
8493
  );
7887
8494
  }
7888
8495
  const tracked = this.executeBatchedToolCalls(dispatchable.requests)
7889
- .catch((error) => {
8496
+ .catch(async (error) => {
7890
8497
  this.toolDispatcherFailure ??= error;
7891
8498
  this.rejectQueuedToolCalls(error);
8499
+ await this.rejectUnresolvedToolCalls(
8500
+ dispatchable.requests,
8501
+ error,
8502
+ );
7892
8503
  })
7893
8504
  .finally(() => {
7894
8505
  for (const laneKey of dispatchable.laneKeys) {
@@ -8059,6 +8670,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8059
8670
 
8060
8671
  const executeTool = async (context?: {
8061
8672
  leaseId?: string | null;
8673
+ retainExternalCallSlot?: (release: () => void) => void;
8062
8674
  }): Promise<unknown> => {
8063
8675
  if (eventWaitHandler) {
8064
8676
  return this.executeIntegrationEventWaitTool(
@@ -8190,6 +8802,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8190
8802
  options?.timeoutMs,
8191
8803
  this.currentAuthoringContractEdition,
8192
8804
  ),
8805
+ retainToolSlot: context?.retainExternalCallSlot,
8193
8806
  ...(directReceiptLeaseId &&
8194
8807
  (this.#options.heartbeatRuntimeStepReceipts ||
8195
8808
  this.#options.getRuntimeStepReceipt ||
@@ -8406,7 +9019,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8406
9019
  shouldPersistFailure: (error) =>
8407
9020
  !(error instanceof ToolExecuteAuthScopeChangedError),
8408
9021
  markRunningBeforeExecute: false,
8409
- execute: ({ leaseId }) => executeTool({ leaseId }),
9022
+ execute: ({ leaseId, retainExternalCallSlot }) =>
9023
+ executeTool({ leaseId, retainExternalCallSlot }),
8410
9024
  runningReceiptWaitMaxAttempts:
8411
9025
  resolveRuntimeToolReceiptWaitMaxAttempts(
8412
9026
  options?.receiptWaitMs !== undefined
@@ -8943,7 +9557,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8943
9557
  assertSecretAuthUsesTls(init.auth, input, 'ctx.fetch');
8944
9558
  const secretHeaderMarkers = secretAuthHeaderMarkers(init.auth);
8945
9559
 
8946
- return this.executeWithRuntimeReceipt<PlayFetchResponse>(
9560
+ const execution = this.executeWithRuntimeReceipt<PlayFetchResponse>(
8947
9561
  'fetch',
8948
9562
  normalizedKey,
8949
9563
  this.currentRunId,
@@ -8961,7 +9575,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8961
9575
  }),
8962
9576
  ),
8963
9577
  staleAfterSeconds: options?.staleAfterSeconds,
8964
- execute: async () => {
9578
+ execute: async ({ retainExternalCallSlot }) => {
8965
9579
  const url = input.toString();
8966
9580
  const method = (init.method ?? 'GET').toUpperCase();
8967
9581
  const secretHeaders = await this.resolveSecretAuth(init.auth);
@@ -8989,6 +9603,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8989
9603
  const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
8990
9604
  if (existing?.kind === 'fetch' && 'output' in existing) {
8991
9605
  this.log(`ctx.fetch(${url}): recovered response from checkpoint`);
9606
+ if (this.durableDirectToolResultsBackedByReceipts) {
9607
+ // The outer durable receipt is the replay authority in hosted
9608
+ // runtimes. A legacy checkpoint fetch may seed that receipt
9609
+ // once, but it must not remain as a second full response-body
9610
+ // cache for the lifetime of a large map.
9611
+ delete this.checkpoint.resolvedBoundaries?.[boundaryId];
9612
+ }
8992
9613
  return existing.output as PlayFetchResponse;
8993
9614
  }
8994
9615
 
@@ -9011,7 +9632,28 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9011
9632
  providerResourceKey: CTX_FETCH_EGRESS_TOOL_ID,
9012
9633
  toolId: CTX_FETCH_EGRESS_TOOL_ID,
9013
9634
  });
9635
+ retainExternalCallSlot(() => egressSlot.release());
9636
+ const finishFetchObservation =
9637
+ rowStore?.mapStallObserver?.fetchStarted(
9638
+ rowStore.mapStallResolverToken,
9639
+ );
9640
+ const fetchDeadline: CtxFetchDeadline = {
9641
+ startedAt: Date.now(),
9642
+ headersMs: positiveTimeoutMs(
9643
+ this.#options.ctxFetchTimeouts?.headersMs,
9644
+ CTX_FETCH_HEADERS_TIMEOUT_MS,
9645
+ ),
9646
+ bodyMs: positiveTimeoutMs(
9647
+ this.#options.ctxFetchTimeouts?.bodyMs,
9648
+ CTX_FETCH_BODY_TIMEOUT_MS,
9649
+ ),
9650
+ totalMs: positiveTimeoutMs(
9651
+ this.#options.ctxFetchTimeouts?.totalMs,
9652
+ CTX_FETCH_TOTAL_TIMEOUT_MS,
9653
+ ),
9654
+ };
9014
9655
  let response: Response | null = null;
9656
+ let bodyText: string | null = null;
9015
9657
  try {
9016
9658
  const canRetryTransport = ['GET', 'HEAD', 'OPTIONS'].includes(
9017
9659
  method,
@@ -9022,13 +9664,40 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9022
9664
  attempt <= FETCH_TRANSPORT_MAX_ATTEMPTS;
9023
9665
  attempt += 1
9024
9666
  ) {
9667
+ const attemptController = new AbortController();
9025
9668
  try {
9026
- response = await safePublicFetch(url, fetchInit, {
9027
- fetchImpl: this.#options.fetchImpl,
9028
- sensitiveHeaders: Object.keys(secretHeaderMarkers),
9669
+ const attemptInit = {
9670
+ ...fetchInit,
9671
+ signal: init.signal
9672
+ ? AbortSignal.any([init.signal, attemptController.signal])
9673
+ : attemptController.signal,
9674
+ };
9675
+ response = await runCtxFetchPhase({
9676
+ callerSignal: init.signal,
9677
+ controller: attemptController,
9678
+ deadline: fetchDeadline,
9679
+ phase: 'headers',
9680
+ run: () =>
9681
+ safePublicFetch(url, attemptInit, {
9682
+ fetchImpl: this.#options.fetchImpl,
9683
+ sensitiveHeaders: Object.keys(secretHeaderMarkers),
9684
+ }),
9685
+ });
9686
+ bodyText = await readCtxFetchBody({
9687
+ callerSignal: init.signal,
9688
+ controller: attemptController,
9689
+ deadline: fetchDeadline,
9690
+ response,
9029
9691
  });
9030
9692
  break;
9031
9693
  } catch (error) {
9694
+ void response?.body?.cancel(error).catch(() => undefined);
9695
+ response = null;
9696
+ bodyText = null;
9697
+ if (error instanceof CtxFetchTimeoutError) throw error;
9698
+ if (init.signal?.aborted) {
9699
+ throw init.signal.reason ?? error;
9700
+ }
9032
9701
  if (isUnsafeOutboundUrlError(error)) {
9033
9702
  throw error;
9034
9703
  }
@@ -9041,12 +9710,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9041
9710
  this.log(
9042
9711
  `ctx.fetch(${method} ${url}) transport failed on attempt ${attempt}/${FETCH_TRANSPORT_MAX_ATTEMPTS}; retrying: ${message}`,
9043
9712
  );
9044
- await new Promise((resolve) =>
9045
- setTimeout(
9046
- resolve,
9047
- FETCH_TRANSPORT_RETRY_DELAY_MS * attempt,
9048
- ),
9049
- );
9713
+ await sleepWithinCtxFetchDeadline({
9714
+ callerSignal: init.signal,
9715
+ deadline: fetchDeadline,
9716
+ delayMs: FETCH_TRANSPORT_RETRY_DELAY_MS * attempt,
9717
+ });
9050
9718
  continue;
9051
9719
  }
9052
9720
  throw new Error(
@@ -9059,7 +9727,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9059
9727
  `ctx.fetch(${method} ${url}) failed before receiving a response.`,
9060
9728
  );
9061
9729
  }
9062
- const bodyText = await response.text();
9730
+ if (bodyText === null) {
9731
+ throw new Error(
9732
+ `ctx.fetch(${method} ${url}) failed while reading the response body.`,
9733
+ );
9734
+ }
9063
9735
  const redactedBodyText = this.secretRedactor.redactString(bodyText);
9064
9736
  const output: PlayFetchResponse = {
9065
9737
  ok: response.ok,
@@ -9075,24 +9747,27 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9075
9747
  ),
9076
9748
  };
9077
9749
 
9078
- this.checkpoint.resolvedBoundaries = {
9079
- ...(this.checkpoint.resolvedBoundaries ?? {}),
9080
- [boundaryId]: {
9081
- kind: 'fetch',
9082
- url,
9083
- method,
9084
- output,
9085
- completedAt: Date.now(),
9086
- },
9087
- };
9088
- this.#options.onBatchComplete?.(this.checkpoint);
9089
- return output;
9090
- } finally {
9091
- egressSlot.release();
9092
- }
9093
- },
9750
+ if (!this.durableDirectToolResultsBackedByReceipts) {
9751
+ this.checkpoint.resolvedBoundaries = {
9752
+ ...(this.checkpoint.resolvedBoundaries ?? {}),
9753
+ [boundaryId]: {
9754
+ kind: 'fetch',
9755
+ url,
9756
+ method,
9757
+ output,
9758
+ completedAt: Date.now(),
9759
+ },
9760
+ };
9761
+ this.#options.onBatchComplete?.(this.checkpoint);
9762
+ }
9763
+ return output;
9764
+ } finally {
9765
+ finishFetchObservation?.();
9766
+ }
9767
+ },
9094
9768
  },
9095
9769
  );
9770
+ return execution;
9096
9771
  }
9097
9772
 
9098
9773
  async step<T>(
@@ -9213,9 +9888,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9213
9888
 
9214
9889
  // ——— Batched tool call execution ———
9215
9890
 
9216
- private startPendingToolReceiptHeartbeat(requests: ToolCallRequest[]): {
9891
+ private startPendingToolReceiptHeartbeat(
9892
+ requests: ToolCallRequest[],
9893
+ claimsEstablishedExecutionFence: boolean,
9894
+ ): {
9217
9895
  stop: () => void;
9218
9896
  leaseLost: () => unknown | null;
9897
+ confirmOwned: () => Promise<void>;
9898
+ assertOwned: () => void;
9219
9899
  } | null {
9220
9900
  const heartbeatReceipts = this.#options.heartbeatRuntimeStepReceipts;
9221
9901
  const ownedRequests = requests.filter(
@@ -9235,66 +9915,87 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9235
9915
  null,
9236
9916
  );
9237
9917
  let leaseLost: unknown | null = null;
9238
- const supervisor = createRuntimeReceiptHeartbeatSupervisor({
9239
- intervalMs: runtimeLeaseHeartbeatIntervalFromExpiry({
9240
- leaseExpiresAt: earliestLeaseExpiry,
9241
- fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
9242
- }),
9243
- heartbeat: async () => {
9244
- // Receipt claims are acquired in bulk before the provider-shaped
9245
- // dispatcher admits every call. Renew the undispatched tail as well as
9246
- // in-flight calls; otherwise a slow provider can let later claims
9247
- // expire while they are still waiting behind the concurrency gate.
9248
- const unsettled = ownedRequests.filter((request) =>
9249
- this.toolCallResolvers.has(request.callId),
9250
- );
9251
- if (unsettled.length === 0) return 'terminal';
9252
- const byReceiptKey = new Map<string, ToolCallRequest[]>();
9253
- for (const request of unsettled) {
9254
- const receiptKey = request.receiptKey!;
9255
- const group = byReceiptKey.get(receiptKey) ?? [];
9256
- group.push(request);
9257
- byReceiptKey.set(receiptKey, group);
9918
+ let ownershipConfirmedUntilMs = Number.isFinite(
9919
+ Date.parse(earliestLeaseExpiry ?? ''),
9920
+ )
9921
+ ? Date.parse(earliestLeaseExpiry!)
9922
+ : Date.now() + PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS;
9923
+ const firstOwnedRequest = ownedRequests[0]!;
9924
+ const heartbeatOwnedRequests = async () => {
9925
+ // Receipt claims are acquired in bulk before the provider-shaped
9926
+ // dispatcher admits every call. Renew the undispatched tail as well as
9927
+ // in-flight calls; otherwise a slow provider can let later claims
9928
+ // expire while they are still waiting behind the concurrency gate.
9929
+ const unsettled = ownedRequests.filter((request) =>
9930
+ this.toolCallResolvers.has(request.callId),
9931
+ );
9932
+ if (unsettled.length === 0) return 'terminal' as const;
9933
+ const byReceiptKey = new Map<string, ToolCallRequest[]>();
9934
+ for (const request of unsettled) {
9935
+ const receiptKey = request.receiptKey!;
9936
+ const group = byReceiptKey.get(receiptKey) ?? [];
9937
+ group.push(request);
9938
+ byReceiptKey.set(receiptKey, group);
9939
+ }
9940
+ // One content-addressed receipt can have many same-run followers. The
9941
+ // store renews unique receipt rows, not request positions, so dedupe by
9942
+ // key while preserving the matching per-row lease id.
9943
+ const entries = [...byReceiptKey];
9944
+ const keys = entries.map(([receiptKey]) => receiptKey);
9945
+ const leaseIds = entries.map(
9946
+ ([, requests]) => requests[0]!.receiptLeaseId!,
9947
+ );
9948
+ const receipts = await heartbeatReceipts({
9949
+ runId: this.currentReceiptOwnerRunId,
9950
+ runAttempt: this.currentRunAttempt,
9951
+ keys,
9952
+ leaseIds,
9953
+ });
9954
+ const renewedExpiries: number[] = [];
9955
+ for (let index = 0; index < entries.length; index += 1) {
9956
+ const [receiptKey, requests] = entries[index]!;
9957
+ const leaseId = leaseIds[index]!;
9958
+ // Completion can race this bulk heartbeat response. A settled
9959
+ // receipt group no longer needs ownership and must not turn that
9960
+ // race into a false lease-loss failure or hold the confirmation
9961
+ // horizon at its old expiry.
9962
+ if (
9963
+ this.runtimeToolReceiptCompletionsInFlight.has(receiptKey) ||
9964
+ !requests.some((request) =>
9965
+ this.toolCallResolvers.has(request.callId),
9966
+ )
9967
+ ) {
9968
+ continue;
9258
9969
  }
9259
- // One content-addressed receipt can have many same-run followers. The
9260
- // store renews unique receipt rows, not request positions, so dedupe by
9261
- // key while preserving the matching per-row lease id.
9262
- const entries = [...byReceiptKey];
9263
- const keys = entries.map(([receiptKey]) => receiptKey);
9264
- const leaseIds = entries.map(
9265
- ([, requests]) => requests[0]!.receiptLeaseId!,
9266
- );
9267
- const receipts = await heartbeatReceipts({
9268
- runId: this.currentReceiptOwnerRunId,
9269
- runAttempt: this.currentRunAttempt,
9270
- keys,
9271
- leaseIds,
9272
- });
9273
- for (let index = 0; index < entries.length; index += 1) {
9274
- const [receiptKey, requests] = entries[index]!;
9275
- const leaseId = leaseIds[index]!;
9276
- // Completion can race this bulk heartbeat response. A settled
9277
- // receipt group no longer needs ownership and must not turn that
9278
- // race into a false lease-loss failure.
9279
- if (
9280
- !requests.some((request) =>
9281
- this.toolCallResolvers.has(request.callId),
9282
- )
9283
- ) {
9284
- continue;
9285
- }
9286
- if (
9287
- !this.runtimeToolReceiptStillOwned(receipts[index] ?? null, leaseId)
9288
- ) {
9289
- throw new RuntimeReceiptLeaseLostError({
9290
- receiptKey,
9291
- runId: this.currentReceiptOwnerRunId,
9292
- leaseId,
9293
- });
9294
- }
9970
+ const receipt = receipts[index] ?? null;
9971
+ const renewedExpiryMs = Date.parse(receipt?.leaseExpiresAt ?? '');
9972
+ if (
9973
+ !this.runtimeToolReceiptStillOwned(receipt, leaseId) ||
9974
+ !Number.isFinite(renewedExpiryMs)
9975
+ ) {
9976
+ throw new RuntimeReceiptLeaseLostError({
9977
+ receiptKey,
9978
+ runId: this.currentReceiptOwnerRunId,
9979
+ leaseId,
9980
+ });
9295
9981
  }
9296
- return 'active';
9297
- },
9982
+ renewedExpiries.push(renewedExpiryMs);
9983
+ }
9984
+ if (renewedExpiries.length === 0) return 'terminal' as const;
9985
+ ownershipConfirmedUntilMs = Math.min(...renewedExpiries);
9986
+ return 'active' as const;
9987
+ };
9988
+ const perCallHeartbeatIntervalMs = runtimeLeaseHeartbeatIntervalFromExpiry({
9989
+ leaseExpiresAt: earliestLeaseExpiry,
9990
+ fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
9991
+ });
9992
+ const supervisor = createRuntimeReceiptHeartbeatSupervisor({
9993
+ // The cohort owns claims that have not reached provider admission yet.
9994
+ // Renew it before the one-call supervisors can win the same short-lease
9995
+ // timer race, so the queued tail remains fenced while the active call
9996
+ // consumes the provider concurrency slot.
9997
+ intervalMs: Math.max(1, Math.floor(perCallHeartbeatIntervalMs / 2)),
9998
+ heartbeat: heartbeatOwnedRequests,
9298
9999
  isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
9299
10000
  onLeaseLost: (error) => {
9300
10001
  leaseLost ??= error;
@@ -9305,10 +10006,32 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9305
10006
  );
9306
10007
  },
9307
10008
  });
9308
- supervisor.start();
9309
10009
  return {
9310
10010
  stop: () => supervisor.stop(),
9311
10011
  leaseLost: () => leaseLost,
10012
+ confirmOwned: async () => {
10013
+ // The scheduler receipt gateway returns `running` from an atomic
10014
+ // claim. That claim is already the provider-execution fence, so short
10015
+ // calls only need scheduled renewal if they approach lease expiry.
10016
+ // Other callers may still return queued/pending claims or omit the
10017
+ // explicit backend capability; retain the immediate ownership check
10018
+ // for those receipts.
10019
+ if (claimsEstablishedExecutionFence) {
10020
+ supervisor.start();
10021
+ return;
10022
+ }
10023
+ const outcome = await heartbeatOwnedRequests();
10024
+ if (outcome === 'active') supervisor.start();
10025
+ },
10026
+ assertOwned: () => {
10027
+ if (leaseLost) throw leaseLost;
10028
+ if (Date.now() < ownershipConfirmedUntilMs) return;
10029
+ throw new RuntimeReceiptLeaseLostError({
10030
+ receiptKey: firstOwnedRequest.receiptKey!,
10031
+ runId: this.currentReceiptOwnerRunId,
10032
+ leaseId: firstOwnedRequest.receiptLeaseId!,
10033
+ });
10034
+ },
9312
10035
  };
9313
10036
  }
9314
10037
 
@@ -9368,6 +10091,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9368
10091
  };
9369
10092
 
9370
10093
  const pendingRequests: ToolCallRequest[] = [];
10094
+ let pendingReceiptClaimsEstablishExecutionFence =
10095
+ this.#options.runtimeReceiptClaimsEstablishExecutionFence === true;
9371
10096
  const recoveredRequests: ToolCallRequest[] = [];
9372
10097
  for (const req of requests) {
9373
10098
  const cached =
@@ -9495,504 +10220,611 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9495
10220
  forceAfterWait: boolean;
9496
10221
  waitMaxAttempts?: number;
9497
10222
  }> = [];
9498
- let durableExistingRunningHandler: Promise<void> | null = null;
10223
+ const durableExistingRunningState: {
10224
+ settlement: Promise<
10225
+ { status: 'fulfilled' } | { status: 'rejected'; reason: unknown }
10226
+ > | null;
10227
+ } = { settlement: null };
10228
+ const durableExistingRunningAbort = new AbortController();
9499
10229
  if (
9500
10230
  pendingRequests.length > 0 &&
9501
10231
  (this.#options.claimRuntimeStepReceipts ||
9502
10232
  this.#options.claimRuntimeStepReceipt)
9503
10233
  ) {
9504
- const requestsByReceiptKey = new Map<
9505
- string,
9506
- {
9507
- requests: ToolCallRequest[];
9508
- forceRefresh: boolean;
9509
- forceFailedRefresh: boolean;
9510
- }
9511
- >();
9512
- const localOnlyRequests: ToolCallRequest[] = [];
9513
- for (const request of pendingRequests) {
9514
- const receiptKey = request.receiptKey?.trim() || null;
9515
- if (!receiptKey) {
9516
- localOnlyRequests.push(request);
9517
- continue;
10234
+ await this.withRuntimeReceiptHydrationTurn(async () => {
10235
+ const requestsByReceiptKey = new Map<
10236
+ string,
10237
+ {
10238
+ requests: ToolCallRequest[];
10239
+ forceRefresh: boolean;
10240
+ forceFailedRefresh: boolean;
10241
+ }
10242
+ >();
10243
+ const localOnlyRequests: ToolCallRequest[] = [];
10244
+ for (const request of pendingRequests) {
10245
+ const receiptKey = request.receiptKey?.trim() || null;
10246
+ if (!receiptKey) {
10247
+ localOnlyRequests.push(request);
10248
+ continue;
10249
+ }
10250
+ const group = requestsByReceiptKey.get(receiptKey) ?? {
10251
+ requests: [],
10252
+ forceRefresh: false,
10253
+ forceFailedRefresh: false,
10254
+ };
10255
+ group.requests.push(request);
10256
+ group.forceRefresh ||= request.force === true;
10257
+ group.forceFailedRefresh ||= request.forceFailedRefresh === true;
10258
+ requestsByReceiptKey.set(receiptKey, group);
9518
10259
  }
9519
- const group = requestsByReceiptKey.get(receiptKey) ?? {
9520
- requests: [],
9521
- forceRefresh: false,
9522
- forceFailedRefresh: false,
9523
- };
9524
- group.requests.push(request);
9525
- group.forceRefresh ||= request.force === true;
9526
- group.forceFailedRefresh ||= request.forceFailedRefresh === true;
9527
- requestsByReceiptKey.set(receiptKey, group);
9528
- }
9529
- const normalReceiptKeys = [...requestsByReceiptKey]
9530
- .filter(
9531
- ([, group]) => !group.forceRefresh && !group.forceFailedRefresh,
9532
- )
9533
- .map(([receiptKey]) => receiptKey);
9534
- const forcedReceiptKeys = [...requestsByReceiptKey]
9535
- .filter(([, group]) => group.forceRefresh)
9536
- .map(([receiptKey]) => receiptKey);
9537
- const failedRefreshReceiptKeys = [...requestsByReceiptKey]
9538
- .filter(
9539
- ([, group]) => !group.forceRefresh && group.forceFailedRefresh,
9540
- )
9541
- .map(([receiptKey]) => receiptKey);
9542
- const forcedExisting = forcedReceiptKeys.length
9543
- ? await this.getRuntimeStepReceipts(forcedReceiptKeys)
9544
- : new Map<string, RuntimeStepReceipt>();
9545
- // A forced refresh must never duplicate a provider call owned by a
9546
- // live lease. It can, however, immediately claim pending/unleased
9547
- // work and any expired lease. Waiting first for those receipts turns
9548
- // an already-recoverable interruption into a five-minute stall.
9549
- const forcedLiveReceiptKeys = new Set(
9550
- [...forcedExisting]
9551
- .filter(([, receipt]) => {
9552
- if (
9553
- receipt.status !== 'queued' &&
9554
- receipt.status !== 'pending' &&
9555
- receipt.status !== 'running'
9556
- ) {
9557
- return false;
9558
- }
9559
- if (!receipt.leaseId?.trim()) return false;
9560
- const expiresAt = receipt.leaseExpiresAt
9561
- ? Date.parse(receipt.leaseExpiresAt)
9562
- : Number.NaN;
9563
- // Keep an unparseable leased receipt conservative. The
9564
- // fenced claim remains the recovery path after the wait.
9565
- return !Number.isFinite(expiresAt) || expiresAt > Date.now();
9566
- })
9567
- .map(([receiptKey]) => receiptKey),
9568
- );
9569
- const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
9570
- (receiptKey) => !forcedLiveReceiptKeys.has(receiptKey),
9571
- );
9572
- const claims =
9573
- normalReceiptKeys.length > 0
9574
- ? await this.claimRuntimeStepReceipts(
9575
- normalReceiptKeys,
9576
- this.currentReceiptOwnerRunId,
9577
- )
9578
- : new Map<string, RuntimeStepReceipt>();
9579
- const forcedClaims =
9580
- claimableForcedReceiptKeys.length > 0
9581
- ? await this.claimRuntimeStepReceipts(
9582
- claimableForcedReceiptKeys,
9583
- this.currentReceiptOwnerRunId,
9584
- false,
9585
- true,
9586
- )
9587
- : new Map<string, RuntimeStepReceipt>();
9588
- const failedRefreshClaims =
9589
- failedRefreshReceiptKeys.length > 0
9590
- ? await this.claimRuntimeStepReceipts(
9591
- failedRefreshReceiptKeys,
9592
- this.currentReceiptOwnerRunId,
9593
- false,
9594
- false,
9595
- true,
9596
- )
10260
+ const normalReceiptKeys = [...requestsByReceiptKey]
10261
+ .filter(
10262
+ ([, group]) => !group.forceRefresh && !group.forceFailedRefresh,
10263
+ )
10264
+ .map(([receiptKey]) => receiptKey);
10265
+ const forcedReceiptKeys = [...requestsByReceiptKey]
10266
+ .filter(([, group]) => group.forceRefresh)
10267
+ .map(([receiptKey]) => receiptKey);
10268
+ const failedRefreshReceiptKeys = [...requestsByReceiptKey]
10269
+ .filter(
10270
+ ([, group]) => !group.forceRefresh && group.forceFailedRefresh,
10271
+ )
10272
+ .map(([receiptKey]) => receiptKey);
10273
+ const forcedExisting = forcedReceiptKeys.length
10274
+ ? await this.getRuntimeStepReceipts(forcedReceiptKeys)
9597
10275
  : new Map<string, RuntimeStepReceipt>();
9598
- for (const [receiptKey, claim] of forcedClaims) {
9599
- claims.set(receiptKey, claim);
9600
- }
9601
- for (const [receiptKey, claim] of failedRefreshClaims) {
9602
- claims.set(receiptKey, claim);
9603
- }
9604
- const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
9605
- const durableRecoveredRequests: ToolCallRequest[] = [];
9606
- const resolveRequestsFromReceipt = async (
9607
- requestsForKey: ToolCallRequest[],
9608
- receipt: RuntimeStepReceipt,
9609
- source: 'cache' | 'in_flight',
9610
- ): Promise<void> => {
9611
- const request = requestsForKey[0];
9612
- if (!request) return;
9613
- const receiptKey = request.receiptKey?.trim() || null;
9614
- if (!receiptKey) {
9615
- throw new Error(`${source} tool recovery needs receipt key.`);
9616
- }
9617
- const wrapped = await this.wrapToolExecutionResult({
9618
- toolId,
9619
- status:
9620
- receipt.output === null || receipt.output === undefined
9621
- ? 'no_result'
9622
- : 'completed',
9623
- result: this.runtimeReceiptOutput(receipt),
9624
- requestInput: request.input,
9625
- execution: toolExecutionMetadataForOutcome({
9626
- kind: source,
9627
- cacheKey: request.cacheKey,
9628
- receiptKey,
9629
- attachedToReceiptKey: receiptKey,
9630
- }),
9631
- });
9632
- this.cacheToolResult(toolId, request.cacheKey, wrapped);
9633
- for (const waitingRequest of requestsForKey) {
9634
- successfulLiveStepCallIds.add(waitingRequest.callId);
9635
- const resolver = this.toolCallResolvers.get(
9636
- waitingRequest.callId,
9637
- );
9638
- if (resolver) {
9639
- resolver.resolve(wrapped);
9640
- this.toolCallResolvers.delete(waitingRequest.callId);
9641
- }
9642
- this.emitScopedFieldMetaUpdate({
9643
- rowId: waitingRequest.rowId,
9644
- key: waitingRequest.rowKey ?? null,
9645
- tableNamespace: waitingRequest.tableNamespace ?? null,
9646
- fieldName: waitingRequest.fieldName,
9647
- status: 'cached',
9648
- rowStatus: 'running',
9649
- stage: toolId,
9650
- provider: null,
9651
- error: null,
9652
- reused: true,
9653
- dataPatch: {},
9654
- });
10276
+ // A forced refresh must never duplicate a provider call owned by a
10277
+ // live lease. It can, however, immediately claim pending/unleased
10278
+ // work and any expired lease. Waiting first for those receipts turns
10279
+ // an already-recoverable interruption into a five-minute stall.
10280
+ const forcedLiveReceiptKeys = new Set(
10281
+ [...forcedExisting]
10282
+ .filter(([, receipt]) => {
10283
+ if (
10284
+ receipt.status !== 'queued' &&
10285
+ receipt.status !== 'pending' &&
10286
+ receipt.status !== 'running'
10287
+ ) {
10288
+ return false;
10289
+ }
10290
+ if (!receipt.leaseId?.trim()) return false;
10291
+ const expiresAt = receipt.leaseExpiresAt
10292
+ ? Date.parse(receipt.leaseExpiresAt)
10293
+ : Number.NaN;
10294
+ // Keep an unparseable leased receipt conservative. The
10295
+ // fenced claim remains the recovery path after the wait.
10296
+ return !Number.isFinite(expiresAt) || expiresAt > Date.now();
10297
+ })
10298
+ .map(([receiptKey]) => receiptKey),
10299
+ );
10300
+ const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
10301
+ (receiptKey) => !forcedLiveReceiptKeys.has(receiptKey),
10302
+ );
10303
+ const claims =
10304
+ normalReceiptKeys.length > 0
10305
+ ? await this.claimRuntimeStepReceipts(
10306
+ normalReceiptKeys,
10307
+ this.currentReceiptOwnerRunId,
10308
+ )
10309
+ : new Map<string, RuntimeStepReceipt>();
10310
+ const forcedClaims =
10311
+ claimableForcedReceiptKeys.length > 0
10312
+ ? await this.claimRuntimeStepReceipts(
10313
+ claimableForcedReceiptKeys,
10314
+ this.currentReceiptOwnerRunId,
10315
+ false,
10316
+ true,
10317
+ )
10318
+ : new Map<string, RuntimeStepReceipt>();
10319
+ const failedRefreshClaims =
10320
+ failedRefreshReceiptKeys.length > 0
10321
+ ? await this.claimRuntimeStepReceipts(
10322
+ failedRefreshReceiptKeys,
10323
+ this.currentReceiptOwnerRunId,
10324
+ false,
10325
+ false,
10326
+ true,
10327
+ )
10328
+ : new Map<string, RuntimeStepReceipt>();
10329
+ for (const [receiptKey, claim] of forcedClaims) {
10330
+ claims.set(receiptKey, claim);
9655
10331
  }
9656
- };
9657
- const claimOwnerWithLiveFollowers = (
9658
- owner: ToolCallRequest | undefined,
9659
- followers: ToolCallRequest[],
9660
- ): void => {
9661
- if (!owner) return;
9662
- claimedRequests.push(owner);
9663
- if (followers.length > 0) {
9664
- liveFollowersByOwnerCallId.set(owner.callId, followers);
10332
+ for (const [receiptKey, claim] of failedRefreshClaims) {
10333
+ claims.set(receiptKey, claim);
9665
10334
  }
9666
- };
9667
- const resolveReceiptWaitMaxAttempts = (
9668
- requestsForKey: ToolCallRequest[],
9669
- ): number | undefined =>
9670
- requestsForKey.length > 0
9671
- ? Math.max(
9672
- ...requestsForKey.map((request) =>
9673
- resolveRuntimeToolReceiptWaitMaxAttempts(
9674
- typeof request.receiptWaitMs === 'number'
9675
- ? { max_wait_ms: request.receiptWaitMs }
9676
- : request.input,
9677
- ),
9678
- ),
9679
- )
9680
- : undefined;
9681
- const queueExistingRunningReceiptWait = (
9682
- receiptKey: string,
9683
- requestsForKey: ToolCallRequest[],
9684
- forceAfterWait = false,
9685
- ): void => {
9686
- durableExistingRunningWaits.push({
9687
- receiptKey,
9688
- requestsForKey,
9689
- forceAfterWait,
9690
- waitMaxAttempts: resolveReceiptWaitMaxAttempts(requestsForKey),
9691
- });
9692
- };
9693
- const reclaimExistingRunningReceiptAfterWait = async (
9694
- receiptKey: string,
9695
- requestsForKey: ToolCallRequest[],
9696
- forceAfterWait = false,
9697
- waitResult?: {
9698
- completedReceipt?: RuntimeStepReceipt;
9699
- skipWait?: boolean;
9700
- },
9701
- ): Promise<void> => {
9702
- if (waitResult?.completedReceipt) {
9703
- if (!forceAfterWait) {
9704
- await resolveRequestsFromReceipt(
9705
- requestsForKey,
9706
- waitResult.completedReceipt,
9707
- 'in_flight',
9708
- );
9709
- return;
10335
+ const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
10336
+ const durableRecoveredRequests: ToolCallRequest[] = [];
10337
+ const resolveRequestsFromReceipt = async (
10338
+ requestsForKey: ToolCallRequest[],
10339
+ receipt: RuntimeStepReceipt,
10340
+ source: 'cache' | 'in_flight',
10341
+ ): Promise<void> => {
10342
+ const request = requestsForKey[0];
10343
+ if (!request) return;
10344
+ const receiptKey = request.receiptKey?.trim() || null;
10345
+ if (!receiptKey) {
10346
+ throw new Error(`${source} tool recovery needs receipt key.`);
9710
10347
  }
9711
- } else if (waitResult?.skipWait !== true) {
9712
- const waitMaxAttempts =
9713
- resolveReceiptWaitMaxAttempts(requestsForKey);
9714
- try {
9715
- const completed = await this.waitForCompletedRuntimeToolReceipt(
10348
+ const wrapped = await this.wrapToolExecutionResult({
10349
+ toolId,
10350
+ status:
10351
+ receipt.output === null || receipt.output === undefined
10352
+ ? 'no_result'
10353
+ : 'completed',
10354
+ result: this.runtimeReceiptOutput(receipt),
10355
+ requestInput: request.input,
10356
+ execution: toolExecutionMetadataForOutcome({
10357
+ kind: source,
10358
+ cacheKey: request.cacheKey,
9716
10359
  receiptKey,
9717
- waitMaxAttempts,
10360
+ attachedToReceiptKey: receiptKey,
10361
+ }),
10362
+ });
10363
+ this.cacheToolResult(toolId, request.cacheKey, wrapped);
10364
+ for (const waitingRequest of requestsForKey) {
10365
+ successfulLiveStepCallIds.add(waitingRequest.callId);
10366
+ const resolver = this.toolCallResolvers.get(
10367
+ waitingRequest.callId,
9718
10368
  );
10369
+ if (resolver) {
10370
+ resolver.resolve(wrapped);
10371
+ this.toolCallResolvers.delete(waitingRequest.callId);
10372
+ }
10373
+ this.emitScopedFieldMetaUpdate({
10374
+ rowId: waitingRequest.rowId,
10375
+ key: waitingRequest.rowKey ?? null,
10376
+ tableNamespace: waitingRequest.tableNamespace ?? null,
10377
+ fieldName: waitingRequest.fieldName,
10378
+ status: 'cached',
10379
+ rowStatus: 'running',
10380
+ stage: toolId,
10381
+ provider: null,
10382
+ error: null,
10383
+ reused: true,
10384
+ dataPatch: {},
10385
+ });
10386
+ }
10387
+ };
10388
+ const claimOwnerWithLiveFollowers = (
10389
+ owner: ToolCallRequest | undefined,
10390
+ followers: ToolCallRequest[],
10391
+ ): void => {
10392
+ if (!owner) return;
10393
+ claimedRequests.push(owner);
10394
+ if (followers.length > 0) {
10395
+ liveFollowersByOwnerCallId.set(owner.callId, followers);
10396
+ }
10397
+ };
10398
+ const resolveReceiptWaitMaxAttempts = (
10399
+ requestsForKey: ToolCallRequest[],
10400
+ ): number | undefined =>
10401
+ requestsForKey.length > 0
10402
+ ? Math.max(
10403
+ ...requestsForKey.map((request) =>
10404
+ resolveRuntimeToolReceiptWaitMaxAttempts(
10405
+ typeof request.receiptWaitMs === 'number'
10406
+ ? { max_wait_ms: request.receiptWaitMs }
10407
+ : request.input,
10408
+ ),
10409
+ ),
10410
+ )
10411
+ : undefined;
10412
+ const queueExistingRunningReceiptWait = (
10413
+ receiptKey: string,
10414
+ requestsForKey: ToolCallRequest[],
10415
+ forceAfterWait = false,
10416
+ ): void => {
10417
+ durableExistingRunningWaits.push({
10418
+ receiptKey,
10419
+ requestsForKey,
10420
+ forceAfterWait,
10421
+ waitMaxAttempts: resolveReceiptWaitMaxAttempts(requestsForKey),
10422
+ });
10423
+ };
10424
+ const reclaimExistingRunningReceiptAfterWait = async (
10425
+ receiptKey: string,
10426
+ requestsForKey: ToolCallRequest[],
10427
+ forceAfterWait = false,
10428
+ waitResult?: {
10429
+ completedReceipt?: RuntimeStepReceipt;
10430
+ skipWait?: boolean;
10431
+ },
10432
+ ): Promise<void> => {
10433
+ if (waitResult?.completedReceipt) {
9719
10434
  if (!forceAfterWait) {
9720
10435
  await resolveRequestsFromReceipt(
9721
10436
  requestsForKey,
9722
- completed,
10437
+ waitResult.completedReceipt,
9723
10438
  'in_flight',
9724
10439
  );
9725
10440
  return;
9726
10441
  }
9727
- } catch (error) {
9728
- if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
9729
- for (const request of requestsForKey) {
9730
- await this.rejectToolCall(toolId, request, error, {
9731
- persistReceiptFailure: false,
9732
- });
10442
+ } else if (waitResult?.skipWait !== true) {
10443
+ const waitMaxAttempts =
10444
+ resolveReceiptWaitMaxAttempts(requestsForKey);
10445
+ try {
10446
+ const completed =
10447
+ await this.waitForCompletedRuntimeToolReceipt(
10448
+ receiptKey,
10449
+ waitMaxAttempts,
10450
+ );
10451
+ if (!forceAfterWait) {
10452
+ await resolveRequestsFromReceipt(
10453
+ requestsForKey,
10454
+ completed,
10455
+ 'in_flight',
10456
+ );
10457
+ return;
10458
+ }
10459
+ } catch (error) {
10460
+ if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
10461
+ for (const request of requestsForKey) {
10462
+ await this.rejectToolCall(toolId, request, error, {
10463
+ persistReceiptFailure: false,
10464
+ });
10465
+ }
10466
+ return;
9733
10467
  }
9734
- return;
9735
10468
  }
9736
10469
  }
9737
- }
9738
- const reclaimed = (
9739
- await this.claimRuntimeStepReceipts(
9740
- [receiptKey],
9741
- this.currentReceiptOwnerRunId,
9742
- true,
9743
- forceAfterWait,
9744
- )
9745
- ).get(receiptKey);
9746
- if (
9747
- reclaimed?.status === 'completed' ||
9748
- reclaimed?.status === 'skipped'
9749
- ) {
9750
- await resolveRequestsFromReceipt(
9751
- requestsForKey,
9752
- reclaimed,
9753
- 'cache',
9754
- );
9755
- durableRecoveredRequests.push(...requestsForKey);
9756
- return;
9757
- }
9758
- if (reclaimed?.status === 'failed') {
10470
+ const reclaimed = (
10471
+ await this.claimRuntimeStepReceipts(
10472
+ [receiptKey],
10473
+ this.currentReceiptOwnerRunId,
10474
+ true,
10475
+ forceAfterWait,
10476
+ )
10477
+ ).get(receiptKey);
10478
+ if (
10479
+ reclaimed?.status === 'completed' ||
10480
+ reclaimed?.status === 'skipped'
10481
+ ) {
10482
+ await resolveRequestsFromReceipt(
10483
+ requestsForKey,
10484
+ reclaimed,
10485
+ 'cache',
10486
+ );
10487
+ durableRecoveredRequests.push(...requestsForKey);
10488
+ return;
10489
+ }
10490
+ if (reclaimed?.status === 'failed') {
10491
+ for (const request of requestsForKey) {
10492
+ await this.rejectToolCall(
10493
+ toolId,
10494
+ request,
10495
+ runtimeReceiptFailureError(
10496
+ reclaimed,
10497
+ 'Durable tool call failed',
10498
+ this.currentToolErrorSchemaVersion,
10499
+ ),
10500
+ );
10501
+ }
10502
+ return;
10503
+ }
10504
+ if (this.isOwnedClaimedRuntimeReceipt(reclaimed)) {
10505
+ const [owner, ...waiters] = requestsForKey;
10506
+ if (!owner) return;
10507
+ if (!reclaimed.leaseId) {
10508
+ throw new RuntimeReceiptLeaseLostError({
10509
+ receiptKey,
10510
+ runId: this.currentReceiptOwnerRunId,
10511
+ leaseId: 'missing',
10512
+ });
10513
+ }
10514
+ owner.receiptLeaseId = reclaimed.leaseId ?? null;
10515
+ owner.receiptLeaseExpiresAt = reclaimed.leaseExpiresAt ?? null;
10516
+ if (waiters.length > 0) {
10517
+ liveFollowersByOwnerCallId.set(owner.callId, waiters);
10518
+ }
10519
+ try {
10520
+ const execution = await this.callToolExecutionAPI(
10521
+ toolId,
10522
+ owner.input,
10523
+ {
10524
+ beforeProviderCall: () =>
10525
+ this.assertRuntimeToolReceiptOwnership([owner]),
10526
+ durableCallReceiptKey: receiptKey,
10527
+ playNodeScope: playNodeScopeForToolCallRequest(owner),
10528
+ executionAuthScopeDigest: owner.executionAuthScopeDigest,
10529
+ receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
10530
+ heartbeatReceipt: () =>
10531
+ this.renewRuntimeToolReceiptOwnership([owner]),
10532
+ providerIdempotencyKey:
10533
+ this.providerIdempotencyKeyForToolCall({
10534
+ cacheKey: owner.cacheKey,
10535
+ force: owner.force === true,
10536
+ leaseId: owner.receiptLeaseId,
10537
+ }),
10538
+ timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
10539
+ owner,
10540
+ ]),
10541
+ },
10542
+ );
10543
+ const result = await this.resolveToolCall(
10544
+ toolId,
10545
+ owner,
10546
+ execution?.result ?? null,
10547
+ execution?.metadata ?? null,
10548
+ execution?.jobId,
10549
+ execution?.meta,
10550
+ );
10551
+ if (result != null) {
10552
+ successfulLiveStepCallIds.add(owner.callId);
10553
+ }
10554
+ resolveLiveFollowers(owner, result);
10555
+ recordToolStep([owner]);
10556
+ this.#options.onBatchComplete?.(this.checkpoint);
10557
+ } catch (error) {
10558
+ await rejectWithLiveFollowers(owner, error);
10559
+ }
10560
+ return;
10561
+ }
9759
10562
  for (const request of requestsForKey) {
9760
10563
  await this.rejectToolCall(
9761
10564
  toolId,
9762
10565
  request,
9763
- runtimeReceiptFailureError(
9764
- reclaimed,
9765
- 'Durable tool call failed',
9766
- this.currentToolErrorSchemaVersion,
9767
- ),
10566
+ new RuntimeReceiptWaitTimeoutError(receiptKey),
10567
+ { persistReceiptFailure: false },
9768
10568
  );
9769
10569
  }
9770
- return;
9771
- }
9772
- if (this.isOwnedClaimedRuntimeReceipt(reclaimed)) {
9773
- const [owner, ...waiters] = requestsForKey;
9774
- if (!owner) return;
9775
- if (!reclaimed.leaseId) {
9776
- throw new RuntimeReceiptLeaseLostError({
9777
- receiptKey,
9778
- runId: this.currentReceiptOwnerRunId,
9779
- leaseId: 'missing',
9780
- });
9781
- }
9782
- owner.receiptLeaseId = reclaimed.leaseId ?? null;
9783
- owner.receiptLeaseExpiresAt = reclaimed.leaseExpiresAt ?? null;
9784
- if (waiters.length > 0) {
9785
- liveFollowersByOwnerCallId.set(owner.callId, waiters);
9786
- }
9787
- try {
9788
- const execution = await this.callToolExecutionAPI(
9789
- toolId,
9790
- owner.input,
9791
- {
9792
- beforeProviderCall: () =>
9793
- this.assertRuntimeToolReceiptOwnership([owner]),
9794
- durableCallReceiptKey: receiptKey,
9795
- playNodeScope: playNodeScopeForToolCallRequest(owner),
9796
- executionAuthScopeDigest: owner.executionAuthScopeDigest,
9797
- receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
9798
- heartbeatReceipt: () =>
9799
- this.renewRuntimeToolReceiptOwnership([owner]),
9800
- providerIdempotencyKey:
9801
- this.providerIdempotencyKeyForToolCall({
9802
- cacheKey: owner.cacheKey,
9803
- force: owner.force === true,
9804
- leaseId: owner.receiptLeaseId,
10570
+ };
10571
+ const processExistingRunningReceiptWaits =
10572
+ async (): Promise<void> => {
10573
+ if (durableExistingRunningWaits.length === 0) return;
10574
+ const waitSignal = durableExistingRunningAbort.signal;
10575
+ const pendingWaits = new Map(
10576
+ durableExistingRunningWaits.map((wait) => [
10577
+ wait.receiptKey,
10578
+ wait,
10579
+ ]),
10580
+ );
10581
+ const waitMaxAttempts =
10582
+ Math.max(
10583
+ ...durableExistingRunningWaits.map(
10584
+ (wait) => wait.waitMaxAttempts ?? 0,
10585
+ ),
10586
+ ) || DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
10587
+ if (runtimeReceiptReadTraceEnabled) {
10588
+ this.log(
10589
+ `[runtime-receipt-wait] phase=start requested=${pendingWaits.size} ` +
10590
+ `request_digests=${[...pendingWaits.keys()]
10591
+ .map(runtimeReceiptKeyDigest)
10592
+ .join(',')} max_attempts=${waitMaxAttempts}`,
10593
+ );
10594
+ }
10595
+ for (
10596
+ let attempt = 0;
10597
+ attempt < waitMaxAttempts && pendingWaits.size > 0;
10598
+ attempt += 1
10599
+ ) {
10600
+ if (waitSignal.aborted) return;
10601
+ if (attempt > 0) {
10602
+ await new Promise<void>((resolve) => {
10603
+ const finish = () => {
10604
+ clearTimeout(timeout);
10605
+ waitSignal.removeEventListener('abort', finish);
10606
+ resolve();
10607
+ };
10608
+ const timeout = setTimeout(
10609
+ finish,
10610
+ DURABLE_RECEIPT_WAIT_DELAY_MS,
10611
+ );
10612
+ waitSignal.addEventListener('abort', finish, {
10613
+ once: true,
10614
+ });
10615
+ });
10616
+ if (waitSignal.aborted) return;
10617
+ }
10618
+ // The long backoff lives outside the hydration lane. Only a
10619
+ // bounded receipt read and delivery hold the lane, so stale
10620
+ // ownership cannot block unrelated cache misses/provider
10621
+ // work while terminal payloads still hydrate one group at a
10622
+ // time.
10623
+ await this.withRuntimeReceiptHydrationTurn(async () => {
10624
+ if (waitSignal.aborted) return;
10625
+ const waited =
10626
+ await this.waitForCompletedRuntimeToolReceipts(
10627
+ [...pendingWaits.keys()],
10628
+ 1,
10629
+ );
10630
+ if (waitSignal.aborted) return;
10631
+ const settledWaits = [...pendingWaits.values()].filter(
10632
+ (wait) =>
10633
+ waited.completed.has(wait.receiptKey) ||
10634
+ waited.failed.has(wait.receiptKey),
10635
+ );
10636
+ const deliverySettlements = await Promise.allSettled(
10637
+ settledWaits.map(async (wait) => {
10638
+ const failed = waited.failed.get(wait.receiptKey);
10639
+ if (failed) {
10640
+ for (const request of wait.requestsForKey) {
10641
+ await this.rejectToolCall(toolId, request, failed, {
10642
+ persistReceiptFailure: false,
10643
+ });
10644
+ }
10645
+ return;
10646
+ }
10647
+ await reclaimExistingRunningReceiptAfterWait(
10648
+ wait.receiptKey,
10649
+ wait.requestsForKey,
10650
+ wait.forceAfterWait,
10651
+ {
10652
+ completedReceipt: waited.completed.get(
10653
+ wait.receiptKey,
10654
+ ),
10655
+ skipWait: true,
10656
+ },
10657
+ );
9805
10658
  }),
9806
- timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([owner]),
9807
- },
10659
+ );
10660
+ for (
10661
+ let index = 0;
10662
+ index < settledWaits.length;
10663
+ index += 1
10664
+ ) {
10665
+ if (deliverySettlements[index]?.status === 'fulfilled') {
10666
+ pendingWaits.delete(settledWaits[index]!.receiptKey);
10667
+ }
10668
+ }
10669
+ if (runtimeReceiptReadTraceEnabled) {
10670
+ const rejected = deliverySettlements.flatMap(
10671
+ (settlement, index) => {
10672
+ if (settlement.status !== 'rejected') return [];
10673
+ const receiptKey = settledWaits[index]?.receiptKey;
10674
+ return receiptKey
10675
+ ? [
10676
+ `${runtimeReceiptKeyDigest(receiptKey)}:${
10677
+ settlement.reason instanceof Error
10678
+ ? settlement.reason.name
10679
+ : 'non_error'
10680
+ }`,
10681
+ ]
10682
+ : ['unknown'];
10683
+ },
10684
+ );
10685
+ this.log(
10686
+ `[runtime-receipt-wait] phase=delivery settled=${deliverySettlements.length} ` +
10687
+ `rejected=${rejected.length} rejected_digests=${rejected.join(',')}`,
10688
+ );
10689
+ }
10690
+ const rejectedDelivery = deliverySettlements.find(
10691
+ (settlement) => settlement.status === 'rejected',
10692
+ );
10693
+ if (rejectedDelivery?.status === 'rejected') {
10694
+ throw rejectedDelivery.reason;
10695
+ }
10696
+ });
10697
+ }
10698
+ if (waitSignal.aborted) return;
10699
+ const reclaimSettlements = await Promise.allSettled(
10700
+ [...pendingWaits.values()].map((wait) =>
10701
+ reclaimExistingRunningReceiptAfterWait(
10702
+ wait.receiptKey,
10703
+ wait.requestsForKey,
10704
+ wait.forceAfterWait,
10705
+ { skipWait: true },
10706
+ ),
10707
+ ),
9808
10708
  );
9809
- const result = await this.resolveToolCall(
9810
- toolId,
9811
- owner,
9812
- execution?.result ?? null,
9813
- execution?.metadata ?? null,
9814
- execution?.jobId,
9815
- execution?.meta,
10709
+ const rejectedReclaim = reclaimSettlements.find(
10710
+ (settlement) => settlement.status === 'rejected',
9816
10711
  );
9817
- if (result != null) {
9818
- successfulLiveStepCallIds.add(owner.callId);
10712
+ if (rejectedReclaim?.status === 'rejected') {
10713
+ throw rejectedReclaim.reason;
9819
10714
  }
9820
- resolveLiveFollowers(owner, result);
9821
- recordToolStep([owner]);
9822
- this.#options.onBatchComplete?.(this.checkpoint);
9823
- } catch (error) {
9824
- await rejectWithLiveFollowers(owner, error);
9825
- }
9826
- return;
9827
- }
9828
- for (const request of requestsForKey) {
9829
- await this.rejectToolCall(
9830
- toolId,
9831
- request,
9832
- new RuntimeReceiptWaitTimeoutError(receiptKey),
9833
- { persistReceiptFailure: false },
9834
- );
9835
- }
9836
- };
9837
- const processExistingRunningReceiptWaits =
9838
- async (): Promise<void> => {
9839
- if (durableExistingRunningWaits.length === 0) return;
9840
- const waitingReceiptKeys = durableExistingRunningWaits.map(
9841
- (wait) => wait.receiptKey,
9842
- );
9843
- const waitMaxAttempts = Math.max(
9844
- ...durableExistingRunningWaits.map(
9845
- (wait) => wait.waitMaxAttempts ?? 0,
9846
- ),
9847
- );
9848
- if (runtimeReceiptReadTraceEnabled) {
9849
- this.log(
9850
- `[runtime-receipt-wait] phase=start requested=${waitingReceiptKeys.length} ` +
9851
- `request_digests=${waitingReceiptKeys
9852
- .map(runtimeReceiptKeyDigest)
9853
- .join(',')} max_attempts=${waitMaxAttempts || 'default'}`,
10715
+ };
10716
+
10717
+ for (const [receiptKey, group] of requestsByReceiptKey) {
10718
+ const requestsForKey = group.requests;
10719
+ const claim = claims.get(receiptKey);
10720
+ if (!claim) {
10721
+ queueExistingRunningReceiptWait(
10722
+ receiptKey,
10723
+ requestsForKey,
10724
+ group.forceRefresh,
9854
10725
  );
10726
+ continue;
9855
10727
  }
9856
- const waited = await this.waitForCompletedRuntimeToolReceipts(
9857
- waitingReceiptKeys,
9858
- waitMaxAttempts > 0 ? waitMaxAttempts : undefined,
9859
- );
9860
- if (runtimeReceiptReadTraceEnabled) {
9861
- this.log(
9862
- `[runtime-receipt-wait] phase=settled completed=${waited.completed.size} ` +
9863
- `failed=${waited.failed.size} timed_out=${waited.timedOut.size} ` +
9864
- `completed_digests=${[...waited.completed.keys()]
9865
- .map(runtimeReceiptKeyDigest)
9866
- .join(',')} ` +
9867
- `timed_out_digests=${[...waited.timedOut]
9868
- .map(runtimeReceiptKeyDigest)
9869
- .join(',')}`,
10728
+ if (
10729
+ claim?.status === 'completed' ||
10730
+ claim?.status === 'skipped'
10731
+ ) {
10732
+ await resolveRequestsFromReceipt(
10733
+ requestsForKey,
10734
+ claim,
10735
+ 'cache',
9870
10736
  );
10737
+ durableRecoveredRequests.push(...requestsForKey);
10738
+ continue;
9871
10739
  }
9872
- const deliverySettlements = await Promise.allSettled(
9873
- durableExistingRunningWaits.map(async (wait) => {
9874
- const failed = waited.failed.get(wait.receiptKey);
9875
- if (failed) {
9876
- for (const request of wait.requestsForKey) {
9877
- await this.rejectToolCall(toolId, request, failed, {
9878
- persistReceiptFailure: false,
9879
- });
9880
- }
9881
- return;
9882
- }
9883
- await reclaimExistingRunningReceiptAfterWait(
9884
- wait.receiptKey,
9885
- wait.requestsForKey,
9886
- wait.forceAfterWait,
9887
- {
9888
- completedReceipt: waited.completed.get(wait.receiptKey),
9889
- skipWait: true,
9890
- },
10740
+ if (claim?.status === 'failed') {
10741
+ for (const request of requestsForKey) {
10742
+ await this.rejectToolCall(
10743
+ toolId,
10744
+ request,
10745
+ runtimeReceiptFailureError(
10746
+ claim,
10747
+ 'Durable tool call failed',
10748
+ this.currentToolErrorSchemaVersion,
10749
+ ),
9891
10750
  );
9892
- }),
9893
- );
9894
- if (runtimeReceiptReadTraceEnabled) {
9895
- const rejected = deliverySettlements.flatMap(
9896
- (settlement, index) => {
9897
- if (settlement.status !== 'rejected') return [];
9898
- const receiptKey =
9899
- durableExistingRunningWaits[index]?.receiptKey;
9900
- return receiptKey
9901
- ? [
9902
- `${runtimeReceiptKeyDigest(receiptKey)}:${
9903
- settlement.reason instanceof Error
9904
- ? settlement.reason.name
9905
- : 'non_error'
9906
- }`,
9907
- ]
9908
- : ['unknown'];
9909
- },
9910
- );
9911
- this.log(
9912
- `[runtime-receipt-wait] phase=delivery settled=${deliverySettlements.length} ` +
9913
- `rejected=${rejected.length} rejected_digests=${rejected.join(',')}`,
9914
- );
10751
+ }
10752
+ continue;
9915
10753
  }
9916
- };
9917
-
9918
- for (const [receiptKey, group] of requestsByReceiptKey) {
9919
- const requestsForKey = group.requests;
9920
- const claim = claims.get(receiptKey);
9921
- if (!claim) {
9922
- queueExistingRunningReceiptWait(
9923
- receiptKey,
9924
- requestsForKey,
9925
- group.forceRefresh,
9926
- );
9927
- continue;
9928
- }
9929
- if (claim?.status === 'completed' || claim?.status === 'skipped') {
9930
- await resolveRequestsFromReceipt(requestsForKey, claim, 'cache');
9931
- durableRecoveredRequests.push(...requestsForKey);
9932
- continue;
9933
- }
9934
- if (claim?.status === 'failed') {
9935
- for (const request of requestsForKey) {
9936
- await this.rejectToolCall(
9937
- toolId,
9938
- request,
9939
- runtimeReceiptFailureError(
9940
- claim,
9941
- 'Durable tool call failed',
9942
- this.currentToolErrorSchemaVersion,
9943
- ),
10754
+ if (
10755
+ (claim?.status === 'queued' ||
10756
+ claim?.status === 'pending' ||
10757
+ claim?.status === 'running') &&
10758
+ claim.claimState === 'existing'
10759
+ ) {
10760
+ queueExistingRunningReceiptWait(
10761
+ receiptKey,
10762
+ requestsForKey,
10763
+ group.forceRefresh,
9944
10764
  );
10765
+ continue;
10766
+ }
10767
+ if (this.isOwnedClaimedRuntimeReceipt(claim)) {
10768
+ const [owner, ...waiters] = requestsForKey;
10769
+ if (!claim.leaseId) {
10770
+ throw new RuntimeReceiptLeaseLostError({
10771
+ receiptKey,
10772
+ runId: this.currentReceiptOwnerRunId,
10773
+ leaseId: 'missing',
10774
+ });
10775
+ }
10776
+ if (owner) {
10777
+ owner.receiptLeaseId = claim.leaseId ?? null;
10778
+ owner.receiptLeaseExpiresAt = claim.leaseExpiresAt ?? null;
10779
+ const claimLeaseExpiresAtMs = Date.parse(
10780
+ claim.leaseExpiresAt ?? '',
10781
+ );
10782
+ pendingReceiptClaimsEstablishExecutionFence &&=
10783
+ claim.claimState === 'claimed' &&
10784
+ claim.status === 'running' &&
10785
+ Number.isFinite(claimLeaseExpiresAtMs) &&
10786
+ claimLeaseExpiresAtMs > Date.now();
10787
+ }
10788
+ claimOwnerWithLiveFollowers(owner, waiters);
10789
+ continue;
9945
10790
  }
9946
- continue;
9947
- }
9948
- if (
9949
- (claim?.status === 'queued' ||
9950
- claim?.status === 'pending' ||
9951
- claim?.status === 'running') &&
9952
- claim.claimState === 'existing'
9953
- ) {
9954
10791
  queueExistingRunningReceiptWait(
9955
10792
  receiptKey,
9956
10793
  requestsForKey,
9957
10794
  group.forceRefresh,
9958
10795
  );
9959
- continue;
9960
- }
9961
- if (this.isOwnedClaimedRuntimeReceipt(claim)) {
9962
- const [owner, ...waiters] = requestsForKey;
9963
- if (!claim.leaseId) {
9964
- throw new RuntimeReceiptLeaseLostError({
9965
- receiptKey,
9966
- runId: this.currentReceiptOwnerRunId,
9967
- leaseId: 'missing',
9968
- });
9969
- }
9970
- if (owner) {
9971
- owner.receiptLeaseId = claim.leaseId ?? null;
9972
- owner.receiptLeaseExpiresAt = claim.leaseExpiresAt ?? null;
9973
- }
9974
- claimOwnerWithLiveFollowers(owner, waiters);
9975
- continue;
9976
10796
  }
9977
- queueExistingRunningReceiptWait(
9978
- receiptKey,
9979
- requestsForKey,
9980
- group.forceRefresh,
9981
- );
9982
- }
9983
10797
 
9984
- pendingRequests.length = 0;
9985
- pendingRequests.push(...claimedRequests);
9986
- durableExistingRunningHandler =
9987
- durableExistingRunningWaits.length > 0
9988
- ? processExistingRunningReceiptWaits()
9989
- : null;
9990
- recordToolStep(durableRecoveredRequests);
10798
+ pendingRequests.length = 0;
10799
+ pendingRequests.push(...claimedRequests);
10800
+ durableExistingRunningState.settlement =
10801
+ durableExistingRunningWaits.length > 0
10802
+ ? processExistingRunningReceiptWaits().then(
10803
+ () => ({ status: 'fulfilled' as const }),
10804
+ (reason: unknown) => ({
10805
+ status: 'rejected' as const,
10806
+ reason,
10807
+ }),
10808
+ )
10809
+ : null;
10810
+ recordToolStep(durableRecoveredRequests);
10811
+ });
9991
10812
  }
9992
10813
 
9993
- const pendingReceiptHeartbeat =
9994
- this.startPendingToolReceiptHeartbeat(pendingRequests);
10814
+ // Claims establish one lease cohort. Revalidate that cohort in one
10815
+ // gateway operation before provider admission, then let the existing
10816
+ // cohort heartbeat keep the undispatched tail alive. Revalidating each
10817
+ // call separately after a tool slot opens fragments the writer into
10818
+ // one-row batches and starves otherwise healthy providers.
10819
+ const pendingReceiptHeartbeat = this.startPendingToolReceiptHeartbeat(
10820
+ pendingRequests,
10821
+ pendingReceiptClaimsEstablishExecutionFence,
10822
+ );
10823
+ let providerFailure: { reason: unknown } | null = null;
9995
10824
  try {
10825
+ if (pendingReceiptHeartbeat) {
10826
+ await pendingReceiptHeartbeat.confirmOwned();
10827
+ }
9996
10828
  if (pendingRequests.length > 0) {
9997
10829
  const strategy =
9998
10830
  this.#options.getBatchOperationStrategy?.(toolId) ?? null;
@@ -10039,59 +10871,72 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10039
10871
  toolId: batch.batchOperation,
10040
10872
  }),
10041
10873
  });
10042
- return await this.callToolAPI(
10043
- batch.batchOperation,
10044
- batch.batchPayload,
10045
- {
10046
- durableCallReceiptKey: aggregateReceiptKey,
10047
- // A native batch is one physical provider request for
10048
- // many rows of the same column, so the batch's node scope
10049
- // is its members' shared scope.
10050
- playNodeScope: batch.memberRequests[0]
10051
- ? playNodeScopeForToolCallRequest(
10052
- batch.memberRequests[0],
10053
- batch.batchOperation,
10054
- )
10055
- : null,
10056
- executionAuthScopeDigest:
10057
- batch.memberRequests[0]?.executionAuthScopeDigest ??
10058
- null,
10059
- providerIdempotencyKey:
10060
- buildDurableToolAggregateProviderIdempotencyKey({
10061
- aggregateReceiptKey,
10062
- receiptKeys,
10063
- providerIdempotencyKeys: batch.memberRequests.map(
10064
- (request) =>
10065
- this.providerIdempotencyKeyForToolCall({
10066
- cacheKey: request.cacheKey,
10067
- force: request.force === true,
10068
- leaseId: request.receiptLeaseId,
10069
- }),
10070
- ),
10071
- }),
10072
- receiptLeaseExpiresAt: batch.memberRequests.reduce<
10073
- string | null
10074
- >((earliest, request) => {
10075
- const expiresAt = request.receiptLeaseExpiresAt ?? null;
10076
- if (!expiresAt) return earliest;
10077
- if (!earliest) return expiresAt;
10078
- return Date.parse(expiresAt) < Date.parse(earliest)
10079
- ? expiresAt
10080
- : earliest;
10081
- }, null),
10082
- timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
10083
- batch.memberRequests,
10084
- ),
10085
- beforeProviderCall: () =>
10086
- this.assertRuntimeToolReceiptOwnership(
10087
- batch.memberRequests,
10088
- ),
10089
- heartbeatReceipt: () =>
10090
- this.renewRuntimeToolReceiptOwnership(
10874
+ let releaseToolSlot: () => void = () => undefined;
10875
+ try {
10876
+ const value = await this.callToolAPI(
10877
+ batch.batchOperation,
10878
+ batch.batchPayload,
10879
+ {
10880
+ durableCallReceiptKey: aggregateReceiptKey,
10881
+ // A native batch is one physical provider request for
10882
+ // many rows of the same column, so the batch's node scope
10883
+ // is its members' shared scope.
10884
+ playNodeScope: batch.memberRequests[0]
10885
+ ? playNodeScopeForToolCallRequest(
10886
+ batch.memberRequests[0],
10887
+ batch.batchOperation,
10888
+ )
10889
+ : null,
10890
+ executionAuthScopeDigest:
10891
+ batch.memberRequests[0]?.executionAuthScopeDigest ??
10892
+ null,
10893
+ providerIdempotencyKey:
10894
+ buildDurableToolAggregateProviderIdempotencyKey({
10895
+ aggregateReceiptKey,
10896
+ receiptKeys,
10897
+ providerIdempotencyKeys: batch.memberRequests.map(
10898
+ (request) =>
10899
+ this.providerIdempotencyKeyForToolCall({
10900
+ cacheKey: request.cacheKey,
10901
+ force: request.force === true,
10902
+ leaseId: request.receiptLeaseId,
10903
+ }),
10904
+ ),
10905
+ }),
10906
+ receiptLeaseExpiresAt: batch.memberRequests.reduce<
10907
+ string | null
10908
+ >((earliest, request) => {
10909
+ const expiresAt =
10910
+ request.receiptLeaseExpiresAt ?? null;
10911
+ if (!expiresAt) return earliest;
10912
+ if (!earliest) return expiresAt;
10913
+ return Date.parse(expiresAt) < Date.parse(earliest)
10914
+ ? expiresAt
10915
+ : earliest;
10916
+ }, null),
10917
+ timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
10091
10918
  batch.memberRequests,
10092
10919
  ),
10093
- },
10094
- );
10920
+ beforeProviderCall: () =>
10921
+ pendingReceiptHeartbeat
10922
+ ? pendingReceiptHeartbeat.assertOwned()
10923
+ : this.assertRuntimeToolReceiptOwnership(
10924
+ batch.memberRequests,
10925
+ ),
10926
+ heartbeatReceipt: () =>
10927
+ this.renewRuntimeToolReceiptOwnership(
10928
+ batch.memberRequests,
10929
+ ),
10930
+ retainToolSlot: (release) => {
10931
+ releaseToolSlot = release;
10932
+ },
10933
+ },
10934
+ );
10935
+ return { value, releaseToolSlot };
10936
+ } catch (error) {
10937
+ releaseToolSlot();
10938
+ throw error;
10939
+ }
10095
10940
  },
10096
10941
  onChunkComplete: async (chunkResults) => {
10097
10942
  for (const entry of chunkResults) {
@@ -10101,29 +10946,35 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10101
10946
  }
10102
10947
  continue;
10103
10948
  }
10104
- const splitResults =
10105
- entry.result != null
10106
- ? entry.request.splitResults(entry.result)
10107
- : entry.request.memberRequests.map(() => null);
10108
- const resolvedResults =
10109
- await this.resolveToolCallBatchResults(
10110
- toolId,
10111
- entry.request.memberRequests.map((request, index) => ({
10112
- request,
10113
- result: splitResults[index] ?? null,
10114
- })),
10115
- );
10949
+ try {
10950
+ const splitResults =
10951
+ entry.result != null
10952
+ ? entry.request.splitResults(entry.result.value)
10953
+ : entry.request.memberRequests.map(() => null);
10954
+ const resolvedResults =
10955
+ await this.resolveToolCallBatchResults(
10956
+ toolId,
10957
+ entry.request.memberRequests.map(
10958
+ (request, index) => ({
10959
+ request,
10960
+ result: splitResults[index] ?? null,
10961
+ }),
10962
+ ),
10963
+ );
10116
10964
 
10117
- for (
10118
- let index = 0;
10119
- index < entry.request.memberRequests.length;
10120
- index += 1
10121
- ) {
10122
- const request = entry.request.memberRequests[index]!;
10123
- if (resolvedResults[index] != null) {
10124
- successfulLiveStepCallIds.add(request.callId);
10965
+ for (
10966
+ let index = 0;
10967
+ index < entry.request.memberRequests.length;
10968
+ index += 1
10969
+ ) {
10970
+ const request = entry.request.memberRequests[index]!;
10971
+ if (resolvedResults[index] != null) {
10972
+ successfulLiveStepCallIds.add(request.callId);
10973
+ }
10974
+ resolveLiveFollowers(request, resolvedResults[index]);
10125
10975
  }
10126
- resolveLiveFollowers(request, resolvedResults[index]);
10976
+ } finally {
10977
+ entry.result?.releaseToolSlot();
10127
10978
  }
10128
10979
  }
10129
10980
 
@@ -10134,6 +10985,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10134
10985
  );
10135
10986
  this.#options.onBatchComplete?.(this.checkpoint);
10136
10987
  },
10988
+ retainResults: false,
10137
10989
  });
10138
10990
  } else {
10139
10991
  const completionBuffer: Array<{
@@ -10235,36 +11087,48 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10235
11087
  this.persistenceLatch,
10236
11088
  );
10237
11089
  }
10238
- const execution = await this.callToolExecutionAPI(
10239
- toolId,
10240
- request.input,
10241
- {
10242
- beforeProviderCall: () =>
10243
- this.assertRuntimeToolReceiptOwnership([request]),
10244
- playNodeScope: playNodeScopeForToolCallRequest(request),
10245
- ...(request.receiptKey
10246
- ? {
10247
- durableCallReceiptKey: request.receiptKey,
10248
- executionAuthScopeDigest:
10249
- request.executionAuthScopeDigest,
10250
- providerIdempotencyKey:
10251
- this.providerIdempotencyKeyForToolCall({
10252
- cacheKey: request.receiptKey,
10253
- force: request.force === true,
10254
- leaseId: request.receiptLeaseId,
10255
- }),
10256
- receiptLeaseExpiresAt:
10257
- request.receiptLeaseExpiresAt,
10258
- heartbeatReceipt: () =>
10259
- this.renewRuntimeToolReceiptOwnership([request]),
10260
- }
10261
- : {}),
10262
- timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
10263
- request,
10264
- ]),
10265
- },
10266
- );
10267
- return await enqueueCompletion(request, execution);
11090
+ let releaseToolSlot: () => void = () => undefined;
11091
+ try {
11092
+ const execution = await this.callToolExecutionAPI(
11093
+ toolId,
11094
+ request.input,
11095
+ {
11096
+ beforeProviderCall: () =>
11097
+ pendingReceiptHeartbeat
11098
+ ? pendingReceiptHeartbeat.assertOwned()
11099
+ : this.assertRuntimeToolReceiptOwnership([request]),
11100
+ playNodeScope: playNodeScopeForToolCallRequest(request),
11101
+ ...(request.receiptKey
11102
+ ? {
11103
+ durableCallReceiptKey: request.receiptKey,
11104
+ executionAuthScopeDigest:
11105
+ request.executionAuthScopeDigest,
11106
+ providerIdempotencyKey:
11107
+ this.providerIdempotencyKeyForToolCall({
11108
+ cacheKey: request.receiptKey,
11109
+ force: request.force === true,
11110
+ leaseId: request.receiptLeaseId,
11111
+ }),
11112
+ receiptLeaseExpiresAt:
11113
+ request.receiptLeaseExpiresAt,
11114
+ heartbeatReceipt: () =>
11115
+ this.renewRuntimeToolReceiptOwnership([
11116
+ request,
11117
+ ]),
11118
+ }
11119
+ : {}),
11120
+ timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
11121
+ request,
11122
+ ]),
11123
+ retainToolSlot: (release) => {
11124
+ releaseToolSlot = release;
11125
+ },
11126
+ },
11127
+ );
11128
+ await enqueueCompletion(request, execution);
11129
+ } finally {
11130
+ releaseToolSlot();
11131
+ }
10268
11132
  },
10269
11133
  );
10270
11134
  const failedEntries = dispatchResults.filter(
@@ -10281,14 +11145,26 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10281
11145
  this.#options.onBatchComplete?.(this.checkpoint);
10282
11146
  }
10283
11147
  }
11148
+ } catch (reason) {
11149
+ providerFailure = { reason };
11150
+ durableExistingRunningAbort.abort(reason);
10284
11151
  } finally {
10285
11152
  pendingReceiptHeartbeat?.stop();
10286
11153
  }
10287
11154
  const pendingReceiptLeaseLost = pendingReceiptHeartbeat?.leaseLost();
10288
- if (pendingReceiptLeaseLost) throw pendingReceiptLeaseLost;
10289
- if (durableExistingRunningHandler) {
10290
- await durableExistingRunningHandler;
11155
+ if (pendingReceiptLeaseLost && providerFailure === null) {
11156
+ providerFailure = { reason: pendingReceiptLeaseLost };
11157
+ durableExistingRunningAbort.abort(pendingReceiptLeaseLost);
11158
+ }
11159
+ const durableExistingRunningSettlement =
11160
+ durableExistingRunningState.settlement;
11161
+ if (durableExistingRunningSettlement) {
11162
+ const settlement = await durableExistingRunningSettlement;
11163
+ if (settlement.status === 'rejected' && providerFailure === null) {
11164
+ providerFailure = { reason: settlement.reason };
11165
+ }
10291
11166
  }
11167
+ if (providerFailure) throw providerFailure.reason;
10292
11168
  }),
10293
11169
  );
10294
11170
  const rejected = toolSettlements.find(
@@ -10497,7 +11373,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10497
11373
  `[perf] tool call id=${toolId} phase=governor_admission elapsed_ms=${Date.now() - admissionStartedAt}`,
10498
11374
  );
10499
11375
  }
11376
+ let toolSlotTransferred = false;
10500
11377
  try {
11378
+ if (options?.retainToolSlot) {
11379
+ options.retainToolSlot(() => toolSlot.release());
11380
+ toolSlotTransferred = true;
11381
+ }
10501
11382
  return await withActiveSpan(
10502
11383
  'plays.tool.execute',
10503
11384
  {
@@ -11271,7 +12152,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11271
12152
  },
11272
12153
  });
11273
12154
  }
11274
- toolSlot.release();
12155
+ if (!toolSlotTransferred) {
12156
+ toolSlot.release();
12157
+ }
11275
12158
  }
11276
12159
  }
11277
12160