deepline 0.2.55 → 0.2.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +14 -0
- package/dist/bundling-sources/sdk/src/http.ts +19 -1
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/stream-reconnect.ts +6 -0
- package/dist/bundling-sources/sdk/src/types.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +146 -12
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +6 -3
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1631 -748
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +81 -52
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +146 -6
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +56 -22
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +14 -11
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +21 -2
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +81 -21
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +27 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +49 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +96 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +341 -67
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +17 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +17 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +16 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +25 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +80 -1
- package/dist/bundling-sources/shared_libs/security/safe-fetch.ts +9 -0
- package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +85 -48
- package/dist/cli/index.js +409 -51
- package/dist/cli/index.mjs +389 -25
- package/dist/index.d.mts +19 -1
- package/dist/index.d.ts +19 -1
- package/dist/index.js +29 -2
- package/dist/index.mjs +29 -2
- package/package.json +1 -1
|
@@ -46,7 +46,10 @@ import type { PlayQueueHint } from './governor/rate-state-backend';
|
|
|
46
46
|
import type { MapRowOutcome } from './durability-store';
|
|
47
47
|
import { stringifyPostgresJson } from './postgres-json';
|
|
48
48
|
import { RUNTIME_ROW_OUTPUT_NOT_JSON_SERIALIZABLE } from './runtime-sheet-row-transition';
|
|
49
|
-
import {
|
|
49
|
+
import {
|
|
50
|
+
RuntimeSheetRowWriter,
|
|
51
|
+
type RuntimeSheetRowWriterDiagnostics,
|
|
52
|
+
} from './runtime-sheet-row-writer';
|
|
50
53
|
import type { WorkReceiptFailureKind } from './work-receipts';
|
|
51
54
|
import {
|
|
52
55
|
completedMapRowOutcome,
|
|
@@ -161,6 +164,8 @@ import {
|
|
|
161
164
|
type PlayAuthoringRuntimeContext,
|
|
162
165
|
} from '../plays/authoring-contract';
|
|
163
166
|
import {
|
|
167
|
+
DURABLE_RECEIPT_WAIT_DELAY_MS,
|
|
168
|
+
DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS,
|
|
164
169
|
RuntimeReceiptLeaseLostError,
|
|
165
170
|
RuntimeReceiptWaitTimeoutError,
|
|
166
171
|
executeWithDurableRuntimeReceipt,
|
|
@@ -331,6 +336,8 @@ const rowContext = new AsyncLocalStorage<{
|
|
|
331
336
|
tableNamespace?: string;
|
|
332
337
|
rowKey?: string;
|
|
333
338
|
mapScope?: MapExecutionScope;
|
|
339
|
+
mapStallObserver?: RuntimeMapStallObserver;
|
|
340
|
+
mapStallResolverToken?: number;
|
|
334
341
|
/**
|
|
335
342
|
* Per-resolver inline-child fan-width guard. This stays in the async row
|
|
336
343
|
* scope, so it becomes collectible as soon as the resolver settles.
|
|
@@ -366,6 +373,7 @@ const MAP_PERSIST_CHUNK_BYTES = 8 * 1024 * 1024;
|
|
|
366
373
|
const MAP_ROW_WRITER_BUFFER_BYTES = 16 * 1024 * 1024;
|
|
367
374
|
const MAP_ROW_WRITER_MAX_FLUSH_MS = 100;
|
|
368
375
|
const MAP_FRAME_FLUSH_INTERVAL_MS = 250;
|
|
376
|
+
const MAP_STALL_LOG_INTERVAL_MS = 60_000;
|
|
369
377
|
// Tool scheduling lanes wait at most this long for same-lane row continuations.
|
|
370
378
|
// The window is fixed from the first item and never resets on later arrivals.
|
|
371
379
|
const TOOL_BATCH_COALESCE_WINDOW_MS = 5;
|
|
@@ -389,6 +397,9 @@ const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
|
|
|
389
397
|
const DEFAULT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000 + 30_000;
|
|
390
398
|
const FETCH_TRANSPORT_MAX_ATTEMPTS = 3;
|
|
391
399
|
const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
|
|
400
|
+
const CTX_FETCH_HEADERS_TIMEOUT_MS = 30_000;
|
|
401
|
+
const CTX_FETCH_BODY_TIMEOUT_MS = 30_000;
|
|
402
|
+
const CTX_FETCH_TOTAL_TIMEOUT_MS = 120_000;
|
|
392
403
|
// cloudflared returns this branded HTML when its connection to the local app
|
|
393
404
|
// resets. It is a transport failure before the app can reach a provider, not a
|
|
394
405
|
// provider 502; require both the branded page and our development tunnel host.
|
|
@@ -427,6 +438,388 @@ class ToolExecuteResponseBodyTimeoutError extends Error {
|
|
|
427
438
|
}
|
|
428
439
|
}
|
|
429
440
|
|
|
441
|
+
export class CtxFetchTimeoutError extends Error {
|
|
442
|
+
readonly code = 'CTX_FETCH_TIMEOUT';
|
|
443
|
+
|
|
444
|
+
constructor(
|
|
445
|
+
readonly phase: 'headers' | 'body' | 'total',
|
|
446
|
+
readonly timeoutMs: number,
|
|
447
|
+
) {
|
|
448
|
+
super(
|
|
449
|
+
`[CTX_FETCH_TIMEOUT phase=${phase} timeout_ms=${timeoutMs}] ` +
|
|
450
|
+
`ctx.fetch exceeded its ${phase} deadline after ${timeoutMs}ms.`,
|
|
451
|
+
);
|
|
452
|
+
this.name = 'CtxFetchTimeoutError';
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
type CtxFetchDeadline = {
|
|
457
|
+
bodyMs: number;
|
|
458
|
+
headersMs: number;
|
|
459
|
+
startedAt: number;
|
|
460
|
+
totalMs: number;
|
|
461
|
+
};
|
|
462
|
+
|
|
463
|
+
function positiveTimeoutMs(
|
|
464
|
+
value: number | undefined,
|
|
465
|
+
fallback: number,
|
|
466
|
+
): number {
|
|
467
|
+
return value === undefined || !Number.isFinite(value) || value <= 0
|
|
468
|
+
? fallback
|
|
469
|
+
: Math.max(1, Math.floor(value));
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
function remainingCtxFetchMs(deadline: CtxFetchDeadline): number {
|
|
473
|
+
return Math.max(0, deadline.totalMs - (Date.now() - deadline.startedAt));
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
async function runCtxFetchPhase<T>(input: {
|
|
477
|
+
callerSignal?: AbortSignal | null;
|
|
478
|
+
controller: AbortController;
|
|
479
|
+
deadline: CtxFetchDeadline;
|
|
480
|
+
phase: 'headers' | 'body';
|
|
481
|
+
run: () => Promise<T>;
|
|
482
|
+
}): Promise<T> {
|
|
483
|
+
const remainingMs = remainingCtxFetchMs(input.deadline);
|
|
484
|
+
if (remainingMs <= 0) {
|
|
485
|
+
throw new CtxFetchTimeoutError('total', input.deadline.totalMs);
|
|
486
|
+
}
|
|
487
|
+
if (input.callerSignal?.aborted) {
|
|
488
|
+
throw (
|
|
489
|
+
input.callerSignal.reason ?? new DOMException('Aborted', 'AbortError')
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
const phaseLimitMs =
|
|
493
|
+
input.phase === 'headers'
|
|
494
|
+
? input.deadline.headersMs
|
|
495
|
+
: input.deadline.bodyMs;
|
|
496
|
+
const timeoutMs = Math.min(phaseLimitMs, remainingMs);
|
|
497
|
+
const timeoutPhase = timeoutMs >= remainingMs ? 'total' : input.phase;
|
|
498
|
+
let timeout: ReturnType<typeof setTimeout> | null = null;
|
|
499
|
+
let onCallerAbort: (() => void) | null = null;
|
|
500
|
+
const phaseTimeout = new Promise<never>((_resolve, reject) => {
|
|
501
|
+
timeout = setTimeout(() => {
|
|
502
|
+
const error = new CtxFetchTimeoutError(
|
|
503
|
+
timeoutPhase,
|
|
504
|
+
timeoutPhase === 'total' ? input.deadline.totalMs : phaseLimitMs,
|
|
505
|
+
);
|
|
506
|
+
reject(error);
|
|
507
|
+
input.controller.abort(error);
|
|
508
|
+
}, timeoutMs);
|
|
509
|
+
});
|
|
510
|
+
const callerAbort = input.callerSignal
|
|
511
|
+
? new Promise<never>((_resolve, reject) => {
|
|
512
|
+
onCallerAbort = () => {
|
|
513
|
+
const reason =
|
|
514
|
+
input.callerSignal?.reason ??
|
|
515
|
+
new DOMException('Aborted', 'AbortError');
|
|
516
|
+
reject(reason);
|
|
517
|
+
input.controller.abort(reason);
|
|
518
|
+
};
|
|
519
|
+
input.callerSignal!.addEventListener('abort', onCallerAbort, {
|
|
520
|
+
once: true,
|
|
521
|
+
});
|
|
522
|
+
})
|
|
523
|
+
: null;
|
|
524
|
+
try {
|
|
525
|
+
return await Promise.race([
|
|
526
|
+
input.run(),
|
|
527
|
+
phaseTimeout,
|
|
528
|
+
...(callerAbort ? [callerAbort] : []),
|
|
529
|
+
]);
|
|
530
|
+
} finally {
|
|
531
|
+
if (timeout) clearTimeout(timeout);
|
|
532
|
+
if (onCallerAbort) {
|
|
533
|
+
input.callerSignal?.removeEventListener('abort', onCallerAbort);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
async function sleepWithinCtxFetchDeadline(input: {
|
|
539
|
+
callerSignal?: AbortSignal | null;
|
|
540
|
+
deadline: CtxFetchDeadline;
|
|
541
|
+
delayMs: number;
|
|
542
|
+
}): Promise<void> {
|
|
543
|
+
const controller = new AbortController();
|
|
544
|
+
await runCtxFetchPhase({
|
|
545
|
+
callerSignal: input.callerSignal,
|
|
546
|
+
controller,
|
|
547
|
+
deadline: input.deadline,
|
|
548
|
+
phase: 'headers',
|
|
549
|
+
run: () =>
|
|
550
|
+
new Promise<void>((resolve) => {
|
|
551
|
+
setTimeout(
|
|
552
|
+
resolve,
|
|
553
|
+
Math.min(input.delayMs, remainingCtxFetchMs(input.deadline)),
|
|
554
|
+
);
|
|
555
|
+
}),
|
|
556
|
+
});
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
async function readCtxFetchBody(input: {
|
|
560
|
+
callerSignal?: AbortSignal | null;
|
|
561
|
+
controller: AbortController;
|
|
562
|
+
deadline: CtxFetchDeadline;
|
|
563
|
+
response: Response;
|
|
564
|
+
}): Promise<string> {
|
|
565
|
+
if (!input.response.body) return '';
|
|
566
|
+
const reader = input.response.body.getReader();
|
|
567
|
+
const decoder = new TextDecoder();
|
|
568
|
+
try {
|
|
569
|
+
return await runCtxFetchPhase({
|
|
570
|
+
callerSignal: input.callerSignal,
|
|
571
|
+
controller: input.controller,
|
|
572
|
+
deadline: input.deadline,
|
|
573
|
+
phase: 'body',
|
|
574
|
+
run: async () => {
|
|
575
|
+
let body = '';
|
|
576
|
+
for (;;) {
|
|
577
|
+
const chunk = await reader.read();
|
|
578
|
+
if (chunk.done) return body + decoder.decode();
|
|
579
|
+
body += decoder.decode(chunk.value, { stream: true });
|
|
580
|
+
}
|
|
581
|
+
},
|
|
582
|
+
});
|
|
583
|
+
} catch (error) {
|
|
584
|
+
// Cancellation is socket hygiene, not part of the timeout contract. A
|
|
585
|
+
// hostile/custom stream may never resolve its cancel hook; awaiting it
|
|
586
|
+
// here would recreate the exact forever-pending ctx.fetch this deadline
|
|
587
|
+
// is meant to eliminate.
|
|
588
|
+
void reader.cancel(error).catch(() => {});
|
|
589
|
+
throw error;
|
|
590
|
+
} finally {
|
|
591
|
+
reader.releaseLock();
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
class RuntimeMapStallObserver {
|
|
596
|
+
readonly #resolvers = new Map<
|
|
597
|
+
number,
|
|
598
|
+
{
|
|
599
|
+
phase: 'resolver' | 'fetch';
|
|
600
|
+
phaseStartedAt: number;
|
|
601
|
+
startedAt: number;
|
|
602
|
+
}
|
|
603
|
+
>();
|
|
604
|
+
readonly #terminalAdmissionStartedAt = new Map<number, number>();
|
|
605
|
+
readonly #terminalCommitStartedAt = new Map<number, number>();
|
|
606
|
+
readonly #getWriterDiagnostics: () => RuntimeSheetRowWriterDiagnostics | null;
|
|
607
|
+
readonly #log: (line: string) => void;
|
|
608
|
+
readonly #mapName: string;
|
|
609
|
+
readonly #pageOffset: number;
|
|
610
|
+
readonly #intervalMs: number;
|
|
611
|
+
#timer: ReturnType<typeof setTimeout> | null = null;
|
|
612
|
+
#lastTerminalCommitAt = Date.now();
|
|
613
|
+
#terminalCommitted = 0;
|
|
614
|
+
#token = 0;
|
|
615
|
+
|
|
616
|
+
constructor(input: {
|
|
617
|
+
getWriterDiagnostics: () => RuntimeSheetRowWriterDiagnostics | null;
|
|
618
|
+
log: (line: string) => void;
|
|
619
|
+
mapName: string;
|
|
620
|
+
pageOffset: number;
|
|
621
|
+
intervalMs?: number;
|
|
622
|
+
}) {
|
|
623
|
+
this.#getWriterDiagnostics = input.getWriterDiagnostics;
|
|
624
|
+
this.#log = input.log;
|
|
625
|
+
this.#mapName = input.mapName;
|
|
626
|
+
this.#pageOffset = input.pageOffset;
|
|
627
|
+
this.#intervalMs = positiveTimeoutMs(
|
|
628
|
+
input.intervalMs,
|
|
629
|
+
MAP_STALL_LOG_INTERVAL_MS,
|
|
630
|
+
);
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
resolverStarted(): { finish: () => void; token: number } {
|
|
634
|
+
const token = ++this.#token;
|
|
635
|
+
const now = Date.now();
|
|
636
|
+
this.#resolvers.set(token, {
|
|
637
|
+
phase: 'resolver',
|
|
638
|
+
phaseStartedAt: now,
|
|
639
|
+
startedAt: now,
|
|
640
|
+
});
|
|
641
|
+
this.#schedule();
|
|
642
|
+
return {
|
|
643
|
+
token,
|
|
644
|
+
finish: () => {
|
|
645
|
+
this.#resolvers.delete(token);
|
|
646
|
+
this.#cancelIfIdle();
|
|
647
|
+
},
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
fetchStarted(resolverToken: number | undefined): () => void {
|
|
652
|
+
if (resolverToken === undefined) return () => {};
|
|
653
|
+
const resolver = this.#resolvers.get(resolverToken);
|
|
654
|
+
if (!resolver) return () => {};
|
|
655
|
+
resolver.phase = 'fetch';
|
|
656
|
+
resolver.phaseStartedAt = Date.now();
|
|
657
|
+
return () => {
|
|
658
|
+
const current = this.#resolvers.get(resolverToken);
|
|
659
|
+
if (!current) return;
|
|
660
|
+
current.phase = 'resolver';
|
|
661
|
+
current.phaseStartedAt = Date.now();
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
terminalQueued(): {
|
|
666
|
+
admitted: () => void;
|
|
667
|
+
committed: () => void;
|
|
668
|
+
failed: () => void;
|
|
669
|
+
} {
|
|
670
|
+
const token = ++this.#token;
|
|
671
|
+
const now = Date.now();
|
|
672
|
+
this.#terminalAdmissionStartedAt.set(token, now);
|
|
673
|
+
this.#schedule();
|
|
674
|
+
return {
|
|
675
|
+
admitted: () => {
|
|
676
|
+
this.#terminalAdmissionStartedAt.delete(token);
|
|
677
|
+
this.#terminalCommitStartedAt.set(token, Date.now());
|
|
678
|
+
},
|
|
679
|
+
committed: () => {
|
|
680
|
+
this.#terminalAdmissionStartedAt.delete(token);
|
|
681
|
+
if (this.#terminalCommitStartedAt.delete(token)) {
|
|
682
|
+
this.#terminalCommitted += 1;
|
|
683
|
+
this.#lastTerminalCommitAt = Date.now();
|
|
684
|
+
}
|
|
685
|
+
this.#cancelIfIdle();
|
|
686
|
+
},
|
|
687
|
+
failed: () => {
|
|
688
|
+
this.#terminalAdmissionStartedAt.delete(token);
|
|
689
|
+
this.#terminalCommitStartedAt.delete(token);
|
|
690
|
+
this.#cancelIfIdle();
|
|
691
|
+
},
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
stop(): void {
|
|
696
|
+
if (this.#timer) clearTimeout(this.#timer);
|
|
697
|
+
this.#timer = null;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
#schedule(): void {
|
|
701
|
+
if (this.#timer) return;
|
|
702
|
+
this.#timer = setTimeout(() => {
|
|
703
|
+
this.#timer = null;
|
|
704
|
+
this.#emitIfStalled();
|
|
705
|
+
if (this.#hasUnresolved()) this.#schedule();
|
|
706
|
+
}, this.#intervalMs);
|
|
707
|
+
if (
|
|
708
|
+
typeof this.#timer === 'object' &&
|
|
709
|
+
'unref' in this.#timer &&
|
|
710
|
+
typeof this.#timer.unref === 'function'
|
|
711
|
+
) {
|
|
712
|
+
this.#timer.unref();
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
#cancelIfIdle(): void {
|
|
717
|
+
if (!this.#hasUnresolved()) this.stop();
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
#hasUnresolved(): boolean {
|
|
721
|
+
return (
|
|
722
|
+
this.#resolvers.size > 0 ||
|
|
723
|
+
this.#terminalAdmissionStartedAt.size > 0 ||
|
|
724
|
+
this.#terminalCommitStartedAt.size > 0
|
|
725
|
+
);
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
#emitIfStalled(): void {
|
|
729
|
+
const now = Date.now();
|
|
730
|
+
if (now - this.#lastTerminalCommitAt < this.#intervalMs) return;
|
|
731
|
+
const writer = this.#getWriterDiagnostics();
|
|
732
|
+
const unresolved =
|
|
733
|
+
this.#resolvers.size +
|
|
734
|
+
this.#terminalAdmissionStartedAt.size +
|
|
735
|
+
this.#terminalCommitStartedAt.size +
|
|
736
|
+
(writer?.activeRows ?? 0) +
|
|
737
|
+
(writer?.queuedTerminalRows ?? 0) +
|
|
738
|
+
(writer?.blockedTerminalRows ?? 0);
|
|
739
|
+
if (unresolved === 0) return;
|
|
740
|
+
const oldest = (...sources: Array<Iterable<number>>): number | null => {
|
|
741
|
+
let value: number | null = null;
|
|
742
|
+
for (const source of sources) {
|
|
743
|
+
for (const candidate of source) {
|
|
744
|
+
value = value === null ? candidate : Math.min(value, candidate);
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
return value;
|
|
748
|
+
};
|
|
749
|
+
const oldestUnsettledAt = oldest(
|
|
750
|
+
[...this.#resolvers.values()].map((resolver) => resolver.startedAt),
|
|
751
|
+
this.#terminalAdmissionStartedAt.values(),
|
|
752
|
+
this.#terminalCommitStartedAt.values(),
|
|
753
|
+
writer?.activeStartedAt === null || writer?.activeStartedAt === undefined
|
|
754
|
+
? []
|
|
755
|
+
: [writer.activeStartedAt],
|
|
756
|
+
);
|
|
757
|
+
const phaseCandidates: Array<{
|
|
758
|
+
at: number;
|
|
759
|
+
phase: 'fetch' | 'resolver' | 'writer_admission' | 'writer_commit';
|
|
760
|
+
}> = [
|
|
761
|
+
...[...this.#resolvers.values()].map((resolver) => ({
|
|
762
|
+
at: resolver.startedAt,
|
|
763
|
+
phase: resolver.phase,
|
|
764
|
+
})),
|
|
765
|
+
...[...this.#terminalAdmissionStartedAt.values()].map((at) => ({
|
|
766
|
+
at,
|
|
767
|
+
phase: 'writer_admission' as const,
|
|
768
|
+
})),
|
|
769
|
+
...[...this.#terminalCommitStartedAt.values()].map((at) => ({
|
|
770
|
+
at,
|
|
771
|
+
phase: 'writer_commit' as const,
|
|
772
|
+
})),
|
|
773
|
+
];
|
|
774
|
+
if (
|
|
775
|
+
writer?.activeKind === 'terminal' &&
|
|
776
|
+
writer.activeRows > 0 &&
|
|
777
|
+
writer.activeStartedAt !== null
|
|
778
|
+
) {
|
|
779
|
+
phaseCandidates.push({
|
|
780
|
+
at: writer.activeStartedAt,
|
|
781
|
+
phase: 'writer_commit',
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
const oldestPhase = phaseCandidates.reduce<
|
|
785
|
+
(typeof phaseCandidates)[number] | null
|
|
786
|
+
>(
|
|
787
|
+
(current, candidate) =>
|
|
788
|
+
current === null || candidate.at < current.at ? candidate : current,
|
|
789
|
+
null,
|
|
790
|
+
)?.phase;
|
|
791
|
+
const fetchResolvers = [...this.#resolvers.values()].filter(
|
|
792
|
+
(resolver) => resolver.phase === 'fetch',
|
|
793
|
+
);
|
|
794
|
+
this.#log(
|
|
795
|
+
`[runtime.map_stall] ${JSON.stringify({
|
|
796
|
+
map: this.#mapName,
|
|
797
|
+
page_offset: this.#pageOffset,
|
|
798
|
+
resolver_active: this.#resolvers.size,
|
|
799
|
+
fetch_active: fetchResolvers.length,
|
|
800
|
+
oldest_fetch_ms: Math.max(
|
|
801
|
+
0,
|
|
802
|
+
now -
|
|
803
|
+
(oldest(
|
|
804
|
+
fetchResolvers.map((resolver) => resolver.phaseStartedAt),
|
|
805
|
+
) ?? now),
|
|
806
|
+
),
|
|
807
|
+
terminal_admission_waiting: this.#terminalAdmissionStartedAt.size,
|
|
808
|
+
terminal_commit_waiting: this.#terminalCommitStartedAt.size,
|
|
809
|
+
terminal_write_active:
|
|
810
|
+
writer?.activeKind === 'terminal' ? writer.activeRows : 0,
|
|
811
|
+
terminal_queued:
|
|
812
|
+
(writer?.queuedTerminalRows ?? 0) +
|
|
813
|
+
(writer?.blockedTerminalRows ?? 0),
|
|
814
|
+
terminal_committed: this.#terminalCommitted,
|
|
815
|
+
oldest_unsettled_ms: Math.max(0, now - (oldestUnsettledAt ?? now)),
|
|
816
|
+
oldest_phase: oldestPhase ?? 'resolver',
|
|
817
|
+
last_terminal_commit_ms: Math.max(0, now - this.#lastTerminalCommitAt),
|
|
818
|
+
})}`,
|
|
819
|
+
);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
|
|
430
823
|
async function readToolExecuteResponseBody<T>(input: {
|
|
431
824
|
toolId: string;
|
|
432
825
|
abortController: AbortController | null;
|
|
@@ -841,6 +1234,12 @@ type ToolExecutionApiOptions = {
|
|
|
841
1234
|
beforeProviderCall?: () => Promise<void> | void;
|
|
842
1235
|
heartbeatReceipt?: () => Promise<void> | void;
|
|
843
1236
|
parkProviderCall?: () => Promise<void> | void;
|
|
1237
|
+
/**
|
|
1238
|
+
* Transfer the logical tool permit to the caller so it can remain held until
|
|
1239
|
+
* the decoded response is durably settled. The release callback is
|
|
1240
|
+
* idempotent. Callers must release it in a finally block.
|
|
1241
|
+
*/
|
|
1242
|
+
retainToolSlot?: (release: () => void) => void;
|
|
844
1243
|
/**
|
|
845
1244
|
* Explicit graph-node attribution for this provider call (ADR 0018). Carried
|
|
846
1245
|
* to the execute route, which stamps it onto the usage event so per-node
|
|
@@ -936,6 +1335,10 @@ function failedMapRowLogLabel(row: PersistableMapRow): string {
|
|
|
936
1335
|
type FieldMapRunResult = {
|
|
937
1336
|
completedRows: PersistableMapRow[];
|
|
938
1337
|
failedRows: PersistableMapRow[];
|
|
1338
|
+
completedRowCount: number;
|
|
1339
|
+
failedRowCount: number;
|
|
1340
|
+
previewRows: Record<string, unknown>[];
|
|
1341
|
+
retainedRowsComplete: boolean;
|
|
939
1342
|
};
|
|
940
1343
|
|
|
941
1344
|
type RuntimeMapRowPersistence = {
|
|
@@ -946,6 +1349,7 @@ type RuntimeMapRowPersistence = {
|
|
|
946
1349
|
isPersisted: (row: PersistableMapRow) => boolean;
|
|
947
1350
|
checkpoint: (updates: PlayRowUpdate[]) => Promise<void>;
|
|
948
1351
|
flush: () => Promise<void>;
|
|
1352
|
+
diagnostics: () => RuntimeSheetRowWriterDiagnostics;
|
|
949
1353
|
};
|
|
950
1354
|
|
|
951
1355
|
function createRuntimeMapRowPersistence(
|
|
@@ -1050,6 +1454,7 @@ function createRuntimeMapRowPersistence(
|
|
|
1050
1454
|
flush: async () => {
|
|
1051
1455
|
await writer.finish();
|
|
1052
1456
|
},
|
|
1457
|
+
diagnostics: () => writer.diagnostics(),
|
|
1053
1458
|
};
|
|
1054
1459
|
}
|
|
1055
1460
|
|
|
@@ -1324,8 +1729,9 @@ class RuntimeDatasetBuilder<T extends Record<string, unknown>> {
|
|
|
1324
1729
|
|
|
1325
1730
|
const WAITING_ROW = Symbol('deepline.waiting_row');
|
|
1326
1731
|
const FAILED_ROW = Symbol('deepline.failed_row');
|
|
1732
|
+
const COMPLETED_ROW = Symbol('deepline.completed_row');
|
|
1327
1733
|
type MapRowExecutionResult =
|
|
1328
|
-
|
|
|
1734
|
+
| typeof COMPLETED_ROW
|
|
1329
1735
|
| typeof WAITING_ROW
|
|
1330
1736
|
| typeof FAILED_ROW;
|
|
1331
1737
|
|
|
@@ -1446,6 +1852,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
1446
1852
|
reject: (error: unknown) => void;
|
|
1447
1853
|
}> = [];
|
|
1448
1854
|
private runtimeToolOwnershipAssertionFlushScheduled = false;
|
|
1855
|
+
private runtimeToolReceiptCompletionsInFlight = new Map<string, number>();
|
|
1449
1856
|
/**
|
|
1450
1857
|
* Direct durable boundaries (including ctx.fetch) arrive independently from
|
|
1451
1858
|
* concurrent map rows. Hold one microtask's worth so they can use the bulk
|
|
@@ -1465,6 +1872,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
1465
1872
|
}>;
|
|
1466
1873
|
}
|
|
1467
1874
|
>();
|
|
1875
|
+
/**
|
|
1876
|
+
* Completed durable receipt payloads can expand many times beyond their wire
|
|
1877
|
+
* size during JSON parsing. Scheduling groups may overlap for provider
|
|
1878
|
+
* throughput, but only one group at a time may hydrate and deliver cached
|
|
1879
|
+
* results. Pending claims leave this section before provider execution, so
|
|
1880
|
+
* cache misses retain normal cross-group concurrency.
|
|
1881
|
+
*/
|
|
1882
|
+
private runtimeReceiptHydrationTurn: Promise<void> = Promise.resolve();
|
|
1468
1883
|
/**
|
|
1469
1884
|
* Fixed, non-resetting coalescing deadlines for ready scheduling lanes. A
|
|
1470
1885
|
* deadline starts when the first request enters an empty batch bucket. New
|
|
@@ -1850,6 +2265,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
1850
2265
|
options.getToolProvider,
|
|
1851
2266
|
),
|
|
1852
2267
|
resolveRateScope: options.getToolRateScope,
|
|
2268
|
+
maxConcurrentExternalCalls: options.maxConcurrentExternalCalls,
|
|
2269
|
+
maxConcurrentRows: options.maxConcurrentRows,
|
|
1853
2270
|
// A root run keeps depth 0 (its first child play is depth 1) but still
|
|
1854
2271
|
// seeds its own play id into the ancestry/currentPlayId so the cycle guard
|
|
1855
2272
|
// catches a child re-invoking the root. createDefaultGovernanceSnapshot
|
|
@@ -3077,6 +3494,22 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3077
3494
|
return byKey;
|
|
3078
3495
|
}
|
|
3079
3496
|
|
|
3497
|
+
private async withRuntimeReceiptHydrationTurn<T>(
|
|
3498
|
+
hydrate: () => Promise<T>,
|
|
3499
|
+
): Promise<T> {
|
|
3500
|
+
const precedingTurn = this.runtimeReceiptHydrationTurn;
|
|
3501
|
+
let releaseTurn!: () => void;
|
|
3502
|
+
this.runtimeReceiptHydrationTurn = new Promise<void>((resolve) => {
|
|
3503
|
+
releaseTurn = resolve;
|
|
3504
|
+
});
|
|
3505
|
+
await precedingTurn;
|
|
3506
|
+
try {
|
|
3507
|
+
return await hydrate();
|
|
3508
|
+
} finally {
|
|
3509
|
+
releaseTurn();
|
|
3510
|
+
}
|
|
3511
|
+
}
|
|
3512
|
+
|
|
3080
3513
|
private async markRuntimeStepReceiptRunning(input: {
|
|
3081
3514
|
key: string;
|
|
3082
3515
|
runId: string;
|
|
@@ -3587,7 +4020,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3587
4020
|
markRunningBeforeExecute?: boolean;
|
|
3588
4021
|
requiresExecutionLock?: boolean;
|
|
3589
4022
|
executionLockTtlMs?: number;
|
|
3590
|
-
execute: (context: {
|
|
4023
|
+
execute: (context: {
|
|
4024
|
+
leaseId: string | null;
|
|
4025
|
+
retainExternalCallSlot: (release: () => void) => void;
|
|
4026
|
+
}) => Promise<T>;
|
|
3591
4027
|
},
|
|
3592
4028
|
): Promise<T> {
|
|
3593
4029
|
const stalePolicy = resolveDurableCallCachePolicy(
|
|
@@ -3609,32 +4045,52 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3609
4045
|
staleAfterSeconds: stalePolicy.staleAfterSeconds,
|
|
3610
4046
|
cacheEpochMs: this.durableCallCacheEpochMs,
|
|
3611
4047
|
});
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
opts.
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
4048
|
+
let releaseExternalCallSlot: (() => void) | null = null;
|
|
4049
|
+
try {
|
|
4050
|
+
return await executeWithDurableRuntimeReceipt<T>({
|
|
4051
|
+
operation,
|
|
4052
|
+
id,
|
|
4053
|
+
runId: this.currentReceiptOwnerRunId,
|
|
4054
|
+
receiptKey,
|
|
4055
|
+
store: this.durableReceiptExecutionStore(),
|
|
4056
|
+
force: opts.force === true || stalePolicy.forceRefresh,
|
|
4057
|
+
repairRunningReceiptForSameRun: opts.repairRunningReceiptForSameRun,
|
|
4058
|
+
repairRunningReceiptForSameRunAfterWaitTimeout:
|
|
4059
|
+
opts.repairRunningReceiptForSameRunAfterWaitTimeout,
|
|
4060
|
+
runningReceiptWaitMaxAttempts: opts.runningReceiptWaitMaxAttempts,
|
|
4061
|
+
runningReceiptWaitDelayMs: opts.runningReceiptWaitDelayMs,
|
|
4062
|
+
reclaimRunning: opts.reclaimRunning,
|
|
4063
|
+
markSkipped: opts.markSkipped,
|
|
4064
|
+
onRecovered: opts.onRecovered,
|
|
4065
|
+
onClaimedResult: opts.onClaimedResult,
|
|
4066
|
+
shouldPersistFailure: opts.shouldPersistFailure,
|
|
4067
|
+
markRunningBeforeExecute: opts.markRunningBeforeExecute,
|
|
4068
|
+
completedCacheOnly: operation === 'tool',
|
|
4069
|
+
withCompletedReceiptHydration: (hydrate) =>
|
|
4070
|
+
this.withRuntimeReceiptHydrationTurn(hydrate),
|
|
4071
|
+
requiresExecutionLock: opts.requiresExecutionLock,
|
|
4072
|
+
executionLockTtlMs: opts.executionLockTtlMs,
|
|
4073
|
+
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
4074
|
+
formatError: (error) => this.formatRuntimeError(error),
|
|
4075
|
+
log: (message) => this.log(message),
|
|
4076
|
+
execute: ({ leaseId }) =>
|
|
4077
|
+
opts.execute({
|
|
4078
|
+
leaseId,
|
|
4079
|
+
retainExternalCallSlot: (release) => {
|
|
4080
|
+
if (releaseExternalCallSlot) {
|
|
4081
|
+
release();
|
|
4082
|
+
throw new Error(
|
|
4083
|
+
`ctx.${operation}(${id}) attempted to retain more than one external-call slot.`,
|
|
4084
|
+
);
|
|
4085
|
+
}
|
|
4086
|
+
releaseExternalCallSlot = release;
|
|
4087
|
+
},
|
|
4088
|
+
}),
|
|
4089
|
+
});
|
|
4090
|
+
} finally {
|
|
4091
|
+
const release = releaseExternalCallSlot as (() => void) | null;
|
|
4092
|
+
release?.();
|
|
4093
|
+
}
|
|
3638
4094
|
}
|
|
3639
4095
|
|
|
3640
4096
|
private get currentRunId(): string {
|
|
@@ -4669,32 +5125,39 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4669
5125
|
}),
|
|
4670
5126
|
requestInput: request.input,
|
|
4671
5127
|
});
|
|
4672
|
-
|
|
4673
|
-
|
|
4674
|
-
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
|
|
4679
|
-
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
4688
|
-
|
|
4689
|
-
|
|
5128
|
+
const releaseCompletionHold = this.holdRuntimeToolReceiptCompletions(
|
|
5129
|
+
receiptKey ? [receiptKey] : [],
|
|
5130
|
+
);
|
|
5131
|
+
try {
|
|
5132
|
+
let completed: RuntimeStepReceipt | null | undefined = null;
|
|
5133
|
+
if (receiptKey) {
|
|
5134
|
+
try {
|
|
5135
|
+
completed = (
|
|
5136
|
+
await this.completeRuntimeStepReceipts([
|
|
5137
|
+
{
|
|
5138
|
+
key: receiptKey,
|
|
5139
|
+
runId: this.currentReceiptOwnerRunId,
|
|
5140
|
+
leaseId: request.receiptLeaseId ?? null,
|
|
5141
|
+
output: serializeToolExecuteResult(wrapped),
|
|
5142
|
+
},
|
|
5143
|
+
])
|
|
5144
|
+
).get(receiptKey);
|
|
5145
|
+
} catch (receiptError) {
|
|
5146
|
+
// Completion failed AFTER a successful (billed) tool call: trip the
|
|
5147
|
+
// breaker so remaining rows stop dispatching new provider calls.
|
|
5148
|
+
tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
|
|
5149
|
+
throw receiptError;
|
|
5150
|
+
}
|
|
4690
5151
|
}
|
|
5152
|
+
return await this.finalizeResolvedToolCall(
|
|
5153
|
+
toolId,
|
|
5154
|
+
request,
|
|
5155
|
+
wrapped,
|
|
5156
|
+
completed,
|
|
5157
|
+
);
|
|
5158
|
+
} finally {
|
|
5159
|
+
releaseCompletionHold();
|
|
4691
5160
|
}
|
|
4692
|
-
return await this.finalizeResolvedToolCall(
|
|
4693
|
-
toolId,
|
|
4694
|
-
request,
|
|
4695
|
-
wrapped,
|
|
4696
|
-
completed,
|
|
4697
|
-
);
|
|
4698
5161
|
}
|
|
4699
5162
|
|
|
4700
5163
|
private async resolveToolCallBatchResults(
|
|
@@ -4739,37 +5202,70 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4739
5202
|
]
|
|
4740
5203
|
: [];
|
|
4741
5204
|
});
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
5205
|
+
const releaseCompletionHold = this.holdRuntimeToolReceiptCompletions(
|
|
5206
|
+
receiptInputs.map((receipt) => receipt.key),
|
|
5207
|
+
);
|
|
5208
|
+
try {
|
|
5209
|
+
let completedByKey = new Map<string, RuntimeStepReceipt>();
|
|
5210
|
+
if (receiptInputs.length > 0) {
|
|
5211
|
+
try {
|
|
5212
|
+
completedByKey =
|
|
5213
|
+
await this.completeRuntimeStepReceipts(receiptInputs);
|
|
5214
|
+
} catch (receiptError) {
|
|
5215
|
+
// Completion failed AFTER successful (billed) tool calls: trip the
|
|
5216
|
+
// breaker so remaining rows stop dispatching new provider calls.
|
|
5217
|
+
tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
|
|
5218
|
+
await Promise.all(
|
|
5219
|
+
wrappedEntries.map((entry) =>
|
|
5220
|
+
this.rejectToolCall(toolId, entry.request, receiptError, {
|
|
5221
|
+
persistReceiptFailure: false,
|
|
5222
|
+
}),
|
|
5223
|
+
),
|
|
5224
|
+
);
|
|
5225
|
+
throw receiptError;
|
|
5226
|
+
}
|
|
5227
|
+
}
|
|
5228
|
+
const results: unknown[] = [];
|
|
5229
|
+
for (const entry of wrappedEntries) {
|
|
5230
|
+
const receiptKey = entry.request.receiptKey?.trim() || null;
|
|
5231
|
+
results.push(
|
|
5232
|
+
await this.finalizeResolvedToolCall(
|
|
5233
|
+
toolId,
|
|
5234
|
+
entry.request,
|
|
5235
|
+
entry.wrapped,
|
|
5236
|
+
receiptKey ? completedByKey.get(receiptKey) : null,
|
|
4755
5237
|
),
|
|
4756
5238
|
);
|
|
4757
|
-
throw receiptError;
|
|
4758
5239
|
}
|
|
5240
|
+
return results;
|
|
5241
|
+
} finally {
|
|
5242
|
+
releaseCompletionHold();
|
|
4759
5243
|
}
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4765
|
-
|
|
4766
|
-
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
),
|
|
5244
|
+
}
|
|
5245
|
+
|
|
5246
|
+
private holdRuntimeToolReceiptCompletions(receiptKeys: string[]): () => void {
|
|
5247
|
+
const uniqueKeys = [
|
|
5248
|
+
...new Set(receiptKeys.map((key) => key.trim())),
|
|
5249
|
+
].filter(Boolean);
|
|
5250
|
+
for (const key of uniqueKeys) {
|
|
5251
|
+
this.runtimeToolReceiptCompletionsInFlight.set(
|
|
5252
|
+
key,
|
|
5253
|
+
(this.runtimeToolReceiptCompletionsInFlight.get(key) ?? 0) + 1,
|
|
4770
5254
|
);
|
|
4771
5255
|
}
|
|
4772
|
-
|
|
5256
|
+
let released = false;
|
|
5257
|
+
return () => {
|
|
5258
|
+
if (released) return;
|
|
5259
|
+
released = true;
|
|
5260
|
+
for (const key of uniqueKeys) {
|
|
5261
|
+
const count = this.runtimeToolReceiptCompletionsInFlight.get(key) ?? 0;
|
|
5262
|
+
if (count <= 1) {
|
|
5263
|
+
this.runtimeToolReceiptCompletionsInFlight.delete(key);
|
|
5264
|
+
} else {
|
|
5265
|
+
this.runtimeToolReceiptCompletionsInFlight.set(key, count - 1);
|
|
5266
|
+
}
|
|
5267
|
+
}
|
|
5268
|
+
};
|
|
4773
5269
|
}
|
|
4774
5270
|
|
|
4775
5271
|
private async finalizeResolvedToolCall(
|
|
@@ -5514,6 +6010,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
5514
6010
|
activeRowsMemoryBudgetBytes:
|
|
5515
6011
|
mapMemoryLimits.activeRowsBudgetBytes,
|
|
5516
6012
|
incrementalPersistence,
|
|
6013
|
+
releasePersistedRowsOnMemoryPressure: true,
|
|
5517
6014
|
},
|
|
5518
6015
|
);
|
|
5519
6016
|
} catch (error) {
|
|
@@ -5560,9 +6057,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
5560
6057
|
this.activeMapCellMeta = null;
|
|
5561
6058
|
this.activeMapCheckpointUpdates = null;
|
|
5562
6059
|
|
|
6060
|
+
for (const materializedRow of mapResult.previewRows) {
|
|
6061
|
+
if (previewRows.length < 5) previewRows.push(materializedRow);
|
|
6062
|
+
}
|
|
6063
|
+
if (!mapResult.retainedRowsComplete) {
|
|
6064
|
+
disableImmediateMaterializedCache(
|
|
6065
|
+
'the current execution page exceeded the in-memory result budget',
|
|
6066
|
+
);
|
|
6067
|
+
}
|
|
5563
6068
|
for (const row of mapResult.completedRows) {
|
|
5564
6069
|
const materializedRow = this.toMaterializedOutputRow(row.data);
|
|
5565
|
-
if (previewRows.length < 5) previewRows.push(materializedRow);
|
|
5566
6070
|
if (
|
|
5567
6071
|
immediateMaterializedCacheEnabled &&
|
|
5568
6072
|
immediateMaterializedRows.length < materializedResultRowLimit
|
|
@@ -5607,8 +6111,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
5607
6111
|
}
|
|
5608
6112
|
}
|
|
5609
6113
|
executedCount += rowsToExecute.length;
|
|
5610
|
-
successfulCount += mapResult.
|
|
5611
|
-
failedCount += mapResult.
|
|
6114
|
+
successfulCount += mapResult.completedRowCount;
|
|
6115
|
+
failedCount += mapResult.failedRowCount;
|
|
5612
6116
|
processedCount += rowsToExecute.length;
|
|
5613
6117
|
page.rows.length = 0;
|
|
5614
6118
|
}
|
|
@@ -6464,6 +6968,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6464
6968
|
progressFailedOffset?: number;
|
|
6465
6969
|
progressTotalRows?: number;
|
|
6466
6970
|
incrementalPersistence?: RuntimeMapRowPersistence | null;
|
|
6971
|
+
/**
|
|
6972
|
+
* Runtime Sheet-backed maps may discard their optional in-memory result
|
|
6973
|
+
* cache once it reaches the resident-byte budget. The durable sheet is
|
|
6974
|
+
* already authoritative; retaining the same completed payloads until a
|
|
6975
|
+
* source page ends must never turn a successful large map into an OOM.
|
|
6976
|
+
*/
|
|
6977
|
+
releasePersistedRowsOnMemoryPressure?: boolean;
|
|
6467
6978
|
},
|
|
6468
6979
|
): Promise<FieldMapRunResult> {
|
|
6469
6980
|
const fieldEntries = Object.entries(definition);
|
|
@@ -6508,12 +7019,41 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6508
7019
|
const failFastOnRowError = runtimeOptions?.onRowError === 'fail';
|
|
6509
7020
|
const completedRowsToPersist: PersistableMapRow[] = [];
|
|
6510
7021
|
const failedRowsToPersist: PersistableMapRow[] = [];
|
|
7022
|
+
const previewRows: Record<string, unknown>[] = [];
|
|
7023
|
+
let completedRowCount = 0;
|
|
7024
|
+
let failedRowCount = 0;
|
|
7025
|
+
let retainedRowsComplete = true;
|
|
6511
7026
|
const incrementalPersistence =
|
|
6512
7027
|
runtimeOptions?.incrementalPersistence ?? null;
|
|
7028
|
+
let mapStallObserver: RuntimeMapStallObserver | null = null;
|
|
6513
7029
|
const retainedRowsMemoryTracker = createRuntimeMapRetainedRowsTracker({
|
|
6514
7030
|
mapName: normalizedTableNamespace,
|
|
6515
7031
|
budgetBytes: runtimeOptions?.retainedRowsMemoryBudgetBytes,
|
|
6516
7032
|
});
|
|
7033
|
+
const retainPersistedRow = (
|
|
7034
|
+
row: PersistableMapRow,
|
|
7035
|
+
target: PersistableMapRow[],
|
|
7036
|
+
): void => {
|
|
7037
|
+
if (!retainedRowsComplete) return;
|
|
7038
|
+
try {
|
|
7039
|
+
retainedRowsMemoryTracker.track(row);
|
|
7040
|
+
target.push(row);
|
|
7041
|
+
} catch (error) {
|
|
7042
|
+
if (
|
|
7043
|
+
!runtimeOptions?.releasePersistedRowsOnMemoryPressure ||
|
|
7044
|
+
!incrementalPersistence ||
|
|
7045
|
+
!(error instanceof RuntimeMapMemoryLimitError)
|
|
7046
|
+
) {
|
|
7047
|
+
throw error;
|
|
7048
|
+
}
|
|
7049
|
+
completedRowsToPersist.length = 0;
|
|
7050
|
+
failedRowsToPersist.length = 0;
|
|
7051
|
+
retainedRowsComplete = false;
|
|
7052
|
+
this.log(
|
|
7053
|
+
`Dataset ${normalizedTableNamespace} released its in-memory result cache after ${completedRowCount + failedRowCount} row(s); using the durable Runtime Sheet.`,
|
|
7054
|
+
);
|
|
7055
|
+
}
|
|
7056
|
+
};
|
|
6517
7057
|
const enqueueIncrementalPersist = (
|
|
6518
7058
|
row: PersistableMapRow,
|
|
6519
7059
|
onCommitted: () => void,
|
|
@@ -6523,11 +7063,24 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6523
7063
|
return Promise.resolve();
|
|
6524
7064
|
}
|
|
6525
7065
|
const settlement = incrementalPersistence.persistRows([row]);
|
|
6526
|
-
|
|
6527
|
-
|
|
6528
|
-
|
|
6529
|
-
|
|
6530
|
-
|
|
7066
|
+
const stall = mapStallObserver?.terminalQueued();
|
|
7067
|
+
void settlement.committed
|
|
7068
|
+
.then(() => {
|
|
7069
|
+
stall?.committed();
|
|
7070
|
+
onCommitted();
|
|
7071
|
+
})
|
|
7072
|
+
.catch(() => {
|
|
7073
|
+
stall?.failed();
|
|
7074
|
+
// The terminal map barrier awaits the writer and surfaces the same
|
|
7075
|
+
// latched failure. This continuation only publishes durable progress.
|
|
7076
|
+
});
|
|
7077
|
+
return settlement.admitted.then(
|
|
7078
|
+
() => stall?.admitted(),
|
|
7079
|
+
(error) => {
|
|
7080
|
+
stall?.failed();
|
|
7081
|
+
throw error;
|
|
7082
|
+
},
|
|
7083
|
+
);
|
|
6531
7084
|
};
|
|
6532
7085
|
|
|
6533
7086
|
if (completedRows > 0 || pendingRows !== totalRows) {
|
|
@@ -6727,6 +7280,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6727
7280
|
}),
|
|
6728
7281
|
),
|
|
6729
7282
|
failedRows: [],
|
|
7283
|
+
completedRowCount: results.length,
|
|
7284
|
+
failedRowCount: 0,
|
|
7285
|
+
previewRows: results.slice(0, 5),
|
|
7286
|
+
retainedRowsComplete: true,
|
|
6730
7287
|
};
|
|
6731
7288
|
}
|
|
6732
7289
|
|
|
@@ -6771,6 +7328,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6771
7328
|
this.resourceGovernor.observe({
|
|
6772
7329
|
memoryBytes: rowAdmission.estimatedResidentBytes,
|
|
6773
7330
|
});
|
|
7331
|
+
mapStallObserver = new RuntimeMapStallObserver({
|
|
7332
|
+
getWriterDiagnostics: () => incrementalPersistence?.diagnostics() ?? null,
|
|
7333
|
+
log: (line) => this.log(line),
|
|
7334
|
+
mapName: normalizedTableNamespace,
|
|
7335
|
+
pageOffset:
|
|
7336
|
+
runtimeOptions?.executionRowIndexes?.[0] ??
|
|
7337
|
+
runtimeOptions?.progressCompletedOffset ??
|
|
7338
|
+
0,
|
|
7339
|
+
intervalMs: this.#options.runtimeMapStallLogIntervalMs,
|
|
7340
|
+
});
|
|
6774
7341
|
const settledResults = new Array<MapRowExecutionResult | undefined>(
|
|
6775
7342
|
items.length,
|
|
6776
7343
|
);
|
|
@@ -6803,6 +7370,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6803
7370
|
const globalRowSlot = await this.resourceGovernor.acquireRow({
|
|
6804
7371
|
estimatedBytes: runtimeMapJsonByteLength(baseRow),
|
|
6805
7372
|
});
|
|
7373
|
+
const resolverObservation = mapStallObserver?.resolverStarted();
|
|
7374
|
+
let resolverObservationActive = resolverObservation !== undefined;
|
|
7375
|
+
const finishResolver = () => {
|
|
7376
|
+
if (!resolverObservationActive) return;
|
|
7377
|
+
resolverObservationActive = false;
|
|
7378
|
+
resolverObservation?.finish();
|
|
7379
|
+
};
|
|
6806
7380
|
try {
|
|
6807
7381
|
for (const [fieldName, resolver] of fieldEntries) {
|
|
6808
7382
|
activeFieldName = fieldName;
|
|
@@ -6834,6 +7408,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6834
7408
|
tableNamespace: normalizedTableNamespace,
|
|
6835
7409
|
rowKey,
|
|
6836
7410
|
mapScope,
|
|
7411
|
+
mapStallObserver: mapStallObserver ?? undefined,
|
|
7412
|
+
mapStallResolverToken: resolverObservation?.token,
|
|
6837
7413
|
},
|
|
6838
7414
|
async () =>
|
|
6839
7415
|
await this.resolveMapFieldValue(
|
|
@@ -6860,7 +7436,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6860
7436
|
|
|
6861
7437
|
value = null;
|
|
6862
7438
|
computedFields[fieldName] = value;
|
|
6863
|
-
this.rowStates.get(idx)?.results.set(fieldName, value);
|
|
6864
7439
|
const formattedError = this.formatRuntimeError(error);
|
|
6865
7440
|
this.emitScopedFieldMetaUpdate({
|
|
6866
7441
|
rowId: idx,
|
|
@@ -6888,8 +7463,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6888
7463
|
: {}),
|
|
6889
7464
|
error: formattedError,
|
|
6890
7465
|
});
|
|
6891
|
-
|
|
6892
|
-
|
|
7466
|
+
// This row has not entered incremental persistence because the
|
|
7467
|
+
// fail-fast error is about to unwind the resolver. Retain this
|
|
7468
|
+
// one terminal fact even if the optional completed-row cache was
|
|
7469
|
+
// already released, so the outer failure barrier can persist it.
|
|
7470
|
+
if (retainedRowsComplete) {
|
|
7471
|
+
retainPersistedRow(failedRow, failedRowsToPersist);
|
|
7472
|
+
}
|
|
7473
|
+
if (!retainedRowsComplete) {
|
|
7474
|
+
failedRowsToPersist.push(failedRow);
|
|
7475
|
+
}
|
|
6893
7476
|
throw error;
|
|
6894
7477
|
}
|
|
6895
7478
|
const failedData = this.toPersistedOutputRow(
|
|
@@ -6903,8 +7486,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6903
7486
|
...(cellMetaPatch ? { cellMetaPatch } : {}),
|
|
6904
7487
|
error: formattedError,
|
|
6905
7488
|
});
|
|
6906
|
-
|
|
6907
|
-
|
|
7489
|
+
failedRowCount += 1;
|
|
7490
|
+
retainPersistedRow(failedRow, failedRowsToPersist);
|
|
6908
7491
|
this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
|
|
6909
7492
|
rowId: idx,
|
|
6910
7493
|
status: 'failed',
|
|
@@ -6913,6 +7496,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6913
7496
|
error: formattedError,
|
|
6914
7497
|
dataPatch: {},
|
|
6915
7498
|
});
|
|
7499
|
+
finishResolver();
|
|
6916
7500
|
await enqueueIncrementalPersist(failedRow, () => {
|
|
6917
7501
|
this.clearActiveMapCheckpointUpdate(
|
|
6918
7502
|
rowKey,
|
|
@@ -6923,7 +7507,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6923
7507
|
return FAILED_ROW;
|
|
6924
7508
|
}
|
|
6925
7509
|
computedFields[fieldName] = cellValue;
|
|
6926
|
-
this.rowStates.get(idx)?.results.set(fieldName, cellValue);
|
|
6927
7510
|
const currentCellMeta =
|
|
6928
7511
|
this.activeMapCellMeta?.get(rowKey)?.[fieldName];
|
|
6929
7512
|
const currentCellRecord =
|
|
@@ -6975,6 +7558,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6975
7558
|
});
|
|
6976
7559
|
}
|
|
6977
7560
|
|
|
7561
|
+
finishResolver();
|
|
6978
7562
|
const merged = cloneCsvAliasedRow(baseRow, computedFields);
|
|
6979
7563
|
activeFieldName = null;
|
|
6980
7564
|
this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
|
|
@@ -6985,7 +7569,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6985
7569
|
error: null,
|
|
6986
7570
|
dataPatch: {},
|
|
6987
7571
|
});
|
|
6988
|
-
const publicRow = this.toPublicOutputRow(merged);
|
|
6989
7572
|
const completedRow: PersistableMapRow = completedMapRowOutcome({
|
|
6990
7573
|
key: rowKey,
|
|
6991
7574
|
inputIndex: rowIndex,
|
|
@@ -6994,13 +7577,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6994
7577
|
? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
|
|
6995
7578
|
: {}),
|
|
6996
7579
|
});
|
|
6997
|
-
|
|
6998
|
-
|
|
7580
|
+
completedRowCount += 1;
|
|
7581
|
+
if (previewRows.length < 5) {
|
|
7582
|
+
previewRows.push(this.toMaterializedOutputRow(completedRow.data));
|
|
7583
|
+
}
|
|
7584
|
+
retainPersistedRow(completedRow, completedRowsToPersist);
|
|
6999
7585
|
await enqueueIncrementalPersist(completedRow, () => {
|
|
7000
7586
|
this.clearActiveMapCheckpointUpdate(rowKey, normalizedTableNamespace);
|
|
7001
7587
|
updateMapFrameProgress({ completedRowKey: rowKey });
|
|
7002
7588
|
});
|
|
7003
|
-
return
|
|
7589
|
+
return COMPLETED_ROW;
|
|
7004
7590
|
} catch (error) {
|
|
7005
7591
|
if (isPlayRowExecutionSuspendedError(error)) {
|
|
7006
7592
|
this.pendingRowEventBoundaries.push(error.boundary);
|
|
@@ -7043,6 +7629,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7043
7629
|
});
|
|
7044
7630
|
throw error;
|
|
7045
7631
|
} finally {
|
|
7632
|
+
finishResolver();
|
|
7046
7633
|
globalRowSlot.release();
|
|
7047
7634
|
}
|
|
7048
7635
|
};
|
|
@@ -7104,16 +7691,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7104
7691
|
const rawItem = this.toOutputRow(
|
|
7105
7692
|
items[index] as Record<string, unknown>,
|
|
7106
7693
|
);
|
|
7694
|
+
const key = executionRowKey(rawItem, index);
|
|
7107
7695
|
if (result === WAITING_ROW) {
|
|
7108
|
-
const key = executionRowKey(rawItem, index);
|
|
7109
7696
|
if (key) {
|
|
7110
7697
|
completedRowKeys.add(key);
|
|
7111
7698
|
}
|
|
7112
7699
|
accountedRowIndexes.add(executionRowIndex(index));
|
|
7113
7700
|
continue;
|
|
7114
7701
|
}
|
|
7115
|
-
const row = result as Record<string, unknown>;
|
|
7116
|
-
const key = executionRowKey(row, index);
|
|
7117
7702
|
if (key) completedRowKeys.add(key);
|
|
7118
7703
|
}
|
|
7119
7704
|
this.setMapFrame({
|
|
@@ -7163,6 +7748,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7163
7748
|
boundaries: uniqueBoundaries,
|
|
7164
7749
|
});
|
|
7165
7750
|
}
|
|
7751
|
+
for (let index = 0; index < rowResults.length; index += 1) {
|
|
7752
|
+
if (rowResults[index] !== WAITING_ROW) {
|
|
7753
|
+
accountedRowIndexes.add(executionRowIndex(index));
|
|
7754
|
+
}
|
|
7755
|
+
}
|
|
7166
7756
|
if (accountedRowIndexes.size !== items.length) {
|
|
7167
7757
|
const expectedRowIndexes = items.map((_item, index) =>
|
|
7168
7758
|
executionRowIndex(index),
|
|
@@ -7176,10 +7766,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7176
7766
|
`missing input index(es): ${missingRowIndexes.slice(0, 10).join(', ') || 'unknown'}.`,
|
|
7177
7767
|
);
|
|
7178
7768
|
}
|
|
7179
|
-
const
|
|
7180
|
-
(result)
|
|
7181
|
-
|
|
7182
|
-
);
|
|
7769
|
+
const succeededRows = rowResults.filter(
|
|
7770
|
+
(result) => result === COMPLETED_ROW,
|
|
7771
|
+
).length;
|
|
7183
7772
|
// A row is acknowledged only after its terminal sheet batch commits. The
|
|
7184
7773
|
// incremental promises publish their frame updates from the commit
|
|
7185
7774
|
// continuation, so this barrier also makes the terminal map event derive
|
|
@@ -7203,13 +7792,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7203
7792
|
);
|
|
7204
7793
|
}
|
|
7205
7794
|
this.log(
|
|
7206
|
-
`Map completed: ${
|
|
7795
|
+
`Map completed: ${succeededRows + completedRows} results (${succeededRows} succeeded, ${failedRowCount} failed, ${completedRows} duplicate keys skipped)`,
|
|
7207
7796
|
);
|
|
7208
7797
|
return {
|
|
7209
7798
|
completedRows: [...completedRowsToPersist].sort(
|
|
7210
7799
|
comparePersistableMapRowsByInputIndex,
|
|
7211
7800
|
),
|
|
7212
7801
|
failedRows: failedRowsToPersist,
|
|
7802
|
+
completedRowCount,
|
|
7803
|
+
failedRowCount,
|
|
7804
|
+
previewRows,
|
|
7805
|
+
retainedRowsComplete,
|
|
7213
7806
|
};
|
|
7214
7807
|
}
|
|
7215
7808
|
|
|
@@ -7719,6 +8312,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7719
8312
|
}
|
|
7720
8313
|
}
|
|
7721
8314
|
|
|
8315
|
+
private async rejectUnresolvedToolCalls(
|
|
8316
|
+
requests: readonly ToolCallRequest[],
|
|
8317
|
+
error: unknown,
|
|
8318
|
+
): Promise<void> {
|
|
8319
|
+
await Promise.all(
|
|
8320
|
+
requests.map(async (request) => {
|
|
8321
|
+
if (!this.toolCallResolvers.has(request.callId)) return;
|
|
8322
|
+
await this.rejectToolCall(request.toolId, request, error, {
|
|
8323
|
+
persistReceiptFailure: false,
|
|
8324
|
+
});
|
|
8325
|
+
}),
|
|
8326
|
+
);
|
|
8327
|
+
}
|
|
8328
|
+
|
|
7722
8329
|
private takeDispatchableToolCalls(
|
|
7723
8330
|
nowMs: number,
|
|
7724
8331
|
blockedLaneKeys: ReadonlySet<string>,
|
|
@@ -7886,9 +8493,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7886
8493
|
);
|
|
7887
8494
|
}
|
|
7888
8495
|
const tracked = this.executeBatchedToolCalls(dispatchable.requests)
|
|
7889
|
-
.catch((error) => {
|
|
8496
|
+
.catch(async (error) => {
|
|
7890
8497
|
this.toolDispatcherFailure ??= error;
|
|
7891
8498
|
this.rejectQueuedToolCalls(error);
|
|
8499
|
+
await this.rejectUnresolvedToolCalls(
|
|
8500
|
+
dispatchable.requests,
|
|
8501
|
+
error,
|
|
8502
|
+
);
|
|
7892
8503
|
})
|
|
7893
8504
|
.finally(() => {
|
|
7894
8505
|
for (const laneKey of dispatchable.laneKeys) {
|
|
@@ -8059,6 +8670,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8059
8670
|
|
|
8060
8671
|
const executeTool = async (context?: {
|
|
8061
8672
|
leaseId?: string | null;
|
|
8673
|
+
retainExternalCallSlot?: (release: () => void) => void;
|
|
8062
8674
|
}): Promise<unknown> => {
|
|
8063
8675
|
if (eventWaitHandler) {
|
|
8064
8676
|
return this.executeIntegrationEventWaitTool(
|
|
@@ -8190,6 +8802,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8190
8802
|
options?.timeoutMs,
|
|
8191
8803
|
this.currentAuthoringContractEdition,
|
|
8192
8804
|
),
|
|
8805
|
+
retainToolSlot: context?.retainExternalCallSlot,
|
|
8193
8806
|
...(directReceiptLeaseId &&
|
|
8194
8807
|
(this.#options.heartbeatRuntimeStepReceipts ||
|
|
8195
8808
|
this.#options.getRuntimeStepReceipt ||
|
|
@@ -8406,7 +9019,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8406
9019
|
shouldPersistFailure: (error) =>
|
|
8407
9020
|
!(error instanceof ToolExecuteAuthScopeChangedError),
|
|
8408
9021
|
markRunningBeforeExecute: false,
|
|
8409
|
-
execute: ({ leaseId }) =>
|
|
9022
|
+
execute: ({ leaseId, retainExternalCallSlot }) =>
|
|
9023
|
+
executeTool({ leaseId, retainExternalCallSlot }),
|
|
8410
9024
|
runningReceiptWaitMaxAttempts:
|
|
8411
9025
|
resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
8412
9026
|
options?.receiptWaitMs !== undefined
|
|
@@ -8943,7 +9557,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8943
9557
|
assertSecretAuthUsesTls(init.auth, input, 'ctx.fetch');
|
|
8944
9558
|
const secretHeaderMarkers = secretAuthHeaderMarkers(init.auth);
|
|
8945
9559
|
|
|
8946
|
-
|
|
9560
|
+
const execution = this.executeWithRuntimeReceipt<PlayFetchResponse>(
|
|
8947
9561
|
'fetch',
|
|
8948
9562
|
normalizedKey,
|
|
8949
9563
|
this.currentRunId,
|
|
@@ -8961,7 +9575,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8961
9575
|
}),
|
|
8962
9576
|
),
|
|
8963
9577
|
staleAfterSeconds: options?.staleAfterSeconds,
|
|
8964
|
-
execute: async () => {
|
|
9578
|
+
execute: async ({ retainExternalCallSlot }) => {
|
|
8965
9579
|
const url = input.toString();
|
|
8966
9580
|
const method = (init.method ?? 'GET').toUpperCase();
|
|
8967
9581
|
const secretHeaders = await this.resolveSecretAuth(init.auth);
|
|
@@ -8989,6 +9603,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8989
9603
|
const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
|
|
8990
9604
|
if (existing?.kind === 'fetch' && 'output' in existing) {
|
|
8991
9605
|
this.log(`ctx.fetch(${url}): recovered response from checkpoint`);
|
|
9606
|
+
if (this.durableDirectToolResultsBackedByReceipts) {
|
|
9607
|
+
// The outer durable receipt is the replay authority in hosted
|
|
9608
|
+
// runtimes. A legacy checkpoint fetch may seed that receipt
|
|
9609
|
+
// once, but it must not remain as a second full response-body
|
|
9610
|
+
// cache for the lifetime of a large map.
|
|
9611
|
+
delete this.checkpoint.resolvedBoundaries?.[boundaryId];
|
|
9612
|
+
}
|
|
8992
9613
|
return existing.output as PlayFetchResponse;
|
|
8993
9614
|
}
|
|
8994
9615
|
|
|
@@ -9011,7 +9632,28 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9011
9632
|
providerResourceKey: CTX_FETCH_EGRESS_TOOL_ID,
|
|
9012
9633
|
toolId: CTX_FETCH_EGRESS_TOOL_ID,
|
|
9013
9634
|
});
|
|
9635
|
+
retainExternalCallSlot(() => egressSlot.release());
|
|
9636
|
+
const finishFetchObservation =
|
|
9637
|
+
rowStore?.mapStallObserver?.fetchStarted(
|
|
9638
|
+
rowStore.mapStallResolverToken,
|
|
9639
|
+
);
|
|
9640
|
+
const fetchDeadline: CtxFetchDeadline = {
|
|
9641
|
+
startedAt: Date.now(),
|
|
9642
|
+
headersMs: positiveTimeoutMs(
|
|
9643
|
+
this.#options.ctxFetchTimeouts?.headersMs,
|
|
9644
|
+
CTX_FETCH_HEADERS_TIMEOUT_MS,
|
|
9645
|
+
),
|
|
9646
|
+
bodyMs: positiveTimeoutMs(
|
|
9647
|
+
this.#options.ctxFetchTimeouts?.bodyMs,
|
|
9648
|
+
CTX_FETCH_BODY_TIMEOUT_MS,
|
|
9649
|
+
),
|
|
9650
|
+
totalMs: positiveTimeoutMs(
|
|
9651
|
+
this.#options.ctxFetchTimeouts?.totalMs,
|
|
9652
|
+
CTX_FETCH_TOTAL_TIMEOUT_MS,
|
|
9653
|
+
),
|
|
9654
|
+
};
|
|
9014
9655
|
let response: Response | null = null;
|
|
9656
|
+
let bodyText: string | null = null;
|
|
9015
9657
|
try {
|
|
9016
9658
|
const canRetryTransport = ['GET', 'HEAD', 'OPTIONS'].includes(
|
|
9017
9659
|
method,
|
|
@@ -9022,13 +9664,40 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9022
9664
|
attempt <= FETCH_TRANSPORT_MAX_ATTEMPTS;
|
|
9023
9665
|
attempt += 1
|
|
9024
9666
|
) {
|
|
9667
|
+
const attemptController = new AbortController();
|
|
9025
9668
|
try {
|
|
9026
|
-
|
|
9027
|
-
|
|
9028
|
-
|
|
9669
|
+
const attemptInit = {
|
|
9670
|
+
...fetchInit,
|
|
9671
|
+
signal: init.signal
|
|
9672
|
+
? AbortSignal.any([init.signal, attemptController.signal])
|
|
9673
|
+
: attemptController.signal,
|
|
9674
|
+
};
|
|
9675
|
+
response = await runCtxFetchPhase({
|
|
9676
|
+
callerSignal: init.signal,
|
|
9677
|
+
controller: attemptController,
|
|
9678
|
+
deadline: fetchDeadline,
|
|
9679
|
+
phase: 'headers',
|
|
9680
|
+
run: () =>
|
|
9681
|
+
safePublicFetch(url, attemptInit, {
|
|
9682
|
+
fetchImpl: this.#options.fetchImpl,
|
|
9683
|
+
sensitiveHeaders: Object.keys(secretHeaderMarkers),
|
|
9684
|
+
}),
|
|
9685
|
+
});
|
|
9686
|
+
bodyText = await readCtxFetchBody({
|
|
9687
|
+
callerSignal: init.signal,
|
|
9688
|
+
controller: attemptController,
|
|
9689
|
+
deadline: fetchDeadline,
|
|
9690
|
+
response,
|
|
9029
9691
|
});
|
|
9030
9692
|
break;
|
|
9031
9693
|
} catch (error) {
|
|
9694
|
+
void response?.body?.cancel(error).catch(() => undefined);
|
|
9695
|
+
response = null;
|
|
9696
|
+
bodyText = null;
|
|
9697
|
+
if (error instanceof CtxFetchTimeoutError) throw error;
|
|
9698
|
+
if (init.signal?.aborted) {
|
|
9699
|
+
throw init.signal.reason ?? error;
|
|
9700
|
+
}
|
|
9032
9701
|
if (isUnsafeOutboundUrlError(error)) {
|
|
9033
9702
|
throw error;
|
|
9034
9703
|
}
|
|
@@ -9041,12 +9710,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9041
9710
|
this.log(
|
|
9042
9711
|
`ctx.fetch(${method} ${url}) transport failed on attempt ${attempt}/${FETCH_TRANSPORT_MAX_ATTEMPTS}; retrying: ${message}`,
|
|
9043
9712
|
);
|
|
9044
|
-
await
|
|
9045
|
-
|
|
9046
|
-
|
|
9047
|
-
|
|
9048
|
-
|
|
9049
|
-
);
|
|
9713
|
+
await sleepWithinCtxFetchDeadline({
|
|
9714
|
+
callerSignal: init.signal,
|
|
9715
|
+
deadline: fetchDeadline,
|
|
9716
|
+
delayMs: FETCH_TRANSPORT_RETRY_DELAY_MS * attempt,
|
|
9717
|
+
});
|
|
9050
9718
|
continue;
|
|
9051
9719
|
}
|
|
9052
9720
|
throw new Error(
|
|
@@ -9059,7 +9727,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9059
9727
|
`ctx.fetch(${method} ${url}) failed before receiving a response.`,
|
|
9060
9728
|
);
|
|
9061
9729
|
}
|
|
9062
|
-
|
|
9730
|
+
if (bodyText === null) {
|
|
9731
|
+
throw new Error(
|
|
9732
|
+
`ctx.fetch(${method} ${url}) failed while reading the response body.`,
|
|
9733
|
+
);
|
|
9734
|
+
}
|
|
9063
9735
|
const redactedBodyText = this.secretRedactor.redactString(bodyText);
|
|
9064
9736
|
const output: PlayFetchResponse = {
|
|
9065
9737
|
ok: response.ok,
|
|
@@ -9075,24 +9747,27 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9075
9747
|
),
|
|
9076
9748
|
};
|
|
9077
9749
|
|
|
9078
|
-
this.
|
|
9079
|
-
|
|
9080
|
-
|
|
9081
|
-
|
|
9082
|
-
|
|
9083
|
-
|
|
9084
|
-
|
|
9085
|
-
|
|
9086
|
-
|
|
9087
|
-
|
|
9088
|
-
|
|
9089
|
-
|
|
9090
|
-
|
|
9091
|
-
|
|
9092
|
-
}
|
|
9093
|
-
|
|
9750
|
+
if (!this.durableDirectToolResultsBackedByReceipts) {
|
|
9751
|
+
this.checkpoint.resolvedBoundaries = {
|
|
9752
|
+
...(this.checkpoint.resolvedBoundaries ?? {}),
|
|
9753
|
+
[boundaryId]: {
|
|
9754
|
+
kind: 'fetch',
|
|
9755
|
+
url,
|
|
9756
|
+
method,
|
|
9757
|
+
output,
|
|
9758
|
+
completedAt: Date.now(),
|
|
9759
|
+
},
|
|
9760
|
+
};
|
|
9761
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
9762
|
+
}
|
|
9763
|
+
return output;
|
|
9764
|
+
} finally {
|
|
9765
|
+
finishFetchObservation?.();
|
|
9766
|
+
}
|
|
9767
|
+
},
|
|
9094
9768
|
},
|
|
9095
9769
|
);
|
|
9770
|
+
return execution;
|
|
9096
9771
|
}
|
|
9097
9772
|
|
|
9098
9773
|
async step<T>(
|
|
@@ -9213,9 +9888,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9213
9888
|
|
|
9214
9889
|
// ——— Batched tool call execution ———
|
|
9215
9890
|
|
|
9216
|
-
private startPendingToolReceiptHeartbeat(
|
|
9891
|
+
private startPendingToolReceiptHeartbeat(
|
|
9892
|
+
requests: ToolCallRequest[],
|
|
9893
|
+
claimsEstablishedExecutionFence: boolean,
|
|
9894
|
+
): {
|
|
9217
9895
|
stop: () => void;
|
|
9218
9896
|
leaseLost: () => unknown | null;
|
|
9897
|
+
confirmOwned: () => Promise<void>;
|
|
9898
|
+
assertOwned: () => void;
|
|
9219
9899
|
} | null {
|
|
9220
9900
|
const heartbeatReceipts = this.#options.heartbeatRuntimeStepReceipts;
|
|
9221
9901
|
const ownedRequests = requests.filter(
|
|
@@ -9235,66 +9915,87 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9235
9915
|
null,
|
|
9236
9916
|
);
|
|
9237
9917
|
let leaseLost: unknown | null = null;
|
|
9238
|
-
|
|
9239
|
-
|
|
9240
|
-
|
|
9241
|
-
|
|
9242
|
-
|
|
9243
|
-
|
|
9244
|
-
|
|
9245
|
-
|
|
9246
|
-
|
|
9247
|
-
|
|
9248
|
-
|
|
9249
|
-
|
|
9250
|
-
)
|
|
9251
|
-
|
|
9252
|
-
|
|
9253
|
-
|
|
9254
|
-
|
|
9255
|
-
|
|
9256
|
-
|
|
9257
|
-
|
|
9918
|
+
let ownershipConfirmedUntilMs = Number.isFinite(
|
|
9919
|
+
Date.parse(earliestLeaseExpiry ?? ''),
|
|
9920
|
+
)
|
|
9921
|
+
? Date.parse(earliestLeaseExpiry!)
|
|
9922
|
+
: Date.now() + PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS;
|
|
9923
|
+
const firstOwnedRequest = ownedRequests[0]!;
|
|
9924
|
+
const heartbeatOwnedRequests = async () => {
|
|
9925
|
+
// Receipt claims are acquired in bulk before the provider-shaped
|
|
9926
|
+
// dispatcher admits every call. Renew the undispatched tail as well as
|
|
9927
|
+
// in-flight calls; otherwise a slow provider can let later claims
|
|
9928
|
+
// expire while they are still waiting behind the concurrency gate.
|
|
9929
|
+
const unsettled = ownedRequests.filter((request) =>
|
|
9930
|
+
this.toolCallResolvers.has(request.callId),
|
|
9931
|
+
);
|
|
9932
|
+
if (unsettled.length === 0) return 'terminal' as const;
|
|
9933
|
+
const byReceiptKey = new Map<string, ToolCallRequest[]>();
|
|
9934
|
+
for (const request of unsettled) {
|
|
9935
|
+
const receiptKey = request.receiptKey!;
|
|
9936
|
+
const group = byReceiptKey.get(receiptKey) ?? [];
|
|
9937
|
+
group.push(request);
|
|
9938
|
+
byReceiptKey.set(receiptKey, group);
|
|
9939
|
+
}
|
|
9940
|
+
// One content-addressed receipt can have many same-run followers. The
|
|
9941
|
+
// store renews unique receipt rows, not request positions, so dedupe by
|
|
9942
|
+
// key while preserving the matching per-row lease id.
|
|
9943
|
+
const entries = [...byReceiptKey];
|
|
9944
|
+
const keys = entries.map(([receiptKey]) => receiptKey);
|
|
9945
|
+
const leaseIds = entries.map(
|
|
9946
|
+
([, requests]) => requests[0]!.receiptLeaseId!,
|
|
9947
|
+
);
|
|
9948
|
+
const receipts = await heartbeatReceipts({
|
|
9949
|
+
runId: this.currentReceiptOwnerRunId,
|
|
9950
|
+
runAttempt: this.currentRunAttempt,
|
|
9951
|
+
keys,
|
|
9952
|
+
leaseIds,
|
|
9953
|
+
});
|
|
9954
|
+
const renewedExpiries: number[] = [];
|
|
9955
|
+
for (let index = 0; index < entries.length; index += 1) {
|
|
9956
|
+
const [receiptKey, requests] = entries[index]!;
|
|
9957
|
+
const leaseId = leaseIds[index]!;
|
|
9958
|
+
// Completion can race this bulk heartbeat response. A settled
|
|
9959
|
+
// receipt group no longer needs ownership and must not turn that
|
|
9960
|
+
// race into a false lease-loss failure or hold the confirmation
|
|
9961
|
+
// horizon at its old expiry.
|
|
9962
|
+
if (
|
|
9963
|
+
this.runtimeToolReceiptCompletionsInFlight.has(receiptKey) ||
|
|
9964
|
+
!requests.some((request) =>
|
|
9965
|
+
this.toolCallResolvers.has(request.callId),
|
|
9966
|
+
)
|
|
9967
|
+
) {
|
|
9968
|
+
continue;
|
|
9258
9969
|
}
|
|
9259
|
-
|
|
9260
|
-
|
|
9261
|
-
|
|
9262
|
-
|
|
9263
|
-
|
|
9264
|
-
|
|
9265
|
-
|
|
9266
|
-
|
|
9267
|
-
|
|
9268
|
-
|
|
9269
|
-
|
|
9270
|
-
keys,
|
|
9271
|
-
leaseIds,
|
|
9272
|
-
});
|
|
9273
|
-
for (let index = 0; index < entries.length; index += 1) {
|
|
9274
|
-
const [receiptKey, requests] = entries[index]!;
|
|
9275
|
-
const leaseId = leaseIds[index]!;
|
|
9276
|
-
// Completion can race this bulk heartbeat response. A settled
|
|
9277
|
-
// receipt group no longer needs ownership and must not turn that
|
|
9278
|
-
// race into a false lease-loss failure.
|
|
9279
|
-
if (
|
|
9280
|
-
!requests.some((request) =>
|
|
9281
|
-
this.toolCallResolvers.has(request.callId),
|
|
9282
|
-
)
|
|
9283
|
-
) {
|
|
9284
|
-
continue;
|
|
9285
|
-
}
|
|
9286
|
-
if (
|
|
9287
|
-
!this.runtimeToolReceiptStillOwned(receipts[index] ?? null, leaseId)
|
|
9288
|
-
) {
|
|
9289
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
9290
|
-
receiptKey,
|
|
9291
|
-
runId: this.currentReceiptOwnerRunId,
|
|
9292
|
-
leaseId,
|
|
9293
|
-
});
|
|
9294
|
-
}
|
|
9970
|
+
const receipt = receipts[index] ?? null;
|
|
9971
|
+
const renewedExpiryMs = Date.parse(receipt?.leaseExpiresAt ?? '');
|
|
9972
|
+
if (
|
|
9973
|
+
!this.runtimeToolReceiptStillOwned(receipt, leaseId) ||
|
|
9974
|
+
!Number.isFinite(renewedExpiryMs)
|
|
9975
|
+
) {
|
|
9976
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
9977
|
+
receiptKey,
|
|
9978
|
+
runId: this.currentReceiptOwnerRunId,
|
|
9979
|
+
leaseId,
|
|
9980
|
+
});
|
|
9295
9981
|
}
|
|
9296
|
-
|
|
9297
|
-
}
|
|
9982
|
+
renewedExpiries.push(renewedExpiryMs);
|
|
9983
|
+
}
|
|
9984
|
+
if (renewedExpiries.length === 0) return 'terminal' as const;
|
|
9985
|
+
ownershipConfirmedUntilMs = Math.min(...renewedExpiries);
|
|
9986
|
+
return 'active' as const;
|
|
9987
|
+
};
|
|
9988
|
+
const perCallHeartbeatIntervalMs = runtimeLeaseHeartbeatIntervalFromExpiry({
|
|
9989
|
+
leaseExpiresAt: earliestLeaseExpiry,
|
|
9990
|
+
fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
9991
|
+
});
|
|
9992
|
+
const supervisor = createRuntimeReceiptHeartbeatSupervisor({
|
|
9993
|
+
// The cohort owns claims that have not reached provider admission yet.
|
|
9994
|
+
// Renew it before the one-call supervisors can win the same short-lease
|
|
9995
|
+
// timer race, so the queued tail remains fenced while the active call
|
|
9996
|
+
// consumes the provider concurrency slot.
|
|
9997
|
+
intervalMs: Math.max(1, Math.floor(perCallHeartbeatIntervalMs / 2)),
|
|
9998
|
+
heartbeat: heartbeatOwnedRequests,
|
|
9298
9999
|
isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
|
|
9299
10000
|
onLeaseLost: (error) => {
|
|
9300
10001
|
leaseLost ??= error;
|
|
@@ -9305,10 +10006,32 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9305
10006
|
);
|
|
9306
10007
|
},
|
|
9307
10008
|
});
|
|
9308
|
-
supervisor.start();
|
|
9309
10009
|
return {
|
|
9310
10010
|
stop: () => supervisor.stop(),
|
|
9311
10011
|
leaseLost: () => leaseLost,
|
|
10012
|
+
confirmOwned: async () => {
|
|
10013
|
+
// The scheduler receipt gateway returns `running` from an atomic
|
|
10014
|
+
// claim. That claim is already the provider-execution fence, so short
|
|
10015
|
+
// calls only need scheduled renewal if they approach lease expiry.
|
|
10016
|
+
// Other callers may still return queued/pending claims or omit the
|
|
10017
|
+
// explicit backend capability; retain the immediate ownership check
|
|
10018
|
+
// for those receipts.
|
|
10019
|
+
if (claimsEstablishedExecutionFence) {
|
|
10020
|
+
supervisor.start();
|
|
10021
|
+
return;
|
|
10022
|
+
}
|
|
10023
|
+
const outcome = await heartbeatOwnedRequests();
|
|
10024
|
+
if (outcome === 'active') supervisor.start();
|
|
10025
|
+
},
|
|
10026
|
+
assertOwned: () => {
|
|
10027
|
+
if (leaseLost) throw leaseLost;
|
|
10028
|
+
if (Date.now() < ownershipConfirmedUntilMs) return;
|
|
10029
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
10030
|
+
receiptKey: firstOwnedRequest.receiptKey!,
|
|
10031
|
+
runId: this.currentReceiptOwnerRunId,
|
|
10032
|
+
leaseId: firstOwnedRequest.receiptLeaseId!,
|
|
10033
|
+
});
|
|
10034
|
+
},
|
|
9312
10035
|
};
|
|
9313
10036
|
}
|
|
9314
10037
|
|
|
@@ -9368,6 +10091,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9368
10091
|
};
|
|
9369
10092
|
|
|
9370
10093
|
const pendingRequests: ToolCallRequest[] = [];
|
|
10094
|
+
let pendingReceiptClaimsEstablishExecutionFence =
|
|
10095
|
+
this.#options.runtimeReceiptClaimsEstablishExecutionFence === true;
|
|
9371
10096
|
const recoveredRequests: ToolCallRequest[] = [];
|
|
9372
10097
|
for (const req of requests) {
|
|
9373
10098
|
const cached =
|
|
@@ -9495,504 +10220,611 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9495
10220
|
forceAfterWait: boolean;
|
|
9496
10221
|
waitMaxAttempts?: number;
|
|
9497
10222
|
}> = [];
|
|
9498
|
-
|
|
10223
|
+
const durableExistingRunningState: {
|
|
10224
|
+
settlement: Promise<
|
|
10225
|
+
{ status: 'fulfilled' } | { status: 'rejected'; reason: unknown }
|
|
10226
|
+
> | null;
|
|
10227
|
+
} = { settlement: null };
|
|
10228
|
+
const durableExistingRunningAbort = new AbortController();
|
|
9499
10229
|
if (
|
|
9500
10230
|
pendingRequests.length > 0 &&
|
|
9501
10231
|
(this.#options.claimRuntimeStepReceipts ||
|
|
9502
10232
|
this.#options.claimRuntimeStepReceipt)
|
|
9503
10233
|
) {
|
|
9504
|
-
|
|
9505
|
-
|
|
9506
|
-
|
|
9507
|
-
|
|
9508
|
-
|
|
9509
|
-
|
|
9510
|
-
|
|
9511
|
-
|
|
9512
|
-
|
|
9513
|
-
|
|
9514
|
-
const
|
|
9515
|
-
|
|
9516
|
-
|
|
9517
|
-
|
|
10234
|
+
await this.withRuntimeReceiptHydrationTurn(async () => {
|
|
10235
|
+
const requestsByReceiptKey = new Map<
|
|
10236
|
+
string,
|
|
10237
|
+
{
|
|
10238
|
+
requests: ToolCallRequest[];
|
|
10239
|
+
forceRefresh: boolean;
|
|
10240
|
+
forceFailedRefresh: boolean;
|
|
10241
|
+
}
|
|
10242
|
+
>();
|
|
10243
|
+
const localOnlyRequests: ToolCallRequest[] = [];
|
|
10244
|
+
for (const request of pendingRequests) {
|
|
10245
|
+
const receiptKey = request.receiptKey?.trim() || null;
|
|
10246
|
+
if (!receiptKey) {
|
|
10247
|
+
localOnlyRequests.push(request);
|
|
10248
|
+
continue;
|
|
10249
|
+
}
|
|
10250
|
+
const group = requestsByReceiptKey.get(receiptKey) ?? {
|
|
10251
|
+
requests: [],
|
|
10252
|
+
forceRefresh: false,
|
|
10253
|
+
forceFailedRefresh: false,
|
|
10254
|
+
};
|
|
10255
|
+
group.requests.push(request);
|
|
10256
|
+
group.forceRefresh ||= request.force === true;
|
|
10257
|
+
group.forceFailedRefresh ||= request.forceFailedRefresh === true;
|
|
10258
|
+
requestsByReceiptKey.set(receiptKey, group);
|
|
9518
10259
|
}
|
|
9519
|
-
const
|
|
9520
|
-
|
|
9521
|
-
|
|
9522
|
-
|
|
9523
|
-
|
|
9524
|
-
|
|
9525
|
-
|
|
9526
|
-
|
|
9527
|
-
requestsByReceiptKey
|
|
9528
|
-
|
|
9529
|
-
|
|
9530
|
-
|
|
9531
|
-
([
|
|
9532
|
-
|
|
9533
|
-
|
|
9534
|
-
const forcedReceiptKeys = [...requestsByReceiptKey]
|
|
9535
|
-
.filter(([, group]) => group.forceRefresh)
|
|
9536
|
-
.map(([receiptKey]) => receiptKey);
|
|
9537
|
-
const failedRefreshReceiptKeys = [...requestsByReceiptKey]
|
|
9538
|
-
.filter(
|
|
9539
|
-
([, group]) => !group.forceRefresh && group.forceFailedRefresh,
|
|
9540
|
-
)
|
|
9541
|
-
.map(([receiptKey]) => receiptKey);
|
|
9542
|
-
const forcedExisting = forcedReceiptKeys.length
|
|
9543
|
-
? await this.getRuntimeStepReceipts(forcedReceiptKeys)
|
|
9544
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
9545
|
-
// A forced refresh must never duplicate a provider call owned by a
|
|
9546
|
-
// live lease. It can, however, immediately claim pending/unleased
|
|
9547
|
-
// work and any expired lease. Waiting first for those receipts turns
|
|
9548
|
-
// an already-recoverable interruption into a five-minute stall.
|
|
9549
|
-
const forcedLiveReceiptKeys = new Set(
|
|
9550
|
-
[...forcedExisting]
|
|
9551
|
-
.filter(([, receipt]) => {
|
|
9552
|
-
if (
|
|
9553
|
-
receipt.status !== 'queued' &&
|
|
9554
|
-
receipt.status !== 'pending' &&
|
|
9555
|
-
receipt.status !== 'running'
|
|
9556
|
-
) {
|
|
9557
|
-
return false;
|
|
9558
|
-
}
|
|
9559
|
-
if (!receipt.leaseId?.trim()) return false;
|
|
9560
|
-
const expiresAt = receipt.leaseExpiresAt
|
|
9561
|
-
? Date.parse(receipt.leaseExpiresAt)
|
|
9562
|
-
: Number.NaN;
|
|
9563
|
-
// Keep an unparseable leased receipt conservative. The
|
|
9564
|
-
// fenced claim remains the recovery path after the wait.
|
|
9565
|
-
return !Number.isFinite(expiresAt) || expiresAt > Date.now();
|
|
9566
|
-
})
|
|
9567
|
-
.map(([receiptKey]) => receiptKey),
|
|
9568
|
-
);
|
|
9569
|
-
const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
|
|
9570
|
-
(receiptKey) => !forcedLiveReceiptKeys.has(receiptKey),
|
|
9571
|
-
);
|
|
9572
|
-
const claims =
|
|
9573
|
-
normalReceiptKeys.length > 0
|
|
9574
|
-
? await this.claimRuntimeStepReceipts(
|
|
9575
|
-
normalReceiptKeys,
|
|
9576
|
-
this.currentReceiptOwnerRunId,
|
|
9577
|
-
)
|
|
9578
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
9579
|
-
const forcedClaims =
|
|
9580
|
-
claimableForcedReceiptKeys.length > 0
|
|
9581
|
-
? await this.claimRuntimeStepReceipts(
|
|
9582
|
-
claimableForcedReceiptKeys,
|
|
9583
|
-
this.currentReceiptOwnerRunId,
|
|
9584
|
-
false,
|
|
9585
|
-
true,
|
|
9586
|
-
)
|
|
9587
|
-
: new Map<string, RuntimeStepReceipt>();
|
|
9588
|
-
const failedRefreshClaims =
|
|
9589
|
-
failedRefreshReceiptKeys.length > 0
|
|
9590
|
-
? await this.claimRuntimeStepReceipts(
|
|
9591
|
-
failedRefreshReceiptKeys,
|
|
9592
|
-
this.currentReceiptOwnerRunId,
|
|
9593
|
-
false,
|
|
9594
|
-
false,
|
|
9595
|
-
true,
|
|
9596
|
-
)
|
|
10260
|
+
const normalReceiptKeys = [...requestsByReceiptKey]
|
|
10261
|
+
.filter(
|
|
10262
|
+
([, group]) => !group.forceRefresh && !group.forceFailedRefresh,
|
|
10263
|
+
)
|
|
10264
|
+
.map(([receiptKey]) => receiptKey);
|
|
10265
|
+
const forcedReceiptKeys = [...requestsByReceiptKey]
|
|
10266
|
+
.filter(([, group]) => group.forceRefresh)
|
|
10267
|
+
.map(([receiptKey]) => receiptKey);
|
|
10268
|
+
const failedRefreshReceiptKeys = [...requestsByReceiptKey]
|
|
10269
|
+
.filter(
|
|
10270
|
+
([, group]) => !group.forceRefresh && group.forceFailedRefresh,
|
|
10271
|
+
)
|
|
10272
|
+
.map(([receiptKey]) => receiptKey);
|
|
10273
|
+
const forcedExisting = forcedReceiptKeys.length
|
|
10274
|
+
? await this.getRuntimeStepReceipts(forcedReceiptKeys)
|
|
9597
10275
|
: new Map<string, RuntimeStepReceipt>();
|
|
9598
|
-
|
|
9599
|
-
|
|
9600
|
-
|
|
9601
|
-
|
|
9602
|
-
|
|
9603
|
-
|
|
9604
|
-
|
|
9605
|
-
|
|
9606
|
-
|
|
9607
|
-
|
|
9608
|
-
|
|
9609
|
-
|
|
9610
|
-
|
|
9611
|
-
|
|
9612
|
-
|
|
9613
|
-
|
|
9614
|
-
|
|
9615
|
-
|
|
9616
|
-
|
|
9617
|
-
|
|
9618
|
-
|
|
9619
|
-
|
|
9620
|
-
|
|
9621
|
-
|
|
9622
|
-
|
|
9623
|
-
|
|
9624
|
-
|
|
9625
|
-
|
|
9626
|
-
|
|
9627
|
-
|
|
9628
|
-
|
|
9629
|
-
|
|
9630
|
-
|
|
9631
|
-
|
|
9632
|
-
|
|
9633
|
-
|
|
9634
|
-
|
|
9635
|
-
|
|
9636
|
-
|
|
9637
|
-
|
|
9638
|
-
|
|
9639
|
-
|
|
9640
|
-
|
|
9641
|
-
|
|
9642
|
-
|
|
9643
|
-
|
|
9644
|
-
|
|
9645
|
-
|
|
9646
|
-
|
|
9647
|
-
|
|
9648
|
-
|
|
9649
|
-
|
|
9650
|
-
|
|
9651
|
-
|
|
9652
|
-
|
|
9653
|
-
dataPatch: {},
|
|
9654
|
-
});
|
|
10276
|
+
// A forced refresh must never duplicate a provider call owned by a
|
|
10277
|
+
// live lease. It can, however, immediately claim pending/unleased
|
|
10278
|
+
// work and any expired lease. Waiting first for those receipts turns
|
|
10279
|
+
// an already-recoverable interruption into a five-minute stall.
|
|
10280
|
+
const forcedLiveReceiptKeys = new Set(
|
|
10281
|
+
[...forcedExisting]
|
|
10282
|
+
.filter(([, receipt]) => {
|
|
10283
|
+
if (
|
|
10284
|
+
receipt.status !== 'queued' &&
|
|
10285
|
+
receipt.status !== 'pending' &&
|
|
10286
|
+
receipt.status !== 'running'
|
|
10287
|
+
) {
|
|
10288
|
+
return false;
|
|
10289
|
+
}
|
|
10290
|
+
if (!receipt.leaseId?.trim()) return false;
|
|
10291
|
+
const expiresAt = receipt.leaseExpiresAt
|
|
10292
|
+
? Date.parse(receipt.leaseExpiresAt)
|
|
10293
|
+
: Number.NaN;
|
|
10294
|
+
// Keep an unparseable leased receipt conservative. The
|
|
10295
|
+
// fenced claim remains the recovery path after the wait.
|
|
10296
|
+
return !Number.isFinite(expiresAt) || expiresAt > Date.now();
|
|
10297
|
+
})
|
|
10298
|
+
.map(([receiptKey]) => receiptKey),
|
|
10299
|
+
);
|
|
10300
|
+
const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
|
|
10301
|
+
(receiptKey) => !forcedLiveReceiptKeys.has(receiptKey),
|
|
10302
|
+
);
|
|
10303
|
+
const claims =
|
|
10304
|
+
normalReceiptKeys.length > 0
|
|
10305
|
+
? await this.claimRuntimeStepReceipts(
|
|
10306
|
+
normalReceiptKeys,
|
|
10307
|
+
this.currentReceiptOwnerRunId,
|
|
10308
|
+
)
|
|
10309
|
+
: new Map<string, RuntimeStepReceipt>();
|
|
10310
|
+
const forcedClaims =
|
|
10311
|
+
claimableForcedReceiptKeys.length > 0
|
|
10312
|
+
? await this.claimRuntimeStepReceipts(
|
|
10313
|
+
claimableForcedReceiptKeys,
|
|
10314
|
+
this.currentReceiptOwnerRunId,
|
|
10315
|
+
false,
|
|
10316
|
+
true,
|
|
10317
|
+
)
|
|
10318
|
+
: new Map<string, RuntimeStepReceipt>();
|
|
10319
|
+
const failedRefreshClaims =
|
|
10320
|
+
failedRefreshReceiptKeys.length > 0
|
|
10321
|
+
? await this.claimRuntimeStepReceipts(
|
|
10322
|
+
failedRefreshReceiptKeys,
|
|
10323
|
+
this.currentReceiptOwnerRunId,
|
|
10324
|
+
false,
|
|
10325
|
+
false,
|
|
10326
|
+
true,
|
|
10327
|
+
)
|
|
10328
|
+
: new Map<string, RuntimeStepReceipt>();
|
|
10329
|
+
for (const [receiptKey, claim] of forcedClaims) {
|
|
10330
|
+
claims.set(receiptKey, claim);
|
|
9655
10331
|
}
|
|
9656
|
-
|
|
9657
|
-
|
|
9658
|
-
owner: ToolCallRequest | undefined,
|
|
9659
|
-
followers: ToolCallRequest[],
|
|
9660
|
-
): void => {
|
|
9661
|
-
if (!owner) return;
|
|
9662
|
-
claimedRequests.push(owner);
|
|
9663
|
-
if (followers.length > 0) {
|
|
9664
|
-
liveFollowersByOwnerCallId.set(owner.callId, followers);
|
|
10332
|
+
for (const [receiptKey, claim] of failedRefreshClaims) {
|
|
10333
|
+
claims.set(receiptKey, claim);
|
|
9665
10334
|
}
|
|
9666
|
-
|
|
9667
|
-
|
|
9668
|
-
|
|
9669
|
-
|
|
9670
|
-
|
|
9671
|
-
|
|
9672
|
-
|
|
9673
|
-
|
|
9674
|
-
|
|
9675
|
-
|
|
9676
|
-
|
|
9677
|
-
|
|
9678
|
-
),
|
|
9679
|
-
)
|
|
9680
|
-
: undefined;
|
|
9681
|
-
const queueExistingRunningReceiptWait = (
|
|
9682
|
-
receiptKey: string,
|
|
9683
|
-
requestsForKey: ToolCallRequest[],
|
|
9684
|
-
forceAfterWait = false,
|
|
9685
|
-
): void => {
|
|
9686
|
-
durableExistingRunningWaits.push({
|
|
9687
|
-
receiptKey,
|
|
9688
|
-
requestsForKey,
|
|
9689
|
-
forceAfterWait,
|
|
9690
|
-
waitMaxAttempts: resolveReceiptWaitMaxAttempts(requestsForKey),
|
|
9691
|
-
});
|
|
9692
|
-
};
|
|
9693
|
-
const reclaimExistingRunningReceiptAfterWait = async (
|
|
9694
|
-
receiptKey: string,
|
|
9695
|
-
requestsForKey: ToolCallRequest[],
|
|
9696
|
-
forceAfterWait = false,
|
|
9697
|
-
waitResult?: {
|
|
9698
|
-
completedReceipt?: RuntimeStepReceipt;
|
|
9699
|
-
skipWait?: boolean;
|
|
9700
|
-
},
|
|
9701
|
-
): Promise<void> => {
|
|
9702
|
-
if (waitResult?.completedReceipt) {
|
|
9703
|
-
if (!forceAfterWait) {
|
|
9704
|
-
await resolveRequestsFromReceipt(
|
|
9705
|
-
requestsForKey,
|
|
9706
|
-
waitResult.completedReceipt,
|
|
9707
|
-
'in_flight',
|
|
9708
|
-
);
|
|
9709
|
-
return;
|
|
10335
|
+
const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
|
|
10336
|
+
const durableRecoveredRequests: ToolCallRequest[] = [];
|
|
10337
|
+
const resolveRequestsFromReceipt = async (
|
|
10338
|
+
requestsForKey: ToolCallRequest[],
|
|
10339
|
+
receipt: RuntimeStepReceipt,
|
|
10340
|
+
source: 'cache' | 'in_flight',
|
|
10341
|
+
): Promise<void> => {
|
|
10342
|
+
const request = requestsForKey[0];
|
|
10343
|
+
if (!request) return;
|
|
10344
|
+
const receiptKey = request.receiptKey?.trim() || null;
|
|
10345
|
+
if (!receiptKey) {
|
|
10346
|
+
throw new Error(`${source} tool recovery needs receipt key.`);
|
|
9710
10347
|
}
|
|
9711
|
-
|
|
9712
|
-
|
|
9713
|
-
|
|
9714
|
-
|
|
9715
|
-
|
|
10348
|
+
const wrapped = await this.wrapToolExecutionResult({
|
|
10349
|
+
toolId,
|
|
10350
|
+
status:
|
|
10351
|
+
receipt.output === null || receipt.output === undefined
|
|
10352
|
+
? 'no_result'
|
|
10353
|
+
: 'completed',
|
|
10354
|
+
result: this.runtimeReceiptOutput(receipt),
|
|
10355
|
+
requestInput: request.input,
|
|
10356
|
+
execution: toolExecutionMetadataForOutcome({
|
|
10357
|
+
kind: source,
|
|
10358
|
+
cacheKey: request.cacheKey,
|
|
9716
10359
|
receiptKey,
|
|
9717
|
-
|
|
10360
|
+
attachedToReceiptKey: receiptKey,
|
|
10361
|
+
}),
|
|
10362
|
+
});
|
|
10363
|
+
this.cacheToolResult(toolId, request.cacheKey, wrapped);
|
|
10364
|
+
for (const waitingRequest of requestsForKey) {
|
|
10365
|
+
successfulLiveStepCallIds.add(waitingRequest.callId);
|
|
10366
|
+
const resolver = this.toolCallResolvers.get(
|
|
10367
|
+
waitingRequest.callId,
|
|
9718
10368
|
);
|
|
10369
|
+
if (resolver) {
|
|
10370
|
+
resolver.resolve(wrapped);
|
|
10371
|
+
this.toolCallResolvers.delete(waitingRequest.callId);
|
|
10372
|
+
}
|
|
10373
|
+
this.emitScopedFieldMetaUpdate({
|
|
10374
|
+
rowId: waitingRequest.rowId,
|
|
10375
|
+
key: waitingRequest.rowKey ?? null,
|
|
10376
|
+
tableNamespace: waitingRequest.tableNamespace ?? null,
|
|
10377
|
+
fieldName: waitingRequest.fieldName,
|
|
10378
|
+
status: 'cached',
|
|
10379
|
+
rowStatus: 'running',
|
|
10380
|
+
stage: toolId,
|
|
10381
|
+
provider: null,
|
|
10382
|
+
error: null,
|
|
10383
|
+
reused: true,
|
|
10384
|
+
dataPatch: {},
|
|
10385
|
+
});
|
|
10386
|
+
}
|
|
10387
|
+
};
|
|
10388
|
+
const claimOwnerWithLiveFollowers = (
|
|
10389
|
+
owner: ToolCallRequest | undefined,
|
|
10390
|
+
followers: ToolCallRequest[],
|
|
10391
|
+
): void => {
|
|
10392
|
+
if (!owner) return;
|
|
10393
|
+
claimedRequests.push(owner);
|
|
10394
|
+
if (followers.length > 0) {
|
|
10395
|
+
liveFollowersByOwnerCallId.set(owner.callId, followers);
|
|
10396
|
+
}
|
|
10397
|
+
};
|
|
10398
|
+
const resolveReceiptWaitMaxAttempts = (
|
|
10399
|
+
requestsForKey: ToolCallRequest[],
|
|
10400
|
+
): number | undefined =>
|
|
10401
|
+
requestsForKey.length > 0
|
|
10402
|
+
? Math.max(
|
|
10403
|
+
...requestsForKey.map((request) =>
|
|
10404
|
+
resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
10405
|
+
typeof request.receiptWaitMs === 'number'
|
|
10406
|
+
? { max_wait_ms: request.receiptWaitMs }
|
|
10407
|
+
: request.input,
|
|
10408
|
+
),
|
|
10409
|
+
),
|
|
10410
|
+
)
|
|
10411
|
+
: undefined;
|
|
10412
|
+
const queueExistingRunningReceiptWait = (
|
|
10413
|
+
receiptKey: string,
|
|
10414
|
+
requestsForKey: ToolCallRequest[],
|
|
10415
|
+
forceAfterWait = false,
|
|
10416
|
+
): void => {
|
|
10417
|
+
durableExistingRunningWaits.push({
|
|
10418
|
+
receiptKey,
|
|
10419
|
+
requestsForKey,
|
|
10420
|
+
forceAfterWait,
|
|
10421
|
+
waitMaxAttempts: resolveReceiptWaitMaxAttempts(requestsForKey),
|
|
10422
|
+
});
|
|
10423
|
+
};
|
|
10424
|
+
const reclaimExistingRunningReceiptAfterWait = async (
|
|
10425
|
+
receiptKey: string,
|
|
10426
|
+
requestsForKey: ToolCallRequest[],
|
|
10427
|
+
forceAfterWait = false,
|
|
10428
|
+
waitResult?: {
|
|
10429
|
+
completedReceipt?: RuntimeStepReceipt;
|
|
10430
|
+
skipWait?: boolean;
|
|
10431
|
+
},
|
|
10432
|
+
): Promise<void> => {
|
|
10433
|
+
if (waitResult?.completedReceipt) {
|
|
9719
10434
|
if (!forceAfterWait) {
|
|
9720
10435
|
await resolveRequestsFromReceipt(
|
|
9721
10436
|
requestsForKey,
|
|
9722
|
-
|
|
10437
|
+
waitResult.completedReceipt,
|
|
9723
10438
|
'in_flight',
|
|
9724
10439
|
);
|
|
9725
10440
|
return;
|
|
9726
10441
|
}
|
|
9727
|
-
}
|
|
9728
|
-
|
|
9729
|
-
|
|
9730
|
-
|
|
9731
|
-
|
|
9732
|
-
|
|
10442
|
+
} else if (waitResult?.skipWait !== true) {
|
|
10443
|
+
const waitMaxAttempts =
|
|
10444
|
+
resolveReceiptWaitMaxAttempts(requestsForKey);
|
|
10445
|
+
try {
|
|
10446
|
+
const completed =
|
|
10447
|
+
await this.waitForCompletedRuntimeToolReceipt(
|
|
10448
|
+
receiptKey,
|
|
10449
|
+
waitMaxAttempts,
|
|
10450
|
+
);
|
|
10451
|
+
if (!forceAfterWait) {
|
|
10452
|
+
await resolveRequestsFromReceipt(
|
|
10453
|
+
requestsForKey,
|
|
10454
|
+
completed,
|
|
10455
|
+
'in_flight',
|
|
10456
|
+
);
|
|
10457
|
+
return;
|
|
10458
|
+
}
|
|
10459
|
+
} catch (error) {
|
|
10460
|
+
if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
|
|
10461
|
+
for (const request of requestsForKey) {
|
|
10462
|
+
await this.rejectToolCall(toolId, request, error, {
|
|
10463
|
+
persistReceiptFailure: false,
|
|
10464
|
+
});
|
|
10465
|
+
}
|
|
10466
|
+
return;
|
|
9733
10467
|
}
|
|
9734
|
-
return;
|
|
9735
10468
|
}
|
|
9736
10469
|
}
|
|
9737
|
-
|
|
9738
|
-
|
|
9739
|
-
|
|
9740
|
-
|
|
9741
|
-
|
|
9742
|
-
|
|
9743
|
-
|
|
9744
|
-
)
|
|
9745
|
-
|
|
9746
|
-
|
|
9747
|
-
|
|
9748
|
-
|
|
9749
|
-
|
|
9750
|
-
|
|
9751
|
-
|
|
9752
|
-
|
|
9753
|
-
|
|
9754
|
-
|
|
9755
|
-
|
|
9756
|
-
|
|
9757
|
-
|
|
9758
|
-
|
|
10470
|
+
const reclaimed = (
|
|
10471
|
+
await this.claimRuntimeStepReceipts(
|
|
10472
|
+
[receiptKey],
|
|
10473
|
+
this.currentReceiptOwnerRunId,
|
|
10474
|
+
true,
|
|
10475
|
+
forceAfterWait,
|
|
10476
|
+
)
|
|
10477
|
+
).get(receiptKey);
|
|
10478
|
+
if (
|
|
10479
|
+
reclaimed?.status === 'completed' ||
|
|
10480
|
+
reclaimed?.status === 'skipped'
|
|
10481
|
+
) {
|
|
10482
|
+
await resolveRequestsFromReceipt(
|
|
10483
|
+
requestsForKey,
|
|
10484
|
+
reclaimed,
|
|
10485
|
+
'cache',
|
|
10486
|
+
);
|
|
10487
|
+
durableRecoveredRequests.push(...requestsForKey);
|
|
10488
|
+
return;
|
|
10489
|
+
}
|
|
10490
|
+
if (reclaimed?.status === 'failed') {
|
|
10491
|
+
for (const request of requestsForKey) {
|
|
10492
|
+
await this.rejectToolCall(
|
|
10493
|
+
toolId,
|
|
10494
|
+
request,
|
|
10495
|
+
runtimeReceiptFailureError(
|
|
10496
|
+
reclaimed,
|
|
10497
|
+
'Durable tool call failed',
|
|
10498
|
+
this.currentToolErrorSchemaVersion,
|
|
10499
|
+
),
|
|
10500
|
+
);
|
|
10501
|
+
}
|
|
10502
|
+
return;
|
|
10503
|
+
}
|
|
10504
|
+
if (this.isOwnedClaimedRuntimeReceipt(reclaimed)) {
|
|
10505
|
+
const [owner, ...waiters] = requestsForKey;
|
|
10506
|
+
if (!owner) return;
|
|
10507
|
+
if (!reclaimed.leaseId) {
|
|
10508
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
10509
|
+
receiptKey,
|
|
10510
|
+
runId: this.currentReceiptOwnerRunId,
|
|
10511
|
+
leaseId: 'missing',
|
|
10512
|
+
});
|
|
10513
|
+
}
|
|
10514
|
+
owner.receiptLeaseId = reclaimed.leaseId ?? null;
|
|
10515
|
+
owner.receiptLeaseExpiresAt = reclaimed.leaseExpiresAt ?? null;
|
|
10516
|
+
if (waiters.length > 0) {
|
|
10517
|
+
liveFollowersByOwnerCallId.set(owner.callId, waiters);
|
|
10518
|
+
}
|
|
10519
|
+
try {
|
|
10520
|
+
const execution = await this.callToolExecutionAPI(
|
|
10521
|
+
toolId,
|
|
10522
|
+
owner.input,
|
|
10523
|
+
{
|
|
10524
|
+
beforeProviderCall: () =>
|
|
10525
|
+
this.assertRuntimeToolReceiptOwnership([owner]),
|
|
10526
|
+
durableCallReceiptKey: receiptKey,
|
|
10527
|
+
playNodeScope: playNodeScopeForToolCallRequest(owner),
|
|
10528
|
+
executionAuthScopeDigest: owner.executionAuthScopeDigest,
|
|
10529
|
+
receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
|
|
10530
|
+
heartbeatReceipt: () =>
|
|
10531
|
+
this.renewRuntimeToolReceiptOwnership([owner]),
|
|
10532
|
+
providerIdempotencyKey:
|
|
10533
|
+
this.providerIdempotencyKeyForToolCall({
|
|
10534
|
+
cacheKey: owner.cacheKey,
|
|
10535
|
+
force: owner.force === true,
|
|
10536
|
+
leaseId: owner.receiptLeaseId,
|
|
10537
|
+
}),
|
|
10538
|
+
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
|
|
10539
|
+
owner,
|
|
10540
|
+
]),
|
|
10541
|
+
},
|
|
10542
|
+
);
|
|
10543
|
+
const result = await this.resolveToolCall(
|
|
10544
|
+
toolId,
|
|
10545
|
+
owner,
|
|
10546
|
+
execution?.result ?? null,
|
|
10547
|
+
execution?.metadata ?? null,
|
|
10548
|
+
execution?.jobId,
|
|
10549
|
+
execution?.meta,
|
|
10550
|
+
);
|
|
10551
|
+
if (result != null) {
|
|
10552
|
+
successfulLiveStepCallIds.add(owner.callId);
|
|
10553
|
+
}
|
|
10554
|
+
resolveLiveFollowers(owner, result);
|
|
10555
|
+
recordToolStep([owner]);
|
|
10556
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
10557
|
+
} catch (error) {
|
|
10558
|
+
await rejectWithLiveFollowers(owner, error);
|
|
10559
|
+
}
|
|
10560
|
+
return;
|
|
10561
|
+
}
|
|
9759
10562
|
for (const request of requestsForKey) {
|
|
9760
10563
|
await this.rejectToolCall(
|
|
9761
10564
|
toolId,
|
|
9762
10565
|
request,
|
|
9763
|
-
|
|
9764
|
-
|
|
9765
|
-
'Durable tool call failed',
|
|
9766
|
-
this.currentToolErrorSchemaVersion,
|
|
9767
|
-
),
|
|
10566
|
+
new RuntimeReceiptWaitTimeoutError(receiptKey),
|
|
10567
|
+
{ persistReceiptFailure: false },
|
|
9768
10568
|
);
|
|
9769
10569
|
}
|
|
9770
|
-
|
|
9771
|
-
|
|
9772
|
-
|
|
9773
|
-
|
|
9774
|
-
|
|
9775
|
-
|
|
9776
|
-
|
|
9777
|
-
|
|
9778
|
-
|
|
9779
|
-
|
|
9780
|
-
|
|
9781
|
-
|
|
9782
|
-
|
|
9783
|
-
|
|
9784
|
-
|
|
9785
|
-
|
|
9786
|
-
|
|
9787
|
-
|
|
9788
|
-
|
|
9789
|
-
|
|
9790
|
-
|
|
9791
|
-
|
|
9792
|
-
|
|
9793
|
-
|
|
9794
|
-
|
|
9795
|
-
|
|
9796
|
-
|
|
9797
|
-
|
|
9798
|
-
|
|
9799
|
-
|
|
9800
|
-
|
|
9801
|
-
|
|
9802
|
-
|
|
9803
|
-
|
|
9804
|
-
|
|
10570
|
+
};
|
|
10571
|
+
const processExistingRunningReceiptWaits =
|
|
10572
|
+
async (): Promise<void> => {
|
|
10573
|
+
if (durableExistingRunningWaits.length === 0) return;
|
|
10574
|
+
const waitSignal = durableExistingRunningAbort.signal;
|
|
10575
|
+
const pendingWaits = new Map(
|
|
10576
|
+
durableExistingRunningWaits.map((wait) => [
|
|
10577
|
+
wait.receiptKey,
|
|
10578
|
+
wait,
|
|
10579
|
+
]),
|
|
10580
|
+
);
|
|
10581
|
+
const waitMaxAttempts =
|
|
10582
|
+
Math.max(
|
|
10583
|
+
...durableExistingRunningWaits.map(
|
|
10584
|
+
(wait) => wait.waitMaxAttempts ?? 0,
|
|
10585
|
+
),
|
|
10586
|
+
) || DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
|
|
10587
|
+
if (runtimeReceiptReadTraceEnabled) {
|
|
10588
|
+
this.log(
|
|
10589
|
+
`[runtime-receipt-wait] phase=start requested=${pendingWaits.size} ` +
|
|
10590
|
+
`request_digests=${[...pendingWaits.keys()]
|
|
10591
|
+
.map(runtimeReceiptKeyDigest)
|
|
10592
|
+
.join(',')} max_attempts=${waitMaxAttempts}`,
|
|
10593
|
+
);
|
|
10594
|
+
}
|
|
10595
|
+
for (
|
|
10596
|
+
let attempt = 0;
|
|
10597
|
+
attempt < waitMaxAttempts && pendingWaits.size > 0;
|
|
10598
|
+
attempt += 1
|
|
10599
|
+
) {
|
|
10600
|
+
if (waitSignal.aborted) return;
|
|
10601
|
+
if (attempt > 0) {
|
|
10602
|
+
await new Promise<void>((resolve) => {
|
|
10603
|
+
const finish = () => {
|
|
10604
|
+
clearTimeout(timeout);
|
|
10605
|
+
waitSignal.removeEventListener('abort', finish);
|
|
10606
|
+
resolve();
|
|
10607
|
+
};
|
|
10608
|
+
const timeout = setTimeout(
|
|
10609
|
+
finish,
|
|
10610
|
+
DURABLE_RECEIPT_WAIT_DELAY_MS,
|
|
10611
|
+
);
|
|
10612
|
+
waitSignal.addEventListener('abort', finish, {
|
|
10613
|
+
once: true,
|
|
10614
|
+
});
|
|
10615
|
+
});
|
|
10616
|
+
if (waitSignal.aborted) return;
|
|
10617
|
+
}
|
|
10618
|
+
// The long backoff lives outside the hydration lane. Only a
|
|
10619
|
+
// bounded receipt read and delivery hold the lane, so stale
|
|
10620
|
+
// ownership cannot block unrelated cache misses/provider
|
|
10621
|
+
// work while terminal payloads still hydrate one group at a
|
|
10622
|
+
// time.
|
|
10623
|
+
await this.withRuntimeReceiptHydrationTurn(async () => {
|
|
10624
|
+
if (waitSignal.aborted) return;
|
|
10625
|
+
const waited =
|
|
10626
|
+
await this.waitForCompletedRuntimeToolReceipts(
|
|
10627
|
+
[...pendingWaits.keys()],
|
|
10628
|
+
1,
|
|
10629
|
+
);
|
|
10630
|
+
if (waitSignal.aborted) return;
|
|
10631
|
+
const settledWaits = [...pendingWaits.values()].filter(
|
|
10632
|
+
(wait) =>
|
|
10633
|
+
waited.completed.has(wait.receiptKey) ||
|
|
10634
|
+
waited.failed.has(wait.receiptKey),
|
|
10635
|
+
);
|
|
10636
|
+
const deliverySettlements = await Promise.allSettled(
|
|
10637
|
+
settledWaits.map(async (wait) => {
|
|
10638
|
+
const failed = waited.failed.get(wait.receiptKey);
|
|
10639
|
+
if (failed) {
|
|
10640
|
+
for (const request of wait.requestsForKey) {
|
|
10641
|
+
await this.rejectToolCall(toolId, request, failed, {
|
|
10642
|
+
persistReceiptFailure: false,
|
|
10643
|
+
});
|
|
10644
|
+
}
|
|
10645
|
+
return;
|
|
10646
|
+
}
|
|
10647
|
+
await reclaimExistingRunningReceiptAfterWait(
|
|
10648
|
+
wait.receiptKey,
|
|
10649
|
+
wait.requestsForKey,
|
|
10650
|
+
wait.forceAfterWait,
|
|
10651
|
+
{
|
|
10652
|
+
completedReceipt: waited.completed.get(
|
|
10653
|
+
wait.receiptKey,
|
|
10654
|
+
),
|
|
10655
|
+
skipWait: true,
|
|
10656
|
+
},
|
|
10657
|
+
);
|
|
9805
10658
|
}),
|
|
9806
|
-
|
|
9807
|
-
|
|
10659
|
+
);
|
|
10660
|
+
for (
|
|
10661
|
+
let index = 0;
|
|
10662
|
+
index < settledWaits.length;
|
|
10663
|
+
index += 1
|
|
10664
|
+
) {
|
|
10665
|
+
if (deliverySettlements[index]?.status === 'fulfilled') {
|
|
10666
|
+
pendingWaits.delete(settledWaits[index]!.receiptKey);
|
|
10667
|
+
}
|
|
10668
|
+
}
|
|
10669
|
+
if (runtimeReceiptReadTraceEnabled) {
|
|
10670
|
+
const rejected = deliverySettlements.flatMap(
|
|
10671
|
+
(settlement, index) => {
|
|
10672
|
+
if (settlement.status !== 'rejected') return [];
|
|
10673
|
+
const receiptKey = settledWaits[index]?.receiptKey;
|
|
10674
|
+
return receiptKey
|
|
10675
|
+
? [
|
|
10676
|
+
`${runtimeReceiptKeyDigest(receiptKey)}:${
|
|
10677
|
+
settlement.reason instanceof Error
|
|
10678
|
+
? settlement.reason.name
|
|
10679
|
+
: 'non_error'
|
|
10680
|
+
}`,
|
|
10681
|
+
]
|
|
10682
|
+
: ['unknown'];
|
|
10683
|
+
},
|
|
10684
|
+
);
|
|
10685
|
+
this.log(
|
|
10686
|
+
`[runtime-receipt-wait] phase=delivery settled=${deliverySettlements.length} ` +
|
|
10687
|
+
`rejected=${rejected.length} rejected_digests=${rejected.join(',')}`,
|
|
10688
|
+
);
|
|
10689
|
+
}
|
|
10690
|
+
const rejectedDelivery = deliverySettlements.find(
|
|
10691
|
+
(settlement) => settlement.status === 'rejected',
|
|
10692
|
+
);
|
|
10693
|
+
if (rejectedDelivery?.status === 'rejected') {
|
|
10694
|
+
throw rejectedDelivery.reason;
|
|
10695
|
+
}
|
|
10696
|
+
});
|
|
10697
|
+
}
|
|
10698
|
+
if (waitSignal.aborted) return;
|
|
10699
|
+
const reclaimSettlements = await Promise.allSettled(
|
|
10700
|
+
[...pendingWaits.values()].map((wait) =>
|
|
10701
|
+
reclaimExistingRunningReceiptAfterWait(
|
|
10702
|
+
wait.receiptKey,
|
|
10703
|
+
wait.requestsForKey,
|
|
10704
|
+
wait.forceAfterWait,
|
|
10705
|
+
{ skipWait: true },
|
|
10706
|
+
),
|
|
10707
|
+
),
|
|
9808
10708
|
);
|
|
9809
|
-
const
|
|
9810
|
-
|
|
9811
|
-
owner,
|
|
9812
|
-
execution?.result ?? null,
|
|
9813
|
-
execution?.metadata ?? null,
|
|
9814
|
-
execution?.jobId,
|
|
9815
|
-
execution?.meta,
|
|
10709
|
+
const rejectedReclaim = reclaimSettlements.find(
|
|
10710
|
+
(settlement) => settlement.status === 'rejected',
|
|
9816
10711
|
);
|
|
9817
|
-
if (
|
|
9818
|
-
|
|
10712
|
+
if (rejectedReclaim?.status === 'rejected') {
|
|
10713
|
+
throw rejectedReclaim.reason;
|
|
9819
10714
|
}
|
|
9820
|
-
|
|
9821
|
-
|
|
9822
|
-
|
|
9823
|
-
|
|
9824
|
-
|
|
9825
|
-
|
|
9826
|
-
|
|
9827
|
-
|
|
9828
|
-
|
|
9829
|
-
|
|
9830
|
-
toolId,
|
|
9831
|
-
request,
|
|
9832
|
-
new RuntimeReceiptWaitTimeoutError(receiptKey),
|
|
9833
|
-
{ persistReceiptFailure: false },
|
|
9834
|
-
);
|
|
9835
|
-
}
|
|
9836
|
-
};
|
|
9837
|
-
const processExistingRunningReceiptWaits =
|
|
9838
|
-
async (): Promise<void> => {
|
|
9839
|
-
if (durableExistingRunningWaits.length === 0) return;
|
|
9840
|
-
const waitingReceiptKeys = durableExistingRunningWaits.map(
|
|
9841
|
-
(wait) => wait.receiptKey,
|
|
9842
|
-
);
|
|
9843
|
-
const waitMaxAttempts = Math.max(
|
|
9844
|
-
...durableExistingRunningWaits.map(
|
|
9845
|
-
(wait) => wait.waitMaxAttempts ?? 0,
|
|
9846
|
-
),
|
|
9847
|
-
);
|
|
9848
|
-
if (runtimeReceiptReadTraceEnabled) {
|
|
9849
|
-
this.log(
|
|
9850
|
-
`[runtime-receipt-wait] phase=start requested=${waitingReceiptKeys.length} ` +
|
|
9851
|
-
`request_digests=${waitingReceiptKeys
|
|
9852
|
-
.map(runtimeReceiptKeyDigest)
|
|
9853
|
-
.join(',')} max_attempts=${waitMaxAttempts || 'default'}`,
|
|
10715
|
+
};
|
|
10716
|
+
|
|
10717
|
+
for (const [receiptKey, group] of requestsByReceiptKey) {
|
|
10718
|
+
const requestsForKey = group.requests;
|
|
10719
|
+
const claim = claims.get(receiptKey);
|
|
10720
|
+
if (!claim) {
|
|
10721
|
+
queueExistingRunningReceiptWait(
|
|
10722
|
+
receiptKey,
|
|
10723
|
+
requestsForKey,
|
|
10724
|
+
group.forceRefresh,
|
|
9854
10725
|
);
|
|
10726
|
+
continue;
|
|
9855
10727
|
}
|
|
9856
|
-
|
|
9857
|
-
|
|
9858
|
-
|
|
9859
|
-
)
|
|
9860
|
-
|
|
9861
|
-
|
|
9862
|
-
|
|
9863
|
-
|
|
9864
|
-
`completed_digests=${[...waited.completed.keys()]
|
|
9865
|
-
.map(runtimeReceiptKeyDigest)
|
|
9866
|
-
.join(',')} ` +
|
|
9867
|
-
`timed_out_digests=${[...waited.timedOut]
|
|
9868
|
-
.map(runtimeReceiptKeyDigest)
|
|
9869
|
-
.join(',')}`,
|
|
10728
|
+
if (
|
|
10729
|
+
claim?.status === 'completed' ||
|
|
10730
|
+
claim?.status === 'skipped'
|
|
10731
|
+
) {
|
|
10732
|
+
await resolveRequestsFromReceipt(
|
|
10733
|
+
requestsForKey,
|
|
10734
|
+
claim,
|
|
10735
|
+
'cache',
|
|
9870
10736
|
);
|
|
10737
|
+
durableRecoveredRequests.push(...requestsForKey);
|
|
10738
|
+
continue;
|
|
9871
10739
|
}
|
|
9872
|
-
|
|
9873
|
-
|
|
9874
|
-
|
|
9875
|
-
|
|
9876
|
-
|
|
9877
|
-
|
|
9878
|
-
|
|
9879
|
-
|
|
9880
|
-
|
|
9881
|
-
|
|
9882
|
-
}
|
|
9883
|
-
await reclaimExistingRunningReceiptAfterWait(
|
|
9884
|
-
wait.receiptKey,
|
|
9885
|
-
wait.requestsForKey,
|
|
9886
|
-
wait.forceAfterWait,
|
|
9887
|
-
{
|
|
9888
|
-
completedReceipt: waited.completed.get(wait.receiptKey),
|
|
9889
|
-
skipWait: true,
|
|
9890
|
-
},
|
|
10740
|
+
if (claim?.status === 'failed') {
|
|
10741
|
+
for (const request of requestsForKey) {
|
|
10742
|
+
await this.rejectToolCall(
|
|
10743
|
+
toolId,
|
|
10744
|
+
request,
|
|
10745
|
+
runtimeReceiptFailureError(
|
|
10746
|
+
claim,
|
|
10747
|
+
'Durable tool call failed',
|
|
10748
|
+
this.currentToolErrorSchemaVersion,
|
|
10749
|
+
),
|
|
9891
10750
|
);
|
|
9892
|
-
}
|
|
9893
|
-
|
|
9894
|
-
if (runtimeReceiptReadTraceEnabled) {
|
|
9895
|
-
const rejected = deliverySettlements.flatMap(
|
|
9896
|
-
(settlement, index) => {
|
|
9897
|
-
if (settlement.status !== 'rejected') return [];
|
|
9898
|
-
const receiptKey =
|
|
9899
|
-
durableExistingRunningWaits[index]?.receiptKey;
|
|
9900
|
-
return receiptKey
|
|
9901
|
-
? [
|
|
9902
|
-
`${runtimeReceiptKeyDigest(receiptKey)}:${
|
|
9903
|
-
settlement.reason instanceof Error
|
|
9904
|
-
? settlement.reason.name
|
|
9905
|
-
: 'non_error'
|
|
9906
|
-
}`,
|
|
9907
|
-
]
|
|
9908
|
-
: ['unknown'];
|
|
9909
|
-
},
|
|
9910
|
-
);
|
|
9911
|
-
this.log(
|
|
9912
|
-
`[runtime-receipt-wait] phase=delivery settled=${deliverySettlements.length} ` +
|
|
9913
|
-
`rejected=${rejected.length} rejected_digests=${rejected.join(',')}`,
|
|
9914
|
-
);
|
|
10751
|
+
}
|
|
10752
|
+
continue;
|
|
9915
10753
|
}
|
|
9916
|
-
|
|
9917
|
-
|
|
9918
|
-
|
|
9919
|
-
|
|
9920
|
-
|
|
9921
|
-
|
|
9922
|
-
|
|
9923
|
-
|
|
9924
|
-
|
|
9925
|
-
|
|
9926
|
-
);
|
|
9927
|
-
continue;
|
|
9928
|
-
}
|
|
9929
|
-
if (claim?.status === 'completed' || claim?.status === 'skipped') {
|
|
9930
|
-
await resolveRequestsFromReceipt(requestsForKey, claim, 'cache');
|
|
9931
|
-
durableRecoveredRequests.push(...requestsForKey);
|
|
9932
|
-
continue;
|
|
9933
|
-
}
|
|
9934
|
-
if (claim?.status === 'failed') {
|
|
9935
|
-
for (const request of requestsForKey) {
|
|
9936
|
-
await this.rejectToolCall(
|
|
9937
|
-
toolId,
|
|
9938
|
-
request,
|
|
9939
|
-
runtimeReceiptFailureError(
|
|
9940
|
-
claim,
|
|
9941
|
-
'Durable tool call failed',
|
|
9942
|
-
this.currentToolErrorSchemaVersion,
|
|
9943
|
-
),
|
|
10754
|
+
if (
|
|
10755
|
+
(claim?.status === 'queued' ||
|
|
10756
|
+
claim?.status === 'pending' ||
|
|
10757
|
+
claim?.status === 'running') &&
|
|
10758
|
+
claim.claimState === 'existing'
|
|
10759
|
+
) {
|
|
10760
|
+
queueExistingRunningReceiptWait(
|
|
10761
|
+
receiptKey,
|
|
10762
|
+
requestsForKey,
|
|
10763
|
+
group.forceRefresh,
|
|
9944
10764
|
);
|
|
10765
|
+
continue;
|
|
10766
|
+
}
|
|
10767
|
+
if (this.isOwnedClaimedRuntimeReceipt(claim)) {
|
|
10768
|
+
const [owner, ...waiters] = requestsForKey;
|
|
10769
|
+
if (!claim.leaseId) {
|
|
10770
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
10771
|
+
receiptKey,
|
|
10772
|
+
runId: this.currentReceiptOwnerRunId,
|
|
10773
|
+
leaseId: 'missing',
|
|
10774
|
+
});
|
|
10775
|
+
}
|
|
10776
|
+
if (owner) {
|
|
10777
|
+
owner.receiptLeaseId = claim.leaseId ?? null;
|
|
10778
|
+
owner.receiptLeaseExpiresAt = claim.leaseExpiresAt ?? null;
|
|
10779
|
+
const claimLeaseExpiresAtMs = Date.parse(
|
|
10780
|
+
claim.leaseExpiresAt ?? '',
|
|
10781
|
+
);
|
|
10782
|
+
pendingReceiptClaimsEstablishExecutionFence &&=
|
|
10783
|
+
claim.claimState === 'claimed' &&
|
|
10784
|
+
claim.status === 'running' &&
|
|
10785
|
+
Number.isFinite(claimLeaseExpiresAtMs) &&
|
|
10786
|
+
claimLeaseExpiresAtMs > Date.now();
|
|
10787
|
+
}
|
|
10788
|
+
claimOwnerWithLiveFollowers(owner, waiters);
|
|
10789
|
+
continue;
|
|
9945
10790
|
}
|
|
9946
|
-
continue;
|
|
9947
|
-
}
|
|
9948
|
-
if (
|
|
9949
|
-
(claim?.status === 'queued' ||
|
|
9950
|
-
claim?.status === 'pending' ||
|
|
9951
|
-
claim?.status === 'running') &&
|
|
9952
|
-
claim.claimState === 'existing'
|
|
9953
|
-
) {
|
|
9954
10791
|
queueExistingRunningReceiptWait(
|
|
9955
10792
|
receiptKey,
|
|
9956
10793
|
requestsForKey,
|
|
9957
10794
|
group.forceRefresh,
|
|
9958
10795
|
);
|
|
9959
|
-
continue;
|
|
9960
|
-
}
|
|
9961
|
-
if (this.isOwnedClaimedRuntimeReceipt(claim)) {
|
|
9962
|
-
const [owner, ...waiters] = requestsForKey;
|
|
9963
|
-
if (!claim.leaseId) {
|
|
9964
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
9965
|
-
receiptKey,
|
|
9966
|
-
runId: this.currentReceiptOwnerRunId,
|
|
9967
|
-
leaseId: 'missing',
|
|
9968
|
-
});
|
|
9969
|
-
}
|
|
9970
|
-
if (owner) {
|
|
9971
|
-
owner.receiptLeaseId = claim.leaseId ?? null;
|
|
9972
|
-
owner.receiptLeaseExpiresAt = claim.leaseExpiresAt ?? null;
|
|
9973
|
-
}
|
|
9974
|
-
claimOwnerWithLiveFollowers(owner, waiters);
|
|
9975
|
-
continue;
|
|
9976
10796
|
}
|
|
9977
|
-
queueExistingRunningReceiptWait(
|
|
9978
|
-
receiptKey,
|
|
9979
|
-
requestsForKey,
|
|
9980
|
-
group.forceRefresh,
|
|
9981
|
-
);
|
|
9982
|
-
}
|
|
9983
10797
|
|
|
9984
|
-
|
|
9985
|
-
|
|
9986
|
-
|
|
9987
|
-
|
|
9988
|
-
|
|
9989
|
-
|
|
9990
|
-
|
|
10798
|
+
pendingRequests.length = 0;
|
|
10799
|
+
pendingRequests.push(...claimedRequests);
|
|
10800
|
+
durableExistingRunningState.settlement =
|
|
10801
|
+
durableExistingRunningWaits.length > 0
|
|
10802
|
+
? processExistingRunningReceiptWaits().then(
|
|
10803
|
+
() => ({ status: 'fulfilled' as const }),
|
|
10804
|
+
(reason: unknown) => ({
|
|
10805
|
+
status: 'rejected' as const,
|
|
10806
|
+
reason,
|
|
10807
|
+
}),
|
|
10808
|
+
)
|
|
10809
|
+
: null;
|
|
10810
|
+
recordToolStep(durableRecoveredRequests);
|
|
10811
|
+
});
|
|
9991
10812
|
}
|
|
9992
10813
|
|
|
9993
|
-
|
|
9994
|
-
|
|
10814
|
+
// Claims establish one lease cohort. Revalidate that cohort in one
|
|
10815
|
+
// gateway operation before provider admission, then let the existing
|
|
10816
|
+
// cohort heartbeat keep the undispatched tail alive. Revalidating each
|
|
10817
|
+
// call separately after a tool slot opens fragments the writer into
|
|
10818
|
+
// one-row batches and starves otherwise healthy providers.
|
|
10819
|
+
const pendingReceiptHeartbeat = this.startPendingToolReceiptHeartbeat(
|
|
10820
|
+
pendingRequests,
|
|
10821
|
+
pendingReceiptClaimsEstablishExecutionFence,
|
|
10822
|
+
);
|
|
10823
|
+
let providerFailure: { reason: unknown } | null = null;
|
|
9995
10824
|
try {
|
|
10825
|
+
if (pendingReceiptHeartbeat) {
|
|
10826
|
+
await pendingReceiptHeartbeat.confirmOwned();
|
|
10827
|
+
}
|
|
9996
10828
|
if (pendingRequests.length > 0) {
|
|
9997
10829
|
const strategy =
|
|
9998
10830
|
this.#options.getBatchOperationStrategy?.(toolId) ?? null;
|
|
@@ -10039,59 +10871,72 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10039
10871
|
toolId: batch.batchOperation,
|
|
10040
10872
|
}),
|
|
10041
10873
|
});
|
|
10042
|
-
|
|
10043
|
-
|
|
10044
|
-
|
|
10045
|
-
|
|
10046
|
-
|
|
10047
|
-
|
|
10048
|
-
|
|
10049
|
-
|
|
10050
|
-
|
|
10051
|
-
|
|
10052
|
-
|
|
10053
|
-
|
|
10054
|
-
|
|
10055
|
-
|
|
10056
|
-
|
|
10057
|
-
|
|
10058
|
-
|
|
10059
|
-
|
|
10060
|
-
|
|
10061
|
-
|
|
10062
|
-
|
|
10063
|
-
|
|
10064
|
-
|
|
10065
|
-
|
|
10066
|
-
|
|
10067
|
-
|
|
10068
|
-
|
|
10069
|
-
|
|
10070
|
-
|
|
10071
|
-
|
|
10072
|
-
|
|
10073
|
-
|
|
10074
|
-
|
|
10075
|
-
|
|
10076
|
-
|
|
10077
|
-
|
|
10078
|
-
|
|
10079
|
-
|
|
10080
|
-
|
|
10081
|
-
|
|
10082
|
-
|
|
10083
|
-
|
|
10084
|
-
|
|
10085
|
-
|
|
10086
|
-
this.assertRuntimeToolReceiptOwnership(
|
|
10087
|
-
batch.memberRequests,
|
|
10088
|
-
),
|
|
10089
|
-
heartbeatReceipt: () =>
|
|
10090
|
-
this.renewRuntimeToolReceiptOwnership(
|
|
10874
|
+
let releaseToolSlot: () => void = () => undefined;
|
|
10875
|
+
try {
|
|
10876
|
+
const value = await this.callToolAPI(
|
|
10877
|
+
batch.batchOperation,
|
|
10878
|
+
batch.batchPayload,
|
|
10879
|
+
{
|
|
10880
|
+
durableCallReceiptKey: aggregateReceiptKey,
|
|
10881
|
+
// A native batch is one physical provider request for
|
|
10882
|
+
// many rows of the same column, so the batch's node scope
|
|
10883
|
+
// is its members' shared scope.
|
|
10884
|
+
playNodeScope: batch.memberRequests[0]
|
|
10885
|
+
? playNodeScopeForToolCallRequest(
|
|
10886
|
+
batch.memberRequests[0],
|
|
10887
|
+
batch.batchOperation,
|
|
10888
|
+
)
|
|
10889
|
+
: null,
|
|
10890
|
+
executionAuthScopeDigest:
|
|
10891
|
+
batch.memberRequests[0]?.executionAuthScopeDigest ??
|
|
10892
|
+
null,
|
|
10893
|
+
providerIdempotencyKey:
|
|
10894
|
+
buildDurableToolAggregateProviderIdempotencyKey({
|
|
10895
|
+
aggregateReceiptKey,
|
|
10896
|
+
receiptKeys,
|
|
10897
|
+
providerIdempotencyKeys: batch.memberRequests.map(
|
|
10898
|
+
(request) =>
|
|
10899
|
+
this.providerIdempotencyKeyForToolCall({
|
|
10900
|
+
cacheKey: request.cacheKey,
|
|
10901
|
+
force: request.force === true,
|
|
10902
|
+
leaseId: request.receiptLeaseId,
|
|
10903
|
+
}),
|
|
10904
|
+
),
|
|
10905
|
+
}),
|
|
10906
|
+
receiptLeaseExpiresAt: batch.memberRequests.reduce<
|
|
10907
|
+
string | null
|
|
10908
|
+
>((earliest, request) => {
|
|
10909
|
+
const expiresAt =
|
|
10910
|
+
request.receiptLeaseExpiresAt ?? null;
|
|
10911
|
+
if (!expiresAt) return earliest;
|
|
10912
|
+
if (!earliest) return expiresAt;
|
|
10913
|
+
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
10914
|
+
? expiresAt
|
|
10915
|
+
: earliest;
|
|
10916
|
+
}, null),
|
|
10917
|
+
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
|
|
10091
10918
|
batch.memberRequests,
|
|
10092
10919
|
),
|
|
10093
|
-
|
|
10094
|
-
|
|
10920
|
+
beforeProviderCall: () =>
|
|
10921
|
+
pendingReceiptHeartbeat
|
|
10922
|
+
? pendingReceiptHeartbeat.assertOwned()
|
|
10923
|
+
: this.assertRuntimeToolReceiptOwnership(
|
|
10924
|
+
batch.memberRequests,
|
|
10925
|
+
),
|
|
10926
|
+
heartbeatReceipt: () =>
|
|
10927
|
+
this.renewRuntimeToolReceiptOwnership(
|
|
10928
|
+
batch.memberRequests,
|
|
10929
|
+
),
|
|
10930
|
+
retainToolSlot: (release) => {
|
|
10931
|
+
releaseToolSlot = release;
|
|
10932
|
+
},
|
|
10933
|
+
},
|
|
10934
|
+
);
|
|
10935
|
+
return { value, releaseToolSlot };
|
|
10936
|
+
} catch (error) {
|
|
10937
|
+
releaseToolSlot();
|
|
10938
|
+
throw error;
|
|
10939
|
+
}
|
|
10095
10940
|
},
|
|
10096
10941
|
onChunkComplete: async (chunkResults) => {
|
|
10097
10942
|
for (const entry of chunkResults) {
|
|
@@ -10101,29 +10946,35 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10101
10946
|
}
|
|
10102
10947
|
continue;
|
|
10103
10948
|
}
|
|
10104
|
-
|
|
10105
|
-
|
|
10106
|
-
|
|
10107
|
-
|
|
10108
|
-
|
|
10109
|
-
|
|
10110
|
-
|
|
10111
|
-
|
|
10112
|
-
request
|
|
10113
|
-
|
|
10114
|
-
|
|
10115
|
-
|
|
10949
|
+
try {
|
|
10950
|
+
const splitResults =
|
|
10951
|
+
entry.result != null
|
|
10952
|
+
? entry.request.splitResults(entry.result.value)
|
|
10953
|
+
: entry.request.memberRequests.map(() => null);
|
|
10954
|
+
const resolvedResults =
|
|
10955
|
+
await this.resolveToolCallBatchResults(
|
|
10956
|
+
toolId,
|
|
10957
|
+
entry.request.memberRequests.map(
|
|
10958
|
+
(request, index) => ({
|
|
10959
|
+
request,
|
|
10960
|
+
result: splitResults[index] ?? null,
|
|
10961
|
+
}),
|
|
10962
|
+
),
|
|
10963
|
+
);
|
|
10116
10964
|
|
|
10117
|
-
|
|
10118
|
-
|
|
10119
|
-
|
|
10120
|
-
|
|
10121
|
-
|
|
10122
|
-
|
|
10123
|
-
|
|
10124
|
-
|
|
10965
|
+
for (
|
|
10966
|
+
let index = 0;
|
|
10967
|
+
index < entry.request.memberRequests.length;
|
|
10968
|
+
index += 1
|
|
10969
|
+
) {
|
|
10970
|
+
const request = entry.request.memberRequests[index]!;
|
|
10971
|
+
if (resolvedResults[index] != null) {
|
|
10972
|
+
successfulLiveStepCallIds.add(request.callId);
|
|
10973
|
+
}
|
|
10974
|
+
resolveLiveFollowers(request, resolvedResults[index]);
|
|
10125
10975
|
}
|
|
10126
|
-
|
|
10976
|
+
} finally {
|
|
10977
|
+
entry.result?.releaseToolSlot();
|
|
10127
10978
|
}
|
|
10128
10979
|
}
|
|
10129
10980
|
|
|
@@ -10134,6 +10985,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10134
10985
|
);
|
|
10135
10986
|
this.#options.onBatchComplete?.(this.checkpoint);
|
|
10136
10987
|
},
|
|
10988
|
+
retainResults: false,
|
|
10137
10989
|
});
|
|
10138
10990
|
} else {
|
|
10139
10991
|
const completionBuffer: Array<{
|
|
@@ -10235,36 +11087,48 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10235
11087
|
this.persistenceLatch,
|
|
10236
11088
|
);
|
|
10237
11089
|
}
|
|
10238
|
-
|
|
10239
|
-
|
|
10240
|
-
|
|
10241
|
-
|
|
10242
|
-
|
|
10243
|
-
|
|
10244
|
-
|
|
10245
|
-
|
|
10246
|
-
|
|
10247
|
-
|
|
10248
|
-
|
|
10249
|
-
|
|
10250
|
-
|
|
10251
|
-
|
|
10252
|
-
|
|
10253
|
-
|
|
10254
|
-
|
|
10255
|
-
|
|
10256
|
-
|
|
10257
|
-
|
|
10258
|
-
|
|
10259
|
-
|
|
10260
|
-
|
|
10261
|
-
|
|
10262
|
-
|
|
10263
|
-
|
|
10264
|
-
|
|
10265
|
-
|
|
10266
|
-
|
|
10267
|
-
|
|
11090
|
+
let releaseToolSlot: () => void = () => undefined;
|
|
11091
|
+
try {
|
|
11092
|
+
const execution = await this.callToolExecutionAPI(
|
|
11093
|
+
toolId,
|
|
11094
|
+
request.input,
|
|
11095
|
+
{
|
|
11096
|
+
beforeProviderCall: () =>
|
|
11097
|
+
pendingReceiptHeartbeat
|
|
11098
|
+
? pendingReceiptHeartbeat.assertOwned()
|
|
11099
|
+
: this.assertRuntimeToolReceiptOwnership([request]),
|
|
11100
|
+
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
11101
|
+
...(request.receiptKey
|
|
11102
|
+
? {
|
|
11103
|
+
durableCallReceiptKey: request.receiptKey,
|
|
11104
|
+
executionAuthScopeDigest:
|
|
11105
|
+
request.executionAuthScopeDigest,
|
|
11106
|
+
providerIdempotencyKey:
|
|
11107
|
+
this.providerIdempotencyKeyForToolCall({
|
|
11108
|
+
cacheKey: request.receiptKey,
|
|
11109
|
+
force: request.force === true,
|
|
11110
|
+
leaseId: request.receiptLeaseId,
|
|
11111
|
+
}),
|
|
11112
|
+
receiptLeaseExpiresAt:
|
|
11113
|
+
request.receiptLeaseExpiresAt,
|
|
11114
|
+
heartbeatReceipt: () =>
|
|
11115
|
+
this.renewRuntimeToolReceiptOwnership([
|
|
11116
|
+
request,
|
|
11117
|
+
]),
|
|
11118
|
+
}
|
|
11119
|
+
: {}),
|
|
11120
|
+
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
|
|
11121
|
+
request,
|
|
11122
|
+
]),
|
|
11123
|
+
retainToolSlot: (release) => {
|
|
11124
|
+
releaseToolSlot = release;
|
|
11125
|
+
},
|
|
11126
|
+
},
|
|
11127
|
+
);
|
|
11128
|
+
await enqueueCompletion(request, execution);
|
|
11129
|
+
} finally {
|
|
11130
|
+
releaseToolSlot();
|
|
11131
|
+
}
|
|
10268
11132
|
},
|
|
10269
11133
|
);
|
|
10270
11134
|
const failedEntries = dispatchResults.filter(
|
|
@@ -10281,14 +11145,26 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10281
11145
|
this.#options.onBatchComplete?.(this.checkpoint);
|
|
10282
11146
|
}
|
|
10283
11147
|
}
|
|
11148
|
+
} catch (reason) {
|
|
11149
|
+
providerFailure = { reason };
|
|
11150
|
+
durableExistingRunningAbort.abort(reason);
|
|
10284
11151
|
} finally {
|
|
10285
11152
|
pendingReceiptHeartbeat?.stop();
|
|
10286
11153
|
}
|
|
10287
11154
|
const pendingReceiptLeaseLost = pendingReceiptHeartbeat?.leaseLost();
|
|
10288
|
-
if (pendingReceiptLeaseLost)
|
|
10289
|
-
|
|
10290
|
-
|
|
11155
|
+
if (pendingReceiptLeaseLost && providerFailure === null) {
|
|
11156
|
+
providerFailure = { reason: pendingReceiptLeaseLost };
|
|
11157
|
+
durableExistingRunningAbort.abort(pendingReceiptLeaseLost);
|
|
11158
|
+
}
|
|
11159
|
+
const durableExistingRunningSettlement =
|
|
11160
|
+
durableExistingRunningState.settlement;
|
|
11161
|
+
if (durableExistingRunningSettlement) {
|
|
11162
|
+
const settlement = await durableExistingRunningSettlement;
|
|
11163
|
+
if (settlement.status === 'rejected' && providerFailure === null) {
|
|
11164
|
+
providerFailure = { reason: settlement.reason };
|
|
11165
|
+
}
|
|
10291
11166
|
}
|
|
11167
|
+
if (providerFailure) throw providerFailure.reason;
|
|
10292
11168
|
}),
|
|
10293
11169
|
);
|
|
10294
11170
|
const rejected = toolSettlements.find(
|
|
@@ -10497,7 +11373,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10497
11373
|
`[perf] tool call id=${toolId} phase=governor_admission elapsed_ms=${Date.now() - admissionStartedAt}`,
|
|
10498
11374
|
);
|
|
10499
11375
|
}
|
|
11376
|
+
let toolSlotTransferred = false;
|
|
10500
11377
|
try {
|
|
11378
|
+
if (options?.retainToolSlot) {
|
|
11379
|
+
options.retainToolSlot(() => toolSlot.release());
|
|
11380
|
+
toolSlotTransferred = true;
|
|
11381
|
+
}
|
|
10501
11382
|
return await withActiveSpan(
|
|
10502
11383
|
'plays.tool.execute',
|
|
10503
11384
|
{
|
|
@@ -11271,7 +12152,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11271
12152
|
},
|
|
11272
12153
|
});
|
|
11273
12154
|
}
|
|
11274
|
-
|
|
12155
|
+
if (!toolSlotTransferred) {
|
|
12156
|
+
toolSlot.release();
|
|
12157
|
+
}
|
|
11275
12158
|
}
|
|
11276
12159
|
}
|
|
11277
12160
|
|