deepline 0.1.243 → 0.1.244

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +1 -1
  2. package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +5 -25
  3. package/dist/bundling-sources/sdk/src/release.ts +23 -21
  4. package/dist/bundling-sources/sdk/src/types.ts +1 -1
  5. package/dist/bundling-sources/shared_libs/play-runtime/backend.ts +2 -21
  6. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -6
  7. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +6 -13
  8. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +9 -26
  9. package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +1 -2
  10. package/dist/bundling-sources/shared_libs/play-runtime/map-row-identity.ts +1 -1
  11. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +191 -29
  12. package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +1 -1
  13. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +1 -36
  14. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +2 -3
  15. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +0 -7
  16. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +1 -12
  17. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +2 -3
  18. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +5 -15
  19. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +6 -21
  20. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +3 -4
  21. package/dist/bundling-sources/shared_libs/plays/artifact-kind-guard.ts +16 -29
  22. package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +4 -8
  23. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +21 -712
  24. package/dist/bundling-sources/shared_libs/plays/bundling/limits.ts +1 -18
  25. package/dist/cli/index.js +29 -71
  26. package/dist/cli/index.mjs +29 -71
  27. package/dist/{compiler-manifest-CYcwzSOJ.d.ts → compiler-manifest-DlE7dnRm.d.mts} +2 -8
  28. package/dist/{compiler-manifest-CYcwzSOJ.d.mts → compiler-manifest-DlE7dnRm.d.ts} +2 -8
  29. package/dist/index.d.mts +2 -2
  30. package/dist/index.d.ts +2 -2
  31. package/dist/index.js +24 -22
  32. package/dist/index.mjs +24 -22
  33. package/dist/plays/bundle-play-file.d.mts +7 -22
  34. package/dist/plays/bundle-play-file.d.ts +7 -22
  35. package/dist/plays/bundle-play-file.mjs +26 -614
  36. package/package.json +2 -2
  37. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +0 -4504
  38. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +0 -1386
  39. package/dist/bundling-sources/apps/play-runner-workers/src/durable-object-deploy-handoff.ts +0 -30
  40. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +0 -8385
  41. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md +0 -21
  42. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts +0 -251
  43. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/coordinator-progress.ts +0 -289
  44. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/csv-rows.ts +0 -85
  45. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +0 -23
  46. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/dataset-handles.ts +0 -528
  47. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +0 -229
  48. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/ledger-event-batches.ts +0 -81
  49. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/live-progress.ts +0 -22
  50. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +0 -920
  51. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts +0 -90
  52. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +0 -475
  53. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +0 -570
  54. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/result-dataset-persistence.ts +0 -50
  55. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/row-isolation.ts +0 -5
  56. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +0 -519
  57. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-batch.ts +0 -107
  58. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +0 -2541
  59. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-http-errors.ts +0 -7
  60. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +0 -227
  61. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +0 -130
  62. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +0 -39
  63. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/workflow-preview.ts +0 -26
  64. package/dist/bundling-sources/apps/play-runner-workers/src/runtime-api-retry.ts +0 -23
  65. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-instance-create.ts +0 -47
  66. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +0 -167
  67. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +0 -91
  68. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +0 -329
  69. package/dist/bundling-sources/sdk/src/worker-play-entry.ts +0 -240
  70. package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +0 -252
  71. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +0 -217
@@ -1,2541 +0,0 @@
1
- import {
2
- compileRequestsWithStrategy,
3
- executeChunkedRequests,
4
- type ChunkExecutionResult,
5
- } from '../../../../shared_libs/play-runtime/batch-runtime';
6
- import { dispatchBounded } from '../../../../shared_libs/play-runtime/bounded-dispatch';
7
- import type { AnyBatchOperationStrategy } from '../../../../shared_libs/play-runtime/batching-types';
8
- import {
9
- buildDurableToolCallAuthScopeDigest,
10
- buildDurableToolCallCacheKey,
11
- buildDurableToolAggregateProviderIdempotencyKey,
12
- buildDurableToolAggregateReceiptKey,
13
- buildDurableToolProviderIdempotencyKey,
14
- buildDurableToolReceiptPrefix,
15
- } from '../../../../shared_libs/play-runtime/durable-call-cache';
16
- import {
17
- RuntimeReceiptWaitTimeoutError,
18
- resolveRuntimeToolReceiptWaitMaxAttempts,
19
- runtimeReceiptFailureKindForError,
20
- waitForCompletedRuntimeReceipt,
21
- } from '../../../../shared_libs/play-runtime/durable-receipt-execution';
22
- import type { PlayExecutionGovernor } from '../../../../shared_libs/play-runtime/governor/governor';
23
- import { ToolExecuteAuthScopeChangedError } from '../../../../shared_libs/play-runtime/tool-execute-retry-policy';
24
- import { getPlayRuntimeBatchStrategy } from '../../../../shared_libs/play-runtime/play-runtime-batching-registry';
25
- import {
26
- RuntimePersistenceCircuitOpenError,
27
- shouldShortenCompleteReceiptLadder,
28
- tripRuntimePersistenceLatch,
29
- type RuntimePersistenceLatch,
30
- } from '../../../../shared_libs/play-runtime/persistence-latch';
31
- import type { ReceiptSalvageBuffer } from '../../../../shared_libs/play-runtime/receipt-salvage';
32
- import type { ResolvedPacingPolicy } from '../../../../shared_libs/play-runtime/pacing';
33
- import type { RuntimeStepReceipt } from '../../../../shared_libs/play-runtime/ctx-types';
34
- import {
35
- isPlayExecutionSuspendedError,
36
- isPlayRowExecutionSuspendedError,
37
- } from '../../../../shared_libs/play-runtime/suspension';
38
- import {
39
- QUERY_RESULT_DATASET_PAGE_SIZE,
40
- isQueryResultDatasetReadRequest,
41
- type QueryResultDatasetRequest,
42
- } from '../../../../shared_libs/play-runtime/query-result-dataset';
43
- import {
44
- createToolExecuteResult,
45
- isToolExecuteResult,
46
- type ToolExecuteResult,
47
- type ToolResultMetadataInput,
48
- } from '../../../../shared_libs/play-runtime/tool-result';
49
- import { toolExecutionMetadataForOutcome } from '../../../../shared_libs/play-runtime/tool-execution-outcome';
50
- import { buildScopedWorkReceiptKey } from '../../../../shared_libs/play-runtime/work-receipts';
51
- import {
52
- canReclaimFailedWorkerToolReceipt,
53
- canReclaimTimedOutWorkerToolReceipt,
54
- markWorkerToolReceiptResultCached,
55
- markWorkerToolReceiptResultExecution,
56
- planWorkerToolReceiptGroups,
57
- resolveRunningReceiptWaitMaxAttempts,
58
- resolveWorkerToolReceiptGroupWaitMaxAttempts,
59
- resolveWorkerToolRuntimeTimeoutMs,
60
- } from './tool-receipts';
61
- import type {
62
- WorkerRuntimeReceipt,
63
- WorkerRuntimeReceiptClaim,
64
- WorkerRuntimeReceiptStore,
65
- } from './receipts';
66
- import {
67
- executeWithWorkerReceiptHeartbeats,
68
- RuntimeLeaseLostError,
69
- } from './receipts';
70
- import { WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL } from './worker-platform-budget';
71
- import type { WorkerWorkBudgetMeter } from './work-budget';
72
-
73
- export type WorkerPacingResolver = (
74
- toolId: string,
75
- ) => Promise<
76
- | (ResolvedPacingPolicy & {
77
- retrySafeTransientHttp: boolean;
78
- r: { bucketId: string; token: string } | null;
79
- })
80
- | null
81
- >;
82
-
83
- export type WorkerToolActionCacheVersionResolver = (
84
- toolId: string,
85
- ) => Promise<string>;
86
-
87
- export type WorkerToolAuthScopeDigestResolver = ((
88
- toolId: string,
89
- ) => Promise<string>) & {
90
- invalidate?: (toolId?: string | null) => void;
91
- };
92
-
93
- export type WorkerToolDispatchCallbacks = {
94
- onToolCalled?: (toolId: string, at?: number) => void;
95
- onToolFailed?: (toolId: string, at?: number) => void;
96
- };
97
-
98
- export type WorkerToolDirectOptions = {
99
- customerDbDataset?: QueryResultDatasetRequest;
100
- attachCustomerDbDataset?: boolean;
101
- };
102
-
103
- export type WorkerToolDispatchRunRequest = {
104
- runId: string;
105
- runAttempt?: number | null;
106
- orgId: string;
107
- playName: string;
108
- };
109
-
110
- export type WorkflowStepLike = object;
111
-
112
- export type WorkerToolDispatchExecuteTool = (input: {
113
- id: string;
114
- toolId: string;
115
- input: Record<string, unknown>;
116
- workflowStep?: WorkflowStepLike;
117
- callbacks?: WorkerToolDispatchCallbacks;
118
- onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>;
119
- onRetryAttempt?: () => void | Promise<void>;
120
- transientHttpRetrySafe?: boolean;
121
- durableCallReceiptKey?: string | null;
122
- executionAuthScopeDigest?: string | null;
123
- providerIdempotencyKey?: string | null;
124
- runtimeTimeoutMs?: number;
125
- directOptions?: WorkerToolDirectOptions;
126
- }) => Promise<ToolExecuteResult>;
127
-
128
- export type WorkerToolDispatchPerfRecorder = (input: {
129
- phase: string;
130
- ms?: number;
131
- extra?: Record<string, unknown>;
132
- }) => void;
133
-
134
- export type ToolDispatchStore = {
135
- receiptStore?: WorkerRuntimeReceiptStore;
136
- serializeDurableValue: <T>(value: T) => T;
137
- deserializeDurableValue: <T>(value: T) => T;
138
- allowLocalRetryReceipts?: boolean;
139
- runtimeReceiptHeartbeatIntervalMs?: number;
140
- runtimeReceiptLeaseTtlMs?: number;
141
- };
142
-
143
- export type ToolDispatchGovernor = {
144
- governor: Pick<
145
- PlayExecutionGovernor,
146
- | 'policy'
147
- | 'acquireToolSlot'
148
- | 'suggestedParallelism'
149
- | 'chargeBudget'
150
- | 'reportProviderBackpressure'
151
- >;
152
- resolvePacing: WorkerPacingResolver;
153
- budgetMeter?: WorkerWorkBudgetMeter;
154
- };
155
-
156
- export type ToolExecutor = {
157
- resolveToolActionCacheVersion: WorkerToolActionCacheVersionResolver;
158
- resolveToolAuthScopeDigest: WorkerToolAuthScopeDigestResolver;
159
- // REQUIRED: the AUTH_SCOPE_CHANGED re-claim depends on evicting the cached
160
- // digest before re-resolving. When this was optional, the root (top-level)
161
- // scheduler was constructed without it: the requeue silently reused the
162
- // stale digest, re-claimed the SAME receipt key, collided with its own
163
- // still-running receipt, and the run stalled until the whole workflow
164
- // invocation was retried minutes later. A missing wiring must be a type
165
- // error, not a 5-minute invisible stall.
166
- invalidateToolAuthScopeDigest: (toolId: string) => void;
167
- executeTool: WorkerToolDispatchExecuteTool;
168
- callbacks?: WorkerToolDispatchCallbacks;
169
- // Observability hook for the bounded AUTH_SCOPE_CHANGED re-claim: called
170
- // once per requeued group so the runner can emit a durable run-log line
171
- // (marker: auth_scope_changed_reclaim). The credential-swap transition must
172
- // be visible in run logs, not a silent continuation.
173
- onAuthScopeReclaim?: (input: {
174
- toolId: string;
175
- requestIds: string[];
176
- previousReceiptKeys: Array<string | null>;
177
- nextReceiptKeys: string[];
178
- }) => void;
179
- };
180
-
181
- export type WorkerToolBatchSchedulerOptions = ToolDispatchStore &
182
- ToolDispatchGovernor &
183
- ToolExecutor & {
184
- req: WorkerToolDispatchRunRequest;
185
- nowMs: () => number;
186
- batchGraceMs?: number;
187
- recordPerfTrace?: WorkerToolDispatchPerfRecorder;
188
- abortSignal?: AbortSignal;
189
- onRequestsSettled?: (count: number) => void;
190
- runtimeDeadlineMs?: number;
191
- persistenceLatch?: RuntimePersistenceLatch;
192
- receiptSalvage?: ReceiptSalvageBuffer;
193
- };
194
-
195
- type WorkerToolBatchRequest = {
196
- id: string;
197
- cacheKey: string;
198
- receiptKey: string | null;
199
- executionAuthScopeDigest: string;
200
- force: boolean;
201
- runForceRefresh: boolean;
202
- runForceFailedRefresh: boolean;
203
- staleAfterSeconds?: number | null;
204
- receiptWaitMaxAttempts: number;
205
- authScopeRetryUsed?: boolean;
206
- runtimeTimeoutMs?: number;
207
- toolId: string;
208
- input: Record<string, unknown>;
209
- workflowStep?: WorkflowStepLike;
210
- directOptions?: WorkerToolDirectOptions;
211
- resolve: (value: unknown) => void;
212
- reject: (error: unknown) => void;
213
- };
214
-
215
- type ClaimedWorkerToolBatchRequest = {
216
- request: WorkerToolBatchRequest;
217
- receiptKey: string | null;
218
- receiptLeaseId: string | null;
219
- receiptStatus: WorkerRuntimeReceipt['status'] | null;
220
- providerForce?: boolean;
221
- followers: WorkerToolBatchRequest[];
222
- };
223
-
224
- type PreparedWorkerToolBatchRequests = {
225
- claimedRequests: ClaimedWorkerToolBatchRequest[];
226
- deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[];
227
- };
228
-
229
- type WorkerToolCompletionResult =
230
- | { ok: true; value: unknown }
231
- | { ok: false; error: unknown };
232
-
233
- const WORKER_TOOL_BATCH_GRACE_MS = 250;
234
- const COMPLETE_RECEIPT_RETRY_BACKOFFS_MS = [1_000, 3_000, 9_000] as const;
235
- const COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS = 10_000;
236
- export const WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_BYTES = 32 * 1024 * 1024;
237
- export const WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_MEMBERS = 500;
238
- export const WORKER_TOOL_RECEIPT_RECOVERY_MAX_CONCURRENT_READS = 25;
239
-
240
- type DurableToolCompletionEntry = {
241
- index: number;
242
- claimed: ClaimedWorkerToolBatchRequest;
243
- ownerResult: unknown;
244
- receipt: {
245
- runId: string;
246
- runAttempt: number;
247
- key: string;
248
- leaseId: string | null;
249
- output: unknown;
250
- };
251
- serializedBytes: number;
252
- };
253
-
254
- type DurableToolCompletionBatch = {
255
- entries: DurableToolCompletionEntry[];
256
- bytes: number;
257
- };
258
-
259
- const jsonByteLength = (value: unknown): number =>
260
- new TextEncoder().encode(JSON.stringify(value)).byteLength;
261
-
262
- async function mapWithConcurrency<T>(
263
- values: T[],
264
- concurrency: number,
265
- run: (value: T, index: number) => Promise<void>,
266
- ): Promise<void> {
267
- const workerCount = Math.max(
268
- 1,
269
- Math.min(Math.floor(concurrency), values.length),
270
- );
271
- let nextIndex = 0;
272
- await Promise.all(
273
- Array.from({ length: workerCount }, async () => {
274
- while (true) {
275
- const index = nextIndex;
276
- nextIndex += 1;
277
- if (index >= values.length) return;
278
- await run(values[index]!, index);
279
- }
280
- }),
281
- );
282
- }
283
-
284
- async function sleepWorkerMs(ms: number): Promise<void> {
285
- await new Promise((resolve) => setTimeout(resolve, ms));
286
- }
287
-
288
- export class RuntimeReceiptPersistenceError extends Error {
289
- readonly receiptKeys: string[];
290
- readonly errors: unknown[];
291
-
292
- constructor(input: {
293
- message: string;
294
- receiptKeys: string[];
295
- errors?: unknown[];
296
- cause?: unknown;
297
- }) {
298
- super(input.message, { cause: input.cause });
299
- this.name = 'RuntimeReceiptPersistenceError';
300
- this.receiptKeys = input.receiptKeys;
301
- this.errors =
302
- input.errors ?? (input.cause === undefined ? [] : [input.cause]);
303
- }
304
- }
305
-
306
- function receiptPersistenceError(input: {
307
- operation: 'mark-running' | 'complete' | 'fail';
308
- receiptKeys: string[];
309
- cause: unknown;
310
- }): RuntimeReceiptPersistenceError {
311
- const keys = input.receiptKeys.filter((key) => key.trim().length > 0);
312
- const keySummary =
313
- keys.length === 0
314
- ? 'unknown receipt'
315
- : keys.length === 1
316
- ? keys[0]!
317
- : `${keys.length} receipts`;
318
- return new RuntimeReceiptPersistenceError({
319
- message: `Runtime receipt ${input.operation} persistence failed for ${keySummary}.`,
320
- receiptKeys: keys,
321
- cause: input.cause,
322
- });
323
- }
324
-
325
- function buildBatchDurableCallReceiptKey(input: {
326
- req: WorkerToolDispatchRunRequest;
327
- batchOperation: string;
328
- receiptKeys: string[];
329
- }): string {
330
- return buildDurableToolAggregateReceiptKey({
331
- receiptKeys: input.receiptKeys,
332
- prefix: 'batch',
333
- aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
334
- orgId: input.req.orgId,
335
- toolId: input.batchOperation,
336
- }),
337
- });
338
- }
339
-
340
- function buildBatchProviderIdempotencyKey(input: {
341
- aggregateReceiptKey: string;
342
- receiptKeys: string[];
343
- providerIdempotencyKeys: string[];
344
- }): string {
345
- return buildDurableToolAggregateProviderIdempotencyKey({
346
- aggregateReceiptKey: input.aggregateReceiptKey,
347
- receiptKeys: input.receiptKeys,
348
- providerIdempotencyKeys: input.providerIdempotencyKeys,
349
- });
350
- }
351
-
352
- function durableCallReceiptKeyForClaimed(input: {
353
- req: WorkerToolDispatchRunRequest;
354
- claimed: ClaimedWorkerToolBatchRequest;
355
- }): string {
356
- return input.claimed.receiptKey ?? input.claimed.request.cacheKey;
357
- }
358
-
359
- function providerIdempotencyKeyForClaimed(input: {
360
- req: WorkerToolDispatchRunRequest;
361
- claimed: ClaimedWorkerToolBatchRequest;
362
- }): string {
363
- const fallbackAttemptId =
364
- (input.claimed.providerForce === true || input.claimed.request.force) &&
365
- !input.claimed.receiptLeaseId
366
- ? input.claimed.request.id
367
- : null;
368
- return buildDurableToolProviderIdempotencyKey({
369
- receiptKey: durableCallReceiptKeyForClaimed(input),
370
- force: input.claimed.providerForce === true || input.claimed.request.force,
371
- receiptLeaseId: input.claimed.receiptLeaseId,
372
- fallbackAttemptId,
373
- });
374
- }
375
-
376
- export function wrapWorkerToolResult(
377
- toolId: string,
378
- result: unknown,
379
- metadata: ToolResultMetadataInput | null,
380
- status = result == null ? 'no_result' : 'completed',
381
- ): ToolExecuteResult {
382
- return createToolExecuteResult({
383
- status,
384
- result,
385
- metadata: metadata ?? { toolId },
386
- execution: toolExecutionMetadataForOutcome({ kind: 'live' }),
387
- });
388
- }
389
-
390
- function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
391
- if (toolId === 'test_rate_limit') {
392
- // Batched members resolve metadata through this fallback because the lean
393
- // worker does not bundle the catalog. It MUST mirror the same
394
- // `email_status: emailStatus({...})` contract registered in
395
- // src/lib/integrations/test/index.ts so batched results normalize to the
396
- // same rich email_status OBJECT (status from statusMap, catch_all as a
397
- // signal) that the single-execute real-metadata path produces.
398
- return {
399
- toolId,
400
- extractors: {
401
- email_status: {
402
- paths: ['email_status'],
403
- emailStatus: {
404
- provider: 'test',
405
- rawStatus: ['email_status'],
406
- catchAll: ['mx_security_gateway'],
407
- statusMap: {
408
- valid: { status: 'valid', verdict: 'send', verified: true },
409
- invalid: { status: 'invalid', verdict: 'drop', verified: false },
410
- catch_all: {
411
- status: 'catch_all',
412
- verdict: 'verify_next',
413
- verified: false,
414
- },
415
- unknown: { status: 'unknown', verdict: 'hold', verified: false },
416
- },
417
- },
418
- },
419
- },
420
- targetGetters: {
421
- email: ['email', 'value'],
422
- email_status: ['email_status'],
423
- },
424
- };
425
- }
426
- return { toolId };
427
- }
428
-
429
- function durableReceiptInputForTool(
430
- toolId: string,
431
- input: Record<string, unknown>,
432
- ): Record<string, unknown> {
433
- if (toolId !== 'test_rate_limit') {
434
- return input;
435
- }
436
- const { simulated_delay_ms: _simulatedDelayMs, ...semanticInput } = input;
437
- return semanticInput;
438
- }
439
-
440
- export function workerDurableToolCallCacheKey(input: {
441
- req: WorkerToolDispatchRunRequest;
442
- toolId: string;
443
- requestInput: Record<string, unknown>;
444
- providerActionVersion: string;
445
- executionAuthScopeDigest: string;
446
- staleAfterSeconds?: number | null;
447
- }): string {
448
- return buildDurableToolCallCacheKey({
449
- orgId: input.req.orgId,
450
- playLocalScope: input.req.playName,
451
- toolId: input.toolId,
452
- requestInput: durableReceiptInputForTool(input.toolId, input.requestInput),
453
- authScopeDigest: buildDurableToolCallAuthScopeDigest({
454
- orgId: input.req.orgId,
455
- toolId: input.toolId,
456
- executionAuthScopeDigest: input.executionAuthScopeDigest,
457
- }),
458
- providerActionVersion: input.providerActionVersion,
459
- staleAfterSeconds: input.staleAfterSeconds,
460
- });
461
- }
462
-
463
- function workerRuntimeReceiptKey(input: {
464
- req: WorkerToolDispatchRunRequest;
465
- key: string;
466
- }): string {
467
- const orgId = input.req.orgId?.trim() || 'org';
468
- if (input.key.startsWith(`ctx:${orgId}:`)) {
469
- return input.key;
470
- }
471
- return buildScopedWorkReceiptKey({
472
- orgId: input.req.orgId,
473
- playName: input.req.playName,
474
- key: input.key,
475
- });
476
- }
477
-
478
- export class WorkerToolBatchScheduler {
479
- private queue: WorkerToolBatchRequest[] = [];
480
- private scheduled = false;
481
- private drainTimer: ReturnType<typeof setTimeout> | null = null;
482
- private immediateDrainQueued = false;
483
-
484
- constructor(private readonly options: WorkerToolBatchSchedulerOptions) {}
485
-
486
- private runtimeDeadlineRemainingMs(): number {
487
- if (typeof this.options.runtimeDeadlineMs !== 'number') {
488
- return Number.POSITIVE_INFINITY;
489
- }
490
- return Math.max(0, this.options.runtimeDeadlineMs - this.options.nowMs());
491
- }
492
-
493
- private async completeReceiptWithRetryLadder<T>(
494
- run: () => Promise<T>,
495
- onGiveUp?: (attempts: number) => void,
496
- ): Promise<T> {
497
- if (!this.options.persistenceLatch && !this.options.receiptSalvage) {
498
- return await run();
499
- }
500
- const latch = this.options.persistenceLatch;
501
- let attempt = 0;
502
- let committedToFullLadder = false;
503
- const giveUp = (error: unknown): never => {
504
- if (latch) {
505
- latch.latchRetryBudgetExhausted = true;
506
- tripRuntimePersistenceLatch(latch, error);
507
- }
508
- onGiveUp?.(attempt);
509
- throw error;
510
- };
511
- while (true) {
512
- try {
513
- return await run();
514
- } catch (error) {
515
- attempt += 1;
516
- if (
517
- this.options.abortSignal?.aborted ||
518
- attempt > COMPLETE_RECEIPT_RETRY_BACKOFFS_MS.length ||
519
- (latch &&
520
- shouldShortenCompleteReceiptLadder(latch, committedToFullLadder))
521
- ) {
522
- return giveUp(error);
523
- }
524
- const backoffMs = COMPLETE_RECEIPT_RETRY_BACKOFFS_MS[attempt - 1]!;
525
- if (
526
- this.runtimeDeadlineRemainingMs() <=
527
- backoffMs + COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS
528
- ) {
529
- return giveUp(error);
530
- }
531
- committedToFullLadder = true;
532
- await sleepWorkerMs(backoffMs);
533
- }
534
- }
535
- }
536
-
537
- private recordReceiptSalvage(input: {
538
- claimed: ClaimedWorkerToolBatchRequest;
539
- output: unknown;
540
- attempts: number;
541
- }): void {
542
- if (!this.options.receiptSalvage || !input.claimed.receiptKey) return;
543
- this.options.receiptSalvage.push({
544
- receiptKey: input.claimed.receiptKey,
545
- toolId: input.claimed.request.toolId,
546
- cacheKey: input.claimed.request.cacheKey,
547
- output: input.output,
548
- completeAttempts: input.attempts,
549
- firstErrorAt: this.options.nowMs(),
550
- });
551
- }
552
-
553
- private runtimePersistenceCircuitOpenError(
554
- skippedCalls: number,
555
- ): Error | null {
556
- const latch = this.options.persistenceLatch;
557
- if (!latch?.tripped) return null;
558
- latch.preventedCallCount += Math.max(1, skippedCalls);
559
- return new RuntimePersistenceCircuitOpenError(latch);
560
- }
561
-
562
- /**
563
- * Report a provider 429 / Retry-After back into the Governor's shared pacer
564
- * so future acquires for this (org, provider) bucket back off across all
565
- * isolates. Provider comes from the same pacing resolver the Governor uses
566
- * (the worker has no local catalog), so callers pass only the toolId.
567
- */
568
- private async reportBackpressure(
569
- toolId: string,
570
- retryAfterMs: number,
571
- ): Promise<void> {
572
- if (retryAfterMs <= 0) return;
573
- const pacing = await this.options.resolvePacing(toolId).catch(() => null);
574
- if (pacing?.provider) {
575
- await this.options.governor.reportProviderBackpressure({
576
- provider: pacing.provider,
577
- toolId,
578
- retryAfterMs,
579
- });
580
- }
581
- }
582
-
583
- async execute(
584
- id: string,
585
- toolId: string,
586
- input: Record<string, unknown>,
587
- workflowStep?: WorkflowStepLike,
588
- options?: {
589
- force?: boolean;
590
- runForceRefresh?: boolean;
591
- runForceFailedRefresh?: boolean;
592
- staleAfterSeconds?: number | null;
593
- timeoutMs?: number;
594
- receiptWaitMs?: number;
595
- },
596
- ): Promise<unknown> {
597
- this.options.budgetMeter?.count('tool');
598
- const [providerActionVersion, executionAuthScopeDigest] = await Promise.all(
599
- [
600
- this.options.resolveToolActionCacheVersion(toolId),
601
- this.options.resolveToolAuthScopeDigest(toolId),
602
- ],
603
- );
604
- return await new Promise((resolve, reject) => {
605
- const queuedAt = this.options.nowMs();
606
- const receiptKey = workerDurableToolCallCacheKey({
607
- req: this.options.req,
608
- toolId,
609
- requestInput: input,
610
- providerActionVersion,
611
- executionAuthScopeDigest,
612
- staleAfterSeconds: options?.staleAfterSeconds,
613
- });
614
- const queryResultDatasetRead = isQueryResultDatasetReadRequest(
615
- toolId,
616
- input,
617
- );
618
- const receiptEligible = !queryResultDatasetRead;
619
- this.queue.push({
620
- id,
621
- cacheKey: receiptEligible ? receiptKey : '',
622
- receiptKey: receiptEligible ? receiptKey : null,
623
- executionAuthScopeDigest,
624
- force: options?.force === true,
625
- runForceRefresh: options?.runForceRefresh === true,
626
- runForceFailedRefresh: options?.runForceFailedRefresh === true,
627
- staleAfterSeconds: options?.staleAfterSeconds ?? null,
628
- receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
629
- typeof options?.receiptWaitMs === 'number'
630
- ? { max_wait_ms: options.receiptWaitMs }
631
- : input,
632
- ),
633
- runtimeTimeoutMs: options?.timeoutMs,
634
- toolId,
635
- input,
636
- workflowStep,
637
- ...(queryResultDatasetRead
638
- ? {
639
- directOptions: {
640
- customerDbDataset: {
641
- offset: 0,
642
- pageSize: QUERY_RESULT_DATASET_PAGE_SIZE,
643
- },
644
- },
645
- }
646
- : {}),
647
- resolve: (value) => {
648
- this.options.recordPerfTrace?.({
649
- phase: 'runner.tool.request',
650
- ms: this.options.nowMs() - queuedAt,
651
- extra: { id, toolId },
652
- });
653
- resolve(value);
654
- },
655
- reject,
656
- });
657
- this.scheduleDrain();
658
- });
659
- }
660
-
661
- private scheduleDrain(): void {
662
- if (this.shouldDrainQueuedToolsImmediately()) {
663
- this.scheduleImmediateDrain();
664
- return;
665
- }
666
- if (this.scheduled) return;
667
- this.scheduled = true;
668
- this.drainTimer = setTimeout(() => {
669
- this.drainTimer = null;
670
- this.scheduleImmediateDrain();
671
- }, this.options.batchGraceMs ?? WORKER_TOOL_BATCH_GRACE_MS);
672
- }
673
-
674
- private scheduleImmediateDrain(): void {
675
- if (this.drainTimer) {
676
- clearTimeout(this.drainTimer);
677
- this.drainTimer = null;
678
- }
679
- if (this.immediateDrainQueued) return;
680
- this.scheduled = true;
681
- this.immediateDrainQueued = true;
682
- queueMicrotask(() => {
683
- this.immediateDrainQueued = false;
684
- void this.drain().catch((error) => {
685
- const queued = this.queue;
686
- this.queue = [];
687
- this.scheduled = false;
688
- if (queued.length > 0) {
689
- this.rejectRawRequests(queued, error);
690
- }
691
- });
692
- });
693
- }
694
-
695
- private shouldDrainQueuedToolsImmediately(): boolean {
696
- if (!this.hasBatchableQueuedTool()) return true;
697
- const byTool = new Map<string, number>();
698
- for (const request of this.queue) {
699
- if (request.toolId === 'test_wait_for_event') continue;
700
- const strategy = getPlayRuntimeBatchStrategy(request.toolId);
701
- if (!strategy) continue;
702
- const nextCount = (byTool.get(request.toolId) ?? 0) + 1;
703
- if (nextCount >= strategy.maxBatchSize) {
704
- return true;
705
- }
706
- byTool.set(request.toolId, nextCount);
707
- }
708
- return false;
709
- }
710
-
711
- private hasBatchableQueuedTool(): boolean {
712
- return this.queue.some(
713
- (request) =>
714
- request.toolId !== 'test_wait_for_event' &&
715
- getPlayRuntimeBatchStrategy(request.toolId) !== null,
716
- );
717
- }
718
-
719
- private async drain(): Promise<void> {
720
- const requests = this.queue;
721
- this.queue = [];
722
- this.scheduled = false;
723
- const drainStartedAt = this.options.nowMs();
724
- await Promise.all(
725
- [...groupWorkerToolRequestsByTool(requests).entries()].map(
726
- async ([toolId, groupedRequests]) => {
727
- try {
728
- await this.executeToolGroup(toolId, groupedRequests);
729
- } catch (error) {
730
- this.rejectRawRequests(groupedRequests, error);
731
- }
732
- },
733
- ),
734
- );
735
- this.options.recordPerfTrace?.({
736
- phase: 'runner.tool.drain',
737
- ms: this.options.nowMs() - drainStartedAt,
738
- extra: { requests: requests.length },
739
- });
740
- if (this.queue.length > 0) {
741
- this.scheduleDrain();
742
- }
743
- }
744
-
745
- private async waitForDurableToolReceipt(input: {
746
- receiptKey: string;
747
- maxAttempts: number;
748
- }): Promise<unknown> {
749
- if (!this.options.receiptStore) {
750
- throw new Error('No receipt store.');
751
- }
752
- if (!this.options.receiptStore.getReceipt) {
753
- throw new Error('No receipt lookup.');
754
- }
755
- this.options.budgetMeter?.count('receipt');
756
- const receipt = await waitForCompletedRuntimeReceipt({
757
- receiptKey: input.receiptKey,
758
- maxAttempts: input.maxAttempts,
759
- abortSignal: this.options.abortSignal,
760
- store: {
761
- getMany: async (receiptKeys) => {
762
- const receipts = new Map<string, RuntimeStepReceipt>();
763
- await Promise.all(
764
- receiptKeys.map(async (key) => {
765
- const receipt = await this.options.receiptStore!.getReceipt!({
766
- playName: this.options.req.playName,
767
- key,
768
- });
769
- if (!receipt) return;
770
- receipts.set(key, {
771
- key: receipt.key,
772
- status: receipt.status,
773
- output: receipt.output,
774
- error: receipt.error ?? undefined,
775
- runId: receipt.runId ?? null,
776
- });
777
- }),
778
- );
779
- return receipts;
780
- },
781
- },
782
- });
783
- return markWorkerToolReceiptResultCached(
784
- this.options.deserializeDurableValue(receipt.output),
785
- input.receiptKey,
786
- input.receiptKey,
787
- );
788
- }
789
-
790
- private settleRequests(
791
- claimed: ClaimedWorkerToolBatchRequest,
792
- result: unknown,
793
- ): void {
794
- claimed.request.resolve(result);
795
- for (const follower of claimed.followers) {
796
- follower.resolve(
797
- claimed.receiptKey
798
- ? markWorkerToolReceiptResultExecution(result, {
799
- kind: 'in_flight',
800
- receiptKey: claimed.receiptKey,
801
- attachedToReceiptKey: claimed.receiptKey,
802
- })
803
- : result,
804
- );
805
- }
806
- this.options.onRequestsSettled?.(1 + claimed.followers.length);
807
- }
808
-
809
- private rejectRequests(
810
- claimed: ClaimedWorkerToolBatchRequest,
811
- error: unknown,
812
- ): void {
813
- claimed.request.reject(error);
814
- for (const follower of claimed.followers) {
815
- follower.reject(error);
816
- }
817
- this.options.onRequestsSettled?.(1 + claimed.followers.length);
818
- }
819
-
820
- private rejectRawRequests(
821
- requests: WorkerToolBatchRequest[],
822
- error: unknown,
823
- ): void {
824
- for (const request of requests) {
825
- request.reject(error);
826
- }
827
- this.options.onRequestsSettled?.(requests.length);
828
- }
829
-
830
- private async requeueAfterAuthScopeChanged(
831
- requests: WorkerToolBatchRequest[],
832
- error: unknown,
833
- ): Promise<boolean> {
834
- if (!(error instanceof ToolExecuteAuthScopeChangedError)) {
835
- return false;
836
- }
837
- const [first] = requests;
838
- if (!first) return false;
839
- if (requests.some((request) => request.authScopeRetryUsed === true)) {
840
- return false;
841
- }
842
- this.options.invalidateToolAuthScopeDigest(first.toolId);
843
- const [providerActionVersion, executionAuthScopeDigest] = await Promise.all(
844
- [
845
- this.options.resolveToolActionCacheVersion(first.toolId),
846
- this.options.resolveToolAuthScopeDigest(first.toolId),
847
- ],
848
- );
849
- const requeued = requests.map((request) => ({
850
- request,
851
- receiptKey: workerDurableToolCallCacheKey({
852
- req: this.options.req,
853
- toolId: request.toolId,
854
- requestInput: request.input,
855
- providerActionVersion,
856
- executionAuthScopeDigest,
857
- staleAfterSeconds: request.staleAfterSeconds,
858
- }),
859
- }));
860
- // If the re-resolved scope produced the exact same receipt keys, the
861
- // credential identity did NOT actually change from this isolate's point of
862
- // view (stale cache, no-op invalidation, or a racing revert). Requeuing
863
- // would claim the same key this group just left `running` and stall on its
864
- // own abandoned receipt until the lease TTL. Refuse instead: the caller
865
- // rejects the request with the typed 409 error — loud and immediate.
866
- if (
867
- requeued.every(
868
- (entry) => entry.receiptKey === (entry.request.receiptKey ?? null),
869
- )
870
- ) {
871
- return false;
872
- }
873
- for (const { request, receiptKey } of requeued) {
874
- this.queue.push({
875
- ...request,
876
- cacheKey: receiptKey,
877
- receiptKey,
878
- executionAuthScopeDigest,
879
- authScopeRetryUsed: true,
880
- });
881
- }
882
- this.options.onAuthScopeReclaim?.({
883
- toolId: first.toolId,
884
- requestIds: requests.map((request) => request.id),
885
- previousReceiptKeys: requests.map(
886
- (request) => request.receiptKey ?? null,
887
- ),
888
- nextReceiptKeys: requeued.map((entry) => entry.receiptKey),
889
- });
890
- this.scheduleDrain();
891
- return true;
892
- }
893
-
894
- private async reclaimTimedOutDurableToolReceiptGroup(input: {
895
- group: WorkerToolBatchRequest[];
896
- receiptKey: string;
897
- runningReceipt: WorkerRuntimeReceipt;
898
- waitError: unknown;
899
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
900
- const [request, ...followers] = input.group;
901
- if (!request || !this.options.receiptStore) {
902
- this.rejectRawRequests(input.group, input.waitError);
903
- return [];
904
- }
905
- if (
906
- !canReclaimTimedOutWorkerToolReceipt({
907
- ownerRunId: input.runningReceipt.runId,
908
- currentRunId: this.options.req.runId,
909
- })
910
- ) {
911
- this.rejectRawRequests(input.group, input.waitError);
912
- return [];
913
- }
914
- let claim: WorkerRuntimeReceiptClaim;
915
- try {
916
- this.options.budgetMeter?.count('receipt');
917
- claim = await this.options.receiptStore.claimReceipt({
918
- playName: this.options.req.playName,
919
- runId: this.options.req.runId,
920
- runAttempt: this.options.req.runAttempt ?? 0,
921
- key: input.receiptKey,
922
- leaseAware: true,
923
- leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
924
- reclaimRunning: true,
925
- });
926
- } catch (error) {
927
- this.rejectRawRequests(input.group, error);
928
- return [];
929
- }
930
- if (claim.disposition === 'claimed') {
931
- return [
932
- {
933
- request,
934
- receiptKey: input.receiptKey,
935
- receiptLeaseId: claim.receipt.leaseId ?? null,
936
- receiptStatus: claim.receipt.status,
937
- providerForce: true,
938
- followers,
939
- },
940
- ];
941
- }
942
- if (claim.disposition === 'reused') {
943
- const result = markWorkerToolReceiptResultCached(
944
- this.options.deserializeDurableValue(claim.receipt.output),
945
- request.cacheKey,
946
- input.receiptKey,
947
- );
948
- for (const pending of input.group) {
949
- pending.resolve(result);
950
- }
951
- this.options.onRequestsSettled?.(input.group.length);
952
- return [];
953
- }
954
- if (claim.disposition === 'failed') {
955
- this.rejectRawRequests(
956
- input.group,
957
- new Error(
958
- `Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
959
- ),
960
- );
961
- return [];
962
- }
963
- this.rejectRawRequests(input.group, input.waitError);
964
- return [];
965
- }
966
-
967
- private async claimForcedDurableToolReceiptGroupAfterWait(input: {
968
- group: WorkerToolBatchRequest[];
969
- receiptKey: string;
970
- }): Promise<ClaimedWorkerToolBatchRequest[]> {
971
- const [request, ...followers] = input.group;
972
- if (!request || !this.options.receiptStore) {
973
- this.rejectRawRequests(
974
- input.group,
975
- new Error(
976
- `No receipt store for forced durable tool call ${input.receiptKey}.`,
977
- ),
978
- );
979
- return [];
980
- }
981
- let claim: WorkerRuntimeReceiptClaim;
982
- try {
983
- this.options.budgetMeter?.count('receipt');
984
- claim = await this.options.receiptStore.claimReceipt({
985
- playName: this.options.req.playName,
986
- runId: this.options.req.runId,
987
- runAttempt: this.options.req.runAttempt ?? 0,
988
- key: input.receiptKey,
989
- leaseAware: true,
990
- leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
991
- forceRefresh: true,
992
- reclaimRunning: true,
993
- });
994
- } catch (error) {
995
- this.rejectRawRequests(input.group, error);
996
- return [];
997
- }
998
- if (claim.disposition === 'claimed') {
999
- return [
1000
- {
1001
- request,
1002
- receiptKey: input.receiptKey,
1003
- receiptLeaseId: claim.receipt.leaseId ?? null,
1004
- receiptStatus: claim.receipt.status,
1005
- providerForce: true,
1006
- followers,
1007
- },
1008
- ];
1009
- }
1010
- const error =
1011
- claim.disposition === 'failed'
1012
- ? new Error(
1013
- `Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
1014
- )
1015
- : new Error(
1016
- `Forced durable tool call ${input.receiptKey} could not claim execution ownership after the active lease resolved; disposition=${claim.disposition}.`,
1017
- );
1018
- this.rejectRawRequests(input.group, error);
1019
- return [];
1020
- }
1021
-
1022
- private async failureForRejectedToolRequest(
1023
- claimed: ClaimedWorkerToolBatchRequest,
1024
- error: unknown,
1025
- ): Promise<unknown> {
1026
- try {
1027
- if (
1028
- isPlayExecutionSuspendedError(error) ||
1029
- isPlayRowExecutionSuspendedError(error)
1030
- ) {
1031
- await this.releaseDurableToolRequest(claimed, error);
1032
- } else {
1033
- await this.failDurableToolRequest(claimed, error);
1034
- }
1035
- return error;
1036
- } catch (receiptError) {
1037
- return receiptPersistenceError({
1038
- operation: 'fail',
1039
- receiptKeys: claimed.receiptKey ? [claimed.receiptKey] : [],
1040
- cause: new AggregateError(
1041
- [error, receiptError],
1042
- 'Receipt fail mark failed',
1043
- ),
1044
- });
1045
- }
1046
- }
1047
-
1048
- private async claimDurableToolReceiptGroups(
1049
- groups: ReturnType<
1050
- typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
1051
- >['durableGroups'],
1052
- ): Promise<
1053
- Array<{
1054
- group: ReturnType<
1055
- typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
1056
- >['durableGroups'][number];
1057
- receiptKey: string;
1058
- claim: WorkerRuntimeReceiptClaim;
1059
- }>
1060
- > {
1061
- if (!this.options.receiptStore) return [];
1062
- const planned = groups.map((group) => ({
1063
- group,
1064
- receiptKey: workerRuntimeReceiptKey({
1065
- req: this.options.req,
1066
- key: group.claimableReceiptKey,
1067
- }),
1068
- }));
1069
- const claimOne = async (entry: (typeof planned)[number]) => {
1070
- this.options.budgetMeter?.count('receipt');
1071
- return {
1072
- ...entry,
1073
- claim: await this.options.receiptStore!.claimReceipt({
1074
- playName: this.options.req.playName,
1075
- runId: this.options.req.runId,
1076
- runAttempt: this.options.req.runAttempt ?? 0,
1077
- key: entry.receiptKey,
1078
- leaseAware: true,
1079
- leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
1080
- ...(entry.group.forceDurableRefresh
1081
- ? { forceRefresh: true }
1082
- : entry.group.forceFailedDurableRefresh
1083
- ? { forceFailedRefresh: true }
1084
- : {}),
1085
- }),
1086
- };
1087
- };
1088
- if (!this.options.receiptStore.claimReceipts) {
1089
- return await Promise.all(planned.map(claimOne));
1090
- }
1091
- const claimed: Array<{
1092
- group: (typeof planned)[number]['group'];
1093
- receiptKey: string;
1094
- claim: WorkerRuntimeReceiptClaim;
1095
- }> = [];
1096
- for (const claimMode of ['normal', 'failed', 'full'] as const) {
1097
- const entries = planned.filter((entry) =>
1098
- claimMode === 'full'
1099
- ? entry.group.forceDurableRefresh
1100
- : claimMode === 'failed'
1101
- ? !entry.group.forceDurableRefresh &&
1102
- entry.group.forceFailedDurableRefresh
1103
- : !entry.group.forceDurableRefresh &&
1104
- !entry.group.forceFailedDurableRefresh,
1105
- );
1106
- if (entries.length === 0) continue;
1107
- // Bulk claim is a single service-binding RPC regardless of key count, so
1108
- // charge one subrequest per RPC issued (once per claimMode page), not one
1109
- // per receipt key.
1110
- this.options.budgetMeter?.count('receipt');
1111
- const claims = await this.options.receiptStore.claimReceipts({
1112
- playName: this.options.req.playName,
1113
- runId: this.options.req.runId,
1114
- runAttempt: this.options.req.runAttempt ?? 0,
1115
- keys: entries.map((entry) => entry.receiptKey),
1116
- leaseAware: true,
1117
- leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
1118
- ...(claimMode === 'full'
1119
- ? { forceRefresh: true }
1120
- : claimMode === 'failed'
1121
- ? { forceFailedRefresh: true }
1122
- : {}),
1123
- });
1124
- entries.forEach((entry, index) => {
1125
- const claim = claims[index];
1126
- if (!claim) {
1127
- throw new Error(
1128
- `Runtime receipt batch claim did not return receipt ${entry.receiptKey}.`,
1129
- );
1130
- }
1131
- claimed.push({ ...entry, claim });
1132
- });
1133
- }
1134
- return claimed;
1135
- }
1136
-
1137
- private async prepareDurableToolRequests(
1138
- requests: WorkerToolBatchRequest[],
1139
- ): Promise<PreparedWorkerToolBatchRequests> {
1140
- if (!this.options.receiptStore) {
1141
- return {
1142
- claimedRequests: requests.map((request) => ({
1143
- request,
1144
- receiptKey: null,
1145
- receiptLeaseId: null,
1146
- receiptStatus: null,
1147
- followers: [],
1148
- })),
1149
- deferredClaimedRequests: [],
1150
- };
1151
- }
1152
-
1153
- const claimedRequests: ClaimedWorkerToolBatchRequest[] = [];
1154
- const deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[] =
1155
- [];
1156
- const receiptGroupPlan = planWorkerToolReceiptGroups(requests, {
1157
- allowLocalRetryReceipts: this.options.allowLocalRetryReceipts === true,
1158
- getReceiptInput: (request) => ({
1159
- durableReceiptKey: request.receiptKey,
1160
- localRetryCacheKey: request.cacheKey,
1161
- force: request.force || request.runForceRefresh,
1162
- forceFailed: request.runForceFailedRefresh,
1163
- }),
1164
- });
1165
- for (const request of receiptGroupPlan.localRequests) {
1166
- claimedRequests.push({
1167
- request,
1168
- receiptKey: null,
1169
- receiptLeaseId: null,
1170
- receiptStatus: null,
1171
- followers: [],
1172
- });
1173
- }
1174
-
1175
- try {
1176
- const claimEntries = await this.claimDurableToolReceiptGroups(
1177
- receiptGroupPlan.durableGroups,
1178
- );
1179
- for (const { group: groupState, receiptKey, claim } of claimEntries) {
1180
- const group = groupState.requests;
1181
- const [first] = group;
1182
- if (!first) continue;
1183
- if (
1184
- claim.disposition === 'failed' &&
1185
- !groupState.forceDurableRefresh &&
1186
- group.some((request) => request.runForceRefresh === true)
1187
- ) {
1188
- deferredClaimedRequests.push(
1189
- this.claimForcedDurableToolReceiptGroupAfterWait({
1190
- group,
1191
- receiptKey,
1192
- }),
1193
- );
1194
- continue;
1195
- }
1196
- if (claim.disposition === 'reused') {
1197
- const result = markWorkerToolReceiptResultCached(
1198
- this.options.deserializeDurableValue(claim.receipt.output),
1199
- first.cacheKey,
1200
- receiptKey,
1201
- );
1202
- for (const request of group) {
1203
- request.resolve(result);
1204
- }
1205
- this.options.onRequestsSettled?.(group.length);
1206
- continue;
1207
- }
1208
- if (claim.disposition === 'failed') {
1209
- if (
1210
- canReclaimFailedWorkerToolReceipt({
1211
- error: claim.receipt.error ?? null,
1212
- })
1213
- ) {
1214
- deferredClaimedRequests.push(
1215
- this.claimForcedDurableToolReceiptGroupAfterWait({
1216
- group,
1217
- receiptKey,
1218
- }),
1219
- );
1220
- continue;
1221
- }
1222
- this.rejectRawRequests(
1223
- group,
1224
- new Error(
1225
- `Durable tool call ${receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
1226
- ),
1227
- );
1228
- continue;
1229
- }
1230
- if (
1231
- claim.disposition === 'running' ||
1232
- claim.disposition === 'blocked_active_lease'
1233
- ) {
1234
- deferredClaimedRequests.push(
1235
- (async (): Promise<ClaimedWorkerToolBatchRequest[]> => {
1236
- let waitError: unknown = new RuntimeReceiptWaitTimeoutError(
1237
- receiptKey,
1238
- );
1239
- try {
1240
- const result = await this.waitForDurableToolReceipt({
1241
- receiptKey,
1242
- maxAttempts: resolveRunningReceiptWaitMaxAttempts({
1243
- ownerRunId: claim.receipt.runId,
1244
- currentRunId: this.options.req.runId,
1245
- updatedAt: claim.receipt.updatedAt,
1246
- requestedMaxAttempts:
1247
- resolveWorkerToolReceiptGroupWaitMaxAttempts(
1248
- group,
1249
- (request) => request.receiptWaitMaxAttempts,
1250
- ),
1251
- nowMs: this.options.nowMs(),
1252
- }),
1253
- });
1254
- if (groupState.forceDurableRefresh) {
1255
- return await this.claimForcedDurableToolReceiptGroupAfterWait(
1256
- {
1257
- group,
1258
- receiptKey,
1259
- },
1260
- );
1261
- }
1262
- for (const request of group) {
1263
- request.resolve(
1264
- markWorkerToolReceiptResultExecution(result, {
1265
- kind: 'in_flight',
1266
- receiptKey,
1267
- attachedToReceiptKey: receiptKey,
1268
- }),
1269
- );
1270
- }
1271
- this.options.onRequestsSettled?.(group.length);
1272
- return [];
1273
- } catch (error) {
1274
- waitError = error;
1275
- if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
1276
- this.rejectRawRequests(group, error);
1277
- return [];
1278
- }
1279
- }
1280
- return await this.reclaimTimedOutDurableToolReceiptGroup({
1281
- group,
1282
- receiptKey,
1283
- runningReceipt: claim.receipt,
1284
- waitError,
1285
- });
1286
- })(),
1287
- );
1288
- continue;
1289
- }
1290
- const [request, ...followers] = group;
1291
- if (!request) continue;
1292
- claimedRequests.push({
1293
- request,
1294
- receiptKey,
1295
- receiptLeaseId: claim.receipt.leaseId ?? null,
1296
- receiptStatus: claim.receipt.status,
1297
- providerForce: groupState.forceDurableRefresh,
1298
- followers,
1299
- });
1300
- }
1301
- } catch (error) {
1302
- for (const group of receiptGroupPlan.durableGroups) {
1303
- this.rejectRawRequests(group.requests, error);
1304
- }
1305
- }
1306
- return { claimedRequests, deferredClaimedRequests };
1307
- }
1308
-
1309
- private async completeDurableToolRequest(
1310
- claimed: ClaimedWorkerToolBatchRequest,
1311
- result: unknown,
1312
- ): Promise<unknown> {
1313
- if (!this.options.receiptStore || !claimed.receiptKey) {
1314
- return result;
1315
- }
1316
- const ownerResult = markWorkerToolReceiptResultExecution(result, {
1317
- kind: 'live',
1318
- receiptKey: claimed.receiptKey,
1319
- });
1320
- this.options.budgetMeter?.count('receipt');
1321
- let completed: WorkerRuntimeReceipt | null;
1322
- try {
1323
- let completeAttempts = 0;
1324
- const output = this.options.serializeDurableValue(ownerResult);
1325
- completed = await this.completeReceiptWithRetryLadder(
1326
- async () => {
1327
- completeAttempts += 1;
1328
- return await this.options.receiptStore!.completeReceipt({
1329
- playName: this.options.req.playName,
1330
- runId: this.options.req.runId,
1331
- runAttempt: this.options.req.runAttempt ?? 0,
1332
- key: claimed.receiptKey!,
1333
- leaseId: claimed.receiptLeaseId,
1334
- output,
1335
- });
1336
- },
1337
- (attempts) =>
1338
- this.recordReceiptSalvage({
1339
- claimed,
1340
- output,
1341
- attempts: Math.max(attempts, completeAttempts),
1342
- }),
1343
- );
1344
- } catch (error) {
1345
- throw receiptPersistenceError({
1346
- operation: 'complete',
1347
- receiptKeys: [claimed.receiptKey],
1348
- cause: error,
1349
- });
1350
- }
1351
- const completedResult = this.completedDurableToolRequestResult({
1352
- claimed,
1353
- ownerResult,
1354
- completed,
1355
- });
1356
- if (completedResult.ok) return completedResult.value;
1357
- throw completedResult.error;
1358
- }
1359
-
1360
- private async markAdmittedToolRequestsRunning(
1361
- claimedRequests: ClaimedWorkerToolBatchRequest[],
1362
- ): Promise<void> {
1363
- const queued = claimedRequests.filter(
1364
- (claimed) =>
1365
- claimed.receiptKey !== null && claimed.receiptStatus !== 'running',
1366
- );
1367
- if (queued.length === 0) return;
1368
-
1369
- const markReceiptsRunning = this.options.receiptStore?.markReceiptsRunning;
1370
- const receiptKeys = queued.map((claimed) => claimed.receiptKey!);
1371
- if (!markReceiptsRunning) {
1372
- throw receiptPersistenceError({
1373
- operation: 'mark-running',
1374
- receiptKeys,
1375
- cause: new Error(
1376
- 'Queued runtime receipts require a fenced mark-running store operation.',
1377
- ),
1378
- });
1379
- }
1380
-
1381
- const runId = this.options.req.runId;
1382
- const runAttempt = this.options.req.runAttempt ?? 0;
1383
- this.options.budgetMeter?.count('receipt');
1384
- let marked: Array<WorkerRuntimeReceipt | null>;
1385
- try {
1386
- marked = await markReceiptsRunning({
1387
- playName: this.options.req.playName,
1388
- leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
1389
- receipts: queued.map((claimed) => ({
1390
- runId,
1391
- runAttempt,
1392
- key: claimed.receiptKey!,
1393
- leaseId: claimed.receiptLeaseId,
1394
- })),
1395
- });
1396
- } catch (error) {
1397
- throw receiptPersistenceError({
1398
- operation: 'mark-running',
1399
- receiptKeys,
1400
- cause: error,
1401
- });
1402
- }
1403
-
1404
- for (let index = 0; index < queued.length; index += 1) {
1405
- const claimed = queued[index]!;
1406
- const receipt = marked[index];
1407
- const leaseId = claimed.receiptLeaseId?.trim() || null;
1408
- const ownerRunId = receipt?.leaseOwnerRunId ?? receipt?.runId ?? null;
1409
- if (
1410
- !receipt ||
1411
- receipt.key !== claimed.receiptKey ||
1412
- receipt.status !== 'running' ||
1413
- !leaseId ||
1414
- receipt.leaseId !== leaseId ||
1415
- ownerRunId !== runId ||
1416
- receipt.leaseOwnerAttempt !== runAttempt
1417
- ) {
1418
- throw new RuntimeLeaseLostError({
1419
- keys: [claimed.receiptKey!],
1420
- runId,
1421
- leaseId: leaseId ?? 'missing',
1422
- });
1423
- }
1424
- }
1425
-
1426
- for (const claimed of queued) {
1427
- claimed.receiptStatus = 'running';
1428
- }
1429
- }
1430
-
1431
- private completedDurableToolRequestResult(input: {
1432
- claimed: ClaimedWorkerToolBatchRequest;
1433
- ownerResult: unknown;
1434
- completed: WorkerRuntimeReceipt | null | undefined;
1435
- }): WorkerToolCompletionResult {
1436
- const { claimed, completed, ownerResult } = input;
1437
- if (
1438
- completed &&
1439
- (completed.status === 'completed' || completed.status === 'skipped') &&
1440
- completed.output !== undefined
1441
- ) {
1442
- const recovered = this.options.deserializeDurableValue(completed.output);
1443
- return {
1444
- ok: true,
1445
- value:
1446
- completed.runId && completed.runId !== this.options.req.runId
1447
- ? markWorkerToolReceiptResultCached(
1448
- recovered,
1449
- claimed.request.cacheKey,
1450
- claimed.receiptKey ?? claimed.request.cacheKey,
1451
- )
1452
- : recovered,
1453
- };
1454
- }
1455
- if (completed?.status === 'failed') {
1456
- const receiptKey = claimed.receiptKey ?? claimed.request.cacheKey;
1457
- return {
1458
- ok: false,
1459
- error: receiptPersistenceError({
1460
- operation: 'complete',
1461
- receiptKeys: [receiptKey],
1462
- cause: completed.error
1463
- ? new Error(completed.error)
1464
- : new Error(
1465
- `Durable tool call ${receiptKey} recovered failed receipt completion without an error message.`,
1466
- ),
1467
- }),
1468
- };
1469
- }
1470
- return {
1471
- ok: true,
1472
- value: ownerResult,
1473
- };
1474
- }
1475
-
1476
- private async recoverCompletedDurableToolRequestResult(input: {
1477
- claimed: ClaimedWorkerToolBatchRequest;
1478
- ownerResult: unknown;
1479
- completed: WorkerRuntimeReceipt | null | undefined;
1480
- completeAttempts: number;
1481
- }): Promise<WorkerToolCompletionResult> {
1482
- const completedResult = this.completedDurableToolRequestResult(input);
1483
- if (completedResult.ok) {
1484
- return completedResult;
1485
- }
1486
-
1487
- const receiptKey = input.claimed.receiptKey;
1488
- if (!receiptKey || !this.options.receiptStore?.getReceipt) {
1489
- return completedResult;
1490
- }
1491
-
1492
- let receipt: WorkerRuntimeReceipt | null = null;
1493
- try {
1494
- this.options.budgetMeter?.count('receipt');
1495
- receipt = await this.options.receiptStore.getReceipt({
1496
- playName: this.options.req.playName,
1497
- key: receiptKey,
1498
- });
1499
- } catch (readError) {
1500
- this.recordReceiptSalvage({
1501
- claimed: input.claimed,
1502
- output: this.options.serializeDurableValue(input.ownerResult),
1503
- attempts: input.completeAttempts,
1504
- });
1505
- return {
1506
- ok: false,
1507
- error: receiptPersistenceError({
1508
- operation: 'complete',
1509
- receiptKeys: [receiptKey],
1510
- cause: new AggregateError(
1511
- [completedResult.error, readError],
1512
- 'Receipt completion returned no terminal receipt and recovery read failed',
1513
- ),
1514
- }),
1515
- };
1516
- }
1517
-
1518
- if (this.canCompleteRecoveredRunningReceipt(input.claimed, receipt)) {
1519
- const output = this.options.serializeDurableValue(input.ownerResult);
1520
- try {
1521
- this.options.budgetMeter?.count('receipt');
1522
- const completed = await this.completeReceiptWithRetryLadder(async () =>
1523
- this.options.receiptStore!.completeReceipt({
1524
- playName: this.options.req.playName,
1525
- runId: this.options.req.runId,
1526
- runAttempt: this.options.req.runAttempt ?? 0,
1527
- key: receiptKey,
1528
- leaseId: input.claimed.receiptLeaseId,
1529
- output,
1530
- }),
1531
- );
1532
- return this.completedDurableToolRequestResult({
1533
- claimed: input.claimed,
1534
- ownerResult: input.ownerResult,
1535
- completed,
1536
- });
1537
- } catch (completeError) {
1538
- this.recordReceiptSalvage({
1539
- claimed: input.claimed,
1540
- output,
1541
- attempts: input.completeAttempts,
1542
- });
1543
- return {
1544
- ok: false,
1545
- error: receiptPersistenceError({
1546
- operation: 'complete',
1547
- receiptKeys: [receiptKey],
1548
- cause: new AggregateError(
1549
- [completedResult.error, completeError],
1550
- 'Receipt completion returned no terminal receipt and owned receipt recovery failed',
1551
- ),
1552
- }),
1553
- };
1554
- }
1555
- }
1556
-
1557
- // Bulk complete can persist terminal receipts, throw once, then return null
1558
- // on retry because ownership is already gone. The durable row is the source
1559
- // of truth for that transition; read it before treating the call as lost.
1560
- const recoveredResult = this.completedDurableToolRequestResult({
1561
- claimed: input.claimed,
1562
- ownerResult: input.ownerResult,
1563
- completed: receipt,
1564
- });
1565
- if (recoveredResult.ok) {
1566
- return recoveredResult;
1567
- }
1568
-
1569
- this.recordReceiptSalvage({
1570
- claimed: input.claimed,
1571
- output: this.options.serializeDurableValue(input.ownerResult),
1572
- attempts: input.completeAttempts,
1573
- });
1574
- if (receipt?.status === 'failed') {
1575
- return {
1576
- ok: false,
1577
- error: receiptPersistenceError({
1578
- operation: 'complete',
1579
- receiptKeys: [receiptKey],
1580
- cause: receipt.error
1581
- ? new AggregateError(
1582
- [completedResult.error, new Error(receipt.error)],
1583
- 'Receipt completion recovered failed terminal receipt',
1584
- )
1585
- : completedResult.error,
1586
- }),
1587
- };
1588
- }
1589
- return completedResult;
1590
- }
1591
-
1592
- private canCompleteRecoveredRunningReceipt(
1593
- claimed: ClaimedWorkerToolBatchRequest,
1594
- receipt: WorkerRuntimeReceipt | null,
1595
- ): boolean {
1596
- if (!claimed.receiptKey || !receipt || receipt.status !== 'running') {
1597
- return false;
1598
- }
1599
- const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
1600
- if (ownerRunId !== this.options.req.runId) {
1601
- return false;
1602
- }
1603
- const ownerAttempt =
1604
- typeof receipt.leaseOwnerAttempt === 'number'
1605
- ? receipt.leaseOwnerAttempt
1606
- : 0;
1607
- if (ownerAttempt !== (this.options.req.runAttempt ?? 0)) {
1608
- return false;
1609
- }
1610
- if (claimed.receiptLeaseId) {
1611
- return receipt.leaseId === claimed.receiptLeaseId;
1612
- }
1613
- return receipt.leaseId == null;
1614
- }
1615
-
1616
- private buildDurableToolCompletionBatches(
1617
- entries: DurableToolCompletionEntry[],
1618
- ): DurableToolCompletionBatch[] {
1619
- const envelopeBytes = jsonByteLength({
1620
- playName: this.options.req.playName,
1621
- receipts: [],
1622
- });
1623
- const batches: DurableToolCompletionBatch[] = [];
1624
- let current: DurableToolCompletionBatch = {
1625
- entries: [],
1626
- bytes: envelopeBytes,
1627
- };
1628
- for (const entry of entries) {
1629
- const entryBytes = entry.serializedBytes;
1630
- const singleBytes = envelopeBytes + entryBytes;
1631
- if (singleBytes > WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_BYTES) {
1632
- throw receiptPersistenceError({
1633
- operation: 'complete',
1634
- receiptKeys: [entry.receipt.key],
1635
- cause: new Error(
1636
- `Runtime receipt completion payload for ${entry.receipt.key} is ${singleBytes} bytes, exceeding the ${WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_BYTES} byte service-binding cap.`,
1637
- ),
1638
- });
1639
- }
1640
- const commaBytes = current.entries.length > 0 ? 1 : 0;
1641
- const wouldExceedBytes =
1642
- current.bytes + commaBytes + entryBytes >
1643
- WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_BYTES;
1644
- const wouldExceedMembers =
1645
- current.entries.length >=
1646
- WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_MEMBERS;
1647
- if (
1648
- current.entries.length > 0 &&
1649
- (wouldExceedBytes || wouldExceedMembers)
1650
- ) {
1651
- batches.push(current);
1652
- current = { entries: [], bytes: envelopeBytes };
1653
- }
1654
- current.bytes += (current.entries.length > 0 ? 1 : 0) + entryBytes;
1655
- current.entries.push(entry);
1656
- }
1657
- if (current.entries.length > 0) {
1658
- batches.push(current);
1659
- }
1660
- return batches;
1661
- }
1662
-
1663
- private async completeDurableToolRequests(
1664
- entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
1665
- ): Promise<WorkerToolCompletionResult[]> {
1666
- const results: WorkerToolCompletionResult[] = new Array(entries.length);
1667
- const durableEntries: DurableToolCompletionEntry[] = [];
1668
- for (let index = 0; index < entries.length; index += 1) {
1669
- const entry = entries[index]!;
1670
- if (!this.options.receiptStore || !entry.claimed.receiptKey) {
1671
- results[index] = { ok: true, value: entry.result };
1672
- continue;
1673
- }
1674
- const ownerResult = markWorkerToolReceiptResultExecution(entry.result, {
1675
- kind: 'live',
1676
- receiptKey: entry.claimed.receiptKey,
1677
- });
1678
- const receipt = {
1679
- runId: this.options.req.runId,
1680
- runAttempt: this.options.req.runAttempt ?? 0,
1681
- key: entry.claimed.receiptKey!,
1682
- leaseId: entry.claimed.receiptLeaseId,
1683
- output: this.options.serializeDurableValue(ownerResult),
1684
- };
1685
- durableEntries.push({
1686
- index,
1687
- claimed: entry.claimed,
1688
- ownerResult,
1689
- receipt,
1690
- serializedBytes: jsonByteLength(receipt),
1691
- });
1692
- }
1693
- if (durableEntries.length === 0) {
1694
- return results;
1695
- }
1696
- if (!this.options.receiptStore?.completeReceipts) {
1697
- await Promise.all(
1698
- durableEntries.map(async (entry) => {
1699
- try {
1700
- results[entry.index] = {
1701
- ok: true,
1702
- value: await this.completeDurableToolRequest(
1703
- entry.claimed,
1704
- entries[entry.index]!.result,
1705
- ),
1706
- };
1707
- } catch (error) {
1708
- results[entry.index] = { ok: false, error };
1709
- }
1710
- }),
1711
- );
1712
- return results;
1713
- }
1714
- const completionBatches =
1715
- this.buildDurableToolCompletionBatches(durableEntries);
1716
- let completeAttempts = 0;
1717
- for (const batch of completionBatches) {
1718
- this.options.budgetMeter?.count('receipt');
1719
- let completed: Array<WorkerRuntimeReceipt | null>;
1720
- try {
1721
- completeAttempts += 1;
1722
- completed = await this.options.receiptStore!.completeReceipts!({
1723
- playName: this.options.req.playName,
1724
- receipts: batch.entries.map((entry) => entry.receipt),
1725
- });
1726
- } catch (error) {
1727
- const persistenceError = receiptPersistenceError({
1728
- operation: 'complete',
1729
- receiptKeys: batch.entries.map((entry) => entry.receipt.key),
1730
- cause: error,
1731
- });
1732
- if (!this.options.receiptStore.getReceipt) {
1733
- for (const entry of batch.entries) {
1734
- this.recordReceiptSalvage({
1735
- claimed: entry.claimed,
1736
- output: entry.receipt.output,
1737
- attempts: completeAttempts,
1738
- });
1739
- results[entry.index] = { ok: false, error: persistenceError };
1740
- }
1741
- continue;
1742
- }
1743
- await mapWithConcurrency(
1744
- batch.entries,
1745
- WORKER_TOOL_RECEIPT_RECOVERY_MAX_CONCURRENT_READS,
1746
- async (entry) => {
1747
- const receiptKey = entry.receipt.key;
1748
- let receipt: WorkerRuntimeReceipt | null = null;
1749
- try {
1750
- this.options.budgetMeter?.count('receipt');
1751
- receipt = await this.options.receiptStore!.getReceipt!({
1752
- playName: this.options.req.playName,
1753
- key: receiptKey,
1754
- });
1755
- } catch (readError) {
1756
- this.recordReceiptSalvage({
1757
- claimed: entry.claimed,
1758
- output: entry.receipt.output,
1759
- attempts: completeAttempts,
1760
- });
1761
- results[entry.index] = {
1762
- ok: false,
1763
- error: receiptPersistenceError({
1764
- operation: 'complete',
1765
- receiptKeys: [receiptKey],
1766
- cause: new AggregateError(
1767
- [error, readError],
1768
- 'Receipt completion failed and receipt recovery read failed',
1769
- ),
1770
- }),
1771
- };
1772
- return;
1773
- }
1774
- if (
1775
- receipt &&
1776
- (receipt.status === 'completed' ||
1777
- receipt.status === 'skipped') &&
1778
- receipt.output !== undefined
1779
- ) {
1780
- results[entry.index] = this.completedDurableToolRequestResult({
1781
- claimed: entry.claimed,
1782
- ownerResult: entry.ownerResult,
1783
- completed: receipt,
1784
- });
1785
- return;
1786
- }
1787
- if (
1788
- this.canCompleteRecoveredRunningReceipt(entry.claimed, receipt)
1789
- ) {
1790
- try {
1791
- results[entry.index] = {
1792
- ok: true,
1793
- value: await this.completeDurableToolRequest(
1794
- entry.claimed,
1795
- entries[entry.index]!.result,
1796
- ),
1797
- };
1798
- return;
1799
- } catch (singleCompleteError) {
1800
- results[entry.index] = {
1801
- ok: false,
1802
- error: receiptPersistenceError({
1803
- operation: 'complete',
1804
- receiptKeys: [receiptKey],
1805
- cause: new AggregateError(
1806
- [error, singleCompleteError],
1807
- 'Receipt completion failed and owned receipt recovery failed',
1808
- ),
1809
- }),
1810
- };
1811
- return;
1812
- }
1813
- }
1814
- this.recordReceiptSalvage({
1815
- claimed: entry.claimed,
1816
- output: entry.receipt.output,
1817
- attempts: completeAttempts,
1818
- });
1819
- results[entry.index] = {
1820
- ok: false,
1821
- error: receiptPersistenceError({
1822
- operation: 'complete',
1823
- receiptKeys: [receiptKey],
1824
- cause: receipt?.error
1825
- ? new AggregateError(
1826
- [error, new Error(receipt.error)],
1827
- 'Receipt completion failed and receipt recovered failed',
1828
- )
1829
- : error,
1830
- }),
1831
- };
1832
- },
1833
- );
1834
- this.options.recordPerfTrace?.({
1835
- phase: 'worker.tool_receipts.partial_complete_recovery',
1836
- extra: {
1837
- recovered: batch.entries.filter((entry) => results[entry.index]?.ok)
1838
- .length,
1839
- failed: batch.entries.filter((entry) => {
1840
- const result = results[entry.index];
1841
- return result && !result.ok;
1842
- }).length,
1843
- total: batch.entries.length,
1844
- failedReceiptKeys: batch.entries
1845
- .filter((entry) => {
1846
- const result = results[entry.index];
1847
- return result && !result.ok;
1848
- })
1849
- .slice(0, 5)
1850
- .map((entry) => entry.receipt.key),
1851
- },
1852
- });
1853
- continue;
1854
- }
1855
- await mapWithConcurrency(
1856
- batch.entries,
1857
- WORKER_TOOL_RECEIPT_RECOVERY_MAX_CONCURRENT_READS,
1858
- async (entry, resultIndex) => {
1859
- results[entry.index] =
1860
- await this.recoverCompletedDurableToolRequestResult({
1861
- claimed: entry.claimed,
1862
- ownerResult: entry.ownerResult,
1863
- completed: completed[resultIndex],
1864
- completeAttempts,
1865
- });
1866
- },
1867
- );
1868
- }
1869
- return results;
1870
- }
1871
-
1872
- private async failDurableToolRequest(
1873
- claimed: ClaimedWorkerToolBatchRequest,
1874
- error: unknown,
1875
- ): Promise<void> {
1876
- if (!this.options.receiptStore || !claimed.receiptKey) {
1877
- return;
1878
- }
1879
- this.options.budgetMeter?.count('receipt');
1880
- try {
1881
- await this.options.receiptStore.failReceipt({
1882
- playName: this.options.req.playName,
1883
- runId: this.options.req.runId,
1884
- runAttempt: this.options.req.runAttempt ?? 0,
1885
- key: claimed.receiptKey,
1886
- leaseId: claimed.receiptLeaseId,
1887
- error: error instanceof Error ? error.message : String(error),
1888
- failureKind: runtimeReceiptFailureKindForError(error),
1889
- });
1890
- } catch (receiptError) {
1891
- if (this.options.persistenceLatch) {
1892
- tripRuntimePersistenceLatch(
1893
- this.options.persistenceLatch,
1894
- receiptError,
1895
- );
1896
- }
1897
- throw receiptPersistenceError({
1898
- operation: 'fail',
1899
- receiptKeys: [claimed.receiptKey],
1900
- cause: receiptError,
1901
- });
1902
- }
1903
- }
1904
-
1905
- private async releaseDurableToolRequest(
1906
- claimed: ClaimedWorkerToolBatchRequest,
1907
- error: unknown,
1908
- ): Promise<void> {
1909
- if (!this.options.receiptStore || !claimed.receiptKey) {
1910
- return;
1911
- }
1912
- if (!this.options.receiptStore.releaseReceipt) {
1913
- throw new Error(
1914
- `Durable tool call ${claimed.receiptKey} suspended but the receipt store cannot release ownership.`,
1915
- );
1916
- }
1917
- this.options.budgetMeter?.count('receipt');
1918
- try {
1919
- const released = await this.options.receiptStore.releaseReceipt({
1920
- playName: this.options.req.playName,
1921
- runId: this.options.req.runId,
1922
- runAttempt: this.options.req.runAttempt ?? 0,
1923
- key: claimed.receiptKey,
1924
- leaseId: claimed.receiptLeaseId,
1925
- });
1926
- if (!released || released.status !== 'pending') {
1927
- throw new Error(
1928
- `Durable tool call ${claimed.receiptKey} suspended but receipt ownership could not be released.`,
1929
- );
1930
- }
1931
- } catch (receiptError) {
1932
- if (this.options.persistenceLatch) {
1933
- tripRuntimePersistenceLatch(
1934
- this.options.persistenceLatch,
1935
- receiptError,
1936
- );
1937
- }
1938
- throw receiptPersistenceError({
1939
- operation: 'fail',
1940
- receiptKeys: [claimed.receiptKey],
1941
- cause: new AggregateError(
1942
- [error, receiptError],
1943
- 'Tool receipt release failed after durable suspension',
1944
- ),
1945
- });
1946
- }
1947
- }
1948
-
1949
- private async failDurableToolRequests(
1950
- claimedRequests: ClaimedWorkerToolBatchRequest[],
1951
- error: unknown,
1952
- ): Promise<void> {
1953
- const durable = claimedRequests.filter(
1954
- (claimed) => this.options.receiptStore && claimed.receiptKey,
1955
- );
1956
- if (durable.length === 0) return;
1957
- if (
1958
- isPlayExecutionSuspendedError(error) ||
1959
- isPlayRowExecutionSuspendedError(error)
1960
- ) {
1961
- await Promise.all(
1962
- durable.map((claimed) =>
1963
- this.releaseDurableToolRequest(claimed, error),
1964
- ),
1965
- );
1966
- return;
1967
- }
1968
- if (!this.options.receiptStore?.failReceipts) {
1969
- await Promise.all(
1970
- durable.map((claimed) => this.failDurableToolRequest(claimed, error)),
1971
- );
1972
- return;
1973
- }
1974
- // Bulk fail is a single service-binding RPC regardless of key count.
1975
- this.options.budgetMeter?.count('receipt');
1976
- try {
1977
- await this.options.receiptStore.failReceipts({
1978
- playName: this.options.req.playName,
1979
- receipts: durable.map((claimed) => ({
1980
- runId: this.options.req.runId,
1981
- runAttempt: this.options.req.runAttempt ?? 0,
1982
- key: claimed.receiptKey!,
1983
- leaseId: claimed.receiptLeaseId,
1984
- error: error instanceof Error ? error.message : String(error),
1985
- failureKind: runtimeReceiptFailureKindForError(error),
1986
- })),
1987
- });
1988
- } catch (receiptError) {
1989
- if (this.options.persistenceLatch) {
1990
- tripRuntimePersistenceLatch(
1991
- this.options.persistenceLatch,
1992
- receiptError,
1993
- );
1994
- }
1995
- throw receiptPersistenceError({
1996
- operation: 'fail',
1997
- receiptKeys: durable
1998
- .map((claimed) => claimed.receiptKey)
1999
- .filter((key): key is string => typeof key === 'string'),
2000
- cause: receiptError,
2001
- });
2002
- }
2003
- }
2004
-
2005
- private async executeToolGroup(
2006
- toolId: string,
2007
- requests: WorkerToolBatchRequest[],
2008
- ): Promise<void> {
2009
- const receiptPrepareStartedAt = this.options.nowMs();
2010
- const { claimedRequests, deferredClaimedRequests } =
2011
- await this.prepareDurableToolRequests(requests);
2012
- this.options.recordPerfTrace?.({
2013
- phase: 'runner.tool.receipt_prepare',
2014
- ms: this.options.nowMs() - receiptPrepareStartedAt,
2015
- extra: { toolId, requests: requests.length },
2016
- });
2017
- await this.executeClaimedToolRequests(
2018
- toolId,
2019
- requests.length,
2020
- claimedRequests,
2021
- );
2022
- if (deferredClaimedRequests.length === 0) {
2023
- return;
2024
- }
2025
- const reclaimedRequests = (
2026
- await Promise.all(deferredClaimedRequests)
2027
- ).flat();
2028
- await this.executeClaimedToolRequests(
2029
- toolId,
2030
- requests.length,
2031
- reclaimedRequests,
2032
- );
2033
- }
2034
-
2035
- private async executeClaimedToolRequests(
2036
- toolId: string,
2037
- requestCount: number,
2038
- claimedRequests: ClaimedWorkerToolBatchRequest[],
2039
- ): Promise<void> {
2040
- if (claimedRequests.length === 0) {
2041
- return;
2042
- }
2043
- const strategy = getPlayRuntimeBatchStrategy(toolId);
2044
- if (
2045
- !strategy ||
2046
- toolId === 'test_wait_for_event' ||
2047
- claimedRequests.length < 2
2048
- ) {
2049
- const groupStartedAt = this.options.nowMs();
2050
- // Seed the dispatch width from the governor's provider-shaped parallelism
2051
- // instead of the flat policy.concurrency.toolCalls ceiling. A flat width
2052
- // launched every claimed row at once into the pacer, so unhinted providers
2053
- // 429'd on the first burst before AIMD could halve the rate. The shaped
2054
- // estimate is derived from the provider's RPS/maxConcurrency pacing rules
2055
- // (see suggestedParallelism in governor.ts), floored at 1 and capped by the
2056
- // global tool-call concurrency ceiling. Per-call pacing still gates each
2057
- // in-flight request, so this only trims the launch burst.
2058
- const toolCallConcurrencyCeiling =
2059
- this.options.governor.policy.concurrency.toolCalls;
2060
- const shapedToolParallelism =
2061
- await this.options.governor.suggestedParallelism(
2062
- toolId,
2063
- toolCallConcurrencyCeiling,
2064
- );
2065
- const dispatchWidth = Math.min(
2066
- toolCallConcurrencyCeiling,
2067
- Math.max(1, shapedToolParallelism),
2068
- );
2069
- await dispatchBounded(claimedRequests, dispatchWidth, async (claimed) => {
2070
- const { request } = claimed;
2071
- const toolContract = await this.options
2072
- .resolvePacing(toolId)
2073
- .catch(() => null);
2074
- const circuitOpenError = this.runtimePersistenceCircuitOpenError(1);
2075
- if (circuitOpenError) {
2076
- this.rejectRequests(
2077
- claimed,
2078
- await this.failureForRejectedToolRequest(claimed, circuitOpenError),
2079
- );
2080
- return;
2081
- }
2082
- // Each unbatched provider call takes its own tool slot: the Governor
2083
- // charges tool budget, holds a global tool-concurrency slot, and
2084
- // applies per-(org,provider) pacing before the call runs.
2085
- let slot: { release: () => void };
2086
- const admissionStartedAt = this.options.nowMs();
2087
- try {
2088
- slot = await this.options.governor.acquireToolSlot(toolId, {
2089
- signal: this.options.abortSignal,
2090
- });
2091
- } catch (error) {
2092
- this.rejectRequests(
2093
- claimed,
2094
- await this.failureForRejectedToolRequest(claimed, error),
2095
- );
2096
- return;
2097
- }
2098
- this.options.recordPerfTrace?.({
2099
- phase: 'runner.tool.admission',
2100
- ms: this.options.nowMs() - admissionStartedAt,
2101
- extra: { toolId, requests: 1 },
2102
- });
2103
- try {
2104
- let result: unknown;
2105
- try {
2106
- await this.markAdmittedToolRequestsRunning([claimed]);
2107
- this.options.budgetMeter?.count('provider');
2108
- this.options.budgetMeter?.count(
2109
- 'egress',
2110
- Math.max(
2111
- 0,
2112
- WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL - 1,
2113
- ),
2114
- );
2115
- const durableReceiptKey = claimed.receiptKey
2116
- ? durableCallReceiptKeyForClaimed({
2117
- req: this.options.req,
2118
- claimed,
2119
- })
2120
- : null;
2121
- const providerStartedAt = this.options.nowMs();
2122
- try {
2123
- result = await executeWithWorkerReceiptHeartbeats({
2124
- playName: this.options.req.playName,
2125
- runId: this.options.req.runId,
2126
- runAttempt: this.options.req.runAttempt ?? 0,
2127
- ...(this.options.receiptStore
2128
- ? { receiptStore: this.options.receiptStore }
2129
- : {}),
2130
- budgetMeter: this.options.budgetMeter,
2131
- heartbeatIntervalMs:
2132
- this.options.runtimeReceiptHeartbeatIntervalMs,
2133
- targets: claimed.receiptKey
2134
- ? [
2135
- {
2136
- key: claimed.receiptKey,
2137
- leaseId: claimed.receiptLeaseId,
2138
- },
2139
- ]
2140
- : [],
2141
- execute: () =>
2142
- this.options.executeTool({
2143
- id: request.id,
2144
- toolId,
2145
- input: request.input,
2146
- workflowStep: request.workflowStep,
2147
- callbacks: this.options.callbacks,
2148
- onProviderBackpressure: (retryAfterMs) =>
2149
- this.reportBackpressure(toolId, retryAfterMs),
2150
- onRetryAttempt: () =>
2151
- this.options.governor.chargeBudget('retry'),
2152
- transientHttpRetrySafe:
2153
- toolContract?.retrySafeTransientHttp === true,
2154
- ...(durableReceiptKey
2155
- ? {
2156
- durableCallReceiptKey: durableReceiptKey,
2157
- executionAuthScopeDigest:
2158
- request.executionAuthScopeDigest,
2159
- providerIdempotencyKey:
2160
- providerIdempotencyKeyForClaimed({
2161
- req: this.options.req,
2162
- claimed,
2163
- }),
2164
- }
2165
- : {}),
2166
- runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
2167
- [claimed],
2168
- {
2169
- resolveOwnerTimeoutMs: (request) =>
2170
- request.runtimeTimeoutMs,
2171
- },
2172
- ),
2173
- directOptions: request.directOptions,
2174
- }),
2175
- });
2176
- } finally {
2177
- this.options.recordPerfTrace?.({
2178
- phase: 'runner.tool.provider',
2179
- ms: this.options.nowMs() - providerStartedAt,
2180
- extra: { toolId, requests: 1 },
2181
- });
2182
- }
2183
- } catch (error) {
2184
- if (error instanceof RuntimeLeaseLostError) {
2185
- this.rejectRequests(claimed, error);
2186
- return;
2187
- }
2188
- if (
2189
- await this.requeueAfterAuthScopeChanged(
2190
- [request, ...claimed.followers],
2191
- error,
2192
- )
2193
- ) {
2194
- return;
2195
- }
2196
- this.rejectRequests(
2197
- claimed,
2198
- await this.failureForRejectedToolRequest(claimed, error),
2199
- );
2200
- return;
2201
- }
2202
- try {
2203
- const receiptCompleteStartedAt = this.options.nowMs();
2204
- this.settleRequests(
2205
- claimed,
2206
- await this.completeDurableToolRequest(claimed, result),
2207
- );
2208
- this.options.recordPerfTrace?.({
2209
- phase: 'runner.tool.receipt_complete',
2210
- ms: this.options.nowMs() - receiptCompleteStartedAt,
2211
- extra: { toolId, requests: 1 },
2212
- });
2213
- } catch (receiptError) {
2214
- this.rejectRequests(claimed, receiptError);
2215
- }
2216
- } finally {
2217
- slot.release();
2218
- }
2219
- });
2220
- this.options.recordPerfTrace?.({
2221
- phase: 'runner.tool.group',
2222
- ms: this.options.nowMs() - groupStartedAt,
2223
- extra: {
2224
- toolId,
2225
- requests: requestCount,
2226
- executed: claimedRequests.length,
2227
- batched: false,
2228
- },
2229
- });
2230
- return;
2231
- }
2232
-
2233
- const batchStartedAt = this.options.nowMs();
2234
- await this.executeBatchedWorkerToolGroup({
2235
- sourceToolId: toolId,
2236
- requests: claimedRequests,
2237
- strategy,
2238
- suggestedParallelism: await this.options.governor.suggestedParallelism(
2239
- toolId,
2240
- this.options.governor.policy.pacing.workerToolBatchDefaultParallelism,
2241
- ),
2242
- });
2243
- this.options.recordPerfTrace?.({
2244
- phase: 'runner.tool.group',
2245
- ms: this.options.nowMs() - batchStartedAt,
2246
- extra: {
2247
- toolId,
2248
- requests: requestCount,
2249
- executed: claimedRequests.length,
2250
- batched: true,
2251
- },
2252
- });
2253
- }
2254
-
2255
- private async executeBatchedWorkerToolGroup(input: {
2256
- sourceToolId: string;
2257
- requests: ClaimedWorkerToolBatchRequest[];
2258
- strategy: AnyBatchOperationStrategy;
2259
- suggestedParallelism: number;
2260
- }): Promise<void> {
2261
- const compiledBatches = compileRequestsWithStrategy({
2262
- requests: input.requests,
2263
- strategy: input.strategy,
2264
- getPayload: (request) => request.request.input,
2265
- });
2266
- this.options.recordPerfTrace?.({
2267
- phase: 'runner.tool.batch.compile',
2268
- ms: 0,
2269
- extra: {
2270
- sourceOperation: input.strategy.sourceOperation,
2271
- batchOperation: input.strategy.batchOperation,
2272
- requests: input.requests.length,
2273
- batches: compiledBatches.length,
2274
- batchSizes: compiledBatches.map((batch) => batch.memberRequests.length),
2275
- },
2276
- });
2277
- await executeChunkedRequests({
2278
- requests: compiledBatches,
2279
- // Chunk parallelism is the Governor's per-tool suggestion (provider rate
2280
- // hints tightened to the policy ceiling), bounded by the batch count.
2281
- batchSize: Math.max(
2282
- 1,
2283
- Math.min(input.suggestedParallelism, compiledBatches.length || 1),
2284
- ),
2285
- execute: async (batch) => {
2286
- const toolContract = await this.options
2287
- .resolvePacing(batch.batchOperation)
2288
- .catch(() => null);
2289
- const circuitOpenError = this.runtimePersistenceCircuitOpenError(
2290
- batch.memberRequests.length,
2291
- );
2292
- if (circuitOpenError) {
2293
- throw circuitOpenError;
2294
- }
2295
- // One provider call per batch -> one tool slot (budget + global
2296
- // concurrency + per-(org,provider) pacing) around the whole batch.
2297
- const admissionStartedAt = this.options.nowMs();
2298
- const slot = await this.options.governor.acquireToolSlot(
2299
- batch.batchOperation,
2300
- {
2301
- signal: this.options.abortSignal,
2302
- },
2303
- );
2304
- this.options.recordPerfTrace?.({
2305
- phase: 'runner.tool.admission',
2306
- ms: this.options.nowMs() - admissionStartedAt,
2307
- extra: {
2308
- toolId: batch.batchOperation,
2309
- requests: batch.memberRequests.length,
2310
- batched: true,
2311
- },
2312
- });
2313
- try {
2314
- await this.markAdmittedToolRequestsRunning(batch.memberRequests);
2315
- this.options.budgetMeter?.count('provider');
2316
- const receiptKeys = batch.memberRequests.map((member) =>
2317
- durableCallReceiptKeyForClaimed({
2318
- req: this.options.req,
2319
- claimed: member,
2320
- }),
2321
- );
2322
- const aggregateReceiptKey = buildBatchDurableCallReceiptKey({
2323
- req: this.options.req,
2324
- batchOperation: batch.batchOperation,
2325
- receiptKeys,
2326
- });
2327
- const providerStartedAt = this.options.nowMs();
2328
- try {
2329
- return await executeWithWorkerReceiptHeartbeats({
2330
- playName: this.options.req.playName,
2331
- runId: this.options.req.runId,
2332
- runAttempt: this.options.req.runAttempt ?? 0,
2333
- ...(this.options.receiptStore
2334
- ? { receiptStore: this.options.receiptStore }
2335
- : {}),
2336
- budgetMeter: this.options.budgetMeter,
2337
- heartbeatIntervalMs:
2338
- this.options.runtimeReceiptHeartbeatIntervalMs,
2339
- targets: batch.memberRequests.map((member) => ({
2340
- key: member.receiptKey,
2341
- leaseId: member.receiptLeaseId,
2342
- })),
2343
- execute: () =>
2344
- this.options.executeTool({
2345
- id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
2346
- toolId: batch.batchOperation,
2347
- input: batch.batchPayload,
2348
- callbacks: this.options.callbacks,
2349
- onProviderBackpressure: (retryAfterMs) =>
2350
- this.reportBackpressure(input.sourceToolId, retryAfterMs),
2351
- onRetryAttempt: () =>
2352
- this.options.governor.chargeBudget('retry'),
2353
- transientHttpRetrySafe:
2354
- toolContract?.retrySafeTransientHttp === true,
2355
- durableCallReceiptKey: aggregateReceiptKey,
2356
- executionAuthScopeDigest:
2357
- batch.memberRequests[0]?.request.executionAuthScopeDigest ??
2358
- null,
2359
- providerIdempotencyKey: buildBatchProviderIdempotencyKey({
2360
- aggregateReceiptKey,
2361
- receiptKeys,
2362
- providerIdempotencyKeys: batch.memberRequests.map(
2363
- (member) =>
2364
- providerIdempotencyKeyForClaimed({
2365
- req: this.options.req,
2366
- claimed: member,
2367
- }),
2368
- ),
2369
- }),
2370
- runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
2371
- batch.memberRequests,
2372
- {
2373
- resolveOwnerTimeoutMs: (request) =>
2374
- request.runtimeTimeoutMs,
2375
- },
2376
- ),
2377
- }),
2378
- });
2379
- } finally {
2380
- this.options.recordPerfTrace?.({
2381
- phase: 'runner.tool.provider',
2382
- ms: this.options.nowMs() - providerStartedAt,
2383
- extra: {
2384
- toolId: batch.batchOperation,
2385
- requests: batch.memberRequests.length,
2386
- batched: true,
2387
- },
2388
- });
2389
- }
2390
- } finally {
2391
- slot.release();
2392
- }
2393
- },
2394
- onChunkComplete: async (
2395
- chunkResults: Array<
2396
- ChunkExecutionResult<(typeof compiledBatches)[number], unknown>
2397
- >,
2398
- ) => {
2399
- for (const entry of chunkResults) {
2400
- if (entry.error !== undefined) {
2401
- // One batch's provider error stays scoped to that batch's member
2402
- // requests. Sibling batches in this chunk keep their results so a
2403
- // single provider hiccup cannot cascade into a whole-map failure.
2404
- if (
2405
- entry.error instanceof ToolExecuteAuthScopeChangedError &&
2406
- entry.request.memberRequests.every(
2407
- (claimed) => claimed.request.authScopeRetryUsed !== true,
2408
- ) &&
2409
- (await this.requeueAfterAuthScopeChanged(
2410
- entry.request.memberRequests.map((claimed) => claimed.request),
2411
- entry.error,
2412
- ))
2413
- ) {
2414
- continue;
2415
- }
2416
- let rejection: unknown = entry.error;
2417
- if (!(entry.error instanceof RuntimeLeaseLostError)) {
2418
- try {
2419
- await this.failDurableToolRequests(
2420
- entry.request.memberRequests,
2421
- entry.error,
2422
- );
2423
- } catch (receiptError) {
2424
- rejection = receiptPersistenceError({
2425
- operation: 'fail',
2426
- receiptKeys: entry.request.memberRequests
2427
- .map((claimed) => claimed.receiptKey)
2428
- .filter((key): key is string => typeof key === 'string'),
2429
- cause: new AggregateError(
2430
- [entry.error, receiptError],
2431
- 'Tool receipts fail mark failed',
2432
- ),
2433
- });
2434
- }
2435
- }
2436
- for (const claimed of entry.request.memberRequests) {
2437
- this.rejectRequests(claimed, rejection);
2438
- }
2439
- continue;
2440
- }
2441
- const batchResult = isToolExecuteResult(entry.result)
2442
- ? entry.result.toolOutput.raw
2443
- : entry.result;
2444
- const splitResults =
2445
- batchResult != null
2446
- ? entry.request.splitResults(batchResult)
2447
- : entry.request.memberRequests.map(() => null);
2448
- let completedResults: WorkerToolCompletionResult[];
2449
- try {
2450
- const receiptCompleteStartedAt = this.options.nowMs();
2451
- completedResults = await this.completeDurableToolRequests(
2452
- entry.request.memberRequests.map((claimed, index) => ({
2453
- claimed,
2454
- result: wrapWorkerToolResult(
2455
- claimed.request.toolId,
2456
- splitResults[index] ?? null,
2457
- toolMetadataFallback(claimed.request.toolId),
2458
- ),
2459
- })),
2460
- );
2461
- this.options.recordPerfTrace?.({
2462
- phase: 'runner.tool.receipt_complete',
2463
- ms: this.options.nowMs() - receiptCompleteStartedAt,
2464
- extra: {
2465
- toolId: entry.request.batchOperation,
2466
- requests: entry.request.memberRequests.length,
2467
- batched: true,
2468
- },
2469
- });
2470
- } catch (receiptError) {
2471
- for (const claimed of entry.request.memberRequests) {
2472
- this.rejectRequests(claimed, receiptError);
2473
- }
2474
- continue;
2475
- }
2476
- for (let index = 0; index < completedResults.length; index += 1) {
2477
- const claimed = entry.request.memberRequests[index]!;
2478
- const completed = completedResults[index];
2479
- if (!completed) {
2480
- this.rejectRequests(
2481
- claimed,
2482
- new Error(
2483
- `Tool batch completion did not return member ${claimed.request.id}.`,
2484
- ),
2485
- );
2486
- } else if (completed.ok) {
2487
- this.settleRequests(claimed, completed.value);
2488
- } else {
2489
- this.rejectRequests(claimed, completed.error);
2490
- }
2491
- }
2492
- }
2493
- },
2494
- }).catch(async (error) => {
2495
- let rejection: unknown = error;
2496
- if (
2497
- error instanceof ToolExecuteAuthScopeChangedError &&
2498
- (await this.requeueAfterAuthScopeChanged(
2499
- input.requests.map((claimed) => claimed.request),
2500
- error,
2501
- ))
2502
- ) {
2503
- return;
2504
- }
2505
- if (!(error instanceof RuntimeLeaseLostError)) {
2506
- try {
2507
- await this.failDurableToolRequests(input.requests, error);
2508
- } catch (receiptError) {
2509
- rejection = receiptPersistenceError({
2510
- operation: 'fail',
2511
- receiptKeys: input.requests
2512
- .map((claimed) => claimed.receiptKey)
2513
- .filter((key): key is string => typeof key === 'string'),
2514
- cause: new AggregateError(
2515
- [error, receiptError],
2516
- 'Tool receipts fail mark failed',
2517
- ),
2518
- });
2519
- }
2520
- }
2521
- for (const claimed of input.requests) {
2522
- this.rejectRequests(claimed, rejection);
2523
- }
2524
- });
2525
- }
2526
- }
2527
-
2528
- function groupWorkerToolRequestsByTool(
2529
- requests: WorkerToolBatchRequest[],
2530
- ): Map<string, WorkerToolBatchRequest[]> {
2531
- const byTool = new Map<string, WorkerToolBatchRequest[]>();
2532
- for (const request of requests) {
2533
- const existing = byTool.get(request.toolId);
2534
- if (existing) {
2535
- existing.push(request);
2536
- } else {
2537
- byTool.set(request.toolId, [request]);
2538
- }
2539
- }
2540
- return byTool;
2541
- }