deepline 0.1.243 → 0.1.244
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +5 -25
- package/dist/bundling-sources/sdk/src/release.ts +23 -21
- package/dist/bundling-sources/sdk/src/types.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/backend.ts +2 -21
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -6
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +6 -13
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +9 -26
- package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +1 -2
- package/dist/bundling-sources/shared_libs/play-runtime/map-row-identity.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +191 -29
- package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +1 -36
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +0 -7
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +1 -12
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +5 -15
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +6 -21
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +3 -4
- package/dist/bundling-sources/shared_libs/plays/artifact-kind-guard.ts +16 -29
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +4 -8
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +21 -712
- package/dist/bundling-sources/shared_libs/plays/bundling/limits.ts +1 -18
- package/dist/cli/index.js +29 -71
- package/dist/cli/index.mjs +29 -71
- package/dist/{compiler-manifest-CYcwzSOJ.d.ts → compiler-manifest-DlE7dnRm.d.mts} +2 -8
- package/dist/{compiler-manifest-CYcwzSOJ.d.mts → compiler-manifest-DlE7dnRm.d.ts} +2 -8
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +24 -22
- package/dist/index.mjs +24 -22
- package/dist/plays/bundle-play-file.d.mts +7 -22
- package/dist/plays/bundle-play-file.d.ts +7 -22
- package/dist/plays/bundle-play-file.mjs +26 -614
- package/package.json +2 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +0 -4504
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +0 -1386
- package/dist/bundling-sources/apps/play-runner-workers/src/durable-object-deploy-handoff.ts +0 -30
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +0 -8385
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md +0 -21
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts +0 -251
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/coordinator-progress.ts +0 -289
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/csv-rows.ts +0 -85
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +0 -23
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/dataset-handles.ts +0 -528
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +0 -229
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/ledger-event-batches.ts +0 -81
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/live-progress.ts +0 -22
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +0 -920
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts +0 -90
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +0 -475
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +0 -570
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/result-dataset-persistence.ts +0 -50
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/row-isolation.ts +0 -5
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +0 -519
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-batch.ts +0 -107
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +0 -2541
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-http-errors.ts +0 -7
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +0 -227
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +0 -130
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +0 -39
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/workflow-preview.ts +0 -26
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime-api-retry.ts +0 -23
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-instance-create.ts +0 -47
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +0 -167
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +0 -91
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +0 -329
- package/dist/bundling-sources/sdk/src/worker-play-entry.ts +0 -240
- package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +0 -252
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +0 -217
|
@@ -1,2541 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
compileRequestsWithStrategy,
|
|
3
|
-
executeChunkedRequests,
|
|
4
|
-
type ChunkExecutionResult,
|
|
5
|
-
} from '../../../../shared_libs/play-runtime/batch-runtime';
|
|
6
|
-
import { dispatchBounded } from '../../../../shared_libs/play-runtime/bounded-dispatch';
|
|
7
|
-
import type { AnyBatchOperationStrategy } from '../../../../shared_libs/play-runtime/batching-types';
|
|
8
|
-
import {
|
|
9
|
-
buildDurableToolCallAuthScopeDigest,
|
|
10
|
-
buildDurableToolCallCacheKey,
|
|
11
|
-
buildDurableToolAggregateProviderIdempotencyKey,
|
|
12
|
-
buildDurableToolAggregateReceiptKey,
|
|
13
|
-
buildDurableToolProviderIdempotencyKey,
|
|
14
|
-
buildDurableToolReceiptPrefix,
|
|
15
|
-
} from '../../../../shared_libs/play-runtime/durable-call-cache';
|
|
16
|
-
import {
|
|
17
|
-
RuntimeReceiptWaitTimeoutError,
|
|
18
|
-
resolveRuntimeToolReceiptWaitMaxAttempts,
|
|
19
|
-
runtimeReceiptFailureKindForError,
|
|
20
|
-
waitForCompletedRuntimeReceipt,
|
|
21
|
-
} from '../../../../shared_libs/play-runtime/durable-receipt-execution';
|
|
22
|
-
import type { PlayExecutionGovernor } from '../../../../shared_libs/play-runtime/governor/governor';
|
|
23
|
-
import { ToolExecuteAuthScopeChangedError } from '../../../../shared_libs/play-runtime/tool-execute-retry-policy';
|
|
24
|
-
import { getPlayRuntimeBatchStrategy } from '../../../../shared_libs/play-runtime/play-runtime-batching-registry';
|
|
25
|
-
import {
|
|
26
|
-
RuntimePersistenceCircuitOpenError,
|
|
27
|
-
shouldShortenCompleteReceiptLadder,
|
|
28
|
-
tripRuntimePersistenceLatch,
|
|
29
|
-
type RuntimePersistenceLatch,
|
|
30
|
-
} from '../../../../shared_libs/play-runtime/persistence-latch';
|
|
31
|
-
import type { ReceiptSalvageBuffer } from '../../../../shared_libs/play-runtime/receipt-salvage';
|
|
32
|
-
import type { ResolvedPacingPolicy } from '../../../../shared_libs/play-runtime/pacing';
|
|
33
|
-
import type { RuntimeStepReceipt } from '../../../../shared_libs/play-runtime/ctx-types';
|
|
34
|
-
import {
|
|
35
|
-
isPlayExecutionSuspendedError,
|
|
36
|
-
isPlayRowExecutionSuspendedError,
|
|
37
|
-
} from '../../../../shared_libs/play-runtime/suspension';
|
|
38
|
-
import {
|
|
39
|
-
QUERY_RESULT_DATASET_PAGE_SIZE,
|
|
40
|
-
isQueryResultDatasetReadRequest,
|
|
41
|
-
type QueryResultDatasetRequest,
|
|
42
|
-
} from '../../../../shared_libs/play-runtime/query-result-dataset';
|
|
43
|
-
import {
|
|
44
|
-
createToolExecuteResult,
|
|
45
|
-
isToolExecuteResult,
|
|
46
|
-
type ToolExecuteResult,
|
|
47
|
-
type ToolResultMetadataInput,
|
|
48
|
-
} from '../../../../shared_libs/play-runtime/tool-result';
|
|
49
|
-
import { toolExecutionMetadataForOutcome } from '../../../../shared_libs/play-runtime/tool-execution-outcome';
|
|
50
|
-
import { buildScopedWorkReceiptKey } from '../../../../shared_libs/play-runtime/work-receipts';
|
|
51
|
-
import {
|
|
52
|
-
canReclaimFailedWorkerToolReceipt,
|
|
53
|
-
canReclaimTimedOutWorkerToolReceipt,
|
|
54
|
-
markWorkerToolReceiptResultCached,
|
|
55
|
-
markWorkerToolReceiptResultExecution,
|
|
56
|
-
planWorkerToolReceiptGroups,
|
|
57
|
-
resolveRunningReceiptWaitMaxAttempts,
|
|
58
|
-
resolveWorkerToolReceiptGroupWaitMaxAttempts,
|
|
59
|
-
resolveWorkerToolRuntimeTimeoutMs,
|
|
60
|
-
} from './tool-receipts';
|
|
61
|
-
import type {
|
|
62
|
-
WorkerRuntimeReceipt,
|
|
63
|
-
WorkerRuntimeReceiptClaim,
|
|
64
|
-
WorkerRuntimeReceiptStore,
|
|
65
|
-
} from './receipts';
|
|
66
|
-
import {
|
|
67
|
-
executeWithWorkerReceiptHeartbeats,
|
|
68
|
-
RuntimeLeaseLostError,
|
|
69
|
-
} from './receipts';
|
|
70
|
-
import { WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL } from './worker-platform-budget';
|
|
71
|
-
import type { WorkerWorkBudgetMeter } from './work-budget';
|
|
72
|
-
|
|
73
|
-
export type WorkerPacingResolver = (
|
|
74
|
-
toolId: string,
|
|
75
|
-
) => Promise<
|
|
76
|
-
| (ResolvedPacingPolicy & {
|
|
77
|
-
retrySafeTransientHttp: boolean;
|
|
78
|
-
r: { bucketId: string; token: string } | null;
|
|
79
|
-
})
|
|
80
|
-
| null
|
|
81
|
-
>;
|
|
82
|
-
|
|
83
|
-
export type WorkerToolActionCacheVersionResolver = (
|
|
84
|
-
toolId: string,
|
|
85
|
-
) => Promise<string>;
|
|
86
|
-
|
|
87
|
-
export type WorkerToolAuthScopeDigestResolver = ((
|
|
88
|
-
toolId: string,
|
|
89
|
-
) => Promise<string>) & {
|
|
90
|
-
invalidate?: (toolId?: string | null) => void;
|
|
91
|
-
};
|
|
92
|
-
|
|
93
|
-
export type WorkerToolDispatchCallbacks = {
|
|
94
|
-
onToolCalled?: (toolId: string, at?: number) => void;
|
|
95
|
-
onToolFailed?: (toolId: string, at?: number) => void;
|
|
96
|
-
};
|
|
97
|
-
|
|
98
|
-
export type WorkerToolDirectOptions = {
|
|
99
|
-
customerDbDataset?: QueryResultDatasetRequest;
|
|
100
|
-
attachCustomerDbDataset?: boolean;
|
|
101
|
-
};
|
|
102
|
-
|
|
103
|
-
export type WorkerToolDispatchRunRequest = {
|
|
104
|
-
runId: string;
|
|
105
|
-
runAttempt?: number | null;
|
|
106
|
-
orgId: string;
|
|
107
|
-
playName: string;
|
|
108
|
-
};
|
|
109
|
-
|
|
110
|
-
export type WorkflowStepLike = object;
|
|
111
|
-
|
|
112
|
-
export type WorkerToolDispatchExecuteTool = (input: {
|
|
113
|
-
id: string;
|
|
114
|
-
toolId: string;
|
|
115
|
-
input: Record<string, unknown>;
|
|
116
|
-
workflowStep?: WorkflowStepLike;
|
|
117
|
-
callbacks?: WorkerToolDispatchCallbacks;
|
|
118
|
-
onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>;
|
|
119
|
-
onRetryAttempt?: () => void | Promise<void>;
|
|
120
|
-
transientHttpRetrySafe?: boolean;
|
|
121
|
-
durableCallReceiptKey?: string | null;
|
|
122
|
-
executionAuthScopeDigest?: string | null;
|
|
123
|
-
providerIdempotencyKey?: string | null;
|
|
124
|
-
runtimeTimeoutMs?: number;
|
|
125
|
-
directOptions?: WorkerToolDirectOptions;
|
|
126
|
-
}) => Promise<ToolExecuteResult>;
|
|
127
|
-
|
|
128
|
-
export type WorkerToolDispatchPerfRecorder = (input: {
|
|
129
|
-
phase: string;
|
|
130
|
-
ms?: number;
|
|
131
|
-
extra?: Record<string, unknown>;
|
|
132
|
-
}) => void;
|
|
133
|
-
|
|
134
|
-
export type ToolDispatchStore = {
|
|
135
|
-
receiptStore?: WorkerRuntimeReceiptStore;
|
|
136
|
-
serializeDurableValue: <T>(value: T) => T;
|
|
137
|
-
deserializeDurableValue: <T>(value: T) => T;
|
|
138
|
-
allowLocalRetryReceipts?: boolean;
|
|
139
|
-
runtimeReceiptHeartbeatIntervalMs?: number;
|
|
140
|
-
runtimeReceiptLeaseTtlMs?: number;
|
|
141
|
-
};
|
|
142
|
-
|
|
143
|
-
export type ToolDispatchGovernor = {
|
|
144
|
-
governor: Pick<
|
|
145
|
-
PlayExecutionGovernor,
|
|
146
|
-
| 'policy'
|
|
147
|
-
| 'acquireToolSlot'
|
|
148
|
-
| 'suggestedParallelism'
|
|
149
|
-
| 'chargeBudget'
|
|
150
|
-
| 'reportProviderBackpressure'
|
|
151
|
-
>;
|
|
152
|
-
resolvePacing: WorkerPacingResolver;
|
|
153
|
-
budgetMeter?: WorkerWorkBudgetMeter;
|
|
154
|
-
};
|
|
155
|
-
|
|
156
|
-
export type ToolExecutor = {
|
|
157
|
-
resolveToolActionCacheVersion: WorkerToolActionCacheVersionResolver;
|
|
158
|
-
resolveToolAuthScopeDigest: WorkerToolAuthScopeDigestResolver;
|
|
159
|
-
// REQUIRED: the AUTH_SCOPE_CHANGED re-claim depends on evicting the cached
|
|
160
|
-
// digest before re-resolving. When this was optional, the root (top-level)
|
|
161
|
-
// scheduler was constructed without it: the requeue silently reused the
|
|
162
|
-
// stale digest, re-claimed the SAME receipt key, collided with its own
|
|
163
|
-
// still-running receipt, and the run stalled until the whole workflow
|
|
164
|
-
// invocation was retried minutes later. A missing wiring must be a type
|
|
165
|
-
// error, not a 5-minute invisible stall.
|
|
166
|
-
invalidateToolAuthScopeDigest: (toolId: string) => void;
|
|
167
|
-
executeTool: WorkerToolDispatchExecuteTool;
|
|
168
|
-
callbacks?: WorkerToolDispatchCallbacks;
|
|
169
|
-
// Observability hook for the bounded AUTH_SCOPE_CHANGED re-claim: called
|
|
170
|
-
// once per requeued group so the runner can emit a durable run-log line
|
|
171
|
-
// (marker: auth_scope_changed_reclaim). The credential-swap transition must
|
|
172
|
-
// be visible in run logs, not a silent continuation.
|
|
173
|
-
onAuthScopeReclaim?: (input: {
|
|
174
|
-
toolId: string;
|
|
175
|
-
requestIds: string[];
|
|
176
|
-
previousReceiptKeys: Array<string | null>;
|
|
177
|
-
nextReceiptKeys: string[];
|
|
178
|
-
}) => void;
|
|
179
|
-
};
|
|
180
|
-
|
|
181
|
-
export type WorkerToolBatchSchedulerOptions = ToolDispatchStore &
|
|
182
|
-
ToolDispatchGovernor &
|
|
183
|
-
ToolExecutor & {
|
|
184
|
-
req: WorkerToolDispatchRunRequest;
|
|
185
|
-
nowMs: () => number;
|
|
186
|
-
batchGraceMs?: number;
|
|
187
|
-
recordPerfTrace?: WorkerToolDispatchPerfRecorder;
|
|
188
|
-
abortSignal?: AbortSignal;
|
|
189
|
-
onRequestsSettled?: (count: number) => void;
|
|
190
|
-
runtimeDeadlineMs?: number;
|
|
191
|
-
persistenceLatch?: RuntimePersistenceLatch;
|
|
192
|
-
receiptSalvage?: ReceiptSalvageBuffer;
|
|
193
|
-
};
|
|
194
|
-
|
|
195
|
-
type WorkerToolBatchRequest = {
|
|
196
|
-
id: string;
|
|
197
|
-
cacheKey: string;
|
|
198
|
-
receiptKey: string | null;
|
|
199
|
-
executionAuthScopeDigest: string;
|
|
200
|
-
force: boolean;
|
|
201
|
-
runForceRefresh: boolean;
|
|
202
|
-
runForceFailedRefresh: boolean;
|
|
203
|
-
staleAfterSeconds?: number | null;
|
|
204
|
-
receiptWaitMaxAttempts: number;
|
|
205
|
-
authScopeRetryUsed?: boolean;
|
|
206
|
-
runtimeTimeoutMs?: number;
|
|
207
|
-
toolId: string;
|
|
208
|
-
input: Record<string, unknown>;
|
|
209
|
-
workflowStep?: WorkflowStepLike;
|
|
210
|
-
directOptions?: WorkerToolDirectOptions;
|
|
211
|
-
resolve: (value: unknown) => void;
|
|
212
|
-
reject: (error: unknown) => void;
|
|
213
|
-
};
|
|
214
|
-
|
|
215
|
-
type ClaimedWorkerToolBatchRequest = {
|
|
216
|
-
request: WorkerToolBatchRequest;
|
|
217
|
-
receiptKey: string | null;
|
|
218
|
-
receiptLeaseId: string | null;
|
|
219
|
-
receiptStatus: WorkerRuntimeReceipt['status'] | null;
|
|
220
|
-
providerForce?: boolean;
|
|
221
|
-
followers: WorkerToolBatchRequest[];
|
|
222
|
-
};
|
|
223
|
-
|
|
224
|
-
type PreparedWorkerToolBatchRequests = {
|
|
225
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[];
|
|
226
|
-
deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[];
|
|
227
|
-
};
|
|
228
|
-
|
|
229
|
-
type WorkerToolCompletionResult =
|
|
230
|
-
| { ok: true; value: unknown }
|
|
231
|
-
| { ok: false; error: unknown };
|
|
232
|
-
|
|
233
|
-
const WORKER_TOOL_BATCH_GRACE_MS = 250;
|
|
234
|
-
const COMPLETE_RECEIPT_RETRY_BACKOFFS_MS = [1_000, 3_000, 9_000] as const;
|
|
235
|
-
const COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS = 10_000;
|
|
236
|
-
export const WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_BYTES = 32 * 1024 * 1024;
|
|
237
|
-
export const WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_MEMBERS = 500;
|
|
238
|
-
export const WORKER_TOOL_RECEIPT_RECOVERY_MAX_CONCURRENT_READS = 25;
|
|
239
|
-
|
|
240
|
-
type DurableToolCompletionEntry = {
|
|
241
|
-
index: number;
|
|
242
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
243
|
-
ownerResult: unknown;
|
|
244
|
-
receipt: {
|
|
245
|
-
runId: string;
|
|
246
|
-
runAttempt: number;
|
|
247
|
-
key: string;
|
|
248
|
-
leaseId: string | null;
|
|
249
|
-
output: unknown;
|
|
250
|
-
};
|
|
251
|
-
serializedBytes: number;
|
|
252
|
-
};
|
|
253
|
-
|
|
254
|
-
type DurableToolCompletionBatch = {
|
|
255
|
-
entries: DurableToolCompletionEntry[];
|
|
256
|
-
bytes: number;
|
|
257
|
-
};
|
|
258
|
-
|
|
259
|
-
const jsonByteLength = (value: unknown): number =>
|
|
260
|
-
new TextEncoder().encode(JSON.stringify(value)).byteLength;
|
|
261
|
-
|
|
262
|
-
async function mapWithConcurrency<T>(
|
|
263
|
-
values: T[],
|
|
264
|
-
concurrency: number,
|
|
265
|
-
run: (value: T, index: number) => Promise<void>,
|
|
266
|
-
): Promise<void> {
|
|
267
|
-
const workerCount = Math.max(
|
|
268
|
-
1,
|
|
269
|
-
Math.min(Math.floor(concurrency), values.length),
|
|
270
|
-
);
|
|
271
|
-
let nextIndex = 0;
|
|
272
|
-
await Promise.all(
|
|
273
|
-
Array.from({ length: workerCount }, async () => {
|
|
274
|
-
while (true) {
|
|
275
|
-
const index = nextIndex;
|
|
276
|
-
nextIndex += 1;
|
|
277
|
-
if (index >= values.length) return;
|
|
278
|
-
await run(values[index]!, index);
|
|
279
|
-
}
|
|
280
|
-
}),
|
|
281
|
-
);
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
async function sleepWorkerMs(ms: number): Promise<void> {
|
|
285
|
-
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
export class RuntimeReceiptPersistenceError extends Error {
|
|
289
|
-
readonly receiptKeys: string[];
|
|
290
|
-
readonly errors: unknown[];
|
|
291
|
-
|
|
292
|
-
constructor(input: {
|
|
293
|
-
message: string;
|
|
294
|
-
receiptKeys: string[];
|
|
295
|
-
errors?: unknown[];
|
|
296
|
-
cause?: unknown;
|
|
297
|
-
}) {
|
|
298
|
-
super(input.message, { cause: input.cause });
|
|
299
|
-
this.name = 'RuntimeReceiptPersistenceError';
|
|
300
|
-
this.receiptKeys = input.receiptKeys;
|
|
301
|
-
this.errors =
|
|
302
|
-
input.errors ?? (input.cause === undefined ? [] : [input.cause]);
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
function receiptPersistenceError(input: {
|
|
307
|
-
operation: 'mark-running' | 'complete' | 'fail';
|
|
308
|
-
receiptKeys: string[];
|
|
309
|
-
cause: unknown;
|
|
310
|
-
}): RuntimeReceiptPersistenceError {
|
|
311
|
-
const keys = input.receiptKeys.filter((key) => key.trim().length > 0);
|
|
312
|
-
const keySummary =
|
|
313
|
-
keys.length === 0
|
|
314
|
-
? 'unknown receipt'
|
|
315
|
-
: keys.length === 1
|
|
316
|
-
? keys[0]!
|
|
317
|
-
: `${keys.length} receipts`;
|
|
318
|
-
return new RuntimeReceiptPersistenceError({
|
|
319
|
-
message: `Runtime receipt ${input.operation} persistence failed for ${keySummary}.`,
|
|
320
|
-
receiptKeys: keys,
|
|
321
|
-
cause: input.cause,
|
|
322
|
-
});
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
function buildBatchDurableCallReceiptKey(input: {
|
|
326
|
-
req: WorkerToolDispatchRunRequest;
|
|
327
|
-
batchOperation: string;
|
|
328
|
-
receiptKeys: string[];
|
|
329
|
-
}): string {
|
|
330
|
-
return buildDurableToolAggregateReceiptKey({
|
|
331
|
-
receiptKeys: input.receiptKeys,
|
|
332
|
-
prefix: 'batch',
|
|
333
|
-
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
334
|
-
orgId: input.req.orgId,
|
|
335
|
-
toolId: input.batchOperation,
|
|
336
|
-
}),
|
|
337
|
-
});
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
function buildBatchProviderIdempotencyKey(input: {
|
|
341
|
-
aggregateReceiptKey: string;
|
|
342
|
-
receiptKeys: string[];
|
|
343
|
-
providerIdempotencyKeys: string[];
|
|
344
|
-
}): string {
|
|
345
|
-
return buildDurableToolAggregateProviderIdempotencyKey({
|
|
346
|
-
aggregateReceiptKey: input.aggregateReceiptKey,
|
|
347
|
-
receiptKeys: input.receiptKeys,
|
|
348
|
-
providerIdempotencyKeys: input.providerIdempotencyKeys,
|
|
349
|
-
});
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
function durableCallReceiptKeyForClaimed(input: {
|
|
353
|
-
req: WorkerToolDispatchRunRequest;
|
|
354
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
355
|
-
}): string {
|
|
356
|
-
return input.claimed.receiptKey ?? input.claimed.request.cacheKey;
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
function providerIdempotencyKeyForClaimed(input: {
|
|
360
|
-
req: WorkerToolDispatchRunRequest;
|
|
361
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
362
|
-
}): string {
|
|
363
|
-
const fallbackAttemptId =
|
|
364
|
-
(input.claimed.providerForce === true || input.claimed.request.force) &&
|
|
365
|
-
!input.claimed.receiptLeaseId
|
|
366
|
-
? input.claimed.request.id
|
|
367
|
-
: null;
|
|
368
|
-
return buildDurableToolProviderIdempotencyKey({
|
|
369
|
-
receiptKey: durableCallReceiptKeyForClaimed(input),
|
|
370
|
-
force: input.claimed.providerForce === true || input.claimed.request.force,
|
|
371
|
-
receiptLeaseId: input.claimed.receiptLeaseId,
|
|
372
|
-
fallbackAttemptId,
|
|
373
|
-
});
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
export function wrapWorkerToolResult(
|
|
377
|
-
toolId: string,
|
|
378
|
-
result: unknown,
|
|
379
|
-
metadata: ToolResultMetadataInput | null,
|
|
380
|
-
status = result == null ? 'no_result' : 'completed',
|
|
381
|
-
): ToolExecuteResult {
|
|
382
|
-
return createToolExecuteResult({
|
|
383
|
-
status,
|
|
384
|
-
result,
|
|
385
|
-
metadata: metadata ?? { toolId },
|
|
386
|
-
execution: toolExecutionMetadataForOutcome({ kind: 'live' }),
|
|
387
|
-
});
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
|
|
391
|
-
if (toolId === 'test_rate_limit') {
|
|
392
|
-
// Batched members resolve metadata through this fallback because the lean
|
|
393
|
-
// worker does not bundle the catalog. It MUST mirror the same
|
|
394
|
-
// `email_status: emailStatus({...})` contract registered in
|
|
395
|
-
// src/lib/integrations/test/index.ts so batched results normalize to the
|
|
396
|
-
// same rich email_status OBJECT (status from statusMap, catch_all as a
|
|
397
|
-
// signal) that the single-execute real-metadata path produces.
|
|
398
|
-
return {
|
|
399
|
-
toolId,
|
|
400
|
-
extractors: {
|
|
401
|
-
email_status: {
|
|
402
|
-
paths: ['email_status'],
|
|
403
|
-
emailStatus: {
|
|
404
|
-
provider: 'test',
|
|
405
|
-
rawStatus: ['email_status'],
|
|
406
|
-
catchAll: ['mx_security_gateway'],
|
|
407
|
-
statusMap: {
|
|
408
|
-
valid: { status: 'valid', verdict: 'send', verified: true },
|
|
409
|
-
invalid: { status: 'invalid', verdict: 'drop', verified: false },
|
|
410
|
-
catch_all: {
|
|
411
|
-
status: 'catch_all',
|
|
412
|
-
verdict: 'verify_next',
|
|
413
|
-
verified: false,
|
|
414
|
-
},
|
|
415
|
-
unknown: { status: 'unknown', verdict: 'hold', verified: false },
|
|
416
|
-
},
|
|
417
|
-
},
|
|
418
|
-
},
|
|
419
|
-
},
|
|
420
|
-
targetGetters: {
|
|
421
|
-
email: ['email', 'value'],
|
|
422
|
-
email_status: ['email_status'],
|
|
423
|
-
},
|
|
424
|
-
};
|
|
425
|
-
}
|
|
426
|
-
return { toolId };
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
function durableReceiptInputForTool(
|
|
430
|
-
toolId: string,
|
|
431
|
-
input: Record<string, unknown>,
|
|
432
|
-
): Record<string, unknown> {
|
|
433
|
-
if (toolId !== 'test_rate_limit') {
|
|
434
|
-
return input;
|
|
435
|
-
}
|
|
436
|
-
const { simulated_delay_ms: _simulatedDelayMs, ...semanticInput } = input;
|
|
437
|
-
return semanticInput;
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
export function workerDurableToolCallCacheKey(input: {
|
|
441
|
-
req: WorkerToolDispatchRunRequest;
|
|
442
|
-
toolId: string;
|
|
443
|
-
requestInput: Record<string, unknown>;
|
|
444
|
-
providerActionVersion: string;
|
|
445
|
-
executionAuthScopeDigest: string;
|
|
446
|
-
staleAfterSeconds?: number | null;
|
|
447
|
-
}): string {
|
|
448
|
-
return buildDurableToolCallCacheKey({
|
|
449
|
-
orgId: input.req.orgId,
|
|
450
|
-
playLocalScope: input.req.playName,
|
|
451
|
-
toolId: input.toolId,
|
|
452
|
-
requestInput: durableReceiptInputForTool(input.toolId, input.requestInput),
|
|
453
|
-
authScopeDigest: buildDurableToolCallAuthScopeDigest({
|
|
454
|
-
orgId: input.req.orgId,
|
|
455
|
-
toolId: input.toolId,
|
|
456
|
-
executionAuthScopeDigest: input.executionAuthScopeDigest,
|
|
457
|
-
}),
|
|
458
|
-
providerActionVersion: input.providerActionVersion,
|
|
459
|
-
staleAfterSeconds: input.staleAfterSeconds,
|
|
460
|
-
});
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
function workerRuntimeReceiptKey(input: {
|
|
464
|
-
req: WorkerToolDispatchRunRequest;
|
|
465
|
-
key: string;
|
|
466
|
-
}): string {
|
|
467
|
-
const orgId = input.req.orgId?.trim() || 'org';
|
|
468
|
-
if (input.key.startsWith(`ctx:${orgId}:`)) {
|
|
469
|
-
return input.key;
|
|
470
|
-
}
|
|
471
|
-
return buildScopedWorkReceiptKey({
|
|
472
|
-
orgId: input.req.orgId,
|
|
473
|
-
playName: input.req.playName,
|
|
474
|
-
key: input.key,
|
|
475
|
-
});
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
export class WorkerToolBatchScheduler {
|
|
479
|
-
private queue: WorkerToolBatchRequest[] = [];
|
|
480
|
-
private scheduled = false;
|
|
481
|
-
private drainTimer: ReturnType<typeof setTimeout> | null = null;
|
|
482
|
-
private immediateDrainQueued = false;
|
|
483
|
-
|
|
484
|
-
constructor(private readonly options: WorkerToolBatchSchedulerOptions) {}
|
|
485
|
-
|
|
486
|
-
private runtimeDeadlineRemainingMs(): number {
|
|
487
|
-
if (typeof this.options.runtimeDeadlineMs !== 'number') {
|
|
488
|
-
return Number.POSITIVE_INFINITY;
|
|
489
|
-
}
|
|
490
|
-
return Math.max(0, this.options.runtimeDeadlineMs - this.options.nowMs());
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
private async completeReceiptWithRetryLadder<T>(
|
|
494
|
-
run: () => Promise<T>,
|
|
495
|
-
onGiveUp?: (attempts: number) => void,
|
|
496
|
-
): Promise<T> {
|
|
497
|
-
if (!this.options.persistenceLatch && !this.options.receiptSalvage) {
|
|
498
|
-
return await run();
|
|
499
|
-
}
|
|
500
|
-
const latch = this.options.persistenceLatch;
|
|
501
|
-
let attempt = 0;
|
|
502
|
-
let committedToFullLadder = false;
|
|
503
|
-
const giveUp = (error: unknown): never => {
|
|
504
|
-
if (latch) {
|
|
505
|
-
latch.latchRetryBudgetExhausted = true;
|
|
506
|
-
tripRuntimePersistenceLatch(latch, error);
|
|
507
|
-
}
|
|
508
|
-
onGiveUp?.(attempt);
|
|
509
|
-
throw error;
|
|
510
|
-
};
|
|
511
|
-
while (true) {
|
|
512
|
-
try {
|
|
513
|
-
return await run();
|
|
514
|
-
} catch (error) {
|
|
515
|
-
attempt += 1;
|
|
516
|
-
if (
|
|
517
|
-
this.options.abortSignal?.aborted ||
|
|
518
|
-
attempt > COMPLETE_RECEIPT_RETRY_BACKOFFS_MS.length ||
|
|
519
|
-
(latch &&
|
|
520
|
-
shouldShortenCompleteReceiptLadder(latch, committedToFullLadder))
|
|
521
|
-
) {
|
|
522
|
-
return giveUp(error);
|
|
523
|
-
}
|
|
524
|
-
const backoffMs = COMPLETE_RECEIPT_RETRY_BACKOFFS_MS[attempt - 1]!;
|
|
525
|
-
if (
|
|
526
|
-
this.runtimeDeadlineRemainingMs() <=
|
|
527
|
-
backoffMs + COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS
|
|
528
|
-
) {
|
|
529
|
-
return giveUp(error);
|
|
530
|
-
}
|
|
531
|
-
committedToFullLadder = true;
|
|
532
|
-
await sleepWorkerMs(backoffMs);
|
|
533
|
-
}
|
|
534
|
-
}
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
private recordReceiptSalvage(input: {
|
|
538
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
539
|
-
output: unknown;
|
|
540
|
-
attempts: number;
|
|
541
|
-
}): void {
|
|
542
|
-
if (!this.options.receiptSalvage || !input.claimed.receiptKey) return;
|
|
543
|
-
this.options.receiptSalvage.push({
|
|
544
|
-
receiptKey: input.claimed.receiptKey,
|
|
545
|
-
toolId: input.claimed.request.toolId,
|
|
546
|
-
cacheKey: input.claimed.request.cacheKey,
|
|
547
|
-
output: input.output,
|
|
548
|
-
completeAttempts: input.attempts,
|
|
549
|
-
firstErrorAt: this.options.nowMs(),
|
|
550
|
-
});
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
private runtimePersistenceCircuitOpenError(
|
|
554
|
-
skippedCalls: number,
|
|
555
|
-
): Error | null {
|
|
556
|
-
const latch = this.options.persistenceLatch;
|
|
557
|
-
if (!latch?.tripped) return null;
|
|
558
|
-
latch.preventedCallCount += Math.max(1, skippedCalls);
|
|
559
|
-
return new RuntimePersistenceCircuitOpenError(latch);
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
/**
|
|
563
|
-
* Report a provider 429 / Retry-After back into the Governor's shared pacer
|
|
564
|
-
* so future acquires for this (org, provider) bucket back off across all
|
|
565
|
-
* isolates. Provider comes from the same pacing resolver the Governor uses
|
|
566
|
-
* (the worker has no local catalog), so callers pass only the toolId.
|
|
567
|
-
*/
|
|
568
|
-
private async reportBackpressure(
|
|
569
|
-
toolId: string,
|
|
570
|
-
retryAfterMs: number,
|
|
571
|
-
): Promise<void> {
|
|
572
|
-
if (retryAfterMs <= 0) return;
|
|
573
|
-
const pacing = await this.options.resolvePacing(toolId).catch(() => null);
|
|
574
|
-
if (pacing?.provider) {
|
|
575
|
-
await this.options.governor.reportProviderBackpressure({
|
|
576
|
-
provider: pacing.provider,
|
|
577
|
-
toolId,
|
|
578
|
-
retryAfterMs,
|
|
579
|
-
});
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
async execute(
|
|
584
|
-
id: string,
|
|
585
|
-
toolId: string,
|
|
586
|
-
input: Record<string, unknown>,
|
|
587
|
-
workflowStep?: WorkflowStepLike,
|
|
588
|
-
options?: {
|
|
589
|
-
force?: boolean;
|
|
590
|
-
runForceRefresh?: boolean;
|
|
591
|
-
runForceFailedRefresh?: boolean;
|
|
592
|
-
staleAfterSeconds?: number | null;
|
|
593
|
-
timeoutMs?: number;
|
|
594
|
-
receiptWaitMs?: number;
|
|
595
|
-
},
|
|
596
|
-
): Promise<unknown> {
|
|
597
|
-
this.options.budgetMeter?.count('tool');
|
|
598
|
-
const [providerActionVersion, executionAuthScopeDigest] = await Promise.all(
|
|
599
|
-
[
|
|
600
|
-
this.options.resolveToolActionCacheVersion(toolId),
|
|
601
|
-
this.options.resolveToolAuthScopeDigest(toolId),
|
|
602
|
-
],
|
|
603
|
-
);
|
|
604
|
-
return await new Promise((resolve, reject) => {
|
|
605
|
-
const queuedAt = this.options.nowMs();
|
|
606
|
-
const receiptKey = workerDurableToolCallCacheKey({
|
|
607
|
-
req: this.options.req,
|
|
608
|
-
toolId,
|
|
609
|
-
requestInput: input,
|
|
610
|
-
providerActionVersion,
|
|
611
|
-
executionAuthScopeDigest,
|
|
612
|
-
staleAfterSeconds: options?.staleAfterSeconds,
|
|
613
|
-
});
|
|
614
|
-
const queryResultDatasetRead = isQueryResultDatasetReadRequest(
|
|
615
|
-
toolId,
|
|
616
|
-
input,
|
|
617
|
-
);
|
|
618
|
-
const receiptEligible = !queryResultDatasetRead;
|
|
619
|
-
this.queue.push({
|
|
620
|
-
id,
|
|
621
|
-
cacheKey: receiptEligible ? receiptKey : '',
|
|
622
|
-
receiptKey: receiptEligible ? receiptKey : null,
|
|
623
|
-
executionAuthScopeDigest,
|
|
624
|
-
force: options?.force === true,
|
|
625
|
-
runForceRefresh: options?.runForceRefresh === true,
|
|
626
|
-
runForceFailedRefresh: options?.runForceFailedRefresh === true,
|
|
627
|
-
staleAfterSeconds: options?.staleAfterSeconds ?? null,
|
|
628
|
-
receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
629
|
-
typeof options?.receiptWaitMs === 'number'
|
|
630
|
-
? { max_wait_ms: options.receiptWaitMs }
|
|
631
|
-
: input,
|
|
632
|
-
),
|
|
633
|
-
runtimeTimeoutMs: options?.timeoutMs,
|
|
634
|
-
toolId,
|
|
635
|
-
input,
|
|
636
|
-
workflowStep,
|
|
637
|
-
...(queryResultDatasetRead
|
|
638
|
-
? {
|
|
639
|
-
directOptions: {
|
|
640
|
-
customerDbDataset: {
|
|
641
|
-
offset: 0,
|
|
642
|
-
pageSize: QUERY_RESULT_DATASET_PAGE_SIZE,
|
|
643
|
-
},
|
|
644
|
-
},
|
|
645
|
-
}
|
|
646
|
-
: {}),
|
|
647
|
-
resolve: (value) => {
|
|
648
|
-
this.options.recordPerfTrace?.({
|
|
649
|
-
phase: 'runner.tool.request',
|
|
650
|
-
ms: this.options.nowMs() - queuedAt,
|
|
651
|
-
extra: { id, toolId },
|
|
652
|
-
});
|
|
653
|
-
resolve(value);
|
|
654
|
-
},
|
|
655
|
-
reject,
|
|
656
|
-
});
|
|
657
|
-
this.scheduleDrain();
|
|
658
|
-
});
|
|
659
|
-
}
|
|
660
|
-
|
|
661
|
-
private scheduleDrain(): void {
|
|
662
|
-
if (this.shouldDrainQueuedToolsImmediately()) {
|
|
663
|
-
this.scheduleImmediateDrain();
|
|
664
|
-
return;
|
|
665
|
-
}
|
|
666
|
-
if (this.scheduled) return;
|
|
667
|
-
this.scheduled = true;
|
|
668
|
-
this.drainTimer = setTimeout(() => {
|
|
669
|
-
this.drainTimer = null;
|
|
670
|
-
this.scheduleImmediateDrain();
|
|
671
|
-
}, this.options.batchGraceMs ?? WORKER_TOOL_BATCH_GRACE_MS);
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
private scheduleImmediateDrain(): void {
|
|
675
|
-
if (this.drainTimer) {
|
|
676
|
-
clearTimeout(this.drainTimer);
|
|
677
|
-
this.drainTimer = null;
|
|
678
|
-
}
|
|
679
|
-
if (this.immediateDrainQueued) return;
|
|
680
|
-
this.scheduled = true;
|
|
681
|
-
this.immediateDrainQueued = true;
|
|
682
|
-
queueMicrotask(() => {
|
|
683
|
-
this.immediateDrainQueued = false;
|
|
684
|
-
void this.drain().catch((error) => {
|
|
685
|
-
const queued = this.queue;
|
|
686
|
-
this.queue = [];
|
|
687
|
-
this.scheduled = false;
|
|
688
|
-
if (queued.length > 0) {
|
|
689
|
-
this.rejectRawRequests(queued, error);
|
|
690
|
-
}
|
|
691
|
-
});
|
|
692
|
-
});
|
|
693
|
-
}
|
|
694
|
-
|
|
695
|
-
private shouldDrainQueuedToolsImmediately(): boolean {
|
|
696
|
-
if (!this.hasBatchableQueuedTool()) return true;
|
|
697
|
-
const byTool = new Map<string, number>();
|
|
698
|
-
for (const request of this.queue) {
|
|
699
|
-
if (request.toolId === 'test_wait_for_event') continue;
|
|
700
|
-
const strategy = getPlayRuntimeBatchStrategy(request.toolId);
|
|
701
|
-
if (!strategy) continue;
|
|
702
|
-
const nextCount = (byTool.get(request.toolId) ?? 0) + 1;
|
|
703
|
-
if (nextCount >= strategy.maxBatchSize) {
|
|
704
|
-
return true;
|
|
705
|
-
}
|
|
706
|
-
byTool.set(request.toolId, nextCount);
|
|
707
|
-
}
|
|
708
|
-
return false;
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
private hasBatchableQueuedTool(): boolean {
|
|
712
|
-
return this.queue.some(
|
|
713
|
-
(request) =>
|
|
714
|
-
request.toolId !== 'test_wait_for_event' &&
|
|
715
|
-
getPlayRuntimeBatchStrategy(request.toolId) !== null,
|
|
716
|
-
);
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
private async drain(): Promise<void> {
|
|
720
|
-
const requests = this.queue;
|
|
721
|
-
this.queue = [];
|
|
722
|
-
this.scheduled = false;
|
|
723
|
-
const drainStartedAt = this.options.nowMs();
|
|
724
|
-
await Promise.all(
|
|
725
|
-
[...groupWorkerToolRequestsByTool(requests).entries()].map(
|
|
726
|
-
async ([toolId, groupedRequests]) => {
|
|
727
|
-
try {
|
|
728
|
-
await this.executeToolGroup(toolId, groupedRequests);
|
|
729
|
-
} catch (error) {
|
|
730
|
-
this.rejectRawRequests(groupedRequests, error);
|
|
731
|
-
}
|
|
732
|
-
},
|
|
733
|
-
),
|
|
734
|
-
);
|
|
735
|
-
this.options.recordPerfTrace?.({
|
|
736
|
-
phase: 'runner.tool.drain',
|
|
737
|
-
ms: this.options.nowMs() - drainStartedAt,
|
|
738
|
-
extra: { requests: requests.length },
|
|
739
|
-
});
|
|
740
|
-
if (this.queue.length > 0) {
|
|
741
|
-
this.scheduleDrain();
|
|
742
|
-
}
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
private async waitForDurableToolReceipt(input: {
|
|
746
|
-
receiptKey: string;
|
|
747
|
-
maxAttempts: number;
|
|
748
|
-
}): Promise<unknown> {
|
|
749
|
-
if (!this.options.receiptStore) {
|
|
750
|
-
throw new Error('No receipt store.');
|
|
751
|
-
}
|
|
752
|
-
if (!this.options.receiptStore.getReceipt) {
|
|
753
|
-
throw new Error('No receipt lookup.');
|
|
754
|
-
}
|
|
755
|
-
this.options.budgetMeter?.count('receipt');
|
|
756
|
-
const receipt = await waitForCompletedRuntimeReceipt({
|
|
757
|
-
receiptKey: input.receiptKey,
|
|
758
|
-
maxAttempts: input.maxAttempts,
|
|
759
|
-
abortSignal: this.options.abortSignal,
|
|
760
|
-
store: {
|
|
761
|
-
getMany: async (receiptKeys) => {
|
|
762
|
-
const receipts = new Map<string, RuntimeStepReceipt>();
|
|
763
|
-
await Promise.all(
|
|
764
|
-
receiptKeys.map(async (key) => {
|
|
765
|
-
const receipt = await this.options.receiptStore!.getReceipt!({
|
|
766
|
-
playName: this.options.req.playName,
|
|
767
|
-
key,
|
|
768
|
-
});
|
|
769
|
-
if (!receipt) return;
|
|
770
|
-
receipts.set(key, {
|
|
771
|
-
key: receipt.key,
|
|
772
|
-
status: receipt.status,
|
|
773
|
-
output: receipt.output,
|
|
774
|
-
error: receipt.error ?? undefined,
|
|
775
|
-
runId: receipt.runId ?? null,
|
|
776
|
-
});
|
|
777
|
-
}),
|
|
778
|
-
);
|
|
779
|
-
return receipts;
|
|
780
|
-
},
|
|
781
|
-
},
|
|
782
|
-
});
|
|
783
|
-
return markWorkerToolReceiptResultCached(
|
|
784
|
-
this.options.deserializeDurableValue(receipt.output),
|
|
785
|
-
input.receiptKey,
|
|
786
|
-
input.receiptKey,
|
|
787
|
-
);
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
private settleRequests(
|
|
791
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
792
|
-
result: unknown,
|
|
793
|
-
): void {
|
|
794
|
-
claimed.request.resolve(result);
|
|
795
|
-
for (const follower of claimed.followers) {
|
|
796
|
-
follower.resolve(
|
|
797
|
-
claimed.receiptKey
|
|
798
|
-
? markWorkerToolReceiptResultExecution(result, {
|
|
799
|
-
kind: 'in_flight',
|
|
800
|
-
receiptKey: claimed.receiptKey,
|
|
801
|
-
attachedToReceiptKey: claimed.receiptKey,
|
|
802
|
-
})
|
|
803
|
-
: result,
|
|
804
|
-
);
|
|
805
|
-
}
|
|
806
|
-
this.options.onRequestsSettled?.(1 + claimed.followers.length);
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
private rejectRequests(
|
|
810
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
811
|
-
error: unknown,
|
|
812
|
-
): void {
|
|
813
|
-
claimed.request.reject(error);
|
|
814
|
-
for (const follower of claimed.followers) {
|
|
815
|
-
follower.reject(error);
|
|
816
|
-
}
|
|
817
|
-
this.options.onRequestsSettled?.(1 + claimed.followers.length);
|
|
818
|
-
}
|
|
819
|
-
|
|
820
|
-
private rejectRawRequests(
|
|
821
|
-
requests: WorkerToolBatchRequest[],
|
|
822
|
-
error: unknown,
|
|
823
|
-
): void {
|
|
824
|
-
for (const request of requests) {
|
|
825
|
-
request.reject(error);
|
|
826
|
-
}
|
|
827
|
-
this.options.onRequestsSettled?.(requests.length);
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
private async requeueAfterAuthScopeChanged(
|
|
831
|
-
requests: WorkerToolBatchRequest[],
|
|
832
|
-
error: unknown,
|
|
833
|
-
): Promise<boolean> {
|
|
834
|
-
if (!(error instanceof ToolExecuteAuthScopeChangedError)) {
|
|
835
|
-
return false;
|
|
836
|
-
}
|
|
837
|
-
const [first] = requests;
|
|
838
|
-
if (!first) return false;
|
|
839
|
-
if (requests.some((request) => request.authScopeRetryUsed === true)) {
|
|
840
|
-
return false;
|
|
841
|
-
}
|
|
842
|
-
this.options.invalidateToolAuthScopeDigest(first.toolId);
|
|
843
|
-
const [providerActionVersion, executionAuthScopeDigest] = await Promise.all(
|
|
844
|
-
[
|
|
845
|
-
this.options.resolveToolActionCacheVersion(first.toolId),
|
|
846
|
-
this.options.resolveToolAuthScopeDigest(first.toolId),
|
|
847
|
-
],
|
|
848
|
-
);
|
|
849
|
-
const requeued = requests.map((request) => ({
|
|
850
|
-
request,
|
|
851
|
-
receiptKey: workerDurableToolCallCacheKey({
|
|
852
|
-
req: this.options.req,
|
|
853
|
-
toolId: request.toolId,
|
|
854
|
-
requestInput: request.input,
|
|
855
|
-
providerActionVersion,
|
|
856
|
-
executionAuthScopeDigest,
|
|
857
|
-
staleAfterSeconds: request.staleAfterSeconds,
|
|
858
|
-
}),
|
|
859
|
-
}));
|
|
860
|
-
// If the re-resolved scope produced the exact same receipt keys, the
|
|
861
|
-
// credential identity did NOT actually change from this isolate's point of
|
|
862
|
-
// view (stale cache, no-op invalidation, or a racing revert). Requeuing
|
|
863
|
-
// would claim the same key this group just left `running` and stall on its
|
|
864
|
-
// own abandoned receipt until the lease TTL. Refuse instead: the caller
|
|
865
|
-
// rejects the request with the typed 409 error — loud and immediate.
|
|
866
|
-
if (
|
|
867
|
-
requeued.every(
|
|
868
|
-
(entry) => entry.receiptKey === (entry.request.receiptKey ?? null),
|
|
869
|
-
)
|
|
870
|
-
) {
|
|
871
|
-
return false;
|
|
872
|
-
}
|
|
873
|
-
for (const { request, receiptKey } of requeued) {
|
|
874
|
-
this.queue.push({
|
|
875
|
-
...request,
|
|
876
|
-
cacheKey: receiptKey,
|
|
877
|
-
receiptKey,
|
|
878
|
-
executionAuthScopeDigest,
|
|
879
|
-
authScopeRetryUsed: true,
|
|
880
|
-
});
|
|
881
|
-
}
|
|
882
|
-
this.options.onAuthScopeReclaim?.({
|
|
883
|
-
toolId: first.toolId,
|
|
884
|
-
requestIds: requests.map((request) => request.id),
|
|
885
|
-
previousReceiptKeys: requests.map(
|
|
886
|
-
(request) => request.receiptKey ?? null,
|
|
887
|
-
),
|
|
888
|
-
nextReceiptKeys: requeued.map((entry) => entry.receiptKey),
|
|
889
|
-
});
|
|
890
|
-
this.scheduleDrain();
|
|
891
|
-
return true;
|
|
892
|
-
}
|
|
893
|
-
|
|
894
|
-
private async reclaimTimedOutDurableToolReceiptGroup(input: {
|
|
895
|
-
group: WorkerToolBatchRequest[];
|
|
896
|
-
receiptKey: string;
|
|
897
|
-
runningReceipt: WorkerRuntimeReceipt;
|
|
898
|
-
waitError: unknown;
|
|
899
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
900
|
-
const [request, ...followers] = input.group;
|
|
901
|
-
if (!request || !this.options.receiptStore) {
|
|
902
|
-
this.rejectRawRequests(input.group, input.waitError);
|
|
903
|
-
return [];
|
|
904
|
-
}
|
|
905
|
-
if (
|
|
906
|
-
!canReclaimTimedOutWorkerToolReceipt({
|
|
907
|
-
ownerRunId: input.runningReceipt.runId,
|
|
908
|
-
currentRunId: this.options.req.runId,
|
|
909
|
-
})
|
|
910
|
-
) {
|
|
911
|
-
this.rejectRawRequests(input.group, input.waitError);
|
|
912
|
-
return [];
|
|
913
|
-
}
|
|
914
|
-
let claim: WorkerRuntimeReceiptClaim;
|
|
915
|
-
try {
|
|
916
|
-
this.options.budgetMeter?.count('receipt');
|
|
917
|
-
claim = await this.options.receiptStore.claimReceipt({
|
|
918
|
-
playName: this.options.req.playName,
|
|
919
|
-
runId: this.options.req.runId,
|
|
920
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
921
|
-
key: input.receiptKey,
|
|
922
|
-
leaseAware: true,
|
|
923
|
-
leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
|
|
924
|
-
reclaimRunning: true,
|
|
925
|
-
});
|
|
926
|
-
} catch (error) {
|
|
927
|
-
this.rejectRawRequests(input.group, error);
|
|
928
|
-
return [];
|
|
929
|
-
}
|
|
930
|
-
if (claim.disposition === 'claimed') {
|
|
931
|
-
return [
|
|
932
|
-
{
|
|
933
|
-
request,
|
|
934
|
-
receiptKey: input.receiptKey,
|
|
935
|
-
receiptLeaseId: claim.receipt.leaseId ?? null,
|
|
936
|
-
receiptStatus: claim.receipt.status,
|
|
937
|
-
providerForce: true,
|
|
938
|
-
followers,
|
|
939
|
-
},
|
|
940
|
-
];
|
|
941
|
-
}
|
|
942
|
-
if (claim.disposition === 'reused') {
|
|
943
|
-
const result = markWorkerToolReceiptResultCached(
|
|
944
|
-
this.options.deserializeDurableValue(claim.receipt.output),
|
|
945
|
-
request.cacheKey,
|
|
946
|
-
input.receiptKey,
|
|
947
|
-
);
|
|
948
|
-
for (const pending of input.group) {
|
|
949
|
-
pending.resolve(result);
|
|
950
|
-
}
|
|
951
|
-
this.options.onRequestsSettled?.(input.group.length);
|
|
952
|
-
return [];
|
|
953
|
-
}
|
|
954
|
-
if (claim.disposition === 'failed') {
|
|
955
|
-
this.rejectRawRequests(
|
|
956
|
-
input.group,
|
|
957
|
-
new Error(
|
|
958
|
-
`Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
|
|
959
|
-
),
|
|
960
|
-
);
|
|
961
|
-
return [];
|
|
962
|
-
}
|
|
963
|
-
this.rejectRawRequests(input.group, input.waitError);
|
|
964
|
-
return [];
|
|
965
|
-
}
|
|
966
|
-
|
|
967
|
-
private async claimForcedDurableToolReceiptGroupAfterWait(input: {
|
|
968
|
-
group: WorkerToolBatchRequest[];
|
|
969
|
-
receiptKey: string;
|
|
970
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
971
|
-
const [request, ...followers] = input.group;
|
|
972
|
-
if (!request || !this.options.receiptStore) {
|
|
973
|
-
this.rejectRawRequests(
|
|
974
|
-
input.group,
|
|
975
|
-
new Error(
|
|
976
|
-
`No receipt store for forced durable tool call ${input.receiptKey}.`,
|
|
977
|
-
),
|
|
978
|
-
);
|
|
979
|
-
return [];
|
|
980
|
-
}
|
|
981
|
-
let claim: WorkerRuntimeReceiptClaim;
|
|
982
|
-
try {
|
|
983
|
-
this.options.budgetMeter?.count('receipt');
|
|
984
|
-
claim = await this.options.receiptStore.claimReceipt({
|
|
985
|
-
playName: this.options.req.playName,
|
|
986
|
-
runId: this.options.req.runId,
|
|
987
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
988
|
-
key: input.receiptKey,
|
|
989
|
-
leaseAware: true,
|
|
990
|
-
leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
|
|
991
|
-
forceRefresh: true,
|
|
992
|
-
reclaimRunning: true,
|
|
993
|
-
});
|
|
994
|
-
} catch (error) {
|
|
995
|
-
this.rejectRawRequests(input.group, error);
|
|
996
|
-
return [];
|
|
997
|
-
}
|
|
998
|
-
if (claim.disposition === 'claimed') {
|
|
999
|
-
return [
|
|
1000
|
-
{
|
|
1001
|
-
request,
|
|
1002
|
-
receiptKey: input.receiptKey,
|
|
1003
|
-
receiptLeaseId: claim.receipt.leaseId ?? null,
|
|
1004
|
-
receiptStatus: claim.receipt.status,
|
|
1005
|
-
providerForce: true,
|
|
1006
|
-
followers,
|
|
1007
|
-
},
|
|
1008
|
-
];
|
|
1009
|
-
}
|
|
1010
|
-
const error =
|
|
1011
|
-
claim.disposition === 'failed'
|
|
1012
|
-
? new Error(
|
|
1013
|
-
`Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
|
|
1014
|
-
)
|
|
1015
|
-
: new Error(
|
|
1016
|
-
`Forced durable tool call ${input.receiptKey} could not claim execution ownership after the active lease resolved; disposition=${claim.disposition}.`,
|
|
1017
|
-
);
|
|
1018
|
-
this.rejectRawRequests(input.group, error);
|
|
1019
|
-
return [];
|
|
1020
|
-
}
|
|
1021
|
-
|
|
1022
|
-
private async failureForRejectedToolRequest(
|
|
1023
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
1024
|
-
error: unknown,
|
|
1025
|
-
): Promise<unknown> {
|
|
1026
|
-
try {
|
|
1027
|
-
if (
|
|
1028
|
-
isPlayExecutionSuspendedError(error) ||
|
|
1029
|
-
isPlayRowExecutionSuspendedError(error)
|
|
1030
|
-
) {
|
|
1031
|
-
await this.releaseDurableToolRequest(claimed, error);
|
|
1032
|
-
} else {
|
|
1033
|
-
await this.failDurableToolRequest(claimed, error);
|
|
1034
|
-
}
|
|
1035
|
-
return error;
|
|
1036
|
-
} catch (receiptError) {
|
|
1037
|
-
return receiptPersistenceError({
|
|
1038
|
-
operation: 'fail',
|
|
1039
|
-
receiptKeys: claimed.receiptKey ? [claimed.receiptKey] : [],
|
|
1040
|
-
cause: new AggregateError(
|
|
1041
|
-
[error, receiptError],
|
|
1042
|
-
'Receipt fail mark failed',
|
|
1043
|
-
),
|
|
1044
|
-
});
|
|
1045
|
-
}
|
|
1046
|
-
}
|
|
1047
|
-
|
|
1048
|
-
private async claimDurableToolReceiptGroups(
|
|
1049
|
-
groups: ReturnType<
|
|
1050
|
-
typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
|
|
1051
|
-
>['durableGroups'],
|
|
1052
|
-
): Promise<
|
|
1053
|
-
Array<{
|
|
1054
|
-
group: ReturnType<
|
|
1055
|
-
typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
|
|
1056
|
-
>['durableGroups'][number];
|
|
1057
|
-
receiptKey: string;
|
|
1058
|
-
claim: WorkerRuntimeReceiptClaim;
|
|
1059
|
-
}>
|
|
1060
|
-
> {
|
|
1061
|
-
if (!this.options.receiptStore) return [];
|
|
1062
|
-
const planned = groups.map((group) => ({
|
|
1063
|
-
group,
|
|
1064
|
-
receiptKey: workerRuntimeReceiptKey({
|
|
1065
|
-
req: this.options.req,
|
|
1066
|
-
key: group.claimableReceiptKey,
|
|
1067
|
-
}),
|
|
1068
|
-
}));
|
|
1069
|
-
const claimOne = async (entry: (typeof planned)[number]) => {
|
|
1070
|
-
this.options.budgetMeter?.count('receipt');
|
|
1071
|
-
return {
|
|
1072
|
-
...entry,
|
|
1073
|
-
claim: await this.options.receiptStore!.claimReceipt({
|
|
1074
|
-
playName: this.options.req.playName,
|
|
1075
|
-
runId: this.options.req.runId,
|
|
1076
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
1077
|
-
key: entry.receiptKey,
|
|
1078
|
-
leaseAware: true,
|
|
1079
|
-
leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
|
|
1080
|
-
...(entry.group.forceDurableRefresh
|
|
1081
|
-
? { forceRefresh: true }
|
|
1082
|
-
: entry.group.forceFailedDurableRefresh
|
|
1083
|
-
? { forceFailedRefresh: true }
|
|
1084
|
-
: {}),
|
|
1085
|
-
}),
|
|
1086
|
-
};
|
|
1087
|
-
};
|
|
1088
|
-
if (!this.options.receiptStore.claimReceipts) {
|
|
1089
|
-
return await Promise.all(planned.map(claimOne));
|
|
1090
|
-
}
|
|
1091
|
-
const claimed: Array<{
|
|
1092
|
-
group: (typeof planned)[number]['group'];
|
|
1093
|
-
receiptKey: string;
|
|
1094
|
-
claim: WorkerRuntimeReceiptClaim;
|
|
1095
|
-
}> = [];
|
|
1096
|
-
for (const claimMode of ['normal', 'failed', 'full'] as const) {
|
|
1097
|
-
const entries = planned.filter((entry) =>
|
|
1098
|
-
claimMode === 'full'
|
|
1099
|
-
? entry.group.forceDurableRefresh
|
|
1100
|
-
: claimMode === 'failed'
|
|
1101
|
-
? !entry.group.forceDurableRefresh &&
|
|
1102
|
-
entry.group.forceFailedDurableRefresh
|
|
1103
|
-
: !entry.group.forceDurableRefresh &&
|
|
1104
|
-
!entry.group.forceFailedDurableRefresh,
|
|
1105
|
-
);
|
|
1106
|
-
if (entries.length === 0) continue;
|
|
1107
|
-
// Bulk claim is a single service-binding RPC regardless of key count, so
|
|
1108
|
-
// charge one subrequest per RPC issued (once per claimMode page), not one
|
|
1109
|
-
// per receipt key.
|
|
1110
|
-
this.options.budgetMeter?.count('receipt');
|
|
1111
|
-
const claims = await this.options.receiptStore.claimReceipts({
|
|
1112
|
-
playName: this.options.req.playName,
|
|
1113
|
-
runId: this.options.req.runId,
|
|
1114
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
1115
|
-
keys: entries.map((entry) => entry.receiptKey),
|
|
1116
|
-
leaseAware: true,
|
|
1117
|
-
leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
|
|
1118
|
-
...(claimMode === 'full'
|
|
1119
|
-
? { forceRefresh: true }
|
|
1120
|
-
: claimMode === 'failed'
|
|
1121
|
-
? { forceFailedRefresh: true }
|
|
1122
|
-
: {}),
|
|
1123
|
-
});
|
|
1124
|
-
entries.forEach((entry, index) => {
|
|
1125
|
-
const claim = claims[index];
|
|
1126
|
-
if (!claim) {
|
|
1127
|
-
throw new Error(
|
|
1128
|
-
`Runtime receipt batch claim did not return receipt ${entry.receiptKey}.`,
|
|
1129
|
-
);
|
|
1130
|
-
}
|
|
1131
|
-
claimed.push({ ...entry, claim });
|
|
1132
|
-
});
|
|
1133
|
-
}
|
|
1134
|
-
return claimed;
|
|
1135
|
-
}
|
|
1136
|
-
|
|
1137
|
-
private async prepareDurableToolRequests(
|
|
1138
|
-
requests: WorkerToolBatchRequest[],
|
|
1139
|
-
): Promise<PreparedWorkerToolBatchRequests> {
|
|
1140
|
-
if (!this.options.receiptStore) {
|
|
1141
|
-
return {
|
|
1142
|
-
claimedRequests: requests.map((request) => ({
|
|
1143
|
-
request,
|
|
1144
|
-
receiptKey: null,
|
|
1145
|
-
receiptLeaseId: null,
|
|
1146
|
-
receiptStatus: null,
|
|
1147
|
-
followers: [],
|
|
1148
|
-
})),
|
|
1149
|
-
deferredClaimedRequests: [],
|
|
1150
|
-
};
|
|
1151
|
-
}
|
|
1152
|
-
|
|
1153
|
-
const claimedRequests: ClaimedWorkerToolBatchRequest[] = [];
|
|
1154
|
-
const deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[] =
|
|
1155
|
-
[];
|
|
1156
|
-
const receiptGroupPlan = planWorkerToolReceiptGroups(requests, {
|
|
1157
|
-
allowLocalRetryReceipts: this.options.allowLocalRetryReceipts === true,
|
|
1158
|
-
getReceiptInput: (request) => ({
|
|
1159
|
-
durableReceiptKey: request.receiptKey,
|
|
1160
|
-
localRetryCacheKey: request.cacheKey,
|
|
1161
|
-
force: request.force || request.runForceRefresh,
|
|
1162
|
-
forceFailed: request.runForceFailedRefresh,
|
|
1163
|
-
}),
|
|
1164
|
-
});
|
|
1165
|
-
for (const request of receiptGroupPlan.localRequests) {
|
|
1166
|
-
claimedRequests.push({
|
|
1167
|
-
request,
|
|
1168
|
-
receiptKey: null,
|
|
1169
|
-
receiptLeaseId: null,
|
|
1170
|
-
receiptStatus: null,
|
|
1171
|
-
followers: [],
|
|
1172
|
-
});
|
|
1173
|
-
}
|
|
1174
|
-
|
|
1175
|
-
try {
|
|
1176
|
-
const claimEntries = await this.claimDurableToolReceiptGroups(
|
|
1177
|
-
receiptGroupPlan.durableGroups,
|
|
1178
|
-
);
|
|
1179
|
-
for (const { group: groupState, receiptKey, claim } of claimEntries) {
|
|
1180
|
-
const group = groupState.requests;
|
|
1181
|
-
const [first] = group;
|
|
1182
|
-
if (!first) continue;
|
|
1183
|
-
if (
|
|
1184
|
-
claim.disposition === 'failed' &&
|
|
1185
|
-
!groupState.forceDurableRefresh &&
|
|
1186
|
-
group.some((request) => request.runForceRefresh === true)
|
|
1187
|
-
) {
|
|
1188
|
-
deferredClaimedRequests.push(
|
|
1189
|
-
this.claimForcedDurableToolReceiptGroupAfterWait({
|
|
1190
|
-
group,
|
|
1191
|
-
receiptKey,
|
|
1192
|
-
}),
|
|
1193
|
-
);
|
|
1194
|
-
continue;
|
|
1195
|
-
}
|
|
1196
|
-
if (claim.disposition === 'reused') {
|
|
1197
|
-
const result = markWorkerToolReceiptResultCached(
|
|
1198
|
-
this.options.deserializeDurableValue(claim.receipt.output),
|
|
1199
|
-
first.cacheKey,
|
|
1200
|
-
receiptKey,
|
|
1201
|
-
);
|
|
1202
|
-
for (const request of group) {
|
|
1203
|
-
request.resolve(result);
|
|
1204
|
-
}
|
|
1205
|
-
this.options.onRequestsSettled?.(group.length);
|
|
1206
|
-
continue;
|
|
1207
|
-
}
|
|
1208
|
-
if (claim.disposition === 'failed') {
|
|
1209
|
-
if (
|
|
1210
|
-
canReclaimFailedWorkerToolReceipt({
|
|
1211
|
-
error: claim.receipt.error ?? null,
|
|
1212
|
-
})
|
|
1213
|
-
) {
|
|
1214
|
-
deferredClaimedRequests.push(
|
|
1215
|
-
this.claimForcedDurableToolReceiptGroupAfterWait({
|
|
1216
|
-
group,
|
|
1217
|
-
receiptKey,
|
|
1218
|
-
}),
|
|
1219
|
-
);
|
|
1220
|
-
continue;
|
|
1221
|
-
}
|
|
1222
|
-
this.rejectRawRequests(
|
|
1223
|
-
group,
|
|
1224
|
-
new Error(
|
|
1225
|
-
`Durable tool call ${receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
|
|
1226
|
-
),
|
|
1227
|
-
);
|
|
1228
|
-
continue;
|
|
1229
|
-
}
|
|
1230
|
-
if (
|
|
1231
|
-
claim.disposition === 'running' ||
|
|
1232
|
-
claim.disposition === 'blocked_active_lease'
|
|
1233
|
-
) {
|
|
1234
|
-
deferredClaimedRequests.push(
|
|
1235
|
-
(async (): Promise<ClaimedWorkerToolBatchRequest[]> => {
|
|
1236
|
-
let waitError: unknown = new RuntimeReceiptWaitTimeoutError(
|
|
1237
|
-
receiptKey,
|
|
1238
|
-
);
|
|
1239
|
-
try {
|
|
1240
|
-
const result = await this.waitForDurableToolReceipt({
|
|
1241
|
-
receiptKey,
|
|
1242
|
-
maxAttempts: resolveRunningReceiptWaitMaxAttempts({
|
|
1243
|
-
ownerRunId: claim.receipt.runId,
|
|
1244
|
-
currentRunId: this.options.req.runId,
|
|
1245
|
-
updatedAt: claim.receipt.updatedAt,
|
|
1246
|
-
requestedMaxAttempts:
|
|
1247
|
-
resolveWorkerToolReceiptGroupWaitMaxAttempts(
|
|
1248
|
-
group,
|
|
1249
|
-
(request) => request.receiptWaitMaxAttempts,
|
|
1250
|
-
),
|
|
1251
|
-
nowMs: this.options.nowMs(),
|
|
1252
|
-
}),
|
|
1253
|
-
});
|
|
1254
|
-
if (groupState.forceDurableRefresh) {
|
|
1255
|
-
return await this.claimForcedDurableToolReceiptGroupAfterWait(
|
|
1256
|
-
{
|
|
1257
|
-
group,
|
|
1258
|
-
receiptKey,
|
|
1259
|
-
},
|
|
1260
|
-
);
|
|
1261
|
-
}
|
|
1262
|
-
for (const request of group) {
|
|
1263
|
-
request.resolve(
|
|
1264
|
-
markWorkerToolReceiptResultExecution(result, {
|
|
1265
|
-
kind: 'in_flight',
|
|
1266
|
-
receiptKey,
|
|
1267
|
-
attachedToReceiptKey: receiptKey,
|
|
1268
|
-
}),
|
|
1269
|
-
);
|
|
1270
|
-
}
|
|
1271
|
-
this.options.onRequestsSettled?.(group.length);
|
|
1272
|
-
return [];
|
|
1273
|
-
} catch (error) {
|
|
1274
|
-
waitError = error;
|
|
1275
|
-
if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
|
|
1276
|
-
this.rejectRawRequests(group, error);
|
|
1277
|
-
return [];
|
|
1278
|
-
}
|
|
1279
|
-
}
|
|
1280
|
-
return await this.reclaimTimedOutDurableToolReceiptGroup({
|
|
1281
|
-
group,
|
|
1282
|
-
receiptKey,
|
|
1283
|
-
runningReceipt: claim.receipt,
|
|
1284
|
-
waitError,
|
|
1285
|
-
});
|
|
1286
|
-
})(),
|
|
1287
|
-
);
|
|
1288
|
-
continue;
|
|
1289
|
-
}
|
|
1290
|
-
const [request, ...followers] = group;
|
|
1291
|
-
if (!request) continue;
|
|
1292
|
-
claimedRequests.push({
|
|
1293
|
-
request,
|
|
1294
|
-
receiptKey,
|
|
1295
|
-
receiptLeaseId: claim.receipt.leaseId ?? null,
|
|
1296
|
-
receiptStatus: claim.receipt.status,
|
|
1297
|
-
providerForce: groupState.forceDurableRefresh,
|
|
1298
|
-
followers,
|
|
1299
|
-
});
|
|
1300
|
-
}
|
|
1301
|
-
} catch (error) {
|
|
1302
|
-
for (const group of receiptGroupPlan.durableGroups) {
|
|
1303
|
-
this.rejectRawRequests(group.requests, error);
|
|
1304
|
-
}
|
|
1305
|
-
}
|
|
1306
|
-
return { claimedRequests, deferredClaimedRequests };
|
|
1307
|
-
}
|
|
1308
|
-
|
|
1309
|
-
private async completeDurableToolRequest(
|
|
1310
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
1311
|
-
result: unknown,
|
|
1312
|
-
): Promise<unknown> {
|
|
1313
|
-
if (!this.options.receiptStore || !claimed.receiptKey) {
|
|
1314
|
-
return result;
|
|
1315
|
-
}
|
|
1316
|
-
const ownerResult = markWorkerToolReceiptResultExecution(result, {
|
|
1317
|
-
kind: 'live',
|
|
1318
|
-
receiptKey: claimed.receiptKey,
|
|
1319
|
-
});
|
|
1320
|
-
this.options.budgetMeter?.count('receipt');
|
|
1321
|
-
let completed: WorkerRuntimeReceipt | null;
|
|
1322
|
-
try {
|
|
1323
|
-
let completeAttempts = 0;
|
|
1324
|
-
const output = this.options.serializeDurableValue(ownerResult);
|
|
1325
|
-
completed = await this.completeReceiptWithRetryLadder(
|
|
1326
|
-
async () => {
|
|
1327
|
-
completeAttempts += 1;
|
|
1328
|
-
return await this.options.receiptStore!.completeReceipt({
|
|
1329
|
-
playName: this.options.req.playName,
|
|
1330
|
-
runId: this.options.req.runId,
|
|
1331
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
1332
|
-
key: claimed.receiptKey!,
|
|
1333
|
-
leaseId: claimed.receiptLeaseId,
|
|
1334
|
-
output,
|
|
1335
|
-
});
|
|
1336
|
-
},
|
|
1337
|
-
(attempts) =>
|
|
1338
|
-
this.recordReceiptSalvage({
|
|
1339
|
-
claimed,
|
|
1340
|
-
output,
|
|
1341
|
-
attempts: Math.max(attempts, completeAttempts),
|
|
1342
|
-
}),
|
|
1343
|
-
);
|
|
1344
|
-
} catch (error) {
|
|
1345
|
-
throw receiptPersistenceError({
|
|
1346
|
-
operation: 'complete',
|
|
1347
|
-
receiptKeys: [claimed.receiptKey],
|
|
1348
|
-
cause: error,
|
|
1349
|
-
});
|
|
1350
|
-
}
|
|
1351
|
-
const completedResult = this.completedDurableToolRequestResult({
|
|
1352
|
-
claimed,
|
|
1353
|
-
ownerResult,
|
|
1354
|
-
completed,
|
|
1355
|
-
});
|
|
1356
|
-
if (completedResult.ok) return completedResult.value;
|
|
1357
|
-
throw completedResult.error;
|
|
1358
|
-
}
|
|
1359
|
-
|
|
1360
|
-
private async markAdmittedToolRequestsRunning(
|
|
1361
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
1362
|
-
): Promise<void> {
|
|
1363
|
-
const queued = claimedRequests.filter(
|
|
1364
|
-
(claimed) =>
|
|
1365
|
-
claimed.receiptKey !== null && claimed.receiptStatus !== 'running',
|
|
1366
|
-
);
|
|
1367
|
-
if (queued.length === 0) return;
|
|
1368
|
-
|
|
1369
|
-
const markReceiptsRunning = this.options.receiptStore?.markReceiptsRunning;
|
|
1370
|
-
const receiptKeys = queued.map((claimed) => claimed.receiptKey!);
|
|
1371
|
-
if (!markReceiptsRunning) {
|
|
1372
|
-
throw receiptPersistenceError({
|
|
1373
|
-
operation: 'mark-running',
|
|
1374
|
-
receiptKeys,
|
|
1375
|
-
cause: new Error(
|
|
1376
|
-
'Queued runtime receipts require a fenced mark-running store operation.',
|
|
1377
|
-
),
|
|
1378
|
-
});
|
|
1379
|
-
}
|
|
1380
|
-
|
|
1381
|
-
const runId = this.options.req.runId;
|
|
1382
|
-
const runAttempt = this.options.req.runAttempt ?? 0;
|
|
1383
|
-
this.options.budgetMeter?.count('receipt');
|
|
1384
|
-
let marked: Array<WorkerRuntimeReceipt | null>;
|
|
1385
|
-
try {
|
|
1386
|
-
marked = await markReceiptsRunning({
|
|
1387
|
-
playName: this.options.req.playName,
|
|
1388
|
-
leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
|
|
1389
|
-
receipts: queued.map((claimed) => ({
|
|
1390
|
-
runId,
|
|
1391
|
-
runAttempt,
|
|
1392
|
-
key: claimed.receiptKey!,
|
|
1393
|
-
leaseId: claimed.receiptLeaseId,
|
|
1394
|
-
})),
|
|
1395
|
-
});
|
|
1396
|
-
} catch (error) {
|
|
1397
|
-
throw receiptPersistenceError({
|
|
1398
|
-
operation: 'mark-running',
|
|
1399
|
-
receiptKeys,
|
|
1400
|
-
cause: error,
|
|
1401
|
-
});
|
|
1402
|
-
}
|
|
1403
|
-
|
|
1404
|
-
for (let index = 0; index < queued.length; index += 1) {
|
|
1405
|
-
const claimed = queued[index]!;
|
|
1406
|
-
const receipt = marked[index];
|
|
1407
|
-
const leaseId = claimed.receiptLeaseId?.trim() || null;
|
|
1408
|
-
const ownerRunId = receipt?.leaseOwnerRunId ?? receipt?.runId ?? null;
|
|
1409
|
-
if (
|
|
1410
|
-
!receipt ||
|
|
1411
|
-
receipt.key !== claimed.receiptKey ||
|
|
1412
|
-
receipt.status !== 'running' ||
|
|
1413
|
-
!leaseId ||
|
|
1414
|
-
receipt.leaseId !== leaseId ||
|
|
1415
|
-
ownerRunId !== runId ||
|
|
1416
|
-
receipt.leaseOwnerAttempt !== runAttempt
|
|
1417
|
-
) {
|
|
1418
|
-
throw new RuntimeLeaseLostError({
|
|
1419
|
-
keys: [claimed.receiptKey!],
|
|
1420
|
-
runId,
|
|
1421
|
-
leaseId: leaseId ?? 'missing',
|
|
1422
|
-
});
|
|
1423
|
-
}
|
|
1424
|
-
}
|
|
1425
|
-
|
|
1426
|
-
for (const claimed of queued) {
|
|
1427
|
-
claimed.receiptStatus = 'running';
|
|
1428
|
-
}
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
|
-
private completedDurableToolRequestResult(input: {
|
|
1432
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
1433
|
-
ownerResult: unknown;
|
|
1434
|
-
completed: WorkerRuntimeReceipt | null | undefined;
|
|
1435
|
-
}): WorkerToolCompletionResult {
|
|
1436
|
-
const { claimed, completed, ownerResult } = input;
|
|
1437
|
-
if (
|
|
1438
|
-
completed &&
|
|
1439
|
-
(completed.status === 'completed' || completed.status === 'skipped') &&
|
|
1440
|
-
completed.output !== undefined
|
|
1441
|
-
) {
|
|
1442
|
-
const recovered = this.options.deserializeDurableValue(completed.output);
|
|
1443
|
-
return {
|
|
1444
|
-
ok: true,
|
|
1445
|
-
value:
|
|
1446
|
-
completed.runId && completed.runId !== this.options.req.runId
|
|
1447
|
-
? markWorkerToolReceiptResultCached(
|
|
1448
|
-
recovered,
|
|
1449
|
-
claimed.request.cacheKey,
|
|
1450
|
-
claimed.receiptKey ?? claimed.request.cacheKey,
|
|
1451
|
-
)
|
|
1452
|
-
: recovered,
|
|
1453
|
-
};
|
|
1454
|
-
}
|
|
1455
|
-
if (completed?.status === 'failed') {
|
|
1456
|
-
const receiptKey = claimed.receiptKey ?? claimed.request.cacheKey;
|
|
1457
|
-
return {
|
|
1458
|
-
ok: false,
|
|
1459
|
-
error: receiptPersistenceError({
|
|
1460
|
-
operation: 'complete',
|
|
1461
|
-
receiptKeys: [receiptKey],
|
|
1462
|
-
cause: completed.error
|
|
1463
|
-
? new Error(completed.error)
|
|
1464
|
-
: new Error(
|
|
1465
|
-
`Durable tool call ${receiptKey} recovered failed receipt completion without an error message.`,
|
|
1466
|
-
),
|
|
1467
|
-
}),
|
|
1468
|
-
};
|
|
1469
|
-
}
|
|
1470
|
-
return {
|
|
1471
|
-
ok: true,
|
|
1472
|
-
value: ownerResult,
|
|
1473
|
-
};
|
|
1474
|
-
}
|
|
1475
|
-
|
|
1476
|
-
private async recoverCompletedDurableToolRequestResult(input: {
|
|
1477
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
1478
|
-
ownerResult: unknown;
|
|
1479
|
-
completed: WorkerRuntimeReceipt | null | undefined;
|
|
1480
|
-
completeAttempts: number;
|
|
1481
|
-
}): Promise<WorkerToolCompletionResult> {
|
|
1482
|
-
const completedResult = this.completedDurableToolRequestResult(input);
|
|
1483
|
-
if (completedResult.ok) {
|
|
1484
|
-
return completedResult;
|
|
1485
|
-
}
|
|
1486
|
-
|
|
1487
|
-
const receiptKey = input.claimed.receiptKey;
|
|
1488
|
-
if (!receiptKey || !this.options.receiptStore?.getReceipt) {
|
|
1489
|
-
return completedResult;
|
|
1490
|
-
}
|
|
1491
|
-
|
|
1492
|
-
let receipt: WorkerRuntimeReceipt | null = null;
|
|
1493
|
-
try {
|
|
1494
|
-
this.options.budgetMeter?.count('receipt');
|
|
1495
|
-
receipt = await this.options.receiptStore.getReceipt({
|
|
1496
|
-
playName: this.options.req.playName,
|
|
1497
|
-
key: receiptKey,
|
|
1498
|
-
});
|
|
1499
|
-
} catch (readError) {
|
|
1500
|
-
this.recordReceiptSalvage({
|
|
1501
|
-
claimed: input.claimed,
|
|
1502
|
-
output: this.options.serializeDurableValue(input.ownerResult),
|
|
1503
|
-
attempts: input.completeAttempts,
|
|
1504
|
-
});
|
|
1505
|
-
return {
|
|
1506
|
-
ok: false,
|
|
1507
|
-
error: receiptPersistenceError({
|
|
1508
|
-
operation: 'complete',
|
|
1509
|
-
receiptKeys: [receiptKey],
|
|
1510
|
-
cause: new AggregateError(
|
|
1511
|
-
[completedResult.error, readError],
|
|
1512
|
-
'Receipt completion returned no terminal receipt and recovery read failed',
|
|
1513
|
-
),
|
|
1514
|
-
}),
|
|
1515
|
-
};
|
|
1516
|
-
}
|
|
1517
|
-
|
|
1518
|
-
if (this.canCompleteRecoveredRunningReceipt(input.claimed, receipt)) {
|
|
1519
|
-
const output = this.options.serializeDurableValue(input.ownerResult);
|
|
1520
|
-
try {
|
|
1521
|
-
this.options.budgetMeter?.count('receipt');
|
|
1522
|
-
const completed = await this.completeReceiptWithRetryLadder(async () =>
|
|
1523
|
-
this.options.receiptStore!.completeReceipt({
|
|
1524
|
-
playName: this.options.req.playName,
|
|
1525
|
-
runId: this.options.req.runId,
|
|
1526
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
1527
|
-
key: receiptKey,
|
|
1528
|
-
leaseId: input.claimed.receiptLeaseId,
|
|
1529
|
-
output,
|
|
1530
|
-
}),
|
|
1531
|
-
);
|
|
1532
|
-
return this.completedDurableToolRequestResult({
|
|
1533
|
-
claimed: input.claimed,
|
|
1534
|
-
ownerResult: input.ownerResult,
|
|
1535
|
-
completed,
|
|
1536
|
-
});
|
|
1537
|
-
} catch (completeError) {
|
|
1538
|
-
this.recordReceiptSalvage({
|
|
1539
|
-
claimed: input.claimed,
|
|
1540
|
-
output,
|
|
1541
|
-
attempts: input.completeAttempts,
|
|
1542
|
-
});
|
|
1543
|
-
return {
|
|
1544
|
-
ok: false,
|
|
1545
|
-
error: receiptPersistenceError({
|
|
1546
|
-
operation: 'complete',
|
|
1547
|
-
receiptKeys: [receiptKey],
|
|
1548
|
-
cause: new AggregateError(
|
|
1549
|
-
[completedResult.error, completeError],
|
|
1550
|
-
'Receipt completion returned no terminal receipt and owned receipt recovery failed',
|
|
1551
|
-
),
|
|
1552
|
-
}),
|
|
1553
|
-
};
|
|
1554
|
-
}
|
|
1555
|
-
}
|
|
1556
|
-
|
|
1557
|
-
// Bulk complete can persist terminal receipts, throw once, then return null
|
|
1558
|
-
// on retry because ownership is already gone. The durable row is the source
|
|
1559
|
-
// of truth for that transition; read it before treating the call as lost.
|
|
1560
|
-
const recoveredResult = this.completedDurableToolRequestResult({
|
|
1561
|
-
claimed: input.claimed,
|
|
1562
|
-
ownerResult: input.ownerResult,
|
|
1563
|
-
completed: receipt,
|
|
1564
|
-
});
|
|
1565
|
-
if (recoveredResult.ok) {
|
|
1566
|
-
return recoveredResult;
|
|
1567
|
-
}
|
|
1568
|
-
|
|
1569
|
-
this.recordReceiptSalvage({
|
|
1570
|
-
claimed: input.claimed,
|
|
1571
|
-
output: this.options.serializeDurableValue(input.ownerResult),
|
|
1572
|
-
attempts: input.completeAttempts,
|
|
1573
|
-
});
|
|
1574
|
-
if (receipt?.status === 'failed') {
|
|
1575
|
-
return {
|
|
1576
|
-
ok: false,
|
|
1577
|
-
error: receiptPersistenceError({
|
|
1578
|
-
operation: 'complete',
|
|
1579
|
-
receiptKeys: [receiptKey],
|
|
1580
|
-
cause: receipt.error
|
|
1581
|
-
? new AggregateError(
|
|
1582
|
-
[completedResult.error, new Error(receipt.error)],
|
|
1583
|
-
'Receipt completion recovered failed terminal receipt',
|
|
1584
|
-
)
|
|
1585
|
-
: completedResult.error,
|
|
1586
|
-
}),
|
|
1587
|
-
};
|
|
1588
|
-
}
|
|
1589
|
-
return completedResult;
|
|
1590
|
-
}
|
|
1591
|
-
|
|
1592
|
-
private canCompleteRecoveredRunningReceipt(
|
|
1593
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
1594
|
-
receipt: WorkerRuntimeReceipt | null,
|
|
1595
|
-
): boolean {
|
|
1596
|
-
if (!claimed.receiptKey || !receipt || receipt.status !== 'running') {
|
|
1597
|
-
return false;
|
|
1598
|
-
}
|
|
1599
|
-
const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
|
|
1600
|
-
if (ownerRunId !== this.options.req.runId) {
|
|
1601
|
-
return false;
|
|
1602
|
-
}
|
|
1603
|
-
const ownerAttempt =
|
|
1604
|
-
typeof receipt.leaseOwnerAttempt === 'number'
|
|
1605
|
-
? receipt.leaseOwnerAttempt
|
|
1606
|
-
: 0;
|
|
1607
|
-
if (ownerAttempt !== (this.options.req.runAttempt ?? 0)) {
|
|
1608
|
-
return false;
|
|
1609
|
-
}
|
|
1610
|
-
if (claimed.receiptLeaseId) {
|
|
1611
|
-
return receipt.leaseId === claimed.receiptLeaseId;
|
|
1612
|
-
}
|
|
1613
|
-
return receipt.leaseId == null;
|
|
1614
|
-
}
|
|
1615
|
-
|
|
1616
|
-
private buildDurableToolCompletionBatches(
|
|
1617
|
-
entries: DurableToolCompletionEntry[],
|
|
1618
|
-
): DurableToolCompletionBatch[] {
|
|
1619
|
-
const envelopeBytes = jsonByteLength({
|
|
1620
|
-
playName: this.options.req.playName,
|
|
1621
|
-
receipts: [],
|
|
1622
|
-
});
|
|
1623
|
-
const batches: DurableToolCompletionBatch[] = [];
|
|
1624
|
-
let current: DurableToolCompletionBatch = {
|
|
1625
|
-
entries: [],
|
|
1626
|
-
bytes: envelopeBytes,
|
|
1627
|
-
};
|
|
1628
|
-
for (const entry of entries) {
|
|
1629
|
-
const entryBytes = entry.serializedBytes;
|
|
1630
|
-
const singleBytes = envelopeBytes + entryBytes;
|
|
1631
|
-
if (singleBytes > WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_BYTES) {
|
|
1632
|
-
throw receiptPersistenceError({
|
|
1633
|
-
operation: 'complete',
|
|
1634
|
-
receiptKeys: [entry.receipt.key],
|
|
1635
|
-
cause: new Error(
|
|
1636
|
-
`Runtime receipt completion payload for ${entry.receipt.key} is ${singleBytes} bytes, exceeding the ${WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_BYTES} byte service-binding cap.`,
|
|
1637
|
-
),
|
|
1638
|
-
});
|
|
1639
|
-
}
|
|
1640
|
-
const commaBytes = current.entries.length > 0 ? 1 : 0;
|
|
1641
|
-
const wouldExceedBytes =
|
|
1642
|
-
current.bytes + commaBytes + entryBytes >
|
|
1643
|
-
WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_BYTES;
|
|
1644
|
-
const wouldExceedMembers =
|
|
1645
|
-
current.entries.length >=
|
|
1646
|
-
WORKER_TOOL_COMPLETE_RECEIPTS_MAX_BATCH_MEMBERS;
|
|
1647
|
-
if (
|
|
1648
|
-
current.entries.length > 0 &&
|
|
1649
|
-
(wouldExceedBytes || wouldExceedMembers)
|
|
1650
|
-
) {
|
|
1651
|
-
batches.push(current);
|
|
1652
|
-
current = { entries: [], bytes: envelopeBytes };
|
|
1653
|
-
}
|
|
1654
|
-
current.bytes += (current.entries.length > 0 ? 1 : 0) + entryBytes;
|
|
1655
|
-
current.entries.push(entry);
|
|
1656
|
-
}
|
|
1657
|
-
if (current.entries.length > 0) {
|
|
1658
|
-
batches.push(current);
|
|
1659
|
-
}
|
|
1660
|
-
return batches;
|
|
1661
|
-
}
|
|
1662
|
-
|
|
1663
|
-
private async completeDurableToolRequests(
|
|
1664
|
-
entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
|
|
1665
|
-
): Promise<WorkerToolCompletionResult[]> {
|
|
1666
|
-
const results: WorkerToolCompletionResult[] = new Array(entries.length);
|
|
1667
|
-
const durableEntries: DurableToolCompletionEntry[] = [];
|
|
1668
|
-
for (let index = 0; index < entries.length; index += 1) {
|
|
1669
|
-
const entry = entries[index]!;
|
|
1670
|
-
if (!this.options.receiptStore || !entry.claimed.receiptKey) {
|
|
1671
|
-
results[index] = { ok: true, value: entry.result };
|
|
1672
|
-
continue;
|
|
1673
|
-
}
|
|
1674
|
-
const ownerResult = markWorkerToolReceiptResultExecution(entry.result, {
|
|
1675
|
-
kind: 'live',
|
|
1676
|
-
receiptKey: entry.claimed.receiptKey,
|
|
1677
|
-
});
|
|
1678
|
-
const receipt = {
|
|
1679
|
-
runId: this.options.req.runId,
|
|
1680
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
1681
|
-
key: entry.claimed.receiptKey!,
|
|
1682
|
-
leaseId: entry.claimed.receiptLeaseId,
|
|
1683
|
-
output: this.options.serializeDurableValue(ownerResult),
|
|
1684
|
-
};
|
|
1685
|
-
durableEntries.push({
|
|
1686
|
-
index,
|
|
1687
|
-
claimed: entry.claimed,
|
|
1688
|
-
ownerResult,
|
|
1689
|
-
receipt,
|
|
1690
|
-
serializedBytes: jsonByteLength(receipt),
|
|
1691
|
-
});
|
|
1692
|
-
}
|
|
1693
|
-
if (durableEntries.length === 0) {
|
|
1694
|
-
return results;
|
|
1695
|
-
}
|
|
1696
|
-
if (!this.options.receiptStore?.completeReceipts) {
|
|
1697
|
-
await Promise.all(
|
|
1698
|
-
durableEntries.map(async (entry) => {
|
|
1699
|
-
try {
|
|
1700
|
-
results[entry.index] = {
|
|
1701
|
-
ok: true,
|
|
1702
|
-
value: await this.completeDurableToolRequest(
|
|
1703
|
-
entry.claimed,
|
|
1704
|
-
entries[entry.index]!.result,
|
|
1705
|
-
),
|
|
1706
|
-
};
|
|
1707
|
-
} catch (error) {
|
|
1708
|
-
results[entry.index] = { ok: false, error };
|
|
1709
|
-
}
|
|
1710
|
-
}),
|
|
1711
|
-
);
|
|
1712
|
-
return results;
|
|
1713
|
-
}
|
|
1714
|
-
const completionBatches =
|
|
1715
|
-
this.buildDurableToolCompletionBatches(durableEntries);
|
|
1716
|
-
let completeAttempts = 0;
|
|
1717
|
-
for (const batch of completionBatches) {
|
|
1718
|
-
this.options.budgetMeter?.count('receipt');
|
|
1719
|
-
let completed: Array<WorkerRuntimeReceipt | null>;
|
|
1720
|
-
try {
|
|
1721
|
-
completeAttempts += 1;
|
|
1722
|
-
completed = await this.options.receiptStore!.completeReceipts!({
|
|
1723
|
-
playName: this.options.req.playName,
|
|
1724
|
-
receipts: batch.entries.map((entry) => entry.receipt),
|
|
1725
|
-
});
|
|
1726
|
-
} catch (error) {
|
|
1727
|
-
const persistenceError = receiptPersistenceError({
|
|
1728
|
-
operation: 'complete',
|
|
1729
|
-
receiptKeys: batch.entries.map((entry) => entry.receipt.key),
|
|
1730
|
-
cause: error,
|
|
1731
|
-
});
|
|
1732
|
-
if (!this.options.receiptStore.getReceipt) {
|
|
1733
|
-
for (const entry of batch.entries) {
|
|
1734
|
-
this.recordReceiptSalvage({
|
|
1735
|
-
claimed: entry.claimed,
|
|
1736
|
-
output: entry.receipt.output,
|
|
1737
|
-
attempts: completeAttempts,
|
|
1738
|
-
});
|
|
1739
|
-
results[entry.index] = { ok: false, error: persistenceError };
|
|
1740
|
-
}
|
|
1741
|
-
continue;
|
|
1742
|
-
}
|
|
1743
|
-
await mapWithConcurrency(
|
|
1744
|
-
batch.entries,
|
|
1745
|
-
WORKER_TOOL_RECEIPT_RECOVERY_MAX_CONCURRENT_READS,
|
|
1746
|
-
async (entry) => {
|
|
1747
|
-
const receiptKey = entry.receipt.key;
|
|
1748
|
-
let receipt: WorkerRuntimeReceipt | null = null;
|
|
1749
|
-
try {
|
|
1750
|
-
this.options.budgetMeter?.count('receipt');
|
|
1751
|
-
receipt = await this.options.receiptStore!.getReceipt!({
|
|
1752
|
-
playName: this.options.req.playName,
|
|
1753
|
-
key: receiptKey,
|
|
1754
|
-
});
|
|
1755
|
-
} catch (readError) {
|
|
1756
|
-
this.recordReceiptSalvage({
|
|
1757
|
-
claimed: entry.claimed,
|
|
1758
|
-
output: entry.receipt.output,
|
|
1759
|
-
attempts: completeAttempts,
|
|
1760
|
-
});
|
|
1761
|
-
results[entry.index] = {
|
|
1762
|
-
ok: false,
|
|
1763
|
-
error: receiptPersistenceError({
|
|
1764
|
-
operation: 'complete',
|
|
1765
|
-
receiptKeys: [receiptKey],
|
|
1766
|
-
cause: new AggregateError(
|
|
1767
|
-
[error, readError],
|
|
1768
|
-
'Receipt completion failed and receipt recovery read failed',
|
|
1769
|
-
),
|
|
1770
|
-
}),
|
|
1771
|
-
};
|
|
1772
|
-
return;
|
|
1773
|
-
}
|
|
1774
|
-
if (
|
|
1775
|
-
receipt &&
|
|
1776
|
-
(receipt.status === 'completed' ||
|
|
1777
|
-
receipt.status === 'skipped') &&
|
|
1778
|
-
receipt.output !== undefined
|
|
1779
|
-
) {
|
|
1780
|
-
results[entry.index] = this.completedDurableToolRequestResult({
|
|
1781
|
-
claimed: entry.claimed,
|
|
1782
|
-
ownerResult: entry.ownerResult,
|
|
1783
|
-
completed: receipt,
|
|
1784
|
-
});
|
|
1785
|
-
return;
|
|
1786
|
-
}
|
|
1787
|
-
if (
|
|
1788
|
-
this.canCompleteRecoveredRunningReceipt(entry.claimed, receipt)
|
|
1789
|
-
) {
|
|
1790
|
-
try {
|
|
1791
|
-
results[entry.index] = {
|
|
1792
|
-
ok: true,
|
|
1793
|
-
value: await this.completeDurableToolRequest(
|
|
1794
|
-
entry.claimed,
|
|
1795
|
-
entries[entry.index]!.result,
|
|
1796
|
-
),
|
|
1797
|
-
};
|
|
1798
|
-
return;
|
|
1799
|
-
} catch (singleCompleteError) {
|
|
1800
|
-
results[entry.index] = {
|
|
1801
|
-
ok: false,
|
|
1802
|
-
error: receiptPersistenceError({
|
|
1803
|
-
operation: 'complete',
|
|
1804
|
-
receiptKeys: [receiptKey],
|
|
1805
|
-
cause: new AggregateError(
|
|
1806
|
-
[error, singleCompleteError],
|
|
1807
|
-
'Receipt completion failed and owned receipt recovery failed',
|
|
1808
|
-
),
|
|
1809
|
-
}),
|
|
1810
|
-
};
|
|
1811
|
-
return;
|
|
1812
|
-
}
|
|
1813
|
-
}
|
|
1814
|
-
this.recordReceiptSalvage({
|
|
1815
|
-
claimed: entry.claimed,
|
|
1816
|
-
output: entry.receipt.output,
|
|
1817
|
-
attempts: completeAttempts,
|
|
1818
|
-
});
|
|
1819
|
-
results[entry.index] = {
|
|
1820
|
-
ok: false,
|
|
1821
|
-
error: receiptPersistenceError({
|
|
1822
|
-
operation: 'complete',
|
|
1823
|
-
receiptKeys: [receiptKey],
|
|
1824
|
-
cause: receipt?.error
|
|
1825
|
-
? new AggregateError(
|
|
1826
|
-
[error, new Error(receipt.error)],
|
|
1827
|
-
'Receipt completion failed and receipt recovered failed',
|
|
1828
|
-
)
|
|
1829
|
-
: error,
|
|
1830
|
-
}),
|
|
1831
|
-
};
|
|
1832
|
-
},
|
|
1833
|
-
);
|
|
1834
|
-
this.options.recordPerfTrace?.({
|
|
1835
|
-
phase: 'worker.tool_receipts.partial_complete_recovery',
|
|
1836
|
-
extra: {
|
|
1837
|
-
recovered: batch.entries.filter((entry) => results[entry.index]?.ok)
|
|
1838
|
-
.length,
|
|
1839
|
-
failed: batch.entries.filter((entry) => {
|
|
1840
|
-
const result = results[entry.index];
|
|
1841
|
-
return result && !result.ok;
|
|
1842
|
-
}).length,
|
|
1843
|
-
total: batch.entries.length,
|
|
1844
|
-
failedReceiptKeys: batch.entries
|
|
1845
|
-
.filter((entry) => {
|
|
1846
|
-
const result = results[entry.index];
|
|
1847
|
-
return result && !result.ok;
|
|
1848
|
-
})
|
|
1849
|
-
.slice(0, 5)
|
|
1850
|
-
.map((entry) => entry.receipt.key),
|
|
1851
|
-
},
|
|
1852
|
-
});
|
|
1853
|
-
continue;
|
|
1854
|
-
}
|
|
1855
|
-
await mapWithConcurrency(
|
|
1856
|
-
batch.entries,
|
|
1857
|
-
WORKER_TOOL_RECEIPT_RECOVERY_MAX_CONCURRENT_READS,
|
|
1858
|
-
async (entry, resultIndex) => {
|
|
1859
|
-
results[entry.index] =
|
|
1860
|
-
await this.recoverCompletedDurableToolRequestResult({
|
|
1861
|
-
claimed: entry.claimed,
|
|
1862
|
-
ownerResult: entry.ownerResult,
|
|
1863
|
-
completed: completed[resultIndex],
|
|
1864
|
-
completeAttempts,
|
|
1865
|
-
});
|
|
1866
|
-
},
|
|
1867
|
-
);
|
|
1868
|
-
}
|
|
1869
|
-
return results;
|
|
1870
|
-
}
|
|
1871
|
-
|
|
1872
|
-
private async failDurableToolRequest(
|
|
1873
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
1874
|
-
error: unknown,
|
|
1875
|
-
): Promise<void> {
|
|
1876
|
-
if (!this.options.receiptStore || !claimed.receiptKey) {
|
|
1877
|
-
return;
|
|
1878
|
-
}
|
|
1879
|
-
this.options.budgetMeter?.count('receipt');
|
|
1880
|
-
try {
|
|
1881
|
-
await this.options.receiptStore.failReceipt({
|
|
1882
|
-
playName: this.options.req.playName,
|
|
1883
|
-
runId: this.options.req.runId,
|
|
1884
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
1885
|
-
key: claimed.receiptKey,
|
|
1886
|
-
leaseId: claimed.receiptLeaseId,
|
|
1887
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1888
|
-
failureKind: runtimeReceiptFailureKindForError(error),
|
|
1889
|
-
});
|
|
1890
|
-
} catch (receiptError) {
|
|
1891
|
-
if (this.options.persistenceLatch) {
|
|
1892
|
-
tripRuntimePersistenceLatch(
|
|
1893
|
-
this.options.persistenceLatch,
|
|
1894
|
-
receiptError,
|
|
1895
|
-
);
|
|
1896
|
-
}
|
|
1897
|
-
throw receiptPersistenceError({
|
|
1898
|
-
operation: 'fail',
|
|
1899
|
-
receiptKeys: [claimed.receiptKey],
|
|
1900
|
-
cause: receiptError,
|
|
1901
|
-
});
|
|
1902
|
-
}
|
|
1903
|
-
}
|
|
1904
|
-
|
|
1905
|
-
private async releaseDurableToolRequest(
|
|
1906
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
1907
|
-
error: unknown,
|
|
1908
|
-
): Promise<void> {
|
|
1909
|
-
if (!this.options.receiptStore || !claimed.receiptKey) {
|
|
1910
|
-
return;
|
|
1911
|
-
}
|
|
1912
|
-
if (!this.options.receiptStore.releaseReceipt) {
|
|
1913
|
-
throw new Error(
|
|
1914
|
-
`Durable tool call ${claimed.receiptKey} suspended but the receipt store cannot release ownership.`,
|
|
1915
|
-
);
|
|
1916
|
-
}
|
|
1917
|
-
this.options.budgetMeter?.count('receipt');
|
|
1918
|
-
try {
|
|
1919
|
-
const released = await this.options.receiptStore.releaseReceipt({
|
|
1920
|
-
playName: this.options.req.playName,
|
|
1921
|
-
runId: this.options.req.runId,
|
|
1922
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
1923
|
-
key: claimed.receiptKey,
|
|
1924
|
-
leaseId: claimed.receiptLeaseId,
|
|
1925
|
-
});
|
|
1926
|
-
if (!released || released.status !== 'pending') {
|
|
1927
|
-
throw new Error(
|
|
1928
|
-
`Durable tool call ${claimed.receiptKey} suspended but receipt ownership could not be released.`,
|
|
1929
|
-
);
|
|
1930
|
-
}
|
|
1931
|
-
} catch (receiptError) {
|
|
1932
|
-
if (this.options.persistenceLatch) {
|
|
1933
|
-
tripRuntimePersistenceLatch(
|
|
1934
|
-
this.options.persistenceLatch,
|
|
1935
|
-
receiptError,
|
|
1936
|
-
);
|
|
1937
|
-
}
|
|
1938
|
-
throw receiptPersistenceError({
|
|
1939
|
-
operation: 'fail',
|
|
1940
|
-
receiptKeys: [claimed.receiptKey],
|
|
1941
|
-
cause: new AggregateError(
|
|
1942
|
-
[error, receiptError],
|
|
1943
|
-
'Tool receipt release failed after durable suspension',
|
|
1944
|
-
),
|
|
1945
|
-
});
|
|
1946
|
-
}
|
|
1947
|
-
}
|
|
1948
|
-
|
|
1949
|
-
private async failDurableToolRequests(
|
|
1950
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
1951
|
-
error: unknown,
|
|
1952
|
-
): Promise<void> {
|
|
1953
|
-
const durable = claimedRequests.filter(
|
|
1954
|
-
(claimed) => this.options.receiptStore && claimed.receiptKey,
|
|
1955
|
-
);
|
|
1956
|
-
if (durable.length === 0) return;
|
|
1957
|
-
if (
|
|
1958
|
-
isPlayExecutionSuspendedError(error) ||
|
|
1959
|
-
isPlayRowExecutionSuspendedError(error)
|
|
1960
|
-
) {
|
|
1961
|
-
await Promise.all(
|
|
1962
|
-
durable.map((claimed) =>
|
|
1963
|
-
this.releaseDurableToolRequest(claimed, error),
|
|
1964
|
-
),
|
|
1965
|
-
);
|
|
1966
|
-
return;
|
|
1967
|
-
}
|
|
1968
|
-
if (!this.options.receiptStore?.failReceipts) {
|
|
1969
|
-
await Promise.all(
|
|
1970
|
-
durable.map((claimed) => this.failDurableToolRequest(claimed, error)),
|
|
1971
|
-
);
|
|
1972
|
-
return;
|
|
1973
|
-
}
|
|
1974
|
-
// Bulk fail is a single service-binding RPC regardless of key count.
|
|
1975
|
-
this.options.budgetMeter?.count('receipt');
|
|
1976
|
-
try {
|
|
1977
|
-
await this.options.receiptStore.failReceipts({
|
|
1978
|
-
playName: this.options.req.playName,
|
|
1979
|
-
receipts: durable.map((claimed) => ({
|
|
1980
|
-
runId: this.options.req.runId,
|
|
1981
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
1982
|
-
key: claimed.receiptKey!,
|
|
1983
|
-
leaseId: claimed.receiptLeaseId,
|
|
1984
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1985
|
-
failureKind: runtimeReceiptFailureKindForError(error),
|
|
1986
|
-
})),
|
|
1987
|
-
});
|
|
1988
|
-
} catch (receiptError) {
|
|
1989
|
-
if (this.options.persistenceLatch) {
|
|
1990
|
-
tripRuntimePersistenceLatch(
|
|
1991
|
-
this.options.persistenceLatch,
|
|
1992
|
-
receiptError,
|
|
1993
|
-
);
|
|
1994
|
-
}
|
|
1995
|
-
throw receiptPersistenceError({
|
|
1996
|
-
operation: 'fail',
|
|
1997
|
-
receiptKeys: durable
|
|
1998
|
-
.map((claimed) => claimed.receiptKey)
|
|
1999
|
-
.filter((key): key is string => typeof key === 'string'),
|
|
2000
|
-
cause: receiptError,
|
|
2001
|
-
});
|
|
2002
|
-
}
|
|
2003
|
-
}
|
|
2004
|
-
|
|
2005
|
-
private async executeToolGroup(
|
|
2006
|
-
toolId: string,
|
|
2007
|
-
requests: WorkerToolBatchRequest[],
|
|
2008
|
-
): Promise<void> {
|
|
2009
|
-
const receiptPrepareStartedAt = this.options.nowMs();
|
|
2010
|
-
const { claimedRequests, deferredClaimedRequests } =
|
|
2011
|
-
await this.prepareDurableToolRequests(requests);
|
|
2012
|
-
this.options.recordPerfTrace?.({
|
|
2013
|
-
phase: 'runner.tool.receipt_prepare',
|
|
2014
|
-
ms: this.options.nowMs() - receiptPrepareStartedAt,
|
|
2015
|
-
extra: { toolId, requests: requests.length },
|
|
2016
|
-
});
|
|
2017
|
-
await this.executeClaimedToolRequests(
|
|
2018
|
-
toolId,
|
|
2019
|
-
requests.length,
|
|
2020
|
-
claimedRequests,
|
|
2021
|
-
);
|
|
2022
|
-
if (deferredClaimedRequests.length === 0) {
|
|
2023
|
-
return;
|
|
2024
|
-
}
|
|
2025
|
-
const reclaimedRequests = (
|
|
2026
|
-
await Promise.all(deferredClaimedRequests)
|
|
2027
|
-
).flat();
|
|
2028
|
-
await this.executeClaimedToolRequests(
|
|
2029
|
-
toolId,
|
|
2030
|
-
requests.length,
|
|
2031
|
-
reclaimedRequests,
|
|
2032
|
-
);
|
|
2033
|
-
}
|
|
2034
|
-
|
|
2035
|
-
private async executeClaimedToolRequests(
|
|
2036
|
-
toolId: string,
|
|
2037
|
-
requestCount: number,
|
|
2038
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
2039
|
-
): Promise<void> {
|
|
2040
|
-
if (claimedRequests.length === 0) {
|
|
2041
|
-
return;
|
|
2042
|
-
}
|
|
2043
|
-
const strategy = getPlayRuntimeBatchStrategy(toolId);
|
|
2044
|
-
if (
|
|
2045
|
-
!strategy ||
|
|
2046
|
-
toolId === 'test_wait_for_event' ||
|
|
2047
|
-
claimedRequests.length < 2
|
|
2048
|
-
) {
|
|
2049
|
-
const groupStartedAt = this.options.nowMs();
|
|
2050
|
-
// Seed the dispatch width from the governor's provider-shaped parallelism
|
|
2051
|
-
// instead of the flat policy.concurrency.toolCalls ceiling. A flat width
|
|
2052
|
-
// launched every claimed row at once into the pacer, so unhinted providers
|
|
2053
|
-
// 429'd on the first burst before AIMD could halve the rate. The shaped
|
|
2054
|
-
// estimate is derived from the provider's RPS/maxConcurrency pacing rules
|
|
2055
|
-
// (see suggestedParallelism in governor.ts), floored at 1 and capped by the
|
|
2056
|
-
// global tool-call concurrency ceiling. Per-call pacing still gates each
|
|
2057
|
-
// in-flight request, so this only trims the launch burst.
|
|
2058
|
-
const toolCallConcurrencyCeiling =
|
|
2059
|
-
this.options.governor.policy.concurrency.toolCalls;
|
|
2060
|
-
const shapedToolParallelism =
|
|
2061
|
-
await this.options.governor.suggestedParallelism(
|
|
2062
|
-
toolId,
|
|
2063
|
-
toolCallConcurrencyCeiling,
|
|
2064
|
-
);
|
|
2065
|
-
const dispatchWidth = Math.min(
|
|
2066
|
-
toolCallConcurrencyCeiling,
|
|
2067
|
-
Math.max(1, shapedToolParallelism),
|
|
2068
|
-
);
|
|
2069
|
-
await dispatchBounded(claimedRequests, dispatchWidth, async (claimed) => {
|
|
2070
|
-
const { request } = claimed;
|
|
2071
|
-
const toolContract = await this.options
|
|
2072
|
-
.resolvePacing(toolId)
|
|
2073
|
-
.catch(() => null);
|
|
2074
|
-
const circuitOpenError = this.runtimePersistenceCircuitOpenError(1);
|
|
2075
|
-
if (circuitOpenError) {
|
|
2076
|
-
this.rejectRequests(
|
|
2077
|
-
claimed,
|
|
2078
|
-
await this.failureForRejectedToolRequest(claimed, circuitOpenError),
|
|
2079
|
-
);
|
|
2080
|
-
return;
|
|
2081
|
-
}
|
|
2082
|
-
// Each unbatched provider call takes its own tool slot: the Governor
|
|
2083
|
-
// charges tool budget, holds a global tool-concurrency slot, and
|
|
2084
|
-
// applies per-(org,provider) pacing before the call runs.
|
|
2085
|
-
let slot: { release: () => void };
|
|
2086
|
-
const admissionStartedAt = this.options.nowMs();
|
|
2087
|
-
try {
|
|
2088
|
-
slot = await this.options.governor.acquireToolSlot(toolId, {
|
|
2089
|
-
signal: this.options.abortSignal,
|
|
2090
|
-
});
|
|
2091
|
-
} catch (error) {
|
|
2092
|
-
this.rejectRequests(
|
|
2093
|
-
claimed,
|
|
2094
|
-
await this.failureForRejectedToolRequest(claimed, error),
|
|
2095
|
-
);
|
|
2096
|
-
return;
|
|
2097
|
-
}
|
|
2098
|
-
this.options.recordPerfTrace?.({
|
|
2099
|
-
phase: 'runner.tool.admission',
|
|
2100
|
-
ms: this.options.nowMs() - admissionStartedAt,
|
|
2101
|
-
extra: { toolId, requests: 1 },
|
|
2102
|
-
});
|
|
2103
|
-
try {
|
|
2104
|
-
let result: unknown;
|
|
2105
|
-
try {
|
|
2106
|
-
await this.markAdmittedToolRequestsRunning([claimed]);
|
|
2107
|
-
this.options.budgetMeter?.count('provider');
|
|
2108
|
-
this.options.budgetMeter?.count(
|
|
2109
|
-
'egress',
|
|
2110
|
-
Math.max(
|
|
2111
|
-
0,
|
|
2112
|
-
WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL - 1,
|
|
2113
|
-
),
|
|
2114
|
-
);
|
|
2115
|
-
const durableReceiptKey = claimed.receiptKey
|
|
2116
|
-
? durableCallReceiptKeyForClaimed({
|
|
2117
|
-
req: this.options.req,
|
|
2118
|
-
claimed,
|
|
2119
|
-
})
|
|
2120
|
-
: null;
|
|
2121
|
-
const providerStartedAt = this.options.nowMs();
|
|
2122
|
-
try {
|
|
2123
|
-
result = await executeWithWorkerReceiptHeartbeats({
|
|
2124
|
-
playName: this.options.req.playName,
|
|
2125
|
-
runId: this.options.req.runId,
|
|
2126
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
2127
|
-
...(this.options.receiptStore
|
|
2128
|
-
? { receiptStore: this.options.receiptStore }
|
|
2129
|
-
: {}),
|
|
2130
|
-
budgetMeter: this.options.budgetMeter,
|
|
2131
|
-
heartbeatIntervalMs:
|
|
2132
|
-
this.options.runtimeReceiptHeartbeatIntervalMs,
|
|
2133
|
-
targets: claimed.receiptKey
|
|
2134
|
-
? [
|
|
2135
|
-
{
|
|
2136
|
-
key: claimed.receiptKey,
|
|
2137
|
-
leaseId: claimed.receiptLeaseId,
|
|
2138
|
-
},
|
|
2139
|
-
]
|
|
2140
|
-
: [],
|
|
2141
|
-
execute: () =>
|
|
2142
|
-
this.options.executeTool({
|
|
2143
|
-
id: request.id,
|
|
2144
|
-
toolId,
|
|
2145
|
-
input: request.input,
|
|
2146
|
-
workflowStep: request.workflowStep,
|
|
2147
|
-
callbacks: this.options.callbacks,
|
|
2148
|
-
onProviderBackpressure: (retryAfterMs) =>
|
|
2149
|
-
this.reportBackpressure(toolId, retryAfterMs),
|
|
2150
|
-
onRetryAttempt: () =>
|
|
2151
|
-
this.options.governor.chargeBudget('retry'),
|
|
2152
|
-
transientHttpRetrySafe:
|
|
2153
|
-
toolContract?.retrySafeTransientHttp === true,
|
|
2154
|
-
...(durableReceiptKey
|
|
2155
|
-
? {
|
|
2156
|
-
durableCallReceiptKey: durableReceiptKey,
|
|
2157
|
-
executionAuthScopeDigest:
|
|
2158
|
-
request.executionAuthScopeDigest,
|
|
2159
|
-
providerIdempotencyKey:
|
|
2160
|
-
providerIdempotencyKeyForClaimed({
|
|
2161
|
-
req: this.options.req,
|
|
2162
|
-
claimed,
|
|
2163
|
-
}),
|
|
2164
|
-
}
|
|
2165
|
-
: {}),
|
|
2166
|
-
runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
|
|
2167
|
-
[claimed],
|
|
2168
|
-
{
|
|
2169
|
-
resolveOwnerTimeoutMs: (request) =>
|
|
2170
|
-
request.runtimeTimeoutMs,
|
|
2171
|
-
},
|
|
2172
|
-
),
|
|
2173
|
-
directOptions: request.directOptions,
|
|
2174
|
-
}),
|
|
2175
|
-
});
|
|
2176
|
-
} finally {
|
|
2177
|
-
this.options.recordPerfTrace?.({
|
|
2178
|
-
phase: 'runner.tool.provider',
|
|
2179
|
-
ms: this.options.nowMs() - providerStartedAt,
|
|
2180
|
-
extra: { toolId, requests: 1 },
|
|
2181
|
-
});
|
|
2182
|
-
}
|
|
2183
|
-
} catch (error) {
|
|
2184
|
-
if (error instanceof RuntimeLeaseLostError) {
|
|
2185
|
-
this.rejectRequests(claimed, error);
|
|
2186
|
-
return;
|
|
2187
|
-
}
|
|
2188
|
-
if (
|
|
2189
|
-
await this.requeueAfterAuthScopeChanged(
|
|
2190
|
-
[request, ...claimed.followers],
|
|
2191
|
-
error,
|
|
2192
|
-
)
|
|
2193
|
-
) {
|
|
2194
|
-
return;
|
|
2195
|
-
}
|
|
2196
|
-
this.rejectRequests(
|
|
2197
|
-
claimed,
|
|
2198
|
-
await this.failureForRejectedToolRequest(claimed, error),
|
|
2199
|
-
);
|
|
2200
|
-
return;
|
|
2201
|
-
}
|
|
2202
|
-
try {
|
|
2203
|
-
const receiptCompleteStartedAt = this.options.nowMs();
|
|
2204
|
-
this.settleRequests(
|
|
2205
|
-
claimed,
|
|
2206
|
-
await this.completeDurableToolRequest(claimed, result),
|
|
2207
|
-
);
|
|
2208
|
-
this.options.recordPerfTrace?.({
|
|
2209
|
-
phase: 'runner.tool.receipt_complete',
|
|
2210
|
-
ms: this.options.nowMs() - receiptCompleteStartedAt,
|
|
2211
|
-
extra: { toolId, requests: 1 },
|
|
2212
|
-
});
|
|
2213
|
-
} catch (receiptError) {
|
|
2214
|
-
this.rejectRequests(claimed, receiptError);
|
|
2215
|
-
}
|
|
2216
|
-
} finally {
|
|
2217
|
-
slot.release();
|
|
2218
|
-
}
|
|
2219
|
-
});
|
|
2220
|
-
this.options.recordPerfTrace?.({
|
|
2221
|
-
phase: 'runner.tool.group',
|
|
2222
|
-
ms: this.options.nowMs() - groupStartedAt,
|
|
2223
|
-
extra: {
|
|
2224
|
-
toolId,
|
|
2225
|
-
requests: requestCount,
|
|
2226
|
-
executed: claimedRequests.length,
|
|
2227
|
-
batched: false,
|
|
2228
|
-
},
|
|
2229
|
-
});
|
|
2230
|
-
return;
|
|
2231
|
-
}
|
|
2232
|
-
|
|
2233
|
-
const batchStartedAt = this.options.nowMs();
|
|
2234
|
-
await this.executeBatchedWorkerToolGroup({
|
|
2235
|
-
sourceToolId: toolId,
|
|
2236
|
-
requests: claimedRequests,
|
|
2237
|
-
strategy,
|
|
2238
|
-
suggestedParallelism: await this.options.governor.suggestedParallelism(
|
|
2239
|
-
toolId,
|
|
2240
|
-
this.options.governor.policy.pacing.workerToolBatchDefaultParallelism,
|
|
2241
|
-
),
|
|
2242
|
-
});
|
|
2243
|
-
this.options.recordPerfTrace?.({
|
|
2244
|
-
phase: 'runner.tool.group',
|
|
2245
|
-
ms: this.options.nowMs() - batchStartedAt,
|
|
2246
|
-
extra: {
|
|
2247
|
-
toolId,
|
|
2248
|
-
requests: requestCount,
|
|
2249
|
-
executed: claimedRequests.length,
|
|
2250
|
-
batched: true,
|
|
2251
|
-
},
|
|
2252
|
-
});
|
|
2253
|
-
}
|
|
2254
|
-
|
|
2255
|
-
private async executeBatchedWorkerToolGroup(input: {
|
|
2256
|
-
sourceToolId: string;
|
|
2257
|
-
requests: ClaimedWorkerToolBatchRequest[];
|
|
2258
|
-
strategy: AnyBatchOperationStrategy;
|
|
2259
|
-
suggestedParallelism: number;
|
|
2260
|
-
}): Promise<void> {
|
|
2261
|
-
const compiledBatches = compileRequestsWithStrategy({
|
|
2262
|
-
requests: input.requests,
|
|
2263
|
-
strategy: input.strategy,
|
|
2264
|
-
getPayload: (request) => request.request.input,
|
|
2265
|
-
});
|
|
2266
|
-
this.options.recordPerfTrace?.({
|
|
2267
|
-
phase: 'runner.tool.batch.compile',
|
|
2268
|
-
ms: 0,
|
|
2269
|
-
extra: {
|
|
2270
|
-
sourceOperation: input.strategy.sourceOperation,
|
|
2271
|
-
batchOperation: input.strategy.batchOperation,
|
|
2272
|
-
requests: input.requests.length,
|
|
2273
|
-
batches: compiledBatches.length,
|
|
2274
|
-
batchSizes: compiledBatches.map((batch) => batch.memberRequests.length),
|
|
2275
|
-
},
|
|
2276
|
-
});
|
|
2277
|
-
await executeChunkedRequests({
|
|
2278
|
-
requests: compiledBatches,
|
|
2279
|
-
// Chunk parallelism is the Governor's per-tool suggestion (provider rate
|
|
2280
|
-
// hints tightened to the policy ceiling), bounded by the batch count.
|
|
2281
|
-
batchSize: Math.max(
|
|
2282
|
-
1,
|
|
2283
|
-
Math.min(input.suggestedParallelism, compiledBatches.length || 1),
|
|
2284
|
-
),
|
|
2285
|
-
execute: async (batch) => {
|
|
2286
|
-
const toolContract = await this.options
|
|
2287
|
-
.resolvePacing(batch.batchOperation)
|
|
2288
|
-
.catch(() => null);
|
|
2289
|
-
const circuitOpenError = this.runtimePersistenceCircuitOpenError(
|
|
2290
|
-
batch.memberRequests.length,
|
|
2291
|
-
);
|
|
2292
|
-
if (circuitOpenError) {
|
|
2293
|
-
throw circuitOpenError;
|
|
2294
|
-
}
|
|
2295
|
-
// One provider call per batch -> one tool slot (budget + global
|
|
2296
|
-
// concurrency + per-(org,provider) pacing) around the whole batch.
|
|
2297
|
-
const admissionStartedAt = this.options.nowMs();
|
|
2298
|
-
const slot = await this.options.governor.acquireToolSlot(
|
|
2299
|
-
batch.batchOperation,
|
|
2300
|
-
{
|
|
2301
|
-
signal: this.options.abortSignal,
|
|
2302
|
-
},
|
|
2303
|
-
);
|
|
2304
|
-
this.options.recordPerfTrace?.({
|
|
2305
|
-
phase: 'runner.tool.admission',
|
|
2306
|
-
ms: this.options.nowMs() - admissionStartedAt,
|
|
2307
|
-
extra: {
|
|
2308
|
-
toolId: batch.batchOperation,
|
|
2309
|
-
requests: batch.memberRequests.length,
|
|
2310
|
-
batched: true,
|
|
2311
|
-
},
|
|
2312
|
-
});
|
|
2313
|
-
try {
|
|
2314
|
-
await this.markAdmittedToolRequestsRunning(batch.memberRequests);
|
|
2315
|
-
this.options.budgetMeter?.count('provider');
|
|
2316
|
-
const receiptKeys = batch.memberRequests.map((member) =>
|
|
2317
|
-
durableCallReceiptKeyForClaimed({
|
|
2318
|
-
req: this.options.req,
|
|
2319
|
-
claimed: member,
|
|
2320
|
-
}),
|
|
2321
|
-
);
|
|
2322
|
-
const aggregateReceiptKey = buildBatchDurableCallReceiptKey({
|
|
2323
|
-
req: this.options.req,
|
|
2324
|
-
batchOperation: batch.batchOperation,
|
|
2325
|
-
receiptKeys,
|
|
2326
|
-
});
|
|
2327
|
-
const providerStartedAt = this.options.nowMs();
|
|
2328
|
-
try {
|
|
2329
|
-
return await executeWithWorkerReceiptHeartbeats({
|
|
2330
|
-
playName: this.options.req.playName,
|
|
2331
|
-
runId: this.options.req.runId,
|
|
2332
|
-
runAttempt: this.options.req.runAttempt ?? 0,
|
|
2333
|
-
...(this.options.receiptStore
|
|
2334
|
-
? { receiptStore: this.options.receiptStore }
|
|
2335
|
-
: {}),
|
|
2336
|
-
budgetMeter: this.options.budgetMeter,
|
|
2337
|
-
heartbeatIntervalMs:
|
|
2338
|
-
this.options.runtimeReceiptHeartbeatIntervalMs,
|
|
2339
|
-
targets: batch.memberRequests.map((member) => ({
|
|
2340
|
-
key: member.receiptKey,
|
|
2341
|
-
leaseId: member.receiptLeaseId,
|
|
2342
|
-
})),
|
|
2343
|
-
execute: () =>
|
|
2344
|
-
this.options.executeTool({
|
|
2345
|
-
id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
|
|
2346
|
-
toolId: batch.batchOperation,
|
|
2347
|
-
input: batch.batchPayload,
|
|
2348
|
-
callbacks: this.options.callbacks,
|
|
2349
|
-
onProviderBackpressure: (retryAfterMs) =>
|
|
2350
|
-
this.reportBackpressure(input.sourceToolId, retryAfterMs),
|
|
2351
|
-
onRetryAttempt: () =>
|
|
2352
|
-
this.options.governor.chargeBudget('retry'),
|
|
2353
|
-
transientHttpRetrySafe:
|
|
2354
|
-
toolContract?.retrySafeTransientHttp === true,
|
|
2355
|
-
durableCallReceiptKey: aggregateReceiptKey,
|
|
2356
|
-
executionAuthScopeDigest:
|
|
2357
|
-
batch.memberRequests[0]?.request.executionAuthScopeDigest ??
|
|
2358
|
-
null,
|
|
2359
|
-
providerIdempotencyKey: buildBatchProviderIdempotencyKey({
|
|
2360
|
-
aggregateReceiptKey,
|
|
2361
|
-
receiptKeys,
|
|
2362
|
-
providerIdempotencyKeys: batch.memberRequests.map(
|
|
2363
|
-
(member) =>
|
|
2364
|
-
providerIdempotencyKeyForClaimed({
|
|
2365
|
-
req: this.options.req,
|
|
2366
|
-
claimed: member,
|
|
2367
|
-
}),
|
|
2368
|
-
),
|
|
2369
|
-
}),
|
|
2370
|
-
runtimeTimeoutMs: resolveWorkerToolRuntimeTimeoutMs(
|
|
2371
|
-
batch.memberRequests,
|
|
2372
|
-
{
|
|
2373
|
-
resolveOwnerTimeoutMs: (request) =>
|
|
2374
|
-
request.runtimeTimeoutMs,
|
|
2375
|
-
},
|
|
2376
|
-
),
|
|
2377
|
-
}),
|
|
2378
|
-
});
|
|
2379
|
-
} finally {
|
|
2380
|
-
this.options.recordPerfTrace?.({
|
|
2381
|
-
phase: 'runner.tool.provider',
|
|
2382
|
-
ms: this.options.nowMs() - providerStartedAt,
|
|
2383
|
-
extra: {
|
|
2384
|
-
toolId: batch.batchOperation,
|
|
2385
|
-
requests: batch.memberRequests.length,
|
|
2386
|
-
batched: true,
|
|
2387
|
-
},
|
|
2388
|
-
});
|
|
2389
|
-
}
|
|
2390
|
-
} finally {
|
|
2391
|
-
slot.release();
|
|
2392
|
-
}
|
|
2393
|
-
},
|
|
2394
|
-
onChunkComplete: async (
|
|
2395
|
-
chunkResults: Array<
|
|
2396
|
-
ChunkExecutionResult<(typeof compiledBatches)[number], unknown>
|
|
2397
|
-
>,
|
|
2398
|
-
) => {
|
|
2399
|
-
for (const entry of chunkResults) {
|
|
2400
|
-
if (entry.error !== undefined) {
|
|
2401
|
-
// One batch's provider error stays scoped to that batch's member
|
|
2402
|
-
// requests. Sibling batches in this chunk keep their results so a
|
|
2403
|
-
// single provider hiccup cannot cascade into a whole-map failure.
|
|
2404
|
-
if (
|
|
2405
|
-
entry.error instanceof ToolExecuteAuthScopeChangedError &&
|
|
2406
|
-
entry.request.memberRequests.every(
|
|
2407
|
-
(claimed) => claimed.request.authScopeRetryUsed !== true,
|
|
2408
|
-
) &&
|
|
2409
|
-
(await this.requeueAfterAuthScopeChanged(
|
|
2410
|
-
entry.request.memberRequests.map((claimed) => claimed.request),
|
|
2411
|
-
entry.error,
|
|
2412
|
-
))
|
|
2413
|
-
) {
|
|
2414
|
-
continue;
|
|
2415
|
-
}
|
|
2416
|
-
let rejection: unknown = entry.error;
|
|
2417
|
-
if (!(entry.error instanceof RuntimeLeaseLostError)) {
|
|
2418
|
-
try {
|
|
2419
|
-
await this.failDurableToolRequests(
|
|
2420
|
-
entry.request.memberRequests,
|
|
2421
|
-
entry.error,
|
|
2422
|
-
);
|
|
2423
|
-
} catch (receiptError) {
|
|
2424
|
-
rejection = receiptPersistenceError({
|
|
2425
|
-
operation: 'fail',
|
|
2426
|
-
receiptKeys: entry.request.memberRequests
|
|
2427
|
-
.map((claimed) => claimed.receiptKey)
|
|
2428
|
-
.filter((key): key is string => typeof key === 'string'),
|
|
2429
|
-
cause: new AggregateError(
|
|
2430
|
-
[entry.error, receiptError],
|
|
2431
|
-
'Tool receipts fail mark failed',
|
|
2432
|
-
),
|
|
2433
|
-
});
|
|
2434
|
-
}
|
|
2435
|
-
}
|
|
2436
|
-
for (const claimed of entry.request.memberRequests) {
|
|
2437
|
-
this.rejectRequests(claimed, rejection);
|
|
2438
|
-
}
|
|
2439
|
-
continue;
|
|
2440
|
-
}
|
|
2441
|
-
const batchResult = isToolExecuteResult(entry.result)
|
|
2442
|
-
? entry.result.toolOutput.raw
|
|
2443
|
-
: entry.result;
|
|
2444
|
-
const splitResults =
|
|
2445
|
-
batchResult != null
|
|
2446
|
-
? entry.request.splitResults(batchResult)
|
|
2447
|
-
: entry.request.memberRequests.map(() => null);
|
|
2448
|
-
let completedResults: WorkerToolCompletionResult[];
|
|
2449
|
-
try {
|
|
2450
|
-
const receiptCompleteStartedAt = this.options.nowMs();
|
|
2451
|
-
completedResults = await this.completeDurableToolRequests(
|
|
2452
|
-
entry.request.memberRequests.map((claimed, index) => ({
|
|
2453
|
-
claimed,
|
|
2454
|
-
result: wrapWorkerToolResult(
|
|
2455
|
-
claimed.request.toolId,
|
|
2456
|
-
splitResults[index] ?? null,
|
|
2457
|
-
toolMetadataFallback(claimed.request.toolId),
|
|
2458
|
-
),
|
|
2459
|
-
})),
|
|
2460
|
-
);
|
|
2461
|
-
this.options.recordPerfTrace?.({
|
|
2462
|
-
phase: 'runner.tool.receipt_complete',
|
|
2463
|
-
ms: this.options.nowMs() - receiptCompleteStartedAt,
|
|
2464
|
-
extra: {
|
|
2465
|
-
toolId: entry.request.batchOperation,
|
|
2466
|
-
requests: entry.request.memberRequests.length,
|
|
2467
|
-
batched: true,
|
|
2468
|
-
},
|
|
2469
|
-
});
|
|
2470
|
-
} catch (receiptError) {
|
|
2471
|
-
for (const claimed of entry.request.memberRequests) {
|
|
2472
|
-
this.rejectRequests(claimed, receiptError);
|
|
2473
|
-
}
|
|
2474
|
-
continue;
|
|
2475
|
-
}
|
|
2476
|
-
for (let index = 0; index < completedResults.length; index += 1) {
|
|
2477
|
-
const claimed = entry.request.memberRequests[index]!;
|
|
2478
|
-
const completed = completedResults[index];
|
|
2479
|
-
if (!completed) {
|
|
2480
|
-
this.rejectRequests(
|
|
2481
|
-
claimed,
|
|
2482
|
-
new Error(
|
|
2483
|
-
`Tool batch completion did not return member ${claimed.request.id}.`,
|
|
2484
|
-
),
|
|
2485
|
-
);
|
|
2486
|
-
} else if (completed.ok) {
|
|
2487
|
-
this.settleRequests(claimed, completed.value);
|
|
2488
|
-
} else {
|
|
2489
|
-
this.rejectRequests(claimed, completed.error);
|
|
2490
|
-
}
|
|
2491
|
-
}
|
|
2492
|
-
}
|
|
2493
|
-
},
|
|
2494
|
-
}).catch(async (error) => {
|
|
2495
|
-
let rejection: unknown = error;
|
|
2496
|
-
if (
|
|
2497
|
-
error instanceof ToolExecuteAuthScopeChangedError &&
|
|
2498
|
-
(await this.requeueAfterAuthScopeChanged(
|
|
2499
|
-
input.requests.map((claimed) => claimed.request),
|
|
2500
|
-
error,
|
|
2501
|
-
))
|
|
2502
|
-
) {
|
|
2503
|
-
return;
|
|
2504
|
-
}
|
|
2505
|
-
if (!(error instanceof RuntimeLeaseLostError)) {
|
|
2506
|
-
try {
|
|
2507
|
-
await this.failDurableToolRequests(input.requests, error);
|
|
2508
|
-
} catch (receiptError) {
|
|
2509
|
-
rejection = receiptPersistenceError({
|
|
2510
|
-
operation: 'fail',
|
|
2511
|
-
receiptKeys: input.requests
|
|
2512
|
-
.map((claimed) => claimed.receiptKey)
|
|
2513
|
-
.filter((key): key is string => typeof key === 'string'),
|
|
2514
|
-
cause: new AggregateError(
|
|
2515
|
-
[error, receiptError],
|
|
2516
|
-
'Tool receipts fail mark failed',
|
|
2517
|
-
),
|
|
2518
|
-
});
|
|
2519
|
-
}
|
|
2520
|
-
}
|
|
2521
|
-
for (const claimed of input.requests) {
|
|
2522
|
-
this.rejectRequests(claimed, rejection);
|
|
2523
|
-
}
|
|
2524
|
-
});
|
|
2525
|
-
}
|
|
2526
|
-
}
|
|
2527
|
-
|
|
2528
|
-
function groupWorkerToolRequestsByTool(
|
|
2529
|
-
requests: WorkerToolBatchRequest[],
|
|
2530
|
-
): Map<string, WorkerToolBatchRequest[]> {
|
|
2531
|
-
const byTool = new Map<string, WorkerToolBatchRequest[]>();
|
|
2532
|
-
for (const request of requests) {
|
|
2533
|
-
const existing = byTool.get(request.toolId);
|
|
2534
|
-
if (existing) {
|
|
2535
|
-
existing.push(request);
|
|
2536
|
-
} else {
|
|
2537
|
-
byTool.set(request.toolId, [request]);
|
|
2538
|
-
}
|
|
2539
|
-
}
|
|
2540
|
-
return byTool;
|
|
2541
|
-
}
|