deepline 0.1.182 → 0.1.184
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +639 -109
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1752 -1899
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
- package/dist/bundling-sources/sdk/src/client.ts +41 -0
- package/dist/bundling-sources/sdk/src/http.ts +23 -8
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +133 -10
- package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
- package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1612 -287
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +105 -6
- package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
- package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
- package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
- package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
- package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2038 -262
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +28 -11
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +38 -6
- package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
- package/dist/cli/index.js +76 -17
- package/dist/cli/index.mjs +76 -17
- package/dist/index.d.mts +2 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +63 -9
- package/dist/index.mjs +63 -9
- package/dist/plays/bundle-play-file.mjs +20 -6
- package/package.json +1 -1
|
@@ -36,12 +36,6 @@ import {
|
|
|
36
36
|
deterministicMapChunkStepName,
|
|
37
37
|
type ExecutionPlan,
|
|
38
38
|
} from '../../../shared_libs/play-runtime/execution-plan';
|
|
39
|
-
import {
|
|
40
|
-
compileRequestsWithStrategy,
|
|
41
|
-
executeChunkedRequests,
|
|
42
|
-
type ChunkExecutionResult,
|
|
43
|
-
} from '../../../shared_libs/play-runtime/batch-runtime';
|
|
44
|
-
import { getPlayRuntimeBatchStrategy } from '../../../shared_libs/play-runtime/play-runtime-batching-registry';
|
|
45
39
|
import {
|
|
46
40
|
STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
|
|
47
41
|
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
|
|
@@ -71,11 +65,10 @@ import {
|
|
|
71
65
|
type WorkflowStepLike,
|
|
72
66
|
} from './child-play-await';
|
|
73
67
|
import { submitChildPlayThroughCoordinator } from './child-play-submit';
|
|
74
|
-
import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
|
|
75
68
|
import {
|
|
76
69
|
attachToolResultListDataset,
|
|
77
|
-
parseToolExecuteResponse,
|
|
78
70
|
createToolExecuteResult,
|
|
71
|
+
parseToolExecuteResponse,
|
|
79
72
|
deserializeToolExecuteResult,
|
|
80
73
|
isSerializedToolExecuteResult,
|
|
81
74
|
isToolExecuteResult,
|
|
@@ -90,29 +83,35 @@ import {
|
|
|
90
83
|
TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS,
|
|
91
84
|
classifyToolExecuteHttpFailure,
|
|
92
85
|
createToolExecuteHttpFailureAttemptTracker,
|
|
86
|
+
parseToolExecuteAuthScopeChangedError,
|
|
93
87
|
} from '../../../shared_libs/play-runtime/tool-execute-retry-policy';
|
|
94
88
|
import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
95
89
|
import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
|
|
96
|
-
import type {
|
|
90
|
+
import type {
|
|
91
|
+
PlayRuntimeManifest,
|
|
92
|
+
PlayRuntimeManifestMap,
|
|
93
|
+
} from '../../../shared_libs/plays/compiler-manifest';
|
|
97
94
|
import {
|
|
98
95
|
deriveToolRequestIdentity,
|
|
99
96
|
derivePlayRowIdentity,
|
|
100
97
|
derivePlayRowIdentityFromKey,
|
|
101
98
|
sha256Hex,
|
|
99
|
+
stableStringify,
|
|
102
100
|
} from '../../../shared_libs/plays/row-identity';
|
|
103
101
|
import { createDeferredPlayDataset } from '../../../shared_libs/plays/dataset';
|
|
104
102
|
import {
|
|
105
103
|
buildDurableCtxCallCacheKey,
|
|
106
|
-
|
|
107
|
-
buildDurableToolCallCacheKey,
|
|
104
|
+
buildDurableRunPlaySemanticKey,
|
|
108
105
|
} from '../../../shared_libs/play-runtime/durable-call-cache';
|
|
106
|
+
import {
|
|
107
|
+
resolveRuntimeToolReceiptWaitMaxAttempts,
|
|
108
|
+
resolveRuntimeToolReceiptWaitTimeoutMs,
|
|
109
|
+
} from '../../../shared_libs/play-runtime/durable-receipt-execution';
|
|
109
110
|
import {
|
|
110
111
|
QUERY_RESULT_DATASET_PAGE_SIZE,
|
|
111
112
|
isCustomerDbDatasetTool,
|
|
112
|
-
isQueryResultDatasetReadRequest,
|
|
113
113
|
isQueryResultDatasetTool,
|
|
114
114
|
} from '../../../shared_libs/play-runtime/query-result-dataset';
|
|
115
|
-
import { buildScopedWorkReceiptKey } from '../../../shared_libs/play-runtime/work-receipts';
|
|
116
115
|
import { DEDUPE_DUPLICATE_KEY_SAMPLE_CAP } from '../../../shared_libs/play-runtime/map-row-identity';
|
|
117
116
|
import {
|
|
118
117
|
getTopLevelPipelineSubsteps,
|
|
@@ -151,26 +150,45 @@ import {
|
|
|
151
150
|
} from './runtime/dataset-handles';
|
|
152
151
|
import {
|
|
153
152
|
runWorkerRuntimeReceiptBoundary,
|
|
154
|
-
|
|
155
|
-
type
|
|
156
|
-
type WorkerRuntimeReceiptStore,
|
|
153
|
+
RuntimeLeaseLostError,
|
|
154
|
+
type WorkerRuntimeReceiptExecutionContext,
|
|
157
155
|
} from './runtime/receipts';
|
|
158
156
|
import {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
} from '
|
|
164
|
-
import
|
|
157
|
+
RuntimeReceiptPersistenceError,
|
|
158
|
+
WorkerToolBatchScheduler,
|
|
159
|
+
type WorkerPacingResolver,
|
|
160
|
+
type WorkerToolActionCacheVersionResolver,
|
|
161
|
+
} from './runtime/tool-dispatch';
|
|
162
|
+
import {
|
|
163
|
+
createWorkerWorkBudgetMeter,
|
|
164
|
+
type WorkerWorkBudgetMeter,
|
|
165
|
+
} from './runtime/work-budget';
|
|
166
|
+
import {
|
|
167
|
+
WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES,
|
|
168
|
+
WorkerRunWorkDispatcher,
|
|
169
|
+
type WorkerRunMapBatchesResult,
|
|
170
|
+
} from './runtime/run-work-dispatcher';
|
|
171
|
+
import { WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL } from './runtime/worker-platform-budget';
|
|
172
|
+
import {
|
|
173
|
+
PLAY_RUNTIME_CONTRACT,
|
|
174
|
+
PLAY_RUNTIME_CONTRACT_HEADER,
|
|
175
|
+
} from '../../../shared_libs/play-runtime/runtime-contract';
|
|
176
|
+
import {
|
|
177
|
+
PLAY_RUNTIME_API_COMPAT_PATH,
|
|
178
|
+
PLAY_RUNTIME_EGRESS_FETCH_COMPAT_PATH,
|
|
179
|
+
} from '../../../shared_libs/play-runtime/runtime-api-paths';
|
|
180
|
+
import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
|
|
181
|
+
import {
|
|
182
|
+
PLAY_RUNTIME_TEST_FAULT_HEADER,
|
|
183
|
+
type RuntimeTestPolicyOverrides,
|
|
184
|
+
} from '../../../shared_libs/play-runtime/test-runtime-seams';
|
|
185
|
+
import { createRuntimeToolAuthScopeDigestResolver } from '../../../shared_libs/play-runtime/auth-scope-resolver';
|
|
165
186
|
import {
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
resolveWorkerToolReceiptGroupWaitMaxAttempts,
|
|
172
|
-
resolveWorkerToolRuntimeTimeoutMs,
|
|
173
|
-
} from './runtime/tool-receipts';
|
|
187
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
188
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
189
|
+
runtimeLeaseHeartbeatIntervalMs,
|
|
190
|
+
} from '../../../shared_libs/play-runtime/lease-policy';
|
|
191
|
+
import { RuntimeSheetRowsBlockedError } from '../../../shared_libs/play-runtime/runtime-sheet-errors';
|
|
174
192
|
// The harness stub forwards leaf calls (validation, runtime-api HTTP) into
|
|
175
193
|
// the long-lived Play Harness Worker via env.HARNESS. We import the
|
|
176
194
|
// `setHarnessBinding` setter eagerly so it's available the moment
|
|
@@ -183,9 +201,12 @@ import {
|
|
|
183
201
|
// re-bundle harness internals into per-play. Keep that in mind.
|
|
184
202
|
import {
|
|
185
203
|
harnessPersistCompletedSheetRows,
|
|
204
|
+
harnessHeartbeatSheetAttempt,
|
|
186
205
|
harnessReadSheetDatasetRowKeys,
|
|
187
206
|
harnessReadSheetDatasetRows,
|
|
188
207
|
harnessReadStagedFileChunk,
|
|
208
|
+
harnessReleaseRuntimeReceipts,
|
|
209
|
+
harnessReleaseSheetAttempt,
|
|
189
210
|
harnessStartSheetDataset,
|
|
190
211
|
setHarnessBinding,
|
|
191
212
|
} from '../../../sdk/src/plays/harness-stub';
|
|
@@ -216,7 +237,10 @@ import {
|
|
|
216
237
|
stripMapRowOutcomeRuntimeFields,
|
|
217
238
|
} from '../../../shared_libs/play-runtime/map-row-outcome';
|
|
218
239
|
import { runtimeSheetSessionScope } from '../../../shared_libs/play-runtime/runtime-sheet-session';
|
|
219
|
-
import {
|
|
240
|
+
import {
|
|
241
|
+
chooseWorkerMapDispatchPlan,
|
|
242
|
+
estimateBatchedToolChunkSubrequests,
|
|
243
|
+
} from './runtime/map-chunk-plan';
|
|
220
244
|
import {
|
|
221
245
|
applyCsvRenameProjection,
|
|
222
246
|
cloneCsvAliasedRow,
|
|
@@ -228,12 +252,7 @@ import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-f
|
|
|
228
252
|
import {
|
|
229
253
|
createRuntimePersistenceLatch,
|
|
230
254
|
isRuntimePersistenceCircuitOpenError,
|
|
231
|
-
partitionDispatchWaves,
|
|
232
|
-
PROVIDER_DISPATCH_WAVE_SIZE,
|
|
233
|
-
RuntimePersistenceCircuitOpenError,
|
|
234
|
-
shouldShortenCompleteReceiptLadder,
|
|
235
255
|
tripRuntimePersistenceLatch,
|
|
236
|
-
type RuntimePersistenceLatch,
|
|
237
256
|
} from '../../../shared_libs/play-runtime/persistence-latch';
|
|
238
257
|
import {
|
|
239
258
|
createReceiptSalvageBuffer,
|
|
@@ -299,8 +318,13 @@ type RunRequest = {
|
|
|
299
318
|
playName: string;
|
|
300
319
|
graphHash?: string | null;
|
|
301
320
|
userEmail: string | null;
|
|
302
|
-
force?: boolean | null;
|
|
303
321
|
runtimeInput: Record<string, unknown>;
|
|
322
|
+
/** Start a fresh run graph and recompute runtime-sheet rows. */
|
|
323
|
+
force?: boolean;
|
|
324
|
+
/** Explicit durable tool receipt refresh that can re-execute completed provider calls. */
|
|
325
|
+
forceToolRefresh?: boolean;
|
|
326
|
+
/** Monotonic coordinator redispatch epoch for receipt and sheet fencing. */
|
|
327
|
+
runAttempt?: number;
|
|
304
328
|
/** Optional inline CSV rows (for plays where ctx.csv was passed inline data). */
|
|
305
329
|
inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
|
|
306
330
|
/** Staged input files keyed by logical filename (used by ctx.csv). */
|
|
@@ -333,6 +357,10 @@ type RunRequest = {
|
|
|
333
357
|
coordinatorUrl?: string | null;
|
|
334
358
|
/** Request-scoped coordinator auth token for preview/dev direct control calls. */
|
|
335
359
|
coordinatorInternalToken?: string | null;
|
|
360
|
+
/** Request-scoped, dev-only runtime fault injection header for black-box tests. */
|
|
361
|
+
runtimeTestFaultHeader?: string | null;
|
|
362
|
+
/** Request-scoped, dev-only policy overrides for black-box durability tests. */
|
|
363
|
+
testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
|
|
336
364
|
/**
|
|
337
365
|
* Total input rows known at submit time, when the dispatcher can count them.
|
|
338
366
|
* `undefined` when the input streams from R2 (unknown row count).
|
|
@@ -348,10 +376,104 @@ type WorkerFileRef = {
|
|
|
348
376
|
bytes?: number | null;
|
|
349
377
|
};
|
|
350
378
|
|
|
379
|
+
function normalizeWorkerRunAttempt(value: unknown): number {
|
|
380
|
+
return typeof value === 'number' && Number.isFinite(value)
|
|
381
|
+
? Math.max(0, Math.floor(value))
|
|
382
|
+
: 0;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
function normalizePositiveInteger(value: unknown): number | undefined {
|
|
386
|
+
return typeof value === 'number' &&
|
|
387
|
+
Number.isFinite(value) &&
|
|
388
|
+
Number.isInteger(value) &&
|
|
389
|
+
value > 0
|
|
390
|
+
? value
|
|
391
|
+
: undefined;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function normalizeRuntimeTestPolicyOverrides(
|
|
395
|
+
value: unknown,
|
|
396
|
+
): RuntimeTestPolicyOverrides | null {
|
|
397
|
+
if (!isRecord(value)) return null;
|
|
398
|
+
const overrides: RuntimeTestPolicyOverrides = {};
|
|
399
|
+
const receiptLeaseTtlMs = normalizePositiveInteger(value.receiptLeaseTtlMs);
|
|
400
|
+
if (receiptLeaseTtlMs !== undefined) {
|
|
401
|
+
overrides.receiptLeaseTtlMs = receiptLeaseTtlMs;
|
|
402
|
+
}
|
|
403
|
+
const sheetAttemptLeaseMs = normalizePositiveInteger(
|
|
404
|
+
value.sheetAttemptLeaseMs,
|
|
405
|
+
);
|
|
406
|
+
if (sheetAttemptLeaseMs !== undefined) {
|
|
407
|
+
overrides.sheetAttemptLeaseMs = sheetAttemptLeaseMs;
|
|
408
|
+
}
|
|
409
|
+
const heartbeatIntervalMs = normalizePositiveInteger(
|
|
410
|
+
value.heartbeatIntervalMs,
|
|
411
|
+
);
|
|
412
|
+
if (heartbeatIntervalMs !== undefined) {
|
|
413
|
+
overrides.heartbeatIntervalMs = heartbeatIntervalMs;
|
|
414
|
+
}
|
|
415
|
+
const batchGraceMs = normalizePositiveInteger(value.batchGraceMs);
|
|
416
|
+
if (batchGraceMs !== undefined) {
|
|
417
|
+
overrides.batchGraceMs = batchGraceMs;
|
|
418
|
+
}
|
|
419
|
+
if (isRecord(value.workBudgetYieldLimits)) {
|
|
420
|
+
const limits: NonNullable<
|
|
421
|
+
RuntimeTestPolicyOverrides['workBudgetYieldLimits']
|
|
422
|
+
> = {};
|
|
423
|
+
for (const key of [
|
|
424
|
+
'elapsed',
|
|
425
|
+
'subrequest',
|
|
426
|
+
'provider',
|
|
427
|
+
'tool',
|
|
428
|
+
'receipt',
|
|
429
|
+
'sheet',
|
|
430
|
+
'log',
|
|
431
|
+
'egress',
|
|
432
|
+
] as const) {
|
|
433
|
+
const limit = normalizePositiveInteger(value.workBudgetYieldLimits[key]);
|
|
434
|
+
if (limit !== undefined) {
|
|
435
|
+
limits[key] = limit;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
if (Object.keys(limits).length > 0) {
|
|
439
|
+
overrides.workBudgetYieldLimits = limits;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
return Object.keys(overrides).length > 0 ? overrides : null;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function runtimeReceiptLeaseTtlMs(req: RunRequest): number | undefined {
|
|
446
|
+
return req.testPolicyOverrides?.receiptLeaseTtlMs;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
function runtimeSheetAttemptLeaseTtlMs(req: RunRequest): number | undefined {
|
|
450
|
+
return req.testPolicyOverrides?.sheetAttemptLeaseMs;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
function runtimePolicyHeartbeatIntervalMs(
|
|
454
|
+
req: RunRequest,
|
|
455
|
+
fallbackLeaseTtlMs: number,
|
|
456
|
+
): number {
|
|
457
|
+
return (
|
|
458
|
+
req.testPolicyOverrides?.heartbeatIntervalMs ??
|
|
459
|
+
runtimeLeaseHeartbeatIntervalMs(fallbackLeaseTtlMs)
|
|
460
|
+
);
|
|
461
|
+
}
|
|
462
|
+
|
|
351
463
|
const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
|
|
352
464
|
const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
|
|
353
465
|
const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
|
|
354
466
|
const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
|
|
467
|
+
// A short durable sleep is enough to force a Cloudflare Workflows step
|
|
468
|
+
// checkpoint. The resume re-enters run() as a new Worker invocation with a
|
|
469
|
+
// fresh per-invocation subrequest budget (see the yield callback below), so
|
|
470
|
+
// there is no need to escalate the sleep to "encourage" re-invocation.
|
|
471
|
+
// A 1ms Workflow sleep is too small to reliably create a new platform
|
|
472
|
+
// invocation in preview; the dispatcher then resets its local budget while
|
|
473
|
+
// Cloudflare's per-invocation subrequest counter is still hot. Use a real
|
|
474
|
+
// boundary so budget-yielded maps resume with fresh platform headroom.
|
|
475
|
+
const WORK_DISPATCHER_YIELD_SLEEP_MS = 1_000;
|
|
476
|
+
|
|
355
477
|
type CustomerDbDatasetRequest = {
|
|
356
478
|
limit?: number;
|
|
357
479
|
offset?: number;
|
|
@@ -418,7 +540,8 @@ type WorkerEnv = {
|
|
|
418
540
|
/**
|
|
419
541
|
* In-process Fetcher constructed by the coordinator and handed to the
|
|
420
542
|
* per-graphHash play Worker. When present, runtime callbacks
|
|
421
|
-
* (`/api/v2/
|
|
543
|
+
* (`/api/v2/internal/play-runtime`,
|
|
544
|
+
* `/api/v2/internal/play-runtime/*`,
|
|
422
545
|
* `/api/v2/plays/runtime-tools/*`) skip the public callback URL and route
|
|
423
546
|
* directly through the coordinator's process to the configured app — saves
|
|
424
547
|
* the *.workers.dev → CF edge → cloudflared → localhost chain on every
|
|
@@ -427,6 +550,7 @@ type WorkerEnv = {
|
|
|
427
550
|
*/
|
|
428
551
|
RUNTIME_API?: {
|
|
429
552
|
runtimeApiCall(input: {
|
|
553
|
+
contract?: number;
|
|
430
554
|
executorToken: string;
|
|
431
555
|
baseUrl?: string;
|
|
432
556
|
path: string;
|
|
@@ -663,7 +787,7 @@ async function fetchRuntimeApi(
|
|
|
663
787
|
? Math.max(1, Math.ceil(options.timeoutMsOverride))
|
|
664
788
|
: path === '/api/v2/plays/run'
|
|
665
789
|
? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
|
|
666
|
-
: path ===
|
|
790
|
+
: path === PLAY_RUNTIME_EGRESS_FETCH_COMPAT_PATH
|
|
667
791
|
? RUNTIME_API_EGRESS_FETCH_TIMEOUT_MS
|
|
668
792
|
: /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
|
|
669
793
|
? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
|
|
@@ -691,6 +815,10 @@ async function fetchRuntimeApi(
|
|
|
691
815
|
);
|
|
692
816
|
}, timeoutMs);
|
|
693
817
|
});
|
|
818
|
+
const relayAbort = () => controller.abort(abortSignal?.reason);
|
|
819
|
+
if (abortSignal) {
|
|
820
|
+
abortSignal.addEventListener('abort', relayAbort, { once: true });
|
|
821
|
+
}
|
|
694
822
|
try {
|
|
695
823
|
const mergedInit: RequestInit = {
|
|
696
824
|
...init,
|
|
@@ -712,11 +840,10 @@ async function fetchRuntimeApi(
|
|
|
712
840
|
timeoutMs,
|
|
713
841
|
},
|
|
714
842
|
);
|
|
715
|
-
// RUNTIME_API service bindings do not consume RequestInit.signal once the RPC
|
|
716
|
-
// is in flight. After dispatch, wait for the owner call to settle so the
|
|
717
|
-
// durable receipt records the real provider result instead of failing early
|
|
718
|
-
// and inviting duplicate provider work on retry.
|
|
719
843
|
const response = await Promise.race([responsePromise, timeoutPromise]);
|
|
844
|
+
if (abortSignal?.aborted) {
|
|
845
|
+
throw abortError();
|
|
846
|
+
}
|
|
720
847
|
if (await isRuntimeApiBindingNotFoundResponse(response)) {
|
|
721
848
|
throw new Error(
|
|
722
849
|
`[play-harness] RUNTIME_API service binding could not route ${path}; coordinator returned not found.`,
|
|
@@ -732,6 +859,9 @@ async function fetchRuntimeApi(
|
|
|
732
859
|
}
|
|
733
860
|
throw err;
|
|
734
861
|
} finally {
|
|
862
|
+
if (abortSignal) {
|
|
863
|
+
abortSignal.removeEventListener('abort', relayAbort);
|
|
864
|
+
}
|
|
735
865
|
if (timeout) clearTimeout(timeout);
|
|
736
866
|
}
|
|
737
867
|
}
|
|
@@ -748,10 +878,23 @@ async function callRuntimeApiRpcBinding(
|
|
|
748
878
|
if (metadata) headers[EXECUTE_TOOL_METADATA_HEADER] = metadata;
|
|
749
879
|
const contract = h.get(EXECUTE_RESPONSE_CONTRACT_HEADER);
|
|
750
880
|
if (contract) headers[EXECUTE_RESPONSE_CONTRACT_HEADER] = contract;
|
|
881
|
+
const idempotencyKey = h.get('x-deepline-idempotency-key');
|
|
882
|
+
if (idempotencyKey) {
|
|
883
|
+
headers['x-deepline-idempotency-key'] = idempotencyKey;
|
|
884
|
+
}
|
|
885
|
+
const cacheControl = h.get('cache-control');
|
|
886
|
+
if (cacheControl) headers['cache-control'] = cacheControl;
|
|
887
|
+
const pragma = h.get('pragma');
|
|
888
|
+
if (pragma) headers.pragma = pragma;
|
|
889
|
+
const runtimeTestFaultHeader = h.get(PLAY_RUNTIME_TEST_FAULT_HEADER);
|
|
890
|
+
if (runtimeTestFaultHeader) {
|
|
891
|
+
headers[PLAY_RUNTIME_TEST_FAULT_HEADER] = runtimeTestFaultHeader;
|
|
892
|
+
}
|
|
751
893
|
const responseIntent = h.get(EXECUTE_RESPONSE_INTENT_HEADER);
|
|
752
894
|
if (responseIntent) headers[EXECUTE_RESPONSE_INTENT_HEADER] = responseIntent;
|
|
753
895
|
const rawBody = typeof init.body === 'string' ? init.body : '';
|
|
754
896
|
const result = await binding.runtimeApiCall({
|
|
897
|
+
contract: PLAY_RUNTIME_CONTRACT,
|
|
755
898
|
executorToken: authorization.replace(/^Bearer\s+/i, '').trim(),
|
|
756
899
|
baseUrl: input.baseUrl,
|
|
757
900
|
path: input.path,
|
|
@@ -784,6 +927,7 @@ function runtimeApiHeaders(
|
|
|
784
927
|
includeVercelBypass: boolean,
|
|
785
928
|
): Headers {
|
|
786
929
|
const next = new Headers(headers);
|
|
930
|
+
next.set(PLAY_RUNTIME_CONTRACT_HEADER, String(PLAY_RUNTIME_CONTRACT));
|
|
787
931
|
if (includeVercelBypass) {
|
|
788
932
|
const bypassToken = cachedVercelProtectionBypassToken();
|
|
789
933
|
if (bypassToken) {
|
|
@@ -976,9 +1120,172 @@ async function drainRunnerPerfTraces(req: RunRequest): Promise<void> {
|
|
|
976
1120
|
]);
|
|
977
1121
|
}
|
|
978
1122
|
|
|
1123
|
+
let workerRequestSequence = 0;
|
|
1124
|
+
|
|
979
1125
|
function makeRequestId(): string {
|
|
980
|
-
|
|
981
|
-
return
|
|
1126
|
+
workerRequestSequence = (workerRequestSequence + 1) % Number.MAX_SAFE_INTEGER;
|
|
1127
|
+
return `worker-request-${workerRequestSequence}`;
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
function makeRuntimeSheetAttempt(input: {
|
|
1131
|
+
runId: string;
|
|
1132
|
+
runAttempt?: number | null;
|
|
1133
|
+
mapName: string;
|
|
1134
|
+
chunkIndex: number | 'finalize';
|
|
1135
|
+
}): {
|
|
1136
|
+
attemptId: string;
|
|
1137
|
+
attemptOwnerRunId: string;
|
|
1138
|
+
attemptExpiresAt: string | null;
|
|
1139
|
+
attemptSeq: number;
|
|
1140
|
+
} {
|
|
1141
|
+
const runAttempt =
|
|
1142
|
+
typeof input.runAttempt === 'number' && Number.isFinite(input.runAttempt)
|
|
1143
|
+
? Math.max(0, Math.floor(input.runAttempt))
|
|
1144
|
+
: 0;
|
|
1145
|
+
const attemptKey = sha256Hex(
|
|
1146
|
+
JSON.stringify({
|
|
1147
|
+
runId: input.runId,
|
|
1148
|
+
runAttempt,
|
|
1149
|
+
mapName: input.mapName,
|
|
1150
|
+
chunkIndex: input.chunkIndex,
|
|
1151
|
+
}),
|
|
1152
|
+
).slice(0, 16);
|
|
1153
|
+
return {
|
|
1154
|
+
attemptId: `sheet:${input.mapName}:${input.chunkIndex}:attempt-${runAttempt}:${attemptKey}`,
|
|
1155
|
+
attemptOwnerRunId: input.runId,
|
|
1156
|
+
attemptExpiresAt: null,
|
|
1157
|
+
attemptSeq: runAttempt,
|
|
1158
|
+
};
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
class RuntimeSheetAttemptLeaseLostError extends Error {
|
|
1162
|
+
readonly rowKeys: string[];
|
|
1163
|
+
readonly runId: string;
|
|
1164
|
+
readonly attemptId: string;
|
|
1165
|
+
readonly tableNamespace: string;
|
|
1166
|
+
|
|
1167
|
+
constructor(input: {
|
|
1168
|
+
rowKeys: string[];
|
|
1169
|
+
runId: string;
|
|
1170
|
+
attemptId: string;
|
|
1171
|
+
tableNamespace: string;
|
|
1172
|
+
cause?: unknown;
|
|
1173
|
+
}) {
|
|
1174
|
+
const rowSummary =
|
|
1175
|
+
input.rowKeys.length === 1
|
|
1176
|
+
? input.rowKeys[0]!
|
|
1177
|
+
: `${input.rowKeys.length} rows`;
|
|
1178
|
+
super(
|
|
1179
|
+
`Runtime sheet attempt ${input.attemptId} lost ownership of ${rowSummary} in ${input.tableNamespace}.`,
|
|
1180
|
+
{ cause: input.cause },
|
|
1181
|
+
);
|
|
1182
|
+
this.name = 'RuntimeSheetAttemptLeaseLostError';
|
|
1183
|
+
this.rowKeys = input.rowKeys;
|
|
1184
|
+
this.runId = input.runId;
|
|
1185
|
+
this.attemptId = input.attemptId;
|
|
1186
|
+
this.tableNamespace = input.tableNamespace;
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
async function executeWithRuntimeSheetAttemptHeartbeat<T>(input: {
|
|
1191
|
+
req: RunRequest;
|
|
1192
|
+
tableNamespace: string;
|
|
1193
|
+
sheetContract: PlaySheetContract;
|
|
1194
|
+
attemptId?: string | null;
|
|
1195
|
+
attemptOwnerRunId?: string | null;
|
|
1196
|
+
attemptSeq?: number | null;
|
|
1197
|
+
rowKeys: string[];
|
|
1198
|
+
execute: () => Promise<T> | T;
|
|
1199
|
+
}): Promise<T> {
|
|
1200
|
+
const attemptId = input.attemptId?.trim();
|
|
1201
|
+
const rowKeys = [
|
|
1202
|
+
...new Set(input.rowKeys.map((key) => key.trim()).filter(Boolean)),
|
|
1203
|
+
];
|
|
1204
|
+
if (!attemptId || rowKeys.length === 0) {
|
|
1205
|
+
return await input.execute();
|
|
1206
|
+
}
|
|
1207
|
+
const heartbeatIntervalMs = runtimePolicyHeartbeatIntervalMs(
|
|
1208
|
+
input.req,
|
|
1209
|
+
runtimeSheetAttemptLeaseTtlMs(input.req) ??
|
|
1210
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
1211
|
+
);
|
|
1212
|
+
|
|
1213
|
+
let stopped = false;
|
|
1214
|
+
let timeout: ReturnType<typeof setTimeout> | null = null;
|
|
1215
|
+
let rejectLeaseLost!: (error: RuntimeSheetAttemptLeaseLostError) => void;
|
|
1216
|
+
const leaseLost = new Promise<never>((_, reject) => {
|
|
1217
|
+
rejectLeaseLost = reject;
|
|
1218
|
+
});
|
|
1219
|
+
|
|
1220
|
+
const stop = () => {
|
|
1221
|
+
stopped = true;
|
|
1222
|
+
if (timeout !== null) {
|
|
1223
|
+
clearTimeout(timeout);
|
|
1224
|
+
timeout = null;
|
|
1225
|
+
}
|
|
1226
|
+
};
|
|
1227
|
+
|
|
1228
|
+
const heartbeatOnce = async () => {
|
|
1229
|
+
const result = await harnessHeartbeatSheetAttempt({
|
|
1230
|
+
...runtimeSheetSessionScope(input.req),
|
|
1231
|
+
tableNamespace: input.tableNamespace,
|
|
1232
|
+
sheetContract: input.sheetContract,
|
|
1233
|
+
runId: input.req.runId,
|
|
1234
|
+
attemptId,
|
|
1235
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? input.req.runId,
|
|
1236
|
+
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
1237
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
|
|
1238
|
+
keys: rowKeys,
|
|
1239
|
+
});
|
|
1240
|
+
const renewed = new Set(result.renewedKeys);
|
|
1241
|
+
const lostKeys = rowKeys.filter((key) => !renewed.has(key));
|
|
1242
|
+
if (lostKeys.length > 0) {
|
|
1243
|
+
throw new RuntimeSheetAttemptLeaseLostError({
|
|
1244
|
+
rowKeys: lostKeys,
|
|
1245
|
+
runId: input.req.runId,
|
|
1246
|
+
attemptId,
|
|
1247
|
+
tableNamespace: input.tableNamespace,
|
|
1248
|
+
});
|
|
1249
|
+
}
|
|
1250
|
+
};
|
|
1251
|
+
|
|
1252
|
+
const schedule = () => {
|
|
1253
|
+
timeout = setTimeout(() => {
|
|
1254
|
+
void (async () => {
|
|
1255
|
+
try {
|
|
1256
|
+
await heartbeatOnce();
|
|
1257
|
+
} catch (error) {
|
|
1258
|
+
stop();
|
|
1259
|
+
if (error instanceof RuntimeSheetAttemptLeaseLostError) {
|
|
1260
|
+
rejectLeaseLost(error);
|
|
1261
|
+
} else {
|
|
1262
|
+
rejectLeaseLost(
|
|
1263
|
+
new RuntimeSheetAttemptLeaseLostError({
|
|
1264
|
+
rowKeys,
|
|
1265
|
+
runId: input.req.runId,
|
|
1266
|
+
attemptId,
|
|
1267
|
+
tableNamespace: input.tableNamespace,
|
|
1268
|
+
cause: error,
|
|
1269
|
+
}),
|
|
1270
|
+
);
|
|
1271
|
+
}
|
|
1272
|
+
return;
|
|
1273
|
+
}
|
|
1274
|
+
if (!stopped) {
|
|
1275
|
+
schedule();
|
|
1276
|
+
}
|
|
1277
|
+
})();
|
|
1278
|
+
}, heartbeatIntervalMs);
|
|
1279
|
+
};
|
|
1280
|
+
|
|
1281
|
+
schedule();
|
|
1282
|
+
const execution = Promise.resolve().then(input.execute);
|
|
1283
|
+
execution.catch(() => {});
|
|
1284
|
+
try {
|
|
1285
|
+
return await Promise.race([execution, leaseLost]);
|
|
1286
|
+
} finally {
|
|
1287
|
+
stop();
|
|
1288
|
+
}
|
|
982
1289
|
}
|
|
983
1290
|
|
|
984
1291
|
/**
|
|
@@ -1004,6 +1311,11 @@ async function postRuntimeApi<T>(
|
|
|
1004
1311
|
baseUrl: string,
|
|
1005
1312
|
executorToken: string,
|
|
1006
1313
|
body: unknown,
|
|
1314
|
+
options: {
|
|
1315
|
+
runtimeTestFaultHeader?: string | null;
|
|
1316
|
+
timeoutMsOverride?: number;
|
|
1317
|
+
timeoutErrorMessage?: string;
|
|
1318
|
+
} = {},
|
|
1007
1319
|
): Promise<T> {
|
|
1008
1320
|
// Routes through the in-process RUNTIME_API service binding. Missing binding
|
|
1009
1321
|
// is an infra error in workers_edge, not a reason to fall back to public HTTP.
|
|
@@ -1016,15 +1328,29 @@ async function postRuntimeApi<T>(
|
|
|
1016
1328
|
) {
|
|
1017
1329
|
let res: Response;
|
|
1018
1330
|
try {
|
|
1019
|
-
res = await fetchRuntimeApi(
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1331
|
+
res = await fetchRuntimeApi(
|
|
1332
|
+
baseUrl,
|
|
1333
|
+
PLAY_RUNTIME_API_COMPAT_PATH,
|
|
1334
|
+
{
|
|
1335
|
+
method: 'POST',
|
|
1336
|
+
headers: {
|
|
1337
|
+
'content-type': 'application/json',
|
|
1338
|
+
authorization: `Bearer ${executorToken}`,
|
|
1339
|
+
'x-deepline-request-id': makeRequestId(),
|
|
1340
|
+
...(options.runtimeTestFaultHeader?.trim()
|
|
1341
|
+
? {
|
|
1342
|
+
[PLAY_RUNTIME_TEST_FAULT_HEADER]:
|
|
1343
|
+
options.runtimeTestFaultHeader.trim(),
|
|
1344
|
+
}
|
|
1345
|
+
: {}),
|
|
1346
|
+
},
|
|
1347
|
+
body: serializedBody,
|
|
1025
1348
|
},
|
|
1026
|
-
|
|
1027
|
-
|
|
1349
|
+
{
|
|
1350
|
+
timeoutMsOverride: options.timeoutMsOverride,
|
|
1351
|
+
timeoutErrorMessage: options.timeoutErrorMessage,
|
|
1352
|
+
},
|
|
1353
|
+
);
|
|
1028
1354
|
} catch (error) {
|
|
1029
1355
|
lastError = error;
|
|
1030
1356
|
if (
|
|
@@ -1108,9 +1434,10 @@ async function postRuntimeApiBestEffort(
|
|
|
1108
1434
|
baseUrl: string,
|
|
1109
1435
|
executorToken: string,
|
|
1110
1436
|
body: unknown,
|
|
1437
|
+
options: { runtimeTestFaultHeader?: string | null } = {},
|
|
1111
1438
|
): Promise<boolean> {
|
|
1112
1439
|
try {
|
|
1113
|
-
await postRuntimeApi(baseUrl, executorToken, body);
|
|
1440
|
+
await postRuntimeApi(baseUrl, executorToken, body, options);
|
|
1114
1441
|
return true;
|
|
1115
1442
|
} catch (error) {
|
|
1116
1443
|
console.error(
|
|
@@ -1122,6 +1449,43 @@ async function postRuntimeApiBestEffort(
|
|
|
1122
1449
|
}
|
|
1123
1450
|
}
|
|
1124
1451
|
|
|
1452
|
+
async function readDurableChildRunTerminalState(input: {
|
|
1453
|
+
baseUrl: string;
|
|
1454
|
+
executorToken: string;
|
|
1455
|
+
parentRunId: string;
|
|
1456
|
+
childRunId: string;
|
|
1457
|
+
childPlayName: string;
|
|
1458
|
+
timeoutMs?: number;
|
|
1459
|
+
}): Promise<{ data?: unknown } | null> {
|
|
1460
|
+
const response = await postRuntimeApi<{
|
|
1461
|
+
state?: {
|
|
1462
|
+
data?: unknown;
|
|
1463
|
+
reason?: string;
|
|
1464
|
+
status?: string;
|
|
1465
|
+
runId?: string;
|
|
1466
|
+
playName?: string;
|
|
1467
|
+
parentRunId?: string | null;
|
|
1468
|
+
} | null;
|
|
1469
|
+
}>(
|
|
1470
|
+
input.baseUrl,
|
|
1471
|
+
input.executorToken,
|
|
1472
|
+
{
|
|
1473
|
+
action: 'read_child_run_terminal_snapshot',
|
|
1474
|
+
parentRunId: input.parentRunId,
|
|
1475
|
+
childRunId: input.childRunId,
|
|
1476
|
+
childPlayName: input.childPlayName,
|
|
1477
|
+
},
|
|
1478
|
+
{
|
|
1479
|
+
timeoutMsOverride: input.timeoutMs,
|
|
1480
|
+
timeoutErrorMessage: `[play-harness] child run terminal snapshot read timed out after ${Math.max(
|
|
1481
|
+
1,
|
|
1482
|
+
Math.ceil(input.timeoutMs ?? RUNTIME_API_TIMEOUT_MS),
|
|
1483
|
+
)}ms. childRunId=${input.childRunId}`,
|
|
1484
|
+
},
|
|
1485
|
+
);
|
|
1486
|
+
return response.state ?? null;
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1125
1489
|
function workflowEventType(name: string): string {
|
|
1126
1490
|
const normalized = name
|
|
1127
1491
|
.trim()
|
|
@@ -1231,6 +1595,27 @@ async function hashChildPlayEventKey(input: unknown): Promise<string> {
|
|
|
1231
1595
|
return (await hashJson(input)).slice(0, 32);
|
|
1232
1596
|
}
|
|
1233
1597
|
|
|
1598
|
+
function workerChildRevisionFingerprint(input: {
|
|
1599
|
+
playName: string;
|
|
1600
|
+
manifest: PlayRuntimeManifest | null | undefined;
|
|
1601
|
+
}): string | null {
|
|
1602
|
+
const manifest = input.manifest;
|
|
1603
|
+
if (!manifest) return null;
|
|
1604
|
+
return sha256Hex(
|
|
1605
|
+
stableStringify({
|
|
1606
|
+
playId: manifest.playName?.trim() || input.playName,
|
|
1607
|
+
codeFormat: 'cjs_module',
|
|
1608
|
+
artifactHash: manifest.artifactHash ?? null,
|
|
1609
|
+
graphHash: manifest.graphHash ?? null,
|
|
1610
|
+
sourceHash: null,
|
|
1611
|
+
sourceCodeHash:
|
|
1612
|
+
typeof manifest.sourceCode === 'string'
|
|
1613
|
+
? sha256Hex(manifest.sourceCode)
|
|
1614
|
+
: null,
|
|
1615
|
+
}),
|
|
1616
|
+
);
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1234
1619
|
async function childPlayEventKey(input: {
|
|
1235
1620
|
key: string;
|
|
1236
1621
|
workflowId: string;
|
|
@@ -1327,7 +1712,14 @@ async function signalParentPlayTerminal(input: {
|
|
|
1327
1712
|
|
|
1328
1713
|
async function executeTool(
|
|
1329
1714
|
req: RunRequest,
|
|
1330
|
-
args: {
|
|
1715
|
+
args: {
|
|
1716
|
+
id: string;
|
|
1717
|
+
toolId: string;
|
|
1718
|
+
input: Record<string, unknown>;
|
|
1719
|
+
durableCallReceiptKey?: string | null;
|
|
1720
|
+
executionAuthScopeDigest?: string | null;
|
|
1721
|
+
providerIdempotencyKey?: string | null;
|
|
1722
|
+
},
|
|
1331
1723
|
workflowStep?: WorkflowStep,
|
|
1332
1724
|
onProviderBackpressure?: (retryAfterMs: number) => void,
|
|
1333
1725
|
onRetryAttempt?: () => void,
|
|
@@ -1365,7 +1757,14 @@ async function executeTool(
|
|
|
1365
1757
|
|
|
1366
1758
|
async function executeToolWithLifecycle(
|
|
1367
1759
|
req: RunRequest,
|
|
1368
|
-
args: {
|
|
1760
|
+
args: {
|
|
1761
|
+
id: string;
|
|
1762
|
+
toolId: string;
|
|
1763
|
+
input: Record<string, unknown>;
|
|
1764
|
+
durableCallReceiptKey?: string | null;
|
|
1765
|
+
executionAuthScopeDigest?: string | null;
|
|
1766
|
+
providerIdempotencyKey?: string | null;
|
|
1767
|
+
},
|
|
1369
1768
|
workflowStep: WorkflowStep | undefined,
|
|
1370
1769
|
callbacks: WorkerCtxCallbacks | undefined,
|
|
1371
1770
|
onProviderBackpressure?: (retryAfterMs: number) => void,
|
|
@@ -1461,24 +1860,29 @@ async function waitForSyntheticIntegrationEvent(
|
|
|
1461
1860
|
typeof input.timeout_ms === 'number' && Number.isFinite(input.timeout_ms)
|
|
1462
1861
|
? Math.max(1, Math.round(input.timeout_ms))
|
|
1463
1862
|
: 30_000;
|
|
1464
|
-
await postRuntimeApiBestEffort(
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1863
|
+
await postRuntimeApiBestEffort(
|
|
1864
|
+
req.baseUrl,
|
|
1865
|
+
req.executorToken,
|
|
1866
|
+
{
|
|
1867
|
+
action: 'append_run_events',
|
|
1868
|
+
playId: req.runId,
|
|
1869
|
+
events: [
|
|
1870
|
+
{
|
|
1871
|
+
type: 'log.appended',
|
|
1872
|
+
runId: req.runId,
|
|
1873
|
+
// 'system' (windowed text-dedupe channel), NOT 'worker': this line is
|
|
1874
|
+
// emitted outside the harness log buffer, so it has no positional
|
|
1875
|
+
// channelOffset and must not pollute the worker channel cursor.
|
|
1876
|
+
source: 'system',
|
|
1877
|
+
occurredAt: nowMs(),
|
|
1878
|
+
lines: [
|
|
1879
|
+
`Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
|
|
1880
|
+
],
|
|
1881
|
+
} satisfies PlayRunLedgerEvent,
|
|
1882
|
+
],
|
|
1883
|
+
},
|
|
1884
|
+
{ runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null },
|
|
1885
|
+
);
|
|
1482
1886
|
try {
|
|
1483
1887
|
const event = (await (
|
|
1484
1888
|
workflowStep.waitForEvent as unknown as (
|
|
@@ -1526,7 +1930,14 @@ async function waitForSyntheticIntegrationEvent(
|
|
|
1526
1930
|
|
|
1527
1931
|
async function callToolDirect(
|
|
1528
1932
|
req: RunRequest,
|
|
1529
|
-
args: {
|
|
1933
|
+
args: {
|
|
1934
|
+
id: string;
|
|
1935
|
+
toolId: string;
|
|
1936
|
+
input: Record<string, unknown>;
|
|
1937
|
+
durableCallReceiptKey?: string | null;
|
|
1938
|
+
executionAuthScopeDigest?: string | null;
|
|
1939
|
+
providerIdempotencyKey?: string | null;
|
|
1940
|
+
},
|
|
1530
1941
|
onProviderBackpressure?: (retryAfterMs: number) => void,
|
|
1531
1942
|
// Invoked once per in-process retry attempt (429 / retryable 5xx / synthetic
|
|
1532
1943
|
// transient) so the Governor charges chargeBudget('retry') per attempt — the
|
|
@@ -1542,6 +1953,16 @@ async function callToolDirect(
|
|
|
1542
1953
|
): Promise<ToolExecuteResult> {
|
|
1543
1954
|
const { id, toolId, input } = args;
|
|
1544
1955
|
const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
|
|
1956
|
+
const durableCallReceiptKey = args.durableCallReceiptKey?.trim() || null;
|
|
1957
|
+
const executionAuthScopeDigest =
|
|
1958
|
+
durableCallReceiptKey && args.executionAuthScopeDigest
|
|
1959
|
+
? args.executionAuthScopeDigest.trim() || null
|
|
1960
|
+
: null;
|
|
1961
|
+
const providerIdempotencyKey =
|
|
1962
|
+
args.providerIdempotencyKey?.trim() || durableCallReceiptKey;
|
|
1963
|
+
const deeplineRequestId = providerIdempotencyKey
|
|
1964
|
+
? `ctx-tool-${sha256Hex(providerIdempotencyKey).slice(0, 32)}`
|
|
1965
|
+
: null;
|
|
1545
1966
|
let lastError: Error | null = null;
|
|
1546
1967
|
const httpFailureAttempts = createToolExecuteHttpFailureAttemptTracker();
|
|
1547
1968
|
let requestAttempt = 0;
|
|
@@ -1564,7 +1985,12 @@ async function callToolDirect(
|
|
|
1564
1985
|
headers: {
|
|
1565
1986
|
'content-type': 'application/json',
|
|
1566
1987
|
authorization: `Bearer ${req.executorToken}`,
|
|
1567
|
-
'x-deepline-request-id':
|
|
1988
|
+
'x-deepline-request-id':
|
|
1989
|
+
deeplineRequestId ??
|
|
1990
|
+
`${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
|
|
1991
|
+
...(providerIdempotencyKey
|
|
1992
|
+
? { 'x-deepline-idempotency-key': providerIdempotencyKey }
|
|
1993
|
+
: {}),
|
|
1568
1994
|
[EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
|
|
1569
1995
|
[EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
|
|
1570
1996
|
[EXECUTE_TOOL_METADATA_HEADER]: 'true',
|
|
@@ -1573,6 +1999,16 @@ async function callToolDirect(
|
|
|
1573
1999
|
payload: input,
|
|
1574
2000
|
metadata: {
|
|
1575
2001
|
parent_run_id: req.runId,
|
|
2002
|
+
...(durableCallReceiptKey
|
|
2003
|
+
? {
|
|
2004
|
+
durable_call_receipt_key: durableCallReceiptKey,
|
|
2005
|
+
...(executionAuthScopeDigest
|
|
2006
|
+
? {
|
|
2007
|
+
execution_auth_scope_digest: executionAuthScopeDigest,
|
|
2008
|
+
}
|
|
2009
|
+
: {}),
|
|
2010
|
+
}
|
|
2011
|
+
: {}),
|
|
1576
2012
|
...(directOptions?.customerDbDataset
|
|
1577
2013
|
? {
|
|
1578
2014
|
query_result_dataset: {
|
|
@@ -1605,6 +2041,9 @@ async function callToolDirect(
|
|
|
1605
2041
|
: {}),
|
|
1606
2042
|
}
|
|
1607
2043
|
: {}),
|
|
2044
|
+
...(providerIdempotencyKey
|
|
2045
|
+
? { provider_idempotency_key: providerIdempotencyKey }
|
|
2046
|
+
: {}),
|
|
1608
2047
|
},
|
|
1609
2048
|
...(req.integrationMode
|
|
1610
2049
|
? { integration_mode: req.integrationMode }
|
|
@@ -1652,6 +2091,13 @@ async function callToolDirect(
|
|
|
1652
2091
|
}
|
|
1653
2092
|
|
|
1654
2093
|
const text = await res.text().catch(() => '');
|
|
2094
|
+
const authScopeChanged = parseToolExecuteAuthScopeChangedError({
|
|
2095
|
+
status: res.status,
|
|
2096
|
+
bodyText: text,
|
|
2097
|
+
});
|
|
2098
|
+
if (authScopeChanged) {
|
|
2099
|
+
throw authScopeChanged;
|
|
2100
|
+
}
|
|
1655
2101
|
const httpFailureAttempt = httpFailureAttempts.next({
|
|
1656
2102
|
toolId,
|
|
1657
2103
|
status: res.status,
|
|
@@ -1685,48 +2131,6 @@ async function callToolDirect(
|
|
|
1685
2131
|
throw lastError ?? new Error(`tool ${toolId} failed before execution.`);
|
|
1686
2132
|
}
|
|
1687
2133
|
|
|
1688
|
-
function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
|
|
1689
|
-
if (toolId === 'test_rate_limit') {
|
|
1690
|
-
// Batched members resolve metadata through this fallback because the lean
|
|
1691
|
-
// worker does not bundle the catalog. It MUST mirror the same
|
|
1692
|
-
// `email_status: emailStatus({...})` contract registered in
|
|
1693
|
-
// src/lib/integrations/test/index.ts so batched results normalize to the
|
|
1694
|
-
// same rich email_status OBJECT (status from statusMap, catch_all as a
|
|
1695
|
-
// signal) that the single-execute real-metadata path produces. The legacy
|
|
1696
|
-
// `transforms:['emailStatus']` + mx_security_gateway→'catch_all' string
|
|
1697
|
-
// override coarsened email_status into a bare string and predates the
|
|
1698
|
-
// emailStatus object contract (#1466).
|
|
1699
|
-
return {
|
|
1700
|
-
toolId,
|
|
1701
|
-
extractors: {
|
|
1702
|
-
email_status: {
|
|
1703
|
-
paths: ['email_status'],
|
|
1704
|
-
emailStatus: {
|
|
1705
|
-
provider: 'test',
|
|
1706
|
-
rawStatus: ['email_status'],
|
|
1707
|
-
catchAll: ['mx_security_gateway'],
|
|
1708
|
-
statusMap: {
|
|
1709
|
-
valid: { status: 'valid', verdict: 'send', verified: true },
|
|
1710
|
-
invalid: { status: 'invalid', verdict: 'drop', verified: false },
|
|
1711
|
-
catch_all: {
|
|
1712
|
-
status: 'catch_all',
|
|
1713
|
-
verdict: 'verify_next',
|
|
1714
|
-
verified: false,
|
|
1715
|
-
},
|
|
1716
|
-
unknown: { status: 'unknown', verdict: 'hold', verified: false },
|
|
1717
|
-
},
|
|
1718
|
-
},
|
|
1719
|
-
},
|
|
1720
|
-
},
|
|
1721
|
-
targetGetters: {
|
|
1722
|
-
email: ['email', 'value'],
|
|
1723
|
-
email_status: ['email_status'],
|
|
1724
|
-
},
|
|
1725
|
-
};
|
|
1726
|
-
}
|
|
1727
|
-
return { toolId };
|
|
1728
|
-
}
|
|
1729
|
-
|
|
1730
2134
|
function wrapWorkerToolResult(
|
|
1731
2135
|
toolId: string,
|
|
1732
2136
|
result: unknown,
|
|
@@ -1767,10 +2171,10 @@ function attachWorkerCustomerDbDatasetResult(
|
|
|
1767
2171
|
typeof options.input.sql === 'string'
|
|
1768
2172
|
? options.input.sql
|
|
1769
2173
|
: typeof options.input.query === 'string'
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
2174
|
+
? options.input.query
|
|
2175
|
+
: typeof raw?.sql === 'string'
|
|
2176
|
+
? raw.sql
|
|
2177
|
+
: null;
|
|
1774
2178
|
if (!dataset || totalRows === null || !sql) {
|
|
1775
2179
|
return result;
|
|
1776
2180
|
}
|
|
@@ -1882,1453 +2286,91 @@ type WorkerStepProgramRecorder = {
|
|
|
1882
2286
|
parentField: string;
|
|
1883
2287
|
path: string[];
|
|
1884
2288
|
outputs: RecordedStepProgramOutput[];
|
|
1885
|
-
onOutput?: (output: RecordedStepProgramOutput) => void | Promise<void>;
|
|
1886
|
-
};
|
|
1887
|
-
|
|
1888
|
-
type WorkerStepResolution = {
|
|
1889
|
-
value: unknown;
|
|
1890
|
-
status?: 'skipped';
|
|
1891
|
-
};
|
|
1892
|
-
|
|
1893
|
-
type WorkerToolBatchRequest = {
|
|
1894
|
-
id: string;
|
|
1895
|
-
cacheKey: string;
|
|
1896
|
-
receiptKey: string | null;
|
|
1897
|
-
force: boolean;
|
|
1898
|
-
receiptWaitMaxAttempts: number;
|
|
1899
|
-
runtimeTimeoutMs?: number;
|
|
1900
|
-
toolId: string;
|
|
1901
|
-
input: Record<string, unknown>;
|
|
1902
|
-
workflowStep?: WorkflowStep;
|
|
1903
|
-
directOptions?: WorkerToolDirectOptions;
|
|
1904
|
-
resolve: (value: unknown) => void;
|
|
1905
|
-
reject: (error: unknown) => void;
|
|
1906
|
-
};
|
|
1907
|
-
|
|
1908
|
-
type ClaimedWorkerToolBatchRequest = {
|
|
1909
|
-
request: WorkerToolBatchRequest;
|
|
1910
|
-
receiptKey: string | null;
|
|
1911
|
-
followers: WorkerToolBatchRequest[];
|
|
1912
|
-
};
|
|
1913
|
-
|
|
1914
|
-
type PreparedWorkerToolBatchRequests = {
|
|
1915
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[];
|
|
1916
|
-
deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[];
|
|
1917
|
-
};
|
|
1918
|
-
|
|
1919
|
-
function resolveClaimedWorkerToolRuntimeTimeoutMs(
|
|
1920
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
1921
|
-
input: { runtimeDeadlineMs?: number },
|
|
1922
|
-
): number | undefined {
|
|
1923
|
-
return resolveWorkerToolRuntimeTimeoutMs(claimedRequests, {
|
|
1924
|
-
resolveOwnerTimeoutMs: (request) =>
|
|
1925
|
-
resolveToolRuntimeApiTimeout({
|
|
1926
|
-
requestInput: request.input,
|
|
1927
|
-
timeoutMs: request.runtimeTimeoutMs,
|
|
1928
|
-
runtimeDeadlineMs: input.runtimeDeadlineMs,
|
|
1929
|
-
}).timeoutMs,
|
|
1930
|
-
});
|
|
1931
|
-
}
|
|
1932
|
-
|
|
1933
|
-
const WORKER_TOOL_BATCH_GRACE_MS = 250;
|
|
1934
|
-
const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
|
|
1935
|
-
const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
|
|
1936
|
-
const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
|
|
1937
|
-
const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 1_000;
|
|
1938
|
-
const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
|
|
1939
|
-
/**
|
|
1940
|
-
* Bounded number of per-row failure samples carried in chunk summaries and the
|
|
1941
|
-
* map's terminal partial-failure log. Every failed row is persisted with its
|
|
1942
|
-
* full error in the runtime sheet; the samples just keep run logs readable.
|
|
1943
|
-
*/
|
|
1944
|
-
const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
|
|
1945
|
-
|
|
1946
|
-
class RuntimeReceiptPersistenceError extends Error {
|
|
1947
|
-
readonly errors: unknown[];
|
|
1948
|
-
|
|
1949
|
-
constructor(message: string, errors: unknown[] = []) {
|
|
1950
|
-
super(message);
|
|
1951
|
-
this.name = 'RuntimeReceiptPersistenceError';
|
|
1952
|
-
this.errors = errors;
|
|
1953
|
-
}
|
|
1954
|
-
}
|
|
1955
|
-
|
|
1956
|
-
function isRuntimeReceiptPersistenceError(error: unknown): boolean {
|
|
1957
|
-
if (!error || typeof error !== 'object') return false;
|
|
1958
|
-
if (error instanceof RuntimeReceiptPersistenceError) return true;
|
|
1959
|
-
if (
|
|
1960
|
-
error instanceof Error &&
|
|
1961
|
-
error.name === 'RuntimeReceiptPersistenceError'
|
|
1962
|
-
) {
|
|
1963
|
-
return true;
|
|
1964
|
-
}
|
|
1965
|
-
const nestedErrors = (error as { errors?: unknown }).errors;
|
|
1966
|
-
return (
|
|
1967
|
-
Array.isArray(nestedErrors) &&
|
|
1968
|
-
nestedErrors.some(isRuntimeReceiptPersistenceError)
|
|
1969
|
-
);
|
|
1970
|
-
}
|
|
1971
|
-
|
|
1972
|
-
function isRunFatalWorkerRowError(error: unknown): boolean {
|
|
1973
|
-
return (
|
|
1974
|
-
isCoordinatorProgressAuthorizationError(error) ||
|
|
1975
|
-
isRowIsolationExemptError(error) ||
|
|
1976
|
-
isRuntimeReceiptPersistenceError(error) ||
|
|
1977
|
-
isRuntimePersistenceCircuitOpenError(error)
|
|
1978
|
-
);
|
|
1979
|
-
}
|
|
1980
|
-
|
|
1981
|
-
/**
|
|
1982
|
-
* completeReceipt salvage retry ladder. On a persistence failure after a billed
|
|
1983
|
-
* call we retry the receipt completion past the server-side 5s connect window:
|
|
1984
|
-
* 1 initial attempt + 3 retries at 1s / 3s / 9s (single isolate — no jitter).
|
|
1985
|
-
* Bounded by the run deadline (`runtimeDeadlineMs`); we stop early unless there
|
|
1986
|
-
* is at least one backoff's worth plus headroom of budget left.
|
|
1987
|
-
*/
|
|
1988
|
-
const COMPLETE_RECEIPT_RETRY_BACKOFFS_MS = [1_000, 3_000, 9_000] as const;
|
|
1989
|
-
const COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS = 10_000;
|
|
1990
|
-
|
|
1991
|
-
// Fallback batch-chunk parallelism when a tool declares no provider rate hints.
|
|
1992
|
-
// Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
|
|
1993
|
-
// their previous batching behavior; declared providers tighten via the
|
|
1994
|
-
// Governor's suggestedParallelism.
|
|
1995
|
-
const WORKER_TOOL_BATCH_DEFAULT_PARALLELISM = 4;
|
|
1996
|
-
|
|
1997
|
-
/**
|
|
1998
|
-
* In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
|
|
1999
|
-
* throughput pacing (provider or Deepline limiter), not a tool defect, so it
|
|
2000
|
-
* gets more patience than the generic 3-attempt budget: with retry-after-aware
|
|
2001
|
-
* escalating delays (capped at 5s) this absorbs roughly 25s of sustained
|
|
2002
|
-
* throttling before the call fails. Every retry still charges the Governor's
|
|
2003
|
-
* retry budget, so a runaway storm stays bounded and loud.
|
|
2004
|
-
*/
|
|
2005
|
-
|
|
2006
|
-
function sleepWorkerMs(ms: number): Promise<void> {
|
|
2007
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2008
|
-
}
|
|
2009
|
-
|
|
2010
|
-
function workerDurableToolCallCacheKey(input: {
|
|
2011
|
-
req: RunRequest;
|
|
2012
|
-
toolId: string;
|
|
2013
|
-
requestInput: Record<string, unknown>;
|
|
2014
|
-
providerActionVersion: string;
|
|
2015
|
-
staleAfterSeconds?: number | null;
|
|
2016
|
-
}): string {
|
|
2017
|
-
return buildDurableToolCallCacheKey({
|
|
2018
|
-
orgId: input.req.orgId,
|
|
2019
|
-
playId: input.req.playName,
|
|
2020
|
-
toolId: input.toolId,
|
|
2021
|
-
requestInput: input.requestInput,
|
|
2022
|
-
authScopeDigest: buildDurableToolCallAuthScopeDigest({
|
|
2023
|
-
orgId: input.req.orgId,
|
|
2024
|
-
userEmail: input.req.userEmail,
|
|
2025
|
-
toolId: input.toolId,
|
|
2026
|
-
}),
|
|
2027
|
-
providerActionVersion: input.providerActionVersion,
|
|
2028
|
-
staleAfterSeconds: input.staleAfterSeconds,
|
|
2029
|
-
});
|
|
2030
|
-
}
|
|
2031
|
-
|
|
2032
|
-
function workerRuntimeReceiptKey(input: {
|
|
2033
|
-
req: RunRequest;
|
|
2034
|
-
key: string;
|
|
2035
|
-
}): string {
|
|
2036
|
-
const orgId = input.req.orgId?.trim() || 'org';
|
|
2037
|
-
if (input.key.startsWith(`ctx:${orgId}:`)) {
|
|
2038
|
-
return input.key;
|
|
2039
|
-
}
|
|
2040
|
-
return buildScopedWorkReceiptKey({
|
|
2041
|
-
orgId: input.req.orgId,
|
|
2042
|
-
playName: input.req.playName,
|
|
2043
|
-
key: input.key,
|
|
2044
|
-
});
|
|
2045
|
-
}
|
|
2046
|
-
|
|
2047
|
-
function stepProgramColumnName(parentField: string, stepId: string): string {
|
|
2048
|
-
return sqlSafePlayColumnName(`${parentField}.${stepId}`);
|
|
2049
|
-
}
|
|
2050
|
-
|
|
2051
|
-
class WorkerToolBatchScheduler {
|
|
2052
|
-
private queue: WorkerToolBatchRequest[] = [];
|
|
2053
|
-
private scheduled = false;
|
|
2054
|
-
|
|
2055
|
-
constructor(
|
|
2056
|
-
private readonly req: RunRequest,
|
|
2057
|
-
private readonly governor: PlayExecutionGovernor,
|
|
2058
|
-
private readonly resolvePacing: WorkerPacingResolver,
|
|
2059
|
-
private readonly resolveToolActionCacheVersion: WorkerToolActionCacheVersionResolver,
|
|
2060
|
-
private readonly abortSignal?: AbortSignal,
|
|
2061
|
-
private readonly onRequestsSettled?: (count: number) => void,
|
|
2062
|
-
private readonly callbacks?: WorkerCtxCallbacks,
|
|
2063
|
-
private readonly receiptStore?: WorkerRuntimeReceiptStore,
|
|
2064
|
-
private readonly allowLocalRetryReceipts = false,
|
|
2065
|
-
private readonly runtimeDeadlineMs?: number,
|
|
2066
|
-
// Map-wide (or ctx-wide for the root scheduler) persistence-failure circuit
|
|
2067
|
-
// breaker. Shared by every row worker so the first persistence failure
|
|
2068
|
-
// halts NEW provider dispatch across the whole map.
|
|
2069
|
-
private readonly latch: RuntimePersistenceLatch = createRuntimePersistenceLatch(),
|
|
2070
|
-
// Run-scoped buffer for orphaned billed results whose receipt could not be
|
|
2071
|
-
// completed even after the retry ladder.
|
|
2072
|
-
private readonly salvage?: ReceiptSalvageBuffer,
|
|
2073
|
-
) {}
|
|
2074
|
-
|
|
2075
|
-
/**
|
|
2076
|
-
* Run `completeReceipt`/`completeReceipts` with the salvage retry ladder.
|
|
2077
|
-
* Retries a receipt-completion failure past the server-side 5s connect window
|
|
2078
|
-
* so a transient blip does not orphan a billed result. Bounded by the run
|
|
2079
|
-
* deadline; skips retries when the latch has already burned a ladder in this
|
|
2080
|
-
* map so many in-flight orphans do not each serialize a long loop.
|
|
2081
|
-
*/
|
|
2082
|
-
private async completeReceiptWithRetryLadder<T>(
|
|
2083
|
-
run: () => Promise<T>,
|
|
2084
|
-
onGiveUp?: (attempts: number) => void,
|
|
2085
|
-
): Promise<T> {
|
|
2086
|
-
let attempt = 0;
|
|
2087
|
-
// A ladder that has already started retrying keeps its full budget even if
|
|
2088
|
-
// a peer trips the latch mid-flight — only FRESH entrants collapse once the
|
|
2089
|
-
// failure is confirmed. This is what guarantees the first ladder rides out
|
|
2090
|
-
// a genuine transient blip.
|
|
2091
|
-
let committedToFullLadder = false;
|
|
2092
|
-
const giveUp = (error: unknown): never => {
|
|
2093
|
-
this.latch.latchRetryBudgetExhausted = true;
|
|
2094
|
-
onGiveUp?.(attempt);
|
|
2095
|
-
throw error;
|
|
2096
|
-
};
|
|
2097
|
-
while (true) {
|
|
2098
|
-
try {
|
|
2099
|
-
return await run();
|
|
2100
|
-
} catch (error) {
|
|
2101
|
-
attempt += 1;
|
|
2102
|
-
if (
|
|
2103
|
-
shouldShortenCompleteReceiptLadder(this.latch, committedToFullLadder) ||
|
|
2104
|
-
attempt > COMPLETE_RECEIPT_RETRY_BACKOFFS_MS.length ||
|
|
2105
|
-
this.abortSignal?.aborted
|
|
2106
|
-
) {
|
|
2107
|
-
return giveUp(error);
|
|
2108
|
-
}
|
|
2109
|
-
const backoffMs = COMPLETE_RECEIPT_RETRY_BACKOFFS_MS[attempt - 1]!;
|
|
2110
|
-
// resolveRuntimeDeadlineRemainingMs THROWS WorkflowAbortError once the
|
|
2111
|
-
// run deadline has passed — the correct loud outcome (there is no
|
|
2112
|
-
// separate per-chunk step timeout). Route that throw through giveUp so
|
|
2113
|
-
// the billed result is still salvaged and the ladder budget flag is
|
|
2114
|
-
// set: a billed output must never lose its durable trace because the
|
|
2115
|
-
// clock ran out mid-ladder.
|
|
2116
|
-
let remainingMs: number;
|
|
2117
|
-
try {
|
|
2118
|
-
remainingMs = resolveRuntimeDeadlineRemainingMs(
|
|
2119
|
-
this.runtimeDeadlineMs,
|
|
2120
|
-
);
|
|
2121
|
-
} catch (deadlineError) {
|
|
2122
|
-
return giveUp(deadlineError);
|
|
2123
|
-
}
|
|
2124
|
-
if (remainingMs <= backoffMs + COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS) {
|
|
2125
|
-
return giveUp(error);
|
|
2126
|
-
}
|
|
2127
|
-
// Past every give-up gate: this ladder is now committed to the full
|
|
2128
|
-
// budget and must not be collapsed by a peer tripping the latch mid-sleep.
|
|
2129
|
-
committedToFullLadder = true;
|
|
2130
|
-
await sleepWorkerMs(backoffMs);
|
|
2131
|
-
}
|
|
2132
|
-
}
|
|
2133
|
-
}
|
|
2134
|
-
|
|
2135
|
-
/**
|
|
2136
|
-
* Record a billed-but-unpersisted tool output into the run-scoped salvage
|
|
2137
|
-
* buffer. Called only after the completeReceipt ladder is exhausted, with the
|
|
2138
|
-
* exact serialized payload a receipt would have stored.
|
|
2139
|
-
*/
|
|
2140
|
-
private recordReceiptSalvage(input: {
|
|
2141
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
2142
|
-
output: unknown;
|
|
2143
|
-
attempts: number;
|
|
2144
|
-
}): void {
|
|
2145
|
-
if (!this.salvage || !input.claimed.receiptKey) return;
|
|
2146
|
-
this.salvage.push({
|
|
2147
|
-
receiptKey: input.claimed.receiptKey,
|
|
2148
|
-
toolId: input.claimed.request.toolId,
|
|
2149
|
-
cacheKey: input.claimed.request.cacheKey,
|
|
2150
|
-
output: input.output,
|
|
2151
|
-
completeAttempts: input.attempts,
|
|
2152
|
-
firstErrorAt: nowMs(),
|
|
2153
|
-
});
|
|
2154
|
-
}
|
|
2155
|
-
|
|
2156
|
-
/**
|
|
2157
|
-
* Report a provider 429 / Retry-After back into the Governor's shared pacer
|
|
2158
|
-
* so future acquires for this (org, provider) bucket back off across all
|
|
2159
|
-
* isolates. Provider comes from the same pacing resolver the Governor uses
|
|
2160
|
-
* (the worker has no local catalog), so callers pass only the toolId.
|
|
2161
|
-
*/
|
|
2162
|
-
private reportBackpressure(toolId: string, retryAfterMs: number): void {
|
|
2163
|
-
if (retryAfterMs <= 0) return;
|
|
2164
|
-
void (async () => {
|
|
2165
|
-
const pacing = await this.resolvePacing(toolId).catch(() => null);
|
|
2166
|
-
if (pacing?.provider) {
|
|
2167
|
-
this.governor.reportProviderBackpressure({
|
|
2168
|
-
provider: pacing.provider,
|
|
2169
|
-
retryAfterMs,
|
|
2170
|
-
});
|
|
2171
|
-
}
|
|
2172
|
-
})();
|
|
2173
|
-
}
|
|
2174
|
-
|
|
2175
|
-
async execute(
|
|
2176
|
-
id: string,
|
|
2177
|
-
toolId: string,
|
|
2178
|
-
input: Record<string, unknown>,
|
|
2179
|
-
workflowStep?: WorkflowStep,
|
|
2180
|
-
options?: { force?: boolean; staleAfterSeconds?: number | null },
|
|
2181
|
-
runtimeOptions?: { timeoutMs?: number; receiptWaitMs?: number },
|
|
2182
|
-
): Promise<unknown> {
|
|
2183
|
-
const providerActionVersion =
|
|
2184
|
-
await this.resolveToolActionCacheVersion(toolId);
|
|
2185
|
-
return await new Promise((resolve, reject) => {
|
|
2186
|
-
const queuedAt = nowMs();
|
|
2187
|
-
const receiptKey = workerDurableToolCallCacheKey({
|
|
2188
|
-
req: this.req,
|
|
2189
|
-
toolId,
|
|
2190
|
-
requestInput: input,
|
|
2191
|
-
providerActionVersion,
|
|
2192
|
-
staleAfterSeconds: options?.staleAfterSeconds,
|
|
2193
|
-
});
|
|
2194
|
-
const queryResultDatasetRead = isQueryResultDatasetReadRequest(
|
|
2195
|
-
toolId,
|
|
2196
|
-
input,
|
|
2197
|
-
);
|
|
2198
|
-
const receiptEligible = !queryResultDatasetRead;
|
|
2199
|
-
this.queue.push({
|
|
2200
|
-
id,
|
|
2201
|
-
cacheKey: receiptEligible ? receiptKey : '',
|
|
2202
|
-
receiptKey: receiptEligible ? receiptKey : null,
|
|
2203
|
-
force: options?.force === true || !!this.req.force,
|
|
2204
|
-
receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
2205
|
-
typeof runtimeOptions?.receiptWaitMs === 'number'
|
|
2206
|
-
? { max_wait_ms: runtimeOptions.receiptWaitMs }
|
|
2207
|
-
: input,
|
|
2208
|
-
),
|
|
2209
|
-
...(typeof runtimeOptions?.timeoutMs === 'number'
|
|
2210
|
-
? { runtimeTimeoutMs: runtimeOptions.timeoutMs }
|
|
2211
|
-
: {}),
|
|
2212
|
-
toolId,
|
|
2213
|
-
input,
|
|
2214
|
-
workflowStep,
|
|
2215
|
-
...(queryResultDatasetRead
|
|
2216
|
-
? {
|
|
2217
|
-
directOptions: {
|
|
2218
|
-
customerDbDataset: {
|
|
2219
|
-
offset: 0,
|
|
2220
|
-
pageSize: QUERY_RESULT_DATASET_PAGE_SIZE,
|
|
2221
|
-
},
|
|
2222
|
-
},
|
|
2223
|
-
}
|
|
2224
|
-
: {}),
|
|
2225
|
-
resolve: (value) => {
|
|
2226
|
-
recordRunnerPerfTrace({
|
|
2227
|
-
req: this.req,
|
|
2228
|
-
phase: 'runner.tool.request',
|
|
2229
|
-
ms: nowMs() - queuedAt,
|
|
2230
|
-
extra: { id, toolId },
|
|
2231
|
-
});
|
|
2232
|
-
resolve(value);
|
|
2233
|
-
},
|
|
2234
|
-
reject,
|
|
2235
|
-
});
|
|
2236
|
-
this.scheduleDrain();
|
|
2237
|
-
});
|
|
2238
|
-
}
|
|
2239
|
-
|
|
2240
|
-
private scheduleDrain(): void {
|
|
2241
|
-
if (this.scheduled) return;
|
|
2242
|
-
this.scheduled = true;
|
|
2243
|
-
queueMicrotask(async () => {
|
|
2244
|
-
if (this.hasBatchableQueuedTool()) {
|
|
2245
|
-
await new Promise((resolve) =>
|
|
2246
|
-
setTimeout(resolve, WORKER_TOOL_BATCH_GRACE_MS),
|
|
2247
|
-
);
|
|
2248
|
-
}
|
|
2249
|
-
void this.drain();
|
|
2250
|
-
});
|
|
2251
|
-
}
|
|
2252
|
-
|
|
2253
|
-
private hasBatchableQueuedTool(): boolean {
|
|
2254
|
-
return this.queue.some(
|
|
2255
|
-
(request) =>
|
|
2256
|
-
request.toolId !== 'test_wait_for_event' &&
|
|
2257
|
-
getPlayRuntimeBatchStrategy(request.toolId) !== null,
|
|
2258
|
-
);
|
|
2259
|
-
}
|
|
2260
|
-
|
|
2261
|
-
private async drain(): Promise<void> {
|
|
2262
|
-
const requests = this.queue;
|
|
2263
|
-
this.queue = [];
|
|
2264
|
-
this.scheduled = false;
|
|
2265
|
-
const drainStartedAt = nowMs();
|
|
2266
|
-
await Promise.all(
|
|
2267
|
-
[...groupWorkerToolRequestsByTool(requests).entries()].map(
|
|
2268
|
-
async ([toolId, groupedRequests]) => {
|
|
2269
|
-
await this.executeToolGroup(toolId, groupedRequests);
|
|
2270
|
-
},
|
|
2271
|
-
),
|
|
2272
|
-
);
|
|
2273
|
-
recordRunnerPerfTrace({
|
|
2274
|
-
req: this.req,
|
|
2275
|
-
phase: 'runner.tool.drain',
|
|
2276
|
-
ms: nowMs() - drainStartedAt,
|
|
2277
|
-
extra: { requests: requests.length },
|
|
2278
|
-
});
|
|
2279
|
-
if (this.queue.length > 0) {
|
|
2280
|
-
this.scheduleDrain();
|
|
2281
|
-
}
|
|
2282
|
-
}
|
|
2283
|
-
|
|
2284
|
-
private async waitForDurableToolReceipt(input: {
|
|
2285
|
-
receiptKey: string;
|
|
2286
|
-
maxAttempts: number;
|
|
2287
|
-
}): Promise<WorkerRuntimeReceipt> {
|
|
2288
|
-
if (!this.receiptStore?.getReceipt) {
|
|
2289
|
-
throw new Error(
|
|
2290
|
-
'Worker durable tool receipt wait requires receipt lookup.',
|
|
2291
|
-
);
|
|
2292
|
-
}
|
|
2293
|
-
const receipt = await waitForCompletedRuntimeReceipt({
|
|
2294
|
-
receiptKey: input.receiptKey,
|
|
2295
|
-
maxAttempts: input.maxAttempts,
|
|
2296
|
-
abortSignal: this.abortSignal,
|
|
2297
|
-
store: {
|
|
2298
|
-
getMany: async (receiptKeys) => {
|
|
2299
|
-
const receipts = new Map<string, RuntimeStepReceipt>();
|
|
2300
|
-
await Promise.all(
|
|
2301
|
-
receiptKeys.map(async (key) => {
|
|
2302
|
-
const receipt = await this.receiptStore!.getReceipt!({
|
|
2303
|
-
playName: this.req.playName,
|
|
2304
|
-
key,
|
|
2305
|
-
});
|
|
2306
|
-
if (!receipt) return;
|
|
2307
|
-
receipts.set(key, {
|
|
2308
|
-
key: receipt.key,
|
|
2309
|
-
status: receipt.status,
|
|
2310
|
-
output: receipt.output,
|
|
2311
|
-
error: receipt.error ?? undefined,
|
|
2312
|
-
runId: receipt.runId ?? null,
|
|
2313
|
-
});
|
|
2314
|
-
}),
|
|
2315
|
-
);
|
|
2316
|
-
return receipts;
|
|
2317
|
-
},
|
|
2318
|
-
},
|
|
2319
|
-
});
|
|
2320
|
-
return receipt;
|
|
2321
|
-
}
|
|
2322
|
-
|
|
2323
|
-
private settleRequests(
|
|
2324
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2325
|
-
result: unknown,
|
|
2326
|
-
): void {
|
|
2327
|
-
claimed.request.resolve(result);
|
|
2328
|
-
for (const follower of claimed.followers) {
|
|
2329
|
-
follower.resolve(
|
|
2330
|
-
claimed.receiptKey
|
|
2331
|
-
? markWorkerToolReceiptResultExecution(result, {
|
|
2332
|
-
kind: 'in_flight',
|
|
2333
|
-
receiptKey: claimed.receiptKey,
|
|
2334
|
-
attachedToReceiptKey: claimed.receiptKey,
|
|
2335
|
-
})
|
|
2336
|
-
: result,
|
|
2337
|
-
);
|
|
2338
|
-
}
|
|
2339
|
-
this.onRequestsSettled?.(1 + claimed.followers.length);
|
|
2340
|
-
}
|
|
2341
|
-
|
|
2342
|
-
private rejectRequests(
|
|
2343
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2344
|
-
error: unknown,
|
|
2345
|
-
): void {
|
|
2346
|
-
claimed.request.reject(error);
|
|
2347
|
-
for (const follower of claimed.followers) {
|
|
2348
|
-
follower.reject(error);
|
|
2349
|
-
}
|
|
2350
|
-
this.onRequestsSettled?.(1 + claimed.followers.length);
|
|
2351
|
-
}
|
|
2352
|
-
|
|
2353
|
-
private rejectRawRequests(
|
|
2354
|
-
requests: WorkerToolBatchRequest[],
|
|
2355
|
-
error: unknown,
|
|
2356
|
-
): void {
|
|
2357
|
-
for (const request of requests) {
|
|
2358
|
-
request.reject(error);
|
|
2359
|
-
}
|
|
2360
|
-
this.onRequestsSettled?.(requests.length);
|
|
2361
|
-
}
|
|
2362
|
-
|
|
2363
|
-
private async resolveCompletedDurableToolReceiptGroup(input: {
|
|
2364
|
-
group: WorkerToolBatchRequest[];
|
|
2365
|
-
receiptKey: string;
|
|
2366
|
-
receipt: WorkerRuntimeReceipt;
|
|
2367
|
-
source: 'cache' | 'in_flight';
|
|
2368
|
-
}): Promise<void> {
|
|
2369
|
-
const [first] = input.group;
|
|
2370
|
-
if (!first) return;
|
|
2371
|
-
const value = deserializeDurableStepValue(input.receipt.output);
|
|
2372
|
-
const result =
|
|
2373
|
-
input.source === 'cache'
|
|
2374
|
-
? markWorkerToolReceiptResultCached(
|
|
2375
|
-
value,
|
|
2376
|
-
first.cacheKey,
|
|
2377
|
-
input.receiptKey,
|
|
2378
|
-
)
|
|
2379
|
-
: markWorkerToolReceiptResultExecution(value, {
|
|
2380
|
-
kind: 'in_flight',
|
|
2381
|
-
receiptKey: input.receiptKey,
|
|
2382
|
-
attachedToReceiptKey: input.receiptKey,
|
|
2383
|
-
});
|
|
2384
|
-
for (const request of input.group) {
|
|
2385
|
-
request.resolve(result);
|
|
2386
|
-
}
|
|
2387
|
-
this.onRequestsSettled?.(input.group.length);
|
|
2388
|
-
}
|
|
2389
|
-
|
|
2390
|
-
private async reclaimRunningDurableToolReceiptGroupAfterWait(input: {
|
|
2391
|
-
group: WorkerToolBatchRequest[];
|
|
2392
|
-
receiptKey: string;
|
|
2393
|
-
runningReceipt: WorkerRuntimeReceipt;
|
|
2394
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
2395
|
-
const [request, ...followers] = input.group;
|
|
2396
|
-
if (!request || !this.receiptStore) {
|
|
2397
|
-
this.rejectRawRequests(
|
|
2398
|
-
input.group,
|
|
2399
|
-
new RuntimeReceiptWaitTimeoutError(input.receiptKey),
|
|
2400
|
-
);
|
|
2401
|
-
return [];
|
|
2402
|
-
}
|
|
2403
|
-
// A `running` receipt left by a DIFFERENT run whose last write is stale will
|
|
2404
|
-
// never be completed by a live worker. Polling it burns one service-binding
|
|
2405
|
-
// subrequest per tick, up to the receipt's wait budget (~1320) — many such
|
|
2406
|
-
// orphans on a resume exhaust Cloudflare's per-invocation subrequest budget
|
|
2407
|
-
// and kill the run. The orchestrator reclaims those immediately (skipping the
|
|
2408
|
-
// poll) and re-executes, matching the "on resume it reclaims and re-executes"
|
|
2409
|
-
// contract; live/fresh/same-run owners still poll before reclaiming. Kept as
|
|
2410
|
-
// an injected helper so the skip-poll decision is unit-testable without
|
|
2411
|
-
// loading this cloudflare:workers Worker entrypoint.
|
|
2412
|
-
return await reclaimRunningReceiptGroupAfterWait<ClaimedWorkerToolBatchRequest>(
|
|
2413
|
-
{
|
|
2414
|
-
ownership: {
|
|
2415
|
-
ownerRunId: input.runningReceipt.runId,
|
|
2416
|
-
currentRunId: this.req.runId,
|
|
2417
|
-
updatedAt: input.runningReceipt.updatedAt,
|
|
2418
|
-
},
|
|
2419
|
-
timeoutError: new RuntimeReceiptWaitTimeoutError(input.receiptKey),
|
|
2420
|
-
poll: async () => {
|
|
2421
|
-
try {
|
|
2422
|
-
const receipt = await this.waitForDurableToolReceipt({
|
|
2423
|
-
receiptKey: input.receiptKey,
|
|
2424
|
-
maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
|
|
2425
|
-
input.group,
|
|
2426
|
-
(groupRequest) => groupRequest.receiptWaitMaxAttempts,
|
|
2427
|
-
),
|
|
2428
|
-
});
|
|
2429
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2430
|
-
group: input.group,
|
|
2431
|
-
receiptKey: input.receiptKey,
|
|
2432
|
-
receipt,
|
|
2433
|
-
source: 'in_flight',
|
|
2434
|
-
});
|
|
2435
|
-
return { kind: 'completed' };
|
|
2436
|
-
} catch (error) {
|
|
2437
|
-
if (error instanceof RuntimeReceiptWaitTimeoutError) {
|
|
2438
|
-
return { kind: 'timed_out', error };
|
|
2439
|
-
}
|
|
2440
|
-
return { kind: 'errored', error };
|
|
2441
|
-
}
|
|
2442
|
-
},
|
|
2443
|
-
reclaim: (rejectionOnFallthrough) =>
|
|
2444
|
-
this.forceReclaimRunningDurableToolReceiptGroup({
|
|
2445
|
-
group: input.group,
|
|
2446
|
-
request,
|
|
2447
|
-
followers,
|
|
2448
|
-
receiptKey: input.receiptKey,
|
|
2449
|
-
rejectionOnFallthrough,
|
|
2450
|
-
}),
|
|
2451
|
-
reject: (error) => this.rejectRawRequests(input.group, error),
|
|
2452
|
-
},
|
|
2453
|
-
);
|
|
2454
|
-
}
|
|
2455
|
-
|
|
2456
|
-
/**
|
|
2457
|
-
* Steal a `running` receipt (reclaimRunning) and route by claim disposition.
|
|
2458
|
-
* Shared by the post-poll-timeout reclaim and the dead-owner fast path, so
|
|
2459
|
-
* both dispose of a reclaimed/reused/failed receipt identically.
|
|
2460
|
-
*/
|
|
2461
|
-
private async forceReclaimRunningDurableToolReceiptGroup(input: {
|
|
2462
|
-
group: WorkerToolBatchRequest[];
|
|
2463
|
-
request: WorkerToolBatchRequest;
|
|
2464
|
-
followers: WorkerToolBatchRequest[];
|
|
2465
|
-
receiptKey: string;
|
|
2466
|
-
rejectionOnFallthrough: unknown;
|
|
2467
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
2468
|
-
if (!this.receiptStore) {
|
|
2469
|
-
this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
|
|
2470
|
-
return [];
|
|
2471
|
-
}
|
|
2472
|
-
let claim: WorkerRuntimeReceiptClaim;
|
|
2473
|
-
try {
|
|
2474
|
-
claim = await this.receiptStore.claimReceipt({
|
|
2475
|
-
playName: this.req.playName,
|
|
2476
|
-
runId: this.req.runId,
|
|
2477
|
-
key: input.receiptKey,
|
|
2478
|
-
reclaimRunning: true,
|
|
2479
|
-
});
|
|
2480
|
-
} catch (error) {
|
|
2481
|
-
this.rejectRawRequests(input.group, error);
|
|
2482
|
-
return [];
|
|
2483
|
-
}
|
|
2484
|
-
if (claim.disposition === 'claimed') {
|
|
2485
|
-
return [
|
|
2486
|
-
{
|
|
2487
|
-
request: input.request,
|
|
2488
|
-
receiptKey: input.receiptKey,
|
|
2489
|
-
followers: input.followers,
|
|
2490
|
-
},
|
|
2491
|
-
];
|
|
2492
|
-
}
|
|
2493
|
-
if (claim.disposition === 'reused') {
|
|
2494
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2495
|
-
group: input.group,
|
|
2496
|
-
receiptKey: input.receiptKey,
|
|
2497
|
-
receipt: claim.receipt,
|
|
2498
|
-
source: 'cache',
|
|
2499
|
-
});
|
|
2500
|
-
return [];
|
|
2501
|
-
}
|
|
2502
|
-
if (claim.disposition === 'failed') {
|
|
2503
|
-
return await this.reclaimFailedDurableToolReceiptGroup({
|
|
2504
|
-
group: input.group,
|
|
2505
|
-
receiptKey: input.receiptKey,
|
|
2506
|
-
failedReceipt: claim.receipt,
|
|
2507
|
-
});
|
|
2508
|
-
}
|
|
2509
|
-
this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
|
|
2510
|
-
return [];
|
|
2511
|
-
}
|
|
2512
|
-
|
|
2513
|
-
private async reclaimFailedDurableToolReceiptGroup(input: {
|
|
2514
|
-
group: WorkerToolBatchRequest[];
|
|
2515
|
-
receiptKey: string;
|
|
2516
|
-
failedReceipt: WorkerRuntimeReceipt;
|
|
2517
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
2518
|
-
const failedError = new Error(
|
|
2519
|
-
`Durable tool call ${input.receiptKey} failed: ${input.failedReceipt.error ?? 'unknown error'}`,
|
|
2520
|
-
);
|
|
2521
|
-
const [request, ...followers] = input.group;
|
|
2522
|
-
if (!request || !this.receiptStore) {
|
|
2523
|
-
this.rejectRawRequests(input.group, failedError);
|
|
2524
|
-
return [];
|
|
2525
|
-
}
|
|
2526
|
-
if (
|
|
2527
|
-
!canReclaimFailedWorkerToolReceipt({
|
|
2528
|
-
error: input.failedReceipt.error,
|
|
2529
|
-
})
|
|
2530
|
-
) {
|
|
2531
|
-
this.rejectRawRequests(input.group, failedError);
|
|
2532
|
-
return [];
|
|
2533
|
-
}
|
|
2534
|
-
let claim: WorkerRuntimeReceiptClaim;
|
|
2535
|
-
try {
|
|
2536
|
-
claim = await this.receiptStore.claimReceipt({
|
|
2537
|
-
playName: this.req.playName,
|
|
2538
|
-
runId: this.req.runId,
|
|
2539
|
-
key: input.receiptKey,
|
|
2540
|
-
});
|
|
2541
|
-
} catch (error) {
|
|
2542
|
-
this.rejectRawRequests(input.group, error);
|
|
2543
|
-
return [];
|
|
2544
|
-
}
|
|
2545
|
-
if (claim.disposition === 'claimed') {
|
|
2546
|
-
return [{ request, receiptKey: input.receiptKey, followers }];
|
|
2547
|
-
}
|
|
2548
|
-
if (claim.disposition === 'reused') {
|
|
2549
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2550
|
-
group: input.group,
|
|
2551
|
-
receiptKey: input.receiptKey,
|
|
2552
|
-
receipt: claim.receipt,
|
|
2553
|
-
source: 'cache',
|
|
2554
|
-
});
|
|
2555
|
-
return [];
|
|
2556
|
-
}
|
|
2557
|
-
if (claim.disposition === 'running') {
|
|
2558
|
-
try {
|
|
2559
|
-
const receipt = await this.waitForDurableToolReceipt({
|
|
2560
|
-
receiptKey: input.receiptKey,
|
|
2561
|
-
maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
|
|
2562
|
-
input.group,
|
|
2563
|
-
(groupRequest) => groupRequest.receiptWaitMaxAttempts,
|
|
2564
|
-
),
|
|
2565
|
-
});
|
|
2566
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2567
|
-
group: input.group,
|
|
2568
|
-
receiptKey: input.receiptKey,
|
|
2569
|
-
receipt,
|
|
2570
|
-
source: 'in_flight',
|
|
2571
|
-
});
|
|
2572
|
-
} catch (error) {
|
|
2573
|
-
this.rejectRawRequests(input.group, error);
|
|
2574
|
-
}
|
|
2575
|
-
return [];
|
|
2576
|
-
}
|
|
2577
|
-
this.rejectRawRequests(
|
|
2578
|
-
input.group,
|
|
2579
|
-
new Error(
|
|
2580
|
-
`Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
|
|
2581
|
-
),
|
|
2582
|
-
);
|
|
2583
|
-
return [];
|
|
2584
|
-
}
|
|
2585
|
-
|
|
2586
|
-
private async failureForRejectedToolRequest(
|
|
2587
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2588
|
-
error: unknown,
|
|
2589
|
-
): Promise<unknown> {
|
|
2590
|
-
try {
|
|
2591
|
-
await this.failDurableToolRequest(claimed, error);
|
|
2592
|
-
return error;
|
|
2593
|
-
} catch (receiptError) {
|
|
2594
|
-
tripRuntimePersistenceLatch(this.latch, receiptError);
|
|
2595
|
-
return new RuntimeReceiptPersistenceError(
|
|
2596
|
-
'Tool call failed and durable receipt could not be marked failed',
|
|
2597
|
-
[error, receiptError],
|
|
2598
|
-
);
|
|
2599
|
-
}
|
|
2600
|
-
}
|
|
2601
|
-
|
|
2602
|
-
private async claimDurableToolReceiptGroups(
|
|
2603
|
-
groups: ReturnType<
|
|
2604
|
-
typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
|
|
2605
|
-
>['durableGroups'],
|
|
2606
|
-
): Promise<
|
|
2607
|
-
Array<{
|
|
2608
|
-
group: ReturnType<
|
|
2609
|
-
typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
|
|
2610
|
-
>['durableGroups'][number];
|
|
2611
|
-
receiptKey: string;
|
|
2612
|
-
claim: WorkerRuntimeReceiptClaim;
|
|
2613
|
-
}>
|
|
2614
|
-
> {
|
|
2615
|
-
if (!this.receiptStore) return [];
|
|
2616
|
-
const planned = groups.map((group) => ({
|
|
2617
|
-
group,
|
|
2618
|
-
receiptKey: workerRuntimeReceiptKey({
|
|
2619
|
-
req: this.req,
|
|
2620
|
-
key: group.claimableReceiptKey,
|
|
2621
|
-
}),
|
|
2622
|
-
}));
|
|
2623
|
-
const claimOne = async (entry: (typeof planned)[number]) => ({
|
|
2624
|
-
...entry,
|
|
2625
|
-
claim: await this.receiptStore!.claimReceipt({
|
|
2626
|
-
playName: this.req.playName,
|
|
2627
|
-
runId: this.req.runId,
|
|
2628
|
-
key: entry.receiptKey,
|
|
2629
|
-
...(entry.group.forceDurableRefresh
|
|
2630
|
-
? { forceRefresh: true, reclaimRunning: true }
|
|
2631
|
-
: {}),
|
|
2632
|
-
}),
|
|
2633
|
-
});
|
|
2634
|
-
if (!this.receiptStore.claimReceipts) {
|
|
2635
|
-
return await Promise.all(planned.map(claimOne));
|
|
2636
|
-
}
|
|
2637
|
-
const claimed: Array<{
|
|
2638
|
-
group: (typeof planned)[number]['group'];
|
|
2639
|
-
receiptKey: string;
|
|
2640
|
-
claim: WorkerRuntimeReceiptClaim;
|
|
2641
|
-
}> = [];
|
|
2642
|
-
for (const forceDurableRefresh of [false, true]) {
|
|
2643
|
-
const entries = planned.filter(
|
|
2644
|
-
(entry) => entry.group.forceDurableRefresh === forceDurableRefresh,
|
|
2645
|
-
);
|
|
2646
|
-
if (entries.length === 0) continue;
|
|
2647
|
-
const claims = await this.receiptStore.claimReceipts({
|
|
2648
|
-
playName: this.req.playName,
|
|
2649
|
-
runId: this.req.runId,
|
|
2650
|
-
keys: entries.map((entry) => entry.receiptKey),
|
|
2651
|
-
...(forceDurableRefresh
|
|
2652
|
-
? { forceRefresh: true, reclaimRunning: true }
|
|
2653
|
-
: {}),
|
|
2654
|
-
});
|
|
2655
|
-
entries.forEach((entry, index) => {
|
|
2656
|
-
const claim = claims[index];
|
|
2657
|
-
if (!claim) {
|
|
2658
|
-
throw new Error(
|
|
2659
|
-
`Runtime receipt batch claim did not return receipt ${entry.receiptKey}.`,
|
|
2660
|
-
);
|
|
2661
|
-
}
|
|
2662
|
-
claimed.push({ ...entry, claim });
|
|
2663
|
-
});
|
|
2664
|
-
}
|
|
2665
|
-
return claimed;
|
|
2666
|
-
}
|
|
2667
|
-
|
|
2668
|
-
private async prepareDurableToolRequests(
|
|
2669
|
-
requests: WorkerToolBatchRequest[],
|
|
2670
|
-
): Promise<PreparedWorkerToolBatchRequests> {
|
|
2671
|
-
if (!this.receiptStore) {
|
|
2672
|
-
return {
|
|
2673
|
-
claimedRequests: requests.map((request) => ({
|
|
2674
|
-
request,
|
|
2675
|
-
receiptKey: null,
|
|
2676
|
-
followers: [],
|
|
2677
|
-
})),
|
|
2678
|
-
deferredClaimedRequests: [],
|
|
2679
|
-
};
|
|
2680
|
-
}
|
|
2681
|
-
|
|
2682
|
-
const claimedRequests: ClaimedWorkerToolBatchRequest[] = [];
|
|
2683
|
-
const deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[] =
|
|
2684
|
-
[];
|
|
2685
|
-
const receiptGroupPlan = planWorkerToolReceiptGroups(requests, {
|
|
2686
|
-
allowLocalRetryReceipts: this.allowLocalRetryReceipts,
|
|
2687
|
-
getReceiptInput: (request) => ({
|
|
2688
|
-
durableReceiptKey: request.receiptKey,
|
|
2689
|
-
localRetryCacheKey: request.cacheKey,
|
|
2690
|
-
force: request.force,
|
|
2691
|
-
}),
|
|
2692
|
-
});
|
|
2693
|
-
for (const request of receiptGroupPlan.localRequests) {
|
|
2694
|
-
claimedRequests.push({
|
|
2695
|
-
request,
|
|
2696
|
-
receiptKey: null,
|
|
2697
|
-
followers: [],
|
|
2698
|
-
});
|
|
2699
|
-
}
|
|
2700
|
-
|
|
2701
|
-
try {
|
|
2702
|
-
const claimEntries = await this.claimDurableToolReceiptGroups(
|
|
2703
|
-
receiptGroupPlan.durableGroups,
|
|
2704
|
-
);
|
|
2705
|
-
for (const { group: groupState, receiptKey, claim } of claimEntries) {
|
|
2706
|
-
const group = groupState.requests;
|
|
2707
|
-
const [first] = group;
|
|
2708
|
-
if (!first) continue;
|
|
2709
|
-
if (claim.disposition === 'reused') {
|
|
2710
|
-
const result = markWorkerToolReceiptResultCached(
|
|
2711
|
-
deserializeDurableStepValue(claim.receipt.output),
|
|
2712
|
-
first.cacheKey,
|
|
2713
|
-
receiptKey,
|
|
2714
|
-
);
|
|
2715
|
-
for (const request of group) {
|
|
2716
|
-
request.resolve(result);
|
|
2717
|
-
}
|
|
2718
|
-
this.onRequestsSettled?.(group.length);
|
|
2719
|
-
continue;
|
|
2720
|
-
}
|
|
2721
|
-
if (claim.disposition === 'failed') {
|
|
2722
|
-
claimedRequests.push(
|
|
2723
|
-
...(await this.reclaimFailedDurableToolReceiptGroup({
|
|
2724
|
-
group,
|
|
2725
|
-
receiptKey,
|
|
2726
|
-
failedReceipt: claim.receipt,
|
|
2727
|
-
})),
|
|
2728
|
-
);
|
|
2729
|
-
continue;
|
|
2730
|
-
}
|
|
2731
|
-
if (claim.disposition === 'running') {
|
|
2732
|
-
deferredClaimedRequests.push(
|
|
2733
|
-
this.reclaimRunningDurableToolReceiptGroupAfterWait({
|
|
2734
|
-
group,
|
|
2735
|
-
receiptKey,
|
|
2736
|
-
runningReceipt: claim.receipt,
|
|
2737
|
-
}),
|
|
2738
|
-
);
|
|
2739
|
-
continue;
|
|
2740
|
-
}
|
|
2741
|
-
const [request, ...followers] = group;
|
|
2742
|
-
if (!request) continue;
|
|
2743
|
-
claimedRequests.push({ request, receiptKey, followers });
|
|
2744
|
-
}
|
|
2745
|
-
} catch (error) {
|
|
2746
|
-
// A receipt claim failure means the tenant Postgres is already
|
|
2747
|
-
// unwritable: trip the breaker so remaining groups/rows stop dispatching.
|
|
2748
|
-
tripRuntimePersistenceLatch(this.latch, error);
|
|
2749
|
-
for (const group of receiptGroupPlan.durableGroups) {
|
|
2750
|
-
this.rejectRawRequests(group.requests, error);
|
|
2751
|
-
}
|
|
2752
|
-
}
|
|
2753
|
-
return { claimedRequests, deferredClaimedRequests };
|
|
2754
|
-
}
|
|
2755
|
-
|
|
2756
|
-
private async completeDurableToolRequest(
|
|
2757
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2758
|
-
result: unknown,
|
|
2759
|
-
): Promise<unknown> {
|
|
2760
|
-
if (!this.receiptStore || !claimed.receiptKey) {
|
|
2761
|
-
return result;
|
|
2762
|
-
}
|
|
2763
|
-
const ownerResult = markWorkerToolReceiptResultExecution(result, {
|
|
2764
|
-
kind: 'live',
|
|
2765
|
-
receiptKey: claimed.receiptKey,
|
|
2766
|
-
});
|
|
2767
|
-
const serializedOutput = serializeDurableStepValue(ownerResult);
|
|
2768
|
-
const completed = await this.completeReceiptWithRetryLadder(
|
|
2769
|
-
() =>
|
|
2770
|
-
this.receiptStore!.completeReceipt({
|
|
2771
|
-
playName: this.req.playName,
|
|
2772
|
-
runId: this.req.runId,
|
|
2773
|
-
key: claimed.receiptKey!,
|
|
2774
|
-
output: serializedOutput,
|
|
2775
|
-
}),
|
|
2776
|
-
// Ladder exhausted: the call billed but the receipt could not land.
|
|
2777
|
-
// Salvage the exact serialized output onto the run record before the
|
|
2778
|
-
// caller turns this into a RuntimeReceiptPersistenceError.
|
|
2779
|
-
(attempts) =>
|
|
2780
|
-
this.recordReceiptSalvage({
|
|
2781
|
-
claimed,
|
|
2782
|
-
output: serializedOutput,
|
|
2783
|
-
attempts,
|
|
2784
|
-
}),
|
|
2785
|
-
);
|
|
2786
|
-
if (
|
|
2787
|
-
completed &&
|
|
2788
|
-
(completed.status === 'completed' || completed.status === 'skipped') &&
|
|
2789
|
-
completed.output !== undefined
|
|
2790
|
-
) {
|
|
2791
|
-
const recovered = deserializeDurableStepValue(completed.output);
|
|
2792
|
-
return completed.runId && completed.runId !== this.req.runId
|
|
2793
|
-
? markWorkerToolReceiptResultCached(
|
|
2794
|
-
recovered,
|
|
2795
|
-
claimed.request.cacheKey,
|
|
2796
|
-
claimed.receiptKey,
|
|
2797
|
-
)
|
|
2798
|
-
: recovered;
|
|
2799
|
-
}
|
|
2800
|
-
return ownerResult;
|
|
2801
|
-
}
|
|
2802
|
-
|
|
2803
|
-
private completedDurableToolRequestResult(input: {
|
|
2804
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
2805
|
-
ownerResult: unknown;
|
|
2806
|
-
completed: WorkerRuntimeReceipt | null | undefined;
|
|
2807
|
-
}): unknown {
|
|
2808
|
-
const { claimed, completed, ownerResult } = input;
|
|
2809
|
-
if (
|
|
2810
|
-
completed &&
|
|
2811
|
-
(completed.status === 'completed' || completed.status === 'skipped') &&
|
|
2812
|
-
completed.output !== undefined
|
|
2813
|
-
) {
|
|
2814
|
-
const recovered = deserializeDurableStepValue(completed.output);
|
|
2815
|
-
return completed.runId && completed.runId !== this.req.runId
|
|
2816
|
-
? markWorkerToolReceiptResultCached(
|
|
2817
|
-
recovered,
|
|
2818
|
-
claimed.request.cacheKey,
|
|
2819
|
-
claimed.receiptKey ?? claimed.request.cacheKey,
|
|
2820
|
-
)
|
|
2821
|
-
: recovered;
|
|
2822
|
-
}
|
|
2823
|
-
return ownerResult;
|
|
2824
|
-
}
|
|
2825
|
-
|
|
2826
|
-
private async completeDurableToolRequests(
|
|
2827
|
-
entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
|
|
2828
|
-
): Promise<unknown[]> {
|
|
2829
|
-
const results: unknown[] = new Array(entries.length);
|
|
2830
|
-
const durableEntries: Array<{
|
|
2831
|
-
index: number;
|
|
2832
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
2833
|
-
ownerResult: unknown;
|
|
2834
|
-
}> = [];
|
|
2835
|
-
for (let index = 0; index < entries.length; index += 1) {
|
|
2836
|
-
const entry = entries[index]!;
|
|
2837
|
-
if (!this.receiptStore || !entry.claimed.receiptKey) {
|
|
2838
|
-
results[index] = entry.result;
|
|
2839
|
-
continue;
|
|
2840
|
-
}
|
|
2841
|
-
const ownerResult = markWorkerToolReceiptResultExecution(entry.result, {
|
|
2842
|
-
kind: 'live',
|
|
2843
|
-
receiptKey: entry.claimed.receiptKey,
|
|
2844
|
-
});
|
|
2845
|
-
durableEntries.push({ index, claimed: entry.claimed, ownerResult });
|
|
2846
|
-
}
|
|
2847
|
-
if (durableEntries.length === 0) {
|
|
2848
|
-
return results;
|
|
2849
|
-
}
|
|
2850
|
-
if (!this.receiptStore?.completeReceipts) {
|
|
2851
|
-
await Promise.all(
|
|
2852
|
-
durableEntries.map(async (entry) => {
|
|
2853
|
-
results[entry.index] = await this.completeDurableToolRequest(
|
|
2854
|
-
entry.claimed,
|
|
2855
|
-
entries[entry.index]!.result,
|
|
2856
|
-
);
|
|
2857
|
-
}),
|
|
2858
|
-
);
|
|
2859
|
-
return results;
|
|
2860
|
-
}
|
|
2861
|
-
const serializedByEntry = durableEntries.map((entry) =>
|
|
2862
|
-
serializeDurableStepValue(entry.ownerResult),
|
|
2863
|
-
);
|
|
2864
|
-
const completed = await this.completeReceiptWithRetryLadder(
|
|
2865
|
-
() =>
|
|
2866
|
-
this.receiptStore!.completeReceipts!({
|
|
2867
|
-
playName: this.req.playName,
|
|
2868
|
-
receipts: durableEntries.map((entry, entryIndex) => ({
|
|
2869
|
-
runId: this.req.runId,
|
|
2870
|
-
key: entry.claimed.receiptKey!,
|
|
2871
|
-
output: serializedByEntry[entryIndex],
|
|
2872
|
-
})),
|
|
2873
|
-
}),
|
|
2874
|
-
// Ladder exhausted: every entry in this batch billed but could not be
|
|
2875
|
-
// marked completed. Salvage each serialized output before the caller
|
|
2876
|
-
// turns this into a RuntimeReceiptPersistenceError.
|
|
2877
|
-
(attempts) => {
|
|
2878
|
-
durableEntries.forEach((entry, entryIndex) => {
|
|
2879
|
-
this.recordReceiptSalvage({
|
|
2880
|
-
claimed: entry.claimed,
|
|
2881
|
-
output: serializedByEntry[entryIndex],
|
|
2882
|
-
attempts,
|
|
2883
|
-
});
|
|
2884
|
-
});
|
|
2885
|
-
},
|
|
2886
|
-
);
|
|
2887
|
-
durableEntries.forEach((entry, resultIndex) => {
|
|
2888
|
-
results[entry.index] = this.completedDurableToolRequestResult({
|
|
2889
|
-
claimed: entry.claimed,
|
|
2890
|
-
ownerResult: entry.ownerResult,
|
|
2891
|
-
completed: completed[resultIndex],
|
|
2892
|
-
});
|
|
2893
|
-
});
|
|
2894
|
-
return results;
|
|
2895
|
-
}
|
|
2896
|
-
|
|
2897
|
-
private async failDurableToolRequest(
|
|
2898
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2899
|
-
error: unknown,
|
|
2900
|
-
): Promise<void> {
|
|
2901
|
-
if (!this.receiptStore || !claimed.receiptKey) {
|
|
2902
|
-
return;
|
|
2903
|
-
}
|
|
2904
|
-
await this.receiptStore.failReceipt({
|
|
2905
|
-
playName: this.req.playName,
|
|
2906
|
-
runId: this.req.runId,
|
|
2907
|
-
key: claimed.receiptKey,
|
|
2908
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2909
|
-
});
|
|
2910
|
-
}
|
|
2911
|
-
|
|
2912
|
-
private async failDurableToolRequests(
|
|
2913
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
2914
|
-
error: unknown,
|
|
2915
|
-
): Promise<void> {
|
|
2916
|
-
const durable = claimedRequests.filter(
|
|
2917
|
-
(claimed) => this.receiptStore && claimed.receiptKey,
|
|
2918
|
-
);
|
|
2919
|
-
if (durable.length === 0) return;
|
|
2920
|
-
if (!this.receiptStore?.failReceipts) {
|
|
2921
|
-
await Promise.all(
|
|
2922
|
-
durable.map((claimed) => this.failDurableToolRequest(claimed, error)),
|
|
2923
|
-
);
|
|
2924
|
-
return;
|
|
2925
|
-
}
|
|
2926
|
-
await this.receiptStore.failReceipts({
|
|
2927
|
-
playName: this.req.playName,
|
|
2928
|
-
receipts: durable.map((claimed) => ({
|
|
2929
|
-
runId: this.req.runId,
|
|
2930
|
-
key: claimed.receiptKey!,
|
|
2931
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2932
|
-
})),
|
|
2933
|
-
});
|
|
2934
|
-
}
|
|
2935
|
-
|
|
2936
|
-
private async executeToolGroup(
|
|
2937
|
-
toolId: string,
|
|
2938
|
-
requests: WorkerToolBatchRequest[],
|
|
2939
|
-
): Promise<void> {
|
|
2940
|
-
// Circuit breaker, gated BEFORE the receipt claim: never-dispatched rows
|
|
2941
|
-
// get NO receipt (not even `running`), so a re-run claims them fresh and
|
|
2942
|
-
// executes them exactly once.
|
|
2943
|
-
if (this.latch.tripped) {
|
|
2944
|
-
this.latch.preventedCallCount += requests.length;
|
|
2945
|
-
this.rejectRawRequests(
|
|
2946
|
-
requests,
|
|
2947
|
-
new RuntimePersistenceCircuitOpenError(this.latch),
|
|
2948
|
-
);
|
|
2949
|
-
return;
|
|
2950
|
-
}
|
|
2951
|
-
const { claimedRequests, deferredClaimedRequests } =
|
|
2952
|
-
await this.prepareDurableToolRequests(requests);
|
|
2953
|
-
await this.executeClaimedToolRequests(
|
|
2954
|
-
toolId,
|
|
2955
|
-
requests.length,
|
|
2956
|
-
claimedRequests,
|
|
2957
|
-
);
|
|
2958
|
-
if (deferredClaimedRequests.length === 0) {
|
|
2959
|
-
return;
|
|
2960
|
-
}
|
|
2961
|
-
const reclaimedRequests = (
|
|
2962
|
-
await Promise.all(deferredClaimedRequests)
|
|
2963
|
-
).flat();
|
|
2964
|
-
await this.executeClaimedToolRequests(
|
|
2965
|
-
toolId,
|
|
2966
|
-
requests.length,
|
|
2967
|
-
reclaimedRequests,
|
|
2968
|
-
);
|
|
2969
|
-
}
|
|
2970
|
-
|
|
2971
|
-
private async executeClaimedToolRequests(
|
|
2972
|
-
toolId: string,
|
|
2973
|
-
requestCount: number,
|
|
2974
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
2975
|
-
): Promise<void> {
|
|
2976
|
-
if (claimedRequests.length === 0) {
|
|
2977
|
-
return;
|
|
2978
|
-
}
|
|
2979
|
-
const strategy = getPlayRuntimeBatchStrategy(toolId);
|
|
2980
|
-
if (
|
|
2981
|
-
!strategy ||
|
|
2982
|
-
toolId === 'test_wait_for_event' ||
|
|
2983
|
-
claimedRequests.length < 2
|
|
2984
|
-
) {
|
|
2985
|
-
const groupStartedAt = nowMs();
|
|
2986
|
-
const dispatchOne = async (
|
|
2987
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2988
|
-
): Promise<void> => {
|
|
2989
|
-
const { request } = claimed;
|
|
2990
|
-
// Circuit breaker: the latch may have tripped after this group was
|
|
2991
|
-
// claimed but before this particular call started. Do NOT call the
|
|
2992
|
-
// provider. This row has a `running` receipt (claimed, never
|
|
2993
|
-
// executed); on resume it reclaims and re-executes — safe, it never
|
|
2994
|
-
// billed.
|
|
2995
|
-
if (this.latch.tripped) {
|
|
2996
|
-
this.latch.preventedCallCount += 1 + claimed.followers.length;
|
|
2997
|
-
this.rejectRequests(
|
|
2998
|
-
claimed,
|
|
2999
|
-
new RuntimePersistenceCircuitOpenError(this.latch),
|
|
3000
|
-
);
|
|
3001
|
-
return;
|
|
3002
|
-
}
|
|
3003
|
-
const toolContract = await this.resolvePacing(toolId).catch(
|
|
3004
|
-
() => null,
|
|
3005
|
-
);
|
|
3006
|
-
// Each unbatched provider call takes its own tool slot: the Governor
|
|
3007
|
-
// charges tool budget, holds a global tool-concurrency slot, and
|
|
3008
|
-
// applies per-(org,provider) pacing before the call runs.
|
|
3009
|
-
const slot = await this.governor.acquireToolSlot(toolId, {
|
|
3010
|
-
signal: this.abortSignal,
|
|
3011
|
-
});
|
|
3012
|
-
try {
|
|
3013
|
-
let result: unknown;
|
|
3014
|
-
try {
|
|
3015
|
-
result = await executeToolWithLifecycle(
|
|
3016
|
-
this.req,
|
|
3017
|
-
{ id: request.id, toolId, input: request.input },
|
|
3018
|
-
request.workflowStep,
|
|
3019
|
-
this.callbacks,
|
|
3020
|
-
(retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
|
|
3021
|
-
() => this.governor.chargeBudget('retry'),
|
|
3022
|
-
toolContract?.retrySafeTransientHttp === true,
|
|
3023
|
-
this.abortSignal,
|
|
3024
|
-
this.runtimeDeadlineMs,
|
|
3025
|
-
resolveClaimedWorkerToolRuntimeTimeoutMs([claimed], {
|
|
3026
|
-
runtimeDeadlineMs: this.runtimeDeadlineMs,
|
|
3027
|
-
}),
|
|
3028
|
-
request.directOptions,
|
|
3029
|
-
);
|
|
3030
|
-
} catch (error) {
|
|
3031
|
-
this.rejectRequests(
|
|
3032
|
-
claimed,
|
|
3033
|
-
await this.failureForRejectedToolRequest(claimed, error),
|
|
3034
|
-
);
|
|
3035
|
-
return;
|
|
3036
|
-
}
|
|
3037
|
-
this.settleRequests(
|
|
3038
|
-
claimed,
|
|
3039
|
-
await this.completeDurableToolRequest(claimed, result),
|
|
3040
|
-
);
|
|
3041
|
-
} catch (receiptError) {
|
|
3042
|
-
tripRuntimePersistenceLatch(this.latch, receiptError);
|
|
3043
|
-
this.rejectRequests(
|
|
3044
|
-
claimed,
|
|
3045
|
-
new RuntimeReceiptPersistenceError(
|
|
3046
|
-
'Tool call succeeded but durable receipt could not be marked completed',
|
|
3047
|
-
[receiptError],
|
|
3048
|
-
),
|
|
3049
|
-
);
|
|
3050
|
-
} finally {
|
|
3051
|
-
slot.release();
|
|
3052
|
-
}
|
|
3053
|
-
};
|
|
3054
|
-
// Bounded dispatch waves: an unbatched tool group can carry a whole chunk's
|
|
3055
|
-
// rows (enrich compiles to one ~245-row chunk). Dispatching them all at
|
|
3056
|
-
// once billed every call before the first receipt-completion failure could
|
|
3057
|
-
// trip the latch (the 250/250 incident). Instead, dispatch one wave, AWAIT
|
|
3058
|
-
// it so any completion failure trips the latch, then read the latch before
|
|
3059
|
-
// the next wave — remaining waves are prevented (counted + rejected) exactly
|
|
3060
|
-
// like the mid-wave gate above. The wave calls stay fully concurrent; the
|
|
3061
|
-
// only added cost on the healthy path is one latch read per wave.
|
|
3062
|
-
for (const wave of partitionDispatchWaves(
|
|
3063
|
-
claimedRequests,
|
|
3064
|
-
() => 1,
|
|
3065
|
-
PROVIDER_DISPATCH_WAVE_SIZE,
|
|
3066
|
-
)) {
|
|
3067
|
-
if (this.latch.tripped) {
|
|
3068
|
-
for (const claimed of wave) {
|
|
3069
|
-
this.latch.preventedCallCount += 1 + claimed.followers.length;
|
|
3070
|
-
this.rejectRequests(
|
|
3071
|
-
claimed,
|
|
3072
|
-
new RuntimePersistenceCircuitOpenError(this.latch),
|
|
3073
|
-
);
|
|
3074
|
-
}
|
|
3075
|
-
continue;
|
|
3076
|
-
}
|
|
3077
|
-
await Promise.all(wave.map(dispatchOne));
|
|
3078
|
-
}
|
|
3079
|
-
recordRunnerPerfTrace({
|
|
3080
|
-
req: this.req,
|
|
3081
|
-
phase: 'runner.tool.group',
|
|
3082
|
-
ms: nowMs() - groupStartedAt,
|
|
3083
|
-
extra: {
|
|
3084
|
-
toolId,
|
|
3085
|
-
requests: requestCount,
|
|
3086
|
-
executed: claimedRequests.length,
|
|
3087
|
-
batched: false,
|
|
3088
|
-
},
|
|
3089
|
-
});
|
|
3090
|
-
return;
|
|
3091
|
-
}
|
|
2289
|
+
onOutput?: (output: RecordedStepProgramOutput) => void | Promise<void>;
|
|
2290
|
+
};
|
|
3092
2291
|
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
toolId,
|
|
3101
|
-
WORKER_TOOL_BATCH_DEFAULT_PARALLELISM,
|
|
3102
|
-
),
|
|
3103
|
-
abortSignal: this.abortSignal,
|
|
3104
|
-
runtimeDeadlineMs: this.runtimeDeadlineMs,
|
|
3105
|
-
reportBackpressure: (retryAfterMs) =>
|
|
3106
|
-
this.reportBackpressure(toolId, retryAfterMs),
|
|
3107
|
-
resolveToolContract: this.resolvePacing,
|
|
3108
|
-
onRequestsSettled: this.onRequestsSettled,
|
|
3109
|
-
callbacks: this.callbacks,
|
|
3110
|
-
completeRequests: async (entries) =>
|
|
3111
|
-
await this.completeDurableToolRequests(entries),
|
|
3112
|
-
failRequests: async (claimedRequests, error) =>
|
|
3113
|
-
await this.failDurableToolRequests(claimedRequests, error),
|
|
3114
|
-
settleRequest: (claimed, result) => this.settleRequests(claimed, result),
|
|
3115
|
-
rejectRequest: (claimed, error) => this.rejectRequests(claimed, error),
|
|
3116
|
-
latch: this.latch,
|
|
3117
|
-
});
|
|
3118
|
-
recordRunnerPerfTrace({
|
|
3119
|
-
req: this.req,
|
|
3120
|
-
phase: 'runner.tool.group',
|
|
3121
|
-
ms: nowMs() - batchStartedAt,
|
|
3122
|
-
extra: {
|
|
3123
|
-
toolId,
|
|
3124
|
-
requests: requestCount,
|
|
3125
|
-
executed: claimedRequests.length,
|
|
3126
|
-
batched: true,
|
|
3127
|
-
},
|
|
3128
|
-
});
|
|
3129
|
-
}
|
|
2292
|
+
type WorkerStepResolution = {
|
|
2293
|
+
value: unknown;
|
|
2294
|
+
status?: 'skipped';
|
|
2295
|
+
};
|
|
2296
|
+
|
|
2297
|
+
function stepProgramColumnName(parentField: string, stepId: string): string {
|
|
2298
|
+
return sqlSafePlayColumnName(`${parentField}.${stepId}`);
|
|
3130
2299
|
}
|
|
3131
2300
|
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
2301
|
+
const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
|
|
2302
|
+
const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
|
|
2303
|
+
const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
|
|
2304
|
+
const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 1_000;
|
|
2305
|
+
const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
|
|
2306
|
+
/**
|
|
2307
|
+
* Bounded number of per-row failure samples carried in chunk summaries and the
|
|
2308
|
+
* map's terminal partial-failure log. Every failed row is persisted with its
|
|
2309
|
+
* full error in the runtime sheet; the samples just keep run logs readable.
|
|
2310
|
+
*/
|
|
2311
|
+
const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
|
|
2312
|
+
|
|
2313
|
+
/**
|
|
2314
|
+
* Bounded-concurrent map-chunk dispatch fan-out for the workers_edge profile
|
|
2315
|
+
* (in-process play execution with no `workflowStep`). On workers_edge the whole
|
|
2316
|
+
* map shares ONE per-invocation subrequest window, so sequential chunk dispatch
|
|
2317
|
+
* buys no durability — it is pure wall-clock cost. Running up to K chunks in
|
|
2318
|
+
* flight overlaps provider batches, sheet writes, and the per-chunk persist
|
|
2319
|
+
* barrier. K is capped at 3 to bound peak memory: each in-flight chunk holds up
|
|
2320
|
+
* to ~5000 rows plus their row buffers, so 3 concurrent chunks is the ceiling
|
|
2321
|
+
* we accept before row payloads risk pressuring the isolate. The native
|
|
2322
|
+
* Cloudflare Workflows path (workflowStep present) keeps K=1: there each chunk
|
|
2323
|
+
* is a durable step and sequential dispatch is what makes replay/resume sound.
|
|
2324
|
+
*/
|
|
2325
|
+
const WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS = 3;
|
|
2326
|
+
const WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES =
|
|
2327
|
+
WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES;
|
|
2328
|
+
const WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES = 128;
|
|
2329
|
+
|
|
2330
|
+
const workerMapResidentRowBytes = (row: unknown): number =>
|
|
2331
|
+
new TextEncoder().encode(JSON.stringify(row)).byteLength +
|
|
2332
|
+
WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES;
|
|
2333
|
+
|
|
2334
|
+
function isRuntimeReceiptPersistenceError(error: unknown): boolean {
|
|
2335
|
+
if (!error || typeof error !== 'object') return false;
|
|
2336
|
+
if (error instanceof RuntimeReceiptPersistenceError) return true;
|
|
2337
|
+
if (
|
|
2338
|
+
error instanceof Error &&
|
|
2339
|
+
error.name === 'RuntimeReceiptPersistenceError'
|
|
2340
|
+
) {
|
|
2341
|
+
return true;
|
|
3143
2342
|
}
|
|
3144
|
-
|
|
2343
|
+
const nestedErrors = (error as { errors?: unknown }).errors;
|
|
2344
|
+
return (
|
|
2345
|
+
Array.isArray(nestedErrors) &&
|
|
2346
|
+
nestedErrors.some(isRuntimeReceiptPersistenceError)
|
|
2347
|
+
);
|
|
3145
2348
|
}
|
|
3146
2349
|
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
completeRequests: (
|
|
3160
|
-
entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
|
|
3161
|
-
) => Promise<unknown[]>;
|
|
3162
|
-
failRequests: (
|
|
3163
|
-
requests: ClaimedWorkerToolBatchRequest[],
|
|
3164
|
-
error: unknown,
|
|
3165
|
-
) => Promise<void>;
|
|
3166
|
-
settleRequest: (
|
|
3167
|
-
request: ClaimedWorkerToolBatchRequest,
|
|
3168
|
-
result: unknown,
|
|
3169
|
-
) => void;
|
|
3170
|
-
rejectRequest: (
|
|
3171
|
-
request: ClaimedWorkerToolBatchRequest,
|
|
3172
|
-
error: unknown,
|
|
3173
|
-
) => void;
|
|
3174
|
-
latch: RuntimePersistenceLatch;
|
|
3175
|
-
}): Promise<void> {
|
|
3176
|
-
const compiledBatches = compileRequestsWithStrategy({
|
|
3177
|
-
requests: input.requests,
|
|
3178
|
-
strategy: input.strategy,
|
|
3179
|
-
getPayload: (request) => request.request.input,
|
|
3180
|
-
});
|
|
3181
|
-
recordRunnerPerfTrace({
|
|
3182
|
-
req: input.req,
|
|
3183
|
-
phase: 'runner.tool.batch.compile',
|
|
3184
|
-
ms: 0,
|
|
3185
|
-
extra: {
|
|
3186
|
-
sourceOperation: input.strategy.sourceOperation,
|
|
3187
|
-
batchOperation: input.strategy.batchOperation,
|
|
3188
|
-
requests: input.requests.length,
|
|
3189
|
-
batches: compiledBatches.length,
|
|
3190
|
-
batchSizes: compiledBatches.map((batch) => batch.memberRequests.length),
|
|
3191
|
-
},
|
|
3192
|
-
});
|
|
2350
|
+
function isRunFatalWorkerRowError(error: unknown): boolean {
|
|
2351
|
+
return (
|
|
2352
|
+
isCoordinatorProgressAuthorizationError(error) ||
|
|
2353
|
+
isRowIsolationExemptError(error) ||
|
|
2354
|
+
isRuntimeReceiptPersistenceError(error) ||
|
|
2355
|
+
isRuntimePersistenceCircuitOpenError(error) ||
|
|
2356
|
+
error instanceof RuntimeSheetAttemptLeaseLostError ||
|
|
2357
|
+
error instanceof RuntimeLeaseLostError ||
|
|
2358
|
+
(error instanceof Error &&
|
|
2359
|
+
error.name === 'RuntimeSheetAttemptLeaseLostError')
|
|
2360
|
+
);
|
|
2361
|
+
}
|
|
3193
2362
|
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
// billing it all. A single over-budget batch still forms its own wave.
|
|
3206
|
-
maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
|
|
3207
|
-
weightOf: (batch) => batch.memberRequests.length,
|
|
3208
|
-
execute: async (batch) => {
|
|
3209
|
-
// Circuit breaker: once a persistence failure has occurred in this map,
|
|
3210
|
-
// do NOT dispatch this batch's provider call. The chunk plumbing rejects
|
|
3211
|
-
// its member requests with the latch error.
|
|
3212
|
-
if (input.latch.tripped) {
|
|
3213
|
-
input.latch.preventedCallCount += batch.memberRequests.length;
|
|
3214
|
-
throw new RuntimePersistenceCircuitOpenError(input.latch);
|
|
3215
|
-
}
|
|
3216
|
-
const toolContract = await input
|
|
3217
|
-
.resolveToolContract(batch.batchOperation)
|
|
3218
|
-
.catch(() => null);
|
|
3219
|
-
// One provider call per batch → one tool slot (budget + global
|
|
3220
|
-
// concurrency + per-(org,provider) pacing) around the whole batch.
|
|
3221
|
-
const slot = await input.governor.acquireToolSlot(batch.batchOperation, {
|
|
3222
|
-
signal: input.abortSignal,
|
|
3223
|
-
});
|
|
3224
|
-
try {
|
|
3225
|
-
input.callbacks?.onToolCalled?.(batch.batchOperation, nowMs());
|
|
3226
|
-
return await executeTool(
|
|
3227
|
-
input.req,
|
|
3228
|
-
{
|
|
3229
|
-
id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
|
|
3230
|
-
toolId: batch.batchOperation,
|
|
3231
|
-
input: batch.batchPayload,
|
|
3232
|
-
},
|
|
3233
|
-
undefined,
|
|
3234
|
-
input.reportBackpressure,
|
|
3235
|
-
() => input.governor.chargeBudget('retry'),
|
|
3236
|
-
toolContract?.retrySafeTransientHttp === true,
|
|
3237
|
-
input.abortSignal,
|
|
3238
|
-
input.runtimeDeadlineMs,
|
|
3239
|
-
resolveClaimedWorkerToolRuntimeTimeoutMs(batch.memberRequests, {
|
|
3240
|
-
runtimeDeadlineMs: input.runtimeDeadlineMs,
|
|
3241
|
-
}),
|
|
3242
|
-
);
|
|
3243
|
-
} catch (error) {
|
|
3244
|
-
input.callbacks?.onToolFailed?.(batch.batchOperation, nowMs());
|
|
3245
|
-
throw error;
|
|
3246
|
-
} finally {
|
|
3247
|
-
slot.release();
|
|
3248
|
-
}
|
|
3249
|
-
},
|
|
3250
|
-
onChunkComplete: async (
|
|
3251
|
-
chunkResults: Array<
|
|
3252
|
-
ChunkExecutionResult<(typeof compiledBatches)[number], unknown>
|
|
3253
|
-
>,
|
|
3254
|
-
) => {
|
|
3255
|
-
for (const entry of chunkResults) {
|
|
3256
|
-
if (entry.error !== undefined) {
|
|
3257
|
-
// One batch's provider error stays scoped to that batch's member
|
|
3258
|
-
// requests. Sibling batches in this chunk keep their results so a
|
|
3259
|
-
// single provider hiccup cannot cascade into a whole-map failure.
|
|
3260
|
-
let rejection: unknown = entry.error;
|
|
3261
|
-
try {
|
|
3262
|
-
await input.failRequests(entry.request.memberRequests, entry.error);
|
|
3263
|
-
} catch (receiptError) {
|
|
3264
|
-
tripRuntimePersistenceLatch(input.latch, receiptError);
|
|
3265
|
-
rejection = new RuntimeReceiptPersistenceError(
|
|
3266
|
-
'Tool call failed and durable receipts could not be marked failed',
|
|
3267
|
-
[entry.error, receiptError],
|
|
3268
|
-
);
|
|
3269
|
-
}
|
|
3270
|
-
for (const claimed of entry.request.memberRequests) {
|
|
3271
|
-
input.rejectRequest(claimed, rejection);
|
|
3272
|
-
}
|
|
3273
|
-
continue;
|
|
3274
|
-
}
|
|
3275
|
-
const batchResult = isToolExecuteResult(entry.result)
|
|
3276
|
-
? entry.result.toolOutput.raw
|
|
3277
|
-
: entry.result;
|
|
3278
|
-
const splitResults =
|
|
3279
|
-
batchResult != null
|
|
3280
|
-
? entry.request.splitResults(batchResult)
|
|
3281
|
-
: entry.request.memberRequests.map(() => null);
|
|
3282
|
-
let completedResults: unknown[];
|
|
3283
|
-
try {
|
|
3284
|
-
completedResults = await input.completeRequests(
|
|
3285
|
-
entry.request.memberRequests.map((claimed, index) => ({
|
|
3286
|
-
claimed,
|
|
3287
|
-
result: wrapWorkerToolResult(
|
|
3288
|
-
claimed.request.toolId,
|
|
3289
|
-
splitResults[index] ?? null,
|
|
3290
|
-
toolMetadataFallback(claimed.request.toolId),
|
|
3291
|
-
splitResults[index] == null ? 'no_result' : 'completed',
|
|
3292
|
-
{
|
|
3293
|
-
req: input.req,
|
|
3294
|
-
input: claimed.request.input,
|
|
3295
|
-
},
|
|
3296
|
-
),
|
|
3297
|
-
})),
|
|
3298
|
-
);
|
|
3299
|
-
} catch (receiptError) {
|
|
3300
|
-
tripRuntimePersistenceLatch(input.latch, receiptError);
|
|
3301
|
-
const rejection = new RuntimeReceiptPersistenceError(
|
|
3302
|
-
'Tool call succeeded but durable receipts could not be marked completed',
|
|
3303
|
-
[receiptError],
|
|
3304
|
-
);
|
|
3305
|
-
for (const claimed of entry.request.memberRequests) {
|
|
3306
|
-
input.rejectRequest(claimed, rejection);
|
|
3307
|
-
}
|
|
3308
|
-
continue;
|
|
3309
|
-
}
|
|
3310
|
-
for (let index = 0; index < completedResults.length; index += 1) {
|
|
3311
|
-
const claimed = entry.request.memberRequests[index]!;
|
|
3312
|
-
const request = claimed.request;
|
|
3313
|
-
input.settleRequest(claimed, completedResults[index]);
|
|
3314
|
-
}
|
|
3315
|
-
}
|
|
3316
|
-
},
|
|
3317
|
-
}).catch(async (error) => {
|
|
3318
|
-
let rejection: unknown = error;
|
|
3319
|
-
try {
|
|
3320
|
-
await input.failRequests(input.requests, error);
|
|
3321
|
-
} catch (receiptError) {
|
|
3322
|
-
tripRuntimePersistenceLatch(input.latch, receiptError);
|
|
3323
|
-
rejection = new RuntimeReceiptPersistenceError(
|
|
3324
|
-
'Tool call failed and durable receipts could not be marked failed',
|
|
3325
|
-
[error, receiptError],
|
|
3326
|
-
);
|
|
3327
|
-
}
|
|
3328
|
-
for (const claimed of input.requests) {
|
|
3329
|
-
input.rejectRequest(claimed, rejection);
|
|
3330
|
-
}
|
|
3331
|
-
});
|
|
2363
|
+
/**
|
|
2364
|
+
* In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
|
|
2365
|
+
* throughput pacing (provider or Deepline limiter), not a tool defect, so it
|
|
2366
|
+
* gets more patience than the generic 3-attempt budget: with retry-after-aware
|
|
2367
|
+
* escalating delays (capped at 5s) this absorbs roughly 25s of sustained
|
|
2368
|
+
* throttling before the call fails. Every retry still charges the Governor's
|
|
2369
|
+
* retry budget, so a runaway storm stays bounded and loud.
|
|
2370
|
+
*/
|
|
2371
|
+
|
|
2372
|
+
function sleepWorkerMs(ms: number): Promise<void> {
|
|
2373
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3332
2374
|
}
|
|
3333
2375
|
|
|
3334
2376
|
function isCompletedWorkerFieldValue(value: unknown): boolean {
|
|
@@ -3339,7 +2381,8 @@ function isCompletedWorkerFieldValue(value: unknown): boolean {
|
|
|
3339
2381
|
);
|
|
3340
2382
|
}
|
|
3341
2383
|
|
|
3342
|
-
type
|
|
2384
|
+
type WorkerMapCompletedChunkSummary<T extends Record<string, unknown>> = {
|
|
2385
|
+
status: 'completed';
|
|
3343
2386
|
chunkIndex: number;
|
|
3344
2387
|
rangeStart: number;
|
|
3345
2388
|
rangeEnd: number;
|
|
@@ -3363,6 +2406,22 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
|
|
|
3363
2406
|
cachedRows?: T[];
|
|
3364
2407
|
};
|
|
3365
2408
|
|
|
2409
|
+
type WorkerMapBlockedChunkSummary = {
|
|
2410
|
+
status: 'blocked';
|
|
2411
|
+
chunkIndex: number;
|
|
2412
|
+
rangeStart: number;
|
|
2413
|
+
rangeEnd: number;
|
|
2414
|
+
rowsRead: number;
|
|
2415
|
+
rowsBlocked: number;
|
|
2416
|
+
rowKeySamples: string[];
|
|
2417
|
+
blockedRowSamples: Record<string, unknown>[];
|
|
2418
|
+
outputDatasetId: string;
|
|
2419
|
+
message: string;
|
|
2420
|
+
};
|
|
2421
|
+
|
|
2422
|
+
type WorkerMapChunkSummary<T extends Record<string, unknown>> =
|
|
2423
|
+
WorkerMapCompletedChunkSummary<T> | WorkerMapBlockedChunkSummary;
|
|
2424
|
+
|
|
3366
2425
|
function serializeDurableStepValue<T>(value: T, depth = 0): T {
|
|
3367
2426
|
if (depth > 20 || value == null) return value;
|
|
3368
2427
|
if (isToolExecuteResult(value)) return serializeToolExecuteResult(value) as T;
|
|
@@ -3420,9 +2479,7 @@ type WorkerConditionalStepResolver = {
|
|
|
3420
2479
|
type WorkerStepProgramStep = {
|
|
3421
2480
|
name: string;
|
|
3422
2481
|
resolver:
|
|
3423
|
-
|
|
|
3424
|
-
| WorkerConditionalStepResolver
|
|
3425
|
-
| WorkerStepProgram;
|
|
2482
|
+
WorkerStepResolver | WorkerConditionalStepResolver | WorkerStepProgram;
|
|
3426
2483
|
};
|
|
3427
2484
|
|
|
3428
2485
|
type WorkerStepProgram = {
|
|
@@ -3575,6 +2632,7 @@ async function executeWorkerStepProgram(
|
|
|
3575
2632
|
...(resolution.status ? { status: resolution.status } : {}),
|
|
3576
2633
|
};
|
|
3577
2634
|
};
|
|
2635
|
+
|
|
3578
2636
|
const executeStep = async (): Promise<WorkerStepResolution> =>
|
|
3579
2637
|
workflowStep
|
|
3580
2638
|
? await (
|
|
@@ -3799,7 +2857,7 @@ async function postRuntimeEgressFetch(
|
|
|
3799
2857
|
) {
|
|
3800
2858
|
const response = await fetchRuntimeApi(
|
|
3801
2859
|
req.baseUrl,
|
|
3802
|
-
|
|
2860
|
+
PLAY_RUNTIME_EGRESS_FETCH_COMPAT_PATH,
|
|
3803
2861
|
{
|
|
3804
2862
|
method: 'POST',
|
|
3805
2863
|
headers: {
|
|
@@ -4358,9 +3416,7 @@ function resolveSheetContractFromReq(
|
|
|
4358
3416
|
// `staticPipeline` field is what `resolveSheetContractForTableNamespace`
|
|
4359
3417
|
// expects. Reach in safely.
|
|
4360
3418
|
const snapshot = req.contractSnapshot as
|
|
4361
|
-
|
|
4362
|
-
| undefined
|
|
4363
|
-
| null;
|
|
3419
|
+
{ staticPipeline?: unknown } | undefined | null;
|
|
4364
3420
|
const pipeline = snapshot?.staticPipeline;
|
|
4365
3421
|
if (!pipeline || typeof pipeline !== 'object' || Array.isArray(pipeline)) {
|
|
4366
3422
|
return null;
|
|
@@ -4375,11 +3431,84 @@ function resolveSheetContractFromReq(
|
|
|
4375
3431
|
}
|
|
4376
3432
|
}
|
|
4377
3433
|
|
|
3434
|
+
/**
|
|
3435
|
+
* Run-fatal teardown: release the runtime-sheet attempt leases and work-receipt
|
|
3436
|
+
* leases this run still holds, so an immediate rerun proceeds without waiting
|
|
3437
|
+
* out the 10-minute lease TTL. Every release is attempt/owner fenced downstream,
|
|
3438
|
+
* so it can never weaken a different run's live lease. Failures are logged loud
|
|
3439
|
+
* and swallowed here — the caller still throws the original run-fatal error.
|
|
3440
|
+
*/
|
|
3441
|
+
async function releaseRuntimeLeasesOnTeardown(input: {
|
|
3442
|
+
req: RunRequest;
|
|
3443
|
+
leasedSheetNamespaces: Set<string>;
|
|
3444
|
+
emit: (event: RunnerEvent) => void;
|
|
3445
|
+
}): Promise<void> {
|
|
3446
|
+
const { req, leasedSheetNamespaces, emit } = input;
|
|
3447
|
+
const scope = runtimeSheetSessionScope(req);
|
|
3448
|
+
for (const tableNamespace of leasedSheetNamespaces) {
|
|
3449
|
+
const sheetContract = resolveSheetContractFromReq(req, tableNamespace);
|
|
3450
|
+
if (!sheetContract) continue;
|
|
3451
|
+
try {
|
|
3452
|
+
const released = await harnessReleaseSheetAttempt({
|
|
3453
|
+
...scope,
|
|
3454
|
+
tableNamespace,
|
|
3455
|
+
sheetContract,
|
|
3456
|
+
runId: req.runId,
|
|
3457
|
+
attemptOwnerRunId: req.runId,
|
|
3458
|
+
attemptSeq: req.runAttempt ?? 0,
|
|
3459
|
+
});
|
|
3460
|
+
if (released.released > 0) {
|
|
3461
|
+
emit({
|
|
3462
|
+
type: 'log',
|
|
3463
|
+
level: 'info',
|
|
3464
|
+
message: `Released ${released.released} runtime sheet row lease(s) for ctx.dataset("${tableNamespace}") on run-fatal teardown.`,
|
|
3465
|
+
ts: nowMs(),
|
|
3466
|
+
});
|
|
3467
|
+
}
|
|
3468
|
+
} catch (releaseError) {
|
|
3469
|
+
emit({
|
|
3470
|
+
type: 'log',
|
|
3471
|
+
level: 'warn',
|
|
3472
|
+
message: `Runtime sheet attempt release failed for ctx.dataset("${tableNamespace}") on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
|
|
3473
|
+
releaseError instanceof Error
|
|
3474
|
+
? releaseError.message
|
|
3475
|
+
: String(releaseError)
|
|
3476
|
+
}`,
|
|
3477
|
+
ts: nowMs(),
|
|
3478
|
+
});
|
|
3479
|
+
}
|
|
3480
|
+
}
|
|
3481
|
+
try {
|
|
3482
|
+
const released = await harnessReleaseRuntimeReceipts({
|
|
3483
|
+
...scope,
|
|
3484
|
+
runId: req.runId,
|
|
3485
|
+
runAttempt: req.runAttempt ?? 0,
|
|
3486
|
+
});
|
|
3487
|
+
if (released.released > 0) {
|
|
3488
|
+
emit({
|
|
3489
|
+
type: 'log',
|
|
3490
|
+
level: 'info',
|
|
3491
|
+
message: `Released ${released.released} work-receipt lease(s) on run-fatal teardown.`,
|
|
3492
|
+
ts: nowMs(),
|
|
3493
|
+
});
|
|
3494
|
+
}
|
|
3495
|
+
} catch (releaseError) {
|
|
3496
|
+
emit({
|
|
3497
|
+
type: 'log',
|
|
3498
|
+
level: 'warn',
|
|
3499
|
+
message: `Work-receipt release failed on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
|
|
3500
|
+
releaseError instanceof Error
|
|
3501
|
+
? releaseError.message
|
|
3502
|
+
: String(releaseError)
|
|
3503
|
+
}`,
|
|
3504
|
+
ts: nowMs(),
|
|
3505
|
+
});
|
|
3506
|
+
}
|
|
3507
|
+
}
|
|
3508
|
+
|
|
4378
3509
|
function staticPipelineFromReq(req: RunRequest): PlayStaticPipeline | null {
|
|
4379
3510
|
const snapshot = req.contractSnapshot as
|
|
4380
|
-
|
|
4381
|
-
| undefined
|
|
4382
|
-
| null;
|
|
3511
|
+
{ staticPipeline?: unknown } | undefined | null;
|
|
4383
3512
|
const pipeline = snapshot?.staticPipeline;
|
|
4384
3513
|
if (!pipeline || typeof pipeline !== 'object' || Array.isArray(pipeline)) {
|
|
4385
3514
|
return null;
|
|
@@ -4417,6 +3546,13 @@ async function persistCompletedMapRows(input: {
|
|
|
4417
3546
|
rows: Record<string, unknown>[];
|
|
4418
3547
|
outputFields: string[];
|
|
4419
3548
|
extraOutputFields?: string[];
|
|
3549
|
+
attemptId?: string | null;
|
|
3550
|
+
attemptOwnerRunId?: string | null;
|
|
3551
|
+
attemptExpiresAt?: string | null;
|
|
3552
|
+
attemptSeq?: number | null;
|
|
3553
|
+
budgetMeter?: WorkerWorkBudgetMeter;
|
|
3554
|
+
force?: boolean;
|
|
3555
|
+
forceFailedRows?: boolean;
|
|
4420
3556
|
}): Promise<{
|
|
4421
3557
|
rows: number;
|
|
4422
3558
|
written: number;
|
|
@@ -4462,23 +3598,28 @@ async function persistCompletedMapRows(input: {
|
|
|
4462
3598
|
sheetContract,
|
|
4463
3599
|
rows,
|
|
4464
3600
|
outputFields,
|
|
3601
|
+
attemptId: input.attemptId ?? null,
|
|
3602
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3603
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3604
|
+
attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
|
|
3605
|
+
forceFailedRows: input.forceFailedRows === true,
|
|
4465
3606
|
};
|
|
3607
|
+
input.budgetMeter?.count('sheet');
|
|
4466
3608
|
const expectedKeys = [
|
|
4467
3609
|
...new Set(
|
|
4468
3610
|
input.rows
|
|
4469
|
-
.map((row) => resolveMapRowOutcomeKey(row))
|
|
4470
|
-
.filter(
|
|
3611
|
+
.map((row) => resolveMapRowOutcomeKey(row) ?? row.key)
|
|
3612
|
+
.filter(Boolean),
|
|
4471
3613
|
),
|
|
4472
|
-
];
|
|
4473
|
-
const expectedVisibleRows =
|
|
4474
|
-
expectedKeys.length > 0 ? expectedKeys.length : rows.length;
|
|
3614
|
+
] as string[];
|
|
3615
|
+
const expectedVisibleRows = expectedKeys.length || rows.length;
|
|
4475
3616
|
const readVisibleRowCount = async () => {
|
|
4476
3617
|
if (expectedKeys.length > 0) {
|
|
4477
3618
|
const result = await harnessReadSheetDatasetRowKeys({
|
|
4478
3619
|
...sessionScope,
|
|
4479
3620
|
tableNamespace,
|
|
4480
3621
|
runId: req.runId,
|
|
4481
|
-
rowMode: '
|
|
3622
|
+
rowMode: 'terminalAnyRun',
|
|
4482
3623
|
keys: expectedKeys,
|
|
4483
3624
|
});
|
|
4484
3625
|
return result.keys.length;
|
|
@@ -4494,6 +3635,24 @@ async function persistCompletedMapRows(input: {
|
|
|
4494
3635
|
return result.rows.length;
|
|
4495
3636
|
};
|
|
4496
3637
|
const result = await harnessPersistCompletedSheetRows(persistRequest);
|
|
3638
|
+
const hasAttemptFence = Boolean(input.attemptId?.trim());
|
|
3639
|
+
const assertAttemptStillOwnsTerminalWrites = (
|
|
3640
|
+
written: number,
|
|
3641
|
+
visible: number | null,
|
|
3642
|
+
) => {
|
|
3643
|
+
// Recovery finalization may include rows completed by an earlier attempt.
|
|
3644
|
+
// A complete terminal readback proves they were not lost.
|
|
3645
|
+
if (
|
|
3646
|
+
!hasAttemptFence ||
|
|
3647
|
+
written === rows.length ||
|
|
3648
|
+
(visible !== null && visible >= expectedVisibleRows)
|
|
3649
|
+
) {
|
|
3650
|
+
return;
|
|
3651
|
+
}
|
|
3652
|
+
throw new Error(
|
|
3653
|
+
`Runtime sheet attempt lost ownership for ${tableNamespace}: wrote ${written}/${rows.length}; run ${req.runId}; attempt ${input.attemptId}.`,
|
|
3654
|
+
);
|
|
3655
|
+
};
|
|
4497
3656
|
console.warn('[play-runner.persist_completed_map_rows.result]', {
|
|
4498
3657
|
runId: req.runId,
|
|
4499
3658
|
playName: req.playName,
|
|
@@ -4518,11 +3677,19 @@ async function persistCompletedMapRows(input: {
|
|
|
4518
3677
|
sheetContract,
|
|
4519
3678
|
rows: input.rows.map((row) => runtimeCsvStorageRow(row)),
|
|
4520
3679
|
runId: req.runId,
|
|
3680
|
+
attemptId: input.attemptId ?? null,
|
|
3681
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3682
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3683
|
+
attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
|
|
3684
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
4521
3685
|
inputOffset: 0,
|
|
4522
3686
|
});
|
|
3687
|
+
input.budgetMeter?.count('sheet');
|
|
4523
3688
|
const retry = await harnessPersistCompletedSheetRows(persistRequest);
|
|
3689
|
+
input.budgetMeter?.count('sheet');
|
|
4524
3690
|
retryWritten = retry.rowsWritten;
|
|
4525
3691
|
retryVisible = await readVisibleRowCount();
|
|
3692
|
+
assertAttemptStillOwnsTerminalWrites(retryWritten, retryVisible);
|
|
4526
3693
|
if (retryVisible >= expectedVisibleRows) {
|
|
4527
3694
|
return {
|
|
4528
3695
|
rows: rows.length,
|
|
@@ -4536,6 +3703,7 @@ async function persistCompletedMapRows(input: {
|
|
|
4536
3703
|
`Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}/${expectedVisibleRows}; run ${req.runId}.`,
|
|
4537
3704
|
);
|
|
4538
3705
|
}
|
|
3706
|
+
assertAttemptStillOwnsTerminalWrites(result.rowsWritten, visibleRows);
|
|
4539
3707
|
}
|
|
4540
3708
|
if (result.rowsWritten !== rows.length && visibleRows < expectedVisibleRows) {
|
|
4541
3709
|
throw new Error(
|
|
@@ -4557,8 +3725,14 @@ async function applyLiveMapRowUpdates(input: {
|
|
|
4557
3725
|
updates: Array<Omit<PlayRowUpdate, 'rowId'> & { runId?: string }>;
|
|
4558
3726
|
outputFields: string[];
|
|
4559
3727
|
extraOutputFields?: string[];
|
|
3728
|
+
attemptId?: string | null;
|
|
3729
|
+
attemptOwnerRunId?: string | null;
|
|
3730
|
+
attemptExpiresAt?: string | null;
|
|
3731
|
+
attemptSeq?: number | null;
|
|
3732
|
+
budgetMeter?: WorkerWorkBudgetMeter;
|
|
4560
3733
|
}): Promise<void> {
|
|
4561
3734
|
if (input.updates.length === 0) return;
|
|
3735
|
+
input.budgetMeter?.count('sheet');
|
|
4562
3736
|
const outputFields = [
|
|
4563
3737
|
...input.outputFields,
|
|
4564
3738
|
...(input.extraOutputFields ?? []).filter(
|
|
@@ -4581,7 +3755,16 @@ async function applyLiveMapRowUpdates(input: {
|
|
|
4581
3755
|
contractSnapshot: input.req.contractSnapshot,
|
|
4582
3756
|
runId: input.req.runId,
|
|
4583
3757
|
userEmail: input.req.userEmail,
|
|
4584
|
-
updates: input.updates
|
|
3758
|
+
updates: input.updates.map((update) => ({
|
|
3759
|
+
...update,
|
|
3760
|
+
attemptId: update.attemptId ?? input.attemptId ?? null,
|
|
3761
|
+
attemptOwnerRunId:
|
|
3762
|
+
update.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
|
|
3763
|
+
attemptExpiresAt:
|
|
3764
|
+
update.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
|
|
3765
|
+
attemptSeq:
|
|
3766
|
+
update.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3767
|
+
})),
|
|
4585
3768
|
},
|
|
4586
3769
|
);
|
|
4587
3770
|
}
|
|
@@ -4592,18 +3775,39 @@ async function prepareMapRows(input: {
|
|
|
4592
3775
|
rows: Record<string, unknown>[];
|
|
4593
3776
|
inputOffset: number;
|
|
4594
3777
|
outputFields: string[];
|
|
3778
|
+
attemptId?: string | null;
|
|
3779
|
+
attemptOwnerRunId?: string | null;
|
|
3780
|
+
attemptExpiresAt?: string | null;
|
|
3781
|
+
attemptSeq?: number | null;
|
|
3782
|
+
budgetMeter?: WorkerWorkBudgetMeter;
|
|
4595
3783
|
force?: boolean;
|
|
4596
3784
|
}): Promise<{
|
|
4597
3785
|
inserted: number;
|
|
4598
3786
|
skipped: number;
|
|
4599
3787
|
pendingRows: Record<string, unknown>[];
|
|
4600
3788
|
completedRows: Record<string, unknown>[];
|
|
3789
|
+
blockedRows: Record<string, unknown>[];
|
|
3790
|
+
attemptId?: string | null;
|
|
3791
|
+
attemptOwnerRunId?: string | null;
|
|
3792
|
+
attemptExpiresAt?: string | null;
|
|
3793
|
+
attemptSeq?: number | null;
|
|
4601
3794
|
}> {
|
|
4602
3795
|
if (input.rows.length === 0) {
|
|
4603
|
-
return {
|
|
3796
|
+
return {
|
|
3797
|
+
inserted: 0,
|
|
3798
|
+
skipped: 0,
|
|
3799
|
+
pendingRows: [],
|
|
3800
|
+
completedRows: [],
|
|
3801
|
+
blockedRows: [],
|
|
3802
|
+
attemptId: input.attemptId ?? null,
|
|
3803
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3804
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3805
|
+
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3806
|
+
};
|
|
4604
3807
|
}
|
|
4605
3808
|
const sessionScope = runtimeSheetSessionScope(input.req);
|
|
4606
3809
|
const rows = input.rows.map((row) => runtimeCsvStorageRow(row));
|
|
3810
|
+
input.budgetMeter?.count('sheet');
|
|
4607
3811
|
const result = await harnessStartSheetDataset({
|
|
4608
3812
|
...sessionScope,
|
|
4609
3813
|
tableNamespace: input.tableNamespace,
|
|
@@ -4614,6 +3818,11 @@ async function prepareMapRows(input: {
|
|
|
4614
3818
|
}),
|
|
4615
3819
|
rows,
|
|
4616
3820
|
inputOffset: input.inputOffset,
|
|
3821
|
+
attemptId: input.attemptId ?? null,
|
|
3822
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3823
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3824
|
+
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3825
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
|
|
4617
3826
|
...(input.force ? { force: true } : {}),
|
|
4618
3827
|
});
|
|
4619
3828
|
for (const timing of result.timings ?? []) {
|
|
@@ -4643,6 +3852,13 @@ async function prepareMapRows(input: {
|
|
|
4643
3852
|
skipped: result.skipped,
|
|
4644
3853
|
pendingRows: result.pendingRows,
|
|
4645
3854
|
completedRows: result.completedRows,
|
|
3855
|
+
blockedRows: result.blockedRows ?? [],
|
|
3856
|
+
attemptId: result.attemptId ?? input.attemptId ?? null,
|
|
3857
|
+
attemptOwnerRunId:
|
|
3858
|
+
result.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
|
|
3859
|
+
attemptExpiresAt: result.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
|
|
3860
|
+
attemptSeq:
|
|
3861
|
+
result.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
4646
3862
|
};
|
|
4647
3863
|
}
|
|
4648
3864
|
|
|
@@ -4788,14 +4004,6 @@ function createCoordinatorRatePort(req: RunRequest): CoordinatorRatePort {
|
|
|
4788
4004
|
* memoized per isolate. No hints → null (pacing is a no-op; the Governor's
|
|
4789
4005
|
* global tool-concurrency slot still applies).
|
|
4790
4006
|
*/
|
|
4791
|
-
type WorkerPacingResolver = (
|
|
4792
|
-
toolId: string,
|
|
4793
|
-
) => Promise<
|
|
4794
|
-
(ResolvedPacingPolicy & { retrySafeTransientHttp: boolean }) | null
|
|
4795
|
-
>;
|
|
4796
|
-
|
|
4797
|
-
type WorkerToolActionCacheVersionResolver = (toolId: string) => Promise<string>;
|
|
4798
|
-
|
|
4799
4007
|
function createWorkerToolActionCacheVersionResolver(
|
|
4800
4008
|
req: RunRequest,
|
|
4801
4009
|
): WorkerToolActionCacheVersionResolver {
|
|
@@ -4803,9 +4011,7 @@ function createWorkerToolActionCacheVersionResolver(
|
|
|
4803
4011
|
return (toolId: string) => {
|
|
4804
4012
|
const normalized = String(toolId || '').trim();
|
|
4805
4013
|
if (!normalized) {
|
|
4806
|
-
return Promise.reject(
|
|
4807
|
-
new Error('Runtime tool metadata lookup requires a non-empty tool id.'),
|
|
4808
|
-
);
|
|
4014
|
+
return Promise.reject(new Error('Tool metadata needs tool id.'));
|
|
4809
4015
|
}
|
|
4810
4016
|
const cached = cache.get(normalized);
|
|
4811
4017
|
if (cached) return cached;
|
|
@@ -4842,6 +4048,44 @@ function createWorkerToolActionCacheVersionResolver(
|
|
|
4842
4048
|
};
|
|
4843
4049
|
}
|
|
4844
4050
|
|
|
4051
|
+
function createWorkerToolAuthScopeDigestResolver(req: RunRequest) {
|
|
4052
|
+
// Return type is inferred from createRuntimeToolAuthScopeDigestResolver
|
|
4053
|
+
// (RuntimeToolAuthScopeDigestResolver), whose `invalidate` member is
|
|
4054
|
+
// REQUIRED — the AUTH_SCOPE_CHANGED re-claim wiring below calls it directly,
|
|
4055
|
+
// and a resolver without eviction support must be a type error.
|
|
4056
|
+
return createRuntimeToolAuthScopeDigestResolver({
|
|
4057
|
+
missingToolIdMessage: 'Tool auth scope needs tool id.',
|
|
4058
|
+
fetchDigest: async (normalized) => {
|
|
4059
|
+
const runScopedPath =
|
|
4060
|
+
`/api/v2/plays/runtime-tools/${encodeURIComponent(normalized)}/auth-scope` +
|
|
4061
|
+
`?runId=${encodeURIComponent(req.runId)}`;
|
|
4062
|
+
const res = await fetchRuntimeApi(req.baseUrl, runScopedPath, {
|
|
4063
|
+
method: 'GET',
|
|
4064
|
+
headers: {
|
|
4065
|
+
authorization: `Bearer ${req.executorToken}`,
|
|
4066
|
+
'cache-control': 'no-store',
|
|
4067
|
+
pragma: 'no-cache',
|
|
4068
|
+
},
|
|
4069
|
+
});
|
|
4070
|
+
if (!res.ok) {
|
|
4071
|
+
throw new Error(
|
|
4072
|
+
`Runtime tool auth scope lookup for ${normalized} failed (${res.status}): ${await res.text()}`,
|
|
4073
|
+
);
|
|
4074
|
+
}
|
|
4075
|
+
const body = (await res.json().catch(() => null)) as {
|
|
4076
|
+
digest?: unknown;
|
|
4077
|
+
} | null;
|
|
4078
|
+
const digest = typeof body?.digest === 'string' ? body.digest.trim() : '';
|
|
4079
|
+
if (!digest) {
|
|
4080
|
+
throw new Error(
|
|
4081
|
+
`Runtime tool auth scope lookup for ${normalized} returned no digest.`,
|
|
4082
|
+
);
|
|
4083
|
+
}
|
|
4084
|
+
return digest;
|
|
4085
|
+
},
|
|
4086
|
+
});
|
|
4087
|
+
}
|
|
4088
|
+
|
|
4845
4089
|
function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
|
|
4846
4090
|
const cache = new Map<
|
|
4847
4091
|
string,
|
|
@@ -4983,13 +4227,22 @@ function createMinimalWorkerCtx(
|
|
|
4983
4227
|
workflowStep?: WorkflowStep,
|
|
4984
4228
|
abortSignal?: AbortSignal,
|
|
4985
4229
|
callbacks?: WorkerCtxCallbacks,
|
|
4230
|
+
workBudgetMeter: WorkerWorkBudgetMeter = createWorkerWorkBudgetMeter({
|
|
4231
|
+
nowMs,
|
|
4232
|
+
}),
|
|
4986
4233
|
runtimeDeadlineMs?: number,
|
|
4234
|
+
// Namespaces this run took a runtime-sheet attempt lease on. The run-fatal
|
|
4235
|
+
// teardown reads this set to release exactly those leases (never a namespace
|
|
4236
|
+
// whose table the run never created), so a rerun proceeds immediately.
|
|
4237
|
+
leasedSheetNamespaces?: Set<string>,
|
|
4987
4238
|
receiptSalvage?: ReceiptSalvageBuffer,
|
|
4988
4239
|
): unknown {
|
|
4989
4240
|
const { governor, resolvePacing: resolveToolPacing } =
|
|
4990
4241
|
createGovernorForRun(req);
|
|
4991
4242
|
const resolveToolActionCacheVersion =
|
|
4992
4243
|
createWorkerToolActionCacheVersionResolver(req);
|
|
4244
|
+
const resolveToolAuthScopeDigest =
|
|
4245
|
+
createWorkerToolAuthScopeDigestResolver(req);
|
|
4993
4246
|
// Play-call depth/count/per-parent budgets, child-play concurrency, and the
|
|
4994
4247
|
// lineage snapshot are owned by the Governor (createGovernorForRun above).
|
|
4995
4248
|
// The worker keeps only substrate mechanism here.
|
|
@@ -5000,7 +4253,7 @@ function createMinimalWorkerCtx(
|
|
|
5000
4253
|
if (!auth) return {};
|
|
5001
4254
|
const response = await fetchRuntimeApi(
|
|
5002
4255
|
req.baseUrl,
|
|
5003
|
-
|
|
4256
|
+
PLAY_RUNTIME_API_COMPAT_PATH,
|
|
5004
4257
|
{
|
|
5005
4258
|
method: 'POST',
|
|
5006
4259
|
headers: {
|
|
@@ -5039,10 +4292,48 @@ function createMinimalWorkerCtx(
|
|
|
5039
4292
|
orgId: req.orgId,
|
|
5040
4293
|
preloadedDbSessions: req.preloadedDbSessions ?? null,
|
|
5041
4294
|
userEmail: req.userEmail,
|
|
4295
|
+
runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null,
|
|
4296
|
+
runAttempt: req.runAttempt ?? 0,
|
|
4297
|
+
receiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
|
|
5042
4298
|
});
|
|
4299
|
+
const executeDispatchedTool = async (input: {
|
|
4300
|
+
id: string;
|
|
4301
|
+
toolId: string;
|
|
4302
|
+
input: Record<string, unknown>;
|
|
4303
|
+
workflowStep?: unknown;
|
|
4304
|
+
callbacks?: WorkerCtxCallbacks;
|
|
4305
|
+
onProviderBackpressure?: (retryAfterMs: number) => void;
|
|
4306
|
+
onRetryAttempt?: () => void;
|
|
4307
|
+
transientHttpRetrySafe?: boolean;
|
|
4308
|
+
durableCallReceiptKey?: string | null;
|
|
4309
|
+
executionAuthScopeDigest?: string | null;
|
|
4310
|
+
providerIdempotencyKey?: string | null;
|
|
4311
|
+
runtimeTimeoutMs?: number;
|
|
4312
|
+
directOptions?: WorkerToolDirectOptions;
|
|
4313
|
+
}): Promise<ToolExecuteResult> =>
|
|
4314
|
+
await executeToolWithLifecycle(
|
|
4315
|
+
req,
|
|
4316
|
+
{
|
|
4317
|
+
id: input.id,
|
|
4318
|
+
toolId: input.toolId,
|
|
4319
|
+
input: input.input,
|
|
4320
|
+
durableCallReceiptKey: input.durableCallReceiptKey,
|
|
4321
|
+
executionAuthScopeDigest: input.executionAuthScopeDigest,
|
|
4322
|
+
providerIdempotencyKey: input.providerIdempotencyKey,
|
|
4323
|
+
},
|
|
4324
|
+
input.workflowStep as WorkflowStep | undefined,
|
|
4325
|
+
input.callbacks,
|
|
4326
|
+
input.onProviderBackpressure,
|
|
4327
|
+
input.onRetryAttempt,
|
|
4328
|
+
input.transientHttpRetrySafe === true,
|
|
4329
|
+
abortSignal,
|
|
4330
|
+
runtimeDeadlineMs,
|
|
4331
|
+
input.runtimeTimeoutMs,
|
|
4332
|
+
input.directOptions,
|
|
4333
|
+
);
|
|
5043
4334
|
const executeWithRuntimeReceipt = async <T>(
|
|
5044
4335
|
key: string,
|
|
5045
|
-
execute: () => Promise<T> | T,
|
|
4336
|
+
execute: (context: WorkerRuntimeReceiptExecutionContext) => Promise<T> | T,
|
|
5046
4337
|
options: {
|
|
5047
4338
|
repairRunningReceiptForSameRun?: boolean;
|
|
5048
4339
|
reclaimRunning?: boolean;
|
|
@@ -5056,7 +4347,8 @@ function createMinimalWorkerCtx(
|
|
|
5056
4347
|
runId: req.runId,
|
|
5057
4348
|
key,
|
|
5058
4349
|
receiptStore,
|
|
5059
|
-
execute: async () =>
|
|
4350
|
+
execute: async (context) =>
|
|
4351
|
+
serializeDurableStepValue(await execute(context)),
|
|
5060
4352
|
repairRunningReceiptForSameRun:
|
|
5061
4353
|
options.repairRunningReceiptForSameRun ?? false,
|
|
5062
4354
|
reclaimRunning: options.reclaimRunning ?? false,
|
|
@@ -5098,24 +4390,54 @@ function createMinimalWorkerCtx(
|
|
|
5098
4390
|
staleAfterSeconds,
|
|
5099
4391
|
});
|
|
5100
4392
|
};
|
|
5101
|
-
|
|
5102
|
-
|
|
5103
|
-
|
|
4393
|
+
const emitAuthScopeReclaimRunLog = (input: {
|
|
4394
|
+
toolId: string;
|
|
4395
|
+
requestIds: string[];
|
|
4396
|
+
}) => {
|
|
4397
|
+
// Durable run-log marker for the bounded AUTH_SCOPE_CHANGED re-claim
|
|
4398
|
+
// (mirrors the shared-runtime marker in shared_libs/play-runtime/context.ts
|
|
4399
|
+
// so run logs expose the credential transition on every execution path).
|
|
4400
|
+
emitEvent({
|
|
4401
|
+
type: 'log',
|
|
4402
|
+
level: 'info',
|
|
4403
|
+
message:
|
|
4404
|
+
`ctx.tools.execute(${input.toolId}): auth_scope_changed_reclaim ` +
|
|
4405
|
+
`(requests: ${input.requestIds.join(', ')}); credential identity ` +
|
|
4406
|
+
`changed mid-run, re-resolved the auth scope and re-claimed fresh ` +
|
|
4407
|
+
`receipts under the new scope.`,
|
|
4408
|
+
ts: nowMs(),
|
|
4409
|
+
});
|
|
4410
|
+
};
|
|
5104
4411
|
const rootPersistenceLatch = createRuntimePersistenceLatch();
|
|
5105
|
-
const rootToolBatchScheduler = new WorkerToolBatchScheduler(
|
|
4412
|
+
const rootToolBatchScheduler = new WorkerToolBatchScheduler({
|
|
5106
4413
|
req,
|
|
5107
4414
|
governor,
|
|
5108
|
-
resolveToolPacing,
|
|
4415
|
+
resolvePacing: resolveToolPacing,
|
|
5109
4416
|
resolveToolActionCacheVersion,
|
|
4417
|
+
resolveToolAuthScopeDigest,
|
|
4418
|
+
invalidateToolAuthScopeDigest: (toolId) =>
|
|
4419
|
+
resolveToolAuthScopeDigest.invalidate(toolId),
|
|
4420
|
+
onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
|
|
4421
|
+
executeTool: executeDispatchedTool,
|
|
4422
|
+
serializeDurableValue: serializeDurableStepValue,
|
|
4423
|
+
deserializeDurableValue: deserializeDurableStepValue,
|
|
4424
|
+
nowMs,
|
|
4425
|
+
batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
|
|
4426
|
+
recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
|
|
5110
4427
|
abortSignal,
|
|
5111
|
-
undefined,
|
|
5112
4428
|
callbacks,
|
|
5113
4429
|
receiptStore,
|
|
5114
|
-
true,
|
|
4430
|
+
allowLocalRetryReceipts: true,
|
|
4431
|
+
runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
|
|
4432
|
+
runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
|
|
4433
|
+
req,
|
|
4434
|
+
runtimeReceiptLeaseTtlMs(req) ?? PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
4435
|
+
),
|
|
4436
|
+
budgetMeter: workBudgetMeter,
|
|
5115
4437
|
runtimeDeadlineMs,
|
|
5116
|
-
rootPersistenceLatch,
|
|
4438
|
+
persistenceLatch: rootPersistenceLatch,
|
|
5117
4439
|
receiptSalvage,
|
|
5118
|
-
);
|
|
4440
|
+
});
|
|
5119
4441
|
// Local ancestry chain that always ENDS with the currently-executing play
|
|
5120
4442
|
// (req.playName). The /api/v2/plays/run lineage validator requires the
|
|
5121
4443
|
// submitted ancestry's tail to equal the executor token's play name (i.e.
|
|
@@ -5150,23 +4472,33 @@ function createMinimalWorkerCtx(
|
|
|
5150
4472
|
): Promise<unknown> => {
|
|
5151
4473
|
const mapStartedAt = nowMs();
|
|
5152
4474
|
const mapNodeId = `map:${name}`;
|
|
5153
|
-
// Map-wide persistence-failure circuit breaker. Shared across every chunk's
|
|
5154
|
-
// scheduler (the scheduler is rebuilt per chunk) and the map's sheet-persist
|
|
5155
|
-
// flush chain, so the first persistence failure halts NEW provider dispatch
|
|
5156
|
-
// for the rest of the map and its prevented-call count surfaces in the
|
|
5157
|
-
// runMap failure message below.
|
|
5158
|
-
const persistenceLatch = createRuntimePersistenceLatch();
|
|
5159
4475
|
const inputRows = rows;
|
|
5160
4476
|
const rowCountHint = datasetRowCountHint(inputRows);
|
|
5161
4477
|
const baseOffset = 0;
|
|
5162
4478
|
const fieldEntries = Object.entries(fieldsDef);
|
|
5163
4479
|
const plan = req.executionPlan;
|
|
5164
|
-
const
|
|
4480
|
+
const mapDispatchPlan = chooseWorkerMapDispatchPlan({
|
|
5165
4481
|
mapName: name,
|
|
5166
4482
|
rowCountHint,
|
|
5167
4483
|
executionPlan: plan,
|
|
5168
4484
|
staticPipeline: staticPipelineFromReq(req),
|
|
5169
4485
|
});
|
|
4486
|
+
const rowsPerChunk = mapDispatchPlan.rowsPerChunk;
|
|
4487
|
+
recordRunnerPerfTrace({
|
|
4488
|
+
req,
|
|
4489
|
+
phase: 'runner.map_dispatch_plan',
|
|
4490
|
+
ms: 0,
|
|
4491
|
+
extra: {
|
|
4492
|
+
mapName: name,
|
|
4493
|
+
rowsPerChunk,
|
|
4494
|
+
rowCountHint,
|
|
4495
|
+
estimatedExternalStepsPerRow:
|
|
4496
|
+
mapDispatchPlan.estimatedExternalStepsPerRow,
|
|
4497
|
+
workEstimate: mapDispatchPlan.workEstimate,
|
|
4498
|
+
reason: mapDispatchPlan.reason,
|
|
4499
|
+
dynamicWorkerArtifactVersion: DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION,
|
|
4500
|
+
},
|
|
4501
|
+
});
|
|
5170
4502
|
const outputFields = fieldEntries.map(([field]) => field);
|
|
5171
4503
|
const updateMapProgress = (
|
|
5172
4504
|
progress: LiveNodeProgressSnapshot,
|
|
@@ -5182,6 +4514,22 @@ function createMinimalWorkerCtx(
|
|
|
5182
4514
|
forceFlush: options?.forceFlush ?? true,
|
|
5183
4515
|
});
|
|
5184
4516
|
};
|
|
4517
|
+
let queuedMapProgress: Promise<void> = Promise.resolve();
|
|
4518
|
+
let queuedMapProgressError: unknown = null;
|
|
4519
|
+
const enqueueMapProgressUpdate = (
|
|
4520
|
+
progress: LiveNodeProgressSnapshot,
|
|
4521
|
+
options?: { forceFlush?: boolean },
|
|
4522
|
+
) => {
|
|
4523
|
+
queuedMapProgress = queuedMapProgress
|
|
4524
|
+
.then(() => Promise.resolve(updateMapProgress(progress, options)))
|
|
4525
|
+
.catch((error) => {
|
|
4526
|
+
queuedMapProgressError ??= error;
|
|
4527
|
+
});
|
|
4528
|
+
};
|
|
4529
|
+
const flushQueuedMapProgressUpdates = async () => {
|
|
4530
|
+
await queuedMapProgress;
|
|
4531
|
+
if (queuedMapProgressError) throw queuedMapProgressError;
|
|
4532
|
+
};
|
|
5185
4533
|
const formatMapProgressMessage = (completed: number, total?: number) =>
|
|
5186
4534
|
typeof total === 'number' && Number.isFinite(total) && total > 0
|
|
5187
4535
|
? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
|
|
@@ -5206,7 +4554,9 @@ function createMinimalWorkerCtx(
|
|
|
5206
4554
|
return formatMapProgressMessage(completed, input.total);
|
|
5207
4555
|
};
|
|
5208
4556
|
const formatMapProcessingMessage = (rowsToExecute: number) =>
|
|
5209
|
-
rowsToExecute > 0
|
|
4557
|
+
rowsToExecute > 0
|
|
4558
|
+
? `Processing ${rowsToExecute.toLocaleString()} rows`
|
|
4559
|
+
: null;
|
|
5210
4560
|
const formatMapExecutionHeartbeatMessage = (input: {
|
|
5211
4561
|
rowsToExecute: number;
|
|
5212
4562
|
startedRows: number;
|
|
@@ -5220,7 +4570,7 @@ function createMinimalWorkerCtx(
|
|
|
5220
4570
|
const waitingRows = Math.max(0, rowsToExecute - startedRows);
|
|
5221
4571
|
const parts = [
|
|
5222
4572
|
activeRows > 0 ? `${activeRows.toLocaleString()} active` : null,
|
|
5223
|
-
waitingRows > 0 ? `${waitingRows.toLocaleString()}
|
|
4573
|
+
waitingRows > 0 ? `${waitingRows.toLocaleString()} waiting` : null,
|
|
5224
4574
|
completedRows > 0 ? `${completedRows.toLocaleString()} done` : null,
|
|
5225
4575
|
].filter((part): part is string => Boolean(part));
|
|
5226
4576
|
const base =
|
|
@@ -5384,6 +4734,12 @@ function createMinimalWorkerCtx(
|
|
|
5384
4734
|
): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
|
|
5385
4735
|
const chunkStartedAt = nowMs();
|
|
5386
4736
|
assertNotAborted(abortSignal);
|
|
4737
|
+
const rowAttempt = makeRuntimeSheetAttempt({
|
|
4738
|
+
runId: req.runId,
|
|
4739
|
+
runAttempt: req.runAttempt,
|
|
4740
|
+
mapName: name,
|
|
4741
|
+
chunkIndex,
|
|
4742
|
+
});
|
|
5387
4743
|
const keyStartedAt = nowMs();
|
|
5388
4744
|
const chunkEntries = chunkRows.map((row, localIndex) => {
|
|
5389
4745
|
const absoluteIndex = baseOffset + chunkStart + localIndex;
|
|
@@ -5411,8 +4767,21 @@ function createMinimalWorkerCtx(
|
|
|
5411
4767
|
...mapRowOutcomeRuntimeFields({ key: rowKey }),
|
|
5412
4768
|
})),
|
|
5413
4769
|
inputOffset: baseOffset + chunkStart,
|
|
4770
|
+
...rowAttempt,
|
|
4771
|
+
budgetMeter: workBudgetMeter,
|
|
5414
4772
|
force: !!req.force,
|
|
5415
4773
|
});
|
|
4774
|
+
const activeRowAttempt = {
|
|
4775
|
+
attemptId: prepared.attemptId ?? rowAttempt.attemptId,
|
|
4776
|
+
attemptOwnerRunId:
|
|
4777
|
+
prepared.attemptOwnerRunId ?? rowAttempt.attemptOwnerRunId,
|
|
4778
|
+
attemptExpiresAt:
|
|
4779
|
+
prepared.attemptExpiresAt ?? rowAttempt.attemptExpiresAt,
|
|
4780
|
+
attemptSeq: prepared.attemptSeq ?? rowAttempt.attemptSeq,
|
|
4781
|
+
};
|
|
4782
|
+
// Record that this run holds attempt leases in `name`'s sheet so a
|
|
4783
|
+
// run-fatal teardown releases them (the table now exists).
|
|
4784
|
+
leasedSheetNamespaces?.add(name);
|
|
5416
4785
|
recordRunnerPerfTrace({
|
|
5417
4786
|
req,
|
|
5418
4787
|
phase: 'runner.map_chunk.prepare_rows',
|
|
@@ -5425,14 +4794,15 @@ function createMinimalWorkerCtx(
|
|
|
5425
4794
|
skipped: prepared.skipped,
|
|
5426
4795
|
pendingRows: prepared.pendingRows.length,
|
|
5427
4796
|
completedRows: prepared.completedRows.length,
|
|
4797
|
+
blockedRows: prepared.blockedRows.length,
|
|
5428
4798
|
},
|
|
5429
4799
|
});
|
|
5430
4800
|
const progressTotalRows = rowCountHint ?? chunkRows.length;
|
|
5431
4801
|
const preparedCompletedRows = Math.min(
|
|
5432
4802
|
progressTotalRows,
|
|
5433
|
-
totalRowsWritten
|
|
4803
|
+
totalRowsWritten,
|
|
5434
4804
|
);
|
|
5435
|
-
|
|
4805
|
+
enqueueMapProgressUpdate({
|
|
5436
4806
|
completed: preparedCompletedRows,
|
|
5437
4807
|
total: progressTotalRows,
|
|
5438
4808
|
startedAt: mapStartedAt,
|
|
@@ -5445,6 +4815,9 @@ function createMinimalWorkerCtx(
|
|
|
5445
4815
|
const pendingKeys = new Set<string>();
|
|
5446
4816
|
const pendingRowsByKey = new Map<string, Record<string, unknown>>();
|
|
5447
4817
|
const completedKeys = new Set<string>();
|
|
4818
|
+
const completedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
4819
|
+
const blockedKeys = new Set<string>();
|
|
4820
|
+
const blockedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
5448
4821
|
const preparedKeys = new Set<string>();
|
|
5449
4822
|
for (const row of prepared.pendingRows) {
|
|
5450
4823
|
const key =
|
|
@@ -5462,14 +4835,95 @@ function createMinimalWorkerCtx(
|
|
|
5462
4835
|
deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
|
|
5463
4836
|
if (key) {
|
|
5464
4837
|
completedKeys.add(key);
|
|
4838
|
+
completedRowsByKey.set(key, row);
|
|
4839
|
+
preparedKeys.add(key);
|
|
4840
|
+
}
|
|
4841
|
+
}
|
|
4842
|
+
for (const row of prepared.blockedRows) {
|
|
4843
|
+
const key =
|
|
4844
|
+
resolveMapRowOutcomeKey(row) ??
|
|
4845
|
+
deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
|
|
4846
|
+
if (key) {
|
|
4847
|
+
blockedKeys.add(key);
|
|
4848
|
+
blockedRowsByKey.set(key, row);
|
|
5465
4849
|
preparedKeys.add(key);
|
|
5466
4850
|
}
|
|
5467
4851
|
}
|
|
4852
|
+
if (blockedKeys.size > 0) {
|
|
4853
|
+
const rowKeySamples = Array.from(blockedKeys).slice(
|
|
4854
|
+
0,
|
|
4855
|
+
MAP_ROW_FAILURE_SAMPLE_LIMIT,
|
|
4856
|
+
);
|
|
4857
|
+
const message =
|
|
4858
|
+
`ctx.dataset("${name}") chunk ${chunkIndex} is blocked by ` +
|
|
4859
|
+
`${blockedKeys.size} active runtime sheet row lease(s) owned by another attempt. ` +
|
|
4860
|
+
`Retry after the owning attempt finishes or the row leases expire.` +
|
|
4861
|
+
(rowKeySamples.length > 0
|
|
4862
|
+
? ` Blocked row keys: ${rowKeySamples.join(', ')}`
|
|
4863
|
+
: '');
|
|
4864
|
+
recordRunnerPerfTrace({
|
|
4865
|
+
req,
|
|
4866
|
+
phase: 'runner.map_chunk.blocked_rows',
|
|
4867
|
+
ms: nowMs() - prepareStartedAt,
|
|
4868
|
+
extra: {
|
|
4869
|
+
mapName: name,
|
|
4870
|
+
chunkIndex,
|
|
4871
|
+
rowsBlocked: blockedKeys.size,
|
|
4872
|
+
rowKeySamples,
|
|
4873
|
+
},
|
|
4874
|
+
});
|
|
4875
|
+
enqueueMapProgressUpdate({
|
|
4876
|
+
completed: preparedCompletedRows,
|
|
4877
|
+
total: progressTotalRows,
|
|
4878
|
+
waitingRows: blockedKeys.size,
|
|
4879
|
+
startedAt: mapStartedAt,
|
|
4880
|
+
message,
|
|
4881
|
+
});
|
|
4882
|
+
return {
|
|
4883
|
+
status: 'blocked',
|
|
4884
|
+
chunkIndex,
|
|
4885
|
+
rangeStart: baseOffset + chunkStart,
|
|
4886
|
+
rangeEnd: baseOffset + chunkStart + chunkRows.length,
|
|
4887
|
+
rowsRead: chunkRows.length,
|
|
4888
|
+
rowsBlocked: blockedKeys.size,
|
|
4889
|
+
rowKeySamples,
|
|
4890
|
+
blockedRowSamples: rowKeySamples.map((rowKey) => {
|
|
4891
|
+
const blockedRow = blockedRowsByKey.get(rowKey);
|
|
4892
|
+
const attemptId =
|
|
4893
|
+
typeof blockedRow?.__deeplineAttemptId === 'string'
|
|
4894
|
+
? blockedRow.__deeplineAttemptId
|
|
4895
|
+
: typeof blockedRow?._attempt_id === 'string'
|
|
4896
|
+
? blockedRow._attempt_id
|
|
4897
|
+
: activeRowAttempt.attemptId;
|
|
4898
|
+
return {
|
|
4899
|
+
__deeplineRowKey: rowKey,
|
|
4900
|
+
__deeplineAttemptId: attemptId,
|
|
4901
|
+
};
|
|
4902
|
+
}),
|
|
4903
|
+
outputDatasetId: `map:${name}`,
|
|
4904
|
+
message,
|
|
4905
|
+
};
|
|
4906
|
+
}
|
|
5468
4907
|
const missingPreparedRows = chunkEntries.filter(
|
|
5469
4908
|
({ rowKey }) => !preparedKeys.has(rowKey),
|
|
5470
4909
|
);
|
|
5471
|
-
|
|
5472
|
-
|
|
4910
|
+
if (missingPreparedRows.length > 0) {
|
|
4911
|
+
const rowKeySamples = missingPreparedRows
|
|
4912
|
+
.map(({ rowKey }) => rowKey)
|
|
4913
|
+
.slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
|
|
4914
|
+
throw new Error(
|
|
4915
|
+
`ctx.dataset("${name}") runtime preparation omitted ${missingPreparedRows.length} row disposition(s). ` +
|
|
4916
|
+
`Every input row must be claimed, completed, or blocked before execution starts.` +
|
|
4917
|
+
(rowKeySamples.length > 0
|
|
4918
|
+
? ` Missing row keys: ${rowKeySamples.join(', ')}`
|
|
4919
|
+
: ''),
|
|
4920
|
+
);
|
|
4921
|
+
}
|
|
4922
|
+
const rowsToExecuteEntries = chunkEntries.filter(({ rowKey }) =>
|
|
4923
|
+
pendingKeys.has(rowKey),
|
|
4924
|
+
);
|
|
4925
|
+
const completedChunkEntries = chunkEntries.filter(({ rowKey }) =>
|
|
4926
|
+
completedKeys.has(rowKey),
|
|
5473
4927
|
);
|
|
5474
4928
|
const uniqueRowsToExecuteEntries = [
|
|
5475
4929
|
...new Map(
|
|
@@ -5486,18 +4940,15 @@ function createMinimalWorkerCtx(
|
|
|
5486
4940
|
rowsToExecute.length,
|
|
5487
4941
|
);
|
|
5488
4942
|
if (processingMessage) {
|
|
5489
|
-
|
|
4943
|
+
enqueueMapProgressUpdate({
|
|
5490
4944
|
completed: preparedCompletedRows,
|
|
5491
4945
|
total: progressTotalRows,
|
|
5492
4946
|
startedAt: mapStartedAt,
|
|
5493
4947
|
message: processingMessage,
|
|
5494
4948
|
});
|
|
5495
4949
|
}
|
|
5496
|
-
const rowsInserted = prepared.inserted
|
|
5497
|
-
const rowsSkipped =
|
|
5498
|
-
0,
|
|
5499
|
-
prepared.skipped - missingPreparedRows.length,
|
|
5500
|
-
);
|
|
4950
|
+
const rowsInserted = prepared.inserted;
|
|
4951
|
+
const rowsSkipped = prepared.skipped;
|
|
5501
4952
|
let completedExecutedRows = 0;
|
|
5502
4953
|
let failedExecutedRows = 0;
|
|
5503
4954
|
let startedExecutedRows = 0;
|
|
@@ -5505,12 +4956,7 @@ function createMinimalWorkerCtx(
|
|
|
5505
4956
|
let lastChunkProgressAt = 0;
|
|
5506
4957
|
let lastExecutionHeartbeatAt = nowMs();
|
|
5507
4958
|
const completedRowsForProgress = () =>
|
|
5508
|
-
Math.min(
|
|
5509
|
-
progressTotalRows,
|
|
5510
|
-
totalRowsWritten +
|
|
5511
|
-
prepared.completedRows.length +
|
|
5512
|
-
completedExecutedRows,
|
|
5513
|
-
);
|
|
4959
|
+
Math.min(progressTotalRows, totalRowsWritten + completedExecutedRows);
|
|
5514
4960
|
const reportExecutionHeartbeat = (force = false) => {
|
|
5515
4961
|
const now = nowMs();
|
|
5516
4962
|
if (
|
|
@@ -5520,7 +4966,7 @@ function createMinimalWorkerCtx(
|
|
|
5520
4966
|
return;
|
|
5521
4967
|
}
|
|
5522
4968
|
lastExecutionHeartbeatAt = now;
|
|
5523
|
-
|
|
4969
|
+
enqueueMapProgressUpdate(
|
|
5524
4970
|
{
|
|
5525
4971
|
completed: completedRowsForProgress(),
|
|
5526
4972
|
total: progressTotalRows,
|
|
@@ -5554,7 +5000,7 @@ function createMinimalWorkerCtx(
|
|
|
5554
5000
|
return;
|
|
5555
5001
|
}
|
|
5556
5002
|
lastChunkProgressAt = now;
|
|
5557
|
-
|
|
5003
|
+
enqueueMapProgressUpdate(
|
|
5558
5004
|
{
|
|
5559
5005
|
completed,
|
|
5560
5006
|
total: progressTotalRows,
|
|
@@ -5585,23 +5031,44 @@ function createMinimalWorkerCtx(
|
|
|
5585
5031
|
const failedRowEntries: Array<
|
|
5586
5032
|
{ row: T & Record<string, unknown>; error: string } | undefined
|
|
5587
5033
|
> = new Array(rowsToExecute.length);
|
|
5034
|
+
const fatalReceiptPersistenceRowEntries: Array<
|
|
5035
|
+
{ row: T & Record<string, unknown>; error: string } | undefined
|
|
5036
|
+
> = new Array(rowsToExecute.length);
|
|
5588
5037
|
const executedCellMetaPatches: Array<
|
|
5589
5038
|
Record<string, WorkerCellMetaPatchEntry> | undefined
|
|
5590
5039
|
> = new Array(rowsToExecute.length);
|
|
5591
|
-
const
|
|
5040
|
+
const persistenceLatch = createRuntimePersistenceLatch();
|
|
5041
|
+
const toolBatchScheduler = new WorkerToolBatchScheduler({
|
|
5592
5042
|
req,
|
|
5593
5043
|
governor,
|
|
5594
|
-
resolveToolPacing,
|
|
5044
|
+
resolvePacing: resolveToolPacing,
|
|
5595
5045
|
resolveToolActionCacheVersion,
|
|
5046
|
+
resolveToolAuthScopeDigest,
|
|
5047
|
+
invalidateToolAuthScopeDigest: (toolId) =>
|
|
5048
|
+
resolveToolAuthScopeDigest.invalidate(toolId),
|
|
5049
|
+
onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
|
|
5050
|
+
executeTool: executeDispatchedTool,
|
|
5051
|
+
serializeDurableValue: serializeDurableStepValue,
|
|
5052
|
+
deserializeDurableValue: deserializeDurableStepValue,
|
|
5053
|
+
nowMs,
|
|
5054
|
+
batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
|
|
5055
|
+
recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
|
|
5596
5056
|
abortSignal,
|
|
5597
|
-
reportSettledToolRequests,
|
|
5057
|
+
onRequestsSettled: reportSettledToolRequests,
|
|
5598
5058
|
callbacks,
|
|
5599
5059
|
receiptStore,
|
|
5600
|
-
false,
|
|
5060
|
+
allowLocalRetryReceipts: false,
|
|
5061
|
+
runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
|
|
5062
|
+
runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
|
|
5063
|
+
req,
|
|
5064
|
+
runtimeReceiptLeaseTtlMs(req) ??
|
|
5065
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
5066
|
+
),
|
|
5067
|
+
budgetMeter: workBudgetMeter,
|
|
5601
5068
|
runtimeDeadlineMs,
|
|
5602
5069
|
persistenceLatch,
|
|
5603
5070
|
receiptSalvage,
|
|
5604
|
-
);
|
|
5071
|
+
});
|
|
5605
5072
|
let stepCellsCompleted = 0;
|
|
5606
5073
|
let stepCellsSkipped = 0;
|
|
5607
5074
|
const generatedOutputFields = new Set<string>();
|
|
@@ -5680,6 +5147,8 @@ function createMinimalWorkerCtx(
|
|
|
5680
5147
|
tableNamespace: name,
|
|
5681
5148
|
outputFields,
|
|
5682
5149
|
extraOutputFields: Array.from(generatedOutputFields),
|
|
5150
|
+
budgetMeter: workBudgetMeter,
|
|
5151
|
+
...activeRowAttempt,
|
|
5683
5152
|
rows: [
|
|
5684
5153
|
...rowsToPersist.map(({ row, executedIndex }) =>
|
|
5685
5154
|
mapRowOutcomeRuntimeRow(
|
|
@@ -5743,9 +5212,6 @@ function createMinimalWorkerCtx(
|
|
|
5743
5212
|
});
|
|
5744
5213
|
persistFlushChain = task.catch((error) => {
|
|
5745
5214
|
persistFailure ??= error;
|
|
5746
|
-
// Output-sheet flush failed: trip the breaker so the scheduler stops
|
|
5747
|
-
// dispatching new provider calls for the rest of this map.
|
|
5748
|
-
tripRuntimePersistenceLatch(persistenceLatch, error);
|
|
5749
5215
|
});
|
|
5750
5216
|
return task;
|
|
5751
5217
|
};
|
|
@@ -5787,6 +5253,8 @@ function createMinimalWorkerCtx(
|
|
|
5787
5253
|
outputFields,
|
|
5788
5254
|
extraOutputFields,
|
|
5789
5255
|
updates,
|
|
5256
|
+
...activeRowAttempt,
|
|
5257
|
+
budgetMeter: workBudgetMeter,
|
|
5790
5258
|
});
|
|
5791
5259
|
} catch (error) {
|
|
5792
5260
|
liveUpdateFailures += 1;
|
|
@@ -5894,18 +5362,11 @@ function createMinimalWorkerCtx(
|
|
|
5894
5362
|
request.input,
|
|
5895
5363
|
workflowStep,
|
|
5896
5364
|
{
|
|
5897
|
-
force: request.force === true
|
|
5365
|
+
force: request.force === true,
|
|
5366
|
+
runForceRefresh: req.forceToolRefresh === true,
|
|
5367
|
+
runForceFailedRefresh: req.force === true,
|
|
5898
5368
|
staleAfterSeconds: request.staleAfterSeconds,
|
|
5899
5369
|
},
|
|
5900
|
-
// Parity with the play-level ctx.tools.execute (and the
|
|
5901
|
-
// cjs runner's map path): the public ToolExecutionRequest
|
|
5902
|
-
// runtime options must not be silently dropped for map
|
|
5903
|
-
// rows — receiptWaitMs bounds how long a resumed run
|
|
5904
|
-
// waits on a stuck `running` receipt before reclaiming.
|
|
5905
|
-
{
|
|
5906
|
-
timeoutMs: request.timeoutMs,
|
|
5907
|
-
receiptWaitMs: request.receiptWaitMs,
|
|
5908
|
-
},
|
|
5909
5370
|
);
|
|
5910
5371
|
},
|
|
5911
5372
|
},
|
|
@@ -6014,8 +5475,28 @@ function createMinimalWorkerCtx(
|
|
|
6014
5475
|
notePersistableRow(enriched);
|
|
6015
5476
|
reportChunkProgress(false);
|
|
6016
5477
|
} catch (rowError) {
|
|
6017
|
-
//
|
|
6018
|
-
//
|
|
5478
|
+
// Receipt persistence errors stay run-fatal, but keep the
|
|
5479
|
+
// row identity so teardown can make the recovery state loud.
|
|
5480
|
+
if (isRuntimeReceiptPersistenceError(rowError)) {
|
|
5481
|
+
const message = formatWorkerRowFailureMessage(rowError);
|
|
5482
|
+
if (activeField) {
|
|
5483
|
+
cellMetaPatch[activeField] = {
|
|
5484
|
+
status: 'failed',
|
|
5485
|
+
stage: activeField,
|
|
5486
|
+
runId: req.runId,
|
|
5487
|
+
error: message,
|
|
5488
|
+
};
|
|
5489
|
+
}
|
|
5490
|
+
executedCellMetaPatches[myIndex] =
|
|
5491
|
+
Object.keys(cellMetaPatch).length > 0
|
|
5492
|
+
? cellMetaPatch
|
|
5493
|
+
: undefined;
|
|
5494
|
+
fatalReceiptPersistenceRowEntries[myIndex] = {
|
|
5495
|
+
row: enriched as T & Record<string, unknown>,
|
|
5496
|
+
error: message,
|
|
5497
|
+
};
|
|
5498
|
+
throw rowError;
|
|
5499
|
+
}
|
|
6019
5500
|
if (isRunFatalWorkerRowError(rowError)) {
|
|
6020
5501
|
throw rowError;
|
|
6021
5502
|
}
|
|
@@ -6095,16 +5576,25 @@ function createMinimalWorkerCtx(
|
|
|
6095
5576
|
return results;
|
|
6096
5577
|
},
|
|
6097
5578
|
);
|
|
6098
|
-
|
|
6099
|
-
|
|
6100
|
-
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
|
|
6107
|
-
|
|
5579
|
+
const workerResults = await executeWithRuntimeSheetAttemptHeartbeat({
|
|
5580
|
+
req,
|
|
5581
|
+
tableNamespace: name,
|
|
5582
|
+
sheetContract: requireSheetContract(req, name),
|
|
5583
|
+
...activeRowAttempt,
|
|
5584
|
+
rowKeys: uniqueRowsToExecuteEntries.map((entry) => entry.rowKey),
|
|
5585
|
+
execute: async () => {
|
|
5586
|
+
while (!workerSettled) {
|
|
5587
|
+
await Promise.race([
|
|
5588
|
+
workerResultsPromise,
|
|
5589
|
+
sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
|
|
5590
|
+
]);
|
|
5591
|
+
if (!workerSettled) {
|
|
5592
|
+
reportExecutionHeartbeat(false);
|
|
5593
|
+
}
|
|
5594
|
+
}
|
|
5595
|
+
return await workerResultsPromise;
|
|
5596
|
+
},
|
|
5597
|
+
});
|
|
6108
5598
|
recordRunnerPerfTrace({
|
|
6109
5599
|
req,
|
|
6110
5600
|
phase: 'runner.map_chunk.execute_workers',
|
|
@@ -6133,26 +5623,89 @@ function createMinimalWorkerCtx(
|
|
|
6133
5623
|
.slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
|
|
6134
5624
|
const fatalMapChunkSummary = async (
|
|
6135
5625
|
error: unknown,
|
|
6136
|
-
): Promise<
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
|
|
6147
|
-
|
|
6148
|
-
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
5626
|
+
): Promise<
|
|
5627
|
+
WorkerMapCompletedChunkSummary<T & Record<string, unknown>>
|
|
5628
|
+
> => {
|
|
5629
|
+
const fatalError = formatWorkerRowFailureMessage(error);
|
|
5630
|
+
const breakerMessage = persistenceLatch.tripped
|
|
5631
|
+
? ` Circuit breaker prevented ${persistenceLatch.preventedCallCount} queued provider call(s).`
|
|
5632
|
+
: '';
|
|
5633
|
+
return {
|
|
5634
|
+
status: 'completed',
|
|
5635
|
+
chunkIndex,
|
|
5636
|
+
rangeStart: baseOffset + chunkStart,
|
|
5637
|
+
rangeEnd: baseOffset + chunkStart,
|
|
5638
|
+
rowsRead: chunkRows.length,
|
|
5639
|
+
rowsWritten: 0,
|
|
5640
|
+
rowsExecuted: rowsToExecute.length,
|
|
5641
|
+
rowsCached: 0,
|
|
5642
|
+
rowsDuplicateReused: duplicateInputReuseCount,
|
|
5643
|
+
rowsInserted,
|
|
5644
|
+
rowsSkipped,
|
|
5645
|
+
rowsFailed: failedExecutedRows,
|
|
5646
|
+
rowFailureSamples: buildRowFailureSamples(),
|
|
5647
|
+
stepCellsCompleted,
|
|
5648
|
+
stepCellsSkipped,
|
|
5649
|
+
outputDatasetId: `map:${name}`,
|
|
5650
|
+
hash: await hashJson([]),
|
|
5651
|
+
fatalError: `${fatalError}${breakerMessage}`,
|
|
5652
|
+
preview: [],
|
|
5653
|
+
};
|
|
5654
|
+
};
|
|
5655
|
+
const markReceiptPersistenceFatalRowsFailed = async (error: unknown) => {
|
|
5656
|
+
const message = formatWorkerRowFailureMessage(error);
|
|
5657
|
+
const rows = uniqueRowsToExecuteEntries
|
|
5658
|
+
.map((entry, executedIndex) => {
|
|
5659
|
+
const failure = fatalReceiptPersistenceRowEntries[executedIndex];
|
|
5660
|
+
const row =
|
|
5661
|
+
failure?.row ??
|
|
5662
|
+
executedRows[executedIndex] ??
|
|
5663
|
+
(runtimeCsvExecutionRow(
|
|
5664
|
+
entry.row,
|
|
5665
|
+
pendingRowsByKey.get(entry.rowKey),
|
|
5666
|
+
) as T & Record<string, unknown>);
|
|
5667
|
+
return mapRowOutcomeRuntimeRow(
|
|
5668
|
+
failedMapRowOutcome({
|
|
5669
|
+
key: entry.rowKey,
|
|
5670
|
+
inputIndex: entry.absoluteIndex,
|
|
5671
|
+
data: row,
|
|
5672
|
+
cellMetaPatch: executedCellMetaPatches[executedIndex],
|
|
5673
|
+
error: failure?.error ?? message,
|
|
5674
|
+
}),
|
|
5675
|
+
);
|
|
5676
|
+
})
|
|
5677
|
+
.filter((row): row is Record<string, unknown> => row !== null);
|
|
5678
|
+
if (rows.length === 0) return;
|
|
5679
|
+
try {
|
|
5680
|
+
const failureStats = await persistCompletedMapRows({
|
|
5681
|
+
req,
|
|
5682
|
+
tableNamespace: name,
|
|
5683
|
+
outputFields,
|
|
5684
|
+
extraOutputFields: Array.from(generatedOutputFields),
|
|
5685
|
+
budgetMeter: workBudgetMeter,
|
|
5686
|
+
...activeRowAttempt,
|
|
5687
|
+
forceFailedRows: true,
|
|
5688
|
+
rows,
|
|
5689
|
+
});
|
|
5690
|
+
emitEvent({
|
|
5691
|
+
type: 'log',
|
|
5692
|
+
level: 'warn',
|
|
5693
|
+
message:
|
|
5694
|
+
`Runtime sheet marked ${failureStats.rows} row(s) failed for ctx.dataset("${name}") ` +
|
|
5695
|
+
`after a run-fatal receipt persistence error.`,
|
|
5696
|
+
ts: nowMs(),
|
|
5697
|
+
});
|
|
5698
|
+
} catch (cleanupError) {
|
|
5699
|
+
emitEvent({
|
|
5700
|
+
type: 'log',
|
|
5701
|
+
level: 'warn',
|
|
5702
|
+
message:
|
|
5703
|
+
`Runtime sheet failed-row cleanup after receipt persistence error failed for ctx.dataset("${name}"): ` +
|
|
5704
|
+
formatWorkerRowFailureMessage(cleanupError),
|
|
5705
|
+
ts: nowMs(),
|
|
5706
|
+
});
|
|
5707
|
+
}
|
|
5708
|
+
};
|
|
6156
5709
|
const persistRowsStartedAt = nowMs();
|
|
6157
5710
|
recordRunnerPerfTrace({
|
|
6158
5711
|
req,
|
|
@@ -6181,6 +5734,7 @@ function createMinimalWorkerCtx(
|
|
|
6181
5734
|
},
|
|
6182
5735
|
});
|
|
6183
5736
|
} catch (error) {
|
|
5737
|
+
tripRuntimePersistenceLatch(persistenceLatch, error);
|
|
6184
5738
|
recordRunnerPerfTrace({
|
|
6185
5739
|
req,
|
|
6186
5740
|
phase: 'runner.map_chunk.persist_rows_error',
|
|
@@ -6199,31 +5753,24 @@ function createMinimalWorkerCtx(
|
|
|
6199
5753
|
result.status === 'rejected',
|
|
6200
5754
|
);
|
|
6201
5755
|
if (rejectedWorker) {
|
|
6202
|
-
if (
|
|
6203
|
-
|
|
6204
|
-
|
|
6205
|
-
|
|
6206
|
-
|
|
6207
|
-
// fatal summary (NOT rethrown into the durable step) so the chunk is
|
|
6208
|
-
// not retried and provider calls are not re-billed.
|
|
5756
|
+
if (isRuntimeReceiptPersistenceError(rejectedWorker.reason)) {
|
|
5757
|
+
await markReceiptPersistenceFatalRowsFailed(rejectedWorker.reason);
|
|
5758
|
+
return await fatalMapChunkSummary(rejectedWorker.reason);
|
|
5759
|
+
}
|
|
5760
|
+
if (isRuntimePersistenceCircuitOpenError(rejectedWorker.reason)) {
|
|
6209
5761
|
return await fatalMapChunkSummary(rejectedWorker.reason);
|
|
6210
5762
|
}
|
|
6211
5763
|
throw rejectedWorker.reason;
|
|
6212
5764
|
}
|
|
6213
5765
|
const resultByKey = new Map<string, T & Record<string, unknown>>();
|
|
6214
|
-
for (const
|
|
6215
|
-
const
|
|
6216
|
-
|
|
6217
|
-
|
|
6218
|
-
|
|
6219
|
-
|
|
6220
|
-
|
|
6221
|
-
|
|
6222
|
-
const cleanedRow = stripMapRowOutcomeRuntimeFields(
|
|
6223
|
-
publicCsvOutputRow(completedRow),
|
|
6224
|
-
);
|
|
6225
|
-
resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
|
|
6226
|
-
}
|
|
5766
|
+
for (const entry of completedChunkEntries) {
|
|
5767
|
+
const completedRow = completedRowsByKey.get(entry.rowKey);
|
|
5768
|
+
if (!completedRow || resultByKey.has(entry.rowKey)) continue;
|
|
5769
|
+
resultByKey.set(
|
|
5770
|
+
entry.rowKey,
|
|
5771
|
+
cloneCsvAliasedRow(entry.row, completedRow) as T &
|
|
5772
|
+
Record<string, unknown>,
|
|
5773
|
+
);
|
|
6227
5774
|
}
|
|
6228
5775
|
for (
|
|
6229
5776
|
let executedIndex = 0;
|
|
@@ -6258,16 +5805,13 @@ function createMinimalWorkerCtx(
|
|
|
6258
5805
|
} => entry !== null,
|
|
6259
5806
|
);
|
|
6260
5807
|
const out = outEntries.map((entry) => entry.row);
|
|
6261
|
-
const executedSuccessCount = Math.max(
|
|
6262
|
-
0,
|
|
6263
|
-
executedRows.length - failedExecutedRows,
|
|
6264
|
-
);
|
|
6265
5808
|
const rowFailureSamples = buildRowFailureSamples();
|
|
6266
5809
|
const publicOut = out.map((row) => publicCsvOutputRow(row));
|
|
6267
5810
|
const keyedOut = outEntries.map(({ key, inputIndex, row }) => ({
|
|
6268
5811
|
...row,
|
|
6269
5812
|
...mapRowOutcomeRuntimeFields({ key, inputIndex }),
|
|
6270
5813
|
}));
|
|
5814
|
+
const completedRowsReused = completedChunkEntries.length;
|
|
6271
5815
|
const hashStartedAt = nowMs();
|
|
6272
5816
|
const hash = await hashJson(publicOut);
|
|
6273
5817
|
const includeCachedRowsInChunkResult = !workflowStep;
|
|
@@ -6298,17 +5842,18 @@ function createMinimalWorkerCtx(
|
|
|
6298
5842
|
rowsWritten: out.length,
|
|
6299
5843
|
rowsExecuted: executedRows.length,
|
|
6300
5844
|
rowsFailed: failedExecutedRows,
|
|
6301
|
-
rowsCached:
|
|
5845
|
+
rowsCached: completedRowsReused,
|
|
6302
5846
|
},
|
|
6303
5847
|
});
|
|
6304
5848
|
return {
|
|
5849
|
+
status: 'completed',
|
|
6305
5850
|
chunkIndex,
|
|
6306
5851
|
rangeStart: baseOffset + chunkStart,
|
|
6307
5852
|
rangeEnd: baseOffset + chunkStart + out.length,
|
|
6308
5853
|
rowsRead: chunkRows.length,
|
|
6309
5854
|
rowsWritten: out.length,
|
|
6310
5855
|
rowsExecuted: executedRows.length,
|
|
6311
|
-
rowsCached:
|
|
5856
|
+
rowsCached: completedRowsReused,
|
|
6312
5857
|
rowsDuplicateReused: duplicateInputReuseCount,
|
|
6313
5858
|
rowsInserted,
|
|
6314
5859
|
rowsSkipped,
|
|
@@ -6340,9 +5885,12 @@ function createMinimalWorkerCtx(
|
|
|
6340
5885
|
let totalRowsInserted = 0;
|
|
6341
5886
|
let totalRowsSkipped = 0;
|
|
6342
5887
|
let totalRowsFailed = 0;
|
|
5888
|
+
let totalRowsBlocked = 0;
|
|
6343
5889
|
let totalStepCellsCompleted = 0;
|
|
6344
5890
|
let totalStepCellsSkipped = 0;
|
|
6345
5891
|
const totalRowFailureSamples: Array<{ rowKey: string; error: string }> = [];
|
|
5892
|
+
const blockedRowKeySamples: string[] = [];
|
|
5893
|
+
const blockedRowSamples: Record<string, unknown>[] = [];
|
|
6346
5894
|
|
|
6347
5895
|
const runChunkStep = async (
|
|
6348
5896
|
chunkRows: T[],
|
|
@@ -6372,6 +5920,7 @@ function createMinimalWorkerCtx(
|
|
|
6372
5920
|
const readPersistedRows = async (input: {
|
|
6373
5921
|
limit: number;
|
|
6374
5922
|
offset: number;
|
|
5923
|
+
rowMode?: 'output' | 'all';
|
|
6375
5924
|
}) => {
|
|
6376
5925
|
const result = await harnessReadSheetDatasetRows({
|
|
6377
5926
|
baseUrl: req.baseUrl,
|
|
@@ -6380,6 +5929,7 @@ function createMinimalWorkerCtx(
|
|
|
6380
5929
|
playName: req.playName,
|
|
6381
5930
|
tableNamespace: name,
|
|
6382
5931
|
runId: req.runId,
|
|
5932
|
+
rowMode: input.rowMode,
|
|
6383
5933
|
limit: input.limit,
|
|
6384
5934
|
offset: input.offset,
|
|
6385
5935
|
userEmail: req.userEmail,
|
|
@@ -6388,19 +5938,61 @@ function createMinimalWorkerCtx(
|
|
|
6388
5938
|
return result.rows as Array<T & Record<string, unknown>>;
|
|
6389
5939
|
};
|
|
6390
5940
|
|
|
6391
|
-
const finalize = async (
|
|
5941
|
+
const finalize = async () => {
|
|
5942
|
+
let finalizedRowsWritten = totalRowsWritten;
|
|
5943
|
+
if (
|
|
5944
|
+
totalRowsFailed === 0 &&
|
|
5945
|
+
rowCountHint !== null &&
|
|
5946
|
+
finalizedRowsWritten > 0 &&
|
|
5947
|
+
finalizedRowsWritten < rowCountHint &&
|
|
5948
|
+
rowCountHint <= WORKER_DATASET_IN_MEMORY_ROWS
|
|
5949
|
+
) {
|
|
5950
|
+
const persistedRows = await readPersistedRows({
|
|
5951
|
+
limit: rowCountHint,
|
|
5952
|
+
offset: 0,
|
|
5953
|
+
rowMode: 'all',
|
|
5954
|
+
});
|
|
5955
|
+
if (persistedRows.length > finalizedRowsWritten) {
|
|
5956
|
+
const inMemoryRowsWritten = finalizedRowsWritten;
|
|
5957
|
+
finalizedRowsWritten = persistedRows.length;
|
|
5958
|
+
totalRowsWritten = finalizedRowsWritten;
|
|
5959
|
+
cachedRows.length = 0;
|
|
5960
|
+
cachedRows.push(...persistedRows);
|
|
5961
|
+
canCacheRows = true;
|
|
5962
|
+
previewRows.length = 0;
|
|
5963
|
+
previewRows.push(
|
|
5964
|
+
...persistedRows.slice(0, WORKER_DATASET_PREVIEW_ROWS),
|
|
5965
|
+
);
|
|
5966
|
+
emitEvent({
|
|
5967
|
+
type: 'log',
|
|
5968
|
+
level: 'warn',
|
|
5969
|
+
message:
|
|
5970
|
+
`Runtime sheet finalization reconciled ctx.dataset("${name}") ` +
|
|
5971
|
+
`from ${inMemoryRowsWritten}/${rowCountHint} in-memory row(s) ` +
|
|
5972
|
+
`to ${finalizedRowsWritten} visible persisted row(s).`,
|
|
5973
|
+
ts: nowMs(),
|
|
5974
|
+
});
|
|
5975
|
+
}
|
|
5976
|
+
if (persistedRows.length < rowCountHint) {
|
|
5977
|
+
throw new Error(
|
|
5978
|
+
`Runtime sheet finalization mismatch ctx.dataset("${name}"): ` +
|
|
5979
|
+
`expected ${rowCountHint}, saw ${persistedRows.length}; ` +
|
|
5980
|
+
`sum ${finalizedRowsWritten}; ${req.runId}`,
|
|
5981
|
+
);
|
|
5982
|
+
}
|
|
5983
|
+
}
|
|
6392
5984
|
const failureSampleSummary =
|
|
6393
5985
|
totalRowFailureSamples.length > 0
|
|
6394
5986
|
? ` First error: ${totalRowFailureSamples[0]!.error}`
|
|
6395
5987
|
: '';
|
|
6396
5988
|
const cacheSummary =
|
|
6397
5989
|
totalRowsFailed > 0
|
|
6398
|
-
? `Map completed with partial failures: ${
|
|
6399
|
-
`${totalRowsFailed} failed (${totalRowsExecuted} executed
|
|
5990
|
+
? `Map completed with partial failures: ${finalizedRowsWritten} succeeded, ` +
|
|
5991
|
+
`${totalRowsFailed} failed (${totalRowsExecuted} executed) ` +
|
|
6400
5992
|
`inserted=${totalRowsInserted} skipped=${totalRowsSkipped}. ` +
|
|
6401
5993
|
`Failed rows are persisted with their errors and re-execute on the next run.${failureSampleSummary}`
|
|
6402
|
-
: `Map completed: ${
|
|
6403
|
-
`(${totalRowsExecuted} executed
|
|
5994
|
+
: `Map completed: ${finalizedRowsWritten} results ` +
|
|
5995
|
+
`(${totalRowsExecuted} executed) ` +
|
|
6404
5996
|
`inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
|
|
6405
5997
|
const completedAt = nowMs();
|
|
6406
5998
|
const totalStepCells = totalStepCellsCompleted + totalStepCellsSkipped;
|
|
@@ -6408,17 +6000,21 @@ function createMinimalWorkerCtx(
|
|
|
6408
6000
|
totalStepCells > 0
|
|
6409
6001
|
? ` cells=${totalStepCells}/${totalStepCellsCompleted}/${totalStepCellsSkipped}`
|
|
6410
6002
|
: '';
|
|
6003
|
+
await flushQueuedMapProgressUpdates();
|
|
6411
6004
|
callbacks?.onMapCompleted?.(mapNodeId, completedAt);
|
|
6412
6005
|
await updateMapProgress({
|
|
6413
|
-
completed:
|
|
6414
|
-
total:
|
|
6006
|
+
completed: finalizedRowsWritten,
|
|
6007
|
+
total: finalizedRowsWritten + totalRowsFailed,
|
|
6415
6008
|
failed: totalRowsFailed,
|
|
6416
6009
|
completedAt,
|
|
6417
6010
|
updatedAt: completedAt,
|
|
6418
6011
|
message:
|
|
6419
6012
|
totalRowsFailed > 0
|
|
6420
|
-
? `${
|
|
6421
|
-
: formatMapProgressMessage(
|
|
6013
|
+
? `${finalizedRowsWritten.toLocaleString()} succeeded, ${totalRowsFailed.toLocaleString()} failed`
|
|
6014
|
+
: formatMapProgressMessage(
|
|
6015
|
+
finalizedRowsWritten,
|
|
6016
|
+
finalizedRowsWritten,
|
|
6017
|
+
),
|
|
6422
6018
|
});
|
|
6423
6019
|
emitEvent({
|
|
6424
6020
|
type: 'log',
|
|
@@ -6427,29 +6023,61 @@ function createMinimalWorkerCtx(
|
|
|
6427
6023
|
ts: nowMs(),
|
|
6428
6024
|
});
|
|
6429
6025
|
if (
|
|
6430
|
-
|
|
6026
|
+
finalizedRowsWritten > 0 &&
|
|
6431
6027
|
totalRowsFailed === 0 &&
|
|
6432
6028
|
canCacheRows &&
|
|
6433
|
-
cachedRows.length ===
|
|
6029
|
+
cachedRows.length === finalizedRowsWritten
|
|
6434
6030
|
) {
|
|
6435
6031
|
const persistedRows = await readPersistedRows({
|
|
6436
|
-
limit:
|
|
6032
|
+
limit: finalizedRowsWritten,
|
|
6437
6033
|
offset: 0,
|
|
6438
6034
|
});
|
|
6439
|
-
if (persistedRows.length <
|
|
6035
|
+
if (persistedRows.length < finalizedRowsWritten) {
|
|
6036
|
+
const repairAttempt = makeRuntimeSheetAttempt({
|
|
6037
|
+
runId: req.runId,
|
|
6038
|
+
runAttempt: req.runAttempt,
|
|
6039
|
+
mapName: name,
|
|
6040
|
+
chunkIndex: 'finalize',
|
|
6041
|
+
});
|
|
6042
|
+
const repairRows = cachedRows.map((row) => runtimeCsvStorageRow(row));
|
|
6043
|
+
workBudgetMeter.count('sheet');
|
|
6044
|
+
const repairStart = await harnessStartSheetDataset({
|
|
6045
|
+
...runtimeSheetSessionScope(req),
|
|
6046
|
+
tableNamespace: name,
|
|
6047
|
+
sheetContract: augmentSheetContractWithDatasetFields({
|
|
6048
|
+
contract: requireSheetContract(req, name),
|
|
6049
|
+
rows: repairRows,
|
|
6050
|
+
outputFields,
|
|
6051
|
+
}),
|
|
6052
|
+
rows: repairRows,
|
|
6053
|
+
runId: req.runId,
|
|
6054
|
+
inputOffset: 0,
|
|
6055
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
6056
|
+
...repairAttempt,
|
|
6057
|
+
});
|
|
6058
|
+
const activeRepairAttempt = {
|
|
6059
|
+
attemptId: repairStart.attemptId ?? repairAttempt.attemptId,
|
|
6060
|
+
attemptOwnerRunId:
|
|
6061
|
+
repairStart.attemptOwnerRunId ?? repairAttempt.attemptOwnerRunId,
|
|
6062
|
+
attemptExpiresAt:
|
|
6063
|
+
repairStart.attemptExpiresAt ?? repairAttempt.attemptExpiresAt,
|
|
6064
|
+
attemptSeq: repairStart.attemptSeq ?? repairAttempt.attemptSeq,
|
|
6065
|
+
};
|
|
6440
6066
|
const repair = await persistCompletedMapRows({
|
|
6441
6067
|
req,
|
|
6442
6068
|
tableNamespace: name,
|
|
6443
6069
|
outputFields,
|
|
6444
6070
|
extraOutputFields: [],
|
|
6445
6071
|
rows: cachedRows,
|
|
6072
|
+
...activeRepairAttempt,
|
|
6073
|
+
budgetMeter: workBudgetMeter,
|
|
6446
6074
|
});
|
|
6447
6075
|
emitEvent({
|
|
6448
6076
|
type: 'log',
|
|
6449
6077
|
level: 'warn',
|
|
6450
6078
|
message:
|
|
6451
6079
|
`Runtime sheet finalization repaired ctx.dataset("${name}") ` +
|
|
6452
|
-
`from ${persistedRows.length}/${
|
|
6080
|
+
`from ${persistedRows.length}/${finalizedRowsWritten} visible row(s) ` +
|
|
6453
6081
|
`(written=${repair.written}, visible=${repair.visible}` +
|
|
6454
6082
|
(repair.retryWritten === null
|
|
6455
6083
|
? ''
|
|
@@ -6465,7 +6093,7 @@ function createMinimalWorkerCtx(
|
|
|
6465
6093
|
return createPersistedDatasetHandle({
|
|
6466
6094
|
playName: req.playName,
|
|
6467
6095
|
name,
|
|
6468
|
-
count:
|
|
6096
|
+
count: finalizedRowsWritten,
|
|
6469
6097
|
// In native Workflows, chunk summaries intentionally omit full row
|
|
6470
6098
|
// payloads, so this preview only contains the bounded chunk samples.
|
|
6471
6099
|
// Do not synchronously page rows back here: service-binding reads have
|
|
@@ -6479,7 +6107,7 @@ function createMinimalWorkerCtx(
|
|
|
6479
6107
|
recordRunnerPerfTrace({ req, phase, ms, extra }),
|
|
6480
6108
|
nowMs,
|
|
6481
6109
|
workProgress: {
|
|
6482
|
-
total:
|
|
6110
|
+
total: finalizedRowsWritten + totalRowsFailed,
|
|
6483
6111
|
executed: totalRowsExecuted,
|
|
6484
6112
|
reused: totalRowsCached,
|
|
6485
6113
|
skipped: totalRowsCached,
|
|
@@ -6493,97 +6121,248 @@ function createMinimalWorkerCtx(
|
|
|
6493
6121
|
};
|
|
6494
6122
|
|
|
6495
6123
|
const failFastRowErrors = opts?.onRowError === 'fail';
|
|
6496
|
-
|
|
6497
|
-
|
|
6498
|
-
|
|
6499
|
-
|
|
6500
|
-
|
|
6501
|
-
|
|
6502
|
-
|
|
6503
|
-
|
|
6504
|
-
|
|
6505
|
-
|
|
6506
|
-
|
|
6507
|
-
|
|
6508
|
-
|
|
6509
|
-
|
|
6510
|
-
|
|
6511
|
-
|
|
6512
|
-
|
|
6513
|
-
|
|
6514
|
-
|
|
6515
|
-
|
|
6516
|
-
|
|
6517
|
-
|
|
6518
|
-
|
|
6519
|
-
|
|
6520
|
-
|
|
6521
|
-
|
|
6522
|
-
|
|
6523
|
-
|
|
6524
|
-
|
|
6525
|
-
`
|
|
6526
|
-
|
|
6527
|
-
);
|
|
6528
|
-
|
|
6529
|
-
|
|
6530
|
-
|
|
6531
|
-
|
|
6532
|
-
|
|
6533
|
-
|
|
6534
|
-
|
|
6535
|
-
|
|
6536
|
-
|
|
6537
|
-
|
|
6538
|
-
|
|
6539
|
-
|
|
6540
|
-
|
|
6124
|
+
// Bounded-concurrent chunk dispatch is only sound off the native Workflows
|
|
6125
|
+
// path: with a `workflowStep` each chunk is a durable step whose sequential
|
|
6126
|
+
// order makes replay/resume deterministic, so that path stays at K=1.
|
|
6127
|
+
// Chunk-level concurrency is otherwise safe under workers_edge because each
|
|
6128
|
+
// chunk owns a disjoint sheet row range with its own runtime-sheet attempt,
|
|
6129
|
+
// receipts complete inside each chunk's persist barrier, and child submits
|
|
6130
|
+
// use per-row durable receipt keys. It is disabled under onRowError:'fail'
|
|
6131
|
+
// on purpose: that mode relies on a first-failure short-circuit to skip the
|
|
6132
|
+
// remaining chunks and stop spending provider credits, and eager concurrent
|
|
6133
|
+
// launch would already have K-1 chunks in flight (credits spent) before the
|
|
6134
|
+
// short-circuit fires.
|
|
6135
|
+
const canRunChunksConcurrently = !workflowStep && !failFastRowErrors;
|
|
6136
|
+
const maxConcurrentMapChunks = canRunChunksConcurrently
|
|
6137
|
+
? WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS
|
|
6138
|
+
: 1;
|
|
6139
|
+
const runWorkDispatcher = new WorkerRunWorkDispatcher({
|
|
6140
|
+
budgetMeter: workBudgetMeter,
|
|
6141
|
+
nowMs,
|
|
6142
|
+
yieldLimits: {
|
|
6143
|
+
elapsed: governor.policy.pacing.workerYieldElapsedMs,
|
|
6144
|
+
...(req.testPolicyOverrides?.workBudgetYieldLimits ?? {}),
|
|
6145
|
+
},
|
|
6146
|
+
recordTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
|
|
6147
|
+
yieldBetweenChunks: async (yieldPoint) => {
|
|
6148
|
+
emitEvent({
|
|
6149
|
+
type: 'log',
|
|
6150
|
+
level: 'info',
|
|
6151
|
+
message:
|
|
6152
|
+
`Runtime dispatcher yielded ctx.dataset("${name}") before chunk ${yieldPoint.nextChunkIndex} ` +
|
|
6153
|
+
`(${yieldPoint.reason} budget).`,
|
|
6154
|
+
ts: nowMs(),
|
|
6155
|
+
});
|
|
6156
|
+
if (workflowStep) {
|
|
6157
|
+
// Cloudflare Workflows durability model: `step.sleep` writes a
|
|
6158
|
+
// durable checkpoint, and the resume re-enters `run()` as a NEW
|
|
6159
|
+
// Worker invocation. Cloudflare's subrequest ceiling is
|
|
6160
|
+
// per-invocation, and on resume the already-completed `step.do`
|
|
6161
|
+
// chunks are served from the durable step cache WITHOUT re-issuing
|
|
6162
|
+
// their subrequests (replay semantics) — so the post-yield chunk
|
|
6163
|
+
// runs against a fresh per-invocation subrequest budget.
|
|
6164
|
+
//
|
|
6165
|
+
// Isolate identity is the WRONG freshness proxy: Cloudflare routinely
|
|
6166
|
+
// reuses warm isolates across invocations, so a genuinely
|
|
6167
|
+
// fresh-budget resume commonly shares the pre-sleep isolate id (this
|
|
6168
|
+
// is exactly why the isolate-identity check reported `false` every
|
|
6169
|
+
// time and hard-failed multi-chunk maps). Crossing the durable
|
|
6170
|
+
// `step.sleep` boundary IS the fresh-budget signal.
|
|
6171
|
+
const sleepMs = WORK_DISPATCHER_YIELD_SLEEP_MS;
|
|
6172
|
+
await (
|
|
6173
|
+
workflowStep.sleep as unknown as (
|
|
6174
|
+
name: string,
|
|
6175
|
+
duration: number,
|
|
6176
|
+
) => Promise<void>
|
|
6177
|
+
)(
|
|
6178
|
+
`work-dispatcher:${name}:${yieldPoint.nextChunkIndex}:${yieldPoint.attempt}`,
|
|
6179
|
+
sleepMs,
|
|
6180
|
+
);
|
|
6181
|
+
recordRunnerPerfTrace({
|
|
6182
|
+
req,
|
|
6183
|
+
phase: 'runner.work_dispatcher.yield_probe',
|
|
6184
|
+
ms: sleepMs,
|
|
6185
|
+
extra: {
|
|
6186
|
+
mapName: name,
|
|
6187
|
+
nextChunkIndex: yieldPoint.nextChunkIndex,
|
|
6188
|
+
reason: yieldPoint.reason,
|
|
6189
|
+
attempt: yieldPoint.attempt,
|
|
6190
|
+
freshBudget: true,
|
|
6191
|
+
invocationBoundary: 'workflow_step_sleep',
|
|
6192
|
+
},
|
|
6193
|
+
});
|
|
6194
|
+
return { freshBudget: true };
|
|
6541
6195
|
}
|
|
6542
|
-
|
|
6543
|
-
|
|
6544
|
-
|
|
6545
|
-
|
|
6546
|
-
|
|
6547
|
-
|
|
6548
|
-
|
|
6549
|
-
|
|
6550
|
-
|
|
6551
|
-
|
|
6552
|
-
|
|
6553
|
-
|
|
6554
|
-
|
|
6555
|
-
|
|
6196
|
+
// No durable Workflow step is available to force a new invocation here,
|
|
6197
|
+
// so we cannot obtain a fresh per-invocation platform budget. Report a
|
|
6198
|
+
// non-fresh yield: the dispatcher escalates and then fails loudly with
|
|
6199
|
+
// WorkerBudgetExhaustedError rather than silently overflowing the
|
|
6200
|
+
// subrequest ceiling mid-map.
|
|
6201
|
+
await sleepWorkerMs(0);
|
|
6202
|
+
return { freshBudget: false };
|
|
6203
|
+
},
|
|
6204
|
+
});
|
|
6205
|
+
let dispatchResult: WorkerRunMapBatchesResult;
|
|
6206
|
+
try {
|
|
6207
|
+
dispatchResult = await runWorkDispatcher.runMapBatches({
|
|
6208
|
+
mapName: name,
|
|
6209
|
+
chunks: iterDatasetChunks(inputRows, rowsPerChunk),
|
|
6210
|
+
maxConcurrentChunks: maxConcurrentMapChunks,
|
|
6211
|
+
maxResidentBatchBytes: WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES,
|
|
6212
|
+
prepareBatch: (rawChunkRows, chunkStart) => {
|
|
6213
|
+
assertNotAborted(abortSignal);
|
|
6214
|
+
if (rawChunkRows.length === 0) return [];
|
|
6215
|
+
// Drop duplicate explicit-key rows before anything downstream observes
|
|
6216
|
+
// them. `chunkStart` keeps advancing by the original (pre-dedupe) chunk
|
|
6217
|
+
// length so cross-chunk key indices and persisted input offsets stay
|
|
6218
|
+
// aligned to the original input stream.
|
|
6219
|
+
return dedupeExplicitRowKeys(rawChunkRows, chunkStart);
|
|
6220
|
+
},
|
|
6221
|
+
estimateRawBatchResidentBytes: (rawChunkRows) =>
|
|
6222
|
+
rawChunkRows.reduce(
|
|
6223
|
+
(total, row) => total + workerMapResidentRowBytes(row),
|
|
6224
|
+
0,
|
|
6225
|
+
),
|
|
6226
|
+
executeBatch: async ({ rows: chunkRows, chunkStart, chunkIndex }) => {
|
|
6227
|
+
assertNotAborted(abortSignal);
|
|
6228
|
+
return await runChunkStep(chunkRows, chunkStart, chunkIndex);
|
|
6229
|
+
},
|
|
6230
|
+
estimateBatchResidentBytes: ({ rows: chunkRows }) =>
|
|
6231
|
+
chunkRows.reduce(
|
|
6232
|
+
(total, row) => total + workerMapResidentRowBytes(row),
|
|
6556
6233
|
0,
|
|
6557
|
-
WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
|
|
6558
6234
|
),
|
|
6235
|
+
estimateBatchSubrequests: ({ rows: chunkRows }) => {
|
|
6236
|
+
const estimate = mapDispatchPlan.workEstimate;
|
|
6237
|
+
const unbatchedProviderSubrequests =
|
|
6238
|
+
estimate.unbatchedProviderToolCallsPerRow *
|
|
6239
|
+
WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL;
|
|
6240
|
+
// Static planning cannot know input-dependent batching buckets, so
|
|
6241
|
+
// this projects worst-case provider-batch fanout while still treating
|
|
6242
|
+
// durable receipt claim/complete/fail as bulk per drained tool group.
|
|
6243
|
+
const batchedProviderSubrequests =
|
|
6244
|
+
estimateBatchedToolChunkSubrequests({
|
|
6245
|
+
batchableToolCallsPerRow: Math.max(
|
|
6246
|
+
0,
|
|
6247
|
+
estimate.providerToolCallsPerRow -
|
|
6248
|
+
estimate.unbatchedProviderToolCallsPerRow,
|
|
6249
|
+
),
|
|
6250
|
+
coalescedBatchableToolCallsPerRow:
|
|
6251
|
+
estimate.coalescedBatchableProviderToolCallsPerRow,
|
|
6252
|
+
rows: chunkRows.length,
|
|
6253
|
+
assumedBatchSize: estimate.assumedBatchSize,
|
|
6254
|
+
});
|
|
6255
|
+
const controlSubrequests =
|
|
6256
|
+
estimate.childPlaySubmitsPerRow + estimate.eventWaitsPerRow;
|
|
6257
|
+
return (
|
|
6258
|
+
chunkRows.length *
|
|
6259
|
+
(unbatchedProviderSubrequests + controlSubrequests) +
|
|
6260
|
+
batchedProviderSubrequests +
|
|
6261
|
+
estimate.runtimeSheetReadsPerChunk +
|
|
6262
|
+
estimate.runtimeSheetWritesPerChunk
|
|
6263
|
+
);
|
|
6264
|
+
},
|
|
6265
|
+
onBatchComplete: async (chunkResult) => {
|
|
6266
|
+
if (chunkResult.status === 'blocked') {
|
|
6267
|
+
totalRowsBlocked += chunkResult.rowsBlocked;
|
|
6268
|
+
for (const key of chunkResult.rowKeySamples) {
|
|
6269
|
+
if (blockedRowKeySamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6270
|
+
break;
|
|
6271
|
+
}
|
|
6272
|
+
blockedRowKeySamples.push(key);
|
|
6273
|
+
}
|
|
6274
|
+
for (const row of chunkResult.blockedRowSamples) {
|
|
6275
|
+
if (blockedRowSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6276
|
+
break;
|
|
6277
|
+
}
|
|
6278
|
+
blockedRowSamples.push(row);
|
|
6279
|
+
}
|
|
6280
|
+
emitEvent({
|
|
6281
|
+
type: 'log',
|
|
6282
|
+
level: 'warn',
|
|
6283
|
+
message: chunkResult.message,
|
|
6284
|
+
ts: nowMs(),
|
|
6285
|
+
});
|
|
6286
|
+
return true;
|
|
6287
|
+
}
|
|
6288
|
+
if (chunkResult.fatalError) {
|
|
6289
|
+
throw new Error(
|
|
6290
|
+
`ctx.dataset("${name}") stopped after a runtime persistence failure ` +
|
|
6291
|
+
`outside the retryable chunk step. Provider calls already executed for ` +
|
|
6292
|
+
`${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
|
|
6293
|
+
`Fix the runtime persistence cause and re-run to resume. ` +
|
|
6294
|
+
`First error: ${chunkResult.fatalError}`,
|
|
6295
|
+
);
|
|
6296
|
+
}
|
|
6297
|
+
totalRowsWritten += chunkResult.rowsWritten;
|
|
6298
|
+
totalRowsExecuted += chunkResult.rowsExecuted;
|
|
6299
|
+
totalRowsCached += chunkResult.rowsCached;
|
|
6300
|
+
totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
|
|
6301
|
+
totalRowsInserted += chunkResult.rowsInserted;
|
|
6302
|
+
totalRowsSkipped += chunkResult.rowsSkipped;
|
|
6303
|
+
totalRowsFailed += chunkResult.rowsFailed ?? 0;
|
|
6304
|
+
totalStepCellsCompleted += chunkResult.stepCellsCompleted ?? 0;
|
|
6305
|
+
totalStepCellsSkipped += chunkResult.stepCellsSkipped ?? 0;
|
|
6306
|
+
for (const sample of chunkResult.rowFailureSamples ?? []) {
|
|
6307
|
+
if (totalRowFailureSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6308
|
+
break;
|
|
6309
|
+
}
|
|
6310
|
+
totalRowFailureSamples.push(sample);
|
|
6311
|
+
}
|
|
6312
|
+
enqueueMapProgressUpdate({
|
|
6313
|
+
completed: totalRowsWritten,
|
|
6314
|
+
total: rowCountHint ?? undefined,
|
|
6315
|
+
...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
|
|
6316
|
+
message: formatMapProgressMessage(
|
|
6317
|
+
totalRowsWritten,
|
|
6318
|
+
rowCountHint ?? undefined,
|
|
6319
|
+
),
|
|
6320
|
+
});
|
|
6321
|
+
if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
|
|
6322
|
+
previewRows.push(
|
|
6323
|
+
...chunkResult.preview.slice(
|
|
6324
|
+
0,
|
|
6325
|
+
WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
|
|
6326
|
+
),
|
|
6327
|
+
);
|
|
6328
|
+
}
|
|
6329
|
+
if (canCacheRows) {
|
|
6330
|
+
const volatileRows = volatileWorkflowChunkRows.get(
|
|
6331
|
+
chunkResult.chunkIndex,
|
|
6332
|
+
);
|
|
6333
|
+
volatileWorkflowChunkRows.delete(chunkResult.chunkIndex);
|
|
6334
|
+
const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
|
|
6335
|
+
if (
|
|
6336
|
+
nextRows.length === chunkResult.rowsWritten &&
|
|
6337
|
+
cachedRows.length + nextRows.length <=
|
|
6338
|
+
WORKER_DATASET_IN_MEMORY_ROWS
|
|
6339
|
+
) {
|
|
6340
|
+
cachedRows.push(...nextRows);
|
|
6341
|
+
} else {
|
|
6342
|
+
cachedRows.length = 0;
|
|
6343
|
+
volatileWorkflowChunkRows.clear();
|
|
6344
|
+
canCacheRows = false;
|
|
6345
|
+
}
|
|
6346
|
+
}
|
|
6347
|
+
// onRowError:'fail' short-circuit: once a chunk reports a row failure,
|
|
6348
|
+
// skip the remaining chunks entirely. The failing chunk itself completed
|
|
6349
|
+
// normally (no chunk-step retry storm) and persisted its rows, but
|
|
6350
|
+
// executing later chunks would keep spending provider credits on a run
|
|
6351
|
+
// the caller asked to fail fast. The post-loop fail-fast throw below
|
|
6352
|
+
// reports what committed before the stop.
|
|
6353
|
+
return failFastRowErrors && totalRowsFailed > 0;
|
|
6354
|
+
},
|
|
6355
|
+
});
|
|
6356
|
+
} catch (error) {
|
|
6357
|
+
try {
|
|
6358
|
+
await flushQueuedMapProgressUpdates();
|
|
6359
|
+
} catch (progressError) {
|
|
6360
|
+
throw new AggregateError(
|
|
6361
|
+
[error, progressError],
|
|
6362
|
+
`ctx.dataset("${name}") failed while flushing queued progress updates.`,
|
|
6559
6363
|
);
|
|
6560
6364
|
}
|
|
6561
|
-
|
|
6562
|
-
const volatileRows = volatileWorkflowChunkRows.get(chunkIndex);
|
|
6563
|
-
volatileWorkflowChunkRows.delete(chunkIndex);
|
|
6564
|
-
const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
|
|
6565
|
-
if (
|
|
6566
|
-
nextRows.length === chunkResult.rowsWritten &&
|
|
6567
|
-
cachedRows.length + nextRows.length <= WORKER_DATASET_IN_MEMORY_ROWS
|
|
6568
|
-
) {
|
|
6569
|
-
cachedRows.push(...nextRows);
|
|
6570
|
-
} else {
|
|
6571
|
-
cachedRows.length = 0;
|
|
6572
|
-
volatileWorkflowChunkRows.clear();
|
|
6573
|
-
canCacheRows = false;
|
|
6574
|
-
}
|
|
6575
|
-
}
|
|
6576
|
-
chunkStart += rawChunkRows.length;
|
|
6577
|
-
chunkIndex += 1;
|
|
6578
|
-
// onRowError:'fail' short-circuit: once a chunk reports a row failure,
|
|
6579
|
-
// skip the remaining chunks entirely. The failing chunk itself completed
|
|
6580
|
-
// normally (no chunk-step retry storm) and persisted its rows, but
|
|
6581
|
-
// executing later chunks would keep spending provider credits on a run
|
|
6582
|
-
// the caller asked to fail fast. The post-loop fail-fast throw below
|
|
6583
|
-
// reports what committed before the stop.
|
|
6584
|
-
if (failFastRowErrors && totalRowsFailed > 0) {
|
|
6585
|
-
break;
|
|
6586
|
-
}
|
|
6365
|
+
throw error;
|
|
6587
6366
|
}
|
|
6588
6367
|
if (totalDuplicateKeysDropped > 0) {
|
|
6589
6368
|
const keySample = droppedDuplicateKeySamples.join(', ');
|
|
@@ -6596,6 +6375,13 @@ function createMinimalWorkerCtx(
|
|
|
6596
6375
|
ts: nowMs(),
|
|
6597
6376
|
});
|
|
6598
6377
|
}
|
|
6378
|
+
if (totalRowsBlocked > 0) {
|
|
6379
|
+
await flushQueuedMapProgressUpdates();
|
|
6380
|
+
throw new RuntimeSheetRowsBlockedError({
|
|
6381
|
+
tableNamespace: name,
|
|
6382
|
+
blockedRows: blockedRowSamples,
|
|
6383
|
+
});
|
|
6384
|
+
}
|
|
6599
6385
|
if (failFastRowErrors && totalRowsFailed > 0 && totalRowsWritten > 0) {
|
|
6600
6386
|
// onRowError:'fail', PARTIAL failure (some rows committed): fail the run
|
|
6601
6387
|
// without finalizing the dataset. The committed rows already persisted
|
|
@@ -6604,6 +6390,7 @@ function createMinimalWorkerCtx(
|
|
|
6604
6390
|
// normally (no per-row throw inside the durable chunk step, so no
|
|
6605
6391
|
// chunk-step retry storm); later chunks were skipped by the fail-fast
|
|
6606
6392
|
// short-circuit in the chunk loop.
|
|
6393
|
+
await flushQueuedMapProgressUpdates();
|
|
6607
6394
|
const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
|
|
6608
6395
|
throw new Error(
|
|
6609
6396
|
`ctx.dataset("${name}") failed for ${totalRowsFailed} executed row(s) under onRowError:'fail'. ` +
|
|
@@ -6621,7 +6408,7 @@ function createMinimalWorkerCtx(
|
|
|
6621
6408
|
// step when no row otherwise succeeded) are summarized and registered as
|
|
6622
6409
|
// a recovered dataset — the failed run then advertises a WORKING export
|
|
6623
6410
|
// instead of a dead end (#15/#27). The run still fails (the throw below).
|
|
6624
|
-
await finalize(
|
|
6411
|
+
await finalize();
|
|
6625
6412
|
const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
|
|
6626
6413
|
throw new Error(
|
|
6627
6414
|
`ctx.dataset("${name}") failed for all ${totalRowsFailed} executed rows. ` +
|
|
@@ -6629,7 +6416,7 @@ function createMinimalWorkerCtx(
|
|
|
6629
6416
|
`(rows are persisted with per-row errors; fix the cause and re-run to resume)`,
|
|
6630
6417
|
);
|
|
6631
6418
|
}
|
|
6632
|
-
const dataset = await finalize(
|
|
6419
|
+
const dataset = await finalize();
|
|
6633
6420
|
recordRunnerPerfTrace({
|
|
6634
6421
|
req,
|
|
6635
6422
|
phase: 'runner.map.total',
|
|
@@ -6638,7 +6425,7 @@ function createMinimalWorkerCtx(
|
|
|
6638
6425
|
mapName: name,
|
|
6639
6426
|
rowsWritten: totalRowsWritten,
|
|
6640
6427
|
inputKind: rowCountHint === null ? 'streaming' : 'known_count',
|
|
6641
|
-
chunks:
|
|
6428
|
+
chunks: dispatchResult.chunksExecuted,
|
|
6642
6429
|
},
|
|
6643
6430
|
});
|
|
6644
6431
|
return dataset;
|
|
@@ -6903,10 +6690,10 @@ function createMinimalWorkerCtx(
|
|
|
6903
6690
|
request.input,
|
|
6904
6691
|
workflowStep,
|
|
6905
6692
|
{
|
|
6906
|
-
force: request.force === true
|
|
6693
|
+
force: request.force === true,
|
|
6694
|
+
runForceRefresh: req.forceToolRefresh === true,
|
|
6695
|
+
runForceFailedRefresh: req.force === true,
|
|
6907
6696
|
staleAfterSeconds: request.staleAfterSeconds,
|
|
6908
|
-
},
|
|
6909
|
-
{
|
|
6910
6697
|
timeoutMs: request.timeoutMs,
|
|
6911
6698
|
receiptWaitMs: request.receiptWaitMs,
|
|
6912
6699
|
},
|
|
@@ -6947,13 +6734,18 @@ function createMinimalWorkerCtx(
|
|
|
6947
6734
|
if (!resolvedName) {
|
|
6948
6735
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
6949
6736
|
}
|
|
6737
|
+
const childManifest = req.childPlayManifests?.[resolvedName];
|
|
6950
6738
|
const receiptKey = buildDurableCtxCallCacheKey({
|
|
6951
6739
|
orgId: req.orgId,
|
|
6952
6740
|
playId: req.playName,
|
|
6953
6741
|
kind: 'runPlay',
|
|
6954
6742
|
id: normalizedKey,
|
|
6955
|
-
semanticKey:
|
|
6743
|
+
semanticKey: buildDurableRunPlaySemanticKey({
|
|
6956
6744
|
childPlayName: resolvedName,
|
|
6745
|
+
childRevisionFingerprint: workerChildRevisionFingerprint({
|
|
6746
|
+
playName: resolvedName,
|
|
6747
|
+
manifest: childManifest,
|
|
6748
|
+
}),
|
|
6957
6749
|
input,
|
|
6958
6750
|
}),
|
|
6959
6751
|
staleAfterSeconds: options?.staleAfterSeconds,
|
|
@@ -6981,7 +6773,6 @@ function createMinimalWorkerCtx(
|
|
|
6981
6773
|
message: `Starting child play ${resolvedName} (${normalizedKey})`,
|
|
6982
6774
|
ts: nowMs(),
|
|
6983
6775
|
});
|
|
6984
|
-
const childManifest = req.childPlayManifests?.[resolvedName];
|
|
6985
6776
|
if (!childManifest) {
|
|
6986
6777
|
throw new Error(
|
|
6987
6778
|
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
|
|
@@ -7019,7 +6810,7 @@ function createMinimalWorkerCtx(
|
|
|
7019
6810
|
});
|
|
7020
6811
|
let childPlaySlot: { release(): void } | null = null;
|
|
7021
6812
|
try {
|
|
7022
|
-
childPlaySlot = await governor.
|
|
6813
|
+
childPlaySlot = await governor.acquireChildSubmitSlot({
|
|
7023
6814
|
signal: abortSignal,
|
|
7024
6815
|
});
|
|
7025
6816
|
const childSubmitStartedAt = nowMs();
|
|
@@ -7044,12 +6835,12 @@ function createMinimalWorkerCtx(
|
|
|
7044
6835
|
options?.timeoutMs == null && !childNeedsWorkflowScheduler,
|
|
7045
6836
|
body: {
|
|
7046
6837
|
name: resolvedName,
|
|
6838
|
+
childIdempotencyKey: receiptKey,
|
|
7047
6839
|
input: isRecord(input) ? input : {},
|
|
7048
6840
|
orgId: req.orgId,
|
|
7049
6841
|
callbackBaseUrl: req.callbackUrl,
|
|
7050
6842
|
baseUrl: req.baseUrl,
|
|
7051
6843
|
integrationMode: req.integrationMode ?? null,
|
|
7052
|
-
forceToolRefresh: req.force === true,
|
|
7053
6844
|
parentExecutorToken: req.executorToken,
|
|
7054
6845
|
userEmail: req.userEmail ?? '',
|
|
7055
6846
|
profile: 'workers_edge',
|
|
@@ -7161,6 +6952,8 @@ function createMinimalWorkerCtx(
|
|
|
7161
6952
|
childNeedsWorkflowScheduler,
|
|
7162
6953
|
},
|
|
7163
6954
|
});
|
|
6955
|
+
childPlaySlot?.release();
|
|
6956
|
+
childPlaySlot = null;
|
|
7164
6957
|
const startedStatus = String(started.status ?? '').toLowerCase();
|
|
7165
6958
|
if (startedStatus === 'completed') {
|
|
7166
6959
|
emitEvent({
|
|
@@ -7189,6 +6982,10 @@ function createMinimalWorkerCtx(
|
|
|
7189
6982
|
}
|
|
7190
6983
|
const childWaitStartedAt = nowMs();
|
|
7191
6984
|
let waitResult: ChildPlayTerminalWaitResult;
|
|
6985
|
+
const readCachedChildTerminalState =
|
|
6986
|
+
cachedCoordinatorBinding?.readChildTerminalState?.bind(
|
|
6987
|
+
cachedCoordinatorBinding,
|
|
6988
|
+
);
|
|
7192
6989
|
try {
|
|
7193
6990
|
waitResult = await awaitChildTerminal({
|
|
7194
6991
|
parentRunId: req.runId,
|
|
@@ -7196,8 +6993,7 @@ function createMinimalWorkerCtx(
|
|
|
7196
6993
|
// the small structural shape ChildPlayAwait needs; bridge it the
|
|
7197
6994
|
// same way the inline implementation did.
|
|
7198
6995
|
workflowStep: workflowStep as unknown as
|
|
7199
|
-
|
|
|
7200
|
-
| undefined,
|
|
6996
|
+
WorkflowStepLike | undefined,
|
|
7201
6997
|
workflowId,
|
|
7202
6998
|
playName: resolvedName,
|
|
7203
6999
|
key: normalizedKey,
|
|
@@ -7205,20 +7001,31 @@ function createMinimalWorkerCtx(
|
|
|
7205
7001
|
1_000,
|
|
7206
7002
|
Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
|
|
7207
7003
|
),
|
|
7208
|
-
coordinator:
|
|
7209
|
-
|
|
7210
|
-
|
|
7211
|
-
|
|
7212
|
-
|
|
7213
|
-
|
|
7214
|
-
|
|
7215
|
-
|
|
7216
|
-
|
|
7217
|
-
|
|
7218
|
-
|
|
7219
|
-
|
|
7220
|
-
|
|
7221
|
-
|
|
7004
|
+
coordinator: {
|
|
7005
|
+
readRunTerminalState: (runId, timeoutMs) =>
|
|
7006
|
+
readDurableChildRunTerminalState({
|
|
7007
|
+
baseUrl: req.baseUrl,
|
|
7008
|
+
executorToken: req.executorToken,
|
|
7009
|
+
parentRunId: req.runId,
|
|
7010
|
+
childRunId: runId,
|
|
7011
|
+
childPlayName: resolvedName,
|
|
7012
|
+
timeoutMs,
|
|
7013
|
+
}),
|
|
7014
|
+
...(readCachedChildTerminalState
|
|
7015
|
+
? {
|
|
7016
|
+
readChildTerminalState: (
|
|
7017
|
+
parentRunId: string,
|
|
7018
|
+
eventKey: string,
|
|
7019
|
+
timeoutMs?: number,
|
|
7020
|
+
) =>
|
|
7021
|
+
readCachedChildTerminalState(
|
|
7022
|
+
parentRunId,
|
|
7023
|
+
eventKey,
|
|
7024
|
+
timeoutMs,
|
|
7025
|
+
),
|
|
7026
|
+
}
|
|
7027
|
+
: {}),
|
|
7028
|
+
},
|
|
7222
7029
|
now: nowMs,
|
|
7223
7030
|
hashJson,
|
|
7224
7031
|
});
|
|
@@ -7401,6 +7208,7 @@ function createMinimalWorkerCtx(
|
|
|
7401
7208
|
// Direct Worker fetch below is still limited to runtime origins and
|
|
7402
7209
|
// IP literals; public IP literals also consume this slot because
|
|
7403
7210
|
// they are customer egress.
|
|
7211
|
+
workBudgetMeter.count('egress');
|
|
7404
7212
|
const egressResponse = await postRuntimeEgressFetch(
|
|
7405
7213
|
req,
|
|
7406
7214
|
{
|
|
@@ -7423,6 +7231,7 @@ function createMinimalWorkerCtx(
|
|
|
7423
7231
|
}
|
|
7424
7232
|
const fetchInit = { ...init, headers };
|
|
7425
7233
|
delete fetchInit.auth;
|
|
7234
|
+
workBudgetMeter.count('egress');
|
|
7426
7235
|
const response = await safeWorkerPublicFetch(url, fetchInit, {
|
|
7427
7236
|
allowedOrigins,
|
|
7428
7237
|
sensitiveHeaders: Object.keys(secretHeaderMarkers),
|
|
@@ -7493,6 +7302,7 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
|
|
|
7493
7302
|
let req: RunRequest;
|
|
7494
7303
|
try {
|
|
7495
7304
|
req = (await request.json()) as RunRequest;
|
|
7305
|
+
req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
|
|
7496
7306
|
} catch {
|
|
7497
7307
|
return new Response('invalid JSON body', { status: 400 });
|
|
7498
7308
|
}
|
|
@@ -7557,6 +7367,7 @@ async function handleRunInline(
|
|
|
7557
7367
|
let req: RunRequest;
|
|
7558
7368
|
try {
|
|
7559
7369
|
req = (await request.json()) as RunRequest;
|
|
7370
|
+
req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
|
|
7560
7371
|
} catch {
|
|
7561
7372
|
return Response.json(
|
|
7562
7373
|
{
|
|
@@ -7704,6 +7515,7 @@ async function executeRunRequest(
|
|
|
7704
7515
|
): Promise<WorkflowRunOutput> {
|
|
7705
7516
|
installProcessExitTrap();
|
|
7706
7517
|
const startedAt = nowMs();
|
|
7518
|
+
const workBudgetMeter = createWorkerWorkBudgetMeter({ nowMs });
|
|
7707
7519
|
recordRunnerPerfTrace({
|
|
7708
7520
|
req,
|
|
7709
7521
|
phase: 'runner.execute_start',
|
|
@@ -7741,6 +7553,7 @@ async function executeRunRequest(
|
|
|
7741
7553
|
const appendRunLogLine = (line: string) => {
|
|
7742
7554
|
const trimmed = redactSecretsFromLogString(line.trim());
|
|
7743
7555
|
if (!trimmed) return;
|
|
7556
|
+
workBudgetMeter.count('log');
|
|
7744
7557
|
totalEmittedLogLines += 1;
|
|
7745
7558
|
runLogBuffer = [...runLogBuffer, trimmed].slice(-RUN_LOG_BUFFER_LIMIT);
|
|
7746
7559
|
pendingRunLogLines = [...pendingRunLogLines, trimmed];
|
|
@@ -7933,7 +7746,10 @@ async function executeRunRequest(
|
|
|
7933
7746
|
const flushTerminalLedgerEvents = async (
|
|
7934
7747
|
terminalEvent: PlayRunLedgerEvent,
|
|
7935
7748
|
): Promise<void> => {
|
|
7936
|
-
|
|
7749
|
+
const shouldPersistTerminalLedger =
|
|
7750
|
+
options?.persistResultDatasets === true ||
|
|
7751
|
+
Boolean(req.playCallGovernance);
|
|
7752
|
+
if (!shouldPersistTerminalLedger) return;
|
|
7937
7753
|
await ledgerFlushInFlight;
|
|
7938
7754
|
const now = nowMs();
|
|
7939
7755
|
dirtyProgressNodeIds = new Set([
|
|
@@ -8008,10 +7824,7 @@ async function executeRunRequest(
|
|
|
8008
7824
|
stepLifecycle?.markPreDatasetStepsStarted(startedAt);
|
|
8009
7825
|
flushLedgerEvents(false);
|
|
8010
7826
|
const runtimeDeadlineMs = nowMs() + STANDARD_PLAY_RUNTIME_LIMIT_MS;
|
|
8011
|
-
|
|
8012
|
-
// receipt could not be marked completed even after the retry ladder. On
|
|
8013
|
-
// terminal failure the buffer is built into a size-capped salvage record and
|
|
8014
|
-
// attached to the run.failed ledger event so billed data is never memory-only.
|
|
7827
|
+
const leasedSheetNamespaces = new Set<string>();
|
|
8015
7828
|
const receiptSalvage = createReceiptSalvageBuffer();
|
|
8016
7829
|
const ctx = createMinimalWorkerCtx(
|
|
8017
7830
|
req,
|
|
@@ -8020,7 +7833,9 @@ async function executeRunRequest(
|
|
|
8020
7833
|
workflowStep,
|
|
8021
7834
|
abortSignal,
|
|
8022
7835
|
workerCallbacks,
|
|
7836
|
+
workBudgetMeter,
|
|
8023
7837
|
runtimeDeadlineMs,
|
|
7838
|
+
leasedSheetNamespaces,
|
|
8024
7839
|
receiptSalvage,
|
|
8025
7840
|
);
|
|
8026
7841
|
// Hard wall-clock cap on active user-code runtime. CF Workflows does not
|
|
@@ -8029,15 +7844,18 @@ async function executeRunRequest(
|
|
|
8029
7844
|
// token expires. Aborting the controller surfaces cooperatively through the
|
|
8030
7845
|
// same assertNotAborted checks used for harness cancellation.
|
|
8031
7846
|
let runtimeLimitExceeded = false;
|
|
8032
|
-
const runtimeDeadlineTimer =
|
|
8033
|
-
|
|
8034
|
-
|
|
8035
|
-
|
|
8036
|
-
|
|
8037
|
-
|
|
8038
|
-
|
|
8039
|
-
|
|
8040
|
-
|
|
7847
|
+
const runtimeDeadlineTimer =
|
|
7848
|
+
workflowStep === undefined || workflowStep === null
|
|
7849
|
+
? setTimeout(
|
|
7850
|
+
() => {
|
|
7851
|
+
runtimeLimitExceeded = true;
|
|
7852
|
+
if (!abortSignal.aborted) {
|
|
7853
|
+
abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
|
|
7854
|
+
}
|
|
7855
|
+
},
|
|
7856
|
+
Math.max(1, runtimeDeadlineMs - nowMs()),
|
|
7857
|
+
)
|
|
7858
|
+
: null;
|
|
8041
7859
|
try {
|
|
8042
7860
|
const playStartedAt = nowMs();
|
|
8043
7861
|
const result = await (
|
|
@@ -8128,7 +7946,6 @@ async function executeRunRequest(
|
|
|
8128
7946
|
}
|
|
8129
7947
|
if (options?.persistResultDatasets) {
|
|
8130
7948
|
await persistProjectedResultDatasets();
|
|
8131
|
-
const parentSignal = startParentTerminalSignal();
|
|
8132
7949
|
// Capped runs settle compute billing BEFORE declaring run.completed: a
|
|
8133
7950
|
// per-run cap denial (422 billing_cap_exceeded) must fail the run as
|
|
8134
7951
|
// its ONLY terminal. Flushing completed first opens a race — watchers
|
|
@@ -8163,6 +7980,7 @@ async function executeRunRequest(
|
|
|
8163
7980
|
ms: nowMs() - terminalUpdateStartedAt,
|
|
8164
7981
|
});
|
|
8165
7982
|
|
|
7983
|
+
const parentSignal = startParentTerminalSignal();
|
|
8166
7984
|
if (!capped) {
|
|
8167
7985
|
const billingStartedAt = nowMs();
|
|
8168
7986
|
const billingPromise = finalizeWorkerComputeBilling({
|
|
@@ -8190,6 +8008,20 @@ async function executeRunRequest(
|
|
|
8190
8008
|
}
|
|
8191
8009
|
}
|
|
8192
8010
|
await parentSignal;
|
|
8011
|
+
} else if (req.playCallGovernance) {
|
|
8012
|
+
const childTerminalStartedAt = nowMs();
|
|
8013
|
+
await flushTerminalLedgerEvents({
|
|
8014
|
+
type: 'run.completed',
|
|
8015
|
+
runId: req.runId,
|
|
8016
|
+
source: 'worker',
|
|
8017
|
+
occurredAt: nowMs(),
|
|
8018
|
+
result: terminalResult,
|
|
8019
|
+
});
|
|
8020
|
+
recordRunnerPerfTrace({
|
|
8021
|
+
req,
|
|
8022
|
+
phase: 'runner.child_terminal_ledger_append',
|
|
8023
|
+
ms: nowMs() - childTerminalStartedAt,
|
|
8024
|
+
});
|
|
8193
8025
|
}
|
|
8194
8026
|
await startParentTerminalSignal();
|
|
8195
8027
|
recordRunnerPerfTrace({
|
|
@@ -8216,16 +8048,8 @@ async function executeRunRequest(
|
|
|
8216
8048
|
stepLifecycle?.markStartedFailed(nowMs());
|
|
8217
8049
|
// A runtime-limit abort is a timeout failure, not a user cancellation, so
|
|
8218
8050
|
// it should be reported as run.failed with the limit message rather than
|
|
8219
|
-
// run.cancelled.
|
|
8220
|
-
|
|
8221
|
-
// runtimeDeadlineTimer macrotask flips the flag — match the limit message
|
|
8222
|
-
// directly so that race can never misclassify a timeout as a cancel (which
|
|
8223
|
-
// would silently drop the salvage buffer of billed-but-unpersisted rows).
|
|
8224
|
-
const runtimeLimitError =
|
|
8225
|
-
runtimeLimitExceeded ||
|
|
8226
|
-
(error instanceof Error &&
|
|
8227
|
-
error.message === STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
|
|
8228
|
-
const aborted = isAbortLikeError(error) && !runtimeLimitError;
|
|
8051
|
+
// run.cancelled.
|
|
8052
|
+
const aborted = isAbortLikeError(error) && !runtimeLimitExceeded;
|
|
8229
8053
|
if (aborted) {
|
|
8230
8054
|
// Flip the controller so any concurrent user code observes the abort
|
|
8231
8055
|
// through ctx.signal. We mark the run cancelled instead of failed.
|
|
@@ -8234,19 +8058,28 @@ async function executeRunRequest(
|
|
|
8234
8058
|
);
|
|
8235
8059
|
}
|
|
8236
8060
|
const failure = normalizePlayRunFailure(error);
|
|
8237
|
-
|
|
8238
|
-
//
|
|
8061
|
+
const message = failure.message;
|
|
8062
|
+
// Controlled run-fatal teardown: release the runtime-sheet row leases and
|
|
8063
|
+
// work-receipt leases this attempt still holds so an immediate rerun is not
|
|
8064
|
+
// blocked for the full lease TTL. Best-effort but LOUD — a release failure
|
|
8065
|
+
// is logged and never masks the original run-fatal error. TTL expiry stays
|
|
8066
|
+
// the recovery path for true crashes (isolate death) where this never runs.
|
|
8067
|
+
await releaseRuntimeLeasesOnTeardown({
|
|
8068
|
+
req,
|
|
8069
|
+
leasedSheetNamespaces,
|
|
8070
|
+
emit: wrappedEmit,
|
|
8071
|
+
});
|
|
8072
|
+
const errorBilling = extractErrorBilling(error);
|
|
8239
8073
|
const salvage: ReceiptSalvage | null = aborted
|
|
8240
8074
|
? null
|
|
8241
8075
|
: receiptSalvage.build();
|
|
8242
|
-
const
|
|
8076
|
+
const terminalMessage =
|
|
8243
8077
|
salvage && salvage.totalEntries > 0
|
|
8244
|
-
? `${
|
|
8245
|
-
:
|
|
8246
|
-
|
|
8247
|
-
if (options?.persistResultDatasets) {
|
|
8078
|
+
? `${message} ${receiptSalvageFailureSuffix(salvage)}`
|
|
8079
|
+
: message;
|
|
8080
|
+
if (options?.persistResultDatasets || req.playCallGovernance) {
|
|
8248
8081
|
appendRunLogLine(
|
|
8249
|
-
`${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(
|
|
8082
|
+
`${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(terminalMessage)}`,
|
|
8250
8083
|
);
|
|
8251
8084
|
const terminalUpdateStartedAt = nowMs();
|
|
8252
8085
|
await flushTerminalLedgerEvents({
|
|
@@ -8254,64 +8087,70 @@ async function executeRunRequest(
|
|
|
8254
8087
|
runId: req.runId,
|
|
8255
8088
|
source: 'worker',
|
|
8256
8089
|
occurredAt: nowMs(),
|
|
8257
|
-
error:
|
|
8090
|
+
error: terminalMessage,
|
|
8258
8091
|
result: aborted
|
|
8259
8092
|
? undefined
|
|
8260
8093
|
: {
|
|
8261
8094
|
success: false,
|
|
8262
8095
|
status: 'failed',
|
|
8263
|
-
error:
|
|
8096
|
+
error: terminalMessage,
|
|
8097
|
+
...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
|
|
8264
8098
|
errors: [
|
|
8265
8099
|
{
|
|
8266
8100
|
code: failure.code,
|
|
8267
8101
|
phase: failure.phase,
|
|
8268
|
-
message,
|
|
8102
|
+
message: terminalMessage,
|
|
8269
8103
|
retryable: failure.retryable,
|
|
8270
8104
|
...(errorBilling ? { billing: errorBilling } : {}),
|
|
8271
8105
|
...(failure.cause ? { cause: failure.cause } : {}),
|
|
8272
8106
|
},
|
|
8273
8107
|
],
|
|
8274
|
-
...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
|
|
8275
8108
|
},
|
|
8276
8109
|
});
|
|
8277
8110
|
recordRunnerPerfTrace({
|
|
8278
8111
|
req,
|
|
8279
|
-
phase:
|
|
8280
|
-
?
|
|
8281
|
-
|
|
8112
|
+
phase: req.playCallGovernance
|
|
8113
|
+
? aborted
|
|
8114
|
+
? 'runner.child_terminal_ledger_append_cancelled'
|
|
8115
|
+
: 'runner.child_terminal_ledger_append_failed'
|
|
8116
|
+
: aborted
|
|
8117
|
+
? 'runner.terminal_ledger_append_cancelled'
|
|
8118
|
+
: 'runner.terminal_ledger_append_failed',
|
|
8282
8119
|
ms: nowMs() - terminalUpdateStartedAt,
|
|
8283
8120
|
extra: {
|
|
8284
8121
|
errorCode: failure.code,
|
|
8285
8122
|
errorPhase: failure.phase,
|
|
8286
8123
|
},
|
|
8287
8124
|
});
|
|
8288
|
-
|
|
8289
|
-
|
|
8290
|
-
|
|
8291
|
-
|
|
8292
|
-
|
|
8293
|
-
|
|
8294
|
-
.catch((finalizeError) => {
|
|
8295
|
-
console.error(
|
|
8296
|
-
`[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
|
|
8297
|
-
finalizeError instanceof Error
|
|
8298
|
-
? finalizeError.message
|
|
8299
|
-
: String(finalizeError)
|
|
8300
|
-
}`,
|
|
8301
|
-
);
|
|
8125
|
+
if (options?.persistResultDatasets) {
|
|
8126
|
+
const billingStartedAt = nowMs();
|
|
8127
|
+
await finalizeWorkerComputeBilling({
|
|
8128
|
+
req,
|
|
8129
|
+
success: false,
|
|
8130
|
+
actionEstimate: 4,
|
|
8302
8131
|
})
|
|
8303
|
-
|
|
8304
|
-
|
|
8305
|
-
|
|
8306
|
-
|
|
8307
|
-
|
|
8132
|
+
.catch((finalizeError) => {
|
|
8133
|
+
console.error(
|
|
8134
|
+
`[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
|
|
8135
|
+
finalizeError instanceof Error
|
|
8136
|
+
? finalizeError.message
|
|
8137
|
+
: String(finalizeError)
|
|
8138
|
+
}`,
|
|
8139
|
+
);
|
|
8140
|
+
})
|
|
8141
|
+
.finally(() => {
|
|
8142
|
+
recordRunnerPerfTrace({
|
|
8143
|
+
req,
|
|
8144
|
+
phase: 'runner.compute_billing_finalize_failed',
|
|
8145
|
+
ms: nowMs() - billingStartedAt,
|
|
8146
|
+
});
|
|
8308
8147
|
});
|
|
8309
|
-
|
|
8148
|
+
}
|
|
8310
8149
|
}
|
|
8311
8150
|
await signalParentPlayTerminal({
|
|
8312
8151
|
req,
|
|
8313
8152
|
status: aborted ? 'cancelled' : 'failed',
|
|
8314
|
-
error:
|
|
8153
|
+
error: terminalMessage,
|
|
8315
8154
|
}).catch(() => null);
|
|
8316
8155
|
recordRunnerPerfTrace({
|
|
8317
8156
|
req,
|
|
@@ -8325,7 +8164,9 @@ async function executeRunRequest(
|
|
|
8325
8164
|
await drainRunnerPerfTraces(req);
|
|
8326
8165
|
throw error;
|
|
8327
8166
|
} finally {
|
|
8328
|
-
|
|
8167
|
+
if (runtimeDeadlineTimer) {
|
|
8168
|
+
clearTimeout(runtimeDeadlineTimer);
|
|
8169
|
+
}
|
|
8329
8170
|
}
|
|
8330
8171
|
}
|
|
8331
8172
|
|
|
@@ -8417,10 +8258,12 @@ function runRequestFromWorkflowParams(
|
|
|
8417
8258
|
orgId: String(params.orgId ?? ''),
|
|
8418
8259
|
playName: String(params.playName ?? ''),
|
|
8419
8260
|
userEmail: typeof params.userEmail === 'string' ? params.userEmail : null,
|
|
8261
|
+
force: params.force === true,
|
|
8262
|
+
forceToolRefresh: params.forceToolRefresh === true,
|
|
8263
|
+
runAttempt: normalizeWorkerRunAttempt(params.runAttempt),
|
|
8420
8264
|
runtimeInput: isRecord(params.input)
|
|
8421
8265
|
? (params.input as Record<string, unknown>)
|
|
8422
8266
|
: {},
|
|
8423
|
-
force: params.forceToolRefresh === true,
|
|
8424
8267
|
inlineCsv: isInlineCsv(params.inlineCsv) ? params.inlineCsv : null,
|
|
8425
8268
|
inputFiles:
|
|
8426
8269
|
inputFile && inputStorageKey
|
|
@@ -8478,6 +8321,14 @@ function runRequestFromWorkflowParams(
|
|
|
8478
8321
|
params.coordinatorInternalToken.trim()
|
|
8479
8322
|
? params.coordinatorInternalToken.trim()
|
|
8480
8323
|
: null,
|
|
8324
|
+
runtimeTestFaultHeader:
|
|
8325
|
+
typeof params.runtimeTestFaultHeader === 'string' &&
|
|
8326
|
+
params.runtimeTestFaultHeader.trim()
|
|
8327
|
+
? params.runtimeTestFaultHeader.trim()
|
|
8328
|
+
: null,
|
|
8329
|
+
testPolicyOverrides: normalizeRuntimeTestPolicyOverrides(
|
|
8330
|
+
params.testPolicyOverrides,
|
|
8331
|
+
),
|
|
8481
8332
|
totalRows:
|
|
8482
8333
|
typeof params.totalRows === 'number' && Number.isFinite(params.totalRows)
|
|
8483
8334
|
? params.totalRows
|
|
@@ -8547,6 +8398,7 @@ async function persistResultDatasets(
|
|
|
8547
8398
|
rows: chunk.map((row) => ({ ...row })),
|
|
8548
8399
|
runId: req.runId,
|
|
8549
8400
|
inputOffset,
|
|
8401
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
8550
8402
|
userEmail: req.userEmail,
|
|
8551
8403
|
preloadedDbSessions: req.preloadedDbSessions ?? null,
|
|
8552
8404
|
});
|
|
@@ -8583,6 +8435,7 @@ async function persistResultDatasets(
|
|
|
8583
8435
|
rows: dataset.rows,
|
|
8584
8436
|
runId: req.runId,
|
|
8585
8437
|
inputOffset: 0,
|
|
8438
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
8586
8439
|
userEmail: req.userEmail,
|
|
8587
8440
|
});
|
|
8588
8441
|
}
|
|
@@ -8864,7 +8717,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
|
|
|
8864
8717
|
try {
|
|
8865
8718
|
const response = await fetchRuntimeApi(
|
|
8866
8719
|
req.baseUrl,
|
|
8867
|
-
|
|
8720
|
+
PLAY_RUNTIME_API_COMPAT_PATH,
|
|
8868
8721
|
{
|
|
8869
8722
|
method: 'POST',
|
|
8870
8723
|
headers: {
|