deepline 0.1.182 → 0.1.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +633 -107
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1747 -1890
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
- package/dist/bundling-sources/sdk/src/client.ts +41 -0
- package/dist/bundling-sources/sdk/src/http.ts +23 -8
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +132 -10
- package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
- package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1605 -281
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +105 -6
- package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
- package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
- package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
- package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
- package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2037 -262
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
- package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
- package/dist/cli/index.js +76 -17
- package/dist/cli/index.mjs +76 -17
- package/dist/index.d.mts +2 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +63 -9
- package/dist/index.mjs +63 -9
- package/dist/plays/bundle-play-file.mjs +2 -2
- package/package.json +1 -1
|
@@ -36,12 +36,6 @@ import {
|
|
|
36
36
|
deterministicMapChunkStepName,
|
|
37
37
|
type ExecutionPlan,
|
|
38
38
|
} from '../../../shared_libs/play-runtime/execution-plan';
|
|
39
|
-
import {
|
|
40
|
-
compileRequestsWithStrategy,
|
|
41
|
-
executeChunkedRequests,
|
|
42
|
-
type ChunkExecutionResult,
|
|
43
|
-
} from '../../../shared_libs/play-runtime/batch-runtime';
|
|
44
|
-
import { getPlayRuntimeBatchStrategy } from '../../../shared_libs/play-runtime/play-runtime-batching-registry';
|
|
45
39
|
import {
|
|
46
40
|
STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
|
|
47
41
|
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
|
|
@@ -71,11 +65,10 @@ import {
|
|
|
71
65
|
type WorkflowStepLike,
|
|
72
66
|
} from './child-play-await';
|
|
73
67
|
import { submitChildPlayThroughCoordinator } from './child-play-submit';
|
|
74
|
-
import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
|
|
75
68
|
import {
|
|
76
69
|
attachToolResultListDataset,
|
|
77
|
-
parseToolExecuteResponse,
|
|
78
70
|
createToolExecuteResult,
|
|
71
|
+
parseToolExecuteResponse,
|
|
79
72
|
deserializeToolExecuteResult,
|
|
80
73
|
isSerializedToolExecuteResult,
|
|
81
74
|
isToolExecuteResult,
|
|
@@ -90,29 +83,35 @@ import {
|
|
|
90
83
|
TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS,
|
|
91
84
|
classifyToolExecuteHttpFailure,
|
|
92
85
|
createToolExecuteHttpFailureAttemptTracker,
|
|
86
|
+
parseToolExecuteAuthScopeChangedError,
|
|
93
87
|
} from '../../../shared_libs/play-runtime/tool-execute-retry-policy';
|
|
94
88
|
import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
95
89
|
import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
|
|
96
|
-
import type {
|
|
90
|
+
import type {
|
|
91
|
+
PlayRuntimeManifest,
|
|
92
|
+
PlayRuntimeManifestMap,
|
|
93
|
+
} from '../../../shared_libs/plays/compiler-manifest';
|
|
97
94
|
import {
|
|
98
95
|
deriveToolRequestIdentity,
|
|
99
96
|
derivePlayRowIdentity,
|
|
100
97
|
derivePlayRowIdentityFromKey,
|
|
101
98
|
sha256Hex,
|
|
99
|
+
stableStringify,
|
|
102
100
|
} from '../../../shared_libs/plays/row-identity';
|
|
103
101
|
import { createDeferredPlayDataset } from '../../../shared_libs/plays/dataset';
|
|
104
102
|
import {
|
|
105
103
|
buildDurableCtxCallCacheKey,
|
|
106
|
-
|
|
107
|
-
buildDurableToolCallCacheKey,
|
|
104
|
+
buildDurableRunPlaySemanticKey,
|
|
108
105
|
} from '../../../shared_libs/play-runtime/durable-call-cache';
|
|
106
|
+
import {
|
|
107
|
+
resolveRuntimeToolReceiptWaitMaxAttempts,
|
|
108
|
+
resolveRuntimeToolReceiptWaitTimeoutMs,
|
|
109
|
+
} from '../../../shared_libs/play-runtime/durable-receipt-execution';
|
|
109
110
|
import {
|
|
110
111
|
QUERY_RESULT_DATASET_PAGE_SIZE,
|
|
111
112
|
isCustomerDbDatasetTool,
|
|
112
|
-
isQueryResultDatasetReadRequest,
|
|
113
113
|
isQueryResultDatasetTool,
|
|
114
114
|
} from '../../../shared_libs/play-runtime/query-result-dataset';
|
|
115
|
-
import { buildScopedWorkReceiptKey } from '../../../shared_libs/play-runtime/work-receipts';
|
|
116
115
|
import { DEDUPE_DUPLICATE_KEY_SAMPLE_CAP } from '../../../shared_libs/play-runtime/map-row-identity';
|
|
117
116
|
import {
|
|
118
117
|
getTopLevelPipelineSubsteps,
|
|
@@ -151,26 +150,41 @@ import {
|
|
|
151
150
|
} from './runtime/dataset-handles';
|
|
152
151
|
import {
|
|
153
152
|
runWorkerRuntimeReceiptBoundary,
|
|
154
|
-
|
|
155
|
-
type
|
|
156
|
-
type WorkerRuntimeReceiptStore,
|
|
153
|
+
RuntimeLeaseLostError,
|
|
154
|
+
type WorkerRuntimeReceiptExecutionContext,
|
|
157
155
|
} from './runtime/receipts';
|
|
158
156
|
import {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
} from '
|
|
164
|
-
import
|
|
157
|
+
RuntimeReceiptPersistenceError,
|
|
158
|
+
WorkerToolBatchScheduler,
|
|
159
|
+
type WorkerPacingResolver,
|
|
160
|
+
type WorkerToolActionCacheVersionResolver,
|
|
161
|
+
} from './runtime/tool-dispatch';
|
|
162
|
+
import {
|
|
163
|
+
createWorkerWorkBudgetMeter,
|
|
164
|
+
type WorkerWorkBudgetMeter,
|
|
165
|
+
} from './runtime/work-budget';
|
|
165
166
|
import {
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
167
|
+
WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES,
|
|
168
|
+
WorkerRunWorkDispatcher,
|
|
169
|
+
type WorkerRunMapBatchesResult,
|
|
170
|
+
} from './runtime/run-work-dispatcher';
|
|
171
|
+
import { WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL } from './runtime/worker-platform-budget';
|
|
172
|
+
import {
|
|
173
|
+
PLAY_RUNTIME_CONTRACT,
|
|
174
|
+
PLAY_RUNTIME_CONTRACT_HEADER,
|
|
175
|
+
} from '../../../shared_libs/play-runtime/runtime-contract';
|
|
176
|
+
import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
|
|
177
|
+
import {
|
|
178
|
+
PLAY_RUNTIME_TEST_FAULT_HEADER,
|
|
179
|
+
type RuntimeTestPolicyOverrides,
|
|
180
|
+
} from '../../../shared_libs/play-runtime/test-runtime-seams';
|
|
181
|
+
import { createRuntimeToolAuthScopeDigestResolver } from '../../../shared_libs/play-runtime/auth-scope-resolver';
|
|
182
|
+
import {
|
|
183
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
184
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
185
|
+
runtimeLeaseHeartbeatIntervalMs,
|
|
186
|
+
} from '../../../shared_libs/play-runtime/lease-policy';
|
|
187
|
+
import { RuntimeSheetRowsBlockedError } from '../../../shared_libs/play-runtime/runtime-sheet-errors';
|
|
174
188
|
// The harness stub forwards leaf calls (validation, runtime-api HTTP) into
|
|
175
189
|
// the long-lived Play Harness Worker via env.HARNESS. We import the
|
|
176
190
|
// `setHarnessBinding` setter eagerly so it's available the moment
|
|
@@ -183,9 +197,12 @@ import {
|
|
|
183
197
|
// re-bundle harness internals into per-play. Keep that in mind.
|
|
184
198
|
import {
|
|
185
199
|
harnessPersistCompletedSheetRows,
|
|
200
|
+
harnessHeartbeatSheetAttempt,
|
|
186
201
|
harnessReadSheetDatasetRowKeys,
|
|
187
202
|
harnessReadSheetDatasetRows,
|
|
188
203
|
harnessReadStagedFileChunk,
|
|
204
|
+
harnessReleaseRuntimeReceipts,
|
|
205
|
+
harnessReleaseSheetAttempt,
|
|
189
206
|
harnessStartSheetDataset,
|
|
190
207
|
setHarnessBinding,
|
|
191
208
|
} from '../../../sdk/src/plays/harness-stub';
|
|
@@ -216,7 +233,10 @@ import {
|
|
|
216
233
|
stripMapRowOutcomeRuntimeFields,
|
|
217
234
|
} from '../../../shared_libs/play-runtime/map-row-outcome';
|
|
218
235
|
import { runtimeSheetSessionScope } from '../../../shared_libs/play-runtime/runtime-sheet-session';
|
|
219
|
-
import {
|
|
236
|
+
import {
|
|
237
|
+
chooseWorkerMapDispatchPlan,
|
|
238
|
+
estimateBatchedToolChunkSubrequests,
|
|
239
|
+
} from './runtime/map-chunk-plan';
|
|
220
240
|
import {
|
|
221
241
|
applyCsvRenameProjection,
|
|
222
242
|
cloneCsvAliasedRow,
|
|
@@ -228,12 +248,7 @@ import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-f
|
|
|
228
248
|
import {
|
|
229
249
|
createRuntimePersistenceLatch,
|
|
230
250
|
isRuntimePersistenceCircuitOpenError,
|
|
231
|
-
partitionDispatchWaves,
|
|
232
|
-
PROVIDER_DISPATCH_WAVE_SIZE,
|
|
233
|
-
RuntimePersistenceCircuitOpenError,
|
|
234
|
-
shouldShortenCompleteReceiptLadder,
|
|
235
251
|
tripRuntimePersistenceLatch,
|
|
236
|
-
type RuntimePersistenceLatch,
|
|
237
252
|
} from '../../../shared_libs/play-runtime/persistence-latch';
|
|
238
253
|
import {
|
|
239
254
|
createReceiptSalvageBuffer,
|
|
@@ -299,8 +314,13 @@ type RunRequest = {
|
|
|
299
314
|
playName: string;
|
|
300
315
|
graphHash?: string | null;
|
|
301
316
|
userEmail: string | null;
|
|
302
|
-
force?: boolean | null;
|
|
303
317
|
runtimeInput: Record<string, unknown>;
|
|
318
|
+
/** Start a fresh run graph and recompute runtime-sheet rows. */
|
|
319
|
+
force?: boolean;
|
|
320
|
+
/** Explicit durable tool receipt refresh that can re-execute completed provider calls. */
|
|
321
|
+
forceToolRefresh?: boolean;
|
|
322
|
+
/** Monotonic coordinator redispatch epoch for receipt and sheet fencing. */
|
|
323
|
+
runAttempt?: number;
|
|
304
324
|
/** Optional inline CSV rows (for plays where ctx.csv was passed inline data). */
|
|
305
325
|
inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
|
|
306
326
|
/** Staged input files keyed by logical filename (used by ctx.csv). */
|
|
@@ -333,6 +353,10 @@ type RunRequest = {
|
|
|
333
353
|
coordinatorUrl?: string | null;
|
|
334
354
|
/** Request-scoped coordinator auth token for preview/dev direct control calls. */
|
|
335
355
|
coordinatorInternalToken?: string | null;
|
|
356
|
+
/** Request-scoped, dev-only runtime fault injection header for black-box tests. */
|
|
357
|
+
runtimeTestFaultHeader?: string | null;
|
|
358
|
+
/** Request-scoped, dev-only policy overrides for black-box durability tests. */
|
|
359
|
+
testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
|
|
336
360
|
/**
|
|
337
361
|
* Total input rows known at submit time, when the dispatcher can count them.
|
|
338
362
|
* `undefined` when the input streams from R2 (unknown row count).
|
|
@@ -348,10 +372,104 @@ type WorkerFileRef = {
|
|
|
348
372
|
bytes?: number | null;
|
|
349
373
|
};
|
|
350
374
|
|
|
375
|
+
function normalizeWorkerRunAttempt(value: unknown): number {
|
|
376
|
+
return typeof value === 'number' && Number.isFinite(value)
|
|
377
|
+
? Math.max(0, Math.floor(value))
|
|
378
|
+
: 0;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function normalizePositiveInteger(value: unknown): number | undefined {
|
|
382
|
+
return typeof value === 'number' &&
|
|
383
|
+
Number.isFinite(value) &&
|
|
384
|
+
Number.isInteger(value) &&
|
|
385
|
+
value > 0
|
|
386
|
+
? value
|
|
387
|
+
: undefined;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function normalizeRuntimeTestPolicyOverrides(
|
|
391
|
+
value: unknown,
|
|
392
|
+
): RuntimeTestPolicyOverrides | null {
|
|
393
|
+
if (!isRecord(value)) return null;
|
|
394
|
+
const overrides: RuntimeTestPolicyOverrides = {};
|
|
395
|
+
const receiptLeaseTtlMs = normalizePositiveInteger(value.receiptLeaseTtlMs);
|
|
396
|
+
if (receiptLeaseTtlMs !== undefined) {
|
|
397
|
+
overrides.receiptLeaseTtlMs = receiptLeaseTtlMs;
|
|
398
|
+
}
|
|
399
|
+
const sheetAttemptLeaseMs = normalizePositiveInteger(
|
|
400
|
+
value.sheetAttemptLeaseMs,
|
|
401
|
+
);
|
|
402
|
+
if (sheetAttemptLeaseMs !== undefined) {
|
|
403
|
+
overrides.sheetAttemptLeaseMs = sheetAttemptLeaseMs;
|
|
404
|
+
}
|
|
405
|
+
const heartbeatIntervalMs = normalizePositiveInteger(
|
|
406
|
+
value.heartbeatIntervalMs,
|
|
407
|
+
);
|
|
408
|
+
if (heartbeatIntervalMs !== undefined) {
|
|
409
|
+
overrides.heartbeatIntervalMs = heartbeatIntervalMs;
|
|
410
|
+
}
|
|
411
|
+
const batchGraceMs = normalizePositiveInteger(value.batchGraceMs);
|
|
412
|
+
if (batchGraceMs !== undefined) {
|
|
413
|
+
overrides.batchGraceMs = batchGraceMs;
|
|
414
|
+
}
|
|
415
|
+
if (isRecord(value.workBudgetYieldLimits)) {
|
|
416
|
+
const limits: NonNullable<
|
|
417
|
+
RuntimeTestPolicyOverrides['workBudgetYieldLimits']
|
|
418
|
+
> = {};
|
|
419
|
+
for (const key of [
|
|
420
|
+
'elapsed',
|
|
421
|
+
'subrequest',
|
|
422
|
+
'provider',
|
|
423
|
+
'tool',
|
|
424
|
+
'receipt',
|
|
425
|
+
'sheet',
|
|
426
|
+
'log',
|
|
427
|
+
'egress',
|
|
428
|
+
] as const) {
|
|
429
|
+
const limit = normalizePositiveInteger(value.workBudgetYieldLimits[key]);
|
|
430
|
+
if (limit !== undefined) {
|
|
431
|
+
limits[key] = limit;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
if (Object.keys(limits).length > 0) {
|
|
435
|
+
overrides.workBudgetYieldLimits = limits;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
return Object.keys(overrides).length > 0 ? overrides : null;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
function runtimeReceiptLeaseTtlMs(req: RunRequest): number | undefined {
|
|
442
|
+
return req.testPolicyOverrides?.receiptLeaseTtlMs;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function runtimeSheetAttemptLeaseTtlMs(req: RunRequest): number | undefined {
|
|
446
|
+
return req.testPolicyOverrides?.sheetAttemptLeaseMs;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
function runtimePolicyHeartbeatIntervalMs(
|
|
450
|
+
req: RunRequest,
|
|
451
|
+
fallbackLeaseTtlMs: number,
|
|
452
|
+
): number {
|
|
453
|
+
return (
|
|
454
|
+
req.testPolicyOverrides?.heartbeatIntervalMs ??
|
|
455
|
+
runtimeLeaseHeartbeatIntervalMs(fallbackLeaseTtlMs)
|
|
456
|
+
);
|
|
457
|
+
}
|
|
458
|
+
|
|
351
459
|
const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
|
|
352
460
|
const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
|
|
353
461
|
const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
|
|
354
462
|
const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
|
|
463
|
+
// A short durable sleep is enough to force a Cloudflare Workflows step
|
|
464
|
+
// checkpoint. The resume re-enters run() as a new Worker invocation with a
|
|
465
|
+
// fresh per-invocation subrequest budget (see the yield callback below), so
|
|
466
|
+
// there is no need to escalate the sleep to "encourage" re-invocation.
|
|
467
|
+
// A 1ms Workflow sleep is too small to reliably create a new platform
|
|
468
|
+
// invocation in preview; the dispatcher then resets its local budget while
|
|
469
|
+
// Cloudflare's per-invocation subrequest counter is still hot. Use a real
|
|
470
|
+
// boundary so budget-yielded maps resume with fresh platform headroom.
|
|
471
|
+
const WORK_DISPATCHER_YIELD_SLEEP_MS = 1_000;
|
|
472
|
+
|
|
355
473
|
type CustomerDbDatasetRequest = {
|
|
356
474
|
limit?: number;
|
|
357
475
|
offset?: number;
|
|
@@ -418,7 +536,8 @@ type WorkerEnv = {
|
|
|
418
536
|
/**
|
|
419
537
|
* In-process Fetcher constructed by the coordinator and handed to the
|
|
420
538
|
* per-graphHash play Worker. When present, runtime callbacks
|
|
421
|
-
* (`/api/v2/
|
|
539
|
+
* (`/api/v2/internal/play-runtime`,
|
|
540
|
+
* `/api/v2/internal/play-runtime/*`,
|
|
422
541
|
* `/api/v2/plays/runtime-tools/*`) skip the public callback URL and route
|
|
423
542
|
* directly through the coordinator's process to the configured app — saves
|
|
424
543
|
* the *.workers.dev → CF edge → cloudflared → localhost chain on every
|
|
@@ -427,6 +546,7 @@ type WorkerEnv = {
|
|
|
427
546
|
*/
|
|
428
547
|
RUNTIME_API?: {
|
|
429
548
|
runtimeApiCall(input: {
|
|
549
|
+
contract?: number;
|
|
430
550
|
executorToken: string;
|
|
431
551
|
baseUrl?: string;
|
|
432
552
|
path: string;
|
|
@@ -663,7 +783,7 @@ async function fetchRuntimeApi(
|
|
|
663
783
|
? Math.max(1, Math.ceil(options.timeoutMsOverride))
|
|
664
784
|
: path === '/api/v2/plays/run'
|
|
665
785
|
? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
|
|
666
|
-
: path === '/api/v2/
|
|
786
|
+
: path === '/api/v2/internal/play-runtime/egress-fetch'
|
|
667
787
|
? RUNTIME_API_EGRESS_FETCH_TIMEOUT_MS
|
|
668
788
|
: /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
|
|
669
789
|
? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
|
|
@@ -691,6 +811,10 @@ async function fetchRuntimeApi(
|
|
|
691
811
|
);
|
|
692
812
|
}, timeoutMs);
|
|
693
813
|
});
|
|
814
|
+
const relayAbort = () => controller.abort(abortSignal?.reason);
|
|
815
|
+
if (abortSignal) {
|
|
816
|
+
abortSignal.addEventListener('abort', relayAbort, { once: true });
|
|
817
|
+
}
|
|
694
818
|
try {
|
|
695
819
|
const mergedInit: RequestInit = {
|
|
696
820
|
...init,
|
|
@@ -712,11 +836,10 @@ async function fetchRuntimeApi(
|
|
|
712
836
|
timeoutMs,
|
|
713
837
|
},
|
|
714
838
|
);
|
|
715
|
-
// RUNTIME_API service bindings do not consume RequestInit.signal once the RPC
|
|
716
|
-
// is in flight. After dispatch, wait for the owner call to settle so the
|
|
717
|
-
// durable receipt records the real provider result instead of failing early
|
|
718
|
-
// and inviting duplicate provider work on retry.
|
|
719
839
|
const response = await Promise.race([responsePromise, timeoutPromise]);
|
|
840
|
+
if (abortSignal?.aborted) {
|
|
841
|
+
throw abortError();
|
|
842
|
+
}
|
|
720
843
|
if (await isRuntimeApiBindingNotFoundResponse(response)) {
|
|
721
844
|
throw new Error(
|
|
722
845
|
`[play-harness] RUNTIME_API service binding could not route ${path}; coordinator returned not found.`,
|
|
@@ -732,6 +855,9 @@ async function fetchRuntimeApi(
|
|
|
732
855
|
}
|
|
733
856
|
throw err;
|
|
734
857
|
} finally {
|
|
858
|
+
if (abortSignal) {
|
|
859
|
+
abortSignal.removeEventListener('abort', relayAbort);
|
|
860
|
+
}
|
|
735
861
|
if (timeout) clearTimeout(timeout);
|
|
736
862
|
}
|
|
737
863
|
}
|
|
@@ -748,10 +874,23 @@ async function callRuntimeApiRpcBinding(
|
|
|
748
874
|
if (metadata) headers[EXECUTE_TOOL_METADATA_HEADER] = metadata;
|
|
749
875
|
const contract = h.get(EXECUTE_RESPONSE_CONTRACT_HEADER);
|
|
750
876
|
if (contract) headers[EXECUTE_RESPONSE_CONTRACT_HEADER] = contract;
|
|
877
|
+
const idempotencyKey = h.get('x-deepline-idempotency-key');
|
|
878
|
+
if (idempotencyKey) {
|
|
879
|
+
headers['x-deepline-idempotency-key'] = idempotencyKey;
|
|
880
|
+
}
|
|
881
|
+
const cacheControl = h.get('cache-control');
|
|
882
|
+
if (cacheControl) headers['cache-control'] = cacheControl;
|
|
883
|
+
const pragma = h.get('pragma');
|
|
884
|
+
if (pragma) headers.pragma = pragma;
|
|
885
|
+
const runtimeTestFaultHeader = h.get(PLAY_RUNTIME_TEST_FAULT_HEADER);
|
|
886
|
+
if (runtimeTestFaultHeader) {
|
|
887
|
+
headers[PLAY_RUNTIME_TEST_FAULT_HEADER] = runtimeTestFaultHeader;
|
|
888
|
+
}
|
|
751
889
|
const responseIntent = h.get(EXECUTE_RESPONSE_INTENT_HEADER);
|
|
752
890
|
if (responseIntent) headers[EXECUTE_RESPONSE_INTENT_HEADER] = responseIntent;
|
|
753
891
|
const rawBody = typeof init.body === 'string' ? init.body : '';
|
|
754
892
|
const result = await binding.runtimeApiCall({
|
|
893
|
+
contract: PLAY_RUNTIME_CONTRACT,
|
|
755
894
|
executorToken: authorization.replace(/^Bearer\s+/i, '').trim(),
|
|
756
895
|
baseUrl: input.baseUrl,
|
|
757
896
|
path: input.path,
|
|
@@ -784,6 +923,7 @@ function runtimeApiHeaders(
|
|
|
784
923
|
includeVercelBypass: boolean,
|
|
785
924
|
): Headers {
|
|
786
925
|
const next = new Headers(headers);
|
|
926
|
+
next.set(PLAY_RUNTIME_CONTRACT_HEADER, String(PLAY_RUNTIME_CONTRACT));
|
|
787
927
|
if (includeVercelBypass) {
|
|
788
928
|
const bypassToken = cachedVercelProtectionBypassToken();
|
|
789
929
|
if (bypassToken) {
|
|
@@ -976,9 +1116,172 @@ async function drainRunnerPerfTraces(req: RunRequest): Promise<void> {
|
|
|
976
1116
|
]);
|
|
977
1117
|
}
|
|
978
1118
|
|
|
1119
|
+
let workerRequestSequence = 0;
|
|
1120
|
+
|
|
979
1121
|
function makeRequestId(): string {
|
|
980
|
-
|
|
981
|
-
return
|
|
1122
|
+
workerRequestSequence = (workerRequestSequence + 1) % Number.MAX_SAFE_INTEGER;
|
|
1123
|
+
return `worker-request-${workerRequestSequence}`;
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
function makeRuntimeSheetAttempt(input: {
|
|
1127
|
+
runId: string;
|
|
1128
|
+
runAttempt?: number | null;
|
|
1129
|
+
mapName: string;
|
|
1130
|
+
chunkIndex: number | 'finalize';
|
|
1131
|
+
}): {
|
|
1132
|
+
attemptId: string;
|
|
1133
|
+
attemptOwnerRunId: string;
|
|
1134
|
+
attemptExpiresAt: string | null;
|
|
1135
|
+
attemptSeq: number;
|
|
1136
|
+
} {
|
|
1137
|
+
const runAttempt =
|
|
1138
|
+
typeof input.runAttempt === 'number' && Number.isFinite(input.runAttempt)
|
|
1139
|
+
? Math.max(0, Math.floor(input.runAttempt))
|
|
1140
|
+
: 0;
|
|
1141
|
+
const attemptKey = sha256Hex(
|
|
1142
|
+
JSON.stringify({
|
|
1143
|
+
runId: input.runId,
|
|
1144
|
+
runAttempt,
|
|
1145
|
+
mapName: input.mapName,
|
|
1146
|
+
chunkIndex: input.chunkIndex,
|
|
1147
|
+
}),
|
|
1148
|
+
).slice(0, 16);
|
|
1149
|
+
return {
|
|
1150
|
+
attemptId: `sheet:${input.mapName}:${input.chunkIndex}:attempt-${runAttempt}:${attemptKey}`,
|
|
1151
|
+
attemptOwnerRunId: input.runId,
|
|
1152
|
+
attemptExpiresAt: null,
|
|
1153
|
+
attemptSeq: runAttempt,
|
|
1154
|
+
};
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
class RuntimeSheetAttemptLeaseLostError extends Error {
|
|
1158
|
+
readonly rowKeys: string[];
|
|
1159
|
+
readonly runId: string;
|
|
1160
|
+
readonly attemptId: string;
|
|
1161
|
+
readonly tableNamespace: string;
|
|
1162
|
+
|
|
1163
|
+
constructor(input: {
|
|
1164
|
+
rowKeys: string[];
|
|
1165
|
+
runId: string;
|
|
1166
|
+
attemptId: string;
|
|
1167
|
+
tableNamespace: string;
|
|
1168
|
+
cause?: unknown;
|
|
1169
|
+
}) {
|
|
1170
|
+
const rowSummary =
|
|
1171
|
+
input.rowKeys.length === 1
|
|
1172
|
+
? input.rowKeys[0]!
|
|
1173
|
+
: `${input.rowKeys.length} rows`;
|
|
1174
|
+
super(
|
|
1175
|
+
`Runtime sheet attempt ${input.attemptId} lost ownership of ${rowSummary} in ${input.tableNamespace}.`,
|
|
1176
|
+
{ cause: input.cause },
|
|
1177
|
+
);
|
|
1178
|
+
this.name = 'RuntimeSheetAttemptLeaseLostError';
|
|
1179
|
+
this.rowKeys = input.rowKeys;
|
|
1180
|
+
this.runId = input.runId;
|
|
1181
|
+
this.attemptId = input.attemptId;
|
|
1182
|
+
this.tableNamespace = input.tableNamespace;
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
async function executeWithRuntimeSheetAttemptHeartbeat<T>(input: {
|
|
1187
|
+
req: RunRequest;
|
|
1188
|
+
tableNamespace: string;
|
|
1189
|
+
sheetContract: PlaySheetContract;
|
|
1190
|
+
attemptId?: string | null;
|
|
1191
|
+
attemptOwnerRunId?: string | null;
|
|
1192
|
+
attemptSeq?: number | null;
|
|
1193
|
+
rowKeys: string[];
|
|
1194
|
+
execute: () => Promise<T> | T;
|
|
1195
|
+
}): Promise<T> {
|
|
1196
|
+
const attemptId = input.attemptId?.trim();
|
|
1197
|
+
const rowKeys = [
|
|
1198
|
+
...new Set(input.rowKeys.map((key) => key.trim()).filter(Boolean)),
|
|
1199
|
+
];
|
|
1200
|
+
if (!attemptId || rowKeys.length === 0) {
|
|
1201
|
+
return await input.execute();
|
|
1202
|
+
}
|
|
1203
|
+
const heartbeatIntervalMs = runtimePolicyHeartbeatIntervalMs(
|
|
1204
|
+
input.req,
|
|
1205
|
+
runtimeSheetAttemptLeaseTtlMs(input.req) ??
|
|
1206
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
1207
|
+
);
|
|
1208
|
+
|
|
1209
|
+
let stopped = false;
|
|
1210
|
+
let timeout: ReturnType<typeof setTimeout> | null = null;
|
|
1211
|
+
let rejectLeaseLost!: (error: RuntimeSheetAttemptLeaseLostError) => void;
|
|
1212
|
+
const leaseLost = new Promise<never>((_, reject) => {
|
|
1213
|
+
rejectLeaseLost = reject;
|
|
1214
|
+
});
|
|
1215
|
+
|
|
1216
|
+
const stop = () => {
|
|
1217
|
+
stopped = true;
|
|
1218
|
+
if (timeout !== null) {
|
|
1219
|
+
clearTimeout(timeout);
|
|
1220
|
+
timeout = null;
|
|
1221
|
+
}
|
|
1222
|
+
};
|
|
1223
|
+
|
|
1224
|
+
const heartbeatOnce = async () => {
|
|
1225
|
+
const result = await harnessHeartbeatSheetAttempt({
|
|
1226
|
+
...runtimeSheetSessionScope(input.req),
|
|
1227
|
+
tableNamespace: input.tableNamespace,
|
|
1228
|
+
sheetContract: input.sheetContract,
|
|
1229
|
+
runId: input.req.runId,
|
|
1230
|
+
attemptId,
|
|
1231
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? input.req.runId,
|
|
1232
|
+
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
1233
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
|
|
1234
|
+
keys: rowKeys,
|
|
1235
|
+
});
|
|
1236
|
+
const renewed = new Set(result.renewedKeys);
|
|
1237
|
+
const lostKeys = rowKeys.filter((key) => !renewed.has(key));
|
|
1238
|
+
if (lostKeys.length > 0) {
|
|
1239
|
+
throw new RuntimeSheetAttemptLeaseLostError({
|
|
1240
|
+
rowKeys: lostKeys,
|
|
1241
|
+
runId: input.req.runId,
|
|
1242
|
+
attemptId,
|
|
1243
|
+
tableNamespace: input.tableNamespace,
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
1246
|
+
};
|
|
1247
|
+
|
|
1248
|
+
const schedule = () => {
|
|
1249
|
+
timeout = setTimeout(() => {
|
|
1250
|
+
void (async () => {
|
|
1251
|
+
try {
|
|
1252
|
+
await heartbeatOnce();
|
|
1253
|
+
} catch (error) {
|
|
1254
|
+
stop();
|
|
1255
|
+
if (error instanceof RuntimeSheetAttemptLeaseLostError) {
|
|
1256
|
+
rejectLeaseLost(error);
|
|
1257
|
+
} else {
|
|
1258
|
+
rejectLeaseLost(
|
|
1259
|
+
new RuntimeSheetAttemptLeaseLostError({
|
|
1260
|
+
rowKeys,
|
|
1261
|
+
runId: input.req.runId,
|
|
1262
|
+
attemptId,
|
|
1263
|
+
tableNamespace: input.tableNamespace,
|
|
1264
|
+
cause: error,
|
|
1265
|
+
}),
|
|
1266
|
+
);
|
|
1267
|
+
}
|
|
1268
|
+
return;
|
|
1269
|
+
}
|
|
1270
|
+
if (!stopped) {
|
|
1271
|
+
schedule();
|
|
1272
|
+
}
|
|
1273
|
+
})();
|
|
1274
|
+
}, heartbeatIntervalMs);
|
|
1275
|
+
};
|
|
1276
|
+
|
|
1277
|
+
schedule();
|
|
1278
|
+
const execution = Promise.resolve().then(input.execute);
|
|
1279
|
+
execution.catch(() => {});
|
|
1280
|
+
try {
|
|
1281
|
+
return await Promise.race([execution, leaseLost]);
|
|
1282
|
+
} finally {
|
|
1283
|
+
stop();
|
|
1284
|
+
}
|
|
982
1285
|
}
|
|
983
1286
|
|
|
984
1287
|
/**
|
|
@@ -1004,6 +1307,11 @@ async function postRuntimeApi<T>(
|
|
|
1004
1307
|
baseUrl: string,
|
|
1005
1308
|
executorToken: string,
|
|
1006
1309
|
body: unknown,
|
|
1310
|
+
options: {
|
|
1311
|
+
runtimeTestFaultHeader?: string | null;
|
|
1312
|
+
timeoutMsOverride?: number;
|
|
1313
|
+
timeoutErrorMessage?: string;
|
|
1314
|
+
} = {},
|
|
1007
1315
|
): Promise<T> {
|
|
1008
1316
|
// Routes through the in-process RUNTIME_API service binding. Missing binding
|
|
1009
1317
|
// is an infra error in workers_edge, not a reason to fall back to public HTTP.
|
|
@@ -1016,15 +1324,29 @@ async function postRuntimeApi<T>(
|
|
|
1016
1324
|
) {
|
|
1017
1325
|
let res: Response;
|
|
1018
1326
|
try {
|
|
1019
|
-
res = await fetchRuntimeApi(
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1327
|
+
res = await fetchRuntimeApi(
|
|
1328
|
+
baseUrl,
|
|
1329
|
+
'/api/v2/internal/play-runtime',
|
|
1330
|
+
{
|
|
1331
|
+
method: 'POST',
|
|
1332
|
+
headers: {
|
|
1333
|
+
'content-type': 'application/json',
|
|
1334
|
+
authorization: `Bearer ${executorToken}`,
|
|
1335
|
+
'x-deepline-request-id': makeRequestId(),
|
|
1336
|
+
...(options.runtimeTestFaultHeader?.trim()
|
|
1337
|
+
? {
|
|
1338
|
+
[PLAY_RUNTIME_TEST_FAULT_HEADER]:
|
|
1339
|
+
options.runtimeTestFaultHeader.trim(),
|
|
1340
|
+
}
|
|
1341
|
+
: {}),
|
|
1342
|
+
},
|
|
1343
|
+
body: serializedBody,
|
|
1025
1344
|
},
|
|
1026
|
-
|
|
1027
|
-
|
|
1345
|
+
{
|
|
1346
|
+
timeoutMsOverride: options.timeoutMsOverride,
|
|
1347
|
+
timeoutErrorMessage: options.timeoutErrorMessage,
|
|
1348
|
+
},
|
|
1349
|
+
);
|
|
1028
1350
|
} catch (error) {
|
|
1029
1351
|
lastError = error;
|
|
1030
1352
|
if (
|
|
@@ -1108,9 +1430,10 @@ async function postRuntimeApiBestEffort(
|
|
|
1108
1430
|
baseUrl: string,
|
|
1109
1431
|
executorToken: string,
|
|
1110
1432
|
body: unknown,
|
|
1433
|
+
options: { runtimeTestFaultHeader?: string | null } = {},
|
|
1111
1434
|
): Promise<boolean> {
|
|
1112
1435
|
try {
|
|
1113
|
-
await postRuntimeApi(baseUrl, executorToken, body);
|
|
1436
|
+
await postRuntimeApi(baseUrl, executorToken, body, options);
|
|
1114
1437
|
return true;
|
|
1115
1438
|
} catch (error) {
|
|
1116
1439
|
console.error(
|
|
@@ -1122,6 +1445,43 @@ async function postRuntimeApiBestEffort(
|
|
|
1122
1445
|
}
|
|
1123
1446
|
}
|
|
1124
1447
|
|
|
1448
|
+
async function readDurableChildRunTerminalState(input: {
|
|
1449
|
+
baseUrl: string;
|
|
1450
|
+
executorToken: string;
|
|
1451
|
+
parentRunId: string;
|
|
1452
|
+
childRunId: string;
|
|
1453
|
+
childPlayName: string;
|
|
1454
|
+
timeoutMs?: number;
|
|
1455
|
+
}): Promise<{ data?: unknown } | null> {
|
|
1456
|
+
const response = await postRuntimeApi<{
|
|
1457
|
+
state?: {
|
|
1458
|
+
data?: unknown;
|
|
1459
|
+
reason?: string;
|
|
1460
|
+
status?: string;
|
|
1461
|
+
runId?: string;
|
|
1462
|
+
playName?: string;
|
|
1463
|
+
parentRunId?: string | null;
|
|
1464
|
+
} | null;
|
|
1465
|
+
}>(
|
|
1466
|
+
input.baseUrl,
|
|
1467
|
+
input.executorToken,
|
|
1468
|
+
{
|
|
1469
|
+
action: 'read_child_run_terminal_snapshot',
|
|
1470
|
+
parentRunId: input.parentRunId,
|
|
1471
|
+
childRunId: input.childRunId,
|
|
1472
|
+
childPlayName: input.childPlayName,
|
|
1473
|
+
},
|
|
1474
|
+
{
|
|
1475
|
+
timeoutMsOverride: input.timeoutMs,
|
|
1476
|
+
timeoutErrorMessage: `[play-harness] child run terminal snapshot read timed out after ${Math.max(
|
|
1477
|
+
1,
|
|
1478
|
+
Math.ceil(input.timeoutMs ?? RUNTIME_API_TIMEOUT_MS),
|
|
1479
|
+
)}ms. childRunId=${input.childRunId}`,
|
|
1480
|
+
},
|
|
1481
|
+
);
|
|
1482
|
+
return response.state ?? null;
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1125
1485
|
function workflowEventType(name: string): string {
|
|
1126
1486
|
const normalized = name
|
|
1127
1487
|
.trim()
|
|
@@ -1231,6 +1591,27 @@ async function hashChildPlayEventKey(input: unknown): Promise<string> {
|
|
|
1231
1591
|
return (await hashJson(input)).slice(0, 32);
|
|
1232
1592
|
}
|
|
1233
1593
|
|
|
1594
|
+
function workerChildRevisionFingerprint(input: {
|
|
1595
|
+
playName: string;
|
|
1596
|
+
manifest: PlayRuntimeManifest | null | undefined;
|
|
1597
|
+
}): string | null {
|
|
1598
|
+
const manifest = input.manifest;
|
|
1599
|
+
if (!manifest) return null;
|
|
1600
|
+
return sha256Hex(
|
|
1601
|
+
stableStringify({
|
|
1602
|
+
playId: manifest.playName?.trim() || input.playName,
|
|
1603
|
+
codeFormat: 'cjs_module',
|
|
1604
|
+
artifactHash: manifest.artifactHash ?? null,
|
|
1605
|
+
graphHash: manifest.graphHash ?? null,
|
|
1606
|
+
sourceHash: null,
|
|
1607
|
+
sourceCodeHash:
|
|
1608
|
+
typeof manifest.sourceCode === 'string'
|
|
1609
|
+
? sha256Hex(manifest.sourceCode)
|
|
1610
|
+
: null,
|
|
1611
|
+
}),
|
|
1612
|
+
);
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1234
1615
|
async function childPlayEventKey(input: {
|
|
1235
1616
|
key: string;
|
|
1236
1617
|
workflowId: string;
|
|
@@ -1327,7 +1708,14 @@ async function signalParentPlayTerminal(input: {
|
|
|
1327
1708
|
|
|
1328
1709
|
async function executeTool(
|
|
1329
1710
|
req: RunRequest,
|
|
1330
|
-
args: {
|
|
1711
|
+
args: {
|
|
1712
|
+
id: string;
|
|
1713
|
+
toolId: string;
|
|
1714
|
+
input: Record<string, unknown>;
|
|
1715
|
+
durableCallReceiptKey?: string | null;
|
|
1716
|
+
executionAuthScopeDigest?: string | null;
|
|
1717
|
+
providerIdempotencyKey?: string | null;
|
|
1718
|
+
},
|
|
1331
1719
|
workflowStep?: WorkflowStep,
|
|
1332
1720
|
onProviderBackpressure?: (retryAfterMs: number) => void,
|
|
1333
1721
|
onRetryAttempt?: () => void,
|
|
@@ -1365,7 +1753,14 @@ async function executeTool(
|
|
|
1365
1753
|
|
|
1366
1754
|
async function executeToolWithLifecycle(
|
|
1367
1755
|
req: RunRequest,
|
|
1368
|
-
args: {
|
|
1756
|
+
args: {
|
|
1757
|
+
id: string;
|
|
1758
|
+
toolId: string;
|
|
1759
|
+
input: Record<string, unknown>;
|
|
1760
|
+
durableCallReceiptKey?: string | null;
|
|
1761
|
+
executionAuthScopeDigest?: string | null;
|
|
1762
|
+
providerIdempotencyKey?: string | null;
|
|
1763
|
+
},
|
|
1369
1764
|
workflowStep: WorkflowStep | undefined,
|
|
1370
1765
|
callbacks: WorkerCtxCallbacks | undefined,
|
|
1371
1766
|
onProviderBackpressure?: (retryAfterMs: number) => void,
|
|
@@ -1461,24 +1856,29 @@ async function waitForSyntheticIntegrationEvent(
|
|
|
1461
1856
|
typeof input.timeout_ms === 'number' && Number.isFinite(input.timeout_ms)
|
|
1462
1857
|
? Math.max(1, Math.round(input.timeout_ms))
|
|
1463
1858
|
: 30_000;
|
|
1464
|
-
await postRuntimeApiBestEffort(
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1859
|
+
await postRuntimeApiBestEffort(
|
|
1860
|
+
req.baseUrl,
|
|
1861
|
+
req.executorToken,
|
|
1862
|
+
{
|
|
1863
|
+
action: 'append_run_events',
|
|
1864
|
+
playId: req.runId,
|
|
1865
|
+
events: [
|
|
1866
|
+
{
|
|
1867
|
+
type: 'log.appended',
|
|
1868
|
+
runId: req.runId,
|
|
1869
|
+
// 'system' (windowed text-dedupe channel), NOT 'worker': this line is
|
|
1870
|
+
// emitted outside the harness log buffer, so it has no positional
|
|
1871
|
+
// channelOffset and must not pollute the worker channel cursor.
|
|
1872
|
+
source: 'system',
|
|
1873
|
+
occurredAt: nowMs(),
|
|
1874
|
+
lines: [
|
|
1875
|
+
`Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
|
|
1876
|
+
],
|
|
1877
|
+
} satisfies PlayRunLedgerEvent,
|
|
1878
|
+
],
|
|
1879
|
+
},
|
|
1880
|
+
{ runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null },
|
|
1881
|
+
);
|
|
1482
1882
|
try {
|
|
1483
1883
|
const event = (await (
|
|
1484
1884
|
workflowStep.waitForEvent as unknown as (
|
|
@@ -1526,7 +1926,14 @@ async function waitForSyntheticIntegrationEvent(
|
|
|
1526
1926
|
|
|
1527
1927
|
async function callToolDirect(
|
|
1528
1928
|
req: RunRequest,
|
|
1529
|
-
args: {
|
|
1929
|
+
args: {
|
|
1930
|
+
id: string;
|
|
1931
|
+
toolId: string;
|
|
1932
|
+
input: Record<string, unknown>;
|
|
1933
|
+
durableCallReceiptKey?: string | null;
|
|
1934
|
+
executionAuthScopeDigest?: string | null;
|
|
1935
|
+
providerIdempotencyKey?: string | null;
|
|
1936
|
+
},
|
|
1530
1937
|
onProviderBackpressure?: (retryAfterMs: number) => void,
|
|
1531
1938
|
// Invoked once per in-process retry attempt (429 / retryable 5xx / synthetic
|
|
1532
1939
|
// transient) so the Governor charges chargeBudget('retry') per attempt — the
|
|
@@ -1542,6 +1949,16 @@ async function callToolDirect(
|
|
|
1542
1949
|
): Promise<ToolExecuteResult> {
|
|
1543
1950
|
const { id, toolId, input } = args;
|
|
1544
1951
|
const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
|
|
1952
|
+
const durableCallReceiptKey = args.durableCallReceiptKey?.trim() || null;
|
|
1953
|
+
const executionAuthScopeDigest =
|
|
1954
|
+
durableCallReceiptKey && args.executionAuthScopeDigest
|
|
1955
|
+
? args.executionAuthScopeDigest.trim() || null
|
|
1956
|
+
: null;
|
|
1957
|
+
const providerIdempotencyKey =
|
|
1958
|
+
args.providerIdempotencyKey?.trim() || durableCallReceiptKey;
|
|
1959
|
+
const deeplineRequestId = providerIdempotencyKey
|
|
1960
|
+
? `ctx-tool-${sha256Hex(providerIdempotencyKey).slice(0, 32)}`
|
|
1961
|
+
: null;
|
|
1545
1962
|
let lastError: Error | null = null;
|
|
1546
1963
|
const httpFailureAttempts = createToolExecuteHttpFailureAttemptTracker();
|
|
1547
1964
|
let requestAttempt = 0;
|
|
@@ -1564,7 +1981,12 @@ async function callToolDirect(
|
|
|
1564
1981
|
headers: {
|
|
1565
1982
|
'content-type': 'application/json',
|
|
1566
1983
|
authorization: `Bearer ${req.executorToken}`,
|
|
1567
|
-
'x-deepline-request-id':
|
|
1984
|
+
'x-deepline-request-id':
|
|
1985
|
+
deeplineRequestId ??
|
|
1986
|
+
`${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
|
|
1987
|
+
...(providerIdempotencyKey
|
|
1988
|
+
? { 'x-deepline-idempotency-key': providerIdempotencyKey }
|
|
1989
|
+
: {}),
|
|
1568
1990
|
[EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
|
|
1569
1991
|
[EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
|
|
1570
1992
|
[EXECUTE_TOOL_METADATA_HEADER]: 'true',
|
|
@@ -1573,6 +1995,16 @@ async function callToolDirect(
|
|
|
1573
1995
|
payload: input,
|
|
1574
1996
|
metadata: {
|
|
1575
1997
|
parent_run_id: req.runId,
|
|
1998
|
+
...(durableCallReceiptKey
|
|
1999
|
+
? {
|
|
2000
|
+
durable_call_receipt_key: durableCallReceiptKey,
|
|
2001
|
+
...(executionAuthScopeDigest
|
|
2002
|
+
? {
|
|
2003
|
+
execution_auth_scope_digest: executionAuthScopeDigest,
|
|
2004
|
+
}
|
|
2005
|
+
: {}),
|
|
2006
|
+
}
|
|
2007
|
+
: {}),
|
|
1576
2008
|
...(directOptions?.customerDbDataset
|
|
1577
2009
|
? {
|
|
1578
2010
|
query_result_dataset: {
|
|
@@ -1605,6 +2037,9 @@ async function callToolDirect(
|
|
|
1605
2037
|
: {}),
|
|
1606
2038
|
}
|
|
1607
2039
|
: {}),
|
|
2040
|
+
...(providerIdempotencyKey
|
|
2041
|
+
? { provider_idempotency_key: providerIdempotencyKey }
|
|
2042
|
+
: {}),
|
|
1608
2043
|
},
|
|
1609
2044
|
...(req.integrationMode
|
|
1610
2045
|
? { integration_mode: req.integrationMode }
|
|
@@ -1652,6 +2087,13 @@ async function callToolDirect(
|
|
|
1652
2087
|
}
|
|
1653
2088
|
|
|
1654
2089
|
const text = await res.text().catch(() => '');
|
|
2090
|
+
const authScopeChanged = parseToolExecuteAuthScopeChangedError({
|
|
2091
|
+
status: res.status,
|
|
2092
|
+
bodyText: text,
|
|
2093
|
+
});
|
|
2094
|
+
if (authScopeChanged) {
|
|
2095
|
+
throw authScopeChanged;
|
|
2096
|
+
}
|
|
1655
2097
|
const httpFailureAttempt = httpFailureAttempts.next({
|
|
1656
2098
|
toolId,
|
|
1657
2099
|
status: res.status,
|
|
@@ -1685,48 +2127,6 @@ async function callToolDirect(
|
|
|
1685
2127
|
throw lastError ?? new Error(`tool ${toolId} failed before execution.`);
|
|
1686
2128
|
}
|
|
1687
2129
|
|
|
1688
|
-
function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
|
|
1689
|
-
if (toolId === 'test_rate_limit') {
|
|
1690
|
-
// Batched members resolve metadata through this fallback because the lean
|
|
1691
|
-
// worker does not bundle the catalog. It MUST mirror the same
|
|
1692
|
-
// `email_status: emailStatus({...})` contract registered in
|
|
1693
|
-
// src/lib/integrations/test/index.ts so batched results normalize to the
|
|
1694
|
-
// same rich email_status OBJECT (status from statusMap, catch_all as a
|
|
1695
|
-
// signal) that the single-execute real-metadata path produces. The legacy
|
|
1696
|
-
// `transforms:['emailStatus']` + mx_security_gateway→'catch_all' string
|
|
1697
|
-
// override coarsened email_status into a bare string and predates the
|
|
1698
|
-
// emailStatus object contract (#1466).
|
|
1699
|
-
return {
|
|
1700
|
-
toolId,
|
|
1701
|
-
extractors: {
|
|
1702
|
-
email_status: {
|
|
1703
|
-
paths: ['email_status'],
|
|
1704
|
-
emailStatus: {
|
|
1705
|
-
provider: 'test',
|
|
1706
|
-
rawStatus: ['email_status'],
|
|
1707
|
-
catchAll: ['mx_security_gateway'],
|
|
1708
|
-
statusMap: {
|
|
1709
|
-
valid: { status: 'valid', verdict: 'send', verified: true },
|
|
1710
|
-
invalid: { status: 'invalid', verdict: 'drop', verified: false },
|
|
1711
|
-
catch_all: {
|
|
1712
|
-
status: 'catch_all',
|
|
1713
|
-
verdict: 'verify_next',
|
|
1714
|
-
verified: false,
|
|
1715
|
-
},
|
|
1716
|
-
unknown: { status: 'unknown', verdict: 'hold', verified: false },
|
|
1717
|
-
},
|
|
1718
|
-
},
|
|
1719
|
-
},
|
|
1720
|
-
},
|
|
1721
|
-
targetGetters: {
|
|
1722
|
-
email: ['email', 'value'],
|
|
1723
|
-
email_status: ['email_status'],
|
|
1724
|
-
},
|
|
1725
|
-
};
|
|
1726
|
-
}
|
|
1727
|
-
return { toolId };
|
|
1728
|
-
}
|
|
1729
|
-
|
|
1730
2130
|
function wrapWorkerToolResult(
|
|
1731
2131
|
toolId: string,
|
|
1732
2132
|
result: unknown,
|
|
@@ -1767,10 +2167,10 @@ function attachWorkerCustomerDbDatasetResult(
|
|
|
1767
2167
|
typeof options.input.sql === 'string'
|
|
1768
2168
|
? options.input.sql
|
|
1769
2169
|
: typeof options.input.query === 'string'
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
2170
|
+
? options.input.query
|
|
2171
|
+
: typeof raw?.sql === 'string'
|
|
2172
|
+
? raw.sql
|
|
2173
|
+
: null;
|
|
1774
2174
|
if (!dataset || totalRows === null || !sql) {
|
|
1775
2175
|
return result;
|
|
1776
2176
|
}
|
|
@@ -1880,1455 +2280,93 @@ type RecordedStepProgramOutput = {
|
|
|
1880
2280
|
|
|
1881
2281
|
type WorkerStepProgramRecorder = {
|
|
1882
2282
|
parentField: string;
|
|
1883
|
-
path: string[];
|
|
1884
|
-
outputs: RecordedStepProgramOutput[];
|
|
1885
|
-
onOutput?: (output: RecordedStepProgramOutput) => void | Promise<void>;
|
|
1886
|
-
};
|
|
1887
|
-
|
|
1888
|
-
type WorkerStepResolution = {
|
|
1889
|
-
value: unknown;
|
|
1890
|
-
status?: 'skipped';
|
|
1891
|
-
};
|
|
1892
|
-
|
|
1893
|
-
type WorkerToolBatchRequest = {
|
|
1894
|
-
id: string;
|
|
1895
|
-
cacheKey: string;
|
|
1896
|
-
receiptKey: string | null;
|
|
1897
|
-
force: boolean;
|
|
1898
|
-
receiptWaitMaxAttempts: number;
|
|
1899
|
-
runtimeTimeoutMs?: number;
|
|
1900
|
-
toolId: string;
|
|
1901
|
-
input: Record<string, unknown>;
|
|
1902
|
-
workflowStep?: WorkflowStep;
|
|
1903
|
-
directOptions?: WorkerToolDirectOptions;
|
|
1904
|
-
resolve: (value: unknown) => void;
|
|
1905
|
-
reject: (error: unknown) => void;
|
|
1906
|
-
};
|
|
1907
|
-
|
|
1908
|
-
type ClaimedWorkerToolBatchRequest = {
|
|
1909
|
-
request: WorkerToolBatchRequest;
|
|
1910
|
-
receiptKey: string | null;
|
|
1911
|
-
followers: WorkerToolBatchRequest[];
|
|
1912
|
-
};
|
|
1913
|
-
|
|
1914
|
-
type PreparedWorkerToolBatchRequests = {
|
|
1915
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[];
|
|
1916
|
-
deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[];
|
|
1917
|
-
};
|
|
1918
|
-
|
|
1919
|
-
function resolveClaimedWorkerToolRuntimeTimeoutMs(
|
|
1920
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
1921
|
-
input: { runtimeDeadlineMs?: number },
|
|
1922
|
-
): number | undefined {
|
|
1923
|
-
return resolveWorkerToolRuntimeTimeoutMs(claimedRequests, {
|
|
1924
|
-
resolveOwnerTimeoutMs: (request) =>
|
|
1925
|
-
resolveToolRuntimeApiTimeout({
|
|
1926
|
-
requestInput: request.input,
|
|
1927
|
-
timeoutMs: request.runtimeTimeoutMs,
|
|
1928
|
-
runtimeDeadlineMs: input.runtimeDeadlineMs,
|
|
1929
|
-
}).timeoutMs,
|
|
1930
|
-
});
|
|
1931
|
-
}
|
|
1932
|
-
|
|
1933
|
-
const WORKER_TOOL_BATCH_GRACE_MS = 250;
|
|
1934
|
-
const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
|
|
1935
|
-
const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
|
|
1936
|
-
const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
|
|
1937
|
-
const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 1_000;
|
|
1938
|
-
const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
|
|
1939
|
-
/**
|
|
1940
|
-
* Bounded number of per-row failure samples carried in chunk summaries and the
|
|
1941
|
-
* map's terminal partial-failure log. Every failed row is persisted with its
|
|
1942
|
-
* full error in the runtime sheet; the samples just keep run logs readable.
|
|
1943
|
-
*/
|
|
1944
|
-
const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
|
|
1945
|
-
|
|
1946
|
-
class RuntimeReceiptPersistenceError extends Error {
|
|
1947
|
-
readonly errors: unknown[];
|
|
1948
|
-
|
|
1949
|
-
constructor(message: string, errors: unknown[] = []) {
|
|
1950
|
-
super(message);
|
|
1951
|
-
this.name = 'RuntimeReceiptPersistenceError';
|
|
1952
|
-
this.errors = errors;
|
|
1953
|
-
}
|
|
1954
|
-
}
|
|
1955
|
-
|
|
1956
|
-
function isRuntimeReceiptPersistenceError(error: unknown): boolean {
|
|
1957
|
-
if (!error || typeof error !== 'object') return false;
|
|
1958
|
-
if (error instanceof RuntimeReceiptPersistenceError) return true;
|
|
1959
|
-
if (
|
|
1960
|
-
error instanceof Error &&
|
|
1961
|
-
error.name === 'RuntimeReceiptPersistenceError'
|
|
1962
|
-
) {
|
|
1963
|
-
return true;
|
|
1964
|
-
}
|
|
1965
|
-
const nestedErrors = (error as { errors?: unknown }).errors;
|
|
1966
|
-
return (
|
|
1967
|
-
Array.isArray(nestedErrors) &&
|
|
1968
|
-
nestedErrors.some(isRuntimeReceiptPersistenceError)
|
|
1969
|
-
);
|
|
1970
|
-
}
|
|
1971
|
-
|
|
1972
|
-
function isRunFatalWorkerRowError(error: unknown): boolean {
|
|
1973
|
-
return (
|
|
1974
|
-
isCoordinatorProgressAuthorizationError(error) ||
|
|
1975
|
-
isRowIsolationExemptError(error) ||
|
|
1976
|
-
isRuntimeReceiptPersistenceError(error) ||
|
|
1977
|
-
isRuntimePersistenceCircuitOpenError(error)
|
|
1978
|
-
);
|
|
1979
|
-
}
|
|
1980
|
-
|
|
1981
|
-
/**
|
|
1982
|
-
* completeReceipt salvage retry ladder. On a persistence failure after a billed
|
|
1983
|
-
* call we retry the receipt completion past the server-side 5s connect window:
|
|
1984
|
-
* 1 initial attempt + 3 retries at 1s / 3s / 9s (single isolate — no jitter).
|
|
1985
|
-
* Bounded by the run deadline (`runtimeDeadlineMs`); we stop early unless there
|
|
1986
|
-
* is at least one backoff's worth plus headroom of budget left.
|
|
1987
|
-
*/
|
|
1988
|
-
const COMPLETE_RECEIPT_RETRY_BACKOFFS_MS = [1_000, 3_000, 9_000] as const;
|
|
1989
|
-
const COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS = 10_000;
|
|
1990
|
-
|
|
1991
|
-
// Fallback batch-chunk parallelism when a tool declares no provider rate hints.
|
|
1992
|
-
// Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
|
|
1993
|
-
// their previous batching behavior; declared providers tighten via the
|
|
1994
|
-
// Governor's suggestedParallelism.
|
|
1995
|
-
const WORKER_TOOL_BATCH_DEFAULT_PARALLELISM = 4;
|
|
1996
|
-
|
|
1997
|
-
/**
|
|
1998
|
-
* In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
|
|
1999
|
-
* throughput pacing (provider or Deepline limiter), not a tool defect, so it
|
|
2000
|
-
* gets more patience than the generic 3-attempt budget: with retry-after-aware
|
|
2001
|
-
* escalating delays (capped at 5s) this absorbs roughly 25s of sustained
|
|
2002
|
-
* throttling before the call fails. Every retry still charges the Governor's
|
|
2003
|
-
* retry budget, so a runaway storm stays bounded and loud.
|
|
2004
|
-
*/
|
|
2005
|
-
|
|
2006
|
-
function sleepWorkerMs(ms: number): Promise<void> {
|
|
2007
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2008
|
-
}
|
|
2009
|
-
|
|
2010
|
-
function workerDurableToolCallCacheKey(input: {
|
|
2011
|
-
req: RunRequest;
|
|
2012
|
-
toolId: string;
|
|
2013
|
-
requestInput: Record<string, unknown>;
|
|
2014
|
-
providerActionVersion: string;
|
|
2015
|
-
staleAfterSeconds?: number | null;
|
|
2016
|
-
}): string {
|
|
2017
|
-
return buildDurableToolCallCacheKey({
|
|
2018
|
-
orgId: input.req.orgId,
|
|
2019
|
-
playId: input.req.playName,
|
|
2020
|
-
toolId: input.toolId,
|
|
2021
|
-
requestInput: input.requestInput,
|
|
2022
|
-
authScopeDigest: buildDurableToolCallAuthScopeDigest({
|
|
2023
|
-
orgId: input.req.orgId,
|
|
2024
|
-
userEmail: input.req.userEmail,
|
|
2025
|
-
toolId: input.toolId,
|
|
2026
|
-
}),
|
|
2027
|
-
providerActionVersion: input.providerActionVersion,
|
|
2028
|
-
staleAfterSeconds: input.staleAfterSeconds,
|
|
2029
|
-
});
|
|
2030
|
-
}
|
|
2031
|
-
|
|
2032
|
-
function workerRuntimeReceiptKey(input: {
|
|
2033
|
-
req: RunRequest;
|
|
2034
|
-
key: string;
|
|
2035
|
-
}): string {
|
|
2036
|
-
const orgId = input.req.orgId?.trim() || 'org';
|
|
2037
|
-
if (input.key.startsWith(`ctx:${orgId}:`)) {
|
|
2038
|
-
return input.key;
|
|
2039
|
-
}
|
|
2040
|
-
return buildScopedWorkReceiptKey({
|
|
2041
|
-
orgId: input.req.orgId,
|
|
2042
|
-
playName: input.req.playName,
|
|
2043
|
-
key: input.key,
|
|
2044
|
-
});
|
|
2045
|
-
}
|
|
2046
|
-
|
|
2047
|
-
function stepProgramColumnName(parentField: string, stepId: string): string {
|
|
2048
|
-
return sqlSafePlayColumnName(`${parentField}.${stepId}`);
|
|
2049
|
-
}
|
|
2050
|
-
|
|
2051
|
-
class WorkerToolBatchScheduler {
|
|
2052
|
-
private queue: WorkerToolBatchRequest[] = [];
|
|
2053
|
-
private scheduled = false;
|
|
2054
|
-
|
|
2055
|
-
constructor(
|
|
2056
|
-
private readonly req: RunRequest,
|
|
2057
|
-
private readonly governor: PlayExecutionGovernor,
|
|
2058
|
-
private readonly resolvePacing: WorkerPacingResolver,
|
|
2059
|
-
private readonly resolveToolActionCacheVersion: WorkerToolActionCacheVersionResolver,
|
|
2060
|
-
private readonly abortSignal?: AbortSignal,
|
|
2061
|
-
private readonly onRequestsSettled?: (count: number) => void,
|
|
2062
|
-
private readonly callbacks?: WorkerCtxCallbacks,
|
|
2063
|
-
private readonly receiptStore?: WorkerRuntimeReceiptStore,
|
|
2064
|
-
private readonly allowLocalRetryReceipts = false,
|
|
2065
|
-
private readonly runtimeDeadlineMs?: number,
|
|
2066
|
-
// Map-wide (or ctx-wide for the root scheduler) persistence-failure circuit
|
|
2067
|
-
// breaker. Shared by every row worker so the first persistence failure
|
|
2068
|
-
// halts NEW provider dispatch across the whole map.
|
|
2069
|
-
private readonly latch: RuntimePersistenceLatch = createRuntimePersistenceLatch(),
|
|
2070
|
-
// Run-scoped buffer for orphaned billed results whose receipt could not be
|
|
2071
|
-
// completed even after the retry ladder.
|
|
2072
|
-
private readonly salvage?: ReceiptSalvageBuffer,
|
|
2073
|
-
) {}
|
|
2074
|
-
|
|
2075
|
-
/**
|
|
2076
|
-
* Run `completeReceipt`/`completeReceipts` with the salvage retry ladder.
|
|
2077
|
-
* Retries a receipt-completion failure past the server-side 5s connect window
|
|
2078
|
-
* so a transient blip does not orphan a billed result. Bounded by the run
|
|
2079
|
-
* deadline; skips retries when the latch has already burned a ladder in this
|
|
2080
|
-
* map so many in-flight orphans do not each serialize a long loop.
|
|
2081
|
-
*/
|
|
2082
|
-
private async completeReceiptWithRetryLadder<T>(
|
|
2083
|
-
run: () => Promise<T>,
|
|
2084
|
-
onGiveUp?: (attempts: number) => void,
|
|
2085
|
-
): Promise<T> {
|
|
2086
|
-
let attempt = 0;
|
|
2087
|
-
// A ladder that has already started retrying keeps its full budget even if
|
|
2088
|
-
// a peer trips the latch mid-flight — only FRESH entrants collapse once the
|
|
2089
|
-
// failure is confirmed. This is what guarantees the first ladder rides out
|
|
2090
|
-
// a genuine transient blip.
|
|
2091
|
-
let committedToFullLadder = false;
|
|
2092
|
-
const giveUp = (error: unknown): never => {
|
|
2093
|
-
this.latch.latchRetryBudgetExhausted = true;
|
|
2094
|
-
onGiveUp?.(attempt);
|
|
2095
|
-
throw error;
|
|
2096
|
-
};
|
|
2097
|
-
while (true) {
|
|
2098
|
-
try {
|
|
2099
|
-
return await run();
|
|
2100
|
-
} catch (error) {
|
|
2101
|
-
attempt += 1;
|
|
2102
|
-
if (
|
|
2103
|
-
shouldShortenCompleteReceiptLadder(this.latch, committedToFullLadder) ||
|
|
2104
|
-
attempt > COMPLETE_RECEIPT_RETRY_BACKOFFS_MS.length ||
|
|
2105
|
-
this.abortSignal?.aborted
|
|
2106
|
-
) {
|
|
2107
|
-
return giveUp(error);
|
|
2108
|
-
}
|
|
2109
|
-
const backoffMs = COMPLETE_RECEIPT_RETRY_BACKOFFS_MS[attempt - 1]!;
|
|
2110
|
-
// resolveRuntimeDeadlineRemainingMs THROWS WorkflowAbortError once the
|
|
2111
|
-
// run deadline has passed — the correct loud outcome (there is no
|
|
2112
|
-
// separate per-chunk step timeout). Route that throw through giveUp so
|
|
2113
|
-
// the billed result is still salvaged and the ladder budget flag is
|
|
2114
|
-
// set: a billed output must never lose its durable trace because the
|
|
2115
|
-
// clock ran out mid-ladder.
|
|
2116
|
-
let remainingMs: number;
|
|
2117
|
-
try {
|
|
2118
|
-
remainingMs = resolveRuntimeDeadlineRemainingMs(
|
|
2119
|
-
this.runtimeDeadlineMs,
|
|
2120
|
-
);
|
|
2121
|
-
} catch (deadlineError) {
|
|
2122
|
-
return giveUp(deadlineError);
|
|
2123
|
-
}
|
|
2124
|
-
if (remainingMs <= backoffMs + COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS) {
|
|
2125
|
-
return giveUp(error);
|
|
2126
|
-
}
|
|
2127
|
-
// Past every give-up gate: this ladder is now committed to the full
|
|
2128
|
-
// budget and must not be collapsed by a peer tripping the latch mid-sleep.
|
|
2129
|
-
committedToFullLadder = true;
|
|
2130
|
-
await sleepWorkerMs(backoffMs);
|
|
2131
|
-
}
|
|
2132
|
-
}
|
|
2133
|
-
}
|
|
2134
|
-
|
|
2135
|
-
/**
|
|
2136
|
-
* Record a billed-but-unpersisted tool output into the run-scoped salvage
|
|
2137
|
-
* buffer. Called only after the completeReceipt ladder is exhausted, with the
|
|
2138
|
-
* exact serialized payload a receipt would have stored.
|
|
2139
|
-
*/
|
|
2140
|
-
private recordReceiptSalvage(input: {
|
|
2141
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
2142
|
-
output: unknown;
|
|
2143
|
-
attempts: number;
|
|
2144
|
-
}): void {
|
|
2145
|
-
if (!this.salvage || !input.claimed.receiptKey) return;
|
|
2146
|
-
this.salvage.push({
|
|
2147
|
-
receiptKey: input.claimed.receiptKey,
|
|
2148
|
-
toolId: input.claimed.request.toolId,
|
|
2149
|
-
cacheKey: input.claimed.request.cacheKey,
|
|
2150
|
-
output: input.output,
|
|
2151
|
-
completeAttempts: input.attempts,
|
|
2152
|
-
firstErrorAt: nowMs(),
|
|
2153
|
-
});
|
|
2154
|
-
}
|
|
2155
|
-
|
|
2156
|
-
/**
|
|
2157
|
-
* Report a provider 429 / Retry-After back into the Governor's shared pacer
|
|
2158
|
-
* so future acquires for this (org, provider) bucket back off across all
|
|
2159
|
-
* isolates. Provider comes from the same pacing resolver the Governor uses
|
|
2160
|
-
* (the worker has no local catalog), so callers pass only the toolId.
|
|
2161
|
-
*/
|
|
2162
|
-
private reportBackpressure(toolId: string, retryAfterMs: number): void {
|
|
2163
|
-
if (retryAfterMs <= 0) return;
|
|
2164
|
-
void (async () => {
|
|
2165
|
-
const pacing = await this.resolvePacing(toolId).catch(() => null);
|
|
2166
|
-
if (pacing?.provider) {
|
|
2167
|
-
this.governor.reportProviderBackpressure({
|
|
2168
|
-
provider: pacing.provider,
|
|
2169
|
-
retryAfterMs,
|
|
2170
|
-
});
|
|
2171
|
-
}
|
|
2172
|
-
})();
|
|
2173
|
-
}
|
|
2174
|
-
|
|
2175
|
-
async execute(
|
|
2176
|
-
id: string,
|
|
2177
|
-
toolId: string,
|
|
2178
|
-
input: Record<string, unknown>,
|
|
2179
|
-
workflowStep?: WorkflowStep,
|
|
2180
|
-
options?: { force?: boolean; staleAfterSeconds?: number | null },
|
|
2181
|
-
runtimeOptions?: { timeoutMs?: number; receiptWaitMs?: number },
|
|
2182
|
-
): Promise<unknown> {
|
|
2183
|
-
const providerActionVersion =
|
|
2184
|
-
await this.resolveToolActionCacheVersion(toolId);
|
|
2185
|
-
return await new Promise((resolve, reject) => {
|
|
2186
|
-
const queuedAt = nowMs();
|
|
2187
|
-
const receiptKey = workerDurableToolCallCacheKey({
|
|
2188
|
-
req: this.req,
|
|
2189
|
-
toolId,
|
|
2190
|
-
requestInput: input,
|
|
2191
|
-
providerActionVersion,
|
|
2192
|
-
staleAfterSeconds: options?.staleAfterSeconds,
|
|
2193
|
-
});
|
|
2194
|
-
const queryResultDatasetRead = isQueryResultDatasetReadRequest(
|
|
2195
|
-
toolId,
|
|
2196
|
-
input,
|
|
2197
|
-
);
|
|
2198
|
-
const receiptEligible = !queryResultDatasetRead;
|
|
2199
|
-
this.queue.push({
|
|
2200
|
-
id,
|
|
2201
|
-
cacheKey: receiptEligible ? receiptKey : '',
|
|
2202
|
-
receiptKey: receiptEligible ? receiptKey : null,
|
|
2203
|
-
force: options?.force === true || !!this.req.force,
|
|
2204
|
-
receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
2205
|
-
typeof runtimeOptions?.receiptWaitMs === 'number'
|
|
2206
|
-
? { max_wait_ms: runtimeOptions.receiptWaitMs }
|
|
2207
|
-
: input,
|
|
2208
|
-
),
|
|
2209
|
-
...(typeof runtimeOptions?.timeoutMs === 'number'
|
|
2210
|
-
? { runtimeTimeoutMs: runtimeOptions.timeoutMs }
|
|
2211
|
-
: {}),
|
|
2212
|
-
toolId,
|
|
2213
|
-
input,
|
|
2214
|
-
workflowStep,
|
|
2215
|
-
...(queryResultDatasetRead
|
|
2216
|
-
? {
|
|
2217
|
-
directOptions: {
|
|
2218
|
-
customerDbDataset: {
|
|
2219
|
-
offset: 0,
|
|
2220
|
-
pageSize: QUERY_RESULT_DATASET_PAGE_SIZE,
|
|
2221
|
-
},
|
|
2222
|
-
},
|
|
2223
|
-
}
|
|
2224
|
-
: {}),
|
|
2225
|
-
resolve: (value) => {
|
|
2226
|
-
recordRunnerPerfTrace({
|
|
2227
|
-
req: this.req,
|
|
2228
|
-
phase: 'runner.tool.request',
|
|
2229
|
-
ms: nowMs() - queuedAt,
|
|
2230
|
-
extra: { id, toolId },
|
|
2231
|
-
});
|
|
2232
|
-
resolve(value);
|
|
2233
|
-
},
|
|
2234
|
-
reject,
|
|
2235
|
-
});
|
|
2236
|
-
this.scheduleDrain();
|
|
2237
|
-
});
|
|
2238
|
-
}
|
|
2239
|
-
|
|
2240
|
-
private scheduleDrain(): void {
|
|
2241
|
-
if (this.scheduled) return;
|
|
2242
|
-
this.scheduled = true;
|
|
2243
|
-
queueMicrotask(async () => {
|
|
2244
|
-
if (this.hasBatchableQueuedTool()) {
|
|
2245
|
-
await new Promise((resolve) =>
|
|
2246
|
-
setTimeout(resolve, WORKER_TOOL_BATCH_GRACE_MS),
|
|
2247
|
-
);
|
|
2248
|
-
}
|
|
2249
|
-
void this.drain();
|
|
2250
|
-
});
|
|
2251
|
-
}
|
|
2252
|
-
|
|
2253
|
-
private hasBatchableQueuedTool(): boolean {
|
|
2254
|
-
return this.queue.some(
|
|
2255
|
-
(request) =>
|
|
2256
|
-
request.toolId !== 'test_wait_for_event' &&
|
|
2257
|
-
getPlayRuntimeBatchStrategy(request.toolId) !== null,
|
|
2258
|
-
);
|
|
2259
|
-
}
|
|
2260
|
-
|
|
2261
|
-
private async drain(): Promise<void> {
|
|
2262
|
-
const requests = this.queue;
|
|
2263
|
-
this.queue = [];
|
|
2264
|
-
this.scheduled = false;
|
|
2265
|
-
const drainStartedAt = nowMs();
|
|
2266
|
-
await Promise.all(
|
|
2267
|
-
[...groupWorkerToolRequestsByTool(requests).entries()].map(
|
|
2268
|
-
async ([toolId, groupedRequests]) => {
|
|
2269
|
-
await this.executeToolGroup(toolId, groupedRequests);
|
|
2270
|
-
},
|
|
2271
|
-
),
|
|
2272
|
-
);
|
|
2273
|
-
recordRunnerPerfTrace({
|
|
2274
|
-
req: this.req,
|
|
2275
|
-
phase: 'runner.tool.drain',
|
|
2276
|
-
ms: nowMs() - drainStartedAt,
|
|
2277
|
-
extra: { requests: requests.length },
|
|
2278
|
-
});
|
|
2279
|
-
if (this.queue.length > 0) {
|
|
2280
|
-
this.scheduleDrain();
|
|
2281
|
-
}
|
|
2282
|
-
}
|
|
2283
|
-
|
|
2284
|
-
private async waitForDurableToolReceipt(input: {
|
|
2285
|
-
receiptKey: string;
|
|
2286
|
-
maxAttempts: number;
|
|
2287
|
-
}): Promise<WorkerRuntimeReceipt> {
|
|
2288
|
-
if (!this.receiptStore?.getReceipt) {
|
|
2289
|
-
throw new Error(
|
|
2290
|
-
'Worker durable tool receipt wait requires receipt lookup.',
|
|
2291
|
-
);
|
|
2292
|
-
}
|
|
2293
|
-
const receipt = await waitForCompletedRuntimeReceipt({
|
|
2294
|
-
receiptKey: input.receiptKey,
|
|
2295
|
-
maxAttempts: input.maxAttempts,
|
|
2296
|
-
abortSignal: this.abortSignal,
|
|
2297
|
-
store: {
|
|
2298
|
-
getMany: async (receiptKeys) => {
|
|
2299
|
-
const receipts = new Map<string, RuntimeStepReceipt>();
|
|
2300
|
-
await Promise.all(
|
|
2301
|
-
receiptKeys.map(async (key) => {
|
|
2302
|
-
const receipt = await this.receiptStore!.getReceipt!({
|
|
2303
|
-
playName: this.req.playName,
|
|
2304
|
-
key,
|
|
2305
|
-
});
|
|
2306
|
-
if (!receipt) return;
|
|
2307
|
-
receipts.set(key, {
|
|
2308
|
-
key: receipt.key,
|
|
2309
|
-
status: receipt.status,
|
|
2310
|
-
output: receipt.output,
|
|
2311
|
-
error: receipt.error ?? undefined,
|
|
2312
|
-
runId: receipt.runId ?? null,
|
|
2313
|
-
});
|
|
2314
|
-
}),
|
|
2315
|
-
);
|
|
2316
|
-
return receipts;
|
|
2317
|
-
},
|
|
2318
|
-
},
|
|
2319
|
-
});
|
|
2320
|
-
return receipt;
|
|
2321
|
-
}
|
|
2322
|
-
|
|
2323
|
-
private settleRequests(
|
|
2324
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2325
|
-
result: unknown,
|
|
2326
|
-
): void {
|
|
2327
|
-
claimed.request.resolve(result);
|
|
2328
|
-
for (const follower of claimed.followers) {
|
|
2329
|
-
follower.resolve(
|
|
2330
|
-
claimed.receiptKey
|
|
2331
|
-
? markWorkerToolReceiptResultExecution(result, {
|
|
2332
|
-
kind: 'in_flight',
|
|
2333
|
-
receiptKey: claimed.receiptKey,
|
|
2334
|
-
attachedToReceiptKey: claimed.receiptKey,
|
|
2335
|
-
})
|
|
2336
|
-
: result,
|
|
2337
|
-
);
|
|
2338
|
-
}
|
|
2339
|
-
this.onRequestsSettled?.(1 + claimed.followers.length);
|
|
2340
|
-
}
|
|
2341
|
-
|
|
2342
|
-
private rejectRequests(
|
|
2343
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2344
|
-
error: unknown,
|
|
2345
|
-
): void {
|
|
2346
|
-
claimed.request.reject(error);
|
|
2347
|
-
for (const follower of claimed.followers) {
|
|
2348
|
-
follower.reject(error);
|
|
2349
|
-
}
|
|
2350
|
-
this.onRequestsSettled?.(1 + claimed.followers.length);
|
|
2351
|
-
}
|
|
2352
|
-
|
|
2353
|
-
private rejectRawRequests(
|
|
2354
|
-
requests: WorkerToolBatchRequest[],
|
|
2355
|
-
error: unknown,
|
|
2356
|
-
): void {
|
|
2357
|
-
for (const request of requests) {
|
|
2358
|
-
request.reject(error);
|
|
2359
|
-
}
|
|
2360
|
-
this.onRequestsSettled?.(requests.length);
|
|
2361
|
-
}
|
|
2362
|
-
|
|
2363
|
-
private async resolveCompletedDurableToolReceiptGroup(input: {
|
|
2364
|
-
group: WorkerToolBatchRequest[];
|
|
2365
|
-
receiptKey: string;
|
|
2366
|
-
receipt: WorkerRuntimeReceipt;
|
|
2367
|
-
source: 'cache' | 'in_flight';
|
|
2368
|
-
}): Promise<void> {
|
|
2369
|
-
const [first] = input.group;
|
|
2370
|
-
if (!first) return;
|
|
2371
|
-
const value = deserializeDurableStepValue(input.receipt.output);
|
|
2372
|
-
const result =
|
|
2373
|
-
input.source === 'cache'
|
|
2374
|
-
? markWorkerToolReceiptResultCached(
|
|
2375
|
-
value,
|
|
2376
|
-
first.cacheKey,
|
|
2377
|
-
input.receiptKey,
|
|
2378
|
-
)
|
|
2379
|
-
: markWorkerToolReceiptResultExecution(value, {
|
|
2380
|
-
kind: 'in_flight',
|
|
2381
|
-
receiptKey: input.receiptKey,
|
|
2382
|
-
attachedToReceiptKey: input.receiptKey,
|
|
2383
|
-
});
|
|
2384
|
-
for (const request of input.group) {
|
|
2385
|
-
request.resolve(result);
|
|
2386
|
-
}
|
|
2387
|
-
this.onRequestsSettled?.(input.group.length);
|
|
2388
|
-
}
|
|
2389
|
-
|
|
2390
|
-
private async reclaimRunningDurableToolReceiptGroupAfterWait(input: {
|
|
2391
|
-
group: WorkerToolBatchRequest[];
|
|
2392
|
-
receiptKey: string;
|
|
2393
|
-
runningReceipt: WorkerRuntimeReceipt;
|
|
2394
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
2395
|
-
const [request, ...followers] = input.group;
|
|
2396
|
-
if (!request || !this.receiptStore) {
|
|
2397
|
-
this.rejectRawRequests(
|
|
2398
|
-
input.group,
|
|
2399
|
-
new RuntimeReceiptWaitTimeoutError(input.receiptKey),
|
|
2400
|
-
);
|
|
2401
|
-
return [];
|
|
2402
|
-
}
|
|
2403
|
-
// A `running` receipt left by a DIFFERENT run whose last write is stale will
|
|
2404
|
-
// never be completed by a live worker. Polling it burns one service-binding
|
|
2405
|
-
// subrequest per tick, up to the receipt's wait budget (~1320) — many such
|
|
2406
|
-
// orphans on a resume exhaust Cloudflare's per-invocation subrequest budget
|
|
2407
|
-
// and kill the run. The orchestrator reclaims those immediately (skipping the
|
|
2408
|
-
// poll) and re-executes, matching the "on resume it reclaims and re-executes"
|
|
2409
|
-
// contract; live/fresh/same-run owners still poll before reclaiming. Kept as
|
|
2410
|
-
// an injected helper so the skip-poll decision is unit-testable without
|
|
2411
|
-
// loading this cloudflare:workers Worker entrypoint.
|
|
2412
|
-
return await reclaimRunningReceiptGroupAfterWait<ClaimedWorkerToolBatchRequest>(
|
|
2413
|
-
{
|
|
2414
|
-
ownership: {
|
|
2415
|
-
ownerRunId: input.runningReceipt.runId,
|
|
2416
|
-
currentRunId: this.req.runId,
|
|
2417
|
-
updatedAt: input.runningReceipt.updatedAt,
|
|
2418
|
-
},
|
|
2419
|
-
timeoutError: new RuntimeReceiptWaitTimeoutError(input.receiptKey),
|
|
2420
|
-
poll: async () => {
|
|
2421
|
-
try {
|
|
2422
|
-
const receipt = await this.waitForDurableToolReceipt({
|
|
2423
|
-
receiptKey: input.receiptKey,
|
|
2424
|
-
maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
|
|
2425
|
-
input.group,
|
|
2426
|
-
(groupRequest) => groupRequest.receiptWaitMaxAttempts,
|
|
2427
|
-
),
|
|
2428
|
-
});
|
|
2429
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2430
|
-
group: input.group,
|
|
2431
|
-
receiptKey: input.receiptKey,
|
|
2432
|
-
receipt,
|
|
2433
|
-
source: 'in_flight',
|
|
2434
|
-
});
|
|
2435
|
-
return { kind: 'completed' };
|
|
2436
|
-
} catch (error) {
|
|
2437
|
-
if (error instanceof RuntimeReceiptWaitTimeoutError) {
|
|
2438
|
-
return { kind: 'timed_out', error };
|
|
2439
|
-
}
|
|
2440
|
-
return { kind: 'errored', error };
|
|
2441
|
-
}
|
|
2442
|
-
},
|
|
2443
|
-
reclaim: (rejectionOnFallthrough) =>
|
|
2444
|
-
this.forceReclaimRunningDurableToolReceiptGroup({
|
|
2445
|
-
group: input.group,
|
|
2446
|
-
request,
|
|
2447
|
-
followers,
|
|
2448
|
-
receiptKey: input.receiptKey,
|
|
2449
|
-
rejectionOnFallthrough,
|
|
2450
|
-
}),
|
|
2451
|
-
reject: (error) => this.rejectRawRequests(input.group, error),
|
|
2452
|
-
},
|
|
2453
|
-
);
|
|
2454
|
-
}
|
|
2455
|
-
|
|
2456
|
-
/**
|
|
2457
|
-
* Steal a `running` receipt (reclaimRunning) and route by claim disposition.
|
|
2458
|
-
* Shared by the post-poll-timeout reclaim and the dead-owner fast path, so
|
|
2459
|
-
* both dispose of a reclaimed/reused/failed receipt identically.
|
|
2460
|
-
*/
|
|
2461
|
-
private async forceReclaimRunningDurableToolReceiptGroup(input: {
|
|
2462
|
-
group: WorkerToolBatchRequest[];
|
|
2463
|
-
request: WorkerToolBatchRequest;
|
|
2464
|
-
followers: WorkerToolBatchRequest[];
|
|
2465
|
-
receiptKey: string;
|
|
2466
|
-
rejectionOnFallthrough: unknown;
|
|
2467
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
2468
|
-
if (!this.receiptStore) {
|
|
2469
|
-
this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
|
|
2470
|
-
return [];
|
|
2471
|
-
}
|
|
2472
|
-
let claim: WorkerRuntimeReceiptClaim;
|
|
2473
|
-
try {
|
|
2474
|
-
claim = await this.receiptStore.claimReceipt({
|
|
2475
|
-
playName: this.req.playName,
|
|
2476
|
-
runId: this.req.runId,
|
|
2477
|
-
key: input.receiptKey,
|
|
2478
|
-
reclaimRunning: true,
|
|
2479
|
-
});
|
|
2480
|
-
} catch (error) {
|
|
2481
|
-
this.rejectRawRequests(input.group, error);
|
|
2482
|
-
return [];
|
|
2483
|
-
}
|
|
2484
|
-
if (claim.disposition === 'claimed') {
|
|
2485
|
-
return [
|
|
2486
|
-
{
|
|
2487
|
-
request: input.request,
|
|
2488
|
-
receiptKey: input.receiptKey,
|
|
2489
|
-
followers: input.followers,
|
|
2490
|
-
},
|
|
2491
|
-
];
|
|
2492
|
-
}
|
|
2493
|
-
if (claim.disposition === 'reused') {
|
|
2494
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2495
|
-
group: input.group,
|
|
2496
|
-
receiptKey: input.receiptKey,
|
|
2497
|
-
receipt: claim.receipt,
|
|
2498
|
-
source: 'cache',
|
|
2499
|
-
});
|
|
2500
|
-
return [];
|
|
2501
|
-
}
|
|
2502
|
-
if (claim.disposition === 'failed') {
|
|
2503
|
-
return await this.reclaimFailedDurableToolReceiptGroup({
|
|
2504
|
-
group: input.group,
|
|
2505
|
-
receiptKey: input.receiptKey,
|
|
2506
|
-
failedReceipt: claim.receipt,
|
|
2507
|
-
});
|
|
2508
|
-
}
|
|
2509
|
-
this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
|
|
2510
|
-
return [];
|
|
2511
|
-
}
|
|
2512
|
-
|
|
2513
|
-
private async reclaimFailedDurableToolReceiptGroup(input: {
|
|
2514
|
-
group: WorkerToolBatchRequest[];
|
|
2515
|
-
receiptKey: string;
|
|
2516
|
-
failedReceipt: WorkerRuntimeReceipt;
|
|
2517
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
2518
|
-
const failedError = new Error(
|
|
2519
|
-
`Durable tool call ${input.receiptKey} failed: ${input.failedReceipt.error ?? 'unknown error'}`,
|
|
2520
|
-
);
|
|
2521
|
-
const [request, ...followers] = input.group;
|
|
2522
|
-
if (!request || !this.receiptStore) {
|
|
2523
|
-
this.rejectRawRequests(input.group, failedError);
|
|
2524
|
-
return [];
|
|
2525
|
-
}
|
|
2526
|
-
if (
|
|
2527
|
-
!canReclaimFailedWorkerToolReceipt({
|
|
2528
|
-
error: input.failedReceipt.error,
|
|
2529
|
-
})
|
|
2530
|
-
) {
|
|
2531
|
-
this.rejectRawRequests(input.group, failedError);
|
|
2532
|
-
return [];
|
|
2533
|
-
}
|
|
2534
|
-
let claim: WorkerRuntimeReceiptClaim;
|
|
2535
|
-
try {
|
|
2536
|
-
claim = await this.receiptStore.claimReceipt({
|
|
2537
|
-
playName: this.req.playName,
|
|
2538
|
-
runId: this.req.runId,
|
|
2539
|
-
key: input.receiptKey,
|
|
2540
|
-
});
|
|
2541
|
-
} catch (error) {
|
|
2542
|
-
this.rejectRawRequests(input.group, error);
|
|
2543
|
-
return [];
|
|
2544
|
-
}
|
|
2545
|
-
if (claim.disposition === 'claimed') {
|
|
2546
|
-
return [{ request, receiptKey: input.receiptKey, followers }];
|
|
2547
|
-
}
|
|
2548
|
-
if (claim.disposition === 'reused') {
|
|
2549
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2550
|
-
group: input.group,
|
|
2551
|
-
receiptKey: input.receiptKey,
|
|
2552
|
-
receipt: claim.receipt,
|
|
2553
|
-
source: 'cache',
|
|
2554
|
-
});
|
|
2555
|
-
return [];
|
|
2556
|
-
}
|
|
2557
|
-
if (claim.disposition === 'running') {
|
|
2558
|
-
try {
|
|
2559
|
-
const receipt = await this.waitForDurableToolReceipt({
|
|
2560
|
-
receiptKey: input.receiptKey,
|
|
2561
|
-
maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
|
|
2562
|
-
input.group,
|
|
2563
|
-
(groupRequest) => groupRequest.receiptWaitMaxAttempts,
|
|
2564
|
-
),
|
|
2565
|
-
});
|
|
2566
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2567
|
-
group: input.group,
|
|
2568
|
-
receiptKey: input.receiptKey,
|
|
2569
|
-
receipt,
|
|
2570
|
-
source: 'in_flight',
|
|
2571
|
-
});
|
|
2572
|
-
} catch (error) {
|
|
2573
|
-
this.rejectRawRequests(input.group, error);
|
|
2574
|
-
}
|
|
2575
|
-
return [];
|
|
2576
|
-
}
|
|
2577
|
-
this.rejectRawRequests(
|
|
2578
|
-
input.group,
|
|
2579
|
-
new Error(
|
|
2580
|
-
`Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
|
|
2581
|
-
),
|
|
2582
|
-
);
|
|
2583
|
-
return [];
|
|
2584
|
-
}
|
|
2585
|
-
|
|
2586
|
-
private async failureForRejectedToolRequest(
|
|
2587
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2588
|
-
error: unknown,
|
|
2589
|
-
): Promise<unknown> {
|
|
2590
|
-
try {
|
|
2591
|
-
await this.failDurableToolRequest(claimed, error);
|
|
2592
|
-
return error;
|
|
2593
|
-
} catch (receiptError) {
|
|
2594
|
-
tripRuntimePersistenceLatch(this.latch, receiptError);
|
|
2595
|
-
return new RuntimeReceiptPersistenceError(
|
|
2596
|
-
'Tool call failed and durable receipt could not be marked failed',
|
|
2597
|
-
[error, receiptError],
|
|
2598
|
-
);
|
|
2599
|
-
}
|
|
2600
|
-
}
|
|
2601
|
-
|
|
2602
|
-
private async claimDurableToolReceiptGroups(
|
|
2603
|
-
groups: ReturnType<
|
|
2604
|
-
typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
|
|
2605
|
-
>['durableGroups'],
|
|
2606
|
-
): Promise<
|
|
2607
|
-
Array<{
|
|
2608
|
-
group: ReturnType<
|
|
2609
|
-
typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
|
|
2610
|
-
>['durableGroups'][number];
|
|
2611
|
-
receiptKey: string;
|
|
2612
|
-
claim: WorkerRuntimeReceiptClaim;
|
|
2613
|
-
}>
|
|
2614
|
-
> {
|
|
2615
|
-
if (!this.receiptStore) return [];
|
|
2616
|
-
const planned = groups.map((group) => ({
|
|
2617
|
-
group,
|
|
2618
|
-
receiptKey: workerRuntimeReceiptKey({
|
|
2619
|
-
req: this.req,
|
|
2620
|
-
key: group.claimableReceiptKey,
|
|
2621
|
-
}),
|
|
2622
|
-
}));
|
|
2623
|
-
const claimOne = async (entry: (typeof planned)[number]) => ({
|
|
2624
|
-
...entry,
|
|
2625
|
-
claim: await this.receiptStore!.claimReceipt({
|
|
2626
|
-
playName: this.req.playName,
|
|
2627
|
-
runId: this.req.runId,
|
|
2628
|
-
key: entry.receiptKey,
|
|
2629
|
-
...(entry.group.forceDurableRefresh
|
|
2630
|
-
? { forceRefresh: true, reclaimRunning: true }
|
|
2631
|
-
: {}),
|
|
2632
|
-
}),
|
|
2633
|
-
});
|
|
2634
|
-
if (!this.receiptStore.claimReceipts) {
|
|
2635
|
-
return await Promise.all(planned.map(claimOne));
|
|
2636
|
-
}
|
|
2637
|
-
const claimed: Array<{
|
|
2638
|
-
group: (typeof planned)[number]['group'];
|
|
2639
|
-
receiptKey: string;
|
|
2640
|
-
claim: WorkerRuntimeReceiptClaim;
|
|
2641
|
-
}> = [];
|
|
2642
|
-
for (const forceDurableRefresh of [false, true]) {
|
|
2643
|
-
const entries = planned.filter(
|
|
2644
|
-
(entry) => entry.group.forceDurableRefresh === forceDurableRefresh,
|
|
2645
|
-
);
|
|
2646
|
-
if (entries.length === 0) continue;
|
|
2647
|
-
const claims = await this.receiptStore.claimReceipts({
|
|
2648
|
-
playName: this.req.playName,
|
|
2649
|
-
runId: this.req.runId,
|
|
2650
|
-
keys: entries.map((entry) => entry.receiptKey),
|
|
2651
|
-
...(forceDurableRefresh
|
|
2652
|
-
? { forceRefresh: true, reclaimRunning: true }
|
|
2653
|
-
: {}),
|
|
2654
|
-
});
|
|
2655
|
-
entries.forEach((entry, index) => {
|
|
2656
|
-
const claim = claims[index];
|
|
2657
|
-
if (!claim) {
|
|
2658
|
-
throw new Error(
|
|
2659
|
-
`Runtime receipt batch claim did not return receipt ${entry.receiptKey}.`,
|
|
2660
|
-
);
|
|
2661
|
-
}
|
|
2662
|
-
claimed.push({ ...entry, claim });
|
|
2663
|
-
});
|
|
2664
|
-
}
|
|
2665
|
-
return claimed;
|
|
2666
|
-
}
|
|
2667
|
-
|
|
2668
|
-
private async prepareDurableToolRequests(
|
|
2669
|
-
requests: WorkerToolBatchRequest[],
|
|
2670
|
-
): Promise<PreparedWorkerToolBatchRequests> {
|
|
2671
|
-
if (!this.receiptStore) {
|
|
2672
|
-
return {
|
|
2673
|
-
claimedRequests: requests.map((request) => ({
|
|
2674
|
-
request,
|
|
2675
|
-
receiptKey: null,
|
|
2676
|
-
followers: [],
|
|
2677
|
-
})),
|
|
2678
|
-
deferredClaimedRequests: [],
|
|
2679
|
-
};
|
|
2680
|
-
}
|
|
2681
|
-
|
|
2682
|
-
const claimedRequests: ClaimedWorkerToolBatchRequest[] = [];
|
|
2683
|
-
const deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[] =
|
|
2684
|
-
[];
|
|
2685
|
-
const receiptGroupPlan = planWorkerToolReceiptGroups(requests, {
|
|
2686
|
-
allowLocalRetryReceipts: this.allowLocalRetryReceipts,
|
|
2687
|
-
getReceiptInput: (request) => ({
|
|
2688
|
-
durableReceiptKey: request.receiptKey,
|
|
2689
|
-
localRetryCacheKey: request.cacheKey,
|
|
2690
|
-
force: request.force,
|
|
2691
|
-
}),
|
|
2692
|
-
});
|
|
2693
|
-
for (const request of receiptGroupPlan.localRequests) {
|
|
2694
|
-
claimedRequests.push({
|
|
2695
|
-
request,
|
|
2696
|
-
receiptKey: null,
|
|
2697
|
-
followers: [],
|
|
2698
|
-
});
|
|
2699
|
-
}
|
|
2700
|
-
|
|
2701
|
-
try {
|
|
2702
|
-
const claimEntries = await this.claimDurableToolReceiptGroups(
|
|
2703
|
-
receiptGroupPlan.durableGroups,
|
|
2704
|
-
);
|
|
2705
|
-
for (const { group: groupState, receiptKey, claim } of claimEntries) {
|
|
2706
|
-
const group = groupState.requests;
|
|
2707
|
-
const [first] = group;
|
|
2708
|
-
if (!first) continue;
|
|
2709
|
-
if (claim.disposition === 'reused') {
|
|
2710
|
-
const result = markWorkerToolReceiptResultCached(
|
|
2711
|
-
deserializeDurableStepValue(claim.receipt.output),
|
|
2712
|
-
first.cacheKey,
|
|
2713
|
-
receiptKey,
|
|
2714
|
-
);
|
|
2715
|
-
for (const request of group) {
|
|
2716
|
-
request.resolve(result);
|
|
2717
|
-
}
|
|
2718
|
-
this.onRequestsSettled?.(group.length);
|
|
2719
|
-
continue;
|
|
2720
|
-
}
|
|
2721
|
-
if (claim.disposition === 'failed') {
|
|
2722
|
-
claimedRequests.push(
|
|
2723
|
-
...(await this.reclaimFailedDurableToolReceiptGroup({
|
|
2724
|
-
group,
|
|
2725
|
-
receiptKey,
|
|
2726
|
-
failedReceipt: claim.receipt,
|
|
2727
|
-
})),
|
|
2728
|
-
);
|
|
2729
|
-
continue;
|
|
2730
|
-
}
|
|
2731
|
-
if (claim.disposition === 'running') {
|
|
2732
|
-
deferredClaimedRequests.push(
|
|
2733
|
-
this.reclaimRunningDurableToolReceiptGroupAfterWait({
|
|
2734
|
-
group,
|
|
2735
|
-
receiptKey,
|
|
2736
|
-
runningReceipt: claim.receipt,
|
|
2737
|
-
}),
|
|
2738
|
-
);
|
|
2739
|
-
continue;
|
|
2740
|
-
}
|
|
2741
|
-
const [request, ...followers] = group;
|
|
2742
|
-
if (!request) continue;
|
|
2743
|
-
claimedRequests.push({ request, receiptKey, followers });
|
|
2744
|
-
}
|
|
2745
|
-
} catch (error) {
|
|
2746
|
-
// A receipt claim failure means the tenant Postgres is already
|
|
2747
|
-
// unwritable: trip the breaker so remaining groups/rows stop dispatching.
|
|
2748
|
-
tripRuntimePersistenceLatch(this.latch, error);
|
|
2749
|
-
for (const group of receiptGroupPlan.durableGroups) {
|
|
2750
|
-
this.rejectRawRequests(group.requests, error);
|
|
2751
|
-
}
|
|
2752
|
-
}
|
|
2753
|
-
return { claimedRequests, deferredClaimedRequests };
|
|
2754
|
-
}
|
|
2755
|
-
|
|
2756
|
-
private async completeDurableToolRequest(
|
|
2757
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2758
|
-
result: unknown,
|
|
2759
|
-
): Promise<unknown> {
|
|
2760
|
-
if (!this.receiptStore || !claimed.receiptKey) {
|
|
2761
|
-
return result;
|
|
2762
|
-
}
|
|
2763
|
-
const ownerResult = markWorkerToolReceiptResultExecution(result, {
|
|
2764
|
-
kind: 'live',
|
|
2765
|
-
receiptKey: claimed.receiptKey,
|
|
2766
|
-
});
|
|
2767
|
-
const serializedOutput = serializeDurableStepValue(ownerResult);
|
|
2768
|
-
const completed = await this.completeReceiptWithRetryLadder(
|
|
2769
|
-
() =>
|
|
2770
|
-
this.receiptStore!.completeReceipt({
|
|
2771
|
-
playName: this.req.playName,
|
|
2772
|
-
runId: this.req.runId,
|
|
2773
|
-
key: claimed.receiptKey!,
|
|
2774
|
-
output: serializedOutput,
|
|
2775
|
-
}),
|
|
2776
|
-
// Ladder exhausted: the call billed but the receipt could not land.
|
|
2777
|
-
// Salvage the exact serialized output onto the run record before the
|
|
2778
|
-
// caller turns this into a RuntimeReceiptPersistenceError.
|
|
2779
|
-
(attempts) =>
|
|
2780
|
-
this.recordReceiptSalvage({
|
|
2781
|
-
claimed,
|
|
2782
|
-
output: serializedOutput,
|
|
2783
|
-
attempts,
|
|
2784
|
-
}),
|
|
2785
|
-
);
|
|
2786
|
-
if (
|
|
2787
|
-
completed &&
|
|
2788
|
-
(completed.status === 'completed' || completed.status === 'skipped') &&
|
|
2789
|
-
completed.output !== undefined
|
|
2790
|
-
) {
|
|
2791
|
-
const recovered = deserializeDurableStepValue(completed.output);
|
|
2792
|
-
return completed.runId && completed.runId !== this.req.runId
|
|
2793
|
-
? markWorkerToolReceiptResultCached(
|
|
2794
|
-
recovered,
|
|
2795
|
-
claimed.request.cacheKey,
|
|
2796
|
-
claimed.receiptKey,
|
|
2797
|
-
)
|
|
2798
|
-
: recovered;
|
|
2799
|
-
}
|
|
2800
|
-
return ownerResult;
|
|
2801
|
-
}
|
|
2802
|
-
|
|
2803
|
-
private completedDurableToolRequestResult(input: {
|
|
2804
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
2805
|
-
ownerResult: unknown;
|
|
2806
|
-
completed: WorkerRuntimeReceipt | null | undefined;
|
|
2807
|
-
}): unknown {
|
|
2808
|
-
const { claimed, completed, ownerResult } = input;
|
|
2809
|
-
if (
|
|
2810
|
-
completed &&
|
|
2811
|
-
(completed.status === 'completed' || completed.status === 'skipped') &&
|
|
2812
|
-
completed.output !== undefined
|
|
2813
|
-
) {
|
|
2814
|
-
const recovered = deserializeDurableStepValue(completed.output);
|
|
2815
|
-
return completed.runId && completed.runId !== this.req.runId
|
|
2816
|
-
? markWorkerToolReceiptResultCached(
|
|
2817
|
-
recovered,
|
|
2818
|
-
claimed.request.cacheKey,
|
|
2819
|
-
claimed.receiptKey ?? claimed.request.cacheKey,
|
|
2820
|
-
)
|
|
2821
|
-
: recovered;
|
|
2822
|
-
}
|
|
2823
|
-
return ownerResult;
|
|
2824
|
-
}
|
|
2825
|
-
|
|
2826
|
-
private async completeDurableToolRequests(
|
|
2827
|
-
entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
|
|
2828
|
-
): Promise<unknown[]> {
|
|
2829
|
-
const results: unknown[] = new Array(entries.length);
|
|
2830
|
-
const durableEntries: Array<{
|
|
2831
|
-
index: number;
|
|
2832
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
2833
|
-
ownerResult: unknown;
|
|
2834
|
-
}> = [];
|
|
2835
|
-
for (let index = 0; index < entries.length; index += 1) {
|
|
2836
|
-
const entry = entries[index]!;
|
|
2837
|
-
if (!this.receiptStore || !entry.claimed.receiptKey) {
|
|
2838
|
-
results[index] = entry.result;
|
|
2839
|
-
continue;
|
|
2840
|
-
}
|
|
2841
|
-
const ownerResult = markWorkerToolReceiptResultExecution(entry.result, {
|
|
2842
|
-
kind: 'live',
|
|
2843
|
-
receiptKey: entry.claimed.receiptKey,
|
|
2844
|
-
});
|
|
2845
|
-
durableEntries.push({ index, claimed: entry.claimed, ownerResult });
|
|
2846
|
-
}
|
|
2847
|
-
if (durableEntries.length === 0) {
|
|
2848
|
-
return results;
|
|
2849
|
-
}
|
|
2850
|
-
if (!this.receiptStore?.completeReceipts) {
|
|
2851
|
-
await Promise.all(
|
|
2852
|
-
durableEntries.map(async (entry) => {
|
|
2853
|
-
results[entry.index] = await this.completeDurableToolRequest(
|
|
2854
|
-
entry.claimed,
|
|
2855
|
-
entries[entry.index]!.result,
|
|
2856
|
-
);
|
|
2857
|
-
}),
|
|
2858
|
-
);
|
|
2859
|
-
return results;
|
|
2860
|
-
}
|
|
2861
|
-
const serializedByEntry = durableEntries.map((entry) =>
|
|
2862
|
-
serializeDurableStepValue(entry.ownerResult),
|
|
2863
|
-
);
|
|
2864
|
-
const completed = await this.completeReceiptWithRetryLadder(
|
|
2865
|
-
() =>
|
|
2866
|
-
this.receiptStore!.completeReceipts!({
|
|
2867
|
-
playName: this.req.playName,
|
|
2868
|
-
receipts: durableEntries.map((entry, entryIndex) => ({
|
|
2869
|
-
runId: this.req.runId,
|
|
2870
|
-
key: entry.claimed.receiptKey!,
|
|
2871
|
-
output: serializedByEntry[entryIndex],
|
|
2872
|
-
})),
|
|
2873
|
-
}),
|
|
2874
|
-
// Ladder exhausted: every entry in this batch billed but could not be
|
|
2875
|
-
// marked completed. Salvage each serialized output before the caller
|
|
2876
|
-
// turns this into a RuntimeReceiptPersistenceError.
|
|
2877
|
-
(attempts) => {
|
|
2878
|
-
durableEntries.forEach((entry, entryIndex) => {
|
|
2879
|
-
this.recordReceiptSalvage({
|
|
2880
|
-
claimed: entry.claimed,
|
|
2881
|
-
output: serializedByEntry[entryIndex],
|
|
2882
|
-
attempts,
|
|
2883
|
-
});
|
|
2884
|
-
});
|
|
2885
|
-
},
|
|
2886
|
-
);
|
|
2887
|
-
durableEntries.forEach((entry, resultIndex) => {
|
|
2888
|
-
results[entry.index] = this.completedDurableToolRequestResult({
|
|
2889
|
-
claimed: entry.claimed,
|
|
2890
|
-
ownerResult: entry.ownerResult,
|
|
2891
|
-
completed: completed[resultIndex],
|
|
2892
|
-
});
|
|
2893
|
-
});
|
|
2894
|
-
return results;
|
|
2895
|
-
}
|
|
2896
|
-
|
|
2897
|
-
private async failDurableToolRequest(
|
|
2898
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2899
|
-
error: unknown,
|
|
2900
|
-
): Promise<void> {
|
|
2901
|
-
if (!this.receiptStore || !claimed.receiptKey) {
|
|
2902
|
-
return;
|
|
2903
|
-
}
|
|
2904
|
-
await this.receiptStore.failReceipt({
|
|
2905
|
-
playName: this.req.playName,
|
|
2906
|
-
runId: this.req.runId,
|
|
2907
|
-
key: claimed.receiptKey,
|
|
2908
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2909
|
-
});
|
|
2910
|
-
}
|
|
2911
|
-
|
|
2912
|
-
private async failDurableToolRequests(
|
|
2913
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
2914
|
-
error: unknown,
|
|
2915
|
-
): Promise<void> {
|
|
2916
|
-
const durable = claimedRequests.filter(
|
|
2917
|
-
(claimed) => this.receiptStore && claimed.receiptKey,
|
|
2918
|
-
);
|
|
2919
|
-
if (durable.length === 0) return;
|
|
2920
|
-
if (!this.receiptStore?.failReceipts) {
|
|
2921
|
-
await Promise.all(
|
|
2922
|
-
durable.map((claimed) => this.failDurableToolRequest(claimed, error)),
|
|
2923
|
-
);
|
|
2924
|
-
return;
|
|
2925
|
-
}
|
|
2926
|
-
await this.receiptStore.failReceipts({
|
|
2927
|
-
playName: this.req.playName,
|
|
2928
|
-
receipts: durable.map((claimed) => ({
|
|
2929
|
-
runId: this.req.runId,
|
|
2930
|
-
key: claimed.receiptKey!,
|
|
2931
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2932
|
-
})),
|
|
2933
|
-
});
|
|
2934
|
-
}
|
|
2935
|
-
|
|
2936
|
-
private async executeToolGroup(
|
|
2937
|
-
toolId: string,
|
|
2938
|
-
requests: WorkerToolBatchRequest[],
|
|
2939
|
-
): Promise<void> {
|
|
2940
|
-
// Circuit breaker, gated BEFORE the receipt claim: never-dispatched rows
|
|
2941
|
-
// get NO receipt (not even `running`), so a re-run claims them fresh and
|
|
2942
|
-
// executes them exactly once.
|
|
2943
|
-
if (this.latch.tripped) {
|
|
2944
|
-
this.latch.preventedCallCount += requests.length;
|
|
2945
|
-
this.rejectRawRequests(
|
|
2946
|
-
requests,
|
|
2947
|
-
new RuntimePersistenceCircuitOpenError(this.latch),
|
|
2948
|
-
);
|
|
2949
|
-
return;
|
|
2950
|
-
}
|
|
2951
|
-
const { claimedRequests, deferredClaimedRequests } =
|
|
2952
|
-
await this.prepareDurableToolRequests(requests);
|
|
2953
|
-
await this.executeClaimedToolRequests(
|
|
2954
|
-
toolId,
|
|
2955
|
-
requests.length,
|
|
2956
|
-
claimedRequests,
|
|
2957
|
-
);
|
|
2958
|
-
if (deferredClaimedRequests.length === 0) {
|
|
2959
|
-
return;
|
|
2960
|
-
}
|
|
2961
|
-
const reclaimedRequests = (
|
|
2962
|
-
await Promise.all(deferredClaimedRequests)
|
|
2963
|
-
).flat();
|
|
2964
|
-
await this.executeClaimedToolRequests(
|
|
2965
|
-
toolId,
|
|
2966
|
-
requests.length,
|
|
2967
|
-
reclaimedRequests,
|
|
2968
|
-
);
|
|
2969
|
-
}
|
|
2970
|
-
|
|
2971
|
-
private async executeClaimedToolRequests(
|
|
2972
|
-
toolId: string,
|
|
2973
|
-
requestCount: number,
|
|
2974
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
2975
|
-
): Promise<void> {
|
|
2976
|
-
if (claimedRequests.length === 0) {
|
|
2977
|
-
return;
|
|
2978
|
-
}
|
|
2979
|
-
const strategy = getPlayRuntimeBatchStrategy(toolId);
|
|
2980
|
-
if (
|
|
2981
|
-
!strategy ||
|
|
2982
|
-
toolId === 'test_wait_for_event' ||
|
|
2983
|
-
claimedRequests.length < 2
|
|
2984
|
-
) {
|
|
2985
|
-
const groupStartedAt = nowMs();
|
|
2986
|
-
const dispatchOne = async (
|
|
2987
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2988
|
-
): Promise<void> => {
|
|
2989
|
-
const { request } = claimed;
|
|
2990
|
-
// Circuit breaker: the latch may have tripped after this group was
|
|
2991
|
-
// claimed but before this particular call started. Do NOT call the
|
|
2992
|
-
// provider. This row has a `running` receipt (claimed, never
|
|
2993
|
-
// executed); on resume it reclaims and re-executes — safe, it never
|
|
2994
|
-
// billed.
|
|
2995
|
-
if (this.latch.tripped) {
|
|
2996
|
-
this.latch.preventedCallCount += 1 + claimed.followers.length;
|
|
2997
|
-
this.rejectRequests(
|
|
2998
|
-
claimed,
|
|
2999
|
-
new RuntimePersistenceCircuitOpenError(this.latch),
|
|
3000
|
-
);
|
|
3001
|
-
return;
|
|
3002
|
-
}
|
|
3003
|
-
const toolContract = await this.resolvePacing(toolId).catch(
|
|
3004
|
-
() => null,
|
|
3005
|
-
);
|
|
3006
|
-
// Each unbatched provider call takes its own tool slot: the Governor
|
|
3007
|
-
// charges tool budget, holds a global tool-concurrency slot, and
|
|
3008
|
-
// applies per-(org,provider) pacing before the call runs.
|
|
3009
|
-
const slot = await this.governor.acquireToolSlot(toolId, {
|
|
3010
|
-
signal: this.abortSignal,
|
|
3011
|
-
});
|
|
3012
|
-
try {
|
|
3013
|
-
let result: unknown;
|
|
3014
|
-
try {
|
|
3015
|
-
result = await executeToolWithLifecycle(
|
|
3016
|
-
this.req,
|
|
3017
|
-
{ id: request.id, toolId, input: request.input },
|
|
3018
|
-
request.workflowStep,
|
|
3019
|
-
this.callbacks,
|
|
3020
|
-
(retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
|
|
3021
|
-
() => this.governor.chargeBudget('retry'),
|
|
3022
|
-
toolContract?.retrySafeTransientHttp === true,
|
|
3023
|
-
this.abortSignal,
|
|
3024
|
-
this.runtimeDeadlineMs,
|
|
3025
|
-
resolveClaimedWorkerToolRuntimeTimeoutMs([claimed], {
|
|
3026
|
-
runtimeDeadlineMs: this.runtimeDeadlineMs,
|
|
3027
|
-
}),
|
|
3028
|
-
request.directOptions,
|
|
3029
|
-
);
|
|
3030
|
-
} catch (error) {
|
|
3031
|
-
this.rejectRequests(
|
|
3032
|
-
claimed,
|
|
3033
|
-
await this.failureForRejectedToolRequest(claimed, error),
|
|
3034
|
-
);
|
|
3035
|
-
return;
|
|
3036
|
-
}
|
|
3037
|
-
this.settleRequests(
|
|
3038
|
-
claimed,
|
|
3039
|
-
await this.completeDurableToolRequest(claimed, result),
|
|
3040
|
-
);
|
|
3041
|
-
} catch (receiptError) {
|
|
3042
|
-
tripRuntimePersistenceLatch(this.latch, receiptError);
|
|
3043
|
-
this.rejectRequests(
|
|
3044
|
-
claimed,
|
|
3045
|
-
new RuntimeReceiptPersistenceError(
|
|
3046
|
-
'Tool call succeeded but durable receipt could not be marked completed',
|
|
3047
|
-
[receiptError],
|
|
3048
|
-
),
|
|
3049
|
-
);
|
|
3050
|
-
} finally {
|
|
3051
|
-
slot.release();
|
|
3052
|
-
}
|
|
3053
|
-
};
|
|
3054
|
-
// Bounded dispatch waves: an unbatched tool group can carry a whole chunk's
|
|
3055
|
-
// rows (enrich compiles to one ~245-row chunk). Dispatching them all at
|
|
3056
|
-
// once billed every call before the first receipt-completion failure could
|
|
3057
|
-
// trip the latch (the 250/250 incident). Instead, dispatch one wave, AWAIT
|
|
3058
|
-
// it so any completion failure trips the latch, then read the latch before
|
|
3059
|
-
// the next wave — remaining waves are prevented (counted + rejected) exactly
|
|
3060
|
-
// like the mid-wave gate above. The wave calls stay fully concurrent; the
|
|
3061
|
-
// only added cost on the healthy path is one latch read per wave.
|
|
3062
|
-
for (const wave of partitionDispatchWaves(
|
|
3063
|
-
claimedRequests,
|
|
3064
|
-
() => 1,
|
|
3065
|
-
PROVIDER_DISPATCH_WAVE_SIZE,
|
|
3066
|
-
)) {
|
|
3067
|
-
if (this.latch.tripped) {
|
|
3068
|
-
for (const claimed of wave) {
|
|
3069
|
-
this.latch.preventedCallCount += 1 + claimed.followers.length;
|
|
3070
|
-
this.rejectRequests(
|
|
3071
|
-
claimed,
|
|
3072
|
-
new RuntimePersistenceCircuitOpenError(this.latch),
|
|
3073
|
-
);
|
|
3074
|
-
}
|
|
3075
|
-
continue;
|
|
3076
|
-
}
|
|
3077
|
-
await Promise.all(wave.map(dispatchOne));
|
|
3078
|
-
}
|
|
3079
|
-
recordRunnerPerfTrace({
|
|
3080
|
-
req: this.req,
|
|
3081
|
-
phase: 'runner.tool.group',
|
|
3082
|
-
ms: nowMs() - groupStartedAt,
|
|
3083
|
-
extra: {
|
|
3084
|
-
toolId,
|
|
3085
|
-
requests: requestCount,
|
|
3086
|
-
executed: claimedRequests.length,
|
|
3087
|
-
batched: false,
|
|
3088
|
-
},
|
|
3089
|
-
});
|
|
3090
|
-
return;
|
|
3091
|
-
}
|
|
2283
|
+
path: string[];
|
|
2284
|
+
outputs: RecordedStepProgramOutput[];
|
|
2285
|
+
onOutput?: (output: RecordedStepProgramOutput) => void | Promise<void>;
|
|
2286
|
+
};
|
|
3092
2287
|
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
toolId,
|
|
3101
|
-
WORKER_TOOL_BATCH_DEFAULT_PARALLELISM,
|
|
3102
|
-
),
|
|
3103
|
-
abortSignal: this.abortSignal,
|
|
3104
|
-
runtimeDeadlineMs: this.runtimeDeadlineMs,
|
|
3105
|
-
reportBackpressure: (retryAfterMs) =>
|
|
3106
|
-
this.reportBackpressure(toolId, retryAfterMs),
|
|
3107
|
-
resolveToolContract: this.resolvePacing,
|
|
3108
|
-
onRequestsSettled: this.onRequestsSettled,
|
|
3109
|
-
callbacks: this.callbacks,
|
|
3110
|
-
completeRequests: async (entries) =>
|
|
3111
|
-
await this.completeDurableToolRequests(entries),
|
|
3112
|
-
failRequests: async (claimedRequests, error) =>
|
|
3113
|
-
await this.failDurableToolRequests(claimedRequests, error),
|
|
3114
|
-
settleRequest: (claimed, result) => this.settleRequests(claimed, result),
|
|
3115
|
-
rejectRequest: (claimed, error) => this.rejectRequests(claimed, error),
|
|
3116
|
-
latch: this.latch,
|
|
3117
|
-
});
|
|
3118
|
-
recordRunnerPerfTrace({
|
|
3119
|
-
req: this.req,
|
|
3120
|
-
phase: 'runner.tool.group',
|
|
3121
|
-
ms: nowMs() - batchStartedAt,
|
|
3122
|
-
extra: {
|
|
3123
|
-
toolId,
|
|
3124
|
-
requests: requestCount,
|
|
3125
|
-
executed: claimedRequests.length,
|
|
3126
|
-
batched: true,
|
|
3127
|
-
},
|
|
3128
|
-
});
|
|
3129
|
-
}
|
|
2288
|
+
type WorkerStepResolution = {
|
|
2289
|
+
value: unknown;
|
|
2290
|
+
status?: 'skipped';
|
|
2291
|
+
};
|
|
2292
|
+
|
|
2293
|
+
function stepProgramColumnName(parentField: string, stepId: string): string {
|
|
2294
|
+
return sqlSafePlayColumnName(`${parentField}.${stepId}`);
|
|
3130
2295
|
}
|
|
3131
2296
|
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
2297
|
+
const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
|
|
2298
|
+
const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
|
|
2299
|
+
const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
|
|
2300
|
+
const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 1_000;
|
|
2301
|
+
const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
|
|
2302
|
+
/**
|
|
2303
|
+
* Bounded number of per-row failure samples carried in chunk summaries and the
|
|
2304
|
+
* map's terminal partial-failure log. Every failed row is persisted with its
|
|
2305
|
+
* full error in the runtime sheet; the samples just keep run logs readable.
|
|
2306
|
+
*/
|
|
2307
|
+
const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
|
|
2308
|
+
|
|
2309
|
+
/**
|
|
2310
|
+
* Bounded-concurrent map-chunk dispatch fan-out for the workers_edge profile
|
|
2311
|
+
* (in-process play execution with no `workflowStep`). On workers_edge the whole
|
|
2312
|
+
* map shares ONE per-invocation subrequest window, so sequential chunk dispatch
|
|
2313
|
+
* buys no durability — it is pure wall-clock cost. Running up to K chunks in
|
|
2314
|
+
* flight overlaps provider batches, sheet writes, and the per-chunk persist
|
|
2315
|
+
* barrier. K is capped at 3 to bound peak memory: each in-flight chunk holds up
|
|
2316
|
+
* to ~5000 rows plus their row buffers, so 3 concurrent chunks is the ceiling
|
|
2317
|
+
* we accept before row payloads risk pressuring the isolate. The native
|
|
2318
|
+
* Cloudflare Workflows path (workflowStep present) keeps K=1: there each chunk
|
|
2319
|
+
* is a durable step and sequential dispatch is what makes replay/resume sound.
|
|
2320
|
+
*/
|
|
2321
|
+
const WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS = 3;
|
|
2322
|
+
const WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES =
|
|
2323
|
+
WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES;
|
|
2324
|
+
const WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES = 128;
|
|
2325
|
+
|
|
2326
|
+
const workerMapResidentRowBytes = (row: unknown): number =>
|
|
2327
|
+
new TextEncoder().encode(JSON.stringify(row)).byteLength +
|
|
2328
|
+
WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES;
|
|
2329
|
+
|
|
2330
|
+
function isRuntimeReceiptPersistenceError(error: unknown): boolean {
|
|
2331
|
+
if (!error || typeof error !== 'object') return false;
|
|
2332
|
+
if (error instanceof RuntimeReceiptPersistenceError) return true;
|
|
2333
|
+
if (
|
|
2334
|
+
error instanceof Error &&
|
|
2335
|
+
error.name === 'RuntimeReceiptPersistenceError'
|
|
2336
|
+
) {
|
|
2337
|
+
return true;
|
|
3143
2338
|
}
|
|
3144
|
-
|
|
2339
|
+
const nestedErrors = (error as { errors?: unknown }).errors;
|
|
2340
|
+
return (
|
|
2341
|
+
Array.isArray(nestedErrors) &&
|
|
2342
|
+
nestedErrors.some(isRuntimeReceiptPersistenceError)
|
|
2343
|
+
);
|
|
3145
2344
|
}
|
|
3146
2345
|
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
completeRequests: (
|
|
3160
|
-
entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
|
|
3161
|
-
) => Promise<unknown[]>;
|
|
3162
|
-
failRequests: (
|
|
3163
|
-
requests: ClaimedWorkerToolBatchRequest[],
|
|
3164
|
-
error: unknown,
|
|
3165
|
-
) => Promise<void>;
|
|
3166
|
-
settleRequest: (
|
|
3167
|
-
request: ClaimedWorkerToolBatchRequest,
|
|
3168
|
-
result: unknown,
|
|
3169
|
-
) => void;
|
|
3170
|
-
rejectRequest: (
|
|
3171
|
-
request: ClaimedWorkerToolBatchRequest,
|
|
3172
|
-
error: unknown,
|
|
3173
|
-
) => void;
|
|
3174
|
-
latch: RuntimePersistenceLatch;
|
|
3175
|
-
}): Promise<void> {
|
|
3176
|
-
const compiledBatches = compileRequestsWithStrategy({
|
|
3177
|
-
requests: input.requests,
|
|
3178
|
-
strategy: input.strategy,
|
|
3179
|
-
getPayload: (request) => request.request.input,
|
|
3180
|
-
});
|
|
3181
|
-
recordRunnerPerfTrace({
|
|
3182
|
-
req: input.req,
|
|
3183
|
-
phase: 'runner.tool.batch.compile',
|
|
3184
|
-
ms: 0,
|
|
3185
|
-
extra: {
|
|
3186
|
-
sourceOperation: input.strategy.sourceOperation,
|
|
3187
|
-
batchOperation: input.strategy.batchOperation,
|
|
3188
|
-
requests: input.requests.length,
|
|
3189
|
-
batches: compiledBatches.length,
|
|
3190
|
-
batchSizes: compiledBatches.map((batch) => batch.memberRequests.length),
|
|
3191
|
-
},
|
|
3192
|
-
});
|
|
2346
|
+
function isRunFatalWorkerRowError(error: unknown): boolean {
|
|
2347
|
+
return (
|
|
2348
|
+
isCoordinatorProgressAuthorizationError(error) ||
|
|
2349
|
+
isRowIsolationExemptError(error) ||
|
|
2350
|
+
isRuntimeReceiptPersistenceError(error) ||
|
|
2351
|
+
isRuntimePersistenceCircuitOpenError(error) ||
|
|
2352
|
+
error instanceof RuntimeSheetAttemptLeaseLostError ||
|
|
2353
|
+
error instanceof RuntimeLeaseLostError ||
|
|
2354
|
+
(error instanceof Error &&
|
|
2355
|
+
error.name === 'RuntimeSheetAttemptLeaseLostError')
|
|
2356
|
+
);
|
|
2357
|
+
}
|
|
3193
2358
|
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
// billing it all. A single over-budget batch still forms its own wave.
|
|
3206
|
-
maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
|
|
3207
|
-
weightOf: (batch) => batch.memberRequests.length,
|
|
3208
|
-
execute: async (batch) => {
|
|
3209
|
-
// Circuit breaker: once a persistence failure has occurred in this map,
|
|
3210
|
-
// do NOT dispatch this batch's provider call. The chunk plumbing rejects
|
|
3211
|
-
// its member requests with the latch error.
|
|
3212
|
-
if (input.latch.tripped) {
|
|
3213
|
-
input.latch.preventedCallCount += batch.memberRequests.length;
|
|
3214
|
-
throw new RuntimePersistenceCircuitOpenError(input.latch);
|
|
3215
|
-
}
|
|
3216
|
-
const toolContract = await input
|
|
3217
|
-
.resolveToolContract(batch.batchOperation)
|
|
3218
|
-
.catch(() => null);
|
|
3219
|
-
// One provider call per batch → one tool slot (budget + global
|
|
3220
|
-
// concurrency + per-(org,provider) pacing) around the whole batch.
|
|
3221
|
-
const slot = await input.governor.acquireToolSlot(batch.batchOperation, {
|
|
3222
|
-
signal: input.abortSignal,
|
|
3223
|
-
});
|
|
3224
|
-
try {
|
|
3225
|
-
input.callbacks?.onToolCalled?.(batch.batchOperation, nowMs());
|
|
3226
|
-
return await executeTool(
|
|
3227
|
-
input.req,
|
|
3228
|
-
{
|
|
3229
|
-
id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
|
|
3230
|
-
toolId: batch.batchOperation,
|
|
3231
|
-
input: batch.batchPayload,
|
|
3232
|
-
},
|
|
3233
|
-
undefined,
|
|
3234
|
-
input.reportBackpressure,
|
|
3235
|
-
() => input.governor.chargeBudget('retry'),
|
|
3236
|
-
toolContract?.retrySafeTransientHttp === true,
|
|
3237
|
-
input.abortSignal,
|
|
3238
|
-
input.runtimeDeadlineMs,
|
|
3239
|
-
resolveClaimedWorkerToolRuntimeTimeoutMs(batch.memberRequests, {
|
|
3240
|
-
runtimeDeadlineMs: input.runtimeDeadlineMs,
|
|
3241
|
-
}),
|
|
3242
|
-
);
|
|
3243
|
-
} catch (error) {
|
|
3244
|
-
input.callbacks?.onToolFailed?.(batch.batchOperation, nowMs());
|
|
3245
|
-
throw error;
|
|
3246
|
-
} finally {
|
|
3247
|
-
slot.release();
|
|
3248
|
-
}
|
|
3249
|
-
},
|
|
3250
|
-
onChunkComplete: async (
|
|
3251
|
-
chunkResults: Array<
|
|
3252
|
-
ChunkExecutionResult<(typeof compiledBatches)[number], unknown>
|
|
3253
|
-
>,
|
|
3254
|
-
) => {
|
|
3255
|
-
for (const entry of chunkResults) {
|
|
3256
|
-
if (entry.error !== undefined) {
|
|
3257
|
-
// One batch's provider error stays scoped to that batch's member
|
|
3258
|
-
// requests. Sibling batches in this chunk keep their results so a
|
|
3259
|
-
// single provider hiccup cannot cascade into a whole-map failure.
|
|
3260
|
-
let rejection: unknown = entry.error;
|
|
3261
|
-
try {
|
|
3262
|
-
await input.failRequests(entry.request.memberRequests, entry.error);
|
|
3263
|
-
} catch (receiptError) {
|
|
3264
|
-
tripRuntimePersistenceLatch(input.latch, receiptError);
|
|
3265
|
-
rejection = new RuntimeReceiptPersistenceError(
|
|
3266
|
-
'Tool call failed and durable receipts could not be marked failed',
|
|
3267
|
-
[entry.error, receiptError],
|
|
3268
|
-
);
|
|
3269
|
-
}
|
|
3270
|
-
for (const claimed of entry.request.memberRequests) {
|
|
3271
|
-
input.rejectRequest(claimed, rejection);
|
|
3272
|
-
}
|
|
3273
|
-
continue;
|
|
3274
|
-
}
|
|
3275
|
-
const batchResult = isToolExecuteResult(entry.result)
|
|
3276
|
-
? entry.result.toolOutput.raw
|
|
3277
|
-
: entry.result;
|
|
3278
|
-
const splitResults =
|
|
3279
|
-
batchResult != null
|
|
3280
|
-
? entry.request.splitResults(batchResult)
|
|
3281
|
-
: entry.request.memberRequests.map(() => null);
|
|
3282
|
-
let completedResults: unknown[];
|
|
3283
|
-
try {
|
|
3284
|
-
completedResults = await input.completeRequests(
|
|
3285
|
-
entry.request.memberRequests.map((claimed, index) => ({
|
|
3286
|
-
claimed,
|
|
3287
|
-
result: wrapWorkerToolResult(
|
|
3288
|
-
claimed.request.toolId,
|
|
3289
|
-
splitResults[index] ?? null,
|
|
3290
|
-
toolMetadataFallback(claimed.request.toolId),
|
|
3291
|
-
splitResults[index] == null ? 'no_result' : 'completed',
|
|
3292
|
-
{
|
|
3293
|
-
req: input.req,
|
|
3294
|
-
input: claimed.request.input,
|
|
3295
|
-
},
|
|
3296
|
-
),
|
|
3297
|
-
})),
|
|
3298
|
-
);
|
|
3299
|
-
} catch (receiptError) {
|
|
3300
|
-
tripRuntimePersistenceLatch(input.latch, receiptError);
|
|
3301
|
-
const rejection = new RuntimeReceiptPersistenceError(
|
|
3302
|
-
'Tool call succeeded but durable receipts could not be marked completed',
|
|
3303
|
-
[receiptError],
|
|
3304
|
-
);
|
|
3305
|
-
for (const claimed of entry.request.memberRequests) {
|
|
3306
|
-
input.rejectRequest(claimed, rejection);
|
|
3307
|
-
}
|
|
3308
|
-
continue;
|
|
3309
|
-
}
|
|
3310
|
-
for (let index = 0; index < completedResults.length; index += 1) {
|
|
3311
|
-
const claimed = entry.request.memberRequests[index]!;
|
|
3312
|
-
const request = claimed.request;
|
|
3313
|
-
input.settleRequest(claimed, completedResults[index]);
|
|
3314
|
-
}
|
|
3315
|
-
}
|
|
3316
|
-
},
|
|
3317
|
-
}).catch(async (error) => {
|
|
3318
|
-
let rejection: unknown = error;
|
|
3319
|
-
try {
|
|
3320
|
-
await input.failRequests(input.requests, error);
|
|
3321
|
-
} catch (receiptError) {
|
|
3322
|
-
tripRuntimePersistenceLatch(input.latch, receiptError);
|
|
3323
|
-
rejection = new RuntimeReceiptPersistenceError(
|
|
3324
|
-
'Tool call failed and durable receipts could not be marked failed',
|
|
3325
|
-
[error, receiptError],
|
|
3326
|
-
);
|
|
3327
|
-
}
|
|
3328
|
-
for (const claimed of input.requests) {
|
|
3329
|
-
input.rejectRequest(claimed, rejection);
|
|
3330
|
-
}
|
|
3331
|
-
});
|
|
2359
|
+
/**
|
|
2360
|
+
* In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
|
|
2361
|
+
* throughput pacing (provider or Deepline limiter), not a tool defect, so it
|
|
2362
|
+
* gets more patience than the generic 3-attempt budget: with retry-after-aware
|
|
2363
|
+
* escalating delays (capped at 5s) this absorbs roughly 25s of sustained
|
|
2364
|
+
* throttling before the call fails. Every retry still charges the Governor's
|
|
2365
|
+
* retry budget, so a runaway storm stays bounded and loud.
|
|
2366
|
+
*/
|
|
2367
|
+
|
|
2368
|
+
function sleepWorkerMs(ms: number): Promise<void> {
|
|
2369
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3332
2370
|
}
|
|
3333
2371
|
|
|
3334
2372
|
function isCompletedWorkerFieldValue(value: unknown): boolean {
|
|
@@ -3339,7 +2377,8 @@ function isCompletedWorkerFieldValue(value: unknown): boolean {
|
|
|
3339
2377
|
);
|
|
3340
2378
|
}
|
|
3341
2379
|
|
|
3342
|
-
type
|
|
2380
|
+
type WorkerMapCompletedChunkSummary<T extends Record<string, unknown>> = {
|
|
2381
|
+
status: 'completed';
|
|
3343
2382
|
chunkIndex: number;
|
|
3344
2383
|
rangeStart: number;
|
|
3345
2384
|
rangeEnd: number;
|
|
@@ -3363,6 +2402,23 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
|
|
|
3363
2402
|
cachedRows?: T[];
|
|
3364
2403
|
};
|
|
3365
2404
|
|
|
2405
|
+
type WorkerMapBlockedChunkSummary = {
|
|
2406
|
+
status: 'blocked';
|
|
2407
|
+
chunkIndex: number;
|
|
2408
|
+
rangeStart: number;
|
|
2409
|
+
rangeEnd: number;
|
|
2410
|
+
rowsRead: number;
|
|
2411
|
+
rowsBlocked: number;
|
|
2412
|
+
rowKeySamples: string[];
|
|
2413
|
+
blockedRowSamples: Record<string, unknown>[];
|
|
2414
|
+
outputDatasetId: string;
|
|
2415
|
+
message: string;
|
|
2416
|
+
};
|
|
2417
|
+
|
|
2418
|
+
type WorkerMapChunkSummary<T extends Record<string, unknown>> =
|
|
2419
|
+
| WorkerMapCompletedChunkSummary<T>
|
|
2420
|
+
| WorkerMapBlockedChunkSummary;
|
|
2421
|
+
|
|
3366
2422
|
function serializeDurableStepValue<T>(value: T, depth = 0): T {
|
|
3367
2423
|
if (depth > 20 || value == null) return value;
|
|
3368
2424
|
if (isToolExecuteResult(value)) return serializeToolExecuteResult(value) as T;
|
|
@@ -3575,6 +2631,7 @@ async function executeWorkerStepProgram(
|
|
|
3575
2631
|
...(resolution.status ? { status: resolution.status } : {}),
|
|
3576
2632
|
};
|
|
3577
2633
|
};
|
|
2634
|
+
|
|
3578
2635
|
const executeStep = async (): Promise<WorkerStepResolution> =>
|
|
3579
2636
|
workflowStep
|
|
3580
2637
|
? await (
|
|
@@ -3799,7 +2856,7 @@ async function postRuntimeEgressFetch(
|
|
|
3799
2856
|
) {
|
|
3800
2857
|
const response = await fetchRuntimeApi(
|
|
3801
2858
|
req.baseUrl,
|
|
3802
|
-
'/api/v2/
|
|
2859
|
+
'/api/v2/internal/play-runtime/egress-fetch',
|
|
3803
2860
|
{
|
|
3804
2861
|
method: 'POST',
|
|
3805
2862
|
headers: {
|
|
@@ -4375,6 +3432,81 @@ function resolveSheetContractFromReq(
|
|
|
4375
3432
|
}
|
|
4376
3433
|
}
|
|
4377
3434
|
|
|
3435
|
+
/**
|
|
3436
|
+
* Run-fatal teardown: release the runtime-sheet attempt leases and work-receipt
|
|
3437
|
+
* leases this run still holds, so an immediate rerun proceeds without waiting
|
|
3438
|
+
* out the 10-minute lease TTL. Every release is attempt/owner fenced downstream,
|
|
3439
|
+
* so it can never weaken a different run's live lease. Failures are logged loud
|
|
3440
|
+
* and swallowed here — the caller still throws the original run-fatal error.
|
|
3441
|
+
*/
|
|
3442
|
+
async function releaseRuntimeLeasesOnTeardown(input: {
|
|
3443
|
+
req: RunRequest;
|
|
3444
|
+
leasedSheetNamespaces: Set<string>;
|
|
3445
|
+
emit: (event: RunnerEvent) => void;
|
|
3446
|
+
}): Promise<void> {
|
|
3447
|
+
const { req, leasedSheetNamespaces, emit } = input;
|
|
3448
|
+
const scope = runtimeSheetSessionScope(req);
|
|
3449
|
+
for (const tableNamespace of leasedSheetNamespaces) {
|
|
3450
|
+
const sheetContract = resolveSheetContractFromReq(req, tableNamespace);
|
|
3451
|
+
if (!sheetContract) continue;
|
|
3452
|
+
try {
|
|
3453
|
+
const released = await harnessReleaseSheetAttempt({
|
|
3454
|
+
...scope,
|
|
3455
|
+
tableNamespace,
|
|
3456
|
+
sheetContract,
|
|
3457
|
+
runId: req.runId,
|
|
3458
|
+
attemptOwnerRunId: req.runId,
|
|
3459
|
+
attemptSeq: req.runAttempt ?? 0,
|
|
3460
|
+
});
|
|
3461
|
+
if (released.released > 0) {
|
|
3462
|
+
emit({
|
|
3463
|
+
type: 'log',
|
|
3464
|
+
level: 'info',
|
|
3465
|
+
message: `Released ${released.released} runtime sheet row lease(s) for ctx.dataset("${tableNamespace}") on run-fatal teardown.`,
|
|
3466
|
+
ts: nowMs(),
|
|
3467
|
+
});
|
|
3468
|
+
}
|
|
3469
|
+
} catch (releaseError) {
|
|
3470
|
+
emit({
|
|
3471
|
+
type: 'log',
|
|
3472
|
+
level: 'warn',
|
|
3473
|
+
message: `Runtime sheet attempt release failed for ctx.dataset("${tableNamespace}") on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
|
|
3474
|
+
releaseError instanceof Error
|
|
3475
|
+
? releaseError.message
|
|
3476
|
+
: String(releaseError)
|
|
3477
|
+
}`,
|
|
3478
|
+
ts: nowMs(),
|
|
3479
|
+
});
|
|
3480
|
+
}
|
|
3481
|
+
}
|
|
3482
|
+
try {
|
|
3483
|
+
const released = await harnessReleaseRuntimeReceipts({
|
|
3484
|
+
...scope,
|
|
3485
|
+
runId: req.runId,
|
|
3486
|
+
runAttempt: req.runAttempt ?? 0,
|
|
3487
|
+
});
|
|
3488
|
+
if (released.released > 0) {
|
|
3489
|
+
emit({
|
|
3490
|
+
type: 'log',
|
|
3491
|
+
level: 'info',
|
|
3492
|
+
message: `Released ${released.released} work-receipt lease(s) on run-fatal teardown.`,
|
|
3493
|
+
ts: nowMs(),
|
|
3494
|
+
});
|
|
3495
|
+
}
|
|
3496
|
+
} catch (releaseError) {
|
|
3497
|
+
emit({
|
|
3498
|
+
type: 'log',
|
|
3499
|
+
level: 'warn',
|
|
3500
|
+
message: `Work-receipt release failed on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
|
|
3501
|
+
releaseError instanceof Error
|
|
3502
|
+
? releaseError.message
|
|
3503
|
+
: String(releaseError)
|
|
3504
|
+
}`,
|
|
3505
|
+
ts: nowMs(),
|
|
3506
|
+
});
|
|
3507
|
+
}
|
|
3508
|
+
}
|
|
3509
|
+
|
|
4378
3510
|
function staticPipelineFromReq(req: RunRequest): PlayStaticPipeline | null {
|
|
4379
3511
|
const snapshot = req.contractSnapshot as
|
|
4380
3512
|
| { staticPipeline?: unknown }
|
|
@@ -4417,6 +3549,13 @@ async function persistCompletedMapRows(input: {
|
|
|
4417
3549
|
rows: Record<string, unknown>[];
|
|
4418
3550
|
outputFields: string[];
|
|
4419
3551
|
extraOutputFields?: string[];
|
|
3552
|
+
attemptId?: string | null;
|
|
3553
|
+
attemptOwnerRunId?: string | null;
|
|
3554
|
+
attemptExpiresAt?: string | null;
|
|
3555
|
+
attemptSeq?: number | null;
|
|
3556
|
+
budgetMeter?: WorkerWorkBudgetMeter;
|
|
3557
|
+
force?: boolean;
|
|
3558
|
+
forceFailedRows?: boolean;
|
|
4420
3559
|
}): Promise<{
|
|
4421
3560
|
rows: number;
|
|
4422
3561
|
written: number;
|
|
@@ -4462,23 +3601,28 @@ async function persistCompletedMapRows(input: {
|
|
|
4462
3601
|
sheetContract,
|
|
4463
3602
|
rows,
|
|
4464
3603
|
outputFields,
|
|
3604
|
+
attemptId: input.attemptId ?? null,
|
|
3605
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3606
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3607
|
+
attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
|
|
3608
|
+
forceFailedRows: input.forceFailedRows === true,
|
|
4465
3609
|
};
|
|
3610
|
+
input.budgetMeter?.count('sheet');
|
|
4466
3611
|
const expectedKeys = [
|
|
4467
3612
|
...new Set(
|
|
4468
3613
|
input.rows
|
|
4469
|
-
.map((row) => resolveMapRowOutcomeKey(row))
|
|
4470
|
-
.filter(
|
|
3614
|
+
.map((row) => resolveMapRowOutcomeKey(row) ?? row.key)
|
|
3615
|
+
.filter(Boolean),
|
|
4471
3616
|
),
|
|
4472
|
-
];
|
|
4473
|
-
const expectedVisibleRows =
|
|
4474
|
-
expectedKeys.length > 0 ? expectedKeys.length : rows.length;
|
|
3617
|
+
] as string[];
|
|
3618
|
+
const expectedVisibleRows = expectedKeys.length || rows.length;
|
|
4475
3619
|
const readVisibleRowCount = async () => {
|
|
4476
3620
|
if (expectedKeys.length > 0) {
|
|
4477
3621
|
const result = await harnessReadSheetDatasetRowKeys({
|
|
4478
3622
|
...sessionScope,
|
|
4479
3623
|
tableNamespace,
|
|
4480
3624
|
runId: req.runId,
|
|
4481
|
-
rowMode: '
|
|
3625
|
+
rowMode: 'terminalAnyRun',
|
|
4482
3626
|
keys: expectedKeys,
|
|
4483
3627
|
});
|
|
4484
3628
|
return result.keys.length;
|
|
@@ -4494,6 +3638,24 @@ async function persistCompletedMapRows(input: {
|
|
|
4494
3638
|
return result.rows.length;
|
|
4495
3639
|
};
|
|
4496
3640
|
const result = await harnessPersistCompletedSheetRows(persistRequest);
|
|
3641
|
+
const hasAttemptFence = Boolean(input.attemptId?.trim());
|
|
3642
|
+
const assertAttemptStillOwnsTerminalWrites = (
|
|
3643
|
+
written: number,
|
|
3644
|
+
visible: number | null,
|
|
3645
|
+
) => {
|
|
3646
|
+
// Recovery finalization may include rows completed by an earlier attempt.
|
|
3647
|
+
// A complete terminal readback proves they were not lost.
|
|
3648
|
+
if (
|
|
3649
|
+
!hasAttemptFence ||
|
|
3650
|
+
written === rows.length ||
|
|
3651
|
+
(visible !== null && visible >= expectedVisibleRows)
|
|
3652
|
+
) {
|
|
3653
|
+
return;
|
|
3654
|
+
}
|
|
3655
|
+
throw new Error(
|
|
3656
|
+
`Runtime sheet attempt lost ownership for ${tableNamespace}: wrote ${written}/${rows.length}; run ${req.runId}; attempt ${input.attemptId}.`,
|
|
3657
|
+
);
|
|
3658
|
+
};
|
|
4497
3659
|
console.warn('[play-runner.persist_completed_map_rows.result]', {
|
|
4498
3660
|
runId: req.runId,
|
|
4499
3661
|
playName: req.playName,
|
|
@@ -4518,11 +3680,19 @@ async function persistCompletedMapRows(input: {
|
|
|
4518
3680
|
sheetContract,
|
|
4519
3681
|
rows: input.rows.map((row) => runtimeCsvStorageRow(row)),
|
|
4520
3682
|
runId: req.runId,
|
|
3683
|
+
attemptId: input.attemptId ?? null,
|
|
3684
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3685
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3686
|
+
attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
|
|
3687
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
4521
3688
|
inputOffset: 0,
|
|
4522
3689
|
});
|
|
3690
|
+
input.budgetMeter?.count('sheet');
|
|
4523
3691
|
const retry = await harnessPersistCompletedSheetRows(persistRequest);
|
|
3692
|
+
input.budgetMeter?.count('sheet');
|
|
4524
3693
|
retryWritten = retry.rowsWritten;
|
|
4525
3694
|
retryVisible = await readVisibleRowCount();
|
|
3695
|
+
assertAttemptStillOwnsTerminalWrites(retryWritten, retryVisible);
|
|
4526
3696
|
if (retryVisible >= expectedVisibleRows) {
|
|
4527
3697
|
return {
|
|
4528
3698
|
rows: rows.length,
|
|
@@ -4536,6 +3706,7 @@ async function persistCompletedMapRows(input: {
|
|
|
4536
3706
|
`Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}/${expectedVisibleRows}; run ${req.runId}.`,
|
|
4537
3707
|
);
|
|
4538
3708
|
}
|
|
3709
|
+
assertAttemptStillOwnsTerminalWrites(result.rowsWritten, visibleRows);
|
|
4539
3710
|
}
|
|
4540
3711
|
if (result.rowsWritten !== rows.length && visibleRows < expectedVisibleRows) {
|
|
4541
3712
|
throw new Error(
|
|
@@ -4557,8 +3728,14 @@ async function applyLiveMapRowUpdates(input: {
|
|
|
4557
3728
|
updates: Array<Omit<PlayRowUpdate, 'rowId'> & { runId?: string }>;
|
|
4558
3729
|
outputFields: string[];
|
|
4559
3730
|
extraOutputFields?: string[];
|
|
3731
|
+
attemptId?: string | null;
|
|
3732
|
+
attemptOwnerRunId?: string | null;
|
|
3733
|
+
attemptExpiresAt?: string | null;
|
|
3734
|
+
attemptSeq?: number | null;
|
|
3735
|
+
budgetMeter?: WorkerWorkBudgetMeter;
|
|
4560
3736
|
}): Promise<void> {
|
|
4561
3737
|
if (input.updates.length === 0) return;
|
|
3738
|
+
input.budgetMeter?.count('sheet');
|
|
4562
3739
|
const outputFields = [
|
|
4563
3740
|
...input.outputFields,
|
|
4564
3741
|
...(input.extraOutputFields ?? []).filter(
|
|
@@ -4581,7 +3758,16 @@ async function applyLiveMapRowUpdates(input: {
|
|
|
4581
3758
|
contractSnapshot: input.req.contractSnapshot,
|
|
4582
3759
|
runId: input.req.runId,
|
|
4583
3760
|
userEmail: input.req.userEmail,
|
|
4584
|
-
updates: input.updates
|
|
3761
|
+
updates: input.updates.map((update) => ({
|
|
3762
|
+
...update,
|
|
3763
|
+
attemptId: update.attemptId ?? input.attemptId ?? null,
|
|
3764
|
+
attemptOwnerRunId:
|
|
3765
|
+
update.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
|
|
3766
|
+
attemptExpiresAt:
|
|
3767
|
+
update.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
|
|
3768
|
+
attemptSeq:
|
|
3769
|
+
update.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3770
|
+
})),
|
|
4585
3771
|
},
|
|
4586
3772
|
);
|
|
4587
3773
|
}
|
|
@@ -4592,18 +3778,39 @@ async function prepareMapRows(input: {
|
|
|
4592
3778
|
rows: Record<string, unknown>[];
|
|
4593
3779
|
inputOffset: number;
|
|
4594
3780
|
outputFields: string[];
|
|
3781
|
+
attemptId?: string | null;
|
|
3782
|
+
attemptOwnerRunId?: string | null;
|
|
3783
|
+
attemptExpiresAt?: string | null;
|
|
3784
|
+
attemptSeq?: number | null;
|
|
3785
|
+
budgetMeter?: WorkerWorkBudgetMeter;
|
|
4595
3786
|
force?: boolean;
|
|
4596
3787
|
}): Promise<{
|
|
4597
3788
|
inserted: number;
|
|
4598
3789
|
skipped: number;
|
|
4599
3790
|
pendingRows: Record<string, unknown>[];
|
|
4600
3791
|
completedRows: Record<string, unknown>[];
|
|
3792
|
+
blockedRows: Record<string, unknown>[];
|
|
3793
|
+
attemptId?: string | null;
|
|
3794
|
+
attemptOwnerRunId?: string | null;
|
|
3795
|
+
attemptExpiresAt?: string | null;
|
|
3796
|
+
attemptSeq?: number | null;
|
|
4601
3797
|
}> {
|
|
4602
3798
|
if (input.rows.length === 0) {
|
|
4603
|
-
return {
|
|
3799
|
+
return {
|
|
3800
|
+
inserted: 0,
|
|
3801
|
+
skipped: 0,
|
|
3802
|
+
pendingRows: [],
|
|
3803
|
+
completedRows: [],
|
|
3804
|
+
blockedRows: [],
|
|
3805
|
+
attemptId: input.attemptId ?? null,
|
|
3806
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3807
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3808
|
+
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3809
|
+
};
|
|
4604
3810
|
}
|
|
4605
3811
|
const sessionScope = runtimeSheetSessionScope(input.req);
|
|
4606
3812
|
const rows = input.rows.map((row) => runtimeCsvStorageRow(row));
|
|
3813
|
+
input.budgetMeter?.count('sheet');
|
|
4607
3814
|
const result = await harnessStartSheetDataset({
|
|
4608
3815
|
...sessionScope,
|
|
4609
3816
|
tableNamespace: input.tableNamespace,
|
|
@@ -4614,6 +3821,11 @@ async function prepareMapRows(input: {
|
|
|
4614
3821
|
}),
|
|
4615
3822
|
rows,
|
|
4616
3823
|
inputOffset: input.inputOffset,
|
|
3824
|
+
attemptId: input.attemptId ?? null,
|
|
3825
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3826
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3827
|
+
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3828
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
|
|
4617
3829
|
...(input.force ? { force: true } : {}),
|
|
4618
3830
|
});
|
|
4619
3831
|
for (const timing of result.timings ?? []) {
|
|
@@ -4643,6 +3855,13 @@ async function prepareMapRows(input: {
|
|
|
4643
3855
|
skipped: result.skipped,
|
|
4644
3856
|
pendingRows: result.pendingRows,
|
|
4645
3857
|
completedRows: result.completedRows,
|
|
3858
|
+
blockedRows: result.blockedRows ?? [],
|
|
3859
|
+
attemptId: result.attemptId ?? input.attemptId ?? null,
|
|
3860
|
+
attemptOwnerRunId:
|
|
3861
|
+
result.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
|
|
3862
|
+
attemptExpiresAt: result.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
|
|
3863
|
+
attemptSeq:
|
|
3864
|
+
result.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
4646
3865
|
};
|
|
4647
3866
|
}
|
|
4648
3867
|
|
|
@@ -4788,14 +4007,6 @@ function createCoordinatorRatePort(req: RunRequest): CoordinatorRatePort {
|
|
|
4788
4007
|
* memoized per isolate. No hints → null (pacing is a no-op; the Governor's
|
|
4789
4008
|
* global tool-concurrency slot still applies).
|
|
4790
4009
|
*/
|
|
4791
|
-
type WorkerPacingResolver = (
|
|
4792
|
-
toolId: string,
|
|
4793
|
-
) => Promise<
|
|
4794
|
-
(ResolvedPacingPolicy & { retrySafeTransientHttp: boolean }) | null
|
|
4795
|
-
>;
|
|
4796
|
-
|
|
4797
|
-
type WorkerToolActionCacheVersionResolver = (toolId: string) => Promise<string>;
|
|
4798
|
-
|
|
4799
4010
|
function createWorkerToolActionCacheVersionResolver(
|
|
4800
4011
|
req: RunRequest,
|
|
4801
4012
|
): WorkerToolActionCacheVersionResolver {
|
|
@@ -4803,9 +4014,7 @@ function createWorkerToolActionCacheVersionResolver(
|
|
|
4803
4014
|
return (toolId: string) => {
|
|
4804
4015
|
const normalized = String(toolId || '').trim();
|
|
4805
4016
|
if (!normalized) {
|
|
4806
|
-
return Promise.reject(
|
|
4807
|
-
new Error('Runtime tool metadata lookup requires a non-empty tool id.'),
|
|
4808
|
-
);
|
|
4017
|
+
return Promise.reject(new Error('Tool metadata needs tool id.'));
|
|
4809
4018
|
}
|
|
4810
4019
|
const cached = cache.get(normalized);
|
|
4811
4020
|
if (cached) return cached;
|
|
@@ -4842,6 +4051,44 @@ function createWorkerToolActionCacheVersionResolver(
|
|
|
4842
4051
|
};
|
|
4843
4052
|
}
|
|
4844
4053
|
|
|
4054
|
+
function createWorkerToolAuthScopeDigestResolver(req: RunRequest) {
|
|
4055
|
+
// Return type is inferred from createRuntimeToolAuthScopeDigestResolver
|
|
4056
|
+
// (RuntimeToolAuthScopeDigestResolver), whose `invalidate` member is
|
|
4057
|
+
// REQUIRED — the AUTH_SCOPE_CHANGED re-claim wiring below calls it directly,
|
|
4058
|
+
// and a resolver without eviction support must be a type error.
|
|
4059
|
+
return createRuntimeToolAuthScopeDigestResolver({
|
|
4060
|
+
missingToolIdMessage: 'Tool auth scope needs tool id.',
|
|
4061
|
+
fetchDigest: async (normalized) => {
|
|
4062
|
+
const runScopedPath =
|
|
4063
|
+
`/api/v2/plays/runtime-tools/${encodeURIComponent(normalized)}/auth-scope` +
|
|
4064
|
+
`?runId=${encodeURIComponent(req.runId)}`;
|
|
4065
|
+
const res = await fetchRuntimeApi(req.baseUrl, runScopedPath, {
|
|
4066
|
+
method: 'GET',
|
|
4067
|
+
headers: {
|
|
4068
|
+
authorization: `Bearer ${req.executorToken}`,
|
|
4069
|
+
'cache-control': 'no-store',
|
|
4070
|
+
pragma: 'no-cache',
|
|
4071
|
+
},
|
|
4072
|
+
});
|
|
4073
|
+
if (!res.ok) {
|
|
4074
|
+
throw new Error(
|
|
4075
|
+
`Runtime tool auth scope lookup for ${normalized} failed (${res.status}): ${await res.text()}`,
|
|
4076
|
+
);
|
|
4077
|
+
}
|
|
4078
|
+
const body = (await res.json().catch(() => null)) as {
|
|
4079
|
+
digest?: unknown;
|
|
4080
|
+
} | null;
|
|
4081
|
+
const digest = typeof body?.digest === 'string' ? body.digest.trim() : '';
|
|
4082
|
+
if (!digest) {
|
|
4083
|
+
throw new Error(
|
|
4084
|
+
`Runtime tool auth scope lookup for ${normalized} returned no digest.`,
|
|
4085
|
+
);
|
|
4086
|
+
}
|
|
4087
|
+
return digest;
|
|
4088
|
+
},
|
|
4089
|
+
});
|
|
4090
|
+
}
|
|
4091
|
+
|
|
4845
4092
|
function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
|
|
4846
4093
|
const cache = new Map<
|
|
4847
4094
|
string,
|
|
@@ -4983,13 +4230,22 @@ function createMinimalWorkerCtx(
|
|
|
4983
4230
|
workflowStep?: WorkflowStep,
|
|
4984
4231
|
abortSignal?: AbortSignal,
|
|
4985
4232
|
callbacks?: WorkerCtxCallbacks,
|
|
4233
|
+
workBudgetMeter: WorkerWorkBudgetMeter = createWorkerWorkBudgetMeter({
|
|
4234
|
+
nowMs,
|
|
4235
|
+
}),
|
|
4986
4236
|
runtimeDeadlineMs?: number,
|
|
4237
|
+
// Namespaces this run took a runtime-sheet attempt lease on. The run-fatal
|
|
4238
|
+
// teardown reads this set to release exactly those leases (never a namespace
|
|
4239
|
+
// whose table the run never created), so a rerun proceeds immediately.
|
|
4240
|
+
leasedSheetNamespaces?: Set<string>,
|
|
4987
4241
|
receiptSalvage?: ReceiptSalvageBuffer,
|
|
4988
4242
|
): unknown {
|
|
4989
4243
|
const { governor, resolvePacing: resolveToolPacing } =
|
|
4990
4244
|
createGovernorForRun(req);
|
|
4991
4245
|
const resolveToolActionCacheVersion =
|
|
4992
4246
|
createWorkerToolActionCacheVersionResolver(req);
|
|
4247
|
+
const resolveToolAuthScopeDigest =
|
|
4248
|
+
createWorkerToolAuthScopeDigestResolver(req);
|
|
4993
4249
|
// Play-call depth/count/per-parent budgets, child-play concurrency, and the
|
|
4994
4250
|
// lineage snapshot are owned by the Governor (createGovernorForRun above).
|
|
4995
4251
|
// The worker keeps only substrate mechanism here.
|
|
@@ -5000,7 +4256,7 @@ function createMinimalWorkerCtx(
|
|
|
5000
4256
|
if (!auth) return {};
|
|
5001
4257
|
const response = await fetchRuntimeApi(
|
|
5002
4258
|
req.baseUrl,
|
|
5003
|
-
'/api/v2/
|
|
4259
|
+
'/api/v2/internal/play-runtime',
|
|
5004
4260
|
{
|
|
5005
4261
|
method: 'POST',
|
|
5006
4262
|
headers: {
|
|
@@ -5039,10 +4295,48 @@ function createMinimalWorkerCtx(
|
|
|
5039
4295
|
orgId: req.orgId,
|
|
5040
4296
|
preloadedDbSessions: req.preloadedDbSessions ?? null,
|
|
5041
4297
|
userEmail: req.userEmail,
|
|
4298
|
+
runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null,
|
|
4299
|
+
runAttempt: req.runAttempt ?? 0,
|
|
4300
|
+
receiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
|
|
5042
4301
|
});
|
|
4302
|
+
const executeDispatchedTool = async (input: {
|
|
4303
|
+
id: string;
|
|
4304
|
+
toolId: string;
|
|
4305
|
+
input: Record<string, unknown>;
|
|
4306
|
+
workflowStep?: unknown;
|
|
4307
|
+
callbacks?: WorkerCtxCallbacks;
|
|
4308
|
+
onProviderBackpressure?: (retryAfterMs: number) => void;
|
|
4309
|
+
onRetryAttempt?: () => void;
|
|
4310
|
+
transientHttpRetrySafe?: boolean;
|
|
4311
|
+
durableCallReceiptKey?: string | null;
|
|
4312
|
+
executionAuthScopeDigest?: string | null;
|
|
4313
|
+
providerIdempotencyKey?: string | null;
|
|
4314
|
+
runtimeTimeoutMs?: number;
|
|
4315
|
+
directOptions?: WorkerToolDirectOptions;
|
|
4316
|
+
}): Promise<ToolExecuteResult> =>
|
|
4317
|
+
await executeToolWithLifecycle(
|
|
4318
|
+
req,
|
|
4319
|
+
{
|
|
4320
|
+
id: input.id,
|
|
4321
|
+
toolId: input.toolId,
|
|
4322
|
+
input: input.input,
|
|
4323
|
+
durableCallReceiptKey: input.durableCallReceiptKey,
|
|
4324
|
+
executionAuthScopeDigest: input.executionAuthScopeDigest,
|
|
4325
|
+
providerIdempotencyKey: input.providerIdempotencyKey,
|
|
4326
|
+
},
|
|
4327
|
+
input.workflowStep as WorkflowStep | undefined,
|
|
4328
|
+
input.callbacks,
|
|
4329
|
+
input.onProviderBackpressure,
|
|
4330
|
+
input.onRetryAttempt,
|
|
4331
|
+
input.transientHttpRetrySafe === true,
|
|
4332
|
+
abortSignal,
|
|
4333
|
+
runtimeDeadlineMs,
|
|
4334
|
+
input.runtimeTimeoutMs,
|
|
4335
|
+
input.directOptions,
|
|
4336
|
+
);
|
|
5043
4337
|
const executeWithRuntimeReceipt = async <T>(
|
|
5044
4338
|
key: string,
|
|
5045
|
-
execute: () => Promise<T> | T,
|
|
4339
|
+
execute: (context: WorkerRuntimeReceiptExecutionContext) => Promise<T> | T,
|
|
5046
4340
|
options: {
|
|
5047
4341
|
repairRunningReceiptForSameRun?: boolean;
|
|
5048
4342
|
reclaimRunning?: boolean;
|
|
@@ -5056,7 +4350,8 @@ function createMinimalWorkerCtx(
|
|
|
5056
4350
|
runId: req.runId,
|
|
5057
4351
|
key,
|
|
5058
4352
|
receiptStore,
|
|
5059
|
-
execute: async () =>
|
|
4353
|
+
execute: async (context) =>
|
|
4354
|
+
serializeDurableStepValue(await execute(context)),
|
|
5060
4355
|
repairRunningReceiptForSameRun:
|
|
5061
4356
|
options.repairRunningReceiptForSameRun ?? false,
|
|
5062
4357
|
reclaimRunning: options.reclaimRunning ?? false,
|
|
@@ -5098,24 +4393,54 @@ function createMinimalWorkerCtx(
|
|
|
5098
4393
|
staleAfterSeconds,
|
|
5099
4394
|
});
|
|
5100
4395
|
};
|
|
5101
|
-
|
|
5102
|
-
|
|
5103
|
-
|
|
4396
|
+
const emitAuthScopeReclaimRunLog = (input: {
|
|
4397
|
+
toolId: string;
|
|
4398
|
+
requestIds: string[];
|
|
4399
|
+
}) => {
|
|
4400
|
+
// Durable run-log marker for the bounded AUTH_SCOPE_CHANGED re-claim
|
|
4401
|
+
// (mirrors the shared-runtime marker in shared_libs/play-runtime/context.ts
|
|
4402
|
+
// so run logs expose the credential transition on every execution path).
|
|
4403
|
+
emitEvent({
|
|
4404
|
+
type: 'log',
|
|
4405
|
+
level: 'info',
|
|
4406
|
+
message:
|
|
4407
|
+
`ctx.tools.execute(${input.toolId}): auth_scope_changed_reclaim ` +
|
|
4408
|
+
`(requests: ${input.requestIds.join(', ')}); credential identity ` +
|
|
4409
|
+
`changed mid-run, re-resolved the auth scope and re-claimed fresh ` +
|
|
4410
|
+
`receipts under the new scope.`,
|
|
4411
|
+
ts: nowMs(),
|
|
4412
|
+
});
|
|
4413
|
+
};
|
|
5104
4414
|
const rootPersistenceLatch = createRuntimePersistenceLatch();
|
|
5105
|
-
const rootToolBatchScheduler = new WorkerToolBatchScheduler(
|
|
4415
|
+
const rootToolBatchScheduler = new WorkerToolBatchScheduler({
|
|
5106
4416
|
req,
|
|
5107
4417
|
governor,
|
|
5108
|
-
resolveToolPacing,
|
|
4418
|
+
resolvePacing: resolveToolPacing,
|
|
5109
4419
|
resolveToolActionCacheVersion,
|
|
4420
|
+
resolveToolAuthScopeDigest,
|
|
4421
|
+
invalidateToolAuthScopeDigest: (toolId) =>
|
|
4422
|
+
resolveToolAuthScopeDigest.invalidate(toolId),
|
|
4423
|
+
onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
|
|
4424
|
+
executeTool: executeDispatchedTool,
|
|
4425
|
+
serializeDurableValue: serializeDurableStepValue,
|
|
4426
|
+
deserializeDurableValue: deserializeDurableStepValue,
|
|
4427
|
+
nowMs,
|
|
4428
|
+
batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
|
|
4429
|
+
recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
|
|
5110
4430
|
abortSignal,
|
|
5111
|
-
undefined,
|
|
5112
4431
|
callbacks,
|
|
5113
4432
|
receiptStore,
|
|
5114
|
-
true,
|
|
4433
|
+
allowLocalRetryReceipts: true,
|
|
4434
|
+
runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
|
|
4435
|
+
runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
|
|
4436
|
+
req,
|
|
4437
|
+
runtimeReceiptLeaseTtlMs(req) ?? PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
4438
|
+
),
|
|
4439
|
+
budgetMeter: workBudgetMeter,
|
|
5115
4440
|
runtimeDeadlineMs,
|
|
5116
|
-
rootPersistenceLatch,
|
|
4441
|
+
persistenceLatch: rootPersistenceLatch,
|
|
5117
4442
|
receiptSalvage,
|
|
5118
|
-
);
|
|
4443
|
+
});
|
|
5119
4444
|
// Local ancestry chain that always ENDS with the currently-executing play
|
|
5120
4445
|
// (req.playName). The /api/v2/plays/run lineage validator requires the
|
|
5121
4446
|
// submitted ancestry's tail to equal the executor token's play name (i.e.
|
|
@@ -5150,23 +4475,33 @@ function createMinimalWorkerCtx(
|
|
|
5150
4475
|
): Promise<unknown> => {
|
|
5151
4476
|
const mapStartedAt = nowMs();
|
|
5152
4477
|
const mapNodeId = `map:${name}`;
|
|
5153
|
-
// Map-wide persistence-failure circuit breaker. Shared across every chunk's
|
|
5154
|
-
// scheduler (the scheduler is rebuilt per chunk) and the map's sheet-persist
|
|
5155
|
-
// flush chain, so the first persistence failure halts NEW provider dispatch
|
|
5156
|
-
// for the rest of the map and its prevented-call count surfaces in the
|
|
5157
|
-
// runMap failure message below.
|
|
5158
|
-
const persistenceLatch = createRuntimePersistenceLatch();
|
|
5159
4478
|
const inputRows = rows;
|
|
5160
4479
|
const rowCountHint = datasetRowCountHint(inputRows);
|
|
5161
4480
|
const baseOffset = 0;
|
|
5162
4481
|
const fieldEntries = Object.entries(fieldsDef);
|
|
5163
4482
|
const plan = req.executionPlan;
|
|
5164
|
-
const
|
|
4483
|
+
const mapDispatchPlan = chooseWorkerMapDispatchPlan({
|
|
5165
4484
|
mapName: name,
|
|
5166
4485
|
rowCountHint,
|
|
5167
4486
|
executionPlan: plan,
|
|
5168
4487
|
staticPipeline: staticPipelineFromReq(req),
|
|
5169
4488
|
});
|
|
4489
|
+
const rowsPerChunk = mapDispatchPlan.rowsPerChunk;
|
|
4490
|
+
recordRunnerPerfTrace({
|
|
4491
|
+
req,
|
|
4492
|
+
phase: 'runner.map_dispatch_plan',
|
|
4493
|
+
ms: 0,
|
|
4494
|
+
extra: {
|
|
4495
|
+
mapName: name,
|
|
4496
|
+
rowsPerChunk,
|
|
4497
|
+
rowCountHint,
|
|
4498
|
+
estimatedExternalStepsPerRow:
|
|
4499
|
+
mapDispatchPlan.estimatedExternalStepsPerRow,
|
|
4500
|
+
workEstimate: mapDispatchPlan.workEstimate,
|
|
4501
|
+
reason: mapDispatchPlan.reason,
|
|
4502
|
+
dynamicWorkerArtifactVersion: DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION,
|
|
4503
|
+
},
|
|
4504
|
+
});
|
|
5170
4505
|
const outputFields = fieldEntries.map(([field]) => field);
|
|
5171
4506
|
const updateMapProgress = (
|
|
5172
4507
|
progress: LiveNodeProgressSnapshot,
|
|
@@ -5182,6 +4517,22 @@ function createMinimalWorkerCtx(
|
|
|
5182
4517
|
forceFlush: options?.forceFlush ?? true,
|
|
5183
4518
|
});
|
|
5184
4519
|
};
|
|
4520
|
+
let queuedMapProgress: Promise<void> = Promise.resolve();
|
|
4521
|
+
let queuedMapProgressError: unknown = null;
|
|
4522
|
+
const enqueueMapProgressUpdate = (
|
|
4523
|
+
progress: LiveNodeProgressSnapshot,
|
|
4524
|
+
options?: { forceFlush?: boolean },
|
|
4525
|
+
) => {
|
|
4526
|
+
queuedMapProgress = queuedMapProgress
|
|
4527
|
+
.then(() => Promise.resolve(updateMapProgress(progress, options)))
|
|
4528
|
+
.catch((error) => {
|
|
4529
|
+
queuedMapProgressError ??= error;
|
|
4530
|
+
});
|
|
4531
|
+
};
|
|
4532
|
+
const flushQueuedMapProgressUpdates = async () => {
|
|
4533
|
+
await queuedMapProgress;
|
|
4534
|
+
if (queuedMapProgressError) throw queuedMapProgressError;
|
|
4535
|
+
};
|
|
5185
4536
|
const formatMapProgressMessage = (completed: number, total?: number) =>
|
|
5186
4537
|
typeof total === 'number' && Number.isFinite(total) && total > 0
|
|
5187
4538
|
? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
|
|
@@ -5206,7 +4557,9 @@ function createMinimalWorkerCtx(
|
|
|
5206
4557
|
return formatMapProgressMessage(completed, input.total);
|
|
5207
4558
|
};
|
|
5208
4559
|
const formatMapProcessingMessage = (rowsToExecute: number) =>
|
|
5209
|
-
rowsToExecute > 0
|
|
4560
|
+
rowsToExecute > 0
|
|
4561
|
+
? `Processing ${rowsToExecute.toLocaleString()} rows`
|
|
4562
|
+
: null;
|
|
5210
4563
|
const formatMapExecutionHeartbeatMessage = (input: {
|
|
5211
4564
|
rowsToExecute: number;
|
|
5212
4565
|
startedRows: number;
|
|
@@ -5220,7 +4573,7 @@ function createMinimalWorkerCtx(
|
|
|
5220
4573
|
const waitingRows = Math.max(0, rowsToExecute - startedRows);
|
|
5221
4574
|
const parts = [
|
|
5222
4575
|
activeRows > 0 ? `${activeRows.toLocaleString()} active` : null,
|
|
5223
|
-
waitingRows > 0 ? `${waitingRows.toLocaleString()}
|
|
4576
|
+
waitingRows > 0 ? `${waitingRows.toLocaleString()} waiting` : null,
|
|
5224
4577
|
completedRows > 0 ? `${completedRows.toLocaleString()} done` : null,
|
|
5225
4578
|
].filter((part): part is string => Boolean(part));
|
|
5226
4579
|
const base =
|
|
@@ -5384,6 +4737,12 @@ function createMinimalWorkerCtx(
|
|
|
5384
4737
|
): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
|
|
5385
4738
|
const chunkStartedAt = nowMs();
|
|
5386
4739
|
assertNotAborted(abortSignal);
|
|
4740
|
+
const rowAttempt = makeRuntimeSheetAttempt({
|
|
4741
|
+
runId: req.runId,
|
|
4742
|
+
runAttempt: req.runAttempt,
|
|
4743
|
+
mapName: name,
|
|
4744
|
+
chunkIndex,
|
|
4745
|
+
});
|
|
5387
4746
|
const keyStartedAt = nowMs();
|
|
5388
4747
|
const chunkEntries = chunkRows.map((row, localIndex) => {
|
|
5389
4748
|
const absoluteIndex = baseOffset + chunkStart + localIndex;
|
|
@@ -5411,8 +4770,21 @@ function createMinimalWorkerCtx(
|
|
|
5411
4770
|
...mapRowOutcomeRuntimeFields({ key: rowKey }),
|
|
5412
4771
|
})),
|
|
5413
4772
|
inputOffset: baseOffset + chunkStart,
|
|
4773
|
+
...rowAttempt,
|
|
4774
|
+
budgetMeter: workBudgetMeter,
|
|
5414
4775
|
force: !!req.force,
|
|
5415
4776
|
});
|
|
4777
|
+
const activeRowAttempt = {
|
|
4778
|
+
attemptId: prepared.attemptId ?? rowAttempt.attemptId,
|
|
4779
|
+
attemptOwnerRunId:
|
|
4780
|
+
prepared.attemptOwnerRunId ?? rowAttempt.attemptOwnerRunId,
|
|
4781
|
+
attemptExpiresAt:
|
|
4782
|
+
prepared.attemptExpiresAt ?? rowAttempt.attemptExpiresAt,
|
|
4783
|
+
attemptSeq: prepared.attemptSeq ?? rowAttempt.attemptSeq,
|
|
4784
|
+
};
|
|
4785
|
+
// Record that this run holds attempt leases in `name`'s sheet so a
|
|
4786
|
+
// run-fatal teardown releases them (the table now exists).
|
|
4787
|
+
leasedSheetNamespaces?.add(name);
|
|
5416
4788
|
recordRunnerPerfTrace({
|
|
5417
4789
|
req,
|
|
5418
4790
|
phase: 'runner.map_chunk.prepare_rows',
|
|
@@ -5425,14 +4797,15 @@ function createMinimalWorkerCtx(
|
|
|
5425
4797
|
skipped: prepared.skipped,
|
|
5426
4798
|
pendingRows: prepared.pendingRows.length,
|
|
5427
4799
|
completedRows: prepared.completedRows.length,
|
|
4800
|
+
blockedRows: prepared.blockedRows.length,
|
|
5428
4801
|
},
|
|
5429
4802
|
});
|
|
5430
4803
|
const progressTotalRows = rowCountHint ?? chunkRows.length;
|
|
5431
4804
|
const preparedCompletedRows = Math.min(
|
|
5432
4805
|
progressTotalRows,
|
|
5433
|
-
totalRowsWritten
|
|
4806
|
+
totalRowsWritten,
|
|
5434
4807
|
);
|
|
5435
|
-
|
|
4808
|
+
enqueueMapProgressUpdate({
|
|
5436
4809
|
completed: preparedCompletedRows,
|
|
5437
4810
|
total: progressTotalRows,
|
|
5438
4811
|
startedAt: mapStartedAt,
|
|
@@ -5445,6 +4818,9 @@ function createMinimalWorkerCtx(
|
|
|
5445
4818
|
const pendingKeys = new Set<string>();
|
|
5446
4819
|
const pendingRowsByKey = new Map<string, Record<string, unknown>>();
|
|
5447
4820
|
const completedKeys = new Set<string>();
|
|
4821
|
+
const completedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
4822
|
+
const blockedKeys = new Set<string>();
|
|
4823
|
+
const blockedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
5448
4824
|
const preparedKeys = new Set<string>();
|
|
5449
4825
|
for (const row of prepared.pendingRows) {
|
|
5450
4826
|
const key =
|
|
@@ -5462,14 +4838,95 @@ function createMinimalWorkerCtx(
|
|
|
5462
4838
|
deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
|
|
5463
4839
|
if (key) {
|
|
5464
4840
|
completedKeys.add(key);
|
|
4841
|
+
completedRowsByKey.set(key, row);
|
|
4842
|
+
preparedKeys.add(key);
|
|
4843
|
+
}
|
|
4844
|
+
}
|
|
4845
|
+
for (const row of prepared.blockedRows) {
|
|
4846
|
+
const key =
|
|
4847
|
+
resolveMapRowOutcomeKey(row) ??
|
|
4848
|
+
deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
|
|
4849
|
+
if (key) {
|
|
4850
|
+
blockedKeys.add(key);
|
|
4851
|
+
blockedRowsByKey.set(key, row);
|
|
5465
4852
|
preparedKeys.add(key);
|
|
5466
4853
|
}
|
|
5467
4854
|
}
|
|
4855
|
+
if (blockedKeys.size > 0) {
|
|
4856
|
+
const rowKeySamples = Array.from(blockedKeys).slice(
|
|
4857
|
+
0,
|
|
4858
|
+
MAP_ROW_FAILURE_SAMPLE_LIMIT,
|
|
4859
|
+
);
|
|
4860
|
+
const message =
|
|
4861
|
+
`ctx.dataset("${name}") chunk ${chunkIndex} is blocked by ` +
|
|
4862
|
+
`${blockedKeys.size} active runtime sheet row lease(s) owned by another attempt. ` +
|
|
4863
|
+
`Retry after the owning attempt finishes or the row leases expire.` +
|
|
4864
|
+
(rowKeySamples.length > 0
|
|
4865
|
+
? ` Blocked row keys: ${rowKeySamples.join(', ')}`
|
|
4866
|
+
: '');
|
|
4867
|
+
recordRunnerPerfTrace({
|
|
4868
|
+
req,
|
|
4869
|
+
phase: 'runner.map_chunk.blocked_rows',
|
|
4870
|
+
ms: nowMs() - prepareStartedAt,
|
|
4871
|
+
extra: {
|
|
4872
|
+
mapName: name,
|
|
4873
|
+
chunkIndex,
|
|
4874
|
+
rowsBlocked: blockedKeys.size,
|
|
4875
|
+
rowKeySamples,
|
|
4876
|
+
},
|
|
4877
|
+
});
|
|
4878
|
+
enqueueMapProgressUpdate({
|
|
4879
|
+
completed: preparedCompletedRows,
|
|
4880
|
+
total: progressTotalRows,
|
|
4881
|
+
waitingRows: blockedKeys.size,
|
|
4882
|
+
startedAt: mapStartedAt,
|
|
4883
|
+
message,
|
|
4884
|
+
});
|
|
4885
|
+
return {
|
|
4886
|
+
status: 'blocked',
|
|
4887
|
+
chunkIndex,
|
|
4888
|
+
rangeStart: baseOffset + chunkStart,
|
|
4889
|
+
rangeEnd: baseOffset + chunkStart + chunkRows.length,
|
|
4890
|
+
rowsRead: chunkRows.length,
|
|
4891
|
+
rowsBlocked: blockedKeys.size,
|
|
4892
|
+
rowKeySamples,
|
|
4893
|
+
blockedRowSamples: rowKeySamples.map((rowKey) => {
|
|
4894
|
+
const blockedRow = blockedRowsByKey.get(rowKey);
|
|
4895
|
+
const attemptId =
|
|
4896
|
+
typeof blockedRow?.__deeplineAttemptId === 'string'
|
|
4897
|
+
? blockedRow.__deeplineAttemptId
|
|
4898
|
+
: typeof blockedRow?._attempt_id === 'string'
|
|
4899
|
+
? blockedRow._attempt_id
|
|
4900
|
+
: activeRowAttempt.attemptId;
|
|
4901
|
+
return {
|
|
4902
|
+
__deeplineRowKey: rowKey,
|
|
4903
|
+
__deeplineAttemptId: attemptId,
|
|
4904
|
+
};
|
|
4905
|
+
}),
|
|
4906
|
+
outputDatasetId: `map:${name}`,
|
|
4907
|
+
message,
|
|
4908
|
+
};
|
|
4909
|
+
}
|
|
5468
4910
|
const missingPreparedRows = chunkEntries.filter(
|
|
5469
4911
|
({ rowKey }) => !preparedKeys.has(rowKey),
|
|
5470
4912
|
);
|
|
5471
|
-
|
|
5472
|
-
|
|
4913
|
+
if (missingPreparedRows.length > 0) {
|
|
4914
|
+
const rowKeySamples = missingPreparedRows
|
|
4915
|
+
.map(({ rowKey }) => rowKey)
|
|
4916
|
+
.slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
|
|
4917
|
+
throw new Error(
|
|
4918
|
+
`ctx.dataset("${name}") runtime preparation omitted ${missingPreparedRows.length} row disposition(s). ` +
|
|
4919
|
+
`Every input row must be claimed, completed, or blocked before execution starts.` +
|
|
4920
|
+
(rowKeySamples.length > 0
|
|
4921
|
+
? ` Missing row keys: ${rowKeySamples.join(', ')}`
|
|
4922
|
+
: ''),
|
|
4923
|
+
);
|
|
4924
|
+
}
|
|
4925
|
+
const rowsToExecuteEntries = chunkEntries.filter(({ rowKey }) =>
|
|
4926
|
+
pendingKeys.has(rowKey),
|
|
4927
|
+
);
|
|
4928
|
+
const completedChunkEntries = chunkEntries.filter(({ rowKey }) =>
|
|
4929
|
+
completedKeys.has(rowKey),
|
|
5473
4930
|
);
|
|
5474
4931
|
const uniqueRowsToExecuteEntries = [
|
|
5475
4932
|
...new Map(
|
|
@@ -5486,18 +4943,15 @@ function createMinimalWorkerCtx(
|
|
|
5486
4943
|
rowsToExecute.length,
|
|
5487
4944
|
);
|
|
5488
4945
|
if (processingMessage) {
|
|
5489
|
-
|
|
4946
|
+
enqueueMapProgressUpdate({
|
|
5490
4947
|
completed: preparedCompletedRows,
|
|
5491
4948
|
total: progressTotalRows,
|
|
5492
4949
|
startedAt: mapStartedAt,
|
|
5493
4950
|
message: processingMessage,
|
|
5494
4951
|
});
|
|
5495
4952
|
}
|
|
5496
|
-
const rowsInserted = prepared.inserted
|
|
5497
|
-
const rowsSkipped =
|
|
5498
|
-
0,
|
|
5499
|
-
prepared.skipped - missingPreparedRows.length,
|
|
5500
|
-
);
|
|
4953
|
+
const rowsInserted = prepared.inserted;
|
|
4954
|
+
const rowsSkipped = prepared.skipped;
|
|
5501
4955
|
let completedExecutedRows = 0;
|
|
5502
4956
|
let failedExecutedRows = 0;
|
|
5503
4957
|
let startedExecutedRows = 0;
|
|
@@ -5505,12 +4959,7 @@ function createMinimalWorkerCtx(
|
|
|
5505
4959
|
let lastChunkProgressAt = 0;
|
|
5506
4960
|
let lastExecutionHeartbeatAt = nowMs();
|
|
5507
4961
|
const completedRowsForProgress = () =>
|
|
5508
|
-
Math.min(
|
|
5509
|
-
progressTotalRows,
|
|
5510
|
-
totalRowsWritten +
|
|
5511
|
-
prepared.completedRows.length +
|
|
5512
|
-
completedExecutedRows,
|
|
5513
|
-
);
|
|
4962
|
+
Math.min(progressTotalRows, totalRowsWritten + completedExecutedRows);
|
|
5514
4963
|
const reportExecutionHeartbeat = (force = false) => {
|
|
5515
4964
|
const now = nowMs();
|
|
5516
4965
|
if (
|
|
@@ -5520,7 +4969,7 @@ function createMinimalWorkerCtx(
|
|
|
5520
4969
|
return;
|
|
5521
4970
|
}
|
|
5522
4971
|
lastExecutionHeartbeatAt = now;
|
|
5523
|
-
|
|
4972
|
+
enqueueMapProgressUpdate(
|
|
5524
4973
|
{
|
|
5525
4974
|
completed: completedRowsForProgress(),
|
|
5526
4975
|
total: progressTotalRows,
|
|
@@ -5554,7 +5003,7 @@ function createMinimalWorkerCtx(
|
|
|
5554
5003
|
return;
|
|
5555
5004
|
}
|
|
5556
5005
|
lastChunkProgressAt = now;
|
|
5557
|
-
|
|
5006
|
+
enqueueMapProgressUpdate(
|
|
5558
5007
|
{
|
|
5559
5008
|
completed,
|
|
5560
5009
|
total: progressTotalRows,
|
|
@@ -5585,23 +5034,44 @@ function createMinimalWorkerCtx(
|
|
|
5585
5034
|
const failedRowEntries: Array<
|
|
5586
5035
|
{ row: T & Record<string, unknown>; error: string } | undefined
|
|
5587
5036
|
> = new Array(rowsToExecute.length);
|
|
5037
|
+
const fatalReceiptPersistenceRowEntries: Array<
|
|
5038
|
+
{ row: T & Record<string, unknown>; error: string } | undefined
|
|
5039
|
+
> = new Array(rowsToExecute.length);
|
|
5588
5040
|
const executedCellMetaPatches: Array<
|
|
5589
5041
|
Record<string, WorkerCellMetaPatchEntry> | undefined
|
|
5590
5042
|
> = new Array(rowsToExecute.length);
|
|
5591
|
-
const
|
|
5043
|
+
const persistenceLatch = createRuntimePersistenceLatch();
|
|
5044
|
+
const toolBatchScheduler = new WorkerToolBatchScheduler({
|
|
5592
5045
|
req,
|
|
5593
5046
|
governor,
|
|
5594
|
-
resolveToolPacing,
|
|
5047
|
+
resolvePacing: resolveToolPacing,
|
|
5595
5048
|
resolveToolActionCacheVersion,
|
|
5049
|
+
resolveToolAuthScopeDigest,
|
|
5050
|
+
invalidateToolAuthScopeDigest: (toolId) =>
|
|
5051
|
+
resolveToolAuthScopeDigest.invalidate(toolId),
|
|
5052
|
+
onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
|
|
5053
|
+
executeTool: executeDispatchedTool,
|
|
5054
|
+
serializeDurableValue: serializeDurableStepValue,
|
|
5055
|
+
deserializeDurableValue: deserializeDurableStepValue,
|
|
5056
|
+
nowMs,
|
|
5057
|
+
batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
|
|
5058
|
+
recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
|
|
5596
5059
|
abortSignal,
|
|
5597
|
-
reportSettledToolRequests,
|
|
5060
|
+
onRequestsSettled: reportSettledToolRequests,
|
|
5598
5061
|
callbacks,
|
|
5599
5062
|
receiptStore,
|
|
5600
|
-
false,
|
|
5063
|
+
allowLocalRetryReceipts: false,
|
|
5064
|
+
runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
|
|
5065
|
+
runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
|
|
5066
|
+
req,
|
|
5067
|
+
runtimeReceiptLeaseTtlMs(req) ??
|
|
5068
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
5069
|
+
),
|
|
5070
|
+
budgetMeter: workBudgetMeter,
|
|
5601
5071
|
runtimeDeadlineMs,
|
|
5602
5072
|
persistenceLatch,
|
|
5603
5073
|
receiptSalvage,
|
|
5604
|
-
);
|
|
5074
|
+
});
|
|
5605
5075
|
let stepCellsCompleted = 0;
|
|
5606
5076
|
let stepCellsSkipped = 0;
|
|
5607
5077
|
const generatedOutputFields = new Set<string>();
|
|
@@ -5680,6 +5150,8 @@ function createMinimalWorkerCtx(
|
|
|
5680
5150
|
tableNamespace: name,
|
|
5681
5151
|
outputFields,
|
|
5682
5152
|
extraOutputFields: Array.from(generatedOutputFields),
|
|
5153
|
+
budgetMeter: workBudgetMeter,
|
|
5154
|
+
...activeRowAttempt,
|
|
5683
5155
|
rows: [
|
|
5684
5156
|
...rowsToPersist.map(({ row, executedIndex }) =>
|
|
5685
5157
|
mapRowOutcomeRuntimeRow(
|
|
@@ -5743,9 +5215,6 @@ function createMinimalWorkerCtx(
|
|
|
5743
5215
|
});
|
|
5744
5216
|
persistFlushChain = task.catch((error) => {
|
|
5745
5217
|
persistFailure ??= error;
|
|
5746
|
-
// Output-sheet flush failed: trip the breaker so the scheduler stops
|
|
5747
|
-
// dispatching new provider calls for the rest of this map.
|
|
5748
|
-
tripRuntimePersistenceLatch(persistenceLatch, error);
|
|
5749
5218
|
});
|
|
5750
5219
|
return task;
|
|
5751
5220
|
};
|
|
@@ -5787,6 +5256,8 @@ function createMinimalWorkerCtx(
|
|
|
5787
5256
|
outputFields,
|
|
5788
5257
|
extraOutputFields,
|
|
5789
5258
|
updates,
|
|
5259
|
+
...activeRowAttempt,
|
|
5260
|
+
budgetMeter: workBudgetMeter,
|
|
5790
5261
|
});
|
|
5791
5262
|
} catch (error) {
|
|
5792
5263
|
liveUpdateFailures += 1;
|
|
@@ -5894,18 +5365,11 @@ function createMinimalWorkerCtx(
|
|
|
5894
5365
|
request.input,
|
|
5895
5366
|
workflowStep,
|
|
5896
5367
|
{
|
|
5897
|
-
force: request.force === true
|
|
5368
|
+
force: request.force === true,
|
|
5369
|
+
runForceRefresh: req.forceToolRefresh === true,
|
|
5370
|
+
runForceFailedRefresh: req.force === true,
|
|
5898
5371
|
staleAfterSeconds: request.staleAfterSeconds,
|
|
5899
5372
|
},
|
|
5900
|
-
// Parity with the play-level ctx.tools.execute (and the
|
|
5901
|
-
// cjs runner's map path): the public ToolExecutionRequest
|
|
5902
|
-
// runtime options must not be silently dropped for map
|
|
5903
|
-
// rows — receiptWaitMs bounds how long a resumed run
|
|
5904
|
-
// waits on a stuck `running` receipt before reclaiming.
|
|
5905
|
-
{
|
|
5906
|
-
timeoutMs: request.timeoutMs,
|
|
5907
|
-
receiptWaitMs: request.receiptWaitMs,
|
|
5908
|
-
},
|
|
5909
5373
|
);
|
|
5910
5374
|
},
|
|
5911
5375
|
},
|
|
@@ -6014,8 +5478,28 @@ function createMinimalWorkerCtx(
|
|
|
6014
5478
|
notePersistableRow(enriched);
|
|
6015
5479
|
reportChunkProgress(false);
|
|
6016
5480
|
} catch (rowError) {
|
|
6017
|
-
//
|
|
6018
|
-
//
|
|
5481
|
+
// Receipt persistence errors stay run-fatal, but keep the
|
|
5482
|
+
// row identity so teardown can make the recovery state loud.
|
|
5483
|
+
if (isRuntimeReceiptPersistenceError(rowError)) {
|
|
5484
|
+
const message = formatWorkerRowFailureMessage(rowError);
|
|
5485
|
+
if (activeField) {
|
|
5486
|
+
cellMetaPatch[activeField] = {
|
|
5487
|
+
status: 'failed',
|
|
5488
|
+
stage: activeField,
|
|
5489
|
+
runId: req.runId,
|
|
5490
|
+
error: message,
|
|
5491
|
+
};
|
|
5492
|
+
}
|
|
5493
|
+
executedCellMetaPatches[myIndex] =
|
|
5494
|
+
Object.keys(cellMetaPatch).length > 0
|
|
5495
|
+
? cellMetaPatch
|
|
5496
|
+
: undefined;
|
|
5497
|
+
fatalReceiptPersistenceRowEntries[myIndex] = {
|
|
5498
|
+
row: enriched as T & Record<string, unknown>,
|
|
5499
|
+
error: message,
|
|
5500
|
+
};
|
|
5501
|
+
throw rowError;
|
|
5502
|
+
}
|
|
6019
5503
|
if (isRunFatalWorkerRowError(rowError)) {
|
|
6020
5504
|
throw rowError;
|
|
6021
5505
|
}
|
|
@@ -6095,16 +5579,25 @@ function createMinimalWorkerCtx(
|
|
|
6095
5579
|
return results;
|
|
6096
5580
|
},
|
|
6097
5581
|
);
|
|
6098
|
-
|
|
6099
|
-
|
|
6100
|
-
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
|
|
6107
|
-
|
|
5582
|
+
const workerResults = await executeWithRuntimeSheetAttemptHeartbeat({
|
|
5583
|
+
req,
|
|
5584
|
+
tableNamespace: name,
|
|
5585
|
+
sheetContract: requireSheetContract(req, name),
|
|
5586
|
+
...activeRowAttempt,
|
|
5587
|
+
rowKeys: uniqueRowsToExecuteEntries.map((entry) => entry.rowKey),
|
|
5588
|
+
execute: async () => {
|
|
5589
|
+
while (!workerSettled) {
|
|
5590
|
+
await Promise.race([
|
|
5591
|
+
workerResultsPromise,
|
|
5592
|
+
sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
|
|
5593
|
+
]);
|
|
5594
|
+
if (!workerSettled) {
|
|
5595
|
+
reportExecutionHeartbeat(false);
|
|
5596
|
+
}
|
|
5597
|
+
}
|
|
5598
|
+
return await workerResultsPromise;
|
|
5599
|
+
},
|
|
5600
|
+
});
|
|
6108
5601
|
recordRunnerPerfTrace({
|
|
6109
5602
|
req,
|
|
6110
5603
|
phase: 'runner.map_chunk.execute_workers',
|
|
@@ -6133,26 +5626,89 @@ function createMinimalWorkerCtx(
|
|
|
6133
5626
|
.slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
|
|
6134
5627
|
const fatalMapChunkSummary = async (
|
|
6135
5628
|
error: unknown,
|
|
6136
|
-
): Promise<
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
|
|
6147
|
-
|
|
6148
|
-
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
5629
|
+
): Promise<
|
|
5630
|
+
WorkerMapCompletedChunkSummary<T & Record<string, unknown>>
|
|
5631
|
+
> => {
|
|
5632
|
+
const fatalError = formatWorkerRowFailureMessage(error);
|
|
5633
|
+
const breakerMessage = persistenceLatch.tripped
|
|
5634
|
+
? ` Circuit breaker prevented ${persistenceLatch.preventedCallCount} queued provider call(s).`
|
|
5635
|
+
: '';
|
|
5636
|
+
return {
|
|
5637
|
+
status: 'completed',
|
|
5638
|
+
chunkIndex,
|
|
5639
|
+
rangeStart: baseOffset + chunkStart,
|
|
5640
|
+
rangeEnd: baseOffset + chunkStart,
|
|
5641
|
+
rowsRead: chunkRows.length,
|
|
5642
|
+
rowsWritten: 0,
|
|
5643
|
+
rowsExecuted: rowsToExecute.length,
|
|
5644
|
+
rowsCached: 0,
|
|
5645
|
+
rowsDuplicateReused: duplicateInputReuseCount,
|
|
5646
|
+
rowsInserted,
|
|
5647
|
+
rowsSkipped,
|
|
5648
|
+
rowsFailed: failedExecutedRows,
|
|
5649
|
+
rowFailureSamples: buildRowFailureSamples(),
|
|
5650
|
+
stepCellsCompleted,
|
|
5651
|
+
stepCellsSkipped,
|
|
5652
|
+
outputDatasetId: `map:${name}`,
|
|
5653
|
+
hash: await hashJson([]),
|
|
5654
|
+
fatalError: `${fatalError}${breakerMessage}`,
|
|
5655
|
+
preview: [],
|
|
5656
|
+
};
|
|
5657
|
+
};
|
|
5658
|
+
const markReceiptPersistenceFatalRowsFailed = async (error: unknown) => {
|
|
5659
|
+
const message = formatWorkerRowFailureMessage(error);
|
|
5660
|
+
const rows = uniqueRowsToExecuteEntries
|
|
5661
|
+
.map((entry, executedIndex) => {
|
|
5662
|
+
const failure = fatalReceiptPersistenceRowEntries[executedIndex];
|
|
5663
|
+
const row =
|
|
5664
|
+
failure?.row ??
|
|
5665
|
+
executedRows[executedIndex] ??
|
|
5666
|
+
(runtimeCsvExecutionRow(
|
|
5667
|
+
entry.row,
|
|
5668
|
+
pendingRowsByKey.get(entry.rowKey),
|
|
5669
|
+
) as T & Record<string, unknown>);
|
|
5670
|
+
return mapRowOutcomeRuntimeRow(
|
|
5671
|
+
failedMapRowOutcome({
|
|
5672
|
+
key: entry.rowKey,
|
|
5673
|
+
inputIndex: entry.absoluteIndex,
|
|
5674
|
+
data: row,
|
|
5675
|
+
cellMetaPatch: executedCellMetaPatches[executedIndex],
|
|
5676
|
+
error: failure?.error ?? message,
|
|
5677
|
+
}),
|
|
5678
|
+
);
|
|
5679
|
+
})
|
|
5680
|
+
.filter((row): row is Record<string, unknown> => row !== null);
|
|
5681
|
+
if (rows.length === 0) return;
|
|
5682
|
+
try {
|
|
5683
|
+
const failureStats = await persistCompletedMapRows({
|
|
5684
|
+
req,
|
|
5685
|
+
tableNamespace: name,
|
|
5686
|
+
outputFields,
|
|
5687
|
+
extraOutputFields: Array.from(generatedOutputFields),
|
|
5688
|
+
budgetMeter: workBudgetMeter,
|
|
5689
|
+
...activeRowAttempt,
|
|
5690
|
+
forceFailedRows: true,
|
|
5691
|
+
rows,
|
|
5692
|
+
});
|
|
5693
|
+
emitEvent({
|
|
5694
|
+
type: 'log',
|
|
5695
|
+
level: 'warn',
|
|
5696
|
+
message:
|
|
5697
|
+
`Runtime sheet marked ${failureStats.rows} row(s) failed for ctx.dataset("${name}") ` +
|
|
5698
|
+
`after a run-fatal receipt persistence error.`,
|
|
5699
|
+
ts: nowMs(),
|
|
5700
|
+
});
|
|
5701
|
+
} catch (cleanupError) {
|
|
5702
|
+
emitEvent({
|
|
5703
|
+
type: 'log',
|
|
5704
|
+
level: 'warn',
|
|
5705
|
+
message:
|
|
5706
|
+
`Runtime sheet failed-row cleanup after receipt persistence error failed for ctx.dataset("${name}"): ` +
|
|
5707
|
+
formatWorkerRowFailureMessage(cleanupError),
|
|
5708
|
+
ts: nowMs(),
|
|
5709
|
+
});
|
|
5710
|
+
}
|
|
5711
|
+
};
|
|
6156
5712
|
const persistRowsStartedAt = nowMs();
|
|
6157
5713
|
recordRunnerPerfTrace({
|
|
6158
5714
|
req,
|
|
@@ -6181,6 +5737,7 @@ function createMinimalWorkerCtx(
|
|
|
6181
5737
|
},
|
|
6182
5738
|
});
|
|
6183
5739
|
} catch (error) {
|
|
5740
|
+
tripRuntimePersistenceLatch(persistenceLatch, error);
|
|
6184
5741
|
recordRunnerPerfTrace({
|
|
6185
5742
|
req,
|
|
6186
5743
|
phase: 'runner.map_chunk.persist_rows_error',
|
|
@@ -6199,31 +5756,24 @@ function createMinimalWorkerCtx(
|
|
|
6199
5756
|
result.status === 'rejected',
|
|
6200
5757
|
);
|
|
6201
5758
|
if (rejectedWorker) {
|
|
6202
|
-
if (
|
|
6203
|
-
|
|
6204
|
-
|
|
6205
|
-
|
|
6206
|
-
|
|
6207
|
-
// fatal summary (NOT rethrown into the durable step) so the chunk is
|
|
6208
|
-
// not retried and provider calls are not re-billed.
|
|
5759
|
+
if (isRuntimeReceiptPersistenceError(rejectedWorker.reason)) {
|
|
5760
|
+
await markReceiptPersistenceFatalRowsFailed(rejectedWorker.reason);
|
|
5761
|
+
return await fatalMapChunkSummary(rejectedWorker.reason);
|
|
5762
|
+
}
|
|
5763
|
+
if (isRuntimePersistenceCircuitOpenError(rejectedWorker.reason)) {
|
|
6209
5764
|
return await fatalMapChunkSummary(rejectedWorker.reason);
|
|
6210
5765
|
}
|
|
6211
5766
|
throw rejectedWorker.reason;
|
|
6212
5767
|
}
|
|
6213
5768
|
const resultByKey = new Map<string, T & Record<string, unknown>>();
|
|
6214
|
-
for (const
|
|
6215
|
-
const
|
|
6216
|
-
|
|
6217
|
-
|
|
6218
|
-
|
|
6219
|
-
|
|
6220
|
-
|
|
6221
|
-
|
|
6222
|
-
const cleanedRow = stripMapRowOutcomeRuntimeFields(
|
|
6223
|
-
publicCsvOutputRow(completedRow),
|
|
6224
|
-
);
|
|
6225
|
-
resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
|
|
6226
|
-
}
|
|
5769
|
+
for (const entry of completedChunkEntries) {
|
|
5770
|
+
const completedRow = completedRowsByKey.get(entry.rowKey);
|
|
5771
|
+
if (!completedRow || resultByKey.has(entry.rowKey)) continue;
|
|
5772
|
+
resultByKey.set(
|
|
5773
|
+
entry.rowKey,
|
|
5774
|
+
cloneCsvAliasedRow(entry.row, completedRow) as T &
|
|
5775
|
+
Record<string, unknown>,
|
|
5776
|
+
);
|
|
6227
5777
|
}
|
|
6228
5778
|
for (
|
|
6229
5779
|
let executedIndex = 0;
|
|
@@ -6258,16 +5808,13 @@ function createMinimalWorkerCtx(
|
|
|
6258
5808
|
} => entry !== null,
|
|
6259
5809
|
);
|
|
6260
5810
|
const out = outEntries.map((entry) => entry.row);
|
|
6261
|
-
const executedSuccessCount = Math.max(
|
|
6262
|
-
0,
|
|
6263
|
-
executedRows.length - failedExecutedRows,
|
|
6264
|
-
);
|
|
6265
5811
|
const rowFailureSamples = buildRowFailureSamples();
|
|
6266
5812
|
const publicOut = out.map((row) => publicCsvOutputRow(row));
|
|
6267
5813
|
const keyedOut = outEntries.map(({ key, inputIndex, row }) => ({
|
|
6268
5814
|
...row,
|
|
6269
5815
|
...mapRowOutcomeRuntimeFields({ key, inputIndex }),
|
|
6270
5816
|
}));
|
|
5817
|
+
const completedRowsReused = completedChunkEntries.length;
|
|
6271
5818
|
const hashStartedAt = nowMs();
|
|
6272
5819
|
const hash = await hashJson(publicOut);
|
|
6273
5820
|
const includeCachedRowsInChunkResult = !workflowStep;
|
|
@@ -6298,17 +5845,18 @@ function createMinimalWorkerCtx(
|
|
|
6298
5845
|
rowsWritten: out.length,
|
|
6299
5846
|
rowsExecuted: executedRows.length,
|
|
6300
5847
|
rowsFailed: failedExecutedRows,
|
|
6301
|
-
rowsCached:
|
|
5848
|
+
rowsCached: completedRowsReused,
|
|
6302
5849
|
},
|
|
6303
5850
|
});
|
|
6304
5851
|
return {
|
|
5852
|
+
status: 'completed',
|
|
6305
5853
|
chunkIndex,
|
|
6306
5854
|
rangeStart: baseOffset + chunkStart,
|
|
6307
5855
|
rangeEnd: baseOffset + chunkStart + out.length,
|
|
6308
5856
|
rowsRead: chunkRows.length,
|
|
6309
5857
|
rowsWritten: out.length,
|
|
6310
5858
|
rowsExecuted: executedRows.length,
|
|
6311
|
-
rowsCached:
|
|
5859
|
+
rowsCached: completedRowsReused,
|
|
6312
5860
|
rowsDuplicateReused: duplicateInputReuseCount,
|
|
6313
5861
|
rowsInserted,
|
|
6314
5862
|
rowsSkipped,
|
|
@@ -6340,9 +5888,12 @@ function createMinimalWorkerCtx(
|
|
|
6340
5888
|
let totalRowsInserted = 0;
|
|
6341
5889
|
let totalRowsSkipped = 0;
|
|
6342
5890
|
let totalRowsFailed = 0;
|
|
5891
|
+
let totalRowsBlocked = 0;
|
|
6343
5892
|
let totalStepCellsCompleted = 0;
|
|
6344
5893
|
let totalStepCellsSkipped = 0;
|
|
6345
5894
|
const totalRowFailureSamples: Array<{ rowKey: string; error: string }> = [];
|
|
5895
|
+
const blockedRowKeySamples: string[] = [];
|
|
5896
|
+
const blockedRowSamples: Record<string, unknown>[] = [];
|
|
6346
5897
|
|
|
6347
5898
|
const runChunkStep = async (
|
|
6348
5899
|
chunkRows: T[],
|
|
@@ -6372,6 +5923,7 @@ function createMinimalWorkerCtx(
|
|
|
6372
5923
|
const readPersistedRows = async (input: {
|
|
6373
5924
|
limit: number;
|
|
6374
5925
|
offset: number;
|
|
5926
|
+
rowMode?: 'output' | 'all';
|
|
6375
5927
|
}) => {
|
|
6376
5928
|
const result = await harnessReadSheetDatasetRows({
|
|
6377
5929
|
baseUrl: req.baseUrl,
|
|
@@ -6380,6 +5932,7 @@ function createMinimalWorkerCtx(
|
|
|
6380
5932
|
playName: req.playName,
|
|
6381
5933
|
tableNamespace: name,
|
|
6382
5934
|
runId: req.runId,
|
|
5935
|
+
rowMode: input.rowMode,
|
|
6383
5936
|
limit: input.limit,
|
|
6384
5937
|
offset: input.offset,
|
|
6385
5938
|
userEmail: req.userEmail,
|
|
@@ -6388,19 +5941,61 @@ function createMinimalWorkerCtx(
|
|
|
6388
5941
|
return result.rows as Array<T & Record<string, unknown>>;
|
|
6389
5942
|
};
|
|
6390
5943
|
|
|
6391
|
-
const finalize = async (
|
|
5944
|
+
const finalize = async () => {
|
|
5945
|
+
let finalizedRowsWritten = totalRowsWritten;
|
|
5946
|
+
if (
|
|
5947
|
+
totalRowsFailed === 0 &&
|
|
5948
|
+
rowCountHint !== null &&
|
|
5949
|
+
finalizedRowsWritten > 0 &&
|
|
5950
|
+
finalizedRowsWritten < rowCountHint &&
|
|
5951
|
+
rowCountHint <= WORKER_DATASET_IN_MEMORY_ROWS
|
|
5952
|
+
) {
|
|
5953
|
+
const persistedRows = await readPersistedRows({
|
|
5954
|
+
limit: rowCountHint,
|
|
5955
|
+
offset: 0,
|
|
5956
|
+
rowMode: 'all',
|
|
5957
|
+
});
|
|
5958
|
+
if (persistedRows.length > finalizedRowsWritten) {
|
|
5959
|
+
const inMemoryRowsWritten = finalizedRowsWritten;
|
|
5960
|
+
finalizedRowsWritten = persistedRows.length;
|
|
5961
|
+
totalRowsWritten = finalizedRowsWritten;
|
|
5962
|
+
cachedRows.length = 0;
|
|
5963
|
+
cachedRows.push(...persistedRows);
|
|
5964
|
+
canCacheRows = true;
|
|
5965
|
+
previewRows.length = 0;
|
|
5966
|
+
previewRows.push(
|
|
5967
|
+
...persistedRows.slice(0, WORKER_DATASET_PREVIEW_ROWS),
|
|
5968
|
+
);
|
|
5969
|
+
emitEvent({
|
|
5970
|
+
type: 'log',
|
|
5971
|
+
level: 'warn',
|
|
5972
|
+
message:
|
|
5973
|
+
`Runtime sheet finalization reconciled ctx.dataset("${name}") ` +
|
|
5974
|
+
`from ${inMemoryRowsWritten}/${rowCountHint} in-memory row(s) ` +
|
|
5975
|
+
`to ${finalizedRowsWritten} visible persisted row(s).`,
|
|
5976
|
+
ts: nowMs(),
|
|
5977
|
+
});
|
|
5978
|
+
}
|
|
5979
|
+
if (persistedRows.length < rowCountHint) {
|
|
5980
|
+
throw new Error(
|
|
5981
|
+
`Runtime sheet finalization mismatch ctx.dataset("${name}"): ` +
|
|
5982
|
+
`expected ${rowCountHint}, saw ${persistedRows.length}; ` +
|
|
5983
|
+
`sum ${finalizedRowsWritten}; ${req.runId}`,
|
|
5984
|
+
);
|
|
5985
|
+
}
|
|
5986
|
+
}
|
|
6392
5987
|
const failureSampleSummary =
|
|
6393
5988
|
totalRowFailureSamples.length > 0
|
|
6394
5989
|
? ` First error: ${totalRowFailureSamples[0]!.error}`
|
|
6395
5990
|
: '';
|
|
6396
5991
|
const cacheSummary =
|
|
6397
5992
|
totalRowsFailed > 0
|
|
6398
|
-
? `Map completed with partial failures: ${
|
|
6399
|
-
`${totalRowsFailed} failed (${totalRowsExecuted} executed
|
|
5993
|
+
? `Map completed with partial failures: ${finalizedRowsWritten} succeeded, ` +
|
|
5994
|
+
`${totalRowsFailed} failed (${totalRowsExecuted} executed) ` +
|
|
6400
5995
|
`inserted=${totalRowsInserted} skipped=${totalRowsSkipped}. ` +
|
|
6401
5996
|
`Failed rows are persisted with their errors and re-execute on the next run.${failureSampleSummary}`
|
|
6402
|
-
: `Map completed: ${
|
|
6403
|
-
`(${totalRowsExecuted} executed
|
|
5997
|
+
: `Map completed: ${finalizedRowsWritten} results ` +
|
|
5998
|
+
`(${totalRowsExecuted} executed) ` +
|
|
6404
5999
|
`inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
|
|
6405
6000
|
const completedAt = nowMs();
|
|
6406
6001
|
const totalStepCells = totalStepCellsCompleted + totalStepCellsSkipped;
|
|
@@ -6408,17 +6003,21 @@ function createMinimalWorkerCtx(
|
|
|
6408
6003
|
totalStepCells > 0
|
|
6409
6004
|
? ` cells=${totalStepCells}/${totalStepCellsCompleted}/${totalStepCellsSkipped}`
|
|
6410
6005
|
: '';
|
|
6006
|
+
await flushQueuedMapProgressUpdates();
|
|
6411
6007
|
callbacks?.onMapCompleted?.(mapNodeId, completedAt);
|
|
6412
6008
|
await updateMapProgress({
|
|
6413
|
-
completed:
|
|
6414
|
-
total:
|
|
6009
|
+
completed: finalizedRowsWritten,
|
|
6010
|
+
total: finalizedRowsWritten + totalRowsFailed,
|
|
6415
6011
|
failed: totalRowsFailed,
|
|
6416
6012
|
completedAt,
|
|
6417
6013
|
updatedAt: completedAt,
|
|
6418
6014
|
message:
|
|
6419
6015
|
totalRowsFailed > 0
|
|
6420
|
-
? `${
|
|
6421
|
-
: formatMapProgressMessage(
|
|
6016
|
+
? `${finalizedRowsWritten.toLocaleString()} succeeded, ${totalRowsFailed.toLocaleString()} failed`
|
|
6017
|
+
: formatMapProgressMessage(
|
|
6018
|
+
finalizedRowsWritten,
|
|
6019
|
+
finalizedRowsWritten,
|
|
6020
|
+
),
|
|
6422
6021
|
});
|
|
6423
6022
|
emitEvent({
|
|
6424
6023
|
type: 'log',
|
|
@@ -6427,29 +6026,61 @@ function createMinimalWorkerCtx(
|
|
|
6427
6026
|
ts: nowMs(),
|
|
6428
6027
|
});
|
|
6429
6028
|
if (
|
|
6430
|
-
|
|
6029
|
+
finalizedRowsWritten > 0 &&
|
|
6431
6030
|
totalRowsFailed === 0 &&
|
|
6432
6031
|
canCacheRows &&
|
|
6433
|
-
cachedRows.length ===
|
|
6032
|
+
cachedRows.length === finalizedRowsWritten
|
|
6434
6033
|
) {
|
|
6435
6034
|
const persistedRows = await readPersistedRows({
|
|
6436
|
-
limit:
|
|
6035
|
+
limit: finalizedRowsWritten,
|
|
6437
6036
|
offset: 0,
|
|
6438
6037
|
});
|
|
6439
|
-
if (persistedRows.length <
|
|
6038
|
+
if (persistedRows.length < finalizedRowsWritten) {
|
|
6039
|
+
const repairAttempt = makeRuntimeSheetAttempt({
|
|
6040
|
+
runId: req.runId,
|
|
6041
|
+
runAttempt: req.runAttempt,
|
|
6042
|
+
mapName: name,
|
|
6043
|
+
chunkIndex: 'finalize',
|
|
6044
|
+
});
|
|
6045
|
+
const repairRows = cachedRows.map((row) => runtimeCsvStorageRow(row));
|
|
6046
|
+
workBudgetMeter.count('sheet');
|
|
6047
|
+
const repairStart = await harnessStartSheetDataset({
|
|
6048
|
+
...runtimeSheetSessionScope(req),
|
|
6049
|
+
tableNamespace: name,
|
|
6050
|
+
sheetContract: augmentSheetContractWithDatasetFields({
|
|
6051
|
+
contract: requireSheetContract(req, name),
|
|
6052
|
+
rows: repairRows,
|
|
6053
|
+
outputFields,
|
|
6054
|
+
}),
|
|
6055
|
+
rows: repairRows,
|
|
6056
|
+
runId: req.runId,
|
|
6057
|
+
inputOffset: 0,
|
|
6058
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
6059
|
+
...repairAttempt,
|
|
6060
|
+
});
|
|
6061
|
+
const activeRepairAttempt = {
|
|
6062
|
+
attemptId: repairStart.attemptId ?? repairAttempt.attemptId,
|
|
6063
|
+
attemptOwnerRunId:
|
|
6064
|
+
repairStart.attemptOwnerRunId ?? repairAttempt.attemptOwnerRunId,
|
|
6065
|
+
attemptExpiresAt:
|
|
6066
|
+
repairStart.attemptExpiresAt ?? repairAttempt.attemptExpiresAt,
|
|
6067
|
+
attemptSeq: repairStart.attemptSeq ?? repairAttempt.attemptSeq,
|
|
6068
|
+
};
|
|
6440
6069
|
const repair = await persistCompletedMapRows({
|
|
6441
6070
|
req,
|
|
6442
6071
|
tableNamespace: name,
|
|
6443
6072
|
outputFields,
|
|
6444
6073
|
extraOutputFields: [],
|
|
6445
6074
|
rows: cachedRows,
|
|
6075
|
+
...activeRepairAttempt,
|
|
6076
|
+
budgetMeter: workBudgetMeter,
|
|
6446
6077
|
});
|
|
6447
6078
|
emitEvent({
|
|
6448
6079
|
type: 'log',
|
|
6449
6080
|
level: 'warn',
|
|
6450
6081
|
message:
|
|
6451
6082
|
`Runtime sheet finalization repaired ctx.dataset("${name}") ` +
|
|
6452
|
-
`from ${persistedRows.length}/${
|
|
6083
|
+
`from ${persistedRows.length}/${finalizedRowsWritten} visible row(s) ` +
|
|
6453
6084
|
`(written=${repair.written}, visible=${repair.visible}` +
|
|
6454
6085
|
(repair.retryWritten === null
|
|
6455
6086
|
? ''
|
|
@@ -6465,7 +6096,7 @@ function createMinimalWorkerCtx(
|
|
|
6465
6096
|
return createPersistedDatasetHandle({
|
|
6466
6097
|
playName: req.playName,
|
|
6467
6098
|
name,
|
|
6468
|
-
count:
|
|
6099
|
+
count: finalizedRowsWritten,
|
|
6469
6100
|
// In native Workflows, chunk summaries intentionally omit full row
|
|
6470
6101
|
// payloads, so this preview only contains the bounded chunk samples.
|
|
6471
6102
|
// Do not synchronously page rows back here: service-binding reads have
|
|
@@ -6479,7 +6110,7 @@ function createMinimalWorkerCtx(
|
|
|
6479
6110
|
recordRunnerPerfTrace({ req, phase, ms, extra }),
|
|
6480
6111
|
nowMs,
|
|
6481
6112
|
workProgress: {
|
|
6482
|
-
total:
|
|
6113
|
+
total: finalizedRowsWritten + totalRowsFailed,
|
|
6483
6114
|
executed: totalRowsExecuted,
|
|
6484
6115
|
reused: totalRowsCached,
|
|
6485
6116
|
skipped: totalRowsCached,
|
|
@@ -6493,97 +6124,248 @@ function createMinimalWorkerCtx(
|
|
|
6493
6124
|
};
|
|
6494
6125
|
|
|
6495
6126
|
const failFastRowErrors = opts?.onRowError === 'fail';
|
|
6496
|
-
|
|
6497
|
-
|
|
6498
|
-
|
|
6499
|
-
|
|
6500
|
-
|
|
6501
|
-
|
|
6502
|
-
|
|
6503
|
-
|
|
6504
|
-
|
|
6505
|
-
|
|
6506
|
-
|
|
6507
|
-
|
|
6508
|
-
|
|
6509
|
-
|
|
6510
|
-
|
|
6511
|
-
|
|
6512
|
-
|
|
6513
|
-
|
|
6514
|
-
|
|
6515
|
-
|
|
6516
|
-
|
|
6517
|
-
|
|
6518
|
-
|
|
6519
|
-
|
|
6520
|
-
|
|
6521
|
-
|
|
6522
|
-
|
|
6523
|
-
|
|
6524
|
-
|
|
6525
|
-
`
|
|
6526
|
-
|
|
6527
|
-
);
|
|
6528
|
-
|
|
6529
|
-
|
|
6530
|
-
|
|
6531
|
-
|
|
6532
|
-
|
|
6533
|
-
|
|
6534
|
-
|
|
6535
|
-
|
|
6536
|
-
|
|
6537
|
-
|
|
6538
|
-
|
|
6539
|
-
|
|
6540
|
-
|
|
6127
|
+
// Bounded-concurrent chunk dispatch is only sound off the native Workflows
|
|
6128
|
+
// path: with a `workflowStep` each chunk is a durable step whose sequential
|
|
6129
|
+
// order makes replay/resume deterministic, so that path stays at K=1.
|
|
6130
|
+
// Chunk-level concurrency is otherwise safe under workers_edge because each
|
|
6131
|
+
// chunk owns a disjoint sheet row range with its own runtime-sheet attempt,
|
|
6132
|
+
// receipts complete inside each chunk's persist barrier, and child submits
|
|
6133
|
+
// use per-row durable receipt keys. It is disabled under onRowError:'fail'
|
|
6134
|
+
// on purpose: that mode relies on a first-failure short-circuit to skip the
|
|
6135
|
+
// remaining chunks and stop spending provider credits, and eager concurrent
|
|
6136
|
+
// launch would already have K-1 chunks in flight (credits spent) before the
|
|
6137
|
+
// short-circuit fires.
|
|
6138
|
+
const canRunChunksConcurrently = !workflowStep && !failFastRowErrors;
|
|
6139
|
+
const maxConcurrentMapChunks = canRunChunksConcurrently
|
|
6140
|
+
? WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS
|
|
6141
|
+
: 1;
|
|
6142
|
+
const runWorkDispatcher = new WorkerRunWorkDispatcher({
|
|
6143
|
+
budgetMeter: workBudgetMeter,
|
|
6144
|
+
nowMs,
|
|
6145
|
+
yieldLimits: {
|
|
6146
|
+
elapsed: governor.policy.pacing.workerYieldElapsedMs,
|
|
6147
|
+
...(req.testPolicyOverrides?.workBudgetYieldLimits ?? {}),
|
|
6148
|
+
},
|
|
6149
|
+
recordTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
|
|
6150
|
+
yieldBetweenChunks: async (yieldPoint) => {
|
|
6151
|
+
emitEvent({
|
|
6152
|
+
type: 'log',
|
|
6153
|
+
level: 'info',
|
|
6154
|
+
message:
|
|
6155
|
+
`Runtime dispatcher yielded ctx.dataset("${name}") before chunk ${yieldPoint.nextChunkIndex} ` +
|
|
6156
|
+
`(${yieldPoint.reason} budget).`,
|
|
6157
|
+
ts: nowMs(),
|
|
6158
|
+
});
|
|
6159
|
+
if (workflowStep) {
|
|
6160
|
+
// Cloudflare Workflows durability model: `step.sleep` writes a
|
|
6161
|
+
// durable checkpoint, and the resume re-enters `run()` as a NEW
|
|
6162
|
+
// Worker invocation. Cloudflare's subrequest ceiling is
|
|
6163
|
+
// per-invocation, and on resume the already-completed `step.do`
|
|
6164
|
+
// chunks are served from the durable step cache WITHOUT re-issuing
|
|
6165
|
+
// their subrequests (replay semantics) — so the post-yield chunk
|
|
6166
|
+
// runs against a fresh per-invocation subrequest budget.
|
|
6167
|
+
//
|
|
6168
|
+
// Isolate identity is the WRONG freshness proxy: Cloudflare routinely
|
|
6169
|
+
// reuses warm isolates across invocations, so a genuinely
|
|
6170
|
+
// fresh-budget resume commonly shares the pre-sleep isolate id (this
|
|
6171
|
+
// is exactly why the isolate-identity check reported `false` every
|
|
6172
|
+
// time and hard-failed multi-chunk maps). Crossing the durable
|
|
6173
|
+
// `step.sleep` boundary IS the fresh-budget signal.
|
|
6174
|
+
const sleepMs = WORK_DISPATCHER_YIELD_SLEEP_MS;
|
|
6175
|
+
await (
|
|
6176
|
+
workflowStep.sleep as unknown as (
|
|
6177
|
+
name: string,
|
|
6178
|
+
duration: number,
|
|
6179
|
+
) => Promise<void>
|
|
6180
|
+
)(
|
|
6181
|
+
`work-dispatcher:${name}:${yieldPoint.nextChunkIndex}:${yieldPoint.attempt}`,
|
|
6182
|
+
sleepMs,
|
|
6183
|
+
);
|
|
6184
|
+
recordRunnerPerfTrace({
|
|
6185
|
+
req,
|
|
6186
|
+
phase: 'runner.work_dispatcher.yield_probe',
|
|
6187
|
+
ms: sleepMs,
|
|
6188
|
+
extra: {
|
|
6189
|
+
mapName: name,
|
|
6190
|
+
nextChunkIndex: yieldPoint.nextChunkIndex,
|
|
6191
|
+
reason: yieldPoint.reason,
|
|
6192
|
+
attempt: yieldPoint.attempt,
|
|
6193
|
+
freshBudget: true,
|
|
6194
|
+
invocationBoundary: 'workflow_step_sleep',
|
|
6195
|
+
},
|
|
6196
|
+
});
|
|
6197
|
+
return { freshBudget: true };
|
|
6541
6198
|
}
|
|
6542
|
-
|
|
6543
|
-
|
|
6544
|
-
|
|
6545
|
-
|
|
6546
|
-
|
|
6547
|
-
|
|
6548
|
-
|
|
6549
|
-
|
|
6550
|
-
|
|
6551
|
-
|
|
6552
|
-
|
|
6553
|
-
|
|
6554
|
-
|
|
6555
|
-
|
|
6199
|
+
// No durable Workflow step is available to force a new invocation here,
|
|
6200
|
+
// so we cannot obtain a fresh per-invocation platform budget. Report a
|
|
6201
|
+
// non-fresh yield: the dispatcher escalates and then fails loudly with
|
|
6202
|
+
// WorkerBudgetExhaustedError rather than silently overflowing the
|
|
6203
|
+
// subrequest ceiling mid-map.
|
|
6204
|
+
await sleepWorkerMs(0);
|
|
6205
|
+
return { freshBudget: false };
|
|
6206
|
+
},
|
|
6207
|
+
});
|
|
6208
|
+
let dispatchResult: WorkerRunMapBatchesResult;
|
|
6209
|
+
try {
|
|
6210
|
+
dispatchResult = await runWorkDispatcher.runMapBatches({
|
|
6211
|
+
mapName: name,
|
|
6212
|
+
chunks: iterDatasetChunks(inputRows, rowsPerChunk),
|
|
6213
|
+
maxConcurrentChunks: maxConcurrentMapChunks,
|
|
6214
|
+
maxResidentBatchBytes: WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES,
|
|
6215
|
+
prepareBatch: (rawChunkRows, chunkStart) => {
|
|
6216
|
+
assertNotAborted(abortSignal);
|
|
6217
|
+
if (rawChunkRows.length === 0) return [];
|
|
6218
|
+
// Drop duplicate explicit-key rows before anything downstream observes
|
|
6219
|
+
// them. `chunkStart` keeps advancing by the original (pre-dedupe) chunk
|
|
6220
|
+
// length so cross-chunk key indices and persisted input offsets stay
|
|
6221
|
+
// aligned to the original input stream.
|
|
6222
|
+
return dedupeExplicitRowKeys(rawChunkRows, chunkStart);
|
|
6223
|
+
},
|
|
6224
|
+
estimateRawBatchResidentBytes: (rawChunkRows) =>
|
|
6225
|
+
rawChunkRows.reduce(
|
|
6226
|
+
(total, row) => total + workerMapResidentRowBytes(row),
|
|
6227
|
+
0,
|
|
6228
|
+
),
|
|
6229
|
+
executeBatch: async ({ rows: chunkRows, chunkStart, chunkIndex }) => {
|
|
6230
|
+
assertNotAborted(abortSignal);
|
|
6231
|
+
return await runChunkStep(chunkRows, chunkStart, chunkIndex);
|
|
6232
|
+
},
|
|
6233
|
+
estimateBatchResidentBytes: ({ rows: chunkRows }) =>
|
|
6234
|
+
chunkRows.reduce(
|
|
6235
|
+
(total, row) => total + workerMapResidentRowBytes(row),
|
|
6556
6236
|
0,
|
|
6557
|
-
WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
|
|
6558
6237
|
),
|
|
6238
|
+
estimateBatchSubrequests: ({ rows: chunkRows }) => {
|
|
6239
|
+
const estimate = mapDispatchPlan.workEstimate;
|
|
6240
|
+
const unbatchedProviderSubrequests =
|
|
6241
|
+
estimate.unbatchedProviderToolCallsPerRow *
|
|
6242
|
+
WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL;
|
|
6243
|
+
// Static planning cannot know input-dependent batching buckets, so
|
|
6244
|
+
// this projects worst-case provider-batch fanout while still treating
|
|
6245
|
+
// durable receipt claim/complete/fail as bulk per drained tool group.
|
|
6246
|
+
const batchedProviderSubrequests =
|
|
6247
|
+
estimateBatchedToolChunkSubrequests({
|
|
6248
|
+
batchableToolCallsPerRow: Math.max(
|
|
6249
|
+
0,
|
|
6250
|
+
estimate.providerToolCallsPerRow -
|
|
6251
|
+
estimate.unbatchedProviderToolCallsPerRow,
|
|
6252
|
+
),
|
|
6253
|
+
coalescedBatchableToolCallsPerRow:
|
|
6254
|
+
estimate.coalescedBatchableProviderToolCallsPerRow,
|
|
6255
|
+
rows: chunkRows.length,
|
|
6256
|
+
assumedBatchSize: estimate.assumedBatchSize,
|
|
6257
|
+
});
|
|
6258
|
+
const controlSubrequests =
|
|
6259
|
+
estimate.childPlaySubmitsPerRow + estimate.eventWaitsPerRow;
|
|
6260
|
+
return (
|
|
6261
|
+
chunkRows.length *
|
|
6262
|
+
(unbatchedProviderSubrequests + controlSubrequests) +
|
|
6263
|
+
batchedProviderSubrequests +
|
|
6264
|
+
estimate.runtimeSheetReadsPerChunk +
|
|
6265
|
+
estimate.runtimeSheetWritesPerChunk
|
|
6266
|
+
);
|
|
6267
|
+
},
|
|
6268
|
+
onBatchComplete: async (chunkResult) => {
|
|
6269
|
+
if (chunkResult.status === 'blocked') {
|
|
6270
|
+
totalRowsBlocked += chunkResult.rowsBlocked;
|
|
6271
|
+
for (const key of chunkResult.rowKeySamples) {
|
|
6272
|
+
if (blockedRowKeySamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6273
|
+
break;
|
|
6274
|
+
}
|
|
6275
|
+
blockedRowKeySamples.push(key);
|
|
6276
|
+
}
|
|
6277
|
+
for (const row of chunkResult.blockedRowSamples) {
|
|
6278
|
+
if (blockedRowSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6279
|
+
break;
|
|
6280
|
+
}
|
|
6281
|
+
blockedRowSamples.push(row);
|
|
6282
|
+
}
|
|
6283
|
+
emitEvent({
|
|
6284
|
+
type: 'log',
|
|
6285
|
+
level: 'warn',
|
|
6286
|
+
message: chunkResult.message,
|
|
6287
|
+
ts: nowMs(),
|
|
6288
|
+
});
|
|
6289
|
+
return true;
|
|
6290
|
+
}
|
|
6291
|
+
if (chunkResult.fatalError) {
|
|
6292
|
+
throw new Error(
|
|
6293
|
+
`ctx.dataset("${name}") stopped after a runtime persistence failure ` +
|
|
6294
|
+
`outside the retryable chunk step. Provider calls already executed for ` +
|
|
6295
|
+
`${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
|
|
6296
|
+
`Fix the runtime persistence cause and re-run to resume. ` +
|
|
6297
|
+
`First error: ${chunkResult.fatalError}`,
|
|
6298
|
+
);
|
|
6299
|
+
}
|
|
6300
|
+
totalRowsWritten += chunkResult.rowsWritten;
|
|
6301
|
+
totalRowsExecuted += chunkResult.rowsExecuted;
|
|
6302
|
+
totalRowsCached += chunkResult.rowsCached;
|
|
6303
|
+
totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
|
|
6304
|
+
totalRowsInserted += chunkResult.rowsInserted;
|
|
6305
|
+
totalRowsSkipped += chunkResult.rowsSkipped;
|
|
6306
|
+
totalRowsFailed += chunkResult.rowsFailed ?? 0;
|
|
6307
|
+
totalStepCellsCompleted += chunkResult.stepCellsCompleted ?? 0;
|
|
6308
|
+
totalStepCellsSkipped += chunkResult.stepCellsSkipped ?? 0;
|
|
6309
|
+
for (const sample of chunkResult.rowFailureSamples ?? []) {
|
|
6310
|
+
if (totalRowFailureSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6311
|
+
break;
|
|
6312
|
+
}
|
|
6313
|
+
totalRowFailureSamples.push(sample);
|
|
6314
|
+
}
|
|
6315
|
+
enqueueMapProgressUpdate({
|
|
6316
|
+
completed: totalRowsWritten,
|
|
6317
|
+
total: rowCountHint ?? undefined,
|
|
6318
|
+
...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
|
|
6319
|
+
message: formatMapProgressMessage(
|
|
6320
|
+
totalRowsWritten,
|
|
6321
|
+
rowCountHint ?? undefined,
|
|
6322
|
+
),
|
|
6323
|
+
});
|
|
6324
|
+
if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
|
|
6325
|
+
previewRows.push(
|
|
6326
|
+
...chunkResult.preview.slice(
|
|
6327
|
+
0,
|
|
6328
|
+
WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
|
|
6329
|
+
),
|
|
6330
|
+
);
|
|
6331
|
+
}
|
|
6332
|
+
if (canCacheRows) {
|
|
6333
|
+
const volatileRows = volatileWorkflowChunkRows.get(
|
|
6334
|
+
chunkResult.chunkIndex,
|
|
6335
|
+
);
|
|
6336
|
+
volatileWorkflowChunkRows.delete(chunkResult.chunkIndex);
|
|
6337
|
+
const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
|
|
6338
|
+
if (
|
|
6339
|
+
nextRows.length === chunkResult.rowsWritten &&
|
|
6340
|
+
cachedRows.length + nextRows.length <=
|
|
6341
|
+
WORKER_DATASET_IN_MEMORY_ROWS
|
|
6342
|
+
) {
|
|
6343
|
+
cachedRows.push(...nextRows);
|
|
6344
|
+
} else {
|
|
6345
|
+
cachedRows.length = 0;
|
|
6346
|
+
volatileWorkflowChunkRows.clear();
|
|
6347
|
+
canCacheRows = false;
|
|
6348
|
+
}
|
|
6349
|
+
}
|
|
6350
|
+
// onRowError:'fail' short-circuit: once a chunk reports a row failure,
|
|
6351
|
+
// skip the remaining chunks entirely. The failing chunk itself completed
|
|
6352
|
+
// normally (no chunk-step retry storm) and persisted its rows, but
|
|
6353
|
+
// executing later chunks would keep spending provider credits on a run
|
|
6354
|
+
// the caller asked to fail fast. The post-loop fail-fast throw below
|
|
6355
|
+
// reports what committed before the stop.
|
|
6356
|
+
return failFastRowErrors && totalRowsFailed > 0;
|
|
6357
|
+
},
|
|
6358
|
+
});
|
|
6359
|
+
} catch (error) {
|
|
6360
|
+
try {
|
|
6361
|
+
await flushQueuedMapProgressUpdates();
|
|
6362
|
+
} catch (progressError) {
|
|
6363
|
+
throw new AggregateError(
|
|
6364
|
+
[error, progressError],
|
|
6365
|
+
`ctx.dataset("${name}") failed while flushing queued progress updates.`,
|
|
6559
6366
|
);
|
|
6560
6367
|
}
|
|
6561
|
-
|
|
6562
|
-
const volatileRows = volatileWorkflowChunkRows.get(chunkIndex);
|
|
6563
|
-
volatileWorkflowChunkRows.delete(chunkIndex);
|
|
6564
|
-
const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
|
|
6565
|
-
if (
|
|
6566
|
-
nextRows.length === chunkResult.rowsWritten &&
|
|
6567
|
-
cachedRows.length + nextRows.length <= WORKER_DATASET_IN_MEMORY_ROWS
|
|
6568
|
-
) {
|
|
6569
|
-
cachedRows.push(...nextRows);
|
|
6570
|
-
} else {
|
|
6571
|
-
cachedRows.length = 0;
|
|
6572
|
-
volatileWorkflowChunkRows.clear();
|
|
6573
|
-
canCacheRows = false;
|
|
6574
|
-
}
|
|
6575
|
-
}
|
|
6576
|
-
chunkStart += rawChunkRows.length;
|
|
6577
|
-
chunkIndex += 1;
|
|
6578
|
-
// onRowError:'fail' short-circuit: once a chunk reports a row failure,
|
|
6579
|
-
// skip the remaining chunks entirely. The failing chunk itself completed
|
|
6580
|
-
// normally (no chunk-step retry storm) and persisted its rows, but
|
|
6581
|
-
// executing later chunks would keep spending provider credits on a run
|
|
6582
|
-
// the caller asked to fail fast. The post-loop fail-fast throw below
|
|
6583
|
-
// reports what committed before the stop.
|
|
6584
|
-
if (failFastRowErrors && totalRowsFailed > 0) {
|
|
6585
|
-
break;
|
|
6586
|
-
}
|
|
6368
|
+
throw error;
|
|
6587
6369
|
}
|
|
6588
6370
|
if (totalDuplicateKeysDropped > 0) {
|
|
6589
6371
|
const keySample = droppedDuplicateKeySamples.join(', ');
|
|
@@ -6596,6 +6378,13 @@ function createMinimalWorkerCtx(
|
|
|
6596
6378
|
ts: nowMs(),
|
|
6597
6379
|
});
|
|
6598
6380
|
}
|
|
6381
|
+
if (totalRowsBlocked > 0) {
|
|
6382
|
+
await flushQueuedMapProgressUpdates();
|
|
6383
|
+
throw new RuntimeSheetRowsBlockedError({
|
|
6384
|
+
tableNamespace: name,
|
|
6385
|
+
blockedRows: blockedRowSamples,
|
|
6386
|
+
});
|
|
6387
|
+
}
|
|
6599
6388
|
if (failFastRowErrors && totalRowsFailed > 0 && totalRowsWritten > 0) {
|
|
6600
6389
|
// onRowError:'fail', PARTIAL failure (some rows committed): fail the run
|
|
6601
6390
|
// without finalizing the dataset. The committed rows already persisted
|
|
@@ -6604,6 +6393,7 @@ function createMinimalWorkerCtx(
|
|
|
6604
6393
|
// normally (no per-row throw inside the durable chunk step, so no
|
|
6605
6394
|
// chunk-step retry storm); later chunks were skipped by the fail-fast
|
|
6606
6395
|
// short-circuit in the chunk loop.
|
|
6396
|
+
await flushQueuedMapProgressUpdates();
|
|
6607
6397
|
const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
|
|
6608
6398
|
throw new Error(
|
|
6609
6399
|
`ctx.dataset("${name}") failed for ${totalRowsFailed} executed row(s) under onRowError:'fail'. ` +
|
|
@@ -6621,7 +6411,7 @@ function createMinimalWorkerCtx(
|
|
|
6621
6411
|
// step when no row otherwise succeeded) are summarized and registered as
|
|
6622
6412
|
// a recovered dataset — the failed run then advertises a WORKING export
|
|
6623
6413
|
// instead of a dead end (#15/#27). The run still fails (the throw below).
|
|
6624
|
-
await finalize(
|
|
6414
|
+
await finalize();
|
|
6625
6415
|
const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
|
|
6626
6416
|
throw new Error(
|
|
6627
6417
|
`ctx.dataset("${name}") failed for all ${totalRowsFailed} executed rows. ` +
|
|
@@ -6629,7 +6419,7 @@ function createMinimalWorkerCtx(
|
|
|
6629
6419
|
`(rows are persisted with per-row errors; fix the cause and re-run to resume)`,
|
|
6630
6420
|
);
|
|
6631
6421
|
}
|
|
6632
|
-
const dataset = await finalize(
|
|
6422
|
+
const dataset = await finalize();
|
|
6633
6423
|
recordRunnerPerfTrace({
|
|
6634
6424
|
req,
|
|
6635
6425
|
phase: 'runner.map.total',
|
|
@@ -6638,7 +6428,7 @@ function createMinimalWorkerCtx(
|
|
|
6638
6428
|
mapName: name,
|
|
6639
6429
|
rowsWritten: totalRowsWritten,
|
|
6640
6430
|
inputKind: rowCountHint === null ? 'streaming' : 'known_count',
|
|
6641
|
-
chunks:
|
|
6431
|
+
chunks: dispatchResult.chunksExecuted,
|
|
6642
6432
|
},
|
|
6643
6433
|
});
|
|
6644
6434
|
return dataset;
|
|
@@ -6903,10 +6693,10 @@ function createMinimalWorkerCtx(
|
|
|
6903
6693
|
request.input,
|
|
6904
6694
|
workflowStep,
|
|
6905
6695
|
{
|
|
6906
|
-
force: request.force === true
|
|
6696
|
+
force: request.force === true,
|
|
6697
|
+
runForceRefresh: req.forceToolRefresh === true,
|
|
6698
|
+
runForceFailedRefresh: req.force === true,
|
|
6907
6699
|
staleAfterSeconds: request.staleAfterSeconds,
|
|
6908
|
-
},
|
|
6909
|
-
{
|
|
6910
6700
|
timeoutMs: request.timeoutMs,
|
|
6911
6701
|
receiptWaitMs: request.receiptWaitMs,
|
|
6912
6702
|
},
|
|
@@ -6947,13 +6737,18 @@ function createMinimalWorkerCtx(
|
|
|
6947
6737
|
if (!resolvedName) {
|
|
6948
6738
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
6949
6739
|
}
|
|
6740
|
+
const childManifest = req.childPlayManifests?.[resolvedName];
|
|
6950
6741
|
const receiptKey = buildDurableCtxCallCacheKey({
|
|
6951
6742
|
orgId: req.orgId,
|
|
6952
6743
|
playId: req.playName,
|
|
6953
6744
|
kind: 'runPlay',
|
|
6954
6745
|
id: normalizedKey,
|
|
6955
|
-
semanticKey:
|
|
6746
|
+
semanticKey: buildDurableRunPlaySemanticKey({
|
|
6956
6747
|
childPlayName: resolvedName,
|
|
6748
|
+
childRevisionFingerprint: workerChildRevisionFingerprint({
|
|
6749
|
+
playName: resolvedName,
|
|
6750
|
+
manifest: childManifest,
|
|
6751
|
+
}),
|
|
6957
6752
|
input,
|
|
6958
6753
|
}),
|
|
6959
6754
|
staleAfterSeconds: options?.staleAfterSeconds,
|
|
@@ -6981,7 +6776,6 @@ function createMinimalWorkerCtx(
|
|
|
6981
6776
|
message: `Starting child play ${resolvedName} (${normalizedKey})`,
|
|
6982
6777
|
ts: nowMs(),
|
|
6983
6778
|
});
|
|
6984
|
-
const childManifest = req.childPlayManifests?.[resolvedName];
|
|
6985
6779
|
if (!childManifest) {
|
|
6986
6780
|
throw new Error(
|
|
6987
6781
|
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
|
|
@@ -7019,7 +6813,7 @@ function createMinimalWorkerCtx(
|
|
|
7019
6813
|
});
|
|
7020
6814
|
let childPlaySlot: { release(): void } | null = null;
|
|
7021
6815
|
try {
|
|
7022
|
-
childPlaySlot = await governor.
|
|
6816
|
+
childPlaySlot = await governor.acquireChildSubmitSlot({
|
|
7023
6817
|
signal: abortSignal,
|
|
7024
6818
|
});
|
|
7025
6819
|
const childSubmitStartedAt = nowMs();
|
|
@@ -7044,12 +6838,12 @@ function createMinimalWorkerCtx(
|
|
|
7044
6838
|
options?.timeoutMs == null && !childNeedsWorkflowScheduler,
|
|
7045
6839
|
body: {
|
|
7046
6840
|
name: resolvedName,
|
|
6841
|
+
childIdempotencyKey: receiptKey,
|
|
7047
6842
|
input: isRecord(input) ? input : {},
|
|
7048
6843
|
orgId: req.orgId,
|
|
7049
6844
|
callbackBaseUrl: req.callbackUrl,
|
|
7050
6845
|
baseUrl: req.baseUrl,
|
|
7051
6846
|
integrationMode: req.integrationMode ?? null,
|
|
7052
|
-
forceToolRefresh: req.force === true,
|
|
7053
6847
|
parentExecutorToken: req.executorToken,
|
|
7054
6848
|
userEmail: req.userEmail ?? '',
|
|
7055
6849
|
profile: 'workers_edge',
|
|
@@ -7161,6 +6955,8 @@ function createMinimalWorkerCtx(
|
|
|
7161
6955
|
childNeedsWorkflowScheduler,
|
|
7162
6956
|
},
|
|
7163
6957
|
});
|
|
6958
|
+
childPlaySlot?.release();
|
|
6959
|
+
childPlaySlot = null;
|
|
7164
6960
|
const startedStatus = String(started.status ?? '').toLowerCase();
|
|
7165
6961
|
if (startedStatus === 'completed') {
|
|
7166
6962
|
emitEvent({
|
|
@@ -7189,6 +6985,10 @@ function createMinimalWorkerCtx(
|
|
|
7189
6985
|
}
|
|
7190
6986
|
const childWaitStartedAt = nowMs();
|
|
7191
6987
|
let waitResult: ChildPlayTerminalWaitResult;
|
|
6988
|
+
const readCachedChildTerminalState =
|
|
6989
|
+
cachedCoordinatorBinding?.readChildTerminalState?.bind(
|
|
6990
|
+
cachedCoordinatorBinding,
|
|
6991
|
+
);
|
|
7192
6992
|
try {
|
|
7193
6993
|
waitResult = await awaitChildTerminal({
|
|
7194
6994
|
parentRunId: req.runId,
|
|
@@ -7205,20 +7005,31 @@ function createMinimalWorkerCtx(
|
|
|
7205
7005
|
1_000,
|
|
7206
7006
|
Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
|
|
7207
7007
|
),
|
|
7208
|
-
coordinator:
|
|
7209
|
-
|
|
7210
|
-
|
|
7211
|
-
|
|
7212
|
-
|
|
7213
|
-
|
|
7214
|
-
|
|
7215
|
-
|
|
7216
|
-
|
|
7217
|
-
|
|
7218
|
-
|
|
7219
|
-
|
|
7220
|
-
|
|
7221
|
-
|
|
7008
|
+
coordinator: {
|
|
7009
|
+
readRunTerminalState: (runId, timeoutMs) =>
|
|
7010
|
+
readDurableChildRunTerminalState({
|
|
7011
|
+
baseUrl: req.baseUrl,
|
|
7012
|
+
executorToken: req.executorToken,
|
|
7013
|
+
parentRunId: req.runId,
|
|
7014
|
+
childRunId: runId,
|
|
7015
|
+
childPlayName: resolvedName,
|
|
7016
|
+
timeoutMs,
|
|
7017
|
+
}),
|
|
7018
|
+
...(readCachedChildTerminalState
|
|
7019
|
+
? {
|
|
7020
|
+
readChildTerminalState: (
|
|
7021
|
+
parentRunId: string,
|
|
7022
|
+
eventKey: string,
|
|
7023
|
+
timeoutMs?: number,
|
|
7024
|
+
) =>
|
|
7025
|
+
readCachedChildTerminalState(
|
|
7026
|
+
parentRunId,
|
|
7027
|
+
eventKey,
|
|
7028
|
+
timeoutMs,
|
|
7029
|
+
),
|
|
7030
|
+
}
|
|
7031
|
+
: {}),
|
|
7032
|
+
},
|
|
7222
7033
|
now: nowMs,
|
|
7223
7034
|
hashJson,
|
|
7224
7035
|
});
|
|
@@ -7401,6 +7212,7 @@ function createMinimalWorkerCtx(
|
|
|
7401
7212
|
// Direct Worker fetch below is still limited to runtime origins and
|
|
7402
7213
|
// IP literals; public IP literals also consume this slot because
|
|
7403
7214
|
// they are customer egress.
|
|
7215
|
+
workBudgetMeter.count('egress');
|
|
7404
7216
|
const egressResponse = await postRuntimeEgressFetch(
|
|
7405
7217
|
req,
|
|
7406
7218
|
{
|
|
@@ -7423,6 +7235,7 @@ function createMinimalWorkerCtx(
|
|
|
7423
7235
|
}
|
|
7424
7236
|
const fetchInit = { ...init, headers };
|
|
7425
7237
|
delete fetchInit.auth;
|
|
7238
|
+
workBudgetMeter.count('egress');
|
|
7426
7239
|
const response = await safeWorkerPublicFetch(url, fetchInit, {
|
|
7427
7240
|
allowedOrigins,
|
|
7428
7241
|
sensitiveHeaders: Object.keys(secretHeaderMarkers),
|
|
@@ -7493,6 +7306,7 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
|
|
|
7493
7306
|
let req: RunRequest;
|
|
7494
7307
|
try {
|
|
7495
7308
|
req = (await request.json()) as RunRequest;
|
|
7309
|
+
req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
|
|
7496
7310
|
} catch {
|
|
7497
7311
|
return new Response('invalid JSON body', { status: 400 });
|
|
7498
7312
|
}
|
|
@@ -7557,6 +7371,7 @@ async function handleRunInline(
|
|
|
7557
7371
|
let req: RunRequest;
|
|
7558
7372
|
try {
|
|
7559
7373
|
req = (await request.json()) as RunRequest;
|
|
7374
|
+
req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
|
|
7560
7375
|
} catch {
|
|
7561
7376
|
return Response.json(
|
|
7562
7377
|
{
|
|
@@ -7704,6 +7519,7 @@ async function executeRunRequest(
|
|
|
7704
7519
|
): Promise<WorkflowRunOutput> {
|
|
7705
7520
|
installProcessExitTrap();
|
|
7706
7521
|
const startedAt = nowMs();
|
|
7522
|
+
const workBudgetMeter = createWorkerWorkBudgetMeter({ nowMs });
|
|
7707
7523
|
recordRunnerPerfTrace({
|
|
7708
7524
|
req,
|
|
7709
7525
|
phase: 'runner.execute_start',
|
|
@@ -7741,6 +7557,7 @@ async function executeRunRequest(
|
|
|
7741
7557
|
const appendRunLogLine = (line: string) => {
|
|
7742
7558
|
const trimmed = redactSecretsFromLogString(line.trim());
|
|
7743
7559
|
if (!trimmed) return;
|
|
7560
|
+
workBudgetMeter.count('log');
|
|
7744
7561
|
totalEmittedLogLines += 1;
|
|
7745
7562
|
runLogBuffer = [...runLogBuffer, trimmed].slice(-RUN_LOG_BUFFER_LIMIT);
|
|
7746
7563
|
pendingRunLogLines = [...pendingRunLogLines, trimmed];
|
|
@@ -7933,7 +7750,10 @@ async function executeRunRequest(
|
|
|
7933
7750
|
const flushTerminalLedgerEvents = async (
|
|
7934
7751
|
terminalEvent: PlayRunLedgerEvent,
|
|
7935
7752
|
): Promise<void> => {
|
|
7936
|
-
|
|
7753
|
+
const shouldPersistTerminalLedger =
|
|
7754
|
+
options?.persistResultDatasets === true ||
|
|
7755
|
+
Boolean(req.playCallGovernance);
|
|
7756
|
+
if (!shouldPersistTerminalLedger) return;
|
|
7937
7757
|
await ledgerFlushInFlight;
|
|
7938
7758
|
const now = nowMs();
|
|
7939
7759
|
dirtyProgressNodeIds = new Set([
|
|
@@ -8008,10 +7828,7 @@ async function executeRunRequest(
|
|
|
8008
7828
|
stepLifecycle?.markPreDatasetStepsStarted(startedAt);
|
|
8009
7829
|
flushLedgerEvents(false);
|
|
8010
7830
|
const runtimeDeadlineMs = nowMs() + STANDARD_PLAY_RUNTIME_LIMIT_MS;
|
|
8011
|
-
|
|
8012
|
-
// receipt could not be marked completed even after the retry ladder. On
|
|
8013
|
-
// terminal failure the buffer is built into a size-capped salvage record and
|
|
8014
|
-
// attached to the run.failed ledger event so billed data is never memory-only.
|
|
7831
|
+
const leasedSheetNamespaces = new Set<string>();
|
|
8015
7832
|
const receiptSalvage = createReceiptSalvageBuffer();
|
|
8016
7833
|
const ctx = createMinimalWorkerCtx(
|
|
8017
7834
|
req,
|
|
@@ -8020,7 +7837,9 @@ async function executeRunRequest(
|
|
|
8020
7837
|
workflowStep,
|
|
8021
7838
|
abortSignal,
|
|
8022
7839
|
workerCallbacks,
|
|
7840
|
+
workBudgetMeter,
|
|
8023
7841
|
runtimeDeadlineMs,
|
|
7842
|
+
leasedSheetNamespaces,
|
|
8024
7843
|
receiptSalvage,
|
|
8025
7844
|
);
|
|
8026
7845
|
// Hard wall-clock cap on active user-code runtime. CF Workflows does not
|
|
@@ -8029,15 +7848,18 @@ async function executeRunRequest(
|
|
|
8029
7848
|
// token expires. Aborting the controller surfaces cooperatively through the
|
|
8030
7849
|
// same assertNotAborted checks used for harness cancellation.
|
|
8031
7850
|
let runtimeLimitExceeded = false;
|
|
8032
|
-
const runtimeDeadlineTimer =
|
|
8033
|
-
|
|
8034
|
-
|
|
8035
|
-
|
|
8036
|
-
|
|
8037
|
-
|
|
8038
|
-
|
|
8039
|
-
|
|
8040
|
-
|
|
7851
|
+
const runtimeDeadlineTimer =
|
|
7852
|
+
workflowStep === undefined || workflowStep === null
|
|
7853
|
+
? setTimeout(
|
|
7854
|
+
() => {
|
|
7855
|
+
runtimeLimitExceeded = true;
|
|
7856
|
+
if (!abortSignal.aborted) {
|
|
7857
|
+
abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
|
|
7858
|
+
}
|
|
7859
|
+
},
|
|
7860
|
+
Math.max(1, runtimeDeadlineMs - nowMs()),
|
|
7861
|
+
)
|
|
7862
|
+
: null;
|
|
8041
7863
|
try {
|
|
8042
7864
|
const playStartedAt = nowMs();
|
|
8043
7865
|
const result = await (
|
|
@@ -8128,7 +7950,6 @@ async function executeRunRequest(
|
|
|
8128
7950
|
}
|
|
8129
7951
|
if (options?.persistResultDatasets) {
|
|
8130
7952
|
await persistProjectedResultDatasets();
|
|
8131
|
-
const parentSignal = startParentTerminalSignal();
|
|
8132
7953
|
// Capped runs settle compute billing BEFORE declaring run.completed: a
|
|
8133
7954
|
// per-run cap denial (422 billing_cap_exceeded) must fail the run as
|
|
8134
7955
|
// its ONLY terminal. Flushing completed first opens a race — watchers
|
|
@@ -8163,6 +7984,7 @@ async function executeRunRequest(
|
|
|
8163
7984
|
ms: nowMs() - terminalUpdateStartedAt,
|
|
8164
7985
|
});
|
|
8165
7986
|
|
|
7987
|
+
const parentSignal = startParentTerminalSignal();
|
|
8166
7988
|
if (!capped) {
|
|
8167
7989
|
const billingStartedAt = nowMs();
|
|
8168
7990
|
const billingPromise = finalizeWorkerComputeBilling({
|
|
@@ -8190,6 +8012,20 @@ async function executeRunRequest(
|
|
|
8190
8012
|
}
|
|
8191
8013
|
}
|
|
8192
8014
|
await parentSignal;
|
|
8015
|
+
} else if (req.playCallGovernance) {
|
|
8016
|
+
const childTerminalStartedAt = nowMs();
|
|
8017
|
+
await flushTerminalLedgerEvents({
|
|
8018
|
+
type: 'run.completed',
|
|
8019
|
+
runId: req.runId,
|
|
8020
|
+
source: 'worker',
|
|
8021
|
+
occurredAt: nowMs(),
|
|
8022
|
+
result: terminalResult,
|
|
8023
|
+
});
|
|
8024
|
+
recordRunnerPerfTrace({
|
|
8025
|
+
req,
|
|
8026
|
+
phase: 'runner.child_terminal_ledger_append',
|
|
8027
|
+
ms: nowMs() - childTerminalStartedAt,
|
|
8028
|
+
});
|
|
8193
8029
|
}
|
|
8194
8030
|
await startParentTerminalSignal();
|
|
8195
8031
|
recordRunnerPerfTrace({
|
|
@@ -8216,16 +8052,8 @@ async function executeRunRequest(
|
|
|
8216
8052
|
stepLifecycle?.markStartedFailed(nowMs());
|
|
8217
8053
|
// A runtime-limit abort is a timeout failure, not a user cancellation, so
|
|
8218
8054
|
// it should be reported as run.failed with the limit message rather than
|
|
8219
|
-
// run.cancelled.
|
|
8220
|
-
|
|
8221
|
-
// runtimeDeadlineTimer macrotask flips the flag — match the limit message
|
|
8222
|
-
// directly so that race can never misclassify a timeout as a cancel (which
|
|
8223
|
-
// would silently drop the salvage buffer of billed-but-unpersisted rows).
|
|
8224
|
-
const runtimeLimitError =
|
|
8225
|
-
runtimeLimitExceeded ||
|
|
8226
|
-
(error instanceof Error &&
|
|
8227
|
-
error.message === STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
|
|
8228
|
-
const aborted = isAbortLikeError(error) && !runtimeLimitError;
|
|
8055
|
+
// run.cancelled.
|
|
8056
|
+
const aborted = isAbortLikeError(error) && !runtimeLimitExceeded;
|
|
8229
8057
|
if (aborted) {
|
|
8230
8058
|
// Flip the controller so any concurrent user code observes the abort
|
|
8231
8059
|
// through ctx.signal. We mark the run cancelled instead of failed.
|
|
@@ -8234,19 +8062,28 @@ async function executeRunRequest(
|
|
|
8234
8062
|
);
|
|
8235
8063
|
}
|
|
8236
8064
|
const failure = normalizePlayRunFailure(error);
|
|
8237
|
-
|
|
8238
|
-
//
|
|
8065
|
+
const message = failure.message;
|
|
8066
|
+
// Controlled run-fatal teardown: release the runtime-sheet row leases and
|
|
8067
|
+
// work-receipt leases this attempt still holds so an immediate rerun is not
|
|
8068
|
+
// blocked for the full lease TTL. Best-effort but LOUD — a release failure
|
|
8069
|
+
// is logged and never masks the original run-fatal error. TTL expiry stays
|
|
8070
|
+
// the recovery path for true crashes (isolate death) where this never runs.
|
|
8071
|
+
await releaseRuntimeLeasesOnTeardown({
|
|
8072
|
+
req,
|
|
8073
|
+
leasedSheetNamespaces,
|
|
8074
|
+
emit: wrappedEmit,
|
|
8075
|
+
});
|
|
8076
|
+
const errorBilling = extractErrorBilling(error);
|
|
8239
8077
|
const salvage: ReceiptSalvage | null = aborted
|
|
8240
8078
|
? null
|
|
8241
8079
|
: receiptSalvage.build();
|
|
8242
|
-
const
|
|
8080
|
+
const terminalMessage =
|
|
8243
8081
|
salvage && salvage.totalEntries > 0
|
|
8244
|
-
? `${
|
|
8245
|
-
:
|
|
8246
|
-
|
|
8247
|
-
if (options?.persistResultDatasets) {
|
|
8082
|
+
? `${message} ${receiptSalvageFailureSuffix(salvage)}`
|
|
8083
|
+
: message;
|
|
8084
|
+
if (options?.persistResultDatasets || req.playCallGovernance) {
|
|
8248
8085
|
appendRunLogLine(
|
|
8249
|
-
`${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(
|
|
8086
|
+
`${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(terminalMessage)}`,
|
|
8250
8087
|
);
|
|
8251
8088
|
const terminalUpdateStartedAt = nowMs();
|
|
8252
8089
|
await flushTerminalLedgerEvents({
|
|
@@ -8254,64 +8091,70 @@ async function executeRunRequest(
|
|
|
8254
8091
|
runId: req.runId,
|
|
8255
8092
|
source: 'worker',
|
|
8256
8093
|
occurredAt: nowMs(),
|
|
8257
|
-
error:
|
|
8094
|
+
error: terminalMessage,
|
|
8258
8095
|
result: aborted
|
|
8259
8096
|
? undefined
|
|
8260
8097
|
: {
|
|
8261
8098
|
success: false,
|
|
8262
8099
|
status: 'failed',
|
|
8263
|
-
error:
|
|
8100
|
+
error: terminalMessage,
|
|
8101
|
+
...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
|
|
8264
8102
|
errors: [
|
|
8265
8103
|
{
|
|
8266
8104
|
code: failure.code,
|
|
8267
8105
|
phase: failure.phase,
|
|
8268
|
-
message,
|
|
8106
|
+
message: terminalMessage,
|
|
8269
8107
|
retryable: failure.retryable,
|
|
8270
8108
|
...(errorBilling ? { billing: errorBilling } : {}),
|
|
8271
8109
|
...(failure.cause ? { cause: failure.cause } : {}),
|
|
8272
8110
|
},
|
|
8273
8111
|
],
|
|
8274
|
-
...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
|
|
8275
8112
|
},
|
|
8276
8113
|
});
|
|
8277
8114
|
recordRunnerPerfTrace({
|
|
8278
8115
|
req,
|
|
8279
|
-
phase:
|
|
8280
|
-
?
|
|
8281
|
-
|
|
8116
|
+
phase: req.playCallGovernance
|
|
8117
|
+
? aborted
|
|
8118
|
+
? 'runner.child_terminal_ledger_append_cancelled'
|
|
8119
|
+
: 'runner.child_terminal_ledger_append_failed'
|
|
8120
|
+
: aborted
|
|
8121
|
+
? 'runner.terminal_ledger_append_cancelled'
|
|
8122
|
+
: 'runner.terminal_ledger_append_failed',
|
|
8282
8123
|
ms: nowMs() - terminalUpdateStartedAt,
|
|
8283
8124
|
extra: {
|
|
8284
8125
|
errorCode: failure.code,
|
|
8285
8126
|
errorPhase: failure.phase,
|
|
8286
8127
|
},
|
|
8287
8128
|
});
|
|
8288
|
-
|
|
8289
|
-
|
|
8290
|
-
|
|
8291
|
-
|
|
8292
|
-
|
|
8293
|
-
|
|
8294
|
-
.catch((finalizeError) => {
|
|
8295
|
-
console.error(
|
|
8296
|
-
`[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
|
|
8297
|
-
finalizeError instanceof Error
|
|
8298
|
-
? finalizeError.message
|
|
8299
|
-
: String(finalizeError)
|
|
8300
|
-
}`,
|
|
8301
|
-
);
|
|
8129
|
+
if (options?.persistResultDatasets) {
|
|
8130
|
+
const billingStartedAt = nowMs();
|
|
8131
|
+
await finalizeWorkerComputeBilling({
|
|
8132
|
+
req,
|
|
8133
|
+
success: false,
|
|
8134
|
+
actionEstimate: 4,
|
|
8302
8135
|
})
|
|
8303
|
-
|
|
8304
|
-
|
|
8305
|
-
|
|
8306
|
-
|
|
8307
|
-
|
|
8136
|
+
.catch((finalizeError) => {
|
|
8137
|
+
console.error(
|
|
8138
|
+
`[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
|
|
8139
|
+
finalizeError instanceof Error
|
|
8140
|
+
? finalizeError.message
|
|
8141
|
+
: String(finalizeError)
|
|
8142
|
+
}`,
|
|
8143
|
+
);
|
|
8144
|
+
})
|
|
8145
|
+
.finally(() => {
|
|
8146
|
+
recordRunnerPerfTrace({
|
|
8147
|
+
req,
|
|
8148
|
+
phase: 'runner.compute_billing_finalize_failed',
|
|
8149
|
+
ms: nowMs() - billingStartedAt,
|
|
8150
|
+
});
|
|
8308
8151
|
});
|
|
8309
|
-
|
|
8152
|
+
}
|
|
8310
8153
|
}
|
|
8311
8154
|
await signalParentPlayTerminal({
|
|
8312
8155
|
req,
|
|
8313
8156
|
status: aborted ? 'cancelled' : 'failed',
|
|
8314
|
-
error:
|
|
8157
|
+
error: terminalMessage,
|
|
8315
8158
|
}).catch(() => null);
|
|
8316
8159
|
recordRunnerPerfTrace({
|
|
8317
8160
|
req,
|
|
@@ -8325,7 +8168,9 @@ async function executeRunRequest(
|
|
|
8325
8168
|
await drainRunnerPerfTraces(req);
|
|
8326
8169
|
throw error;
|
|
8327
8170
|
} finally {
|
|
8328
|
-
|
|
8171
|
+
if (runtimeDeadlineTimer) {
|
|
8172
|
+
clearTimeout(runtimeDeadlineTimer);
|
|
8173
|
+
}
|
|
8329
8174
|
}
|
|
8330
8175
|
}
|
|
8331
8176
|
|
|
@@ -8417,10 +8262,12 @@ function runRequestFromWorkflowParams(
|
|
|
8417
8262
|
orgId: String(params.orgId ?? ''),
|
|
8418
8263
|
playName: String(params.playName ?? ''),
|
|
8419
8264
|
userEmail: typeof params.userEmail === 'string' ? params.userEmail : null,
|
|
8265
|
+
force: params.force === true,
|
|
8266
|
+
forceToolRefresh: params.forceToolRefresh === true,
|
|
8267
|
+
runAttempt: normalizeWorkerRunAttempt(params.runAttempt),
|
|
8420
8268
|
runtimeInput: isRecord(params.input)
|
|
8421
8269
|
? (params.input as Record<string, unknown>)
|
|
8422
8270
|
: {},
|
|
8423
|
-
force: params.forceToolRefresh === true,
|
|
8424
8271
|
inlineCsv: isInlineCsv(params.inlineCsv) ? params.inlineCsv : null,
|
|
8425
8272
|
inputFiles:
|
|
8426
8273
|
inputFile && inputStorageKey
|
|
@@ -8478,6 +8325,14 @@ function runRequestFromWorkflowParams(
|
|
|
8478
8325
|
params.coordinatorInternalToken.trim()
|
|
8479
8326
|
? params.coordinatorInternalToken.trim()
|
|
8480
8327
|
: null,
|
|
8328
|
+
runtimeTestFaultHeader:
|
|
8329
|
+
typeof params.runtimeTestFaultHeader === 'string' &&
|
|
8330
|
+
params.runtimeTestFaultHeader.trim()
|
|
8331
|
+
? params.runtimeTestFaultHeader.trim()
|
|
8332
|
+
: null,
|
|
8333
|
+
testPolicyOverrides: normalizeRuntimeTestPolicyOverrides(
|
|
8334
|
+
params.testPolicyOverrides,
|
|
8335
|
+
),
|
|
8481
8336
|
totalRows:
|
|
8482
8337
|
typeof params.totalRows === 'number' && Number.isFinite(params.totalRows)
|
|
8483
8338
|
? params.totalRows
|
|
@@ -8547,6 +8402,7 @@ async function persistResultDatasets(
|
|
|
8547
8402
|
rows: chunk.map((row) => ({ ...row })),
|
|
8548
8403
|
runId: req.runId,
|
|
8549
8404
|
inputOffset,
|
|
8405
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
8550
8406
|
userEmail: req.userEmail,
|
|
8551
8407
|
preloadedDbSessions: req.preloadedDbSessions ?? null,
|
|
8552
8408
|
});
|
|
@@ -8583,6 +8439,7 @@ async function persistResultDatasets(
|
|
|
8583
8439
|
rows: dataset.rows,
|
|
8584
8440
|
runId: req.runId,
|
|
8585
8441
|
inputOffset: 0,
|
|
8442
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
8586
8443
|
userEmail: req.userEmail,
|
|
8587
8444
|
});
|
|
8588
8445
|
}
|
|
@@ -8864,7 +8721,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
|
|
|
8864
8721
|
try {
|
|
8865
8722
|
const response = await fetchRuntimeApi(
|
|
8866
8723
|
req.baseUrl,
|
|
8867
|
-
'/api/v2/
|
|
8724
|
+
'/api/v2/internal/play-runtime',
|
|
8868
8725
|
{
|
|
8869
8726
|
method: 'POST',
|
|
8870
8727
|
headers: {
|