deepline 0.1.181 → 0.1.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +633 -107
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1932 -1838
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +124 -12
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
- package/dist/bundling-sources/sdk/src/client.ts +48 -5
- package/dist/bundling-sources/sdk/src/http.ts +23 -8
- package/dist/bundling-sources/sdk/src/play.ts +200 -26
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +132 -10
- package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
- package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1857 -314
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +106 -6
- package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
- package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
- package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/query-result-dataset.ts +120 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
- package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
- package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2037 -262
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +139 -17
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
- package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
- package/dist/cli/index.js +85 -23
- package/dist/cli/index.mjs +85 -23
- package/dist/index.d.mts +4 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.js +274 -35
- package/dist/index.mjs +274 -35
- package/dist/plays/bundle-play-file.mjs +2 -2
- package/package.json +1 -1
|
@@ -36,12 +36,6 @@ import {
|
|
|
36
36
|
deterministicMapChunkStepName,
|
|
37
37
|
type ExecutionPlan,
|
|
38
38
|
} from '../../../shared_libs/play-runtime/execution-plan';
|
|
39
|
-
import {
|
|
40
|
-
compileRequestsWithStrategy,
|
|
41
|
-
executeChunkedRequests,
|
|
42
|
-
type ChunkExecutionResult,
|
|
43
|
-
} from '../../../shared_libs/play-runtime/batch-runtime';
|
|
44
|
-
import { getPlayRuntimeBatchStrategy } from '../../../shared_libs/play-runtime/play-runtime-batching-registry';
|
|
45
39
|
import {
|
|
46
40
|
STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
|
|
47
41
|
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
|
|
@@ -71,10 +65,10 @@ import {
|
|
|
71
65
|
type WorkflowStepLike,
|
|
72
66
|
} from './child-play-await';
|
|
73
67
|
import { submitChildPlayThroughCoordinator } from './child-play-submit';
|
|
74
|
-
import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
|
|
75
68
|
import {
|
|
76
|
-
|
|
69
|
+
attachToolResultListDataset,
|
|
77
70
|
createToolExecuteResult,
|
|
71
|
+
parseToolExecuteResponse,
|
|
78
72
|
deserializeToolExecuteResult,
|
|
79
73
|
isSerializedToolExecuteResult,
|
|
80
74
|
isToolExecuteResult,
|
|
@@ -89,21 +83,35 @@ import {
|
|
|
89
83
|
TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS,
|
|
90
84
|
classifyToolExecuteHttpFailure,
|
|
91
85
|
createToolExecuteHttpFailureAttemptTracker,
|
|
86
|
+
parseToolExecuteAuthScopeChangedError,
|
|
92
87
|
} from '../../../shared_libs/play-runtime/tool-execute-retry-policy';
|
|
93
88
|
import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
94
89
|
import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
|
|
95
|
-
import type {
|
|
90
|
+
import type {
|
|
91
|
+
PlayRuntimeManifest,
|
|
92
|
+
PlayRuntimeManifestMap,
|
|
93
|
+
} from '../../../shared_libs/plays/compiler-manifest';
|
|
96
94
|
import {
|
|
97
95
|
deriveToolRequestIdentity,
|
|
98
96
|
derivePlayRowIdentity,
|
|
99
97
|
derivePlayRowIdentityFromKey,
|
|
98
|
+
sha256Hex,
|
|
99
|
+
stableStringify,
|
|
100
100
|
} from '../../../shared_libs/plays/row-identity';
|
|
101
|
+
import { createDeferredPlayDataset } from '../../../shared_libs/plays/dataset';
|
|
101
102
|
import {
|
|
102
103
|
buildDurableCtxCallCacheKey,
|
|
103
|
-
|
|
104
|
-
buildDurableToolCallCacheKey,
|
|
104
|
+
buildDurableRunPlaySemanticKey,
|
|
105
105
|
} from '../../../shared_libs/play-runtime/durable-call-cache';
|
|
106
|
-
import {
|
|
106
|
+
import {
|
|
107
|
+
resolveRuntimeToolReceiptWaitMaxAttempts,
|
|
108
|
+
resolveRuntimeToolReceiptWaitTimeoutMs,
|
|
109
|
+
} from '../../../shared_libs/play-runtime/durable-receipt-execution';
|
|
110
|
+
import {
|
|
111
|
+
QUERY_RESULT_DATASET_PAGE_SIZE,
|
|
112
|
+
isCustomerDbDatasetTool,
|
|
113
|
+
isQueryResultDatasetTool,
|
|
114
|
+
} from '../../../shared_libs/play-runtime/query-result-dataset';
|
|
107
115
|
import { DEDUPE_DUPLICATE_KEY_SAMPLE_CAP } from '../../../shared_libs/play-runtime/map-row-identity';
|
|
108
116
|
import {
|
|
109
117
|
getTopLevelPipelineSubsteps,
|
|
@@ -142,26 +150,41 @@ import {
|
|
|
142
150
|
} from './runtime/dataset-handles';
|
|
143
151
|
import {
|
|
144
152
|
runWorkerRuntimeReceiptBoundary,
|
|
145
|
-
|
|
146
|
-
type
|
|
147
|
-
type WorkerRuntimeReceiptStore,
|
|
153
|
+
RuntimeLeaseLostError,
|
|
154
|
+
type WorkerRuntimeReceiptExecutionContext,
|
|
148
155
|
} from './runtime/receipts';
|
|
149
156
|
import {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
} from '
|
|
155
|
-
import
|
|
157
|
+
RuntimeReceiptPersistenceError,
|
|
158
|
+
WorkerToolBatchScheduler,
|
|
159
|
+
type WorkerPacingResolver,
|
|
160
|
+
type WorkerToolActionCacheVersionResolver,
|
|
161
|
+
} from './runtime/tool-dispatch';
|
|
162
|
+
import {
|
|
163
|
+
createWorkerWorkBudgetMeter,
|
|
164
|
+
type WorkerWorkBudgetMeter,
|
|
165
|
+
} from './runtime/work-budget';
|
|
166
|
+
import {
|
|
167
|
+
WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES,
|
|
168
|
+
WorkerRunWorkDispatcher,
|
|
169
|
+
type WorkerRunMapBatchesResult,
|
|
170
|
+
} from './runtime/run-work-dispatcher';
|
|
171
|
+
import { WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL } from './runtime/worker-platform-budget';
|
|
172
|
+
import {
|
|
173
|
+
PLAY_RUNTIME_CONTRACT,
|
|
174
|
+
PLAY_RUNTIME_CONTRACT_HEADER,
|
|
175
|
+
} from '../../../shared_libs/play-runtime/runtime-contract';
|
|
176
|
+
import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
|
|
177
|
+
import {
|
|
178
|
+
PLAY_RUNTIME_TEST_FAULT_HEADER,
|
|
179
|
+
type RuntimeTestPolicyOverrides,
|
|
180
|
+
} from '../../../shared_libs/play-runtime/test-runtime-seams';
|
|
181
|
+
import { createRuntimeToolAuthScopeDigestResolver } from '../../../shared_libs/play-runtime/auth-scope-resolver';
|
|
156
182
|
import {
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
resolveWorkerToolReceiptGroupWaitMaxAttempts,
|
|
163
|
-
resolveWorkerToolRuntimeTimeoutMs,
|
|
164
|
-
} from './runtime/tool-receipts';
|
|
183
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
184
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
185
|
+
runtimeLeaseHeartbeatIntervalMs,
|
|
186
|
+
} from '../../../shared_libs/play-runtime/lease-policy';
|
|
187
|
+
import { RuntimeSheetRowsBlockedError } from '../../../shared_libs/play-runtime/runtime-sheet-errors';
|
|
165
188
|
// The harness stub forwards leaf calls (validation, runtime-api HTTP) into
|
|
166
189
|
// the long-lived Play Harness Worker via env.HARNESS. We import the
|
|
167
190
|
// `setHarnessBinding` setter eagerly so it's available the moment
|
|
@@ -174,9 +197,12 @@ import {
|
|
|
174
197
|
// re-bundle harness internals into per-play. Keep that in mind.
|
|
175
198
|
import {
|
|
176
199
|
harnessPersistCompletedSheetRows,
|
|
200
|
+
harnessHeartbeatSheetAttempt,
|
|
177
201
|
harnessReadSheetDatasetRowKeys,
|
|
178
202
|
harnessReadSheetDatasetRows,
|
|
179
203
|
harnessReadStagedFileChunk,
|
|
204
|
+
harnessReleaseRuntimeReceipts,
|
|
205
|
+
harnessReleaseSheetAttempt,
|
|
180
206
|
harnessStartSheetDataset,
|
|
181
207
|
setHarnessBinding,
|
|
182
208
|
} from '../../../sdk/src/plays/harness-stub';
|
|
@@ -207,7 +233,10 @@ import {
|
|
|
207
233
|
stripMapRowOutcomeRuntimeFields,
|
|
208
234
|
} from '../../../shared_libs/play-runtime/map-row-outcome';
|
|
209
235
|
import { runtimeSheetSessionScope } from '../../../shared_libs/play-runtime/runtime-sheet-session';
|
|
210
|
-
import {
|
|
236
|
+
import {
|
|
237
|
+
chooseWorkerMapDispatchPlan,
|
|
238
|
+
estimateBatchedToolChunkSubrequests,
|
|
239
|
+
} from './runtime/map-chunk-plan';
|
|
211
240
|
import {
|
|
212
241
|
applyCsvRenameProjection,
|
|
213
242
|
cloneCsvAliasedRow,
|
|
@@ -219,12 +248,7 @@ import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-f
|
|
|
219
248
|
import {
|
|
220
249
|
createRuntimePersistenceLatch,
|
|
221
250
|
isRuntimePersistenceCircuitOpenError,
|
|
222
|
-
partitionDispatchWaves,
|
|
223
|
-
PROVIDER_DISPATCH_WAVE_SIZE,
|
|
224
|
-
RuntimePersistenceCircuitOpenError,
|
|
225
|
-
shouldShortenCompleteReceiptLadder,
|
|
226
251
|
tripRuntimePersistenceLatch,
|
|
227
|
-
type RuntimePersistenceLatch,
|
|
228
252
|
} from '../../../shared_libs/play-runtime/persistence-latch';
|
|
229
253
|
import {
|
|
230
254
|
createReceiptSalvageBuffer,
|
|
@@ -290,8 +314,13 @@ type RunRequest = {
|
|
|
290
314
|
playName: string;
|
|
291
315
|
graphHash?: string | null;
|
|
292
316
|
userEmail: string | null;
|
|
293
|
-
force?: boolean | null;
|
|
294
317
|
runtimeInput: Record<string, unknown>;
|
|
318
|
+
/** Start a fresh run graph and recompute runtime-sheet rows. */
|
|
319
|
+
force?: boolean;
|
|
320
|
+
/** Explicit durable tool receipt refresh that can re-execute completed provider calls. */
|
|
321
|
+
forceToolRefresh?: boolean;
|
|
322
|
+
/** Monotonic coordinator redispatch epoch for receipt and sheet fencing. */
|
|
323
|
+
runAttempt?: number;
|
|
295
324
|
/** Optional inline CSV rows (for plays where ctx.csv was passed inline data). */
|
|
296
325
|
inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
|
|
297
326
|
/** Staged input files keyed by logical filename (used by ctx.csv). */
|
|
@@ -324,6 +353,10 @@ type RunRequest = {
|
|
|
324
353
|
coordinatorUrl?: string | null;
|
|
325
354
|
/** Request-scoped coordinator auth token for preview/dev direct control calls. */
|
|
326
355
|
coordinatorInternalToken?: string | null;
|
|
356
|
+
/** Request-scoped, dev-only runtime fault injection header for black-box tests. */
|
|
357
|
+
runtimeTestFaultHeader?: string | null;
|
|
358
|
+
/** Request-scoped, dev-only policy overrides for black-box durability tests. */
|
|
359
|
+
testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
|
|
327
360
|
/**
|
|
328
361
|
* Total input rows known at submit time, when the dispatcher can count them.
|
|
329
362
|
* `undefined` when the input streams from R2 (unknown row count).
|
|
@@ -339,9 +372,136 @@ type WorkerFileRef = {
|
|
|
339
372
|
bytes?: number | null;
|
|
340
373
|
};
|
|
341
374
|
|
|
375
|
+
function normalizeWorkerRunAttempt(value: unknown): number {
|
|
376
|
+
return typeof value === 'number' && Number.isFinite(value)
|
|
377
|
+
? Math.max(0, Math.floor(value))
|
|
378
|
+
: 0;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function normalizePositiveInteger(value: unknown): number | undefined {
|
|
382
|
+
return typeof value === 'number' &&
|
|
383
|
+
Number.isFinite(value) &&
|
|
384
|
+
Number.isInteger(value) &&
|
|
385
|
+
value > 0
|
|
386
|
+
? value
|
|
387
|
+
: undefined;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function normalizeRuntimeTestPolicyOverrides(
|
|
391
|
+
value: unknown,
|
|
392
|
+
): RuntimeTestPolicyOverrides | null {
|
|
393
|
+
if (!isRecord(value)) return null;
|
|
394
|
+
const overrides: RuntimeTestPolicyOverrides = {};
|
|
395
|
+
const receiptLeaseTtlMs = normalizePositiveInteger(value.receiptLeaseTtlMs);
|
|
396
|
+
if (receiptLeaseTtlMs !== undefined) {
|
|
397
|
+
overrides.receiptLeaseTtlMs = receiptLeaseTtlMs;
|
|
398
|
+
}
|
|
399
|
+
const sheetAttemptLeaseMs = normalizePositiveInteger(
|
|
400
|
+
value.sheetAttemptLeaseMs,
|
|
401
|
+
);
|
|
402
|
+
if (sheetAttemptLeaseMs !== undefined) {
|
|
403
|
+
overrides.sheetAttemptLeaseMs = sheetAttemptLeaseMs;
|
|
404
|
+
}
|
|
405
|
+
const heartbeatIntervalMs = normalizePositiveInteger(
|
|
406
|
+
value.heartbeatIntervalMs,
|
|
407
|
+
);
|
|
408
|
+
if (heartbeatIntervalMs !== undefined) {
|
|
409
|
+
overrides.heartbeatIntervalMs = heartbeatIntervalMs;
|
|
410
|
+
}
|
|
411
|
+
const batchGraceMs = normalizePositiveInteger(value.batchGraceMs);
|
|
412
|
+
if (batchGraceMs !== undefined) {
|
|
413
|
+
overrides.batchGraceMs = batchGraceMs;
|
|
414
|
+
}
|
|
415
|
+
if (isRecord(value.workBudgetYieldLimits)) {
|
|
416
|
+
const limits: NonNullable<
|
|
417
|
+
RuntimeTestPolicyOverrides['workBudgetYieldLimits']
|
|
418
|
+
> = {};
|
|
419
|
+
for (const key of [
|
|
420
|
+
'elapsed',
|
|
421
|
+
'subrequest',
|
|
422
|
+
'provider',
|
|
423
|
+
'tool',
|
|
424
|
+
'receipt',
|
|
425
|
+
'sheet',
|
|
426
|
+
'log',
|
|
427
|
+
'egress',
|
|
428
|
+
] as const) {
|
|
429
|
+
const limit = normalizePositiveInteger(value.workBudgetYieldLimits[key]);
|
|
430
|
+
if (limit !== undefined) {
|
|
431
|
+
limits[key] = limit;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
if (Object.keys(limits).length > 0) {
|
|
435
|
+
overrides.workBudgetYieldLimits = limits;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
return Object.keys(overrides).length > 0 ? overrides : null;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
function runtimeReceiptLeaseTtlMs(req: RunRequest): number | undefined {
|
|
442
|
+
return req.testPolicyOverrides?.receiptLeaseTtlMs;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function runtimeSheetAttemptLeaseTtlMs(req: RunRequest): number | undefined {
|
|
446
|
+
return req.testPolicyOverrides?.sheetAttemptLeaseMs;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
function runtimePolicyHeartbeatIntervalMs(
|
|
450
|
+
req: RunRequest,
|
|
451
|
+
fallbackLeaseTtlMs: number,
|
|
452
|
+
): number {
|
|
453
|
+
return (
|
|
454
|
+
req.testPolicyOverrides?.heartbeatIntervalMs ??
|
|
455
|
+
runtimeLeaseHeartbeatIntervalMs(fallbackLeaseTtlMs)
|
|
456
|
+
);
|
|
457
|
+
}
|
|
458
|
+
|
|
342
459
|
const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
|
|
343
460
|
const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
|
|
461
|
+
const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
|
|
344
462
|
const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
|
|
463
|
+
// A short durable sleep is enough to force a Cloudflare Workflows step
|
|
464
|
+
// checkpoint. The resume re-enters run() as a new Worker invocation with a
|
|
465
|
+
// fresh per-invocation subrequest budget (see the yield callback below), so
|
|
466
|
+
// there is no need to escalate the sleep to "encourage" re-invocation.
|
|
467
|
+
// A 1ms Workflow sleep is too small to reliably create a new platform
|
|
468
|
+
// invocation in preview; the dispatcher then resets its local budget while
|
|
469
|
+
// Cloudflare's per-invocation subrequest counter is still hot. Use a real
|
|
470
|
+
// boundary so budget-yielded maps resume with fresh platform headroom.
|
|
471
|
+
const WORK_DISPATCHER_YIELD_SLEEP_MS = 1_000;
|
|
472
|
+
|
|
473
|
+
type CustomerDbDatasetRequest = {
|
|
474
|
+
limit?: number;
|
|
475
|
+
offset?: number;
|
|
476
|
+
pageSize?: number;
|
|
477
|
+
totalRows?: number;
|
|
478
|
+
};
|
|
479
|
+
|
|
480
|
+
type WorkerToolDirectOptions = {
|
|
481
|
+
customerDbDataset?: CustomerDbDatasetRequest;
|
|
482
|
+
attachCustomerDbDataset?: boolean;
|
|
483
|
+
};
|
|
484
|
+
|
|
485
|
+
function rowsFromUnknown(value: unknown): Record<string, unknown>[] {
|
|
486
|
+
if (!Array.isArray(value)) return [];
|
|
487
|
+
return value.map((row) =>
|
|
488
|
+
row && typeof row === 'object' && !Array.isArray(row)
|
|
489
|
+
? (row as Record<string, unknown>)
|
|
490
|
+
: { value: row },
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
function finiteNonNegativeInteger(value: unknown): number | null {
|
|
495
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
496
|
+
return null;
|
|
497
|
+
}
|
|
498
|
+
return Math.floor(value);
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
function finitePositiveInteger(value: unknown): number | null {
|
|
502
|
+
const integer = finiteNonNegativeInteger(value);
|
|
503
|
+
return integer !== null && integer > 0 ? integer : null;
|
|
504
|
+
}
|
|
345
505
|
|
|
346
506
|
function getStringField(value: unknown, key: string): string | null {
|
|
347
507
|
if (!isRecord(value)) return null;
|
|
@@ -376,7 +536,8 @@ type WorkerEnv = {
|
|
|
376
536
|
/**
|
|
377
537
|
* In-process Fetcher constructed by the coordinator and handed to the
|
|
378
538
|
* per-graphHash play Worker. When present, runtime callbacks
|
|
379
|
-
* (`/api/v2/
|
|
539
|
+
* (`/api/v2/internal/play-runtime`,
|
|
540
|
+
* `/api/v2/internal/play-runtime/*`,
|
|
380
541
|
* `/api/v2/plays/runtime-tools/*`) skip the public callback URL and route
|
|
381
542
|
* directly through the coordinator's process to the configured app — saves
|
|
382
543
|
* the *.workers.dev → CF edge → cloudflared → localhost chain on every
|
|
@@ -385,6 +546,7 @@ type WorkerEnv = {
|
|
|
385
546
|
*/
|
|
386
547
|
RUNTIME_API?: {
|
|
387
548
|
runtimeApiCall(input: {
|
|
549
|
+
contract?: number;
|
|
388
550
|
executorToken: string;
|
|
389
551
|
baseUrl?: string;
|
|
390
552
|
path: string;
|
|
@@ -621,7 +783,7 @@ async function fetchRuntimeApi(
|
|
|
621
783
|
? Math.max(1, Math.ceil(options.timeoutMsOverride))
|
|
622
784
|
: path === '/api/v2/plays/run'
|
|
623
785
|
? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
|
|
624
|
-
: path === '/api/v2/
|
|
786
|
+
: path === '/api/v2/internal/play-runtime/egress-fetch'
|
|
625
787
|
? RUNTIME_API_EGRESS_FETCH_TIMEOUT_MS
|
|
626
788
|
: /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
|
|
627
789
|
? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
|
|
@@ -649,6 +811,10 @@ async function fetchRuntimeApi(
|
|
|
649
811
|
);
|
|
650
812
|
}, timeoutMs);
|
|
651
813
|
});
|
|
814
|
+
const relayAbort = () => controller.abort(abortSignal?.reason);
|
|
815
|
+
if (abortSignal) {
|
|
816
|
+
abortSignal.addEventListener('abort', relayAbort, { once: true });
|
|
817
|
+
}
|
|
652
818
|
try {
|
|
653
819
|
const mergedInit: RequestInit = {
|
|
654
820
|
...init,
|
|
@@ -670,11 +836,10 @@ async function fetchRuntimeApi(
|
|
|
670
836
|
timeoutMs,
|
|
671
837
|
},
|
|
672
838
|
);
|
|
673
|
-
// RUNTIME_API service bindings do not consume RequestInit.signal once the RPC
|
|
674
|
-
// is in flight. After dispatch, wait for the owner call to settle so the
|
|
675
|
-
// durable receipt records the real provider result instead of failing early
|
|
676
|
-
// and inviting duplicate provider work on retry.
|
|
677
839
|
const response = await Promise.race([responsePromise, timeoutPromise]);
|
|
840
|
+
if (abortSignal?.aborted) {
|
|
841
|
+
throw abortError();
|
|
842
|
+
}
|
|
678
843
|
if (await isRuntimeApiBindingNotFoundResponse(response)) {
|
|
679
844
|
throw new Error(
|
|
680
845
|
`[play-harness] RUNTIME_API service binding could not route ${path}; coordinator returned not found.`,
|
|
@@ -690,6 +855,9 @@ async function fetchRuntimeApi(
|
|
|
690
855
|
}
|
|
691
856
|
throw err;
|
|
692
857
|
} finally {
|
|
858
|
+
if (abortSignal) {
|
|
859
|
+
abortSignal.removeEventListener('abort', relayAbort);
|
|
860
|
+
}
|
|
693
861
|
if (timeout) clearTimeout(timeout);
|
|
694
862
|
}
|
|
695
863
|
}
|
|
@@ -706,8 +874,23 @@ async function callRuntimeApiRpcBinding(
|
|
|
706
874
|
if (metadata) headers[EXECUTE_TOOL_METADATA_HEADER] = metadata;
|
|
707
875
|
const contract = h.get(EXECUTE_RESPONSE_CONTRACT_HEADER);
|
|
708
876
|
if (contract) headers[EXECUTE_RESPONSE_CONTRACT_HEADER] = contract;
|
|
877
|
+
const idempotencyKey = h.get('x-deepline-idempotency-key');
|
|
878
|
+
if (idempotencyKey) {
|
|
879
|
+
headers['x-deepline-idempotency-key'] = idempotencyKey;
|
|
880
|
+
}
|
|
881
|
+
const cacheControl = h.get('cache-control');
|
|
882
|
+
if (cacheControl) headers['cache-control'] = cacheControl;
|
|
883
|
+
const pragma = h.get('pragma');
|
|
884
|
+
if (pragma) headers.pragma = pragma;
|
|
885
|
+
const runtimeTestFaultHeader = h.get(PLAY_RUNTIME_TEST_FAULT_HEADER);
|
|
886
|
+
if (runtimeTestFaultHeader) {
|
|
887
|
+
headers[PLAY_RUNTIME_TEST_FAULT_HEADER] = runtimeTestFaultHeader;
|
|
888
|
+
}
|
|
889
|
+
const responseIntent = h.get(EXECUTE_RESPONSE_INTENT_HEADER);
|
|
890
|
+
if (responseIntent) headers[EXECUTE_RESPONSE_INTENT_HEADER] = responseIntent;
|
|
709
891
|
const rawBody = typeof init.body === 'string' ? init.body : '';
|
|
710
892
|
const result = await binding.runtimeApiCall({
|
|
893
|
+
contract: PLAY_RUNTIME_CONTRACT,
|
|
711
894
|
executorToken: authorization.replace(/^Bearer\s+/i, '').trim(),
|
|
712
895
|
baseUrl: input.baseUrl,
|
|
713
896
|
path: input.path,
|
|
@@ -740,6 +923,7 @@ function runtimeApiHeaders(
|
|
|
740
923
|
includeVercelBypass: boolean,
|
|
741
924
|
): Headers {
|
|
742
925
|
const next = new Headers(headers);
|
|
926
|
+
next.set(PLAY_RUNTIME_CONTRACT_HEADER, String(PLAY_RUNTIME_CONTRACT));
|
|
743
927
|
if (includeVercelBypass) {
|
|
744
928
|
const bypassToken = cachedVercelProtectionBypassToken();
|
|
745
929
|
if (bypassToken) {
|
|
@@ -932,9 +1116,172 @@ async function drainRunnerPerfTraces(req: RunRequest): Promise<void> {
|
|
|
932
1116
|
]);
|
|
933
1117
|
}
|
|
934
1118
|
|
|
1119
|
+
let workerRequestSequence = 0;
|
|
1120
|
+
|
|
935
1121
|
function makeRequestId(): string {
|
|
936
|
-
|
|
937
|
-
return
|
|
1122
|
+
workerRequestSequence = (workerRequestSequence + 1) % Number.MAX_SAFE_INTEGER;
|
|
1123
|
+
return `worker-request-${workerRequestSequence}`;
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
function makeRuntimeSheetAttempt(input: {
|
|
1127
|
+
runId: string;
|
|
1128
|
+
runAttempt?: number | null;
|
|
1129
|
+
mapName: string;
|
|
1130
|
+
chunkIndex: number | 'finalize';
|
|
1131
|
+
}): {
|
|
1132
|
+
attemptId: string;
|
|
1133
|
+
attemptOwnerRunId: string;
|
|
1134
|
+
attemptExpiresAt: string | null;
|
|
1135
|
+
attemptSeq: number;
|
|
1136
|
+
} {
|
|
1137
|
+
const runAttempt =
|
|
1138
|
+
typeof input.runAttempt === 'number' && Number.isFinite(input.runAttempt)
|
|
1139
|
+
? Math.max(0, Math.floor(input.runAttempt))
|
|
1140
|
+
: 0;
|
|
1141
|
+
const attemptKey = sha256Hex(
|
|
1142
|
+
JSON.stringify({
|
|
1143
|
+
runId: input.runId,
|
|
1144
|
+
runAttempt,
|
|
1145
|
+
mapName: input.mapName,
|
|
1146
|
+
chunkIndex: input.chunkIndex,
|
|
1147
|
+
}),
|
|
1148
|
+
).slice(0, 16);
|
|
1149
|
+
return {
|
|
1150
|
+
attemptId: `sheet:${input.mapName}:${input.chunkIndex}:attempt-${runAttempt}:${attemptKey}`,
|
|
1151
|
+
attemptOwnerRunId: input.runId,
|
|
1152
|
+
attemptExpiresAt: null,
|
|
1153
|
+
attemptSeq: runAttempt,
|
|
1154
|
+
};
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
class RuntimeSheetAttemptLeaseLostError extends Error {
|
|
1158
|
+
readonly rowKeys: string[];
|
|
1159
|
+
readonly runId: string;
|
|
1160
|
+
readonly attemptId: string;
|
|
1161
|
+
readonly tableNamespace: string;
|
|
1162
|
+
|
|
1163
|
+
constructor(input: {
|
|
1164
|
+
rowKeys: string[];
|
|
1165
|
+
runId: string;
|
|
1166
|
+
attemptId: string;
|
|
1167
|
+
tableNamespace: string;
|
|
1168
|
+
cause?: unknown;
|
|
1169
|
+
}) {
|
|
1170
|
+
const rowSummary =
|
|
1171
|
+
input.rowKeys.length === 1
|
|
1172
|
+
? input.rowKeys[0]!
|
|
1173
|
+
: `${input.rowKeys.length} rows`;
|
|
1174
|
+
super(
|
|
1175
|
+
`Runtime sheet attempt ${input.attemptId} lost ownership of ${rowSummary} in ${input.tableNamespace}.`,
|
|
1176
|
+
{ cause: input.cause },
|
|
1177
|
+
);
|
|
1178
|
+
this.name = 'RuntimeSheetAttemptLeaseLostError';
|
|
1179
|
+
this.rowKeys = input.rowKeys;
|
|
1180
|
+
this.runId = input.runId;
|
|
1181
|
+
this.attemptId = input.attemptId;
|
|
1182
|
+
this.tableNamespace = input.tableNamespace;
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
async function executeWithRuntimeSheetAttemptHeartbeat<T>(input: {
|
|
1187
|
+
req: RunRequest;
|
|
1188
|
+
tableNamespace: string;
|
|
1189
|
+
sheetContract: PlaySheetContract;
|
|
1190
|
+
attemptId?: string | null;
|
|
1191
|
+
attemptOwnerRunId?: string | null;
|
|
1192
|
+
attemptSeq?: number | null;
|
|
1193
|
+
rowKeys: string[];
|
|
1194
|
+
execute: () => Promise<T> | T;
|
|
1195
|
+
}): Promise<T> {
|
|
1196
|
+
const attemptId = input.attemptId?.trim();
|
|
1197
|
+
const rowKeys = [
|
|
1198
|
+
...new Set(input.rowKeys.map((key) => key.trim()).filter(Boolean)),
|
|
1199
|
+
];
|
|
1200
|
+
if (!attemptId || rowKeys.length === 0) {
|
|
1201
|
+
return await input.execute();
|
|
1202
|
+
}
|
|
1203
|
+
const heartbeatIntervalMs = runtimePolicyHeartbeatIntervalMs(
|
|
1204
|
+
input.req,
|
|
1205
|
+
runtimeSheetAttemptLeaseTtlMs(input.req) ??
|
|
1206
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
1207
|
+
);
|
|
1208
|
+
|
|
1209
|
+
let stopped = false;
|
|
1210
|
+
let timeout: ReturnType<typeof setTimeout> | null = null;
|
|
1211
|
+
let rejectLeaseLost!: (error: RuntimeSheetAttemptLeaseLostError) => void;
|
|
1212
|
+
const leaseLost = new Promise<never>((_, reject) => {
|
|
1213
|
+
rejectLeaseLost = reject;
|
|
1214
|
+
});
|
|
1215
|
+
|
|
1216
|
+
const stop = () => {
|
|
1217
|
+
stopped = true;
|
|
1218
|
+
if (timeout !== null) {
|
|
1219
|
+
clearTimeout(timeout);
|
|
1220
|
+
timeout = null;
|
|
1221
|
+
}
|
|
1222
|
+
};
|
|
1223
|
+
|
|
1224
|
+
const heartbeatOnce = async () => {
|
|
1225
|
+
const result = await harnessHeartbeatSheetAttempt({
|
|
1226
|
+
...runtimeSheetSessionScope(input.req),
|
|
1227
|
+
tableNamespace: input.tableNamespace,
|
|
1228
|
+
sheetContract: input.sheetContract,
|
|
1229
|
+
runId: input.req.runId,
|
|
1230
|
+
attemptId,
|
|
1231
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? input.req.runId,
|
|
1232
|
+
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
1233
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
|
|
1234
|
+
keys: rowKeys,
|
|
1235
|
+
});
|
|
1236
|
+
const renewed = new Set(result.renewedKeys);
|
|
1237
|
+
const lostKeys = rowKeys.filter((key) => !renewed.has(key));
|
|
1238
|
+
if (lostKeys.length > 0) {
|
|
1239
|
+
throw new RuntimeSheetAttemptLeaseLostError({
|
|
1240
|
+
rowKeys: lostKeys,
|
|
1241
|
+
runId: input.req.runId,
|
|
1242
|
+
attemptId,
|
|
1243
|
+
tableNamespace: input.tableNamespace,
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
1246
|
+
};
|
|
1247
|
+
|
|
1248
|
+
const schedule = () => {
|
|
1249
|
+
timeout = setTimeout(() => {
|
|
1250
|
+
void (async () => {
|
|
1251
|
+
try {
|
|
1252
|
+
await heartbeatOnce();
|
|
1253
|
+
} catch (error) {
|
|
1254
|
+
stop();
|
|
1255
|
+
if (error instanceof RuntimeSheetAttemptLeaseLostError) {
|
|
1256
|
+
rejectLeaseLost(error);
|
|
1257
|
+
} else {
|
|
1258
|
+
rejectLeaseLost(
|
|
1259
|
+
new RuntimeSheetAttemptLeaseLostError({
|
|
1260
|
+
rowKeys,
|
|
1261
|
+
runId: input.req.runId,
|
|
1262
|
+
attemptId,
|
|
1263
|
+
tableNamespace: input.tableNamespace,
|
|
1264
|
+
cause: error,
|
|
1265
|
+
}),
|
|
1266
|
+
);
|
|
1267
|
+
}
|
|
1268
|
+
return;
|
|
1269
|
+
}
|
|
1270
|
+
if (!stopped) {
|
|
1271
|
+
schedule();
|
|
1272
|
+
}
|
|
1273
|
+
})();
|
|
1274
|
+
}, heartbeatIntervalMs);
|
|
1275
|
+
};
|
|
1276
|
+
|
|
1277
|
+
schedule();
|
|
1278
|
+
const execution = Promise.resolve().then(input.execute);
|
|
1279
|
+
execution.catch(() => {});
|
|
1280
|
+
try {
|
|
1281
|
+
return await Promise.race([execution, leaseLost]);
|
|
1282
|
+
} finally {
|
|
1283
|
+
stop();
|
|
1284
|
+
}
|
|
938
1285
|
}
|
|
939
1286
|
|
|
940
1287
|
/**
|
|
@@ -960,6 +1307,11 @@ async function postRuntimeApi<T>(
|
|
|
960
1307
|
baseUrl: string,
|
|
961
1308
|
executorToken: string,
|
|
962
1309
|
body: unknown,
|
|
1310
|
+
options: {
|
|
1311
|
+
runtimeTestFaultHeader?: string | null;
|
|
1312
|
+
timeoutMsOverride?: number;
|
|
1313
|
+
timeoutErrorMessage?: string;
|
|
1314
|
+
} = {},
|
|
963
1315
|
): Promise<T> {
|
|
964
1316
|
// Routes through the in-process RUNTIME_API service binding. Missing binding
|
|
965
1317
|
// is an infra error in workers_edge, not a reason to fall back to public HTTP.
|
|
@@ -972,15 +1324,29 @@ async function postRuntimeApi<T>(
|
|
|
972
1324
|
) {
|
|
973
1325
|
let res: Response;
|
|
974
1326
|
try {
|
|
975
|
-
res = await fetchRuntimeApi(
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
1327
|
+
res = await fetchRuntimeApi(
|
|
1328
|
+
baseUrl,
|
|
1329
|
+
'/api/v2/internal/play-runtime',
|
|
1330
|
+
{
|
|
1331
|
+
method: 'POST',
|
|
1332
|
+
headers: {
|
|
1333
|
+
'content-type': 'application/json',
|
|
1334
|
+
authorization: `Bearer ${executorToken}`,
|
|
1335
|
+
'x-deepline-request-id': makeRequestId(),
|
|
1336
|
+
...(options.runtimeTestFaultHeader?.trim()
|
|
1337
|
+
? {
|
|
1338
|
+
[PLAY_RUNTIME_TEST_FAULT_HEADER]:
|
|
1339
|
+
options.runtimeTestFaultHeader.trim(),
|
|
1340
|
+
}
|
|
1341
|
+
: {}),
|
|
1342
|
+
},
|
|
1343
|
+
body: serializedBody,
|
|
981
1344
|
},
|
|
982
|
-
|
|
983
|
-
|
|
1345
|
+
{
|
|
1346
|
+
timeoutMsOverride: options.timeoutMsOverride,
|
|
1347
|
+
timeoutErrorMessage: options.timeoutErrorMessage,
|
|
1348
|
+
},
|
|
1349
|
+
);
|
|
984
1350
|
} catch (error) {
|
|
985
1351
|
lastError = error;
|
|
986
1352
|
if (
|
|
@@ -1064,9 +1430,10 @@ async function postRuntimeApiBestEffort(
|
|
|
1064
1430
|
baseUrl: string,
|
|
1065
1431
|
executorToken: string,
|
|
1066
1432
|
body: unknown,
|
|
1433
|
+
options: { runtimeTestFaultHeader?: string | null } = {},
|
|
1067
1434
|
): Promise<boolean> {
|
|
1068
1435
|
try {
|
|
1069
|
-
await postRuntimeApi(baseUrl, executorToken, body);
|
|
1436
|
+
await postRuntimeApi(baseUrl, executorToken, body, options);
|
|
1070
1437
|
return true;
|
|
1071
1438
|
} catch (error) {
|
|
1072
1439
|
console.error(
|
|
@@ -1078,6 +1445,43 @@ async function postRuntimeApiBestEffort(
|
|
|
1078
1445
|
}
|
|
1079
1446
|
}
|
|
1080
1447
|
|
|
1448
|
+
async function readDurableChildRunTerminalState(input: {
|
|
1449
|
+
baseUrl: string;
|
|
1450
|
+
executorToken: string;
|
|
1451
|
+
parentRunId: string;
|
|
1452
|
+
childRunId: string;
|
|
1453
|
+
childPlayName: string;
|
|
1454
|
+
timeoutMs?: number;
|
|
1455
|
+
}): Promise<{ data?: unknown } | null> {
|
|
1456
|
+
const response = await postRuntimeApi<{
|
|
1457
|
+
state?: {
|
|
1458
|
+
data?: unknown;
|
|
1459
|
+
reason?: string;
|
|
1460
|
+
status?: string;
|
|
1461
|
+
runId?: string;
|
|
1462
|
+
playName?: string;
|
|
1463
|
+
parentRunId?: string | null;
|
|
1464
|
+
} | null;
|
|
1465
|
+
}>(
|
|
1466
|
+
input.baseUrl,
|
|
1467
|
+
input.executorToken,
|
|
1468
|
+
{
|
|
1469
|
+
action: 'read_child_run_terminal_snapshot',
|
|
1470
|
+
parentRunId: input.parentRunId,
|
|
1471
|
+
childRunId: input.childRunId,
|
|
1472
|
+
childPlayName: input.childPlayName,
|
|
1473
|
+
},
|
|
1474
|
+
{
|
|
1475
|
+
timeoutMsOverride: input.timeoutMs,
|
|
1476
|
+
timeoutErrorMessage: `[play-harness] child run terminal snapshot read timed out after ${Math.max(
|
|
1477
|
+
1,
|
|
1478
|
+
Math.ceil(input.timeoutMs ?? RUNTIME_API_TIMEOUT_MS),
|
|
1479
|
+
)}ms. childRunId=${input.childRunId}`,
|
|
1480
|
+
},
|
|
1481
|
+
);
|
|
1482
|
+
return response.state ?? null;
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1081
1485
|
function workflowEventType(name: string): string {
|
|
1082
1486
|
const normalized = name
|
|
1083
1487
|
.trim()
|
|
@@ -1187,6 +1591,27 @@ async function hashChildPlayEventKey(input: unknown): Promise<string> {
|
|
|
1187
1591
|
return (await hashJson(input)).slice(0, 32);
|
|
1188
1592
|
}
|
|
1189
1593
|
|
|
1594
|
+
function workerChildRevisionFingerprint(input: {
|
|
1595
|
+
playName: string;
|
|
1596
|
+
manifest: PlayRuntimeManifest | null | undefined;
|
|
1597
|
+
}): string | null {
|
|
1598
|
+
const manifest = input.manifest;
|
|
1599
|
+
if (!manifest) return null;
|
|
1600
|
+
return sha256Hex(
|
|
1601
|
+
stableStringify({
|
|
1602
|
+
playId: manifest.playName?.trim() || input.playName,
|
|
1603
|
+
codeFormat: 'cjs_module',
|
|
1604
|
+
artifactHash: manifest.artifactHash ?? null,
|
|
1605
|
+
graphHash: manifest.graphHash ?? null,
|
|
1606
|
+
sourceHash: null,
|
|
1607
|
+
sourceCodeHash:
|
|
1608
|
+
typeof manifest.sourceCode === 'string'
|
|
1609
|
+
? sha256Hex(manifest.sourceCode)
|
|
1610
|
+
: null,
|
|
1611
|
+
}),
|
|
1612
|
+
);
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1190
1615
|
async function childPlayEventKey(input: {
|
|
1191
1616
|
key: string;
|
|
1192
1617
|
workflowId: string;
|
|
@@ -1283,7 +1708,14 @@ async function signalParentPlayTerminal(input: {
|
|
|
1283
1708
|
|
|
1284
1709
|
async function executeTool(
|
|
1285
1710
|
req: RunRequest,
|
|
1286
|
-
args: {
|
|
1711
|
+
args: {
|
|
1712
|
+
id: string;
|
|
1713
|
+
toolId: string;
|
|
1714
|
+
input: Record<string, unknown>;
|
|
1715
|
+
durableCallReceiptKey?: string | null;
|
|
1716
|
+
executionAuthScopeDigest?: string | null;
|
|
1717
|
+
providerIdempotencyKey?: string | null;
|
|
1718
|
+
},
|
|
1287
1719
|
workflowStep?: WorkflowStep,
|
|
1288
1720
|
onProviderBackpressure?: (retryAfterMs: number) => void,
|
|
1289
1721
|
onRetryAttempt?: () => void,
|
|
@@ -1291,6 +1723,7 @@ async function executeTool(
|
|
|
1291
1723
|
abortSignal?: AbortSignal,
|
|
1292
1724
|
runtimeDeadlineMs?: number,
|
|
1293
1725
|
runtimeTimeoutMs?: number,
|
|
1726
|
+
directOptions?: WorkerToolDirectOptions,
|
|
1294
1727
|
): Promise<ToolExecuteResult> {
|
|
1295
1728
|
if (args.toolId === 'test_wait_for_event' && workflowStep) {
|
|
1296
1729
|
const result = await waitForSyntheticIntegrationEvent(
|
|
@@ -1314,12 +1747,20 @@ async function executeTool(
|
|
|
1314
1747
|
abortSignal,
|
|
1315
1748
|
runtimeDeadlineMs,
|
|
1316
1749
|
runtimeTimeoutMs,
|
|
1750
|
+
directOptions,
|
|
1317
1751
|
);
|
|
1318
1752
|
}
|
|
1319
1753
|
|
|
1320
1754
|
async function executeToolWithLifecycle(
|
|
1321
1755
|
req: RunRequest,
|
|
1322
|
-
args: {
|
|
1756
|
+
args: {
|
|
1757
|
+
id: string;
|
|
1758
|
+
toolId: string;
|
|
1759
|
+
input: Record<string, unknown>;
|
|
1760
|
+
durableCallReceiptKey?: string | null;
|
|
1761
|
+
executionAuthScopeDigest?: string | null;
|
|
1762
|
+
providerIdempotencyKey?: string | null;
|
|
1763
|
+
},
|
|
1323
1764
|
workflowStep: WorkflowStep | undefined,
|
|
1324
1765
|
callbacks: WorkerCtxCallbacks | undefined,
|
|
1325
1766
|
onProviderBackpressure?: (retryAfterMs: number) => void,
|
|
@@ -1328,6 +1769,7 @@ async function executeToolWithLifecycle(
|
|
|
1328
1769
|
abortSignal?: AbortSignal,
|
|
1329
1770
|
runtimeDeadlineMs?: number,
|
|
1330
1771
|
runtimeTimeoutMs?: number,
|
|
1772
|
+
directOptions?: WorkerToolDirectOptions,
|
|
1331
1773
|
): Promise<ToolExecuteResult> {
|
|
1332
1774
|
callbacks?.onToolCalled?.(args.toolId, nowMs());
|
|
1333
1775
|
try {
|
|
@@ -1341,6 +1783,7 @@ async function executeToolWithLifecycle(
|
|
|
1341
1783
|
abortSignal,
|
|
1342
1784
|
runtimeDeadlineMs,
|
|
1343
1785
|
runtimeTimeoutMs,
|
|
1786
|
+
directOptions,
|
|
1344
1787
|
);
|
|
1345
1788
|
} catch (error) {
|
|
1346
1789
|
callbacks?.onToolFailed?.(args.toolId, nowMs());
|
|
@@ -1413,24 +1856,29 @@ async function waitForSyntheticIntegrationEvent(
|
|
|
1413
1856
|
typeof input.timeout_ms === 'number' && Number.isFinite(input.timeout_ms)
|
|
1414
1857
|
? Math.max(1, Math.round(input.timeout_ms))
|
|
1415
1858
|
: 30_000;
|
|
1416
|
-
await postRuntimeApiBestEffort(
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1859
|
+
await postRuntimeApiBestEffort(
|
|
1860
|
+
req.baseUrl,
|
|
1861
|
+
req.executorToken,
|
|
1862
|
+
{
|
|
1863
|
+
action: 'append_run_events',
|
|
1864
|
+
playId: req.runId,
|
|
1865
|
+
events: [
|
|
1866
|
+
{
|
|
1867
|
+
type: 'log.appended',
|
|
1868
|
+
runId: req.runId,
|
|
1869
|
+
// 'system' (windowed text-dedupe channel), NOT 'worker': this line is
|
|
1870
|
+
// emitted outside the harness log buffer, so it has no positional
|
|
1871
|
+
// channelOffset and must not pollute the worker channel cursor.
|
|
1872
|
+
source: 'system',
|
|
1873
|
+
occurredAt: nowMs(),
|
|
1874
|
+
lines: [
|
|
1875
|
+
`Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
|
|
1876
|
+
],
|
|
1877
|
+
} satisfies PlayRunLedgerEvent,
|
|
1878
|
+
],
|
|
1879
|
+
},
|
|
1880
|
+
{ runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null },
|
|
1881
|
+
);
|
|
1434
1882
|
try {
|
|
1435
1883
|
const event = (await (
|
|
1436
1884
|
workflowStep.waitForEvent as unknown as (
|
|
@@ -1478,7 +1926,14 @@ async function waitForSyntheticIntegrationEvent(
|
|
|
1478
1926
|
|
|
1479
1927
|
async function callToolDirect(
|
|
1480
1928
|
req: RunRequest,
|
|
1481
|
-
args: {
|
|
1929
|
+
args: {
|
|
1930
|
+
id: string;
|
|
1931
|
+
toolId: string;
|
|
1932
|
+
input: Record<string, unknown>;
|
|
1933
|
+
durableCallReceiptKey?: string | null;
|
|
1934
|
+
executionAuthScopeDigest?: string | null;
|
|
1935
|
+
providerIdempotencyKey?: string | null;
|
|
1936
|
+
},
|
|
1482
1937
|
onProviderBackpressure?: (retryAfterMs: number) => void,
|
|
1483
1938
|
// Invoked once per in-process retry attempt (429 / retryable 5xx / synthetic
|
|
1484
1939
|
// transient) so the Governor charges chargeBudget('retry') per attempt — the
|
|
@@ -1490,9 +1945,20 @@ async function callToolDirect(
|
|
|
1490
1945
|
abortSignal?: AbortSignal,
|
|
1491
1946
|
runtimeDeadlineMs?: number,
|
|
1492
1947
|
runtimeTimeoutMs?: number,
|
|
1948
|
+
directOptions?: WorkerToolDirectOptions,
|
|
1493
1949
|
): Promise<ToolExecuteResult> {
|
|
1494
1950
|
const { id, toolId, input } = args;
|
|
1495
1951
|
const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
|
|
1952
|
+
const durableCallReceiptKey = args.durableCallReceiptKey?.trim() || null;
|
|
1953
|
+
const executionAuthScopeDigest =
|
|
1954
|
+
durableCallReceiptKey && args.executionAuthScopeDigest
|
|
1955
|
+
? args.executionAuthScopeDigest.trim() || null
|
|
1956
|
+
: null;
|
|
1957
|
+
const providerIdempotencyKey =
|
|
1958
|
+
args.providerIdempotencyKey?.trim() || durableCallReceiptKey;
|
|
1959
|
+
const deeplineRequestId = providerIdempotencyKey
|
|
1960
|
+
? `ctx-tool-${sha256Hex(providerIdempotencyKey).slice(0, 32)}`
|
|
1961
|
+
: null;
|
|
1496
1962
|
let lastError: Error | null = null;
|
|
1497
1963
|
const httpFailureAttempts = createToolExecuteHttpFailureAttemptTracker();
|
|
1498
1964
|
let requestAttempt = 0;
|
|
@@ -1515,13 +1981,66 @@ async function callToolDirect(
|
|
|
1515
1981
|
headers: {
|
|
1516
1982
|
'content-type': 'application/json',
|
|
1517
1983
|
authorization: `Bearer ${req.executorToken}`,
|
|
1518
|
-
'x-deepline-request-id':
|
|
1984
|
+
'x-deepline-request-id':
|
|
1985
|
+
deeplineRequestId ??
|
|
1986
|
+
`${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
|
|
1987
|
+
...(providerIdempotencyKey
|
|
1988
|
+
? { 'x-deepline-idempotency-key': providerIdempotencyKey }
|
|
1989
|
+
: {}),
|
|
1519
1990
|
[EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
|
|
1991
|
+
[EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
|
|
1520
1992
|
[EXECUTE_TOOL_METADATA_HEADER]: 'true',
|
|
1521
1993
|
},
|
|
1522
1994
|
body: JSON.stringify({
|
|
1523
1995
|
payload: input,
|
|
1524
|
-
metadata: {
|
|
1996
|
+
metadata: {
|
|
1997
|
+
parent_run_id: req.runId,
|
|
1998
|
+
...(durableCallReceiptKey
|
|
1999
|
+
? {
|
|
2000
|
+
durable_call_receipt_key: durableCallReceiptKey,
|
|
2001
|
+
...(executionAuthScopeDigest
|
|
2002
|
+
? {
|
|
2003
|
+
execution_auth_scope_digest: executionAuthScopeDigest,
|
|
2004
|
+
}
|
|
2005
|
+
: {}),
|
|
2006
|
+
}
|
|
2007
|
+
: {}),
|
|
2008
|
+
...(directOptions?.customerDbDataset
|
|
2009
|
+
? {
|
|
2010
|
+
query_result_dataset: {
|
|
2011
|
+
limit: directOptions.customerDbDataset.limit,
|
|
2012
|
+
offset: directOptions.customerDbDataset.offset,
|
|
2013
|
+
page_size: directOptions.customerDbDataset.pageSize,
|
|
2014
|
+
...(directOptions.customerDbDataset.totalRows !==
|
|
2015
|
+
undefined
|
|
2016
|
+
? {
|
|
2017
|
+
total_rows:
|
|
2018
|
+
directOptions.customerDbDataset.totalRows,
|
|
2019
|
+
}
|
|
2020
|
+
: {}),
|
|
2021
|
+
},
|
|
2022
|
+
...(isCustomerDbDatasetTool(toolId)
|
|
2023
|
+
? {
|
|
2024
|
+
customer_db_dataset: {
|
|
2025
|
+
limit: directOptions.customerDbDataset.limit,
|
|
2026
|
+
offset: directOptions.customerDbDataset.offset,
|
|
2027
|
+
page_size: directOptions.customerDbDataset.pageSize,
|
|
2028
|
+
...(directOptions.customerDbDataset.totalRows !==
|
|
2029
|
+
undefined
|
|
2030
|
+
? {
|
|
2031
|
+
total_rows:
|
|
2032
|
+
directOptions.customerDbDataset.totalRows,
|
|
2033
|
+
}
|
|
2034
|
+
: {}),
|
|
2035
|
+
},
|
|
2036
|
+
}
|
|
2037
|
+
: {}),
|
|
2038
|
+
}
|
|
2039
|
+
: {}),
|
|
2040
|
+
...(providerIdempotencyKey
|
|
2041
|
+
? { provider_idempotency_key: providerIdempotencyKey }
|
|
2042
|
+
: {}),
|
|
2043
|
+
},
|
|
1525
2044
|
...(req.integrationMode
|
|
1526
2045
|
? { integration_mode: req.integrationMode }
|
|
1527
2046
|
: {}),
|
|
@@ -1561,10 +2080,20 @@ async function callToolDirect(
|
|
|
1561
2080
|
parsed.result,
|
|
1562
2081
|
parsed.metadata,
|
|
1563
2082
|
parsed.status,
|
|
2083
|
+
directOptions?.attachCustomerDbDataset === false
|
|
2084
|
+
? undefined
|
|
2085
|
+
: { req, input },
|
|
1564
2086
|
);
|
|
1565
2087
|
}
|
|
1566
2088
|
|
|
1567
2089
|
const text = await res.text().catch(() => '');
|
|
2090
|
+
const authScopeChanged = parseToolExecuteAuthScopeChangedError({
|
|
2091
|
+
status: res.status,
|
|
2092
|
+
bodyText: text,
|
|
2093
|
+
});
|
|
2094
|
+
if (authScopeChanged) {
|
|
2095
|
+
throw authScopeChanged;
|
|
2096
|
+
}
|
|
1568
2097
|
const httpFailureAttempt = httpFailureAttempts.next({
|
|
1569
2098
|
toolId,
|
|
1570
2099
|
status: res.status,
|
|
@@ -1598,60 +2127,146 @@ async function callToolDirect(
|
|
|
1598
2127
|
throw lastError ?? new Error(`tool ${toolId} failed before execution.`);
|
|
1599
2128
|
}
|
|
1600
2129
|
|
|
1601
|
-
function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
|
|
1602
|
-
if (toolId === 'test_rate_limit') {
|
|
1603
|
-
// Batched members resolve metadata through this fallback because the lean
|
|
1604
|
-
// worker does not bundle the catalog. It MUST mirror the same
|
|
1605
|
-
// `email_status: emailStatus({...})` contract registered in
|
|
1606
|
-
// src/lib/integrations/test/index.ts so batched results normalize to the
|
|
1607
|
-
// same rich email_status OBJECT (status from statusMap, catch_all as a
|
|
1608
|
-
// signal) that the single-execute real-metadata path produces. The legacy
|
|
1609
|
-
// `transforms:['emailStatus']` + mx_security_gateway→'catch_all' string
|
|
1610
|
-
// override coarsened email_status into a bare string and predates the
|
|
1611
|
-
// emailStatus object contract (#1466).
|
|
1612
|
-
return {
|
|
1613
|
-
toolId,
|
|
1614
|
-
extractors: {
|
|
1615
|
-
email_status: {
|
|
1616
|
-
paths: ['email_status'],
|
|
1617
|
-
emailStatus: {
|
|
1618
|
-
provider: 'test',
|
|
1619
|
-
rawStatus: ['email_status'],
|
|
1620
|
-
catchAll: ['mx_security_gateway'],
|
|
1621
|
-
statusMap: {
|
|
1622
|
-
valid: { status: 'valid', verdict: 'send', verified: true },
|
|
1623
|
-
invalid: { status: 'invalid', verdict: 'drop', verified: false },
|
|
1624
|
-
catch_all: {
|
|
1625
|
-
status: 'catch_all',
|
|
1626
|
-
verdict: 'verify_next',
|
|
1627
|
-
verified: false,
|
|
1628
|
-
},
|
|
1629
|
-
unknown: { status: 'unknown', verdict: 'hold', verified: false },
|
|
1630
|
-
},
|
|
1631
|
-
},
|
|
1632
|
-
},
|
|
1633
|
-
},
|
|
1634
|
-
targetGetters: {
|
|
1635
|
-
email: ['email', 'value'],
|
|
1636
|
-
email_status: ['email_status'],
|
|
1637
|
-
},
|
|
1638
|
-
};
|
|
1639
|
-
}
|
|
1640
|
-
return { toolId };
|
|
1641
|
-
}
|
|
1642
|
-
|
|
1643
2130
|
function wrapWorkerToolResult(
|
|
1644
2131
|
toolId: string,
|
|
1645
2132
|
result: unknown,
|
|
1646
2133
|
metadata: ToolResultMetadataInput | null,
|
|
1647
2134
|
status = result == null ? 'no_result' : 'completed',
|
|
2135
|
+
customerDbDatasetOptions?: {
|
|
2136
|
+
req: RunRequest;
|
|
2137
|
+
input: Record<string, unknown>;
|
|
2138
|
+
},
|
|
1648
2139
|
): ToolExecuteResult {
|
|
1649
|
-
|
|
2140
|
+
const wrapped = createToolExecuteResult({
|
|
1650
2141
|
status,
|
|
1651
2142
|
result,
|
|
1652
2143
|
metadata: metadata ?? { toolId },
|
|
1653
2144
|
execution: toolExecutionMetadataForOutcome({ kind: 'live' }),
|
|
1654
2145
|
});
|
|
2146
|
+
return attachWorkerCustomerDbDatasetResult(
|
|
2147
|
+
toolId,
|
|
2148
|
+
wrapped,
|
|
2149
|
+
customerDbDatasetOptions,
|
|
2150
|
+
);
|
|
2151
|
+
}
|
|
2152
|
+
|
|
2153
|
+
function attachWorkerCustomerDbDatasetResult(
|
|
2154
|
+
toolId: string,
|
|
2155
|
+
result: ToolExecuteResult,
|
|
2156
|
+
options?: {
|
|
2157
|
+
req: RunRequest;
|
|
2158
|
+
input: Record<string, unknown>;
|
|
2159
|
+
},
|
|
2160
|
+
): ToolExecuteResult {
|
|
2161
|
+
if (!options || !isQueryResultDatasetTool(toolId)) return result;
|
|
2162
|
+
const raw = isRecord(result.toolOutput.raw) ? result.toolOutput.raw : null;
|
|
2163
|
+
const dataset = isRecord(raw?.dataset) ? raw.dataset : null;
|
|
2164
|
+
const rows = rowsFromUnknown(raw?.rows);
|
|
2165
|
+
const totalRows = finiteNonNegativeInteger(dataset?.total_rows);
|
|
2166
|
+
const sql =
|
|
2167
|
+
typeof options.input.sql === 'string'
|
|
2168
|
+
? options.input.sql
|
|
2169
|
+
: typeof options.input.query === 'string'
|
|
2170
|
+
? options.input.query
|
|
2171
|
+
: typeof raw?.sql === 'string'
|
|
2172
|
+
? raw.sql
|
|
2173
|
+
: null;
|
|
2174
|
+
if (!dataset || totalRows === null || !sql) {
|
|
2175
|
+
return result;
|
|
2176
|
+
}
|
|
2177
|
+
|
|
2178
|
+
const datasetLimit =
|
|
2179
|
+
finitePositiveInteger(dataset.returned_limit) ?? totalRows;
|
|
2180
|
+
const previewRows = rows.slice(0, Math.min(rows.length, 25));
|
|
2181
|
+
const jobId = typeof result.job_id === 'string' ? result.job_id.trim() : '';
|
|
2182
|
+
const datasetId = `worker-tool-list:${sha256Hex(
|
|
2183
|
+
`${options.req.runId}:${jobId || Math.random()}`,
|
|
2184
|
+
)}`;
|
|
2185
|
+
const fetchPage = async (
|
|
2186
|
+
offset: number,
|
|
2187
|
+
limit: number,
|
|
2188
|
+
): Promise<Record<string, unknown>[]> => {
|
|
2189
|
+
if (limit <= 0 || offset >= totalRows) return [];
|
|
2190
|
+
const page = await callToolDirect(
|
|
2191
|
+
options.req,
|
|
2192
|
+
{
|
|
2193
|
+
id: 'customer-db-dataset-page',
|
|
2194
|
+
toolId,
|
|
2195
|
+
input: options.input,
|
|
2196
|
+
},
|
|
2197
|
+
undefined,
|
|
2198
|
+
undefined,
|
|
2199
|
+
false,
|
|
2200
|
+
undefined,
|
|
2201
|
+
undefined,
|
|
2202
|
+
undefined,
|
|
2203
|
+
{
|
|
2204
|
+
attachCustomerDbDataset: false,
|
|
2205
|
+
customerDbDataset: {
|
|
2206
|
+
limit: datasetLimit,
|
|
2207
|
+
offset,
|
|
2208
|
+
pageSize: Math.min(limit, QUERY_RESULT_DATASET_PAGE_SIZE),
|
|
2209
|
+
totalRows,
|
|
2210
|
+
},
|
|
2211
|
+
},
|
|
2212
|
+
);
|
|
2213
|
+
const pageRaw = isRecord(page.toolOutput.raw) ? page.toolOutput.raw : null;
|
|
2214
|
+
return rowsFromUnknown(pageRaw?.rows);
|
|
2215
|
+
};
|
|
2216
|
+
const collectRows = async (
|
|
2217
|
+
limit: number | undefined,
|
|
2218
|
+
): Promise<Record<string, unknown>[]> => {
|
|
2219
|
+
const target = Math.min(limit ?? totalRows, totalRows, datasetLimit);
|
|
2220
|
+
const collected: Record<string, unknown>[] = [];
|
|
2221
|
+
for (
|
|
2222
|
+
let offset = 0;
|
|
2223
|
+
offset < target;
|
|
2224
|
+
offset += QUERY_RESULT_DATASET_PAGE_SIZE
|
|
2225
|
+
) {
|
|
2226
|
+
collected.push(
|
|
2227
|
+
...(await fetchPage(
|
|
2228
|
+
offset,
|
|
2229
|
+
Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, target - offset),
|
|
2230
|
+
)),
|
|
2231
|
+
);
|
|
2232
|
+
}
|
|
2233
|
+
return collected.slice(0, target);
|
|
2234
|
+
};
|
|
2235
|
+
const playDataset = createDeferredPlayDataset({
|
|
2236
|
+
datasetKind: 'csv',
|
|
2237
|
+
datasetId,
|
|
2238
|
+
count: Math.min(totalRows, datasetLimit),
|
|
2239
|
+
previewRows,
|
|
2240
|
+
sourceLabel: 'toolResponse.raw.rows',
|
|
2241
|
+
tableNamespace: null,
|
|
2242
|
+
resolvers: {
|
|
2243
|
+
count: async () => Math.min(totalRows, datasetLimit),
|
|
2244
|
+
peek: async (limit) => collectRows(limit),
|
|
2245
|
+
materialize: async (limit) => collectRows(limit),
|
|
2246
|
+
iterate: () =>
|
|
2247
|
+
({
|
|
2248
|
+
async *[Symbol.asyncIterator]() {
|
|
2249
|
+
const count = Math.min(totalRows, datasetLimit);
|
|
2250
|
+
for (
|
|
2251
|
+
let offset = 0;
|
|
2252
|
+
offset < count;
|
|
2253
|
+
offset += QUERY_RESULT_DATASET_PAGE_SIZE
|
|
2254
|
+
) {
|
|
2255
|
+
yield* await fetchPage(
|
|
2256
|
+
offset,
|
|
2257
|
+
Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, count - offset),
|
|
2258
|
+
);
|
|
2259
|
+
}
|
|
2260
|
+
},
|
|
2261
|
+
}) as AsyncIterable<Record<string, unknown>>,
|
|
2262
|
+
},
|
|
2263
|
+
});
|
|
2264
|
+
return attachToolResultListDataset(result, {
|
|
2265
|
+
name: 'rows',
|
|
2266
|
+
path: 'toolResponse.raw.rows',
|
|
2267
|
+
dataset: playDataset,
|
|
2268
|
+
count: Math.min(totalRows, datasetLimit),
|
|
2269
|
+
});
|
|
1655
2270
|
}
|
|
1656
2271
|
|
|
1657
2272
|
type RecordedStepProgramOutput = {
|
|
@@ -1675,46 +2290,10 @@ type WorkerStepResolution = {
|
|
|
1675
2290
|
status?: 'skipped';
|
|
1676
2291
|
};
|
|
1677
2292
|
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
cacheKey: string;
|
|
1681
|
-
receiptKey: string | null;
|
|
1682
|
-
force: boolean;
|
|
1683
|
-
receiptWaitMaxAttempts: number;
|
|
1684
|
-
runtimeTimeoutMs?: number;
|
|
1685
|
-
toolId: string;
|
|
1686
|
-
input: Record<string, unknown>;
|
|
1687
|
-
workflowStep?: WorkflowStep;
|
|
1688
|
-
resolve: (value: unknown) => void;
|
|
1689
|
-
reject: (error: unknown) => void;
|
|
1690
|
-
};
|
|
1691
|
-
|
|
1692
|
-
type ClaimedWorkerToolBatchRequest = {
|
|
1693
|
-
request: WorkerToolBatchRequest;
|
|
1694
|
-
receiptKey: string | null;
|
|
1695
|
-
followers: WorkerToolBatchRequest[];
|
|
1696
|
-
};
|
|
1697
|
-
|
|
1698
|
-
type PreparedWorkerToolBatchRequests = {
|
|
1699
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[];
|
|
1700
|
-
deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[];
|
|
1701
|
-
};
|
|
1702
|
-
|
|
1703
|
-
function resolveClaimedWorkerToolRuntimeTimeoutMs(
|
|
1704
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
1705
|
-
input: { runtimeDeadlineMs?: number },
|
|
1706
|
-
): number | undefined {
|
|
1707
|
-
return resolveWorkerToolRuntimeTimeoutMs(claimedRequests, {
|
|
1708
|
-
resolveOwnerTimeoutMs: (request) =>
|
|
1709
|
-
resolveToolRuntimeApiTimeout({
|
|
1710
|
-
requestInput: request.input,
|
|
1711
|
-
timeoutMs: request.runtimeTimeoutMs,
|
|
1712
|
-
runtimeDeadlineMs: input.runtimeDeadlineMs,
|
|
1713
|
-
}).timeoutMs,
|
|
1714
|
-
});
|
|
2293
|
+
function stepProgramColumnName(parentField: string, stepId: string): string {
|
|
2294
|
+
return sqlSafePlayColumnName(`${parentField}.${stepId}`);
|
|
1715
2295
|
}
|
|
1716
2296
|
|
|
1717
|
-
const WORKER_TOOL_BATCH_GRACE_MS = 250;
|
|
1718
2297
|
const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
|
|
1719
2298
|
const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
|
|
1720
2299
|
const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
|
|
@@ -1727,1371 +2306,67 @@ const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
|
|
|
1727
2306
|
*/
|
|
1728
2307
|
const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
|
|
1729
2308
|
|
|
1730
|
-
class RuntimeReceiptPersistenceError extends Error {
|
|
1731
|
-
readonly errors: unknown[];
|
|
1732
|
-
|
|
1733
|
-
constructor(message: string, errors: unknown[] = []) {
|
|
1734
|
-
super(message);
|
|
1735
|
-
this.name = 'RuntimeReceiptPersistenceError';
|
|
1736
|
-
this.errors = errors;
|
|
1737
|
-
}
|
|
1738
|
-
}
|
|
1739
|
-
|
|
1740
|
-
function isRuntimeReceiptPersistenceError(error: unknown): boolean {
|
|
1741
|
-
if (!error || typeof error !== 'object') return false;
|
|
1742
|
-
if (error instanceof RuntimeReceiptPersistenceError) return true;
|
|
1743
|
-
if (
|
|
1744
|
-
error instanceof Error &&
|
|
1745
|
-
error.name === 'RuntimeReceiptPersistenceError'
|
|
1746
|
-
) {
|
|
1747
|
-
return true;
|
|
1748
|
-
}
|
|
1749
|
-
const nestedErrors = (error as { errors?: unknown }).errors;
|
|
1750
|
-
return (
|
|
1751
|
-
Array.isArray(nestedErrors) &&
|
|
1752
|
-
nestedErrors.some(isRuntimeReceiptPersistenceError)
|
|
1753
|
-
);
|
|
1754
|
-
}
|
|
1755
|
-
|
|
1756
|
-
function isRunFatalWorkerRowError(error: unknown): boolean {
|
|
1757
|
-
return (
|
|
1758
|
-
isCoordinatorProgressAuthorizationError(error) ||
|
|
1759
|
-
isRowIsolationExemptError(error) ||
|
|
1760
|
-
isRuntimeReceiptPersistenceError(error) ||
|
|
1761
|
-
isRuntimePersistenceCircuitOpenError(error)
|
|
1762
|
-
);
|
|
1763
|
-
}
|
|
1764
|
-
|
|
1765
|
-
/**
|
|
1766
|
-
* completeReceipt salvage retry ladder. On a persistence failure after a billed
|
|
1767
|
-
* call we retry the receipt completion past the server-side 5s connect window:
|
|
1768
|
-
* 1 initial attempt + 3 retries at 1s / 3s / 9s (single isolate — no jitter).
|
|
1769
|
-
* Bounded by the run deadline (`runtimeDeadlineMs`); we stop early unless there
|
|
1770
|
-
* is at least one backoff's worth plus headroom of budget left.
|
|
1771
|
-
*/
|
|
1772
|
-
const COMPLETE_RECEIPT_RETRY_BACKOFFS_MS = [1_000, 3_000, 9_000] as const;
|
|
1773
|
-
const COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS = 10_000;
|
|
1774
|
-
|
|
1775
|
-
// Fallback batch-chunk parallelism when a tool declares no provider rate hints.
|
|
1776
|
-
// Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
|
|
1777
|
-
// their previous batching behavior; declared providers tighten via the
|
|
1778
|
-
// Governor's suggestedParallelism.
|
|
1779
|
-
const WORKER_TOOL_BATCH_DEFAULT_PARALLELISM = 4;
|
|
1780
|
-
|
|
1781
2309
|
/**
|
|
1782
|
-
*
|
|
1783
|
-
*
|
|
1784
|
-
*
|
|
1785
|
-
*
|
|
1786
|
-
*
|
|
1787
|
-
*
|
|
2310
|
+
* Bounded-concurrent map-chunk dispatch fan-out for the workers_edge profile
|
|
2311
|
+
* (in-process play execution with no `workflowStep`). On workers_edge the whole
|
|
2312
|
+
* map shares ONE per-invocation subrequest window, so sequential chunk dispatch
|
|
2313
|
+
* buys no durability — it is pure wall-clock cost. Running up to K chunks in
|
|
2314
|
+
* flight overlaps provider batches, sheet writes, and the per-chunk persist
|
|
2315
|
+
* barrier. K is capped at 3 to bound peak memory: each in-flight chunk holds up
|
|
2316
|
+
* to ~5000 rows plus their row buffers, so 3 concurrent chunks is the ceiling
|
|
2317
|
+
* we accept before row payloads risk pressuring the isolate. The native
|
|
2318
|
+
* Cloudflare Workflows path (workflowStep present) keeps K=1: there each chunk
|
|
2319
|
+
* is a durable step and sequential dispatch is what makes replay/resume sound.
|
|
1788
2320
|
*/
|
|
2321
|
+
const WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS = 3;
|
|
2322
|
+
const WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES =
|
|
2323
|
+
WORKER_RUN_DISPATCHER_DEFAULT_MAX_RESIDENT_ROW_BYTES;
|
|
2324
|
+
const WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES = 128;
|
|
1789
2325
|
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
function workerDurableToolCallCacheKey(input: {
|
|
1795
|
-
req: RunRequest;
|
|
1796
|
-
toolId: string;
|
|
1797
|
-
requestInput: Record<string, unknown>;
|
|
1798
|
-
providerActionVersion: string;
|
|
1799
|
-
staleAfterSeconds?: number | null;
|
|
1800
|
-
}): string {
|
|
1801
|
-
return buildDurableToolCallCacheKey({
|
|
1802
|
-
orgId: input.req.orgId,
|
|
1803
|
-
playId: input.req.playName,
|
|
1804
|
-
toolId: input.toolId,
|
|
1805
|
-
requestInput: input.requestInput,
|
|
1806
|
-
authScopeDigest: buildDurableToolCallAuthScopeDigest({
|
|
1807
|
-
orgId: input.req.orgId,
|
|
1808
|
-
userEmail: input.req.userEmail,
|
|
1809
|
-
toolId: input.toolId,
|
|
1810
|
-
}),
|
|
1811
|
-
providerActionVersion: input.providerActionVersion,
|
|
1812
|
-
staleAfterSeconds: input.staleAfterSeconds,
|
|
1813
|
-
});
|
|
1814
|
-
}
|
|
1815
|
-
|
|
1816
|
-
function workerRuntimeReceiptKey(input: {
|
|
1817
|
-
req: RunRequest;
|
|
1818
|
-
key: string;
|
|
1819
|
-
}): string {
|
|
1820
|
-
const orgId = input.req.orgId?.trim() || 'org';
|
|
1821
|
-
if (input.key.startsWith(`ctx:${orgId}:`)) {
|
|
1822
|
-
return input.key;
|
|
1823
|
-
}
|
|
1824
|
-
return buildScopedWorkReceiptKey({
|
|
1825
|
-
orgId: input.req.orgId,
|
|
1826
|
-
playName: input.req.playName,
|
|
1827
|
-
key: input.key,
|
|
1828
|
-
});
|
|
1829
|
-
}
|
|
1830
|
-
|
|
1831
|
-
function stepProgramColumnName(parentField: string, stepId: string): string {
|
|
1832
|
-
return sqlSafePlayColumnName(`${parentField}.${stepId}`);
|
|
1833
|
-
}
|
|
1834
|
-
|
|
1835
|
-
class WorkerToolBatchScheduler {
|
|
1836
|
-
private queue: WorkerToolBatchRequest[] = [];
|
|
1837
|
-
private scheduled = false;
|
|
1838
|
-
|
|
1839
|
-
constructor(
|
|
1840
|
-
private readonly req: RunRequest,
|
|
1841
|
-
private readonly governor: PlayExecutionGovernor,
|
|
1842
|
-
private readonly resolvePacing: WorkerPacingResolver,
|
|
1843
|
-
private readonly resolveToolActionCacheVersion: WorkerToolActionCacheVersionResolver,
|
|
1844
|
-
private readonly abortSignal?: AbortSignal,
|
|
1845
|
-
private readonly onRequestsSettled?: (count: number) => void,
|
|
1846
|
-
private readonly callbacks?: WorkerCtxCallbacks,
|
|
1847
|
-
private readonly receiptStore?: WorkerRuntimeReceiptStore,
|
|
1848
|
-
private readonly allowLocalRetryReceipts = false,
|
|
1849
|
-
private readonly runtimeDeadlineMs?: number,
|
|
1850
|
-
// Map-wide (or ctx-wide for the root scheduler) persistence-failure circuit
|
|
1851
|
-
// breaker. Shared by every row worker so the first persistence failure
|
|
1852
|
-
// halts NEW provider dispatch across the whole map.
|
|
1853
|
-
private readonly latch: RuntimePersistenceLatch = createRuntimePersistenceLatch(),
|
|
1854
|
-
// Run-scoped buffer for orphaned billed results whose receipt could not be
|
|
1855
|
-
// completed even after the retry ladder.
|
|
1856
|
-
private readonly salvage?: ReceiptSalvageBuffer,
|
|
1857
|
-
) {}
|
|
1858
|
-
|
|
1859
|
-
/**
|
|
1860
|
-
* Run `completeReceipt`/`completeReceipts` with the salvage retry ladder.
|
|
1861
|
-
* Retries a receipt-completion failure past the server-side 5s connect window
|
|
1862
|
-
* so a transient blip does not orphan a billed result. Bounded by the run
|
|
1863
|
-
* deadline; skips retries when the latch has already burned a ladder in this
|
|
1864
|
-
* map so many in-flight orphans do not each serialize a long loop.
|
|
1865
|
-
*/
|
|
1866
|
-
private async completeReceiptWithRetryLadder<T>(
|
|
1867
|
-
run: () => Promise<T>,
|
|
1868
|
-
onGiveUp?: (attempts: number) => void,
|
|
1869
|
-
): Promise<T> {
|
|
1870
|
-
let attempt = 0;
|
|
1871
|
-
// A ladder that has already started retrying keeps its full budget even if
|
|
1872
|
-
// a peer trips the latch mid-flight — only FRESH entrants collapse once the
|
|
1873
|
-
// failure is confirmed. This is what guarantees the first ladder rides out
|
|
1874
|
-
// a genuine transient blip.
|
|
1875
|
-
let committedToFullLadder = false;
|
|
1876
|
-
const giveUp = (error: unknown): never => {
|
|
1877
|
-
this.latch.latchRetryBudgetExhausted = true;
|
|
1878
|
-
onGiveUp?.(attempt);
|
|
1879
|
-
throw error;
|
|
1880
|
-
};
|
|
1881
|
-
while (true) {
|
|
1882
|
-
try {
|
|
1883
|
-
return await run();
|
|
1884
|
-
} catch (error) {
|
|
1885
|
-
attempt += 1;
|
|
1886
|
-
if (
|
|
1887
|
-
shouldShortenCompleteReceiptLadder(this.latch, committedToFullLadder) ||
|
|
1888
|
-
attempt > COMPLETE_RECEIPT_RETRY_BACKOFFS_MS.length ||
|
|
1889
|
-
this.abortSignal?.aborted
|
|
1890
|
-
) {
|
|
1891
|
-
return giveUp(error);
|
|
1892
|
-
}
|
|
1893
|
-
const backoffMs = COMPLETE_RECEIPT_RETRY_BACKOFFS_MS[attempt - 1]!;
|
|
1894
|
-
// resolveRuntimeDeadlineRemainingMs THROWS WorkflowAbortError once the
|
|
1895
|
-
// run deadline has passed — the correct loud outcome (there is no
|
|
1896
|
-
// separate per-chunk step timeout). Route that throw through giveUp so
|
|
1897
|
-
// the billed result is still salvaged and the ladder budget flag is
|
|
1898
|
-
// set: a billed output must never lose its durable trace because the
|
|
1899
|
-
// clock ran out mid-ladder.
|
|
1900
|
-
let remainingMs: number;
|
|
1901
|
-
try {
|
|
1902
|
-
remainingMs = resolveRuntimeDeadlineRemainingMs(
|
|
1903
|
-
this.runtimeDeadlineMs,
|
|
1904
|
-
);
|
|
1905
|
-
} catch (deadlineError) {
|
|
1906
|
-
return giveUp(deadlineError);
|
|
1907
|
-
}
|
|
1908
|
-
if (remainingMs <= backoffMs + COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS) {
|
|
1909
|
-
return giveUp(error);
|
|
1910
|
-
}
|
|
1911
|
-
// Past every give-up gate: this ladder is now committed to the full
|
|
1912
|
-
// budget and must not be collapsed by a peer tripping the latch mid-sleep.
|
|
1913
|
-
committedToFullLadder = true;
|
|
1914
|
-
await sleepWorkerMs(backoffMs);
|
|
1915
|
-
}
|
|
1916
|
-
}
|
|
1917
|
-
}
|
|
1918
|
-
|
|
1919
|
-
/**
|
|
1920
|
-
* Record a billed-but-unpersisted tool output into the run-scoped salvage
|
|
1921
|
-
* buffer. Called only after the completeReceipt ladder is exhausted, with the
|
|
1922
|
-
* exact serialized payload a receipt would have stored.
|
|
1923
|
-
*/
|
|
1924
|
-
private recordReceiptSalvage(input: {
|
|
1925
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
1926
|
-
output: unknown;
|
|
1927
|
-
attempts: number;
|
|
1928
|
-
}): void {
|
|
1929
|
-
if (!this.salvage || !input.claimed.receiptKey) return;
|
|
1930
|
-
this.salvage.push({
|
|
1931
|
-
receiptKey: input.claimed.receiptKey,
|
|
1932
|
-
toolId: input.claimed.request.toolId,
|
|
1933
|
-
cacheKey: input.claimed.request.cacheKey,
|
|
1934
|
-
output: input.output,
|
|
1935
|
-
completeAttempts: input.attempts,
|
|
1936
|
-
firstErrorAt: nowMs(),
|
|
1937
|
-
});
|
|
1938
|
-
}
|
|
1939
|
-
|
|
1940
|
-
/**
|
|
1941
|
-
* Report a provider 429 / Retry-After back into the Governor's shared pacer
|
|
1942
|
-
* so future acquires for this (org, provider) bucket back off across all
|
|
1943
|
-
* isolates. Provider comes from the same pacing resolver the Governor uses
|
|
1944
|
-
* (the worker has no local catalog), so callers pass only the toolId.
|
|
1945
|
-
*/
|
|
1946
|
-
private reportBackpressure(toolId: string, retryAfterMs: number): void {
|
|
1947
|
-
if (retryAfterMs <= 0) return;
|
|
1948
|
-
void (async () => {
|
|
1949
|
-
const pacing = await this.resolvePacing(toolId).catch(() => null);
|
|
1950
|
-
if (pacing?.provider) {
|
|
1951
|
-
this.governor.reportProviderBackpressure({
|
|
1952
|
-
provider: pacing.provider,
|
|
1953
|
-
retryAfterMs,
|
|
1954
|
-
});
|
|
1955
|
-
}
|
|
1956
|
-
})();
|
|
1957
|
-
}
|
|
1958
|
-
|
|
1959
|
-
async execute(
|
|
1960
|
-
id: string,
|
|
1961
|
-
toolId: string,
|
|
1962
|
-
input: Record<string, unknown>,
|
|
1963
|
-
workflowStep?: WorkflowStep,
|
|
1964
|
-
options?: { force?: boolean; staleAfterSeconds?: number | null },
|
|
1965
|
-
runtimeOptions?: { timeoutMs?: number; receiptWaitMs?: number },
|
|
1966
|
-
): Promise<unknown> {
|
|
1967
|
-
const providerActionVersion =
|
|
1968
|
-
await this.resolveToolActionCacheVersion(toolId);
|
|
1969
|
-
return await new Promise((resolve, reject) => {
|
|
1970
|
-
const queuedAt = nowMs();
|
|
1971
|
-
const receiptKey = workerDurableToolCallCacheKey({
|
|
1972
|
-
req: this.req,
|
|
1973
|
-
toolId,
|
|
1974
|
-
requestInput: input,
|
|
1975
|
-
providerActionVersion,
|
|
1976
|
-
staleAfterSeconds: options?.staleAfterSeconds,
|
|
1977
|
-
});
|
|
1978
|
-
this.queue.push({
|
|
1979
|
-
id,
|
|
1980
|
-
cacheKey: receiptKey,
|
|
1981
|
-
receiptKey,
|
|
1982
|
-
force: options?.force === true || !!this.req.force,
|
|
1983
|
-
receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
1984
|
-
typeof runtimeOptions?.receiptWaitMs === 'number'
|
|
1985
|
-
? { max_wait_ms: runtimeOptions.receiptWaitMs }
|
|
1986
|
-
: input,
|
|
1987
|
-
),
|
|
1988
|
-
...(typeof runtimeOptions?.timeoutMs === 'number'
|
|
1989
|
-
? { runtimeTimeoutMs: runtimeOptions.timeoutMs }
|
|
1990
|
-
: {}),
|
|
1991
|
-
toolId,
|
|
1992
|
-
input,
|
|
1993
|
-
workflowStep,
|
|
1994
|
-
resolve: (value) => {
|
|
1995
|
-
recordRunnerPerfTrace({
|
|
1996
|
-
req: this.req,
|
|
1997
|
-
phase: 'runner.tool.request',
|
|
1998
|
-
ms: nowMs() - queuedAt,
|
|
1999
|
-
extra: { id, toolId },
|
|
2000
|
-
});
|
|
2001
|
-
resolve(value);
|
|
2002
|
-
},
|
|
2003
|
-
reject,
|
|
2004
|
-
});
|
|
2005
|
-
this.scheduleDrain();
|
|
2006
|
-
});
|
|
2007
|
-
}
|
|
2008
|
-
|
|
2009
|
-
private scheduleDrain(): void {
|
|
2010
|
-
if (this.scheduled) return;
|
|
2011
|
-
this.scheduled = true;
|
|
2012
|
-
queueMicrotask(async () => {
|
|
2013
|
-
if (this.hasBatchableQueuedTool()) {
|
|
2014
|
-
await new Promise((resolve) =>
|
|
2015
|
-
setTimeout(resolve, WORKER_TOOL_BATCH_GRACE_MS),
|
|
2016
|
-
);
|
|
2017
|
-
}
|
|
2018
|
-
void this.drain();
|
|
2019
|
-
});
|
|
2020
|
-
}
|
|
2021
|
-
|
|
2022
|
-
private hasBatchableQueuedTool(): boolean {
|
|
2023
|
-
return this.queue.some(
|
|
2024
|
-
(request) =>
|
|
2025
|
-
request.toolId !== 'test_wait_for_event' &&
|
|
2026
|
-
getPlayRuntimeBatchStrategy(request.toolId) !== null,
|
|
2027
|
-
);
|
|
2028
|
-
}
|
|
2029
|
-
|
|
2030
|
-
private async drain(): Promise<void> {
|
|
2031
|
-
const requests = this.queue;
|
|
2032
|
-
this.queue = [];
|
|
2033
|
-
this.scheduled = false;
|
|
2034
|
-
const drainStartedAt = nowMs();
|
|
2035
|
-
await Promise.all(
|
|
2036
|
-
[...groupWorkerToolRequestsByTool(requests).entries()].map(
|
|
2037
|
-
async ([toolId, groupedRequests]) => {
|
|
2038
|
-
await this.executeToolGroup(toolId, groupedRequests);
|
|
2039
|
-
},
|
|
2040
|
-
),
|
|
2041
|
-
);
|
|
2042
|
-
recordRunnerPerfTrace({
|
|
2043
|
-
req: this.req,
|
|
2044
|
-
phase: 'runner.tool.drain',
|
|
2045
|
-
ms: nowMs() - drainStartedAt,
|
|
2046
|
-
extra: { requests: requests.length },
|
|
2047
|
-
});
|
|
2048
|
-
if (this.queue.length > 0) {
|
|
2049
|
-
this.scheduleDrain();
|
|
2050
|
-
}
|
|
2051
|
-
}
|
|
2052
|
-
|
|
2053
|
-
private async waitForDurableToolReceipt(input: {
|
|
2054
|
-
receiptKey: string;
|
|
2055
|
-
maxAttempts: number;
|
|
2056
|
-
}): Promise<WorkerRuntimeReceipt> {
|
|
2057
|
-
if (!this.receiptStore?.getReceipt) {
|
|
2058
|
-
throw new Error(
|
|
2059
|
-
'Worker durable tool receipt wait requires receipt lookup.',
|
|
2060
|
-
);
|
|
2061
|
-
}
|
|
2062
|
-
const receipt = await waitForCompletedRuntimeReceipt({
|
|
2063
|
-
receiptKey: input.receiptKey,
|
|
2064
|
-
maxAttempts: input.maxAttempts,
|
|
2065
|
-
abortSignal: this.abortSignal,
|
|
2066
|
-
store: {
|
|
2067
|
-
getMany: async (receiptKeys) => {
|
|
2068
|
-
const receipts = new Map<string, RuntimeStepReceipt>();
|
|
2069
|
-
await Promise.all(
|
|
2070
|
-
receiptKeys.map(async (key) => {
|
|
2071
|
-
const receipt = await this.receiptStore!.getReceipt!({
|
|
2072
|
-
playName: this.req.playName,
|
|
2073
|
-
key,
|
|
2074
|
-
});
|
|
2075
|
-
if (!receipt) return;
|
|
2076
|
-
receipts.set(key, {
|
|
2077
|
-
key: receipt.key,
|
|
2078
|
-
status: receipt.status,
|
|
2079
|
-
output: receipt.output,
|
|
2080
|
-
error: receipt.error ?? undefined,
|
|
2081
|
-
runId: receipt.runId ?? null,
|
|
2082
|
-
});
|
|
2083
|
-
}),
|
|
2084
|
-
);
|
|
2085
|
-
return receipts;
|
|
2086
|
-
},
|
|
2087
|
-
},
|
|
2088
|
-
});
|
|
2089
|
-
return receipt;
|
|
2090
|
-
}
|
|
2091
|
-
|
|
2092
|
-
private settleRequests(
|
|
2093
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2094
|
-
result: unknown,
|
|
2095
|
-
): void {
|
|
2096
|
-
claimed.request.resolve(result);
|
|
2097
|
-
for (const follower of claimed.followers) {
|
|
2098
|
-
follower.resolve(
|
|
2099
|
-
claimed.receiptKey
|
|
2100
|
-
? markWorkerToolReceiptResultExecution(result, {
|
|
2101
|
-
kind: 'in_flight',
|
|
2102
|
-
receiptKey: claimed.receiptKey,
|
|
2103
|
-
attachedToReceiptKey: claimed.receiptKey,
|
|
2104
|
-
})
|
|
2105
|
-
: result,
|
|
2106
|
-
);
|
|
2107
|
-
}
|
|
2108
|
-
this.onRequestsSettled?.(1 + claimed.followers.length);
|
|
2109
|
-
}
|
|
2110
|
-
|
|
2111
|
-
private rejectRequests(
|
|
2112
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2113
|
-
error: unknown,
|
|
2114
|
-
): void {
|
|
2115
|
-
claimed.request.reject(error);
|
|
2116
|
-
for (const follower of claimed.followers) {
|
|
2117
|
-
follower.reject(error);
|
|
2118
|
-
}
|
|
2119
|
-
this.onRequestsSettled?.(1 + claimed.followers.length);
|
|
2120
|
-
}
|
|
2121
|
-
|
|
2122
|
-
private rejectRawRequests(
|
|
2123
|
-
requests: WorkerToolBatchRequest[],
|
|
2124
|
-
error: unknown,
|
|
2125
|
-
): void {
|
|
2126
|
-
for (const request of requests) {
|
|
2127
|
-
request.reject(error);
|
|
2128
|
-
}
|
|
2129
|
-
this.onRequestsSettled?.(requests.length);
|
|
2130
|
-
}
|
|
2131
|
-
|
|
2132
|
-
private async resolveCompletedDurableToolReceiptGroup(input: {
|
|
2133
|
-
group: WorkerToolBatchRequest[];
|
|
2134
|
-
receiptKey: string;
|
|
2135
|
-
receipt: WorkerRuntimeReceipt;
|
|
2136
|
-
source: 'cache' | 'in_flight';
|
|
2137
|
-
}): Promise<void> {
|
|
2138
|
-
const [first] = input.group;
|
|
2139
|
-
if (!first) return;
|
|
2140
|
-
const value = deserializeDurableStepValue(input.receipt.output);
|
|
2141
|
-
const result =
|
|
2142
|
-
input.source === 'cache'
|
|
2143
|
-
? markWorkerToolReceiptResultCached(
|
|
2144
|
-
value,
|
|
2145
|
-
first.cacheKey,
|
|
2146
|
-
input.receiptKey,
|
|
2147
|
-
)
|
|
2148
|
-
: markWorkerToolReceiptResultExecution(value, {
|
|
2149
|
-
kind: 'in_flight',
|
|
2150
|
-
receiptKey: input.receiptKey,
|
|
2151
|
-
attachedToReceiptKey: input.receiptKey,
|
|
2152
|
-
});
|
|
2153
|
-
for (const request of input.group) {
|
|
2154
|
-
request.resolve(result);
|
|
2155
|
-
}
|
|
2156
|
-
this.onRequestsSettled?.(input.group.length);
|
|
2157
|
-
}
|
|
2158
|
-
|
|
2159
|
-
private async reclaimRunningDurableToolReceiptGroupAfterWait(input: {
|
|
2160
|
-
group: WorkerToolBatchRequest[];
|
|
2161
|
-
receiptKey: string;
|
|
2162
|
-
runningReceipt: WorkerRuntimeReceipt;
|
|
2163
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
2164
|
-
const [request, ...followers] = input.group;
|
|
2165
|
-
if (!request || !this.receiptStore) {
|
|
2166
|
-
this.rejectRawRequests(
|
|
2167
|
-
input.group,
|
|
2168
|
-
new RuntimeReceiptWaitTimeoutError(input.receiptKey),
|
|
2169
|
-
);
|
|
2170
|
-
return [];
|
|
2171
|
-
}
|
|
2172
|
-
// A `running` receipt left by a DIFFERENT run whose last write is stale will
|
|
2173
|
-
// never be completed by a live worker. Polling it burns one service-binding
|
|
2174
|
-
// subrequest per tick, up to the receipt's wait budget (~1320) — many such
|
|
2175
|
-
// orphans on a resume exhaust Cloudflare's per-invocation subrequest budget
|
|
2176
|
-
// and kill the run. The orchestrator reclaims those immediately (skipping the
|
|
2177
|
-
// poll) and re-executes, matching the "on resume it reclaims and re-executes"
|
|
2178
|
-
// contract; live/fresh/same-run owners still poll before reclaiming. Kept as
|
|
2179
|
-
// an injected helper so the skip-poll decision is unit-testable without
|
|
2180
|
-
// loading this cloudflare:workers Worker entrypoint.
|
|
2181
|
-
return await reclaimRunningReceiptGroupAfterWait<ClaimedWorkerToolBatchRequest>(
|
|
2182
|
-
{
|
|
2183
|
-
ownership: {
|
|
2184
|
-
ownerRunId: input.runningReceipt.runId,
|
|
2185
|
-
currentRunId: this.req.runId,
|
|
2186
|
-
updatedAt: input.runningReceipt.updatedAt,
|
|
2187
|
-
},
|
|
2188
|
-
timeoutError: new RuntimeReceiptWaitTimeoutError(input.receiptKey),
|
|
2189
|
-
poll: async () => {
|
|
2190
|
-
try {
|
|
2191
|
-
const receipt = await this.waitForDurableToolReceipt({
|
|
2192
|
-
receiptKey: input.receiptKey,
|
|
2193
|
-
maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
|
|
2194
|
-
input.group,
|
|
2195
|
-
(groupRequest) => groupRequest.receiptWaitMaxAttempts,
|
|
2196
|
-
),
|
|
2197
|
-
});
|
|
2198
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2199
|
-
group: input.group,
|
|
2200
|
-
receiptKey: input.receiptKey,
|
|
2201
|
-
receipt,
|
|
2202
|
-
source: 'in_flight',
|
|
2203
|
-
});
|
|
2204
|
-
return { kind: 'completed' };
|
|
2205
|
-
} catch (error) {
|
|
2206
|
-
if (error instanceof RuntimeReceiptWaitTimeoutError) {
|
|
2207
|
-
return { kind: 'timed_out', error };
|
|
2208
|
-
}
|
|
2209
|
-
return { kind: 'errored', error };
|
|
2210
|
-
}
|
|
2211
|
-
},
|
|
2212
|
-
reclaim: (rejectionOnFallthrough) =>
|
|
2213
|
-
this.forceReclaimRunningDurableToolReceiptGroup({
|
|
2214
|
-
group: input.group,
|
|
2215
|
-
request,
|
|
2216
|
-
followers,
|
|
2217
|
-
receiptKey: input.receiptKey,
|
|
2218
|
-
rejectionOnFallthrough,
|
|
2219
|
-
}),
|
|
2220
|
-
reject: (error) => this.rejectRawRequests(input.group, error),
|
|
2221
|
-
},
|
|
2222
|
-
);
|
|
2223
|
-
}
|
|
2224
|
-
|
|
2225
|
-
/**
|
|
2226
|
-
* Steal a `running` receipt (reclaimRunning) and route by claim disposition.
|
|
2227
|
-
* Shared by the post-poll-timeout reclaim and the dead-owner fast path, so
|
|
2228
|
-
* both dispose of a reclaimed/reused/failed receipt identically.
|
|
2229
|
-
*/
|
|
2230
|
-
private async forceReclaimRunningDurableToolReceiptGroup(input: {
|
|
2231
|
-
group: WorkerToolBatchRequest[];
|
|
2232
|
-
request: WorkerToolBatchRequest;
|
|
2233
|
-
followers: WorkerToolBatchRequest[];
|
|
2234
|
-
receiptKey: string;
|
|
2235
|
-
rejectionOnFallthrough: unknown;
|
|
2236
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
2237
|
-
if (!this.receiptStore) {
|
|
2238
|
-
this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
|
|
2239
|
-
return [];
|
|
2240
|
-
}
|
|
2241
|
-
let claim: WorkerRuntimeReceiptClaim;
|
|
2242
|
-
try {
|
|
2243
|
-
claim = await this.receiptStore.claimReceipt({
|
|
2244
|
-
playName: this.req.playName,
|
|
2245
|
-
runId: this.req.runId,
|
|
2246
|
-
key: input.receiptKey,
|
|
2247
|
-
reclaimRunning: true,
|
|
2248
|
-
});
|
|
2249
|
-
} catch (error) {
|
|
2250
|
-
this.rejectRawRequests(input.group, error);
|
|
2251
|
-
return [];
|
|
2252
|
-
}
|
|
2253
|
-
if (claim.disposition === 'claimed') {
|
|
2254
|
-
return [
|
|
2255
|
-
{
|
|
2256
|
-
request: input.request,
|
|
2257
|
-
receiptKey: input.receiptKey,
|
|
2258
|
-
followers: input.followers,
|
|
2259
|
-
},
|
|
2260
|
-
];
|
|
2261
|
-
}
|
|
2262
|
-
if (claim.disposition === 'reused') {
|
|
2263
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2264
|
-
group: input.group,
|
|
2265
|
-
receiptKey: input.receiptKey,
|
|
2266
|
-
receipt: claim.receipt,
|
|
2267
|
-
source: 'cache',
|
|
2268
|
-
});
|
|
2269
|
-
return [];
|
|
2270
|
-
}
|
|
2271
|
-
if (claim.disposition === 'failed') {
|
|
2272
|
-
return await this.reclaimFailedDurableToolReceiptGroup({
|
|
2273
|
-
group: input.group,
|
|
2274
|
-
receiptKey: input.receiptKey,
|
|
2275
|
-
failedReceipt: claim.receipt,
|
|
2276
|
-
});
|
|
2277
|
-
}
|
|
2278
|
-
this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
|
|
2279
|
-
return [];
|
|
2280
|
-
}
|
|
2281
|
-
|
|
2282
|
-
private async reclaimFailedDurableToolReceiptGroup(input: {
|
|
2283
|
-
group: WorkerToolBatchRequest[];
|
|
2284
|
-
receiptKey: string;
|
|
2285
|
-
failedReceipt: WorkerRuntimeReceipt;
|
|
2286
|
-
}): Promise<ClaimedWorkerToolBatchRequest[]> {
|
|
2287
|
-
const failedError = new Error(
|
|
2288
|
-
`Durable tool call ${input.receiptKey} failed: ${input.failedReceipt.error ?? 'unknown error'}`,
|
|
2289
|
-
);
|
|
2290
|
-
const [request, ...followers] = input.group;
|
|
2291
|
-
if (!request || !this.receiptStore) {
|
|
2292
|
-
this.rejectRawRequests(input.group, failedError);
|
|
2293
|
-
return [];
|
|
2294
|
-
}
|
|
2295
|
-
if (
|
|
2296
|
-
!canReclaimFailedWorkerToolReceipt({
|
|
2297
|
-
error: input.failedReceipt.error,
|
|
2298
|
-
})
|
|
2299
|
-
) {
|
|
2300
|
-
this.rejectRawRequests(input.group, failedError);
|
|
2301
|
-
return [];
|
|
2302
|
-
}
|
|
2303
|
-
let claim: WorkerRuntimeReceiptClaim;
|
|
2304
|
-
try {
|
|
2305
|
-
claim = await this.receiptStore.claimReceipt({
|
|
2306
|
-
playName: this.req.playName,
|
|
2307
|
-
runId: this.req.runId,
|
|
2308
|
-
key: input.receiptKey,
|
|
2309
|
-
});
|
|
2310
|
-
} catch (error) {
|
|
2311
|
-
this.rejectRawRequests(input.group, error);
|
|
2312
|
-
return [];
|
|
2313
|
-
}
|
|
2314
|
-
if (claim.disposition === 'claimed') {
|
|
2315
|
-
return [{ request, receiptKey: input.receiptKey, followers }];
|
|
2316
|
-
}
|
|
2317
|
-
if (claim.disposition === 'reused') {
|
|
2318
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2319
|
-
group: input.group,
|
|
2320
|
-
receiptKey: input.receiptKey,
|
|
2321
|
-
receipt: claim.receipt,
|
|
2322
|
-
source: 'cache',
|
|
2323
|
-
});
|
|
2324
|
-
return [];
|
|
2325
|
-
}
|
|
2326
|
-
if (claim.disposition === 'running') {
|
|
2327
|
-
try {
|
|
2328
|
-
const receipt = await this.waitForDurableToolReceipt({
|
|
2329
|
-
receiptKey: input.receiptKey,
|
|
2330
|
-
maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
|
|
2331
|
-
input.group,
|
|
2332
|
-
(groupRequest) => groupRequest.receiptWaitMaxAttempts,
|
|
2333
|
-
),
|
|
2334
|
-
});
|
|
2335
|
-
await this.resolveCompletedDurableToolReceiptGroup({
|
|
2336
|
-
group: input.group,
|
|
2337
|
-
receiptKey: input.receiptKey,
|
|
2338
|
-
receipt,
|
|
2339
|
-
source: 'in_flight',
|
|
2340
|
-
});
|
|
2341
|
-
} catch (error) {
|
|
2342
|
-
this.rejectRawRequests(input.group, error);
|
|
2343
|
-
}
|
|
2344
|
-
return [];
|
|
2345
|
-
}
|
|
2346
|
-
this.rejectRawRequests(
|
|
2347
|
-
input.group,
|
|
2348
|
-
new Error(
|
|
2349
|
-
`Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
|
|
2350
|
-
),
|
|
2351
|
-
);
|
|
2352
|
-
return [];
|
|
2353
|
-
}
|
|
2354
|
-
|
|
2355
|
-
private async failureForRejectedToolRequest(
|
|
2356
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2357
|
-
error: unknown,
|
|
2358
|
-
): Promise<unknown> {
|
|
2359
|
-
try {
|
|
2360
|
-
await this.failDurableToolRequest(claimed, error);
|
|
2361
|
-
return error;
|
|
2362
|
-
} catch (receiptError) {
|
|
2363
|
-
tripRuntimePersistenceLatch(this.latch, receiptError);
|
|
2364
|
-
return new RuntimeReceiptPersistenceError(
|
|
2365
|
-
'Tool call failed and durable receipt could not be marked failed',
|
|
2366
|
-
[error, receiptError],
|
|
2367
|
-
);
|
|
2368
|
-
}
|
|
2369
|
-
}
|
|
2370
|
-
|
|
2371
|
-
private async claimDurableToolReceiptGroups(
|
|
2372
|
-
groups: ReturnType<
|
|
2373
|
-
typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
|
|
2374
|
-
>['durableGroups'],
|
|
2375
|
-
): Promise<
|
|
2376
|
-
Array<{
|
|
2377
|
-
group: ReturnType<
|
|
2378
|
-
typeof planWorkerToolReceiptGroups<WorkerToolBatchRequest>
|
|
2379
|
-
>['durableGroups'][number];
|
|
2380
|
-
receiptKey: string;
|
|
2381
|
-
claim: WorkerRuntimeReceiptClaim;
|
|
2382
|
-
}>
|
|
2383
|
-
> {
|
|
2384
|
-
if (!this.receiptStore) return [];
|
|
2385
|
-
const planned = groups.map((group) => ({
|
|
2386
|
-
group,
|
|
2387
|
-
receiptKey: workerRuntimeReceiptKey({
|
|
2388
|
-
req: this.req,
|
|
2389
|
-
key: group.claimableReceiptKey,
|
|
2390
|
-
}),
|
|
2391
|
-
}));
|
|
2392
|
-
const claimOne = async (entry: (typeof planned)[number]) => ({
|
|
2393
|
-
...entry,
|
|
2394
|
-
claim: await this.receiptStore!.claimReceipt({
|
|
2395
|
-
playName: this.req.playName,
|
|
2396
|
-
runId: this.req.runId,
|
|
2397
|
-
key: entry.receiptKey,
|
|
2398
|
-
...(entry.group.forceDurableRefresh
|
|
2399
|
-
? { forceRefresh: true, reclaimRunning: true }
|
|
2400
|
-
: {}),
|
|
2401
|
-
}),
|
|
2402
|
-
});
|
|
2403
|
-
if (!this.receiptStore.claimReceipts) {
|
|
2404
|
-
return await Promise.all(planned.map(claimOne));
|
|
2405
|
-
}
|
|
2406
|
-
const claimed: Array<{
|
|
2407
|
-
group: (typeof planned)[number]['group'];
|
|
2408
|
-
receiptKey: string;
|
|
2409
|
-
claim: WorkerRuntimeReceiptClaim;
|
|
2410
|
-
}> = [];
|
|
2411
|
-
for (const forceDurableRefresh of [false, true]) {
|
|
2412
|
-
const entries = planned.filter(
|
|
2413
|
-
(entry) => entry.group.forceDurableRefresh === forceDurableRefresh,
|
|
2414
|
-
);
|
|
2415
|
-
if (entries.length === 0) continue;
|
|
2416
|
-
const claims = await this.receiptStore.claimReceipts({
|
|
2417
|
-
playName: this.req.playName,
|
|
2418
|
-
runId: this.req.runId,
|
|
2419
|
-
keys: entries.map((entry) => entry.receiptKey),
|
|
2420
|
-
...(forceDurableRefresh
|
|
2421
|
-
? { forceRefresh: true, reclaimRunning: true }
|
|
2422
|
-
: {}),
|
|
2423
|
-
});
|
|
2424
|
-
entries.forEach((entry, index) => {
|
|
2425
|
-
const claim = claims[index];
|
|
2426
|
-
if (!claim) {
|
|
2427
|
-
throw new Error(
|
|
2428
|
-
`Runtime receipt batch claim did not return receipt ${entry.receiptKey}.`,
|
|
2429
|
-
);
|
|
2430
|
-
}
|
|
2431
|
-
claimed.push({ ...entry, claim });
|
|
2432
|
-
});
|
|
2433
|
-
}
|
|
2434
|
-
return claimed;
|
|
2435
|
-
}
|
|
2436
|
-
|
|
2437
|
-
private async prepareDurableToolRequests(
|
|
2438
|
-
requests: WorkerToolBatchRequest[],
|
|
2439
|
-
): Promise<PreparedWorkerToolBatchRequests> {
|
|
2440
|
-
if (!this.receiptStore) {
|
|
2441
|
-
return {
|
|
2442
|
-
claimedRequests: requests.map((request) => ({
|
|
2443
|
-
request,
|
|
2444
|
-
receiptKey: null,
|
|
2445
|
-
followers: [],
|
|
2446
|
-
})),
|
|
2447
|
-
deferredClaimedRequests: [],
|
|
2448
|
-
};
|
|
2449
|
-
}
|
|
2450
|
-
|
|
2451
|
-
const claimedRequests: ClaimedWorkerToolBatchRequest[] = [];
|
|
2452
|
-
const deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[] =
|
|
2453
|
-
[];
|
|
2454
|
-
const receiptGroupPlan = planWorkerToolReceiptGroups(requests, {
|
|
2455
|
-
allowLocalRetryReceipts: this.allowLocalRetryReceipts,
|
|
2456
|
-
getReceiptInput: (request) => ({
|
|
2457
|
-
durableReceiptKey: request.receiptKey,
|
|
2458
|
-
localRetryCacheKey: request.cacheKey,
|
|
2459
|
-
force: request.force,
|
|
2460
|
-
}),
|
|
2461
|
-
});
|
|
2462
|
-
for (const request of receiptGroupPlan.localRequests) {
|
|
2463
|
-
claimedRequests.push({
|
|
2464
|
-
request,
|
|
2465
|
-
receiptKey: null,
|
|
2466
|
-
followers: [],
|
|
2467
|
-
});
|
|
2468
|
-
}
|
|
2469
|
-
|
|
2470
|
-
try {
|
|
2471
|
-
const claimEntries = await this.claimDurableToolReceiptGroups(
|
|
2472
|
-
receiptGroupPlan.durableGroups,
|
|
2473
|
-
);
|
|
2474
|
-
for (const { group: groupState, receiptKey, claim } of claimEntries) {
|
|
2475
|
-
const group = groupState.requests;
|
|
2476
|
-
const [first] = group;
|
|
2477
|
-
if (!first) continue;
|
|
2478
|
-
if (claim.disposition === 'reused') {
|
|
2479
|
-
const result = markWorkerToolReceiptResultCached(
|
|
2480
|
-
deserializeDurableStepValue(claim.receipt.output),
|
|
2481
|
-
first.cacheKey,
|
|
2482
|
-
receiptKey,
|
|
2483
|
-
);
|
|
2484
|
-
for (const request of group) {
|
|
2485
|
-
request.resolve(result);
|
|
2486
|
-
}
|
|
2487
|
-
this.onRequestsSettled?.(group.length);
|
|
2488
|
-
continue;
|
|
2489
|
-
}
|
|
2490
|
-
if (claim.disposition === 'failed') {
|
|
2491
|
-
claimedRequests.push(
|
|
2492
|
-
...(await this.reclaimFailedDurableToolReceiptGroup({
|
|
2493
|
-
group,
|
|
2494
|
-
receiptKey,
|
|
2495
|
-
failedReceipt: claim.receipt,
|
|
2496
|
-
})),
|
|
2497
|
-
);
|
|
2498
|
-
continue;
|
|
2499
|
-
}
|
|
2500
|
-
if (claim.disposition === 'running') {
|
|
2501
|
-
deferredClaimedRequests.push(
|
|
2502
|
-
this.reclaimRunningDurableToolReceiptGroupAfterWait({
|
|
2503
|
-
group,
|
|
2504
|
-
receiptKey,
|
|
2505
|
-
runningReceipt: claim.receipt,
|
|
2506
|
-
}),
|
|
2507
|
-
);
|
|
2508
|
-
continue;
|
|
2509
|
-
}
|
|
2510
|
-
const [request, ...followers] = group;
|
|
2511
|
-
if (!request) continue;
|
|
2512
|
-
claimedRequests.push({ request, receiptKey, followers });
|
|
2513
|
-
}
|
|
2514
|
-
} catch (error) {
|
|
2515
|
-
// A receipt claim failure means the tenant Postgres is already
|
|
2516
|
-
// unwritable: trip the breaker so remaining groups/rows stop dispatching.
|
|
2517
|
-
tripRuntimePersistenceLatch(this.latch, error);
|
|
2518
|
-
for (const group of receiptGroupPlan.durableGroups) {
|
|
2519
|
-
this.rejectRawRequests(group.requests, error);
|
|
2520
|
-
}
|
|
2521
|
-
}
|
|
2522
|
-
return { claimedRequests, deferredClaimedRequests };
|
|
2523
|
-
}
|
|
2524
|
-
|
|
2525
|
-
private async completeDurableToolRequest(
|
|
2526
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2527
|
-
result: unknown,
|
|
2528
|
-
): Promise<unknown> {
|
|
2529
|
-
if (!this.receiptStore || !claimed.receiptKey) {
|
|
2530
|
-
return result;
|
|
2531
|
-
}
|
|
2532
|
-
const ownerResult = markWorkerToolReceiptResultExecution(result, {
|
|
2533
|
-
kind: 'live',
|
|
2534
|
-
receiptKey: claimed.receiptKey,
|
|
2535
|
-
});
|
|
2536
|
-
const serializedOutput = serializeDurableStepValue(ownerResult);
|
|
2537
|
-
const completed = await this.completeReceiptWithRetryLadder(
|
|
2538
|
-
() =>
|
|
2539
|
-
this.receiptStore!.completeReceipt({
|
|
2540
|
-
playName: this.req.playName,
|
|
2541
|
-
runId: this.req.runId,
|
|
2542
|
-
key: claimed.receiptKey!,
|
|
2543
|
-
output: serializedOutput,
|
|
2544
|
-
}),
|
|
2545
|
-
// Ladder exhausted: the call billed but the receipt could not land.
|
|
2546
|
-
// Salvage the exact serialized output onto the run record before the
|
|
2547
|
-
// caller turns this into a RuntimeReceiptPersistenceError.
|
|
2548
|
-
(attempts) =>
|
|
2549
|
-
this.recordReceiptSalvage({
|
|
2550
|
-
claimed,
|
|
2551
|
-
output: serializedOutput,
|
|
2552
|
-
attempts,
|
|
2553
|
-
}),
|
|
2554
|
-
);
|
|
2555
|
-
if (
|
|
2556
|
-
completed &&
|
|
2557
|
-
(completed.status === 'completed' || completed.status === 'skipped') &&
|
|
2558
|
-
completed.output !== undefined
|
|
2559
|
-
) {
|
|
2560
|
-
const recovered = deserializeDurableStepValue(completed.output);
|
|
2561
|
-
return completed.runId && completed.runId !== this.req.runId
|
|
2562
|
-
? markWorkerToolReceiptResultCached(
|
|
2563
|
-
recovered,
|
|
2564
|
-
claimed.request.cacheKey,
|
|
2565
|
-
claimed.receiptKey,
|
|
2566
|
-
)
|
|
2567
|
-
: recovered;
|
|
2568
|
-
}
|
|
2569
|
-
return ownerResult;
|
|
2570
|
-
}
|
|
2571
|
-
|
|
2572
|
-
private completedDurableToolRequestResult(input: {
|
|
2573
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
2574
|
-
ownerResult: unknown;
|
|
2575
|
-
completed: WorkerRuntimeReceipt | null | undefined;
|
|
2576
|
-
}): unknown {
|
|
2577
|
-
const { claimed, completed, ownerResult } = input;
|
|
2578
|
-
if (
|
|
2579
|
-
completed &&
|
|
2580
|
-
(completed.status === 'completed' || completed.status === 'skipped') &&
|
|
2581
|
-
completed.output !== undefined
|
|
2582
|
-
) {
|
|
2583
|
-
const recovered = deserializeDurableStepValue(completed.output);
|
|
2584
|
-
return completed.runId && completed.runId !== this.req.runId
|
|
2585
|
-
? markWorkerToolReceiptResultCached(
|
|
2586
|
-
recovered,
|
|
2587
|
-
claimed.request.cacheKey,
|
|
2588
|
-
claimed.receiptKey ?? claimed.request.cacheKey,
|
|
2589
|
-
)
|
|
2590
|
-
: recovered;
|
|
2591
|
-
}
|
|
2592
|
-
return ownerResult;
|
|
2593
|
-
}
|
|
2594
|
-
|
|
2595
|
-
private async completeDurableToolRequests(
|
|
2596
|
-
entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
|
|
2597
|
-
): Promise<unknown[]> {
|
|
2598
|
-
const results: unknown[] = new Array(entries.length);
|
|
2599
|
-
const durableEntries: Array<{
|
|
2600
|
-
index: number;
|
|
2601
|
-
claimed: ClaimedWorkerToolBatchRequest;
|
|
2602
|
-
ownerResult: unknown;
|
|
2603
|
-
}> = [];
|
|
2604
|
-
for (let index = 0; index < entries.length; index += 1) {
|
|
2605
|
-
const entry = entries[index]!;
|
|
2606
|
-
if (!this.receiptStore || !entry.claimed.receiptKey) {
|
|
2607
|
-
results[index] = entry.result;
|
|
2608
|
-
continue;
|
|
2609
|
-
}
|
|
2610
|
-
const ownerResult = markWorkerToolReceiptResultExecution(entry.result, {
|
|
2611
|
-
kind: 'live',
|
|
2612
|
-
receiptKey: entry.claimed.receiptKey,
|
|
2613
|
-
});
|
|
2614
|
-
durableEntries.push({ index, claimed: entry.claimed, ownerResult });
|
|
2615
|
-
}
|
|
2616
|
-
if (durableEntries.length === 0) {
|
|
2617
|
-
return results;
|
|
2618
|
-
}
|
|
2619
|
-
if (!this.receiptStore?.completeReceipts) {
|
|
2620
|
-
await Promise.all(
|
|
2621
|
-
durableEntries.map(async (entry) => {
|
|
2622
|
-
results[entry.index] = await this.completeDurableToolRequest(
|
|
2623
|
-
entry.claimed,
|
|
2624
|
-
entries[entry.index]!.result,
|
|
2625
|
-
);
|
|
2626
|
-
}),
|
|
2627
|
-
);
|
|
2628
|
-
return results;
|
|
2629
|
-
}
|
|
2630
|
-
const serializedByEntry = durableEntries.map((entry) =>
|
|
2631
|
-
serializeDurableStepValue(entry.ownerResult),
|
|
2632
|
-
);
|
|
2633
|
-
const completed = await this.completeReceiptWithRetryLadder(
|
|
2634
|
-
() =>
|
|
2635
|
-
this.receiptStore!.completeReceipts!({
|
|
2636
|
-
playName: this.req.playName,
|
|
2637
|
-
receipts: durableEntries.map((entry, entryIndex) => ({
|
|
2638
|
-
runId: this.req.runId,
|
|
2639
|
-
key: entry.claimed.receiptKey!,
|
|
2640
|
-
output: serializedByEntry[entryIndex],
|
|
2641
|
-
})),
|
|
2642
|
-
}),
|
|
2643
|
-
// Ladder exhausted: every entry in this batch billed but could not be
|
|
2644
|
-
// marked completed. Salvage each serialized output before the caller
|
|
2645
|
-
// turns this into a RuntimeReceiptPersistenceError.
|
|
2646
|
-
(attempts) => {
|
|
2647
|
-
durableEntries.forEach((entry, entryIndex) => {
|
|
2648
|
-
this.recordReceiptSalvage({
|
|
2649
|
-
claimed: entry.claimed,
|
|
2650
|
-
output: serializedByEntry[entryIndex],
|
|
2651
|
-
attempts,
|
|
2652
|
-
});
|
|
2653
|
-
});
|
|
2654
|
-
},
|
|
2655
|
-
);
|
|
2656
|
-
durableEntries.forEach((entry, resultIndex) => {
|
|
2657
|
-
results[entry.index] = this.completedDurableToolRequestResult({
|
|
2658
|
-
claimed: entry.claimed,
|
|
2659
|
-
ownerResult: entry.ownerResult,
|
|
2660
|
-
completed: completed[resultIndex],
|
|
2661
|
-
});
|
|
2662
|
-
});
|
|
2663
|
-
return results;
|
|
2664
|
-
}
|
|
2665
|
-
|
|
2666
|
-
private async failDurableToolRequest(
|
|
2667
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2668
|
-
error: unknown,
|
|
2669
|
-
): Promise<void> {
|
|
2670
|
-
if (!this.receiptStore || !claimed.receiptKey) {
|
|
2671
|
-
return;
|
|
2672
|
-
}
|
|
2673
|
-
await this.receiptStore.failReceipt({
|
|
2674
|
-
playName: this.req.playName,
|
|
2675
|
-
runId: this.req.runId,
|
|
2676
|
-
key: claimed.receiptKey,
|
|
2677
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2678
|
-
});
|
|
2679
|
-
}
|
|
2680
|
-
|
|
2681
|
-
private async failDurableToolRequests(
|
|
2682
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
2683
|
-
error: unknown,
|
|
2684
|
-
): Promise<void> {
|
|
2685
|
-
const durable = claimedRequests.filter(
|
|
2686
|
-
(claimed) => this.receiptStore && claimed.receiptKey,
|
|
2687
|
-
);
|
|
2688
|
-
if (durable.length === 0) return;
|
|
2689
|
-
if (!this.receiptStore?.failReceipts) {
|
|
2690
|
-
await Promise.all(
|
|
2691
|
-
durable.map((claimed) => this.failDurableToolRequest(claimed, error)),
|
|
2692
|
-
);
|
|
2693
|
-
return;
|
|
2694
|
-
}
|
|
2695
|
-
await this.receiptStore.failReceipts({
|
|
2696
|
-
playName: this.req.playName,
|
|
2697
|
-
receipts: durable.map((claimed) => ({
|
|
2698
|
-
runId: this.req.runId,
|
|
2699
|
-
key: claimed.receiptKey!,
|
|
2700
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2701
|
-
})),
|
|
2702
|
-
});
|
|
2703
|
-
}
|
|
2704
|
-
|
|
2705
|
-
private async executeToolGroup(
|
|
2706
|
-
toolId: string,
|
|
2707
|
-
requests: WorkerToolBatchRequest[],
|
|
2708
|
-
): Promise<void> {
|
|
2709
|
-
// Circuit breaker, gated BEFORE the receipt claim: never-dispatched rows
|
|
2710
|
-
// get NO receipt (not even `running`), so a re-run claims them fresh and
|
|
2711
|
-
// executes them exactly once.
|
|
2712
|
-
if (this.latch.tripped) {
|
|
2713
|
-
this.latch.preventedCallCount += requests.length;
|
|
2714
|
-
this.rejectRawRequests(
|
|
2715
|
-
requests,
|
|
2716
|
-
new RuntimePersistenceCircuitOpenError(this.latch),
|
|
2717
|
-
);
|
|
2718
|
-
return;
|
|
2719
|
-
}
|
|
2720
|
-
const { claimedRequests, deferredClaimedRequests } =
|
|
2721
|
-
await this.prepareDurableToolRequests(requests);
|
|
2722
|
-
await this.executeClaimedToolRequests(
|
|
2723
|
-
toolId,
|
|
2724
|
-
requests.length,
|
|
2725
|
-
claimedRequests,
|
|
2726
|
-
);
|
|
2727
|
-
if (deferredClaimedRequests.length === 0) {
|
|
2728
|
-
return;
|
|
2729
|
-
}
|
|
2730
|
-
const reclaimedRequests = (
|
|
2731
|
-
await Promise.all(deferredClaimedRequests)
|
|
2732
|
-
).flat();
|
|
2733
|
-
await this.executeClaimedToolRequests(
|
|
2734
|
-
toolId,
|
|
2735
|
-
requests.length,
|
|
2736
|
-
reclaimedRequests,
|
|
2737
|
-
);
|
|
2738
|
-
}
|
|
2739
|
-
|
|
2740
|
-
private async executeClaimedToolRequests(
|
|
2741
|
-
toolId: string,
|
|
2742
|
-
requestCount: number,
|
|
2743
|
-
claimedRequests: ClaimedWorkerToolBatchRequest[],
|
|
2744
|
-
): Promise<void> {
|
|
2745
|
-
if (claimedRequests.length === 0) {
|
|
2746
|
-
return;
|
|
2747
|
-
}
|
|
2748
|
-
const strategy = getPlayRuntimeBatchStrategy(toolId);
|
|
2749
|
-
if (
|
|
2750
|
-
!strategy ||
|
|
2751
|
-
toolId === 'test_wait_for_event' ||
|
|
2752
|
-
claimedRequests.length < 2
|
|
2753
|
-
) {
|
|
2754
|
-
const groupStartedAt = nowMs();
|
|
2755
|
-
const dispatchOne = async (
|
|
2756
|
-
claimed: ClaimedWorkerToolBatchRequest,
|
|
2757
|
-
): Promise<void> => {
|
|
2758
|
-
const { request } = claimed;
|
|
2759
|
-
// Circuit breaker: the latch may have tripped after this group was
|
|
2760
|
-
// claimed but before this particular call started. Do NOT call the
|
|
2761
|
-
// provider. This row has a `running` receipt (claimed, never
|
|
2762
|
-
// executed); on resume it reclaims and re-executes — safe, it never
|
|
2763
|
-
// billed.
|
|
2764
|
-
if (this.latch.tripped) {
|
|
2765
|
-
this.latch.preventedCallCount += 1 + claimed.followers.length;
|
|
2766
|
-
this.rejectRequests(
|
|
2767
|
-
claimed,
|
|
2768
|
-
new RuntimePersistenceCircuitOpenError(this.latch),
|
|
2769
|
-
);
|
|
2770
|
-
return;
|
|
2771
|
-
}
|
|
2772
|
-
const toolContract = await this.resolvePacing(toolId).catch(
|
|
2773
|
-
() => null,
|
|
2774
|
-
);
|
|
2775
|
-
// Each unbatched provider call takes its own tool slot: the Governor
|
|
2776
|
-
// charges tool budget, holds a global tool-concurrency slot, and
|
|
2777
|
-
// applies per-(org,provider) pacing before the call runs.
|
|
2778
|
-
const slot = await this.governor.acquireToolSlot(toolId, {
|
|
2779
|
-
signal: this.abortSignal,
|
|
2780
|
-
});
|
|
2781
|
-
try {
|
|
2782
|
-
let result: unknown;
|
|
2783
|
-
try {
|
|
2784
|
-
result = await executeToolWithLifecycle(
|
|
2785
|
-
this.req,
|
|
2786
|
-
{ id: request.id, toolId, input: request.input },
|
|
2787
|
-
request.workflowStep,
|
|
2788
|
-
this.callbacks,
|
|
2789
|
-
(retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
|
|
2790
|
-
() => this.governor.chargeBudget('retry'),
|
|
2791
|
-
toolContract?.retrySafeTransientHttp === true,
|
|
2792
|
-
this.abortSignal,
|
|
2793
|
-
this.runtimeDeadlineMs,
|
|
2794
|
-
resolveClaimedWorkerToolRuntimeTimeoutMs([claimed], {
|
|
2795
|
-
runtimeDeadlineMs: this.runtimeDeadlineMs,
|
|
2796
|
-
}),
|
|
2797
|
-
);
|
|
2798
|
-
} catch (error) {
|
|
2799
|
-
this.rejectRequests(
|
|
2800
|
-
claimed,
|
|
2801
|
-
await this.failureForRejectedToolRequest(claimed, error),
|
|
2802
|
-
);
|
|
2803
|
-
return;
|
|
2804
|
-
}
|
|
2805
|
-
this.settleRequests(
|
|
2806
|
-
claimed,
|
|
2807
|
-
await this.completeDurableToolRequest(claimed, result),
|
|
2808
|
-
);
|
|
2809
|
-
} catch (receiptError) {
|
|
2810
|
-
tripRuntimePersistenceLatch(this.latch, receiptError);
|
|
2811
|
-
this.rejectRequests(
|
|
2812
|
-
claimed,
|
|
2813
|
-
new RuntimeReceiptPersistenceError(
|
|
2814
|
-
'Tool call succeeded but durable receipt could not be marked completed',
|
|
2815
|
-
[receiptError],
|
|
2816
|
-
),
|
|
2817
|
-
);
|
|
2818
|
-
} finally {
|
|
2819
|
-
slot.release();
|
|
2820
|
-
}
|
|
2821
|
-
};
|
|
2822
|
-
// Bounded dispatch waves: an unbatched tool group can carry a whole chunk's
|
|
2823
|
-
// rows (enrich compiles to one ~245-row chunk). Dispatching them all at
|
|
2824
|
-
// once billed every call before the first receipt-completion failure could
|
|
2825
|
-
// trip the latch (the 250/250 incident). Instead, dispatch one wave, AWAIT
|
|
2826
|
-
// it so any completion failure trips the latch, then read the latch before
|
|
2827
|
-
// the next wave — remaining waves are prevented (counted + rejected) exactly
|
|
2828
|
-
// like the mid-wave gate above. The wave calls stay fully concurrent; the
|
|
2829
|
-
// only added cost on the healthy path is one latch read per wave.
|
|
2830
|
-
for (const wave of partitionDispatchWaves(
|
|
2831
|
-
claimedRequests,
|
|
2832
|
-
() => 1,
|
|
2833
|
-
PROVIDER_DISPATCH_WAVE_SIZE,
|
|
2834
|
-
)) {
|
|
2835
|
-
if (this.latch.tripped) {
|
|
2836
|
-
for (const claimed of wave) {
|
|
2837
|
-
this.latch.preventedCallCount += 1 + claimed.followers.length;
|
|
2838
|
-
this.rejectRequests(
|
|
2839
|
-
claimed,
|
|
2840
|
-
new RuntimePersistenceCircuitOpenError(this.latch),
|
|
2841
|
-
);
|
|
2842
|
-
}
|
|
2843
|
-
continue;
|
|
2844
|
-
}
|
|
2845
|
-
await Promise.all(wave.map(dispatchOne));
|
|
2846
|
-
}
|
|
2847
|
-
recordRunnerPerfTrace({
|
|
2848
|
-
req: this.req,
|
|
2849
|
-
phase: 'runner.tool.group',
|
|
2850
|
-
ms: nowMs() - groupStartedAt,
|
|
2851
|
-
extra: {
|
|
2852
|
-
toolId,
|
|
2853
|
-
requests: requestCount,
|
|
2854
|
-
executed: claimedRequests.length,
|
|
2855
|
-
batched: false,
|
|
2856
|
-
},
|
|
2857
|
-
});
|
|
2858
|
-
return;
|
|
2859
|
-
}
|
|
2326
|
+
const workerMapResidentRowBytes = (row: unknown): number =>
|
|
2327
|
+
new TextEncoder().encode(JSON.stringify(row)).byteLength +
|
|
2328
|
+
WORKERS_EDGE_MAP_RESIDENT_ROW_STRUCTURAL_OVERHEAD_BYTES;
|
|
2860
2329
|
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
WORKER_TOOL_BATCH_DEFAULT_PARALLELISM,
|
|
2870
|
-
),
|
|
2871
|
-
abortSignal: this.abortSignal,
|
|
2872
|
-
runtimeDeadlineMs: this.runtimeDeadlineMs,
|
|
2873
|
-
reportBackpressure: (retryAfterMs) =>
|
|
2874
|
-
this.reportBackpressure(toolId, retryAfterMs),
|
|
2875
|
-
resolveToolContract: this.resolvePacing,
|
|
2876
|
-
onRequestsSettled: this.onRequestsSettled,
|
|
2877
|
-
callbacks: this.callbacks,
|
|
2878
|
-
completeRequests: async (entries) =>
|
|
2879
|
-
await this.completeDurableToolRequests(entries),
|
|
2880
|
-
failRequests: async (claimedRequests, error) =>
|
|
2881
|
-
await this.failDurableToolRequests(claimedRequests, error),
|
|
2882
|
-
settleRequest: (claimed, result) => this.settleRequests(claimed, result),
|
|
2883
|
-
rejectRequest: (claimed, error) => this.rejectRequests(claimed, error),
|
|
2884
|
-
latch: this.latch,
|
|
2885
|
-
});
|
|
2886
|
-
recordRunnerPerfTrace({
|
|
2887
|
-
req: this.req,
|
|
2888
|
-
phase: 'runner.tool.group',
|
|
2889
|
-
ms: nowMs() - batchStartedAt,
|
|
2890
|
-
extra: {
|
|
2891
|
-
toolId,
|
|
2892
|
-
requests: requestCount,
|
|
2893
|
-
executed: claimedRequests.length,
|
|
2894
|
-
batched: true,
|
|
2895
|
-
},
|
|
2896
|
-
});
|
|
2330
|
+
function isRuntimeReceiptPersistenceError(error: unknown): boolean {
|
|
2331
|
+
if (!error || typeof error !== 'object') return false;
|
|
2332
|
+
if (error instanceof RuntimeReceiptPersistenceError) return true;
|
|
2333
|
+
if (
|
|
2334
|
+
error instanceof Error &&
|
|
2335
|
+
error.name === 'RuntimeReceiptPersistenceError'
|
|
2336
|
+
) {
|
|
2337
|
+
return true;
|
|
2897
2338
|
}
|
|
2339
|
+
const nestedErrors = (error as { errors?: unknown }).errors;
|
|
2340
|
+
return (
|
|
2341
|
+
Array.isArray(nestedErrors) &&
|
|
2342
|
+
nestedErrors.some(isRuntimeReceiptPersistenceError)
|
|
2343
|
+
);
|
|
2898
2344
|
}
|
|
2899
2345
|
|
|
2900
|
-
function
|
|
2901
|
-
|
|
2902
|
-
)
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
}
|
|
2912
|
-
return byTool;
|
|
2346
|
+
function isRunFatalWorkerRowError(error: unknown): boolean {
|
|
2347
|
+
return (
|
|
2348
|
+
isCoordinatorProgressAuthorizationError(error) ||
|
|
2349
|
+
isRowIsolationExemptError(error) ||
|
|
2350
|
+
isRuntimeReceiptPersistenceError(error) ||
|
|
2351
|
+
isRuntimePersistenceCircuitOpenError(error) ||
|
|
2352
|
+
error instanceof RuntimeSheetAttemptLeaseLostError ||
|
|
2353
|
+
error instanceof RuntimeLeaseLostError ||
|
|
2354
|
+
(error instanceof Error &&
|
|
2355
|
+
error.name === 'RuntimeSheetAttemptLeaseLostError')
|
|
2356
|
+
);
|
|
2913
2357
|
}
|
|
2914
2358
|
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
reportBackpressure: (retryAfterMs: number) => void;
|
|
2924
|
-
resolveToolContract: WorkerPacingResolver;
|
|
2925
|
-
onRequestsSettled?: (count: number) => void;
|
|
2926
|
-
callbacks?: WorkerCtxCallbacks;
|
|
2927
|
-
completeRequests: (
|
|
2928
|
-
entries: Array<{ claimed: ClaimedWorkerToolBatchRequest; result: unknown }>,
|
|
2929
|
-
) => Promise<unknown[]>;
|
|
2930
|
-
failRequests: (
|
|
2931
|
-
requests: ClaimedWorkerToolBatchRequest[],
|
|
2932
|
-
error: unknown,
|
|
2933
|
-
) => Promise<void>;
|
|
2934
|
-
settleRequest: (
|
|
2935
|
-
request: ClaimedWorkerToolBatchRequest,
|
|
2936
|
-
result: unknown,
|
|
2937
|
-
) => void;
|
|
2938
|
-
rejectRequest: (
|
|
2939
|
-
request: ClaimedWorkerToolBatchRequest,
|
|
2940
|
-
error: unknown,
|
|
2941
|
-
) => void;
|
|
2942
|
-
latch: RuntimePersistenceLatch;
|
|
2943
|
-
}): Promise<void> {
|
|
2944
|
-
const compiledBatches = compileRequestsWithStrategy({
|
|
2945
|
-
requests: input.requests,
|
|
2946
|
-
strategy: input.strategy,
|
|
2947
|
-
getPayload: (request) => request.request.input,
|
|
2948
|
-
});
|
|
2949
|
-
recordRunnerPerfTrace({
|
|
2950
|
-
req: input.req,
|
|
2951
|
-
phase: 'runner.tool.batch.compile',
|
|
2952
|
-
ms: 0,
|
|
2953
|
-
extra: {
|
|
2954
|
-
sourceOperation: input.strategy.sourceOperation,
|
|
2955
|
-
batchOperation: input.strategy.batchOperation,
|
|
2956
|
-
requests: input.requests.length,
|
|
2957
|
-
batches: compiledBatches.length,
|
|
2958
|
-
batchSizes: compiledBatches.map((batch) => batch.memberRequests.length),
|
|
2959
|
-
},
|
|
2960
|
-
});
|
|
2359
|
+
/**
|
|
2360
|
+
* In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
|
|
2361
|
+
* throughput pacing (provider or Deepline limiter), not a tool defect, so it
|
|
2362
|
+
* gets more patience than the generic 3-attempt budget: with retry-after-aware
|
|
2363
|
+
* escalating delays (capped at 5s) this absorbs roughly 25s of sustained
|
|
2364
|
+
* throttling before the call fails. Every retry still charges the Governor's
|
|
2365
|
+
* retry budget, so a runaway storm stays bounded and loud.
|
|
2366
|
+
*/
|
|
2961
2367
|
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
// Chunk parallelism is the Governor's per-tool suggestion (provider rate
|
|
2965
|
-
// hints tightened to the policy ceiling), bounded by the batch count.
|
|
2966
|
-
batchSize: Math.max(
|
|
2967
|
-
1,
|
|
2968
|
-
Math.min(input.suggestedParallelism, compiledBatches.length || 1),
|
|
2969
|
-
),
|
|
2970
|
-
// Bounded dispatch wave: cap the provider calls (counted by batch member
|
|
2971
|
-
// size) a single chunk dispatches before the latch is read again, so a
|
|
2972
|
-
// persistence failure prevents the rest of a large batch group instead of
|
|
2973
|
-
// billing it all. A single over-budget batch still forms its own wave.
|
|
2974
|
-
maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
|
|
2975
|
-
weightOf: (batch) => batch.memberRequests.length,
|
|
2976
|
-
execute: async (batch) => {
|
|
2977
|
-
// Circuit breaker: once a persistence failure has occurred in this map,
|
|
2978
|
-
// do NOT dispatch this batch's provider call. The chunk plumbing rejects
|
|
2979
|
-
// its member requests with the latch error.
|
|
2980
|
-
if (input.latch.tripped) {
|
|
2981
|
-
input.latch.preventedCallCount += batch.memberRequests.length;
|
|
2982
|
-
throw new RuntimePersistenceCircuitOpenError(input.latch);
|
|
2983
|
-
}
|
|
2984
|
-
const toolContract = await input
|
|
2985
|
-
.resolveToolContract(batch.batchOperation)
|
|
2986
|
-
.catch(() => null);
|
|
2987
|
-
// One provider call per batch → one tool slot (budget + global
|
|
2988
|
-
// concurrency + per-(org,provider) pacing) around the whole batch.
|
|
2989
|
-
const slot = await input.governor.acquireToolSlot(batch.batchOperation, {
|
|
2990
|
-
signal: input.abortSignal,
|
|
2991
|
-
});
|
|
2992
|
-
try {
|
|
2993
|
-
input.callbacks?.onToolCalled?.(batch.batchOperation, nowMs());
|
|
2994
|
-
return await executeTool(
|
|
2995
|
-
input.req,
|
|
2996
|
-
{
|
|
2997
|
-
id: `batch:${batch.memberRequests.map((request) => request.request.id).join('|')}`,
|
|
2998
|
-
toolId: batch.batchOperation,
|
|
2999
|
-
input: batch.batchPayload,
|
|
3000
|
-
},
|
|
3001
|
-
undefined,
|
|
3002
|
-
input.reportBackpressure,
|
|
3003
|
-
() => input.governor.chargeBudget('retry'),
|
|
3004
|
-
toolContract?.retrySafeTransientHttp === true,
|
|
3005
|
-
input.abortSignal,
|
|
3006
|
-
input.runtimeDeadlineMs,
|
|
3007
|
-
resolveClaimedWorkerToolRuntimeTimeoutMs(batch.memberRequests, {
|
|
3008
|
-
runtimeDeadlineMs: input.runtimeDeadlineMs,
|
|
3009
|
-
}),
|
|
3010
|
-
);
|
|
3011
|
-
} catch (error) {
|
|
3012
|
-
input.callbacks?.onToolFailed?.(batch.batchOperation, nowMs());
|
|
3013
|
-
throw error;
|
|
3014
|
-
} finally {
|
|
3015
|
-
slot.release();
|
|
3016
|
-
}
|
|
3017
|
-
},
|
|
3018
|
-
onChunkComplete: async (
|
|
3019
|
-
chunkResults: Array<
|
|
3020
|
-
ChunkExecutionResult<(typeof compiledBatches)[number], unknown>
|
|
3021
|
-
>,
|
|
3022
|
-
) => {
|
|
3023
|
-
for (const entry of chunkResults) {
|
|
3024
|
-
if (entry.error !== undefined) {
|
|
3025
|
-
// One batch's provider error stays scoped to that batch's member
|
|
3026
|
-
// requests. Sibling batches in this chunk keep their results so a
|
|
3027
|
-
// single provider hiccup cannot cascade into a whole-map failure.
|
|
3028
|
-
let rejection: unknown = entry.error;
|
|
3029
|
-
try {
|
|
3030
|
-
await input.failRequests(entry.request.memberRequests, entry.error);
|
|
3031
|
-
} catch (receiptError) {
|
|
3032
|
-
tripRuntimePersistenceLatch(input.latch, receiptError);
|
|
3033
|
-
rejection = new RuntimeReceiptPersistenceError(
|
|
3034
|
-
'Tool call failed and durable receipts could not be marked failed',
|
|
3035
|
-
[entry.error, receiptError],
|
|
3036
|
-
);
|
|
3037
|
-
}
|
|
3038
|
-
for (const claimed of entry.request.memberRequests) {
|
|
3039
|
-
input.rejectRequest(claimed, rejection);
|
|
3040
|
-
}
|
|
3041
|
-
continue;
|
|
3042
|
-
}
|
|
3043
|
-
const batchResult = isToolExecuteResult(entry.result)
|
|
3044
|
-
? entry.result.toolOutput.raw
|
|
3045
|
-
: entry.result;
|
|
3046
|
-
const splitResults =
|
|
3047
|
-
batchResult != null
|
|
3048
|
-
? entry.request.splitResults(batchResult)
|
|
3049
|
-
: entry.request.memberRequests.map(() => null);
|
|
3050
|
-
let completedResults: unknown[];
|
|
3051
|
-
try {
|
|
3052
|
-
completedResults = await input.completeRequests(
|
|
3053
|
-
entry.request.memberRequests.map((claimed, index) => ({
|
|
3054
|
-
claimed,
|
|
3055
|
-
result: wrapWorkerToolResult(
|
|
3056
|
-
claimed.request.toolId,
|
|
3057
|
-
splitResults[index] ?? null,
|
|
3058
|
-
toolMetadataFallback(claimed.request.toolId),
|
|
3059
|
-
),
|
|
3060
|
-
})),
|
|
3061
|
-
);
|
|
3062
|
-
} catch (receiptError) {
|
|
3063
|
-
tripRuntimePersistenceLatch(input.latch, receiptError);
|
|
3064
|
-
const rejection = new RuntimeReceiptPersistenceError(
|
|
3065
|
-
'Tool call succeeded but durable receipts could not be marked completed',
|
|
3066
|
-
[receiptError],
|
|
3067
|
-
);
|
|
3068
|
-
for (const claimed of entry.request.memberRequests) {
|
|
3069
|
-
input.rejectRequest(claimed, rejection);
|
|
3070
|
-
}
|
|
3071
|
-
continue;
|
|
3072
|
-
}
|
|
3073
|
-
for (let index = 0; index < completedResults.length; index += 1) {
|
|
3074
|
-
const claimed = entry.request.memberRequests[index]!;
|
|
3075
|
-
const request = claimed.request;
|
|
3076
|
-
input.settleRequest(claimed, completedResults[index]);
|
|
3077
|
-
}
|
|
3078
|
-
}
|
|
3079
|
-
},
|
|
3080
|
-
}).catch(async (error) => {
|
|
3081
|
-
let rejection: unknown = error;
|
|
3082
|
-
try {
|
|
3083
|
-
await input.failRequests(input.requests, error);
|
|
3084
|
-
} catch (receiptError) {
|
|
3085
|
-
tripRuntimePersistenceLatch(input.latch, receiptError);
|
|
3086
|
-
rejection = new RuntimeReceiptPersistenceError(
|
|
3087
|
-
'Tool call failed and durable receipts could not be marked failed',
|
|
3088
|
-
[error, receiptError],
|
|
3089
|
-
);
|
|
3090
|
-
}
|
|
3091
|
-
for (const claimed of input.requests) {
|
|
3092
|
-
input.rejectRequest(claimed, rejection);
|
|
3093
|
-
}
|
|
3094
|
-
});
|
|
2368
|
+
function sleepWorkerMs(ms: number): Promise<void> {
|
|
2369
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3095
2370
|
}
|
|
3096
2371
|
|
|
3097
2372
|
function isCompletedWorkerFieldValue(value: unknown): boolean {
|
|
@@ -3102,7 +2377,8 @@ function isCompletedWorkerFieldValue(value: unknown): boolean {
|
|
|
3102
2377
|
);
|
|
3103
2378
|
}
|
|
3104
2379
|
|
|
3105
|
-
type
|
|
2380
|
+
type WorkerMapCompletedChunkSummary<T extends Record<string, unknown>> = {
|
|
2381
|
+
status: 'completed';
|
|
3106
2382
|
chunkIndex: number;
|
|
3107
2383
|
rangeStart: number;
|
|
3108
2384
|
rangeEnd: number;
|
|
@@ -3126,6 +2402,23 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
|
|
|
3126
2402
|
cachedRows?: T[];
|
|
3127
2403
|
};
|
|
3128
2404
|
|
|
2405
|
+
type WorkerMapBlockedChunkSummary = {
|
|
2406
|
+
status: 'blocked';
|
|
2407
|
+
chunkIndex: number;
|
|
2408
|
+
rangeStart: number;
|
|
2409
|
+
rangeEnd: number;
|
|
2410
|
+
rowsRead: number;
|
|
2411
|
+
rowsBlocked: number;
|
|
2412
|
+
rowKeySamples: string[];
|
|
2413
|
+
blockedRowSamples: Record<string, unknown>[];
|
|
2414
|
+
outputDatasetId: string;
|
|
2415
|
+
message: string;
|
|
2416
|
+
};
|
|
2417
|
+
|
|
2418
|
+
type WorkerMapChunkSummary<T extends Record<string, unknown>> =
|
|
2419
|
+
| WorkerMapCompletedChunkSummary<T>
|
|
2420
|
+
| WorkerMapBlockedChunkSummary;
|
|
2421
|
+
|
|
3129
2422
|
function serializeDurableStepValue<T>(value: T, depth = 0): T {
|
|
3130
2423
|
if (depth > 20 || value == null) return value;
|
|
3131
2424
|
if (isToolExecuteResult(value)) return serializeToolExecuteResult(value) as T;
|
|
@@ -3338,6 +2631,7 @@ async function executeWorkerStepProgram(
|
|
|
3338
2631
|
...(resolution.status ? { status: resolution.status } : {}),
|
|
3339
2632
|
};
|
|
3340
2633
|
};
|
|
2634
|
+
|
|
3341
2635
|
const executeStep = async (): Promise<WorkerStepResolution> =>
|
|
3342
2636
|
workflowStep
|
|
3343
2637
|
? await (
|
|
@@ -3562,7 +2856,7 @@ async function postRuntimeEgressFetch(
|
|
|
3562
2856
|
) {
|
|
3563
2857
|
const response = await fetchRuntimeApi(
|
|
3564
2858
|
req.baseUrl,
|
|
3565
|
-
'/api/v2/
|
|
2859
|
+
'/api/v2/internal/play-runtime/egress-fetch',
|
|
3566
2860
|
{
|
|
3567
2861
|
method: 'POST',
|
|
3568
2862
|
headers: {
|
|
@@ -4138,6 +3432,81 @@ function resolveSheetContractFromReq(
|
|
|
4138
3432
|
}
|
|
4139
3433
|
}
|
|
4140
3434
|
|
|
3435
|
+
/**
|
|
3436
|
+
* Run-fatal teardown: release the runtime-sheet attempt leases and work-receipt
|
|
3437
|
+
* leases this run still holds, so an immediate rerun proceeds without waiting
|
|
3438
|
+
* out the 10-minute lease TTL. Every release is attempt/owner fenced downstream,
|
|
3439
|
+
* so it can never weaken a different run's live lease. Failures are logged loud
|
|
3440
|
+
* and swallowed here — the caller still throws the original run-fatal error.
|
|
3441
|
+
*/
|
|
3442
|
+
async function releaseRuntimeLeasesOnTeardown(input: {
|
|
3443
|
+
req: RunRequest;
|
|
3444
|
+
leasedSheetNamespaces: Set<string>;
|
|
3445
|
+
emit: (event: RunnerEvent) => void;
|
|
3446
|
+
}): Promise<void> {
|
|
3447
|
+
const { req, leasedSheetNamespaces, emit } = input;
|
|
3448
|
+
const scope = runtimeSheetSessionScope(req);
|
|
3449
|
+
for (const tableNamespace of leasedSheetNamespaces) {
|
|
3450
|
+
const sheetContract = resolveSheetContractFromReq(req, tableNamespace);
|
|
3451
|
+
if (!sheetContract) continue;
|
|
3452
|
+
try {
|
|
3453
|
+
const released = await harnessReleaseSheetAttempt({
|
|
3454
|
+
...scope,
|
|
3455
|
+
tableNamespace,
|
|
3456
|
+
sheetContract,
|
|
3457
|
+
runId: req.runId,
|
|
3458
|
+
attemptOwnerRunId: req.runId,
|
|
3459
|
+
attemptSeq: req.runAttempt ?? 0,
|
|
3460
|
+
});
|
|
3461
|
+
if (released.released > 0) {
|
|
3462
|
+
emit({
|
|
3463
|
+
type: 'log',
|
|
3464
|
+
level: 'info',
|
|
3465
|
+
message: `Released ${released.released} runtime sheet row lease(s) for ctx.dataset("${tableNamespace}") on run-fatal teardown.`,
|
|
3466
|
+
ts: nowMs(),
|
|
3467
|
+
});
|
|
3468
|
+
}
|
|
3469
|
+
} catch (releaseError) {
|
|
3470
|
+
emit({
|
|
3471
|
+
type: 'log',
|
|
3472
|
+
level: 'warn',
|
|
3473
|
+
message: `Runtime sheet attempt release failed for ctx.dataset("${tableNamespace}") on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
|
|
3474
|
+
releaseError instanceof Error
|
|
3475
|
+
? releaseError.message
|
|
3476
|
+
: String(releaseError)
|
|
3477
|
+
}`,
|
|
3478
|
+
ts: nowMs(),
|
|
3479
|
+
});
|
|
3480
|
+
}
|
|
3481
|
+
}
|
|
3482
|
+
try {
|
|
3483
|
+
const released = await harnessReleaseRuntimeReceipts({
|
|
3484
|
+
...scope,
|
|
3485
|
+
runId: req.runId,
|
|
3486
|
+
runAttempt: req.runAttempt ?? 0,
|
|
3487
|
+
});
|
|
3488
|
+
if (released.released > 0) {
|
|
3489
|
+
emit({
|
|
3490
|
+
type: 'log',
|
|
3491
|
+
level: 'info',
|
|
3492
|
+
message: `Released ${released.released} work-receipt lease(s) on run-fatal teardown.`,
|
|
3493
|
+
ts: nowMs(),
|
|
3494
|
+
});
|
|
3495
|
+
}
|
|
3496
|
+
} catch (releaseError) {
|
|
3497
|
+
emit({
|
|
3498
|
+
type: 'log',
|
|
3499
|
+
level: 'warn',
|
|
3500
|
+
message: `Work-receipt release failed on run-fatal teardown (rerun will fall back to lease-TTL expiry): ${
|
|
3501
|
+
releaseError instanceof Error
|
|
3502
|
+
? releaseError.message
|
|
3503
|
+
: String(releaseError)
|
|
3504
|
+
}`,
|
|
3505
|
+
ts: nowMs(),
|
|
3506
|
+
});
|
|
3507
|
+
}
|
|
3508
|
+
}
|
|
3509
|
+
|
|
4141
3510
|
function staticPipelineFromReq(req: RunRequest): PlayStaticPipeline | null {
|
|
4142
3511
|
const snapshot = req.contractSnapshot as
|
|
4143
3512
|
| { staticPipeline?: unknown }
|
|
@@ -4180,6 +3549,13 @@ async function persistCompletedMapRows(input: {
|
|
|
4180
3549
|
rows: Record<string, unknown>[];
|
|
4181
3550
|
outputFields: string[];
|
|
4182
3551
|
extraOutputFields?: string[];
|
|
3552
|
+
attemptId?: string | null;
|
|
3553
|
+
attemptOwnerRunId?: string | null;
|
|
3554
|
+
attemptExpiresAt?: string | null;
|
|
3555
|
+
attemptSeq?: number | null;
|
|
3556
|
+
budgetMeter?: WorkerWorkBudgetMeter;
|
|
3557
|
+
force?: boolean;
|
|
3558
|
+
forceFailedRows?: boolean;
|
|
4183
3559
|
}): Promise<{
|
|
4184
3560
|
rows: number;
|
|
4185
3561
|
written: number;
|
|
@@ -4225,23 +3601,28 @@ async function persistCompletedMapRows(input: {
|
|
|
4225
3601
|
sheetContract,
|
|
4226
3602
|
rows,
|
|
4227
3603
|
outputFields,
|
|
3604
|
+
attemptId: input.attemptId ?? null,
|
|
3605
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3606
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3607
|
+
attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
|
|
3608
|
+
forceFailedRows: input.forceFailedRows === true,
|
|
4228
3609
|
};
|
|
3610
|
+
input.budgetMeter?.count('sheet');
|
|
4229
3611
|
const expectedKeys = [
|
|
4230
3612
|
...new Set(
|
|
4231
3613
|
input.rows
|
|
4232
|
-
.map((row) => resolveMapRowOutcomeKey(row))
|
|
4233
|
-
.filter(
|
|
3614
|
+
.map((row) => resolveMapRowOutcomeKey(row) ?? row.key)
|
|
3615
|
+
.filter(Boolean),
|
|
4234
3616
|
),
|
|
4235
|
-
];
|
|
4236
|
-
const expectedVisibleRows =
|
|
4237
|
-
expectedKeys.length > 0 ? expectedKeys.length : rows.length;
|
|
3617
|
+
] as string[];
|
|
3618
|
+
const expectedVisibleRows = expectedKeys.length || rows.length;
|
|
4238
3619
|
const readVisibleRowCount = async () => {
|
|
4239
3620
|
if (expectedKeys.length > 0) {
|
|
4240
3621
|
const result = await harnessReadSheetDatasetRowKeys({
|
|
4241
3622
|
...sessionScope,
|
|
4242
3623
|
tableNamespace,
|
|
4243
3624
|
runId: req.runId,
|
|
4244
|
-
rowMode: '
|
|
3625
|
+
rowMode: 'terminalAnyRun',
|
|
4245
3626
|
keys: expectedKeys,
|
|
4246
3627
|
});
|
|
4247
3628
|
return result.keys.length;
|
|
@@ -4257,6 +3638,24 @@ async function persistCompletedMapRows(input: {
|
|
|
4257
3638
|
return result.rows.length;
|
|
4258
3639
|
};
|
|
4259
3640
|
const result = await harnessPersistCompletedSheetRows(persistRequest);
|
|
3641
|
+
const hasAttemptFence = Boolean(input.attemptId?.trim());
|
|
3642
|
+
const assertAttemptStillOwnsTerminalWrites = (
|
|
3643
|
+
written: number,
|
|
3644
|
+
visible: number | null,
|
|
3645
|
+
) => {
|
|
3646
|
+
// Recovery finalization may include rows completed by an earlier attempt.
|
|
3647
|
+
// A complete terminal readback proves they were not lost.
|
|
3648
|
+
if (
|
|
3649
|
+
!hasAttemptFence ||
|
|
3650
|
+
written === rows.length ||
|
|
3651
|
+
(visible !== null && visible >= expectedVisibleRows)
|
|
3652
|
+
) {
|
|
3653
|
+
return;
|
|
3654
|
+
}
|
|
3655
|
+
throw new Error(
|
|
3656
|
+
`Runtime sheet attempt lost ownership for ${tableNamespace}: wrote ${written}/${rows.length}; run ${req.runId}; attempt ${input.attemptId}.`,
|
|
3657
|
+
);
|
|
3658
|
+
};
|
|
4260
3659
|
console.warn('[play-runner.persist_completed_map_rows.result]', {
|
|
4261
3660
|
runId: req.runId,
|
|
4262
3661
|
playName: req.playName,
|
|
@@ -4281,11 +3680,19 @@ async function persistCompletedMapRows(input: {
|
|
|
4281
3680
|
sheetContract,
|
|
4282
3681
|
rows: input.rows.map((row) => runtimeCsvStorageRow(row)),
|
|
4283
3682
|
runId: req.runId,
|
|
3683
|
+
attemptId: input.attemptId ?? null,
|
|
3684
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3685
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3686
|
+
attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
|
|
3687
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
4284
3688
|
inputOffset: 0,
|
|
4285
3689
|
});
|
|
3690
|
+
input.budgetMeter?.count('sheet');
|
|
4286
3691
|
const retry = await harnessPersistCompletedSheetRows(persistRequest);
|
|
3692
|
+
input.budgetMeter?.count('sheet');
|
|
4287
3693
|
retryWritten = retry.rowsWritten;
|
|
4288
3694
|
retryVisible = await readVisibleRowCount();
|
|
3695
|
+
assertAttemptStillOwnsTerminalWrites(retryWritten, retryVisible);
|
|
4289
3696
|
if (retryVisible >= expectedVisibleRows) {
|
|
4290
3697
|
return {
|
|
4291
3698
|
rows: rows.length,
|
|
@@ -4299,6 +3706,7 @@ async function persistCompletedMapRows(input: {
|
|
|
4299
3706
|
`Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}/${expectedVisibleRows}; run ${req.runId}.`,
|
|
4300
3707
|
);
|
|
4301
3708
|
}
|
|
3709
|
+
assertAttemptStillOwnsTerminalWrites(result.rowsWritten, visibleRows);
|
|
4302
3710
|
}
|
|
4303
3711
|
if (result.rowsWritten !== rows.length && visibleRows < expectedVisibleRows) {
|
|
4304
3712
|
throw new Error(
|
|
@@ -4320,8 +3728,14 @@ async function applyLiveMapRowUpdates(input: {
|
|
|
4320
3728
|
updates: Array<Omit<PlayRowUpdate, 'rowId'> & { runId?: string }>;
|
|
4321
3729
|
outputFields: string[];
|
|
4322
3730
|
extraOutputFields?: string[];
|
|
3731
|
+
attemptId?: string | null;
|
|
3732
|
+
attemptOwnerRunId?: string | null;
|
|
3733
|
+
attemptExpiresAt?: string | null;
|
|
3734
|
+
attemptSeq?: number | null;
|
|
3735
|
+
budgetMeter?: WorkerWorkBudgetMeter;
|
|
4323
3736
|
}): Promise<void> {
|
|
4324
3737
|
if (input.updates.length === 0) return;
|
|
3738
|
+
input.budgetMeter?.count('sheet');
|
|
4325
3739
|
const outputFields = [
|
|
4326
3740
|
...input.outputFields,
|
|
4327
3741
|
...(input.extraOutputFields ?? []).filter(
|
|
@@ -4344,7 +3758,16 @@ async function applyLiveMapRowUpdates(input: {
|
|
|
4344
3758
|
contractSnapshot: input.req.contractSnapshot,
|
|
4345
3759
|
runId: input.req.runId,
|
|
4346
3760
|
userEmail: input.req.userEmail,
|
|
4347
|
-
updates: input.updates
|
|
3761
|
+
updates: input.updates.map((update) => ({
|
|
3762
|
+
...update,
|
|
3763
|
+
attemptId: update.attemptId ?? input.attemptId ?? null,
|
|
3764
|
+
attemptOwnerRunId:
|
|
3765
|
+
update.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
|
|
3766
|
+
attemptExpiresAt:
|
|
3767
|
+
update.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
|
|
3768
|
+
attemptSeq:
|
|
3769
|
+
update.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3770
|
+
})),
|
|
4348
3771
|
},
|
|
4349
3772
|
);
|
|
4350
3773
|
}
|
|
@@ -4355,18 +3778,39 @@ async function prepareMapRows(input: {
|
|
|
4355
3778
|
rows: Record<string, unknown>[];
|
|
4356
3779
|
inputOffset: number;
|
|
4357
3780
|
outputFields: string[];
|
|
3781
|
+
attemptId?: string | null;
|
|
3782
|
+
attemptOwnerRunId?: string | null;
|
|
3783
|
+
attemptExpiresAt?: string | null;
|
|
3784
|
+
attemptSeq?: number | null;
|
|
3785
|
+
budgetMeter?: WorkerWorkBudgetMeter;
|
|
4358
3786
|
force?: boolean;
|
|
4359
3787
|
}): Promise<{
|
|
4360
3788
|
inserted: number;
|
|
4361
3789
|
skipped: number;
|
|
4362
3790
|
pendingRows: Record<string, unknown>[];
|
|
4363
3791
|
completedRows: Record<string, unknown>[];
|
|
3792
|
+
blockedRows: Record<string, unknown>[];
|
|
3793
|
+
attemptId?: string | null;
|
|
3794
|
+
attemptOwnerRunId?: string | null;
|
|
3795
|
+
attemptExpiresAt?: string | null;
|
|
3796
|
+
attemptSeq?: number | null;
|
|
4364
3797
|
}> {
|
|
4365
3798
|
if (input.rows.length === 0) {
|
|
4366
|
-
return {
|
|
3799
|
+
return {
|
|
3800
|
+
inserted: 0,
|
|
3801
|
+
skipped: 0,
|
|
3802
|
+
pendingRows: [],
|
|
3803
|
+
completedRows: [],
|
|
3804
|
+
blockedRows: [],
|
|
3805
|
+
attemptId: input.attemptId ?? null,
|
|
3806
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3807
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3808
|
+
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3809
|
+
};
|
|
4367
3810
|
}
|
|
4368
3811
|
const sessionScope = runtimeSheetSessionScope(input.req);
|
|
4369
3812
|
const rows = input.rows.map((row) => runtimeCsvStorageRow(row));
|
|
3813
|
+
input.budgetMeter?.count('sheet');
|
|
4370
3814
|
const result = await harnessStartSheetDataset({
|
|
4371
3815
|
...sessionScope,
|
|
4372
3816
|
tableNamespace: input.tableNamespace,
|
|
@@ -4377,6 +3821,11 @@ async function prepareMapRows(input: {
|
|
|
4377
3821
|
}),
|
|
4378
3822
|
rows,
|
|
4379
3823
|
inputOffset: input.inputOffset,
|
|
3824
|
+
attemptId: input.attemptId ?? null,
|
|
3825
|
+
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3826
|
+
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3827
|
+
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3828
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
|
|
4380
3829
|
...(input.force ? { force: true } : {}),
|
|
4381
3830
|
});
|
|
4382
3831
|
for (const timing of result.timings ?? []) {
|
|
@@ -4406,6 +3855,13 @@ async function prepareMapRows(input: {
|
|
|
4406
3855
|
skipped: result.skipped,
|
|
4407
3856
|
pendingRows: result.pendingRows,
|
|
4408
3857
|
completedRows: result.completedRows,
|
|
3858
|
+
blockedRows: result.blockedRows ?? [],
|
|
3859
|
+
attemptId: result.attemptId ?? input.attemptId ?? null,
|
|
3860
|
+
attemptOwnerRunId:
|
|
3861
|
+
result.attemptOwnerRunId ?? input.attemptOwnerRunId ?? null,
|
|
3862
|
+
attemptExpiresAt: result.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
|
|
3863
|
+
attemptSeq:
|
|
3864
|
+
result.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
4409
3865
|
};
|
|
4410
3866
|
}
|
|
4411
3867
|
|
|
@@ -4551,14 +4007,6 @@ function createCoordinatorRatePort(req: RunRequest): CoordinatorRatePort {
|
|
|
4551
4007
|
* memoized per isolate. No hints → null (pacing is a no-op; the Governor's
|
|
4552
4008
|
* global tool-concurrency slot still applies).
|
|
4553
4009
|
*/
|
|
4554
|
-
type WorkerPacingResolver = (
|
|
4555
|
-
toolId: string,
|
|
4556
|
-
) => Promise<
|
|
4557
|
-
(ResolvedPacingPolicy & { retrySafeTransientHttp: boolean }) | null
|
|
4558
|
-
>;
|
|
4559
|
-
|
|
4560
|
-
type WorkerToolActionCacheVersionResolver = (toolId: string) => Promise<string>;
|
|
4561
|
-
|
|
4562
4010
|
function createWorkerToolActionCacheVersionResolver(
|
|
4563
4011
|
req: RunRequest,
|
|
4564
4012
|
): WorkerToolActionCacheVersionResolver {
|
|
@@ -4566,9 +4014,7 @@ function createWorkerToolActionCacheVersionResolver(
|
|
|
4566
4014
|
return (toolId: string) => {
|
|
4567
4015
|
const normalized = String(toolId || '').trim();
|
|
4568
4016
|
if (!normalized) {
|
|
4569
|
-
return Promise.reject(
|
|
4570
|
-
new Error('Runtime tool metadata lookup requires a non-empty tool id.'),
|
|
4571
|
-
);
|
|
4017
|
+
return Promise.reject(new Error('Tool metadata needs tool id.'));
|
|
4572
4018
|
}
|
|
4573
4019
|
const cached = cache.get(normalized);
|
|
4574
4020
|
if (cached) return cached;
|
|
@@ -4605,6 +4051,44 @@ function createWorkerToolActionCacheVersionResolver(
|
|
|
4605
4051
|
};
|
|
4606
4052
|
}
|
|
4607
4053
|
|
|
4054
|
+
function createWorkerToolAuthScopeDigestResolver(req: RunRequest) {
|
|
4055
|
+
// Return type is inferred from createRuntimeToolAuthScopeDigestResolver
|
|
4056
|
+
// (RuntimeToolAuthScopeDigestResolver), whose `invalidate` member is
|
|
4057
|
+
// REQUIRED — the AUTH_SCOPE_CHANGED re-claim wiring below calls it directly,
|
|
4058
|
+
// and a resolver without eviction support must be a type error.
|
|
4059
|
+
return createRuntimeToolAuthScopeDigestResolver({
|
|
4060
|
+
missingToolIdMessage: 'Tool auth scope needs tool id.',
|
|
4061
|
+
fetchDigest: async (normalized) => {
|
|
4062
|
+
const runScopedPath =
|
|
4063
|
+
`/api/v2/plays/runtime-tools/${encodeURIComponent(normalized)}/auth-scope` +
|
|
4064
|
+
`?runId=${encodeURIComponent(req.runId)}`;
|
|
4065
|
+
const res = await fetchRuntimeApi(req.baseUrl, runScopedPath, {
|
|
4066
|
+
method: 'GET',
|
|
4067
|
+
headers: {
|
|
4068
|
+
authorization: `Bearer ${req.executorToken}`,
|
|
4069
|
+
'cache-control': 'no-store',
|
|
4070
|
+
pragma: 'no-cache',
|
|
4071
|
+
},
|
|
4072
|
+
});
|
|
4073
|
+
if (!res.ok) {
|
|
4074
|
+
throw new Error(
|
|
4075
|
+
`Runtime tool auth scope lookup for ${normalized} failed (${res.status}): ${await res.text()}`,
|
|
4076
|
+
);
|
|
4077
|
+
}
|
|
4078
|
+
const body = (await res.json().catch(() => null)) as {
|
|
4079
|
+
digest?: unknown;
|
|
4080
|
+
} | null;
|
|
4081
|
+
const digest = typeof body?.digest === 'string' ? body.digest.trim() : '';
|
|
4082
|
+
if (!digest) {
|
|
4083
|
+
throw new Error(
|
|
4084
|
+
`Runtime tool auth scope lookup for ${normalized} returned no digest.`,
|
|
4085
|
+
);
|
|
4086
|
+
}
|
|
4087
|
+
return digest;
|
|
4088
|
+
},
|
|
4089
|
+
});
|
|
4090
|
+
}
|
|
4091
|
+
|
|
4608
4092
|
function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
|
|
4609
4093
|
const cache = new Map<
|
|
4610
4094
|
string,
|
|
@@ -4746,13 +4230,22 @@ function createMinimalWorkerCtx(
|
|
|
4746
4230
|
workflowStep?: WorkflowStep,
|
|
4747
4231
|
abortSignal?: AbortSignal,
|
|
4748
4232
|
callbacks?: WorkerCtxCallbacks,
|
|
4233
|
+
workBudgetMeter: WorkerWorkBudgetMeter = createWorkerWorkBudgetMeter({
|
|
4234
|
+
nowMs,
|
|
4235
|
+
}),
|
|
4749
4236
|
runtimeDeadlineMs?: number,
|
|
4237
|
+
// Namespaces this run took a runtime-sheet attempt lease on. The run-fatal
|
|
4238
|
+
// teardown reads this set to release exactly those leases (never a namespace
|
|
4239
|
+
// whose table the run never created), so a rerun proceeds immediately.
|
|
4240
|
+
leasedSheetNamespaces?: Set<string>,
|
|
4750
4241
|
receiptSalvage?: ReceiptSalvageBuffer,
|
|
4751
4242
|
): unknown {
|
|
4752
4243
|
const { governor, resolvePacing: resolveToolPacing } =
|
|
4753
4244
|
createGovernorForRun(req);
|
|
4754
4245
|
const resolveToolActionCacheVersion =
|
|
4755
4246
|
createWorkerToolActionCacheVersionResolver(req);
|
|
4247
|
+
const resolveToolAuthScopeDigest =
|
|
4248
|
+
createWorkerToolAuthScopeDigestResolver(req);
|
|
4756
4249
|
// Play-call depth/count/per-parent budgets, child-play concurrency, and the
|
|
4757
4250
|
// lineage snapshot are owned by the Governor (createGovernorForRun above).
|
|
4758
4251
|
// The worker keeps only substrate mechanism here.
|
|
@@ -4763,7 +4256,7 @@ function createMinimalWorkerCtx(
|
|
|
4763
4256
|
if (!auth) return {};
|
|
4764
4257
|
const response = await fetchRuntimeApi(
|
|
4765
4258
|
req.baseUrl,
|
|
4766
|
-
'/api/v2/
|
|
4259
|
+
'/api/v2/internal/play-runtime',
|
|
4767
4260
|
{
|
|
4768
4261
|
method: 'POST',
|
|
4769
4262
|
headers: {
|
|
@@ -4802,10 +4295,48 @@ function createMinimalWorkerCtx(
|
|
|
4802
4295
|
orgId: req.orgId,
|
|
4803
4296
|
preloadedDbSessions: req.preloadedDbSessions ?? null,
|
|
4804
4297
|
userEmail: req.userEmail,
|
|
4298
|
+
runtimeTestFaultHeader: req.runtimeTestFaultHeader ?? null,
|
|
4299
|
+
runAttempt: req.runAttempt ?? 0,
|
|
4300
|
+
receiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
|
|
4805
4301
|
});
|
|
4302
|
+
const executeDispatchedTool = async (input: {
|
|
4303
|
+
id: string;
|
|
4304
|
+
toolId: string;
|
|
4305
|
+
input: Record<string, unknown>;
|
|
4306
|
+
workflowStep?: unknown;
|
|
4307
|
+
callbacks?: WorkerCtxCallbacks;
|
|
4308
|
+
onProviderBackpressure?: (retryAfterMs: number) => void;
|
|
4309
|
+
onRetryAttempt?: () => void;
|
|
4310
|
+
transientHttpRetrySafe?: boolean;
|
|
4311
|
+
durableCallReceiptKey?: string | null;
|
|
4312
|
+
executionAuthScopeDigest?: string | null;
|
|
4313
|
+
providerIdempotencyKey?: string | null;
|
|
4314
|
+
runtimeTimeoutMs?: number;
|
|
4315
|
+
directOptions?: WorkerToolDirectOptions;
|
|
4316
|
+
}): Promise<ToolExecuteResult> =>
|
|
4317
|
+
await executeToolWithLifecycle(
|
|
4318
|
+
req,
|
|
4319
|
+
{
|
|
4320
|
+
id: input.id,
|
|
4321
|
+
toolId: input.toolId,
|
|
4322
|
+
input: input.input,
|
|
4323
|
+
durableCallReceiptKey: input.durableCallReceiptKey,
|
|
4324
|
+
executionAuthScopeDigest: input.executionAuthScopeDigest,
|
|
4325
|
+
providerIdempotencyKey: input.providerIdempotencyKey,
|
|
4326
|
+
},
|
|
4327
|
+
input.workflowStep as WorkflowStep | undefined,
|
|
4328
|
+
input.callbacks,
|
|
4329
|
+
input.onProviderBackpressure,
|
|
4330
|
+
input.onRetryAttempt,
|
|
4331
|
+
input.transientHttpRetrySafe === true,
|
|
4332
|
+
abortSignal,
|
|
4333
|
+
runtimeDeadlineMs,
|
|
4334
|
+
input.runtimeTimeoutMs,
|
|
4335
|
+
input.directOptions,
|
|
4336
|
+
);
|
|
4806
4337
|
const executeWithRuntimeReceipt = async <T>(
|
|
4807
4338
|
key: string,
|
|
4808
|
-
execute: () => Promise<T> | T,
|
|
4339
|
+
execute: (context: WorkerRuntimeReceiptExecutionContext) => Promise<T> | T,
|
|
4809
4340
|
options: {
|
|
4810
4341
|
repairRunningReceiptForSameRun?: boolean;
|
|
4811
4342
|
reclaimRunning?: boolean;
|
|
@@ -4819,7 +4350,8 @@ function createMinimalWorkerCtx(
|
|
|
4819
4350
|
runId: req.runId,
|
|
4820
4351
|
key,
|
|
4821
4352
|
receiptStore,
|
|
4822
|
-
execute: async () =>
|
|
4353
|
+
execute: async (context) =>
|
|
4354
|
+
serializeDurableStepValue(await execute(context)),
|
|
4823
4355
|
repairRunningReceiptForSameRun:
|
|
4824
4356
|
options.repairRunningReceiptForSameRun ?? false,
|
|
4825
4357
|
reclaimRunning: options.reclaimRunning ?? false,
|
|
@@ -4861,24 +4393,54 @@ function createMinimalWorkerCtx(
|
|
|
4861
4393
|
staleAfterSeconds,
|
|
4862
4394
|
});
|
|
4863
4395
|
};
|
|
4864
|
-
|
|
4865
|
-
|
|
4866
|
-
|
|
4396
|
+
const emitAuthScopeReclaimRunLog = (input: {
|
|
4397
|
+
toolId: string;
|
|
4398
|
+
requestIds: string[];
|
|
4399
|
+
}) => {
|
|
4400
|
+
// Durable run-log marker for the bounded AUTH_SCOPE_CHANGED re-claim
|
|
4401
|
+
// (mirrors the shared-runtime marker in shared_libs/play-runtime/context.ts
|
|
4402
|
+
// so run logs expose the credential transition on every execution path).
|
|
4403
|
+
emitEvent({
|
|
4404
|
+
type: 'log',
|
|
4405
|
+
level: 'info',
|
|
4406
|
+
message:
|
|
4407
|
+
`ctx.tools.execute(${input.toolId}): auth_scope_changed_reclaim ` +
|
|
4408
|
+
`(requests: ${input.requestIds.join(', ')}); credential identity ` +
|
|
4409
|
+
`changed mid-run, re-resolved the auth scope and re-claimed fresh ` +
|
|
4410
|
+
`receipts under the new scope.`,
|
|
4411
|
+
ts: nowMs(),
|
|
4412
|
+
});
|
|
4413
|
+
};
|
|
4867
4414
|
const rootPersistenceLatch = createRuntimePersistenceLatch();
|
|
4868
|
-
const rootToolBatchScheduler = new WorkerToolBatchScheduler(
|
|
4415
|
+
const rootToolBatchScheduler = new WorkerToolBatchScheduler({
|
|
4869
4416
|
req,
|
|
4870
4417
|
governor,
|
|
4871
|
-
resolveToolPacing,
|
|
4418
|
+
resolvePacing: resolveToolPacing,
|
|
4872
4419
|
resolveToolActionCacheVersion,
|
|
4420
|
+
resolveToolAuthScopeDigest,
|
|
4421
|
+
invalidateToolAuthScopeDigest: (toolId) =>
|
|
4422
|
+
resolveToolAuthScopeDigest.invalidate(toolId),
|
|
4423
|
+
onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
|
|
4424
|
+
executeTool: executeDispatchedTool,
|
|
4425
|
+
serializeDurableValue: serializeDurableStepValue,
|
|
4426
|
+
deserializeDurableValue: deserializeDurableStepValue,
|
|
4427
|
+
nowMs,
|
|
4428
|
+
batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
|
|
4429
|
+
recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
|
|
4873
4430
|
abortSignal,
|
|
4874
|
-
undefined,
|
|
4875
4431
|
callbacks,
|
|
4876
4432
|
receiptStore,
|
|
4877
|
-
true,
|
|
4433
|
+
allowLocalRetryReceipts: true,
|
|
4434
|
+
runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
|
|
4435
|
+
runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
|
|
4436
|
+
req,
|
|
4437
|
+
runtimeReceiptLeaseTtlMs(req) ?? PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
4438
|
+
),
|
|
4439
|
+
budgetMeter: workBudgetMeter,
|
|
4878
4440
|
runtimeDeadlineMs,
|
|
4879
|
-
rootPersistenceLatch,
|
|
4441
|
+
persistenceLatch: rootPersistenceLatch,
|
|
4880
4442
|
receiptSalvage,
|
|
4881
|
-
);
|
|
4443
|
+
});
|
|
4882
4444
|
// Local ancestry chain that always ENDS with the currently-executing play
|
|
4883
4445
|
// (req.playName). The /api/v2/plays/run lineage validator requires the
|
|
4884
4446
|
// submitted ancestry's tail to equal the executor token's play name (i.e.
|
|
@@ -4913,23 +4475,33 @@ function createMinimalWorkerCtx(
|
|
|
4913
4475
|
): Promise<unknown> => {
|
|
4914
4476
|
const mapStartedAt = nowMs();
|
|
4915
4477
|
const mapNodeId = `map:${name}`;
|
|
4916
|
-
// Map-wide persistence-failure circuit breaker. Shared across every chunk's
|
|
4917
|
-
// scheduler (the scheduler is rebuilt per chunk) and the map's sheet-persist
|
|
4918
|
-
// flush chain, so the first persistence failure halts NEW provider dispatch
|
|
4919
|
-
// for the rest of the map and its prevented-call count surfaces in the
|
|
4920
|
-
// runMap failure message below.
|
|
4921
|
-
const persistenceLatch = createRuntimePersistenceLatch();
|
|
4922
4478
|
const inputRows = rows;
|
|
4923
4479
|
const rowCountHint = datasetRowCountHint(inputRows);
|
|
4924
4480
|
const baseOffset = 0;
|
|
4925
4481
|
const fieldEntries = Object.entries(fieldsDef);
|
|
4926
4482
|
const plan = req.executionPlan;
|
|
4927
|
-
const
|
|
4483
|
+
const mapDispatchPlan = chooseWorkerMapDispatchPlan({
|
|
4928
4484
|
mapName: name,
|
|
4929
4485
|
rowCountHint,
|
|
4930
4486
|
executionPlan: plan,
|
|
4931
4487
|
staticPipeline: staticPipelineFromReq(req),
|
|
4932
4488
|
});
|
|
4489
|
+
const rowsPerChunk = mapDispatchPlan.rowsPerChunk;
|
|
4490
|
+
recordRunnerPerfTrace({
|
|
4491
|
+
req,
|
|
4492
|
+
phase: 'runner.map_dispatch_plan',
|
|
4493
|
+
ms: 0,
|
|
4494
|
+
extra: {
|
|
4495
|
+
mapName: name,
|
|
4496
|
+
rowsPerChunk,
|
|
4497
|
+
rowCountHint,
|
|
4498
|
+
estimatedExternalStepsPerRow:
|
|
4499
|
+
mapDispatchPlan.estimatedExternalStepsPerRow,
|
|
4500
|
+
workEstimate: mapDispatchPlan.workEstimate,
|
|
4501
|
+
reason: mapDispatchPlan.reason,
|
|
4502
|
+
dynamicWorkerArtifactVersion: DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION,
|
|
4503
|
+
},
|
|
4504
|
+
});
|
|
4933
4505
|
const outputFields = fieldEntries.map(([field]) => field);
|
|
4934
4506
|
const updateMapProgress = (
|
|
4935
4507
|
progress: LiveNodeProgressSnapshot,
|
|
@@ -4945,6 +4517,22 @@ function createMinimalWorkerCtx(
|
|
|
4945
4517
|
forceFlush: options?.forceFlush ?? true,
|
|
4946
4518
|
});
|
|
4947
4519
|
};
|
|
4520
|
+
let queuedMapProgress: Promise<void> = Promise.resolve();
|
|
4521
|
+
let queuedMapProgressError: unknown = null;
|
|
4522
|
+
const enqueueMapProgressUpdate = (
|
|
4523
|
+
progress: LiveNodeProgressSnapshot,
|
|
4524
|
+
options?: { forceFlush?: boolean },
|
|
4525
|
+
) => {
|
|
4526
|
+
queuedMapProgress = queuedMapProgress
|
|
4527
|
+
.then(() => Promise.resolve(updateMapProgress(progress, options)))
|
|
4528
|
+
.catch((error) => {
|
|
4529
|
+
queuedMapProgressError ??= error;
|
|
4530
|
+
});
|
|
4531
|
+
};
|
|
4532
|
+
const flushQueuedMapProgressUpdates = async () => {
|
|
4533
|
+
await queuedMapProgress;
|
|
4534
|
+
if (queuedMapProgressError) throw queuedMapProgressError;
|
|
4535
|
+
};
|
|
4948
4536
|
const formatMapProgressMessage = (completed: number, total?: number) =>
|
|
4949
4537
|
typeof total === 'number' && Number.isFinite(total) && total > 0
|
|
4950
4538
|
? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
|
|
@@ -4969,7 +4557,9 @@ function createMinimalWorkerCtx(
|
|
|
4969
4557
|
return formatMapProgressMessage(completed, input.total);
|
|
4970
4558
|
};
|
|
4971
4559
|
const formatMapProcessingMessage = (rowsToExecute: number) =>
|
|
4972
|
-
rowsToExecute > 0
|
|
4560
|
+
rowsToExecute > 0
|
|
4561
|
+
? `Processing ${rowsToExecute.toLocaleString()} rows`
|
|
4562
|
+
: null;
|
|
4973
4563
|
const formatMapExecutionHeartbeatMessage = (input: {
|
|
4974
4564
|
rowsToExecute: number;
|
|
4975
4565
|
startedRows: number;
|
|
@@ -4983,7 +4573,7 @@ function createMinimalWorkerCtx(
|
|
|
4983
4573
|
const waitingRows = Math.max(0, rowsToExecute - startedRows);
|
|
4984
4574
|
const parts = [
|
|
4985
4575
|
activeRows > 0 ? `${activeRows.toLocaleString()} active` : null,
|
|
4986
|
-
waitingRows > 0 ? `${waitingRows.toLocaleString()}
|
|
4576
|
+
waitingRows > 0 ? `${waitingRows.toLocaleString()} waiting` : null,
|
|
4987
4577
|
completedRows > 0 ? `${completedRows.toLocaleString()} done` : null,
|
|
4988
4578
|
].filter((part): part is string => Boolean(part));
|
|
4989
4579
|
const base =
|
|
@@ -5147,6 +4737,12 @@ function createMinimalWorkerCtx(
|
|
|
5147
4737
|
): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
|
|
5148
4738
|
const chunkStartedAt = nowMs();
|
|
5149
4739
|
assertNotAborted(abortSignal);
|
|
4740
|
+
const rowAttempt = makeRuntimeSheetAttempt({
|
|
4741
|
+
runId: req.runId,
|
|
4742
|
+
runAttempt: req.runAttempt,
|
|
4743
|
+
mapName: name,
|
|
4744
|
+
chunkIndex,
|
|
4745
|
+
});
|
|
5150
4746
|
const keyStartedAt = nowMs();
|
|
5151
4747
|
const chunkEntries = chunkRows.map((row, localIndex) => {
|
|
5152
4748
|
const absoluteIndex = baseOffset + chunkStart + localIndex;
|
|
@@ -5174,8 +4770,21 @@ function createMinimalWorkerCtx(
|
|
|
5174
4770
|
...mapRowOutcomeRuntimeFields({ key: rowKey }),
|
|
5175
4771
|
})),
|
|
5176
4772
|
inputOffset: baseOffset + chunkStart,
|
|
4773
|
+
...rowAttempt,
|
|
4774
|
+
budgetMeter: workBudgetMeter,
|
|
5177
4775
|
force: !!req.force,
|
|
5178
4776
|
});
|
|
4777
|
+
const activeRowAttempt = {
|
|
4778
|
+
attemptId: prepared.attemptId ?? rowAttempt.attemptId,
|
|
4779
|
+
attemptOwnerRunId:
|
|
4780
|
+
prepared.attemptOwnerRunId ?? rowAttempt.attemptOwnerRunId,
|
|
4781
|
+
attemptExpiresAt:
|
|
4782
|
+
prepared.attemptExpiresAt ?? rowAttempt.attemptExpiresAt,
|
|
4783
|
+
attemptSeq: prepared.attemptSeq ?? rowAttempt.attemptSeq,
|
|
4784
|
+
};
|
|
4785
|
+
// Record that this run holds attempt leases in `name`'s sheet so a
|
|
4786
|
+
// run-fatal teardown releases them (the table now exists).
|
|
4787
|
+
leasedSheetNamespaces?.add(name);
|
|
5179
4788
|
recordRunnerPerfTrace({
|
|
5180
4789
|
req,
|
|
5181
4790
|
phase: 'runner.map_chunk.prepare_rows',
|
|
@@ -5188,14 +4797,15 @@ function createMinimalWorkerCtx(
|
|
|
5188
4797
|
skipped: prepared.skipped,
|
|
5189
4798
|
pendingRows: prepared.pendingRows.length,
|
|
5190
4799
|
completedRows: prepared.completedRows.length,
|
|
4800
|
+
blockedRows: prepared.blockedRows.length,
|
|
5191
4801
|
},
|
|
5192
4802
|
});
|
|
5193
4803
|
const progressTotalRows = rowCountHint ?? chunkRows.length;
|
|
5194
4804
|
const preparedCompletedRows = Math.min(
|
|
5195
4805
|
progressTotalRows,
|
|
5196
|
-
totalRowsWritten
|
|
4806
|
+
totalRowsWritten,
|
|
5197
4807
|
);
|
|
5198
|
-
|
|
4808
|
+
enqueueMapProgressUpdate({
|
|
5199
4809
|
completed: preparedCompletedRows,
|
|
5200
4810
|
total: progressTotalRows,
|
|
5201
4811
|
startedAt: mapStartedAt,
|
|
@@ -5208,6 +4818,9 @@ function createMinimalWorkerCtx(
|
|
|
5208
4818
|
const pendingKeys = new Set<string>();
|
|
5209
4819
|
const pendingRowsByKey = new Map<string, Record<string, unknown>>();
|
|
5210
4820
|
const completedKeys = new Set<string>();
|
|
4821
|
+
const completedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
4822
|
+
const blockedKeys = new Set<string>();
|
|
4823
|
+
const blockedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
5211
4824
|
const preparedKeys = new Set<string>();
|
|
5212
4825
|
for (const row of prepared.pendingRows) {
|
|
5213
4826
|
const key =
|
|
@@ -5225,14 +4838,95 @@ function createMinimalWorkerCtx(
|
|
|
5225
4838
|
deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
|
|
5226
4839
|
if (key) {
|
|
5227
4840
|
completedKeys.add(key);
|
|
4841
|
+
completedRowsByKey.set(key, row);
|
|
5228
4842
|
preparedKeys.add(key);
|
|
5229
4843
|
}
|
|
5230
4844
|
}
|
|
4845
|
+
for (const row of prepared.blockedRows) {
|
|
4846
|
+
const key =
|
|
4847
|
+
resolveMapRowOutcomeKey(row) ??
|
|
4848
|
+
deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
|
|
4849
|
+
if (key) {
|
|
4850
|
+
blockedKeys.add(key);
|
|
4851
|
+
blockedRowsByKey.set(key, row);
|
|
4852
|
+
preparedKeys.add(key);
|
|
4853
|
+
}
|
|
4854
|
+
}
|
|
4855
|
+
if (blockedKeys.size > 0) {
|
|
4856
|
+
const rowKeySamples = Array.from(blockedKeys).slice(
|
|
4857
|
+
0,
|
|
4858
|
+
MAP_ROW_FAILURE_SAMPLE_LIMIT,
|
|
4859
|
+
);
|
|
4860
|
+
const message =
|
|
4861
|
+
`ctx.dataset("${name}") chunk ${chunkIndex} is blocked by ` +
|
|
4862
|
+
`${blockedKeys.size} active runtime sheet row lease(s) owned by another attempt. ` +
|
|
4863
|
+
`Retry after the owning attempt finishes or the row leases expire.` +
|
|
4864
|
+
(rowKeySamples.length > 0
|
|
4865
|
+
? ` Blocked row keys: ${rowKeySamples.join(', ')}`
|
|
4866
|
+
: '');
|
|
4867
|
+
recordRunnerPerfTrace({
|
|
4868
|
+
req,
|
|
4869
|
+
phase: 'runner.map_chunk.blocked_rows',
|
|
4870
|
+
ms: nowMs() - prepareStartedAt,
|
|
4871
|
+
extra: {
|
|
4872
|
+
mapName: name,
|
|
4873
|
+
chunkIndex,
|
|
4874
|
+
rowsBlocked: blockedKeys.size,
|
|
4875
|
+
rowKeySamples,
|
|
4876
|
+
},
|
|
4877
|
+
});
|
|
4878
|
+
enqueueMapProgressUpdate({
|
|
4879
|
+
completed: preparedCompletedRows,
|
|
4880
|
+
total: progressTotalRows,
|
|
4881
|
+
waitingRows: blockedKeys.size,
|
|
4882
|
+
startedAt: mapStartedAt,
|
|
4883
|
+
message,
|
|
4884
|
+
});
|
|
4885
|
+
return {
|
|
4886
|
+
status: 'blocked',
|
|
4887
|
+
chunkIndex,
|
|
4888
|
+
rangeStart: baseOffset + chunkStart,
|
|
4889
|
+
rangeEnd: baseOffset + chunkStart + chunkRows.length,
|
|
4890
|
+
rowsRead: chunkRows.length,
|
|
4891
|
+
rowsBlocked: blockedKeys.size,
|
|
4892
|
+
rowKeySamples,
|
|
4893
|
+
blockedRowSamples: rowKeySamples.map((rowKey) => {
|
|
4894
|
+
const blockedRow = blockedRowsByKey.get(rowKey);
|
|
4895
|
+
const attemptId =
|
|
4896
|
+
typeof blockedRow?.__deeplineAttemptId === 'string'
|
|
4897
|
+
? blockedRow.__deeplineAttemptId
|
|
4898
|
+
: typeof blockedRow?._attempt_id === 'string'
|
|
4899
|
+
? blockedRow._attempt_id
|
|
4900
|
+
: activeRowAttempt.attemptId;
|
|
4901
|
+
return {
|
|
4902
|
+
__deeplineRowKey: rowKey,
|
|
4903
|
+
__deeplineAttemptId: attemptId,
|
|
4904
|
+
};
|
|
4905
|
+
}),
|
|
4906
|
+
outputDatasetId: `map:${name}`,
|
|
4907
|
+
message,
|
|
4908
|
+
};
|
|
4909
|
+
}
|
|
5231
4910
|
const missingPreparedRows = chunkEntries.filter(
|
|
5232
4911
|
({ rowKey }) => !preparedKeys.has(rowKey),
|
|
5233
4912
|
);
|
|
5234
|
-
|
|
5235
|
-
|
|
4913
|
+
if (missingPreparedRows.length > 0) {
|
|
4914
|
+
const rowKeySamples = missingPreparedRows
|
|
4915
|
+
.map(({ rowKey }) => rowKey)
|
|
4916
|
+
.slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
|
|
4917
|
+
throw new Error(
|
|
4918
|
+
`ctx.dataset("${name}") runtime preparation omitted ${missingPreparedRows.length} row disposition(s). ` +
|
|
4919
|
+
`Every input row must be claimed, completed, or blocked before execution starts.` +
|
|
4920
|
+
(rowKeySamples.length > 0
|
|
4921
|
+
? ` Missing row keys: ${rowKeySamples.join(', ')}`
|
|
4922
|
+
: ''),
|
|
4923
|
+
);
|
|
4924
|
+
}
|
|
4925
|
+
const rowsToExecuteEntries = chunkEntries.filter(({ rowKey }) =>
|
|
4926
|
+
pendingKeys.has(rowKey),
|
|
4927
|
+
);
|
|
4928
|
+
const completedChunkEntries = chunkEntries.filter(({ rowKey }) =>
|
|
4929
|
+
completedKeys.has(rowKey),
|
|
5236
4930
|
);
|
|
5237
4931
|
const uniqueRowsToExecuteEntries = [
|
|
5238
4932
|
...new Map(
|
|
@@ -5249,18 +4943,15 @@ function createMinimalWorkerCtx(
|
|
|
5249
4943
|
rowsToExecute.length,
|
|
5250
4944
|
);
|
|
5251
4945
|
if (processingMessage) {
|
|
5252
|
-
|
|
4946
|
+
enqueueMapProgressUpdate({
|
|
5253
4947
|
completed: preparedCompletedRows,
|
|
5254
4948
|
total: progressTotalRows,
|
|
5255
4949
|
startedAt: mapStartedAt,
|
|
5256
4950
|
message: processingMessage,
|
|
5257
4951
|
});
|
|
5258
4952
|
}
|
|
5259
|
-
const rowsInserted = prepared.inserted
|
|
5260
|
-
const rowsSkipped =
|
|
5261
|
-
0,
|
|
5262
|
-
prepared.skipped - missingPreparedRows.length,
|
|
5263
|
-
);
|
|
4953
|
+
const rowsInserted = prepared.inserted;
|
|
4954
|
+
const rowsSkipped = prepared.skipped;
|
|
5264
4955
|
let completedExecutedRows = 0;
|
|
5265
4956
|
let failedExecutedRows = 0;
|
|
5266
4957
|
let startedExecutedRows = 0;
|
|
@@ -5268,12 +4959,7 @@ function createMinimalWorkerCtx(
|
|
|
5268
4959
|
let lastChunkProgressAt = 0;
|
|
5269
4960
|
let lastExecutionHeartbeatAt = nowMs();
|
|
5270
4961
|
const completedRowsForProgress = () =>
|
|
5271
|
-
Math.min(
|
|
5272
|
-
progressTotalRows,
|
|
5273
|
-
totalRowsWritten +
|
|
5274
|
-
prepared.completedRows.length +
|
|
5275
|
-
completedExecutedRows,
|
|
5276
|
-
);
|
|
4962
|
+
Math.min(progressTotalRows, totalRowsWritten + completedExecutedRows);
|
|
5277
4963
|
const reportExecutionHeartbeat = (force = false) => {
|
|
5278
4964
|
const now = nowMs();
|
|
5279
4965
|
if (
|
|
@@ -5283,7 +4969,7 @@ function createMinimalWorkerCtx(
|
|
|
5283
4969
|
return;
|
|
5284
4970
|
}
|
|
5285
4971
|
lastExecutionHeartbeatAt = now;
|
|
5286
|
-
|
|
4972
|
+
enqueueMapProgressUpdate(
|
|
5287
4973
|
{
|
|
5288
4974
|
completed: completedRowsForProgress(),
|
|
5289
4975
|
total: progressTotalRows,
|
|
@@ -5317,7 +5003,7 @@ function createMinimalWorkerCtx(
|
|
|
5317
5003
|
return;
|
|
5318
5004
|
}
|
|
5319
5005
|
lastChunkProgressAt = now;
|
|
5320
|
-
|
|
5006
|
+
enqueueMapProgressUpdate(
|
|
5321
5007
|
{
|
|
5322
5008
|
completed,
|
|
5323
5009
|
total: progressTotalRows,
|
|
@@ -5348,23 +5034,44 @@ function createMinimalWorkerCtx(
|
|
|
5348
5034
|
const failedRowEntries: Array<
|
|
5349
5035
|
{ row: T & Record<string, unknown>; error: string } | undefined
|
|
5350
5036
|
> = new Array(rowsToExecute.length);
|
|
5037
|
+
const fatalReceiptPersistenceRowEntries: Array<
|
|
5038
|
+
{ row: T & Record<string, unknown>; error: string } | undefined
|
|
5039
|
+
> = new Array(rowsToExecute.length);
|
|
5351
5040
|
const executedCellMetaPatches: Array<
|
|
5352
5041
|
Record<string, WorkerCellMetaPatchEntry> | undefined
|
|
5353
5042
|
> = new Array(rowsToExecute.length);
|
|
5354
|
-
const
|
|
5043
|
+
const persistenceLatch = createRuntimePersistenceLatch();
|
|
5044
|
+
const toolBatchScheduler = new WorkerToolBatchScheduler({
|
|
5355
5045
|
req,
|
|
5356
5046
|
governor,
|
|
5357
|
-
resolveToolPacing,
|
|
5047
|
+
resolvePacing: resolveToolPacing,
|
|
5358
5048
|
resolveToolActionCacheVersion,
|
|
5049
|
+
resolveToolAuthScopeDigest,
|
|
5050
|
+
invalidateToolAuthScopeDigest: (toolId) =>
|
|
5051
|
+
resolveToolAuthScopeDigest.invalidate(toolId),
|
|
5052
|
+
onAuthScopeReclaim: emitAuthScopeReclaimRunLog,
|
|
5053
|
+
executeTool: executeDispatchedTool,
|
|
5054
|
+
serializeDurableValue: serializeDurableStepValue,
|
|
5055
|
+
deserializeDurableValue: deserializeDurableStepValue,
|
|
5056
|
+
nowMs,
|
|
5057
|
+
batchGraceMs: req.testPolicyOverrides?.batchGraceMs,
|
|
5058
|
+
recordPerfTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
|
|
5359
5059
|
abortSignal,
|
|
5360
|
-
reportSettledToolRequests,
|
|
5060
|
+
onRequestsSettled: reportSettledToolRequests,
|
|
5361
5061
|
callbacks,
|
|
5362
5062
|
receiptStore,
|
|
5363
|
-
false,
|
|
5063
|
+
allowLocalRetryReceipts: false,
|
|
5064
|
+
runtimeReceiptLeaseTtlMs: runtimeReceiptLeaseTtlMs(req),
|
|
5065
|
+
runtimeReceiptHeartbeatIntervalMs: runtimePolicyHeartbeatIntervalMs(
|
|
5066
|
+
req,
|
|
5067
|
+
runtimeReceiptLeaseTtlMs(req) ??
|
|
5068
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
5069
|
+
),
|
|
5070
|
+
budgetMeter: workBudgetMeter,
|
|
5364
5071
|
runtimeDeadlineMs,
|
|
5365
5072
|
persistenceLatch,
|
|
5366
5073
|
receiptSalvage,
|
|
5367
|
-
);
|
|
5074
|
+
});
|
|
5368
5075
|
let stepCellsCompleted = 0;
|
|
5369
5076
|
let stepCellsSkipped = 0;
|
|
5370
5077
|
const generatedOutputFields = new Set<string>();
|
|
@@ -5443,6 +5150,8 @@ function createMinimalWorkerCtx(
|
|
|
5443
5150
|
tableNamespace: name,
|
|
5444
5151
|
outputFields,
|
|
5445
5152
|
extraOutputFields: Array.from(generatedOutputFields),
|
|
5153
|
+
budgetMeter: workBudgetMeter,
|
|
5154
|
+
...activeRowAttempt,
|
|
5446
5155
|
rows: [
|
|
5447
5156
|
...rowsToPersist.map(({ row, executedIndex }) =>
|
|
5448
5157
|
mapRowOutcomeRuntimeRow(
|
|
@@ -5506,9 +5215,6 @@ function createMinimalWorkerCtx(
|
|
|
5506
5215
|
});
|
|
5507
5216
|
persistFlushChain = task.catch((error) => {
|
|
5508
5217
|
persistFailure ??= error;
|
|
5509
|
-
// Output-sheet flush failed: trip the breaker so the scheduler stops
|
|
5510
|
-
// dispatching new provider calls for the rest of this map.
|
|
5511
|
-
tripRuntimePersistenceLatch(persistenceLatch, error);
|
|
5512
5218
|
});
|
|
5513
5219
|
return task;
|
|
5514
5220
|
};
|
|
@@ -5550,6 +5256,8 @@ function createMinimalWorkerCtx(
|
|
|
5550
5256
|
outputFields,
|
|
5551
5257
|
extraOutputFields,
|
|
5552
5258
|
updates,
|
|
5259
|
+
...activeRowAttempt,
|
|
5260
|
+
budgetMeter: workBudgetMeter,
|
|
5553
5261
|
});
|
|
5554
5262
|
} catch (error) {
|
|
5555
5263
|
liveUpdateFailures += 1;
|
|
@@ -5657,18 +5365,11 @@ function createMinimalWorkerCtx(
|
|
|
5657
5365
|
request.input,
|
|
5658
5366
|
workflowStep,
|
|
5659
5367
|
{
|
|
5660
|
-
force: request.force === true
|
|
5368
|
+
force: request.force === true,
|
|
5369
|
+
runForceRefresh: req.forceToolRefresh === true,
|
|
5370
|
+
runForceFailedRefresh: req.force === true,
|
|
5661
5371
|
staleAfterSeconds: request.staleAfterSeconds,
|
|
5662
5372
|
},
|
|
5663
|
-
// Parity with the play-level ctx.tools.execute (and the
|
|
5664
|
-
// cjs runner's map path): the public ToolExecutionRequest
|
|
5665
|
-
// runtime options must not be silently dropped for map
|
|
5666
|
-
// rows — receiptWaitMs bounds how long a resumed run
|
|
5667
|
-
// waits on a stuck `running` receipt before reclaiming.
|
|
5668
|
-
{
|
|
5669
|
-
timeoutMs: request.timeoutMs,
|
|
5670
|
-
receiptWaitMs: request.receiptWaitMs,
|
|
5671
|
-
},
|
|
5672
5373
|
);
|
|
5673
5374
|
},
|
|
5674
5375
|
},
|
|
@@ -5777,8 +5478,28 @@ function createMinimalWorkerCtx(
|
|
|
5777
5478
|
notePersistableRow(enriched);
|
|
5778
5479
|
reportChunkProgress(false);
|
|
5779
5480
|
} catch (rowError) {
|
|
5780
|
-
//
|
|
5781
|
-
//
|
|
5481
|
+
// Receipt persistence errors stay run-fatal, but keep the
|
|
5482
|
+
// row identity so teardown can make the recovery state loud.
|
|
5483
|
+
if (isRuntimeReceiptPersistenceError(rowError)) {
|
|
5484
|
+
const message = formatWorkerRowFailureMessage(rowError);
|
|
5485
|
+
if (activeField) {
|
|
5486
|
+
cellMetaPatch[activeField] = {
|
|
5487
|
+
status: 'failed',
|
|
5488
|
+
stage: activeField,
|
|
5489
|
+
runId: req.runId,
|
|
5490
|
+
error: message,
|
|
5491
|
+
};
|
|
5492
|
+
}
|
|
5493
|
+
executedCellMetaPatches[myIndex] =
|
|
5494
|
+
Object.keys(cellMetaPatch).length > 0
|
|
5495
|
+
? cellMetaPatch
|
|
5496
|
+
: undefined;
|
|
5497
|
+
fatalReceiptPersistenceRowEntries[myIndex] = {
|
|
5498
|
+
row: enriched as T & Record<string, unknown>,
|
|
5499
|
+
error: message,
|
|
5500
|
+
};
|
|
5501
|
+
throw rowError;
|
|
5502
|
+
}
|
|
5782
5503
|
if (isRunFatalWorkerRowError(rowError)) {
|
|
5783
5504
|
throw rowError;
|
|
5784
5505
|
}
|
|
@@ -5858,16 +5579,25 @@ function createMinimalWorkerCtx(
|
|
|
5858
5579
|
return results;
|
|
5859
5580
|
},
|
|
5860
5581
|
);
|
|
5861
|
-
|
|
5862
|
-
|
|
5863
|
-
|
|
5864
|
-
|
|
5865
|
-
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
5869
|
-
|
|
5870
|
-
|
|
5582
|
+
const workerResults = await executeWithRuntimeSheetAttemptHeartbeat({
|
|
5583
|
+
req,
|
|
5584
|
+
tableNamespace: name,
|
|
5585
|
+
sheetContract: requireSheetContract(req, name),
|
|
5586
|
+
...activeRowAttempt,
|
|
5587
|
+
rowKeys: uniqueRowsToExecuteEntries.map((entry) => entry.rowKey),
|
|
5588
|
+
execute: async () => {
|
|
5589
|
+
while (!workerSettled) {
|
|
5590
|
+
await Promise.race([
|
|
5591
|
+
workerResultsPromise,
|
|
5592
|
+
sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
|
|
5593
|
+
]);
|
|
5594
|
+
if (!workerSettled) {
|
|
5595
|
+
reportExecutionHeartbeat(false);
|
|
5596
|
+
}
|
|
5597
|
+
}
|
|
5598
|
+
return await workerResultsPromise;
|
|
5599
|
+
},
|
|
5600
|
+
});
|
|
5871
5601
|
recordRunnerPerfTrace({
|
|
5872
5602
|
req,
|
|
5873
5603
|
phase: 'runner.map_chunk.execute_workers',
|
|
@@ -5896,26 +5626,89 @@ function createMinimalWorkerCtx(
|
|
|
5896
5626
|
.slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
|
|
5897
5627
|
const fatalMapChunkSummary = async (
|
|
5898
5628
|
error: unknown,
|
|
5899
|
-
): Promise<
|
|
5900
|
-
|
|
5901
|
-
|
|
5902
|
-
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
|
|
5910
|
-
|
|
5911
|
-
|
|
5912
|
-
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
|
|
5917
|
-
|
|
5918
|
-
|
|
5629
|
+
): Promise<
|
|
5630
|
+
WorkerMapCompletedChunkSummary<T & Record<string, unknown>>
|
|
5631
|
+
> => {
|
|
5632
|
+
const fatalError = formatWorkerRowFailureMessage(error);
|
|
5633
|
+
const breakerMessage = persistenceLatch.tripped
|
|
5634
|
+
? ` Circuit breaker prevented ${persistenceLatch.preventedCallCount} queued provider call(s).`
|
|
5635
|
+
: '';
|
|
5636
|
+
return {
|
|
5637
|
+
status: 'completed',
|
|
5638
|
+
chunkIndex,
|
|
5639
|
+
rangeStart: baseOffset + chunkStart,
|
|
5640
|
+
rangeEnd: baseOffset + chunkStart,
|
|
5641
|
+
rowsRead: chunkRows.length,
|
|
5642
|
+
rowsWritten: 0,
|
|
5643
|
+
rowsExecuted: rowsToExecute.length,
|
|
5644
|
+
rowsCached: 0,
|
|
5645
|
+
rowsDuplicateReused: duplicateInputReuseCount,
|
|
5646
|
+
rowsInserted,
|
|
5647
|
+
rowsSkipped,
|
|
5648
|
+
rowsFailed: failedExecutedRows,
|
|
5649
|
+
rowFailureSamples: buildRowFailureSamples(),
|
|
5650
|
+
stepCellsCompleted,
|
|
5651
|
+
stepCellsSkipped,
|
|
5652
|
+
outputDatasetId: `map:${name}`,
|
|
5653
|
+
hash: await hashJson([]),
|
|
5654
|
+
fatalError: `${fatalError}${breakerMessage}`,
|
|
5655
|
+
preview: [],
|
|
5656
|
+
};
|
|
5657
|
+
};
|
|
5658
|
+
const markReceiptPersistenceFatalRowsFailed = async (error: unknown) => {
|
|
5659
|
+
const message = formatWorkerRowFailureMessage(error);
|
|
5660
|
+
const rows = uniqueRowsToExecuteEntries
|
|
5661
|
+
.map((entry, executedIndex) => {
|
|
5662
|
+
const failure = fatalReceiptPersistenceRowEntries[executedIndex];
|
|
5663
|
+
const row =
|
|
5664
|
+
failure?.row ??
|
|
5665
|
+
executedRows[executedIndex] ??
|
|
5666
|
+
(runtimeCsvExecutionRow(
|
|
5667
|
+
entry.row,
|
|
5668
|
+
pendingRowsByKey.get(entry.rowKey),
|
|
5669
|
+
) as T & Record<string, unknown>);
|
|
5670
|
+
return mapRowOutcomeRuntimeRow(
|
|
5671
|
+
failedMapRowOutcome({
|
|
5672
|
+
key: entry.rowKey,
|
|
5673
|
+
inputIndex: entry.absoluteIndex,
|
|
5674
|
+
data: row,
|
|
5675
|
+
cellMetaPatch: executedCellMetaPatches[executedIndex],
|
|
5676
|
+
error: failure?.error ?? message,
|
|
5677
|
+
}),
|
|
5678
|
+
);
|
|
5679
|
+
})
|
|
5680
|
+
.filter((row): row is Record<string, unknown> => row !== null);
|
|
5681
|
+
if (rows.length === 0) return;
|
|
5682
|
+
try {
|
|
5683
|
+
const failureStats = await persistCompletedMapRows({
|
|
5684
|
+
req,
|
|
5685
|
+
tableNamespace: name,
|
|
5686
|
+
outputFields,
|
|
5687
|
+
extraOutputFields: Array.from(generatedOutputFields),
|
|
5688
|
+
budgetMeter: workBudgetMeter,
|
|
5689
|
+
...activeRowAttempt,
|
|
5690
|
+
forceFailedRows: true,
|
|
5691
|
+
rows,
|
|
5692
|
+
});
|
|
5693
|
+
emitEvent({
|
|
5694
|
+
type: 'log',
|
|
5695
|
+
level: 'warn',
|
|
5696
|
+
message:
|
|
5697
|
+
`Runtime sheet marked ${failureStats.rows} row(s) failed for ctx.dataset("${name}") ` +
|
|
5698
|
+
`after a run-fatal receipt persistence error.`,
|
|
5699
|
+
ts: nowMs(),
|
|
5700
|
+
});
|
|
5701
|
+
} catch (cleanupError) {
|
|
5702
|
+
emitEvent({
|
|
5703
|
+
type: 'log',
|
|
5704
|
+
level: 'warn',
|
|
5705
|
+
message:
|
|
5706
|
+
`Runtime sheet failed-row cleanup after receipt persistence error failed for ctx.dataset("${name}"): ` +
|
|
5707
|
+
formatWorkerRowFailureMessage(cleanupError),
|
|
5708
|
+
ts: nowMs(),
|
|
5709
|
+
});
|
|
5710
|
+
}
|
|
5711
|
+
};
|
|
5919
5712
|
const persistRowsStartedAt = nowMs();
|
|
5920
5713
|
recordRunnerPerfTrace({
|
|
5921
5714
|
req,
|
|
@@ -5944,6 +5737,7 @@ function createMinimalWorkerCtx(
|
|
|
5944
5737
|
},
|
|
5945
5738
|
});
|
|
5946
5739
|
} catch (error) {
|
|
5740
|
+
tripRuntimePersistenceLatch(persistenceLatch, error);
|
|
5947
5741
|
recordRunnerPerfTrace({
|
|
5948
5742
|
req,
|
|
5949
5743
|
phase: 'runner.map_chunk.persist_rows_error',
|
|
@@ -5962,31 +5756,24 @@ function createMinimalWorkerCtx(
|
|
|
5962
5756
|
result.status === 'rejected',
|
|
5963
5757
|
);
|
|
5964
5758
|
if (rejectedWorker) {
|
|
5965
|
-
if (
|
|
5966
|
-
|
|
5967
|
-
|
|
5968
|
-
|
|
5969
|
-
|
|
5970
|
-
// fatal summary (NOT rethrown into the durable step) so the chunk is
|
|
5971
|
-
// not retried and provider calls are not re-billed.
|
|
5759
|
+
if (isRuntimeReceiptPersistenceError(rejectedWorker.reason)) {
|
|
5760
|
+
await markReceiptPersistenceFatalRowsFailed(rejectedWorker.reason);
|
|
5761
|
+
return await fatalMapChunkSummary(rejectedWorker.reason);
|
|
5762
|
+
}
|
|
5763
|
+
if (isRuntimePersistenceCircuitOpenError(rejectedWorker.reason)) {
|
|
5972
5764
|
return await fatalMapChunkSummary(rejectedWorker.reason);
|
|
5973
5765
|
}
|
|
5974
5766
|
throw rejectedWorker.reason;
|
|
5975
5767
|
}
|
|
5976
5768
|
const resultByKey = new Map<string, T & Record<string, unknown>>();
|
|
5977
|
-
for (const
|
|
5978
|
-
const
|
|
5979
|
-
|
|
5980
|
-
|
|
5981
|
-
|
|
5982
|
-
|
|
5983
|
-
|
|
5984
|
-
|
|
5985
|
-
const cleanedRow = stripMapRowOutcomeRuntimeFields(
|
|
5986
|
-
publicCsvOutputRow(completedRow),
|
|
5987
|
-
);
|
|
5988
|
-
resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
|
|
5989
|
-
}
|
|
5769
|
+
for (const entry of completedChunkEntries) {
|
|
5770
|
+
const completedRow = completedRowsByKey.get(entry.rowKey);
|
|
5771
|
+
if (!completedRow || resultByKey.has(entry.rowKey)) continue;
|
|
5772
|
+
resultByKey.set(
|
|
5773
|
+
entry.rowKey,
|
|
5774
|
+
cloneCsvAliasedRow(entry.row, completedRow) as T &
|
|
5775
|
+
Record<string, unknown>,
|
|
5776
|
+
);
|
|
5990
5777
|
}
|
|
5991
5778
|
for (
|
|
5992
5779
|
let executedIndex = 0;
|
|
@@ -6021,16 +5808,13 @@ function createMinimalWorkerCtx(
|
|
|
6021
5808
|
} => entry !== null,
|
|
6022
5809
|
);
|
|
6023
5810
|
const out = outEntries.map((entry) => entry.row);
|
|
6024
|
-
const executedSuccessCount = Math.max(
|
|
6025
|
-
0,
|
|
6026
|
-
executedRows.length - failedExecutedRows,
|
|
6027
|
-
);
|
|
6028
5811
|
const rowFailureSamples = buildRowFailureSamples();
|
|
6029
5812
|
const publicOut = out.map((row) => publicCsvOutputRow(row));
|
|
6030
5813
|
const keyedOut = outEntries.map(({ key, inputIndex, row }) => ({
|
|
6031
5814
|
...row,
|
|
6032
5815
|
...mapRowOutcomeRuntimeFields({ key, inputIndex }),
|
|
6033
5816
|
}));
|
|
5817
|
+
const completedRowsReused = completedChunkEntries.length;
|
|
6034
5818
|
const hashStartedAt = nowMs();
|
|
6035
5819
|
const hash = await hashJson(publicOut);
|
|
6036
5820
|
const includeCachedRowsInChunkResult = !workflowStep;
|
|
@@ -6061,17 +5845,18 @@ function createMinimalWorkerCtx(
|
|
|
6061
5845
|
rowsWritten: out.length,
|
|
6062
5846
|
rowsExecuted: executedRows.length,
|
|
6063
5847
|
rowsFailed: failedExecutedRows,
|
|
6064
|
-
rowsCached:
|
|
5848
|
+
rowsCached: completedRowsReused,
|
|
6065
5849
|
},
|
|
6066
5850
|
});
|
|
6067
5851
|
return {
|
|
5852
|
+
status: 'completed',
|
|
6068
5853
|
chunkIndex,
|
|
6069
5854
|
rangeStart: baseOffset + chunkStart,
|
|
6070
5855
|
rangeEnd: baseOffset + chunkStart + out.length,
|
|
6071
5856
|
rowsRead: chunkRows.length,
|
|
6072
5857
|
rowsWritten: out.length,
|
|
6073
5858
|
rowsExecuted: executedRows.length,
|
|
6074
|
-
rowsCached:
|
|
5859
|
+
rowsCached: completedRowsReused,
|
|
6075
5860
|
rowsDuplicateReused: duplicateInputReuseCount,
|
|
6076
5861
|
rowsInserted,
|
|
6077
5862
|
rowsSkipped,
|
|
@@ -6103,9 +5888,12 @@ function createMinimalWorkerCtx(
|
|
|
6103
5888
|
let totalRowsInserted = 0;
|
|
6104
5889
|
let totalRowsSkipped = 0;
|
|
6105
5890
|
let totalRowsFailed = 0;
|
|
5891
|
+
let totalRowsBlocked = 0;
|
|
6106
5892
|
let totalStepCellsCompleted = 0;
|
|
6107
5893
|
let totalStepCellsSkipped = 0;
|
|
6108
5894
|
const totalRowFailureSamples: Array<{ rowKey: string; error: string }> = [];
|
|
5895
|
+
const blockedRowKeySamples: string[] = [];
|
|
5896
|
+
const blockedRowSamples: Record<string, unknown>[] = [];
|
|
6109
5897
|
|
|
6110
5898
|
const runChunkStep = async (
|
|
6111
5899
|
chunkRows: T[],
|
|
@@ -6135,6 +5923,7 @@ function createMinimalWorkerCtx(
|
|
|
6135
5923
|
const readPersistedRows = async (input: {
|
|
6136
5924
|
limit: number;
|
|
6137
5925
|
offset: number;
|
|
5926
|
+
rowMode?: 'output' | 'all';
|
|
6138
5927
|
}) => {
|
|
6139
5928
|
const result = await harnessReadSheetDatasetRows({
|
|
6140
5929
|
baseUrl: req.baseUrl,
|
|
@@ -6143,6 +5932,7 @@ function createMinimalWorkerCtx(
|
|
|
6143
5932
|
playName: req.playName,
|
|
6144
5933
|
tableNamespace: name,
|
|
6145
5934
|
runId: req.runId,
|
|
5935
|
+
rowMode: input.rowMode,
|
|
6146
5936
|
limit: input.limit,
|
|
6147
5937
|
offset: input.offset,
|
|
6148
5938
|
userEmail: req.userEmail,
|
|
@@ -6151,19 +5941,61 @@ function createMinimalWorkerCtx(
|
|
|
6151
5941
|
return result.rows as Array<T & Record<string, unknown>>;
|
|
6152
5942
|
};
|
|
6153
5943
|
|
|
6154
|
-
const finalize = async (
|
|
5944
|
+
const finalize = async () => {
|
|
5945
|
+
let finalizedRowsWritten = totalRowsWritten;
|
|
5946
|
+
if (
|
|
5947
|
+
totalRowsFailed === 0 &&
|
|
5948
|
+
rowCountHint !== null &&
|
|
5949
|
+
finalizedRowsWritten > 0 &&
|
|
5950
|
+
finalizedRowsWritten < rowCountHint &&
|
|
5951
|
+
rowCountHint <= WORKER_DATASET_IN_MEMORY_ROWS
|
|
5952
|
+
) {
|
|
5953
|
+
const persistedRows = await readPersistedRows({
|
|
5954
|
+
limit: rowCountHint,
|
|
5955
|
+
offset: 0,
|
|
5956
|
+
rowMode: 'all',
|
|
5957
|
+
});
|
|
5958
|
+
if (persistedRows.length > finalizedRowsWritten) {
|
|
5959
|
+
const inMemoryRowsWritten = finalizedRowsWritten;
|
|
5960
|
+
finalizedRowsWritten = persistedRows.length;
|
|
5961
|
+
totalRowsWritten = finalizedRowsWritten;
|
|
5962
|
+
cachedRows.length = 0;
|
|
5963
|
+
cachedRows.push(...persistedRows);
|
|
5964
|
+
canCacheRows = true;
|
|
5965
|
+
previewRows.length = 0;
|
|
5966
|
+
previewRows.push(
|
|
5967
|
+
...persistedRows.slice(0, WORKER_DATASET_PREVIEW_ROWS),
|
|
5968
|
+
);
|
|
5969
|
+
emitEvent({
|
|
5970
|
+
type: 'log',
|
|
5971
|
+
level: 'warn',
|
|
5972
|
+
message:
|
|
5973
|
+
`Runtime sheet finalization reconciled ctx.dataset("${name}") ` +
|
|
5974
|
+
`from ${inMemoryRowsWritten}/${rowCountHint} in-memory row(s) ` +
|
|
5975
|
+
`to ${finalizedRowsWritten} visible persisted row(s).`,
|
|
5976
|
+
ts: nowMs(),
|
|
5977
|
+
});
|
|
5978
|
+
}
|
|
5979
|
+
if (persistedRows.length < rowCountHint) {
|
|
5980
|
+
throw new Error(
|
|
5981
|
+
`Runtime sheet finalization mismatch ctx.dataset("${name}"): ` +
|
|
5982
|
+
`expected ${rowCountHint}, saw ${persistedRows.length}; ` +
|
|
5983
|
+
`sum ${finalizedRowsWritten}; ${req.runId}`,
|
|
5984
|
+
);
|
|
5985
|
+
}
|
|
5986
|
+
}
|
|
6155
5987
|
const failureSampleSummary =
|
|
6156
5988
|
totalRowFailureSamples.length > 0
|
|
6157
5989
|
? ` First error: ${totalRowFailureSamples[0]!.error}`
|
|
6158
5990
|
: '';
|
|
6159
5991
|
const cacheSummary =
|
|
6160
5992
|
totalRowsFailed > 0
|
|
6161
|
-
? `Map completed with partial failures: ${
|
|
6162
|
-
`${totalRowsFailed} failed (${totalRowsExecuted} executed
|
|
5993
|
+
? `Map completed with partial failures: ${finalizedRowsWritten} succeeded, ` +
|
|
5994
|
+
`${totalRowsFailed} failed (${totalRowsExecuted} executed) ` +
|
|
6163
5995
|
`inserted=${totalRowsInserted} skipped=${totalRowsSkipped}. ` +
|
|
6164
5996
|
`Failed rows are persisted with their errors and re-execute on the next run.${failureSampleSummary}`
|
|
6165
|
-
: `Map completed: ${
|
|
6166
|
-
`(${totalRowsExecuted} executed
|
|
5997
|
+
: `Map completed: ${finalizedRowsWritten} results ` +
|
|
5998
|
+
`(${totalRowsExecuted} executed) ` +
|
|
6167
5999
|
`inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
|
|
6168
6000
|
const completedAt = nowMs();
|
|
6169
6001
|
const totalStepCells = totalStepCellsCompleted + totalStepCellsSkipped;
|
|
@@ -6171,17 +6003,21 @@ function createMinimalWorkerCtx(
|
|
|
6171
6003
|
totalStepCells > 0
|
|
6172
6004
|
? ` cells=${totalStepCells}/${totalStepCellsCompleted}/${totalStepCellsSkipped}`
|
|
6173
6005
|
: '';
|
|
6006
|
+
await flushQueuedMapProgressUpdates();
|
|
6174
6007
|
callbacks?.onMapCompleted?.(mapNodeId, completedAt);
|
|
6175
6008
|
await updateMapProgress({
|
|
6176
|
-
completed:
|
|
6177
|
-
total:
|
|
6009
|
+
completed: finalizedRowsWritten,
|
|
6010
|
+
total: finalizedRowsWritten + totalRowsFailed,
|
|
6178
6011
|
failed: totalRowsFailed,
|
|
6179
6012
|
completedAt,
|
|
6180
6013
|
updatedAt: completedAt,
|
|
6181
6014
|
message:
|
|
6182
6015
|
totalRowsFailed > 0
|
|
6183
|
-
? `${
|
|
6184
|
-
: formatMapProgressMessage(
|
|
6016
|
+
? `${finalizedRowsWritten.toLocaleString()} succeeded, ${totalRowsFailed.toLocaleString()} failed`
|
|
6017
|
+
: formatMapProgressMessage(
|
|
6018
|
+
finalizedRowsWritten,
|
|
6019
|
+
finalizedRowsWritten,
|
|
6020
|
+
),
|
|
6185
6021
|
});
|
|
6186
6022
|
emitEvent({
|
|
6187
6023
|
type: 'log',
|
|
@@ -6190,29 +6026,61 @@ function createMinimalWorkerCtx(
|
|
|
6190
6026
|
ts: nowMs(),
|
|
6191
6027
|
});
|
|
6192
6028
|
if (
|
|
6193
|
-
|
|
6029
|
+
finalizedRowsWritten > 0 &&
|
|
6194
6030
|
totalRowsFailed === 0 &&
|
|
6195
6031
|
canCacheRows &&
|
|
6196
|
-
cachedRows.length ===
|
|
6032
|
+
cachedRows.length === finalizedRowsWritten
|
|
6197
6033
|
) {
|
|
6198
6034
|
const persistedRows = await readPersistedRows({
|
|
6199
|
-
limit:
|
|
6035
|
+
limit: finalizedRowsWritten,
|
|
6200
6036
|
offset: 0,
|
|
6201
6037
|
});
|
|
6202
|
-
if (persistedRows.length <
|
|
6038
|
+
if (persistedRows.length < finalizedRowsWritten) {
|
|
6039
|
+
const repairAttempt = makeRuntimeSheetAttempt({
|
|
6040
|
+
runId: req.runId,
|
|
6041
|
+
runAttempt: req.runAttempt,
|
|
6042
|
+
mapName: name,
|
|
6043
|
+
chunkIndex: 'finalize',
|
|
6044
|
+
});
|
|
6045
|
+
const repairRows = cachedRows.map((row) => runtimeCsvStorageRow(row));
|
|
6046
|
+
workBudgetMeter.count('sheet');
|
|
6047
|
+
const repairStart = await harnessStartSheetDataset({
|
|
6048
|
+
...runtimeSheetSessionScope(req),
|
|
6049
|
+
tableNamespace: name,
|
|
6050
|
+
sheetContract: augmentSheetContractWithDatasetFields({
|
|
6051
|
+
contract: requireSheetContract(req, name),
|
|
6052
|
+
rows: repairRows,
|
|
6053
|
+
outputFields,
|
|
6054
|
+
}),
|
|
6055
|
+
rows: repairRows,
|
|
6056
|
+
runId: req.runId,
|
|
6057
|
+
inputOffset: 0,
|
|
6058
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
6059
|
+
...repairAttempt,
|
|
6060
|
+
});
|
|
6061
|
+
const activeRepairAttempt = {
|
|
6062
|
+
attemptId: repairStart.attemptId ?? repairAttempt.attemptId,
|
|
6063
|
+
attemptOwnerRunId:
|
|
6064
|
+
repairStart.attemptOwnerRunId ?? repairAttempt.attemptOwnerRunId,
|
|
6065
|
+
attemptExpiresAt:
|
|
6066
|
+
repairStart.attemptExpiresAt ?? repairAttempt.attemptExpiresAt,
|
|
6067
|
+
attemptSeq: repairStart.attemptSeq ?? repairAttempt.attemptSeq,
|
|
6068
|
+
};
|
|
6203
6069
|
const repair = await persistCompletedMapRows({
|
|
6204
6070
|
req,
|
|
6205
6071
|
tableNamespace: name,
|
|
6206
6072
|
outputFields,
|
|
6207
6073
|
extraOutputFields: [],
|
|
6208
6074
|
rows: cachedRows,
|
|
6075
|
+
...activeRepairAttempt,
|
|
6076
|
+
budgetMeter: workBudgetMeter,
|
|
6209
6077
|
});
|
|
6210
6078
|
emitEvent({
|
|
6211
6079
|
type: 'log',
|
|
6212
6080
|
level: 'warn',
|
|
6213
6081
|
message:
|
|
6214
6082
|
`Runtime sheet finalization repaired ctx.dataset("${name}") ` +
|
|
6215
|
-
`from ${persistedRows.length}/${
|
|
6083
|
+
`from ${persistedRows.length}/${finalizedRowsWritten} visible row(s) ` +
|
|
6216
6084
|
`(written=${repair.written}, visible=${repair.visible}` +
|
|
6217
6085
|
(repair.retryWritten === null
|
|
6218
6086
|
? ''
|
|
@@ -6228,7 +6096,7 @@ function createMinimalWorkerCtx(
|
|
|
6228
6096
|
return createPersistedDatasetHandle({
|
|
6229
6097
|
playName: req.playName,
|
|
6230
6098
|
name,
|
|
6231
|
-
count:
|
|
6099
|
+
count: finalizedRowsWritten,
|
|
6232
6100
|
// In native Workflows, chunk summaries intentionally omit full row
|
|
6233
6101
|
// payloads, so this preview only contains the bounded chunk samples.
|
|
6234
6102
|
// Do not synchronously page rows back here: service-binding reads have
|
|
@@ -6242,7 +6110,7 @@ function createMinimalWorkerCtx(
|
|
|
6242
6110
|
recordRunnerPerfTrace({ req, phase, ms, extra }),
|
|
6243
6111
|
nowMs,
|
|
6244
6112
|
workProgress: {
|
|
6245
|
-
total:
|
|
6113
|
+
total: finalizedRowsWritten + totalRowsFailed,
|
|
6246
6114
|
executed: totalRowsExecuted,
|
|
6247
6115
|
reused: totalRowsCached,
|
|
6248
6116
|
skipped: totalRowsCached,
|
|
@@ -6256,97 +6124,248 @@ function createMinimalWorkerCtx(
|
|
|
6256
6124
|
};
|
|
6257
6125
|
|
|
6258
6126
|
const failFastRowErrors = opts?.onRowError === 'fail';
|
|
6259
|
-
|
|
6260
|
-
|
|
6261
|
-
|
|
6262
|
-
|
|
6263
|
-
|
|
6264
|
-
|
|
6265
|
-
|
|
6266
|
-
|
|
6267
|
-
|
|
6268
|
-
|
|
6269
|
-
|
|
6270
|
-
|
|
6271
|
-
|
|
6272
|
-
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
6276
|
-
|
|
6277
|
-
|
|
6278
|
-
|
|
6279
|
-
|
|
6280
|
-
|
|
6281
|
-
|
|
6282
|
-
|
|
6283
|
-
|
|
6284
|
-
|
|
6285
|
-
|
|
6286
|
-
|
|
6287
|
-
|
|
6288
|
-
`
|
|
6289
|
-
|
|
6290
|
-
);
|
|
6291
|
-
|
|
6292
|
-
|
|
6293
|
-
|
|
6294
|
-
|
|
6295
|
-
|
|
6296
|
-
|
|
6297
|
-
|
|
6298
|
-
|
|
6299
|
-
|
|
6300
|
-
|
|
6301
|
-
|
|
6302
|
-
|
|
6303
|
-
|
|
6127
|
+
// Bounded-concurrent chunk dispatch is only sound off the native Workflows
|
|
6128
|
+
// path: with a `workflowStep` each chunk is a durable step whose sequential
|
|
6129
|
+
// order makes replay/resume deterministic, so that path stays at K=1.
|
|
6130
|
+
// Chunk-level concurrency is otherwise safe under workers_edge because each
|
|
6131
|
+
// chunk owns a disjoint sheet row range with its own runtime-sheet attempt,
|
|
6132
|
+
// receipts complete inside each chunk's persist barrier, and child submits
|
|
6133
|
+
// use per-row durable receipt keys. It is disabled under onRowError:'fail'
|
|
6134
|
+
// on purpose: that mode relies on a first-failure short-circuit to skip the
|
|
6135
|
+
// remaining chunks and stop spending provider credits, and eager concurrent
|
|
6136
|
+
// launch would already have K-1 chunks in flight (credits spent) before the
|
|
6137
|
+
// short-circuit fires.
|
|
6138
|
+
const canRunChunksConcurrently = !workflowStep && !failFastRowErrors;
|
|
6139
|
+
const maxConcurrentMapChunks = canRunChunksConcurrently
|
|
6140
|
+
? WORKERS_EDGE_MAX_CONCURRENT_MAP_CHUNKS
|
|
6141
|
+
: 1;
|
|
6142
|
+
const runWorkDispatcher = new WorkerRunWorkDispatcher({
|
|
6143
|
+
budgetMeter: workBudgetMeter,
|
|
6144
|
+
nowMs,
|
|
6145
|
+
yieldLimits: {
|
|
6146
|
+
elapsed: governor.policy.pacing.workerYieldElapsedMs,
|
|
6147
|
+
...(req.testPolicyOverrides?.workBudgetYieldLimits ?? {}),
|
|
6148
|
+
},
|
|
6149
|
+
recordTrace: (trace) => recordRunnerPerfTrace({ req, ...trace }),
|
|
6150
|
+
yieldBetweenChunks: async (yieldPoint) => {
|
|
6151
|
+
emitEvent({
|
|
6152
|
+
type: 'log',
|
|
6153
|
+
level: 'info',
|
|
6154
|
+
message:
|
|
6155
|
+
`Runtime dispatcher yielded ctx.dataset("${name}") before chunk ${yieldPoint.nextChunkIndex} ` +
|
|
6156
|
+
`(${yieldPoint.reason} budget).`,
|
|
6157
|
+
ts: nowMs(),
|
|
6158
|
+
});
|
|
6159
|
+
if (workflowStep) {
|
|
6160
|
+
// Cloudflare Workflows durability model: `step.sleep` writes a
|
|
6161
|
+
// durable checkpoint, and the resume re-enters `run()` as a NEW
|
|
6162
|
+
// Worker invocation. Cloudflare's subrequest ceiling is
|
|
6163
|
+
// per-invocation, and on resume the already-completed `step.do`
|
|
6164
|
+
// chunks are served from the durable step cache WITHOUT re-issuing
|
|
6165
|
+
// their subrequests (replay semantics) — so the post-yield chunk
|
|
6166
|
+
// runs against a fresh per-invocation subrequest budget.
|
|
6167
|
+
//
|
|
6168
|
+
// Isolate identity is the WRONG freshness proxy: Cloudflare routinely
|
|
6169
|
+
// reuses warm isolates across invocations, so a genuinely
|
|
6170
|
+
// fresh-budget resume commonly shares the pre-sleep isolate id (this
|
|
6171
|
+
// is exactly why the isolate-identity check reported `false` every
|
|
6172
|
+
// time and hard-failed multi-chunk maps). Crossing the durable
|
|
6173
|
+
// `step.sleep` boundary IS the fresh-budget signal.
|
|
6174
|
+
const sleepMs = WORK_DISPATCHER_YIELD_SLEEP_MS;
|
|
6175
|
+
await (
|
|
6176
|
+
workflowStep.sleep as unknown as (
|
|
6177
|
+
name: string,
|
|
6178
|
+
duration: number,
|
|
6179
|
+
) => Promise<void>
|
|
6180
|
+
)(
|
|
6181
|
+
`work-dispatcher:${name}:${yieldPoint.nextChunkIndex}:${yieldPoint.attempt}`,
|
|
6182
|
+
sleepMs,
|
|
6183
|
+
);
|
|
6184
|
+
recordRunnerPerfTrace({
|
|
6185
|
+
req,
|
|
6186
|
+
phase: 'runner.work_dispatcher.yield_probe',
|
|
6187
|
+
ms: sleepMs,
|
|
6188
|
+
extra: {
|
|
6189
|
+
mapName: name,
|
|
6190
|
+
nextChunkIndex: yieldPoint.nextChunkIndex,
|
|
6191
|
+
reason: yieldPoint.reason,
|
|
6192
|
+
attempt: yieldPoint.attempt,
|
|
6193
|
+
freshBudget: true,
|
|
6194
|
+
invocationBoundary: 'workflow_step_sleep',
|
|
6195
|
+
},
|
|
6196
|
+
});
|
|
6197
|
+
return { freshBudget: true };
|
|
6304
6198
|
}
|
|
6305
|
-
|
|
6306
|
-
|
|
6307
|
-
|
|
6308
|
-
|
|
6309
|
-
|
|
6310
|
-
|
|
6311
|
-
|
|
6312
|
-
|
|
6313
|
-
|
|
6314
|
-
|
|
6315
|
-
|
|
6316
|
-
|
|
6317
|
-
|
|
6318
|
-
|
|
6199
|
+
// No durable Workflow step is available to force a new invocation here,
|
|
6200
|
+
// so we cannot obtain a fresh per-invocation platform budget. Report a
|
|
6201
|
+
// non-fresh yield: the dispatcher escalates and then fails loudly with
|
|
6202
|
+
// WorkerBudgetExhaustedError rather than silently overflowing the
|
|
6203
|
+
// subrequest ceiling mid-map.
|
|
6204
|
+
await sleepWorkerMs(0);
|
|
6205
|
+
return { freshBudget: false };
|
|
6206
|
+
},
|
|
6207
|
+
});
|
|
6208
|
+
let dispatchResult: WorkerRunMapBatchesResult;
|
|
6209
|
+
try {
|
|
6210
|
+
dispatchResult = await runWorkDispatcher.runMapBatches({
|
|
6211
|
+
mapName: name,
|
|
6212
|
+
chunks: iterDatasetChunks(inputRows, rowsPerChunk),
|
|
6213
|
+
maxConcurrentChunks: maxConcurrentMapChunks,
|
|
6214
|
+
maxResidentBatchBytes: WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES,
|
|
6215
|
+
prepareBatch: (rawChunkRows, chunkStart) => {
|
|
6216
|
+
assertNotAborted(abortSignal);
|
|
6217
|
+
if (rawChunkRows.length === 0) return [];
|
|
6218
|
+
// Drop duplicate explicit-key rows before anything downstream observes
|
|
6219
|
+
// them. `chunkStart` keeps advancing by the original (pre-dedupe) chunk
|
|
6220
|
+
// length so cross-chunk key indices and persisted input offsets stay
|
|
6221
|
+
// aligned to the original input stream.
|
|
6222
|
+
return dedupeExplicitRowKeys(rawChunkRows, chunkStart);
|
|
6223
|
+
},
|
|
6224
|
+
estimateRawBatchResidentBytes: (rawChunkRows) =>
|
|
6225
|
+
rawChunkRows.reduce(
|
|
6226
|
+
(total, row) => total + workerMapResidentRowBytes(row),
|
|
6227
|
+
0,
|
|
6228
|
+
),
|
|
6229
|
+
executeBatch: async ({ rows: chunkRows, chunkStart, chunkIndex }) => {
|
|
6230
|
+
assertNotAborted(abortSignal);
|
|
6231
|
+
return await runChunkStep(chunkRows, chunkStart, chunkIndex);
|
|
6232
|
+
},
|
|
6233
|
+
estimateBatchResidentBytes: ({ rows: chunkRows }) =>
|
|
6234
|
+
chunkRows.reduce(
|
|
6235
|
+
(total, row) => total + workerMapResidentRowBytes(row),
|
|
6319
6236
|
0,
|
|
6320
|
-
WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
|
|
6321
6237
|
),
|
|
6238
|
+
estimateBatchSubrequests: ({ rows: chunkRows }) => {
|
|
6239
|
+
const estimate = mapDispatchPlan.workEstimate;
|
|
6240
|
+
const unbatchedProviderSubrequests =
|
|
6241
|
+
estimate.unbatchedProviderToolCallsPerRow *
|
|
6242
|
+
WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL;
|
|
6243
|
+
// Static planning cannot know input-dependent batching buckets, so
|
|
6244
|
+
// this projects worst-case provider-batch fanout while still treating
|
|
6245
|
+
// durable receipt claim/complete/fail as bulk per drained tool group.
|
|
6246
|
+
const batchedProviderSubrequests =
|
|
6247
|
+
estimateBatchedToolChunkSubrequests({
|
|
6248
|
+
batchableToolCallsPerRow: Math.max(
|
|
6249
|
+
0,
|
|
6250
|
+
estimate.providerToolCallsPerRow -
|
|
6251
|
+
estimate.unbatchedProviderToolCallsPerRow,
|
|
6252
|
+
),
|
|
6253
|
+
coalescedBatchableToolCallsPerRow:
|
|
6254
|
+
estimate.coalescedBatchableProviderToolCallsPerRow,
|
|
6255
|
+
rows: chunkRows.length,
|
|
6256
|
+
assumedBatchSize: estimate.assumedBatchSize,
|
|
6257
|
+
});
|
|
6258
|
+
const controlSubrequests =
|
|
6259
|
+
estimate.childPlaySubmitsPerRow + estimate.eventWaitsPerRow;
|
|
6260
|
+
return (
|
|
6261
|
+
chunkRows.length *
|
|
6262
|
+
(unbatchedProviderSubrequests + controlSubrequests) +
|
|
6263
|
+
batchedProviderSubrequests +
|
|
6264
|
+
estimate.runtimeSheetReadsPerChunk +
|
|
6265
|
+
estimate.runtimeSheetWritesPerChunk
|
|
6266
|
+
);
|
|
6267
|
+
},
|
|
6268
|
+
onBatchComplete: async (chunkResult) => {
|
|
6269
|
+
if (chunkResult.status === 'blocked') {
|
|
6270
|
+
totalRowsBlocked += chunkResult.rowsBlocked;
|
|
6271
|
+
for (const key of chunkResult.rowKeySamples) {
|
|
6272
|
+
if (blockedRowKeySamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6273
|
+
break;
|
|
6274
|
+
}
|
|
6275
|
+
blockedRowKeySamples.push(key);
|
|
6276
|
+
}
|
|
6277
|
+
for (const row of chunkResult.blockedRowSamples) {
|
|
6278
|
+
if (blockedRowSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6279
|
+
break;
|
|
6280
|
+
}
|
|
6281
|
+
blockedRowSamples.push(row);
|
|
6282
|
+
}
|
|
6283
|
+
emitEvent({
|
|
6284
|
+
type: 'log',
|
|
6285
|
+
level: 'warn',
|
|
6286
|
+
message: chunkResult.message,
|
|
6287
|
+
ts: nowMs(),
|
|
6288
|
+
});
|
|
6289
|
+
return true;
|
|
6290
|
+
}
|
|
6291
|
+
if (chunkResult.fatalError) {
|
|
6292
|
+
throw new Error(
|
|
6293
|
+
`ctx.dataset("${name}") stopped after a runtime persistence failure ` +
|
|
6294
|
+
`outside the retryable chunk step. Provider calls already executed for ` +
|
|
6295
|
+
`${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
|
|
6296
|
+
`Fix the runtime persistence cause and re-run to resume. ` +
|
|
6297
|
+
`First error: ${chunkResult.fatalError}`,
|
|
6298
|
+
);
|
|
6299
|
+
}
|
|
6300
|
+
totalRowsWritten += chunkResult.rowsWritten;
|
|
6301
|
+
totalRowsExecuted += chunkResult.rowsExecuted;
|
|
6302
|
+
totalRowsCached += chunkResult.rowsCached;
|
|
6303
|
+
totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
|
|
6304
|
+
totalRowsInserted += chunkResult.rowsInserted;
|
|
6305
|
+
totalRowsSkipped += chunkResult.rowsSkipped;
|
|
6306
|
+
totalRowsFailed += chunkResult.rowsFailed ?? 0;
|
|
6307
|
+
totalStepCellsCompleted += chunkResult.stepCellsCompleted ?? 0;
|
|
6308
|
+
totalStepCellsSkipped += chunkResult.stepCellsSkipped ?? 0;
|
|
6309
|
+
for (const sample of chunkResult.rowFailureSamples ?? []) {
|
|
6310
|
+
if (totalRowFailureSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6311
|
+
break;
|
|
6312
|
+
}
|
|
6313
|
+
totalRowFailureSamples.push(sample);
|
|
6314
|
+
}
|
|
6315
|
+
enqueueMapProgressUpdate({
|
|
6316
|
+
completed: totalRowsWritten,
|
|
6317
|
+
total: rowCountHint ?? undefined,
|
|
6318
|
+
...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
|
|
6319
|
+
message: formatMapProgressMessage(
|
|
6320
|
+
totalRowsWritten,
|
|
6321
|
+
rowCountHint ?? undefined,
|
|
6322
|
+
),
|
|
6323
|
+
});
|
|
6324
|
+
if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
|
|
6325
|
+
previewRows.push(
|
|
6326
|
+
...chunkResult.preview.slice(
|
|
6327
|
+
0,
|
|
6328
|
+
WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
|
|
6329
|
+
),
|
|
6330
|
+
);
|
|
6331
|
+
}
|
|
6332
|
+
if (canCacheRows) {
|
|
6333
|
+
const volatileRows = volatileWorkflowChunkRows.get(
|
|
6334
|
+
chunkResult.chunkIndex,
|
|
6335
|
+
);
|
|
6336
|
+
volatileWorkflowChunkRows.delete(chunkResult.chunkIndex);
|
|
6337
|
+
const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
|
|
6338
|
+
if (
|
|
6339
|
+
nextRows.length === chunkResult.rowsWritten &&
|
|
6340
|
+
cachedRows.length + nextRows.length <=
|
|
6341
|
+
WORKER_DATASET_IN_MEMORY_ROWS
|
|
6342
|
+
) {
|
|
6343
|
+
cachedRows.push(...nextRows);
|
|
6344
|
+
} else {
|
|
6345
|
+
cachedRows.length = 0;
|
|
6346
|
+
volatileWorkflowChunkRows.clear();
|
|
6347
|
+
canCacheRows = false;
|
|
6348
|
+
}
|
|
6349
|
+
}
|
|
6350
|
+
// onRowError:'fail' short-circuit: once a chunk reports a row failure,
|
|
6351
|
+
// skip the remaining chunks entirely. The failing chunk itself completed
|
|
6352
|
+
// normally (no chunk-step retry storm) and persisted its rows, but
|
|
6353
|
+
// executing later chunks would keep spending provider credits on a run
|
|
6354
|
+
// the caller asked to fail fast. The post-loop fail-fast throw below
|
|
6355
|
+
// reports what committed before the stop.
|
|
6356
|
+
return failFastRowErrors && totalRowsFailed > 0;
|
|
6357
|
+
},
|
|
6358
|
+
});
|
|
6359
|
+
} catch (error) {
|
|
6360
|
+
try {
|
|
6361
|
+
await flushQueuedMapProgressUpdates();
|
|
6362
|
+
} catch (progressError) {
|
|
6363
|
+
throw new AggregateError(
|
|
6364
|
+
[error, progressError],
|
|
6365
|
+
`ctx.dataset("${name}") failed while flushing queued progress updates.`,
|
|
6322
6366
|
);
|
|
6323
6367
|
}
|
|
6324
|
-
|
|
6325
|
-
const volatileRows = volatileWorkflowChunkRows.get(chunkIndex);
|
|
6326
|
-
volatileWorkflowChunkRows.delete(chunkIndex);
|
|
6327
|
-
const nextRows = chunkResult.cachedRows ?? volatileRows ?? [];
|
|
6328
|
-
if (
|
|
6329
|
-
nextRows.length === chunkResult.rowsWritten &&
|
|
6330
|
-
cachedRows.length + nextRows.length <= WORKER_DATASET_IN_MEMORY_ROWS
|
|
6331
|
-
) {
|
|
6332
|
-
cachedRows.push(...nextRows);
|
|
6333
|
-
} else {
|
|
6334
|
-
cachedRows.length = 0;
|
|
6335
|
-
volatileWorkflowChunkRows.clear();
|
|
6336
|
-
canCacheRows = false;
|
|
6337
|
-
}
|
|
6338
|
-
}
|
|
6339
|
-
chunkStart += rawChunkRows.length;
|
|
6340
|
-
chunkIndex += 1;
|
|
6341
|
-
// onRowError:'fail' short-circuit: once a chunk reports a row failure,
|
|
6342
|
-
// skip the remaining chunks entirely. The failing chunk itself completed
|
|
6343
|
-
// normally (no chunk-step retry storm) and persisted its rows, but
|
|
6344
|
-
// executing later chunks would keep spending provider credits on a run
|
|
6345
|
-
// the caller asked to fail fast. The post-loop fail-fast throw below
|
|
6346
|
-
// reports what committed before the stop.
|
|
6347
|
-
if (failFastRowErrors && totalRowsFailed > 0) {
|
|
6348
|
-
break;
|
|
6349
|
-
}
|
|
6368
|
+
throw error;
|
|
6350
6369
|
}
|
|
6351
6370
|
if (totalDuplicateKeysDropped > 0) {
|
|
6352
6371
|
const keySample = droppedDuplicateKeySamples.join(', ');
|
|
@@ -6359,6 +6378,13 @@ function createMinimalWorkerCtx(
|
|
|
6359
6378
|
ts: nowMs(),
|
|
6360
6379
|
});
|
|
6361
6380
|
}
|
|
6381
|
+
if (totalRowsBlocked > 0) {
|
|
6382
|
+
await flushQueuedMapProgressUpdates();
|
|
6383
|
+
throw new RuntimeSheetRowsBlockedError({
|
|
6384
|
+
tableNamespace: name,
|
|
6385
|
+
blockedRows: blockedRowSamples,
|
|
6386
|
+
});
|
|
6387
|
+
}
|
|
6362
6388
|
if (failFastRowErrors && totalRowsFailed > 0 && totalRowsWritten > 0) {
|
|
6363
6389
|
// onRowError:'fail', PARTIAL failure (some rows committed): fail the run
|
|
6364
6390
|
// without finalizing the dataset. The committed rows already persisted
|
|
@@ -6367,6 +6393,7 @@ function createMinimalWorkerCtx(
|
|
|
6367
6393
|
// normally (no per-row throw inside the durable chunk step, so no
|
|
6368
6394
|
// chunk-step retry storm); later chunks were skipped by the fail-fast
|
|
6369
6395
|
// short-circuit in the chunk loop.
|
|
6396
|
+
await flushQueuedMapProgressUpdates();
|
|
6370
6397
|
const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
|
|
6371
6398
|
throw new Error(
|
|
6372
6399
|
`ctx.dataset("${name}") failed for ${totalRowsFailed} executed row(s) under onRowError:'fail'. ` +
|
|
@@ -6384,7 +6411,7 @@ function createMinimalWorkerCtx(
|
|
|
6384
6411
|
// step when no row otherwise succeeded) are summarized and registered as
|
|
6385
6412
|
// a recovered dataset — the failed run then advertises a WORKING export
|
|
6386
6413
|
// instead of a dead end (#15/#27). The run still fails (the throw below).
|
|
6387
|
-
await finalize(
|
|
6414
|
+
await finalize();
|
|
6388
6415
|
const firstError = totalRowFailureSamples[0]?.error ?? 'unknown error';
|
|
6389
6416
|
throw new Error(
|
|
6390
6417
|
`ctx.dataset("${name}") failed for all ${totalRowsFailed} executed rows. ` +
|
|
@@ -6392,7 +6419,7 @@ function createMinimalWorkerCtx(
|
|
|
6392
6419
|
`(rows are persisted with per-row errors; fix the cause and re-run to resume)`,
|
|
6393
6420
|
);
|
|
6394
6421
|
}
|
|
6395
|
-
const dataset = await finalize(
|
|
6422
|
+
const dataset = await finalize();
|
|
6396
6423
|
recordRunnerPerfTrace({
|
|
6397
6424
|
req,
|
|
6398
6425
|
phase: 'runner.map.total',
|
|
@@ -6401,7 +6428,7 @@ function createMinimalWorkerCtx(
|
|
|
6401
6428
|
mapName: name,
|
|
6402
6429
|
rowsWritten: totalRowsWritten,
|
|
6403
6430
|
inputKind: rowCountHint === null ? 'streaming' : 'known_count',
|
|
6404
|
-
chunks:
|
|
6431
|
+
chunks: dispatchResult.chunksExecuted,
|
|
6405
6432
|
},
|
|
6406
6433
|
});
|
|
6407
6434
|
return dataset;
|
|
@@ -6666,10 +6693,10 @@ function createMinimalWorkerCtx(
|
|
|
6666
6693
|
request.input,
|
|
6667
6694
|
workflowStep,
|
|
6668
6695
|
{
|
|
6669
|
-
force: request.force === true
|
|
6696
|
+
force: request.force === true,
|
|
6697
|
+
runForceRefresh: req.forceToolRefresh === true,
|
|
6698
|
+
runForceFailedRefresh: req.force === true,
|
|
6670
6699
|
staleAfterSeconds: request.staleAfterSeconds,
|
|
6671
|
-
},
|
|
6672
|
-
{
|
|
6673
6700
|
timeoutMs: request.timeoutMs,
|
|
6674
6701
|
receiptWaitMs: request.receiptWaitMs,
|
|
6675
6702
|
},
|
|
@@ -6710,13 +6737,18 @@ function createMinimalWorkerCtx(
|
|
|
6710
6737
|
if (!resolvedName) {
|
|
6711
6738
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
6712
6739
|
}
|
|
6740
|
+
const childManifest = req.childPlayManifests?.[resolvedName];
|
|
6713
6741
|
const receiptKey = buildDurableCtxCallCacheKey({
|
|
6714
6742
|
orgId: req.orgId,
|
|
6715
6743
|
playId: req.playName,
|
|
6716
6744
|
kind: 'runPlay',
|
|
6717
6745
|
id: normalizedKey,
|
|
6718
|
-
semanticKey:
|
|
6746
|
+
semanticKey: buildDurableRunPlaySemanticKey({
|
|
6719
6747
|
childPlayName: resolvedName,
|
|
6748
|
+
childRevisionFingerprint: workerChildRevisionFingerprint({
|
|
6749
|
+
playName: resolvedName,
|
|
6750
|
+
manifest: childManifest,
|
|
6751
|
+
}),
|
|
6720
6752
|
input,
|
|
6721
6753
|
}),
|
|
6722
6754
|
staleAfterSeconds: options?.staleAfterSeconds,
|
|
@@ -6744,7 +6776,6 @@ function createMinimalWorkerCtx(
|
|
|
6744
6776
|
message: `Starting child play ${resolvedName} (${normalizedKey})`,
|
|
6745
6777
|
ts: nowMs(),
|
|
6746
6778
|
});
|
|
6747
|
-
const childManifest = req.childPlayManifests?.[resolvedName];
|
|
6748
6779
|
if (!childManifest) {
|
|
6749
6780
|
throw new Error(
|
|
6750
6781
|
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
|
|
@@ -6782,7 +6813,7 @@ function createMinimalWorkerCtx(
|
|
|
6782
6813
|
});
|
|
6783
6814
|
let childPlaySlot: { release(): void } | null = null;
|
|
6784
6815
|
try {
|
|
6785
|
-
childPlaySlot = await governor.
|
|
6816
|
+
childPlaySlot = await governor.acquireChildSubmitSlot({
|
|
6786
6817
|
signal: abortSignal,
|
|
6787
6818
|
});
|
|
6788
6819
|
const childSubmitStartedAt = nowMs();
|
|
@@ -6807,12 +6838,12 @@ function createMinimalWorkerCtx(
|
|
|
6807
6838
|
options?.timeoutMs == null && !childNeedsWorkflowScheduler,
|
|
6808
6839
|
body: {
|
|
6809
6840
|
name: resolvedName,
|
|
6841
|
+
childIdempotencyKey: receiptKey,
|
|
6810
6842
|
input: isRecord(input) ? input : {},
|
|
6811
6843
|
orgId: req.orgId,
|
|
6812
6844
|
callbackBaseUrl: req.callbackUrl,
|
|
6813
6845
|
baseUrl: req.baseUrl,
|
|
6814
6846
|
integrationMode: req.integrationMode ?? null,
|
|
6815
|
-
forceToolRefresh: req.force === true,
|
|
6816
6847
|
parentExecutorToken: req.executorToken,
|
|
6817
6848
|
userEmail: req.userEmail ?? '',
|
|
6818
6849
|
profile: 'workers_edge',
|
|
@@ -6924,6 +6955,8 @@ function createMinimalWorkerCtx(
|
|
|
6924
6955
|
childNeedsWorkflowScheduler,
|
|
6925
6956
|
},
|
|
6926
6957
|
});
|
|
6958
|
+
childPlaySlot?.release();
|
|
6959
|
+
childPlaySlot = null;
|
|
6927
6960
|
const startedStatus = String(started.status ?? '').toLowerCase();
|
|
6928
6961
|
if (startedStatus === 'completed') {
|
|
6929
6962
|
emitEvent({
|
|
@@ -6952,6 +6985,10 @@ function createMinimalWorkerCtx(
|
|
|
6952
6985
|
}
|
|
6953
6986
|
const childWaitStartedAt = nowMs();
|
|
6954
6987
|
let waitResult: ChildPlayTerminalWaitResult;
|
|
6988
|
+
const readCachedChildTerminalState =
|
|
6989
|
+
cachedCoordinatorBinding?.readChildTerminalState?.bind(
|
|
6990
|
+
cachedCoordinatorBinding,
|
|
6991
|
+
);
|
|
6955
6992
|
try {
|
|
6956
6993
|
waitResult = await awaitChildTerminal({
|
|
6957
6994
|
parentRunId: req.runId,
|
|
@@ -6968,20 +7005,31 @@ function createMinimalWorkerCtx(
|
|
|
6968
7005
|
1_000,
|
|
6969
7006
|
Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
|
|
6970
7007
|
),
|
|
6971
|
-
coordinator:
|
|
6972
|
-
|
|
6973
|
-
|
|
6974
|
-
|
|
6975
|
-
|
|
6976
|
-
|
|
6977
|
-
|
|
6978
|
-
|
|
6979
|
-
|
|
6980
|
-
|
|
6981
|
-
|
|
6982
|
-
|
|
6983
|
-
|
|
6984
|
-
|
|
7008
|
+
coordinator: {
|
|
7009
|
+
readRunTerminalState: (runId, timeoutMs) =>
|
|
7010
|
+
readDurableChildRunTerminalState({
|
|
7011
|
+
baseUrl: req.baseUrl,
|
|
7012
|
+
executorToken: req.executorToken,
|
|
7013
|
+
parentRunId: req.runId,
|
|
7014
|
+
childRunId: runId,
|
|
7015
|
+
childPlayName: resolvedName,
|
|
7016
|
+
timeoutMs,
|
|
7017
|
+
}),
|
|
7018
|
+
...(readCachedChildTerminalState
|
|
7019
|
+
? {
|
|
7020
|
+
readChildTerminalState: (
|
|
7021
|
+
parentRunId: string,
|
|
7022
|
+
eventKey: string,
|
|
7023
|
+
timeoutMs?: number,
|
|
7024
|
+
) =>
|
|
7025
|
+
readCachedChildTerminalState(
|
|
7026
|
+
parentRunId,
|
|
7027
|
+
eventKey,
|
|
7028
|
+
timeoutMs,
|
|
7029
|
+
),
|
|
7030
|
+
}
|
|
7031
|
+
: {}),
|
|
7032
|
+
},
|
|
6985
7033
|
now: nowMs,
|
|
6986
7034
|
hashJson,
|
|
6987
7035
|
});
|
|
@@ -7164,6 +7212,7 @@ function createMinimalWorkerCtx(
|
|
|
7164
7212
|
// Direct Worker fetch below is still limited to runtime origins and
|
|
7165
7213
|
// IP literals; public IP literals also consume this slot because
|
|
7166
7214
|
// they are customer egress.
|
|
7215
|
+
workBudgetMeter.count('egress');
|
|
7167
7216
|
const egressResponse = await postRuntimeEgressFetch(
|
|
7168
7217
|
req,
|
|
7169
7218
|
{
|
|
@@ -7186,6 +7235,7 @@ function createMinimalWorkerCtx(
|
|
|
7186
7235
|
}
|
|
7187
7236
|
const fetchInit = { ...init, headers };
|
|
7188
7237
|
delete fetchInit.auth;
|
|
7238
|
+
workBudgetMeter.count('egress');
|
|
7189
7239
|
const response = await safeWorkerPublicFetch(url, fetchInit, {
|
|
7190
7240
|
allowedOrigins,
|
|
7191
7241
|
sensitiveHeaders: Object.keys(secretHeaderMarkers),
|
|
@@ -7256,6 +7306,7 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
|
|
|
7256
7306
|
let req: RunRequest;
|
|
7257
7307
|
try {
|
|
7258
7308
|
req = (await request.json()) as RunRequest;
|
|
7309
|
+
req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
|
|
7259
7310
|
} catch {
|
|
7260
7311
|
return new Response('invalid JSON body', { status: 400 });
|
|
7261
7312
|
}
|
|
@@ -7320,6 +7371,7 @@ async function handleRunInline(
|
|
|
7320
7371
|
let req: RunRequest;
|
|
7321
7372
|
try {
|
|
7322
7373
|
req = (await request.json()) as RunRequest;
|
|
7374
|
+
req.runAttempt = normalizeWorkerRunAttempt(req.runAttempt);
|
|
7323
7375
|
} catch {
|
|
7324
7376
|
return Response.json(
|
|
7325
7377
|
{
|
|
@@ -7467,6 +7519,7 @@ async function executeRunRequest(
|
|
|
7467
7519
|
): Promise<WorkflowRunOutput> {
|
|
7468
7520
|
installProcessExitTrap();
|
|
7469
7521
|
const startedAt = nowMs();
|
|
7522
|
+
const workBudgetMeter = createWorkerWorkBudgetMeter({ nowMs });
|
|
7470
7523
|
recordRunnerPerfTrace({
|
|
7471
7524
|
req,
|
|
7472
7525
|
phase: 'runner.execute_start',
|
|
@@ -7504,6 +7557,7 @@ async function executeRunRequest(
|
|
|
7504
7557
|
const appendRunLogLine = (line: string) => {
|
|
7505
7558
|
const trimmed = redactSecretsFromLogString(line.trim());
|
|
7506
7559
|
if (!trimmed) return;
|
|
7560
|
+
workBudgetMeter.count('log');
|
|
7507
7561
|
totalEmittedLogLines += 1;
|
|
7508
7562
|
runLogBuffer = [...runLogBuffer, trimmed].slice(-RUN_LOG_BUFFER_LIMIT);
|
|
7509
7563
|
pendingRunLogLines = [...pendingRunLogLines, trimmed];
|
|
@@ -7696,7 +7750,10 @@ async function executeRunRequest(
|
|
|
7696
7750
|
const flushTerminalLedgerEvents = async (
|
|
7697
7751
|
terminalEvent: PlayRunLedgerEvent,
|
|
7698
7752
|
): Promise<void> => {
|
|
7699
|
-
|
|
7753
|
+
const shouldPersistTerminalLedger =
|
|
7754
|
+
options?.persistResultDatasets === true ||
|
|
7755
|
+
Boolean(req.playCallGovernance);
|
|
7756
|
+
if (!shouldPersistTerminalLedger) return;
|
|
7700
7757
|
await ledgerFlushInFlight;
|
|
7701
7758
|
const now = nowMs();
|
|
7702
7759
|
dirtyProgressNodeIds = new Set([
|
|
@@ -7771,10 +7828,7 @@ async function executeRunRequest(
|
|
|
7771
7828
|
stepLifecycle?.markPreDatasetStepsStarted(startedAt);
|
|
7772
7829
|
flushLedgerEvents(false);
|
|
7773
7830
|
const runtimeDeadlineMs = nowMs() + STANDARD_PLAY_RUNTIME_LIMIT_MS;
|
|
7774
|
-
|
|
7775
|
-
// receipt could not be marked completed even after the retry ladder. On
|
|
7776
|
-
// terminal failure the buffer is built into a size-capped salvage record and
|
|
7777
|
-
// attached to the run.failed ledger event so billed data is never memory-only.
|
|
7831
|
+
const leasedSheetNamespaces = new Set<string>();
|
|
7778
7832
|
const receiptSalvage = createReceiptSalvageBuffer();
|
|
7779
7833
|
const ctx = createMinimalWorkerCtx(
|
|
7780
7834
|
req,
|
|
@@ -7783,7 +7837,9 @@ async function executeRunRequest(
|
|
|
7783
7837
|
workflowStep,
|
|
7784
7838
|
abortSignal,
|
|
7785
7839
|
workerCallbacks,
|
|
7840
|
+
workBudgetMeter,
|
|
7786
7841
|
runtimeDeadlineMs,
|
|
7842
|
+
leasedSheetNamespaces,
|
|
7787
7843
|
receiptSalvage,
|
|
7788
7844
|
);
|
|
7789
7845
|
// Hard wall-clock cap on active user-code runtime. CF Workflows does not
|
|
@@ -7792,15 +7848,18 @@ async function executeRunRequest(
|
|
|
7792
7848
|
// token expires. Aborting the controller surfaces cooperatively through the
|
|
7793
7849
|
// same assertNotAborted checks used for harness cancellation.
|
|
7794
7850
|
let runtimeLimitExceeded = false;
|
|
7795
|
-
const runtimeDeadlineTimer =
|
|
7796
|
-
|
|
7797
|
-
|
|
7798
|
-
|
|
7799
|
-
|
|
7800
|
-
|
|
7801
|
-
|
|
7802
|
-
|
|
7803
|
-
|
|
7851
|
+
const runtimeDeadlineTimer =
|
|
7852
|
+
workflowStep === undefined || workflowStep === null
|
|
7853
|
+
? setTimeout(
|
|
7854
|
+
() => {
|
|
7855
|
+
runtimeLimitExceeded = true;
|
|
7856
|
+
if (!abortSignal.aborted) {
|
|
7857
|
+
abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
|
|
7858
|
+
}
|
|
7859
|
+
},
|
|
7860
|
+
Math.max(1, runtimeDeadlineMs - nowMs()),
|
|
7861
|
+
)
|
|
7862
|
+
: null;
|
|
7804
7863
|
try {
|
|
7805
7864
|
const playStartedAt = nowMs();
|
|
7806
7865
|
const result = await (
|
|
@@ -7891,7 +7950,6 @@ async function executeRunRequest(
|
|
|
7891
7950
|
}
|
|
7892
7951
|
if (options?.persistResultDatasets) {
|
|
7893
7952
|
await persistProjectedResultDatasets();
|
|
7894
|
-
const parentSignal = startParentTerminalSignal();
|
|
7895
7953
|
// Capped runs settle compute billing BEFORE declaring run.completed: a
|
|
7896
7954
|
// per-run cap denial (422 billing_cap_exceeded) must fail the run as
|
|
7897
7955
|
// its ONLY terminal. Flushing completed first opens a race — watchers
|
|
@@ -7926,6 +7984,7 @@ async function executeRunRequest(
|
|
|
7926
7984
|
ms: nowMs() - terminalUpdateStartedAt,
|
|
7927
7985
|
});
|
|
7928
7986
|
|
|
7987
|
+
const parentSignal = startParentTerminalSignal();
|
|
7929
7988
|
if (!capped) {
|
|
7930
7989
|
const billingStartedAt = nowMs();
|
|
7931
7990
|
const billingPromise = finalizeWorkerComputeBilling({
|
|
@@ -7953,6 +8012,20 @@ async function executeRunRequest(
|
|
|
7953
8012
|
}
|
|
7954
8013
|
}
|
|
7955
8014
|
await parentSignal;
|
|
8015
|
+
} else if (req.playCallGovernance) {
|
|
8016
|
+
const childTerminalStartedAt = nowMs();
|
|
8017
|
+
await flushTerminalLedgerEvents({
|
|
8018
|
+
type: 'run.completed',
|
|
8019
|
+
runId: req.runId,
|
|
8020
|
+
source: 'worker',
|
|
8021
|
+
occurredAt: nowMs(),
|
|
8022
|
+
result: terminalResult,
|
|
8023
|
+
});
|
|
8024
|
+
recordRunnerPerfTrace({
|
|
8025
|
+
req,
|
|
8026
|
+
phase: 'runner.child_terminal_ledger_append',
|
|
8027
|
+
ms: nowMs() - childTerminalStartedAt,
|
|
8028
|
+
});
|
|
7956
8029
|
}
|
|
7957
8030
|
await startParentTerminalSignal();
|
|
7958
8031
|
recordRunnerPerfTrace({
|
|
@@ -7979,16 +8052,8 @@ async function executeRunRequest(
|
|
|
7979
8052
|
stepLifecycle?.markStartedFailed(nowMs());
|
|
7980
8053
|
// A runtime-limit abort is a timeout failure, not a user cancellation, so
|
|
7981
8054
|
// it should be reported as run.failed with the limit message rather than
|
|
7982
|
-
// run.cancelled.
|
|
7983
|
-
|
|
7984
|
-
// runtimeDeadlineTimer macrotask flips the flag — match the limit message
|
|
7985
|
-
// directly so that race can never misclassify a timeout as a cancel (which
|
|
7986
|
-
// would silently drop the salvage buffer of billed-but-unpersisted rows).
|
|
7987
|
-
const runtimeLimitError =
|
|
7988
|
-
runtimeLimitExceeded ||
|
|
7989
|
-
(error instanceof Error &&
|
|
7990
|
-
error.message === STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
|
|
7991
|
-
const aborted = isAbortLikeError(error) && !runtimeLimitError;
|
|
8055
|
+
// run.cancelled.
|
|
8056
|
+
const aborted = isAbortLikeError(error) && !runtimeLimitExceeded;
|
|
7992
8057
|
if (aborted) {
|
|
7993
8058
|
// Flip the controller so any concurrent user code observes the abort
|
|
7994
8059
|
// through ctx.signal. We mark the run cancelled instead of failed.
|
|
@@ -7997,19 +8062,28 @@ async function executeRunRequest(
|
|
|
7997
8062
|
);
|
|
7998
8063
|
}
|
|
7999
8064
|
const failure = normalizePlayRunFailure(error);
|
|
8000
|
-
|
|
8001
|
-
//
|
|
8065
|
+
const message = failure.message;
|
|
8066
|
+
// Controlled run-fatal teardown: release the runtime-sheet row leases and
|
|
8067
|
+
// work-receipt leases this attempt still holds so an immediate rerun is not
|
|
8068
|
+
// blocked for the full lease TTL. Best-effort but LOUD — a release failure
|
|
8069
|
+
// is logged and never masks the original run-fatal error. TTL expiry stays
|
|
8070
|
+
// the recovery path for true crashes (isolate death) where this never runs.
|
|
8071
|
+
await releaseRuntimeLeasesOnTeardown({
|
|
8072
|
+
req,
|
|
8073
|
+
leasedSheetNamespaces,
|
|
8074
|
+
emit: wrappedEmit,
|
|
8075
|
+
});
|
|
8076
|
+
const errorBilling = extractErrorBilling(error);
|
|
8002
8077
|
const salvage: ReceiptSalvage | null = aborted
|
|
8003
8078
|
? null
|
|
8004
8079
|
: receiptSalvage.build();
|
|
8005
|
-
const
|
|
8080
|
+
const terminalMessage =
|
|
8006
8081
|
salvage && salvage.totalEntries > 0
|
|
8007
|
-
? `${
|
|
8008
|
-
:
|
|
8009
|
-
|
|
8010
|
-
if (options?.persistResultDatasets) {
|
|
8082
|
+
? `${message} ${receiptSalvageFailureSuffix(salvage)}`
|
|
8083
|
+
: message;
|
|
8084
|
+
if (options?.persistResultDatasets || req.playCallGovernance) {
|
|
8011
8085
|
appendRunLogLine(
|
|
8012
|
-
`${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(
|
|
8086
|
+
`${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(terminalMessage)}`,
|
|
8013
8087
|
);
|
|
8014
8088
|
const terminalUpdateStartedAt = nowMs();
|
|
8015
8089
|
await flushTerminalLedgerEvents({
|
|
@@ -8017,64 +8091,70 @@ async function executeRunRequest(
|
|
|
8017
8091
|
runId: req.runId,
|
|
8018
8092
|
source: 'worker',
|
|
8019
8093
|
occurredAt: nowMs(),
|
|
8020
|
-
error:
|
|
8094
|
+
error: terminalMessage,
|
|
8021
8095
|
result: aborted
|
|
8022
8096
|
? undefined
|
|
8023
8097
|
: {
|
|
8024
8098
|
success: false,
|
|
8025
8099
|
status: 'failed',
|
|
8026
|
-
error:
|
|
8100
|
+
error: terminalMessage,
|
|
8101
|
+
...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
|
|
8027
8102
|
errors: [
|
|
8028
8103
|
{
|
|
8029
8104
|
code: failure.code,
|
|
8030
8105
|
phase: failure.phase,
|
|
8031
|
-
message,
|
|
8106
|
+
message: terminalMessage,
|
|
8032
8107
|
retryable: failure.retryable,
|
|
8033
8108
|
...(errorBilling ? { billing: errorBilling } : {}),
|
|
8034
8109
|
...(failure.cause ? { cause: failure.cause } : {}),
|
|
8035
8110
|
},
|
|
8036
8111
|
],
|
|
8037
|
-
...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
|
|
8038
8112
|
},
|
|
8039
8113
|
});
|
|
8040
8114
|
recordRunnerPerfTrace({
|
|
8041
8115
|
req,
|
|
8042
|
-
phase:
|
|
8043
|
-
?
|
|
8044
|
-
|
|
8116
|
+
phase: req.playCallGovernance
|
|
8117
|
+
? aborted
|
|
8118
|
+
? 'runner.child_terminal_ledger_append_cancelled'
|
|
8119
|
+
: 'runner.child_terminal_ledger_append_failed'
|
|
8120
|
+
: aborted
|
|
8121
|
+
? 'runner.terminal_ledger_append_cancelled'
|
|
8122
|
+
: 'runner.terminal_ledger_append_failed',
|
|
8045
8123
|
ms: nowMs() - terminalUpdateStartedAt,
|
|
8046
8124
|
extra: {
|
|
8047
8125
|
errorCode: failure.code,
|
|
8048
8126
|
errorPhase: failure.phase,
|
|
8049
8127
|
},
|
|
8050
8128
|
});
|
|
8051
|
-
|
|
8052
|
-
|
|
8053
|
-
|
|
8054
|
-
|
|
8055
|
-
|
|
8056
|
-
|
|
8057
|
-
.catch((finalizeError) => {
|
|
8058
|
-
console.error(
|
|
8059
|
-
`[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
|
|
8060
|
-
finalizeError instanceof Error
|
|
8061
|
-
? finalizeError.message
|
|
8062
|
-
: String(finalizeError)
|
|
8063
|
-
}`,
|
|
8064
|
-
);
|
|
8129
|
+
if (options?.persistResultDatasets) {
|
|
8130
|
+
const billingStartedAt = nowMs();
|
|
8131
|
+
await finalizeWorkerComputeBilling({
|
|
8132
|
+
req,
|
|
8133
|
+
success: false,
|
|
8134
|
+
actionEstimate: 4,
|
|
8065
8135
|
})
|
|
8066
|
-
|
|
8067
|
-
|
|
8068
|
-
|
|
8069
|
-
|
|
8070
|
-
|
|
8136
|
+
.catch((finalizeError) => {
|
|
8137
|
+
console.error(
|
|
8138
|
+
`[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
|
|
8139
|
+
finalizeError instanceof Error
|
|
8140
|
+
? finalizeError.message
|
|
8141
|
+
: String(finalizeError)
|
|
8142
|
+
}`,
|
|
8143
|
+
);
|
|
8144
|
+
})
|
|
8145
|
+
.finally(() => {
|
|
8146
|
+
recordRunnerPerfTrace({
|
|
8147
|
+
req,
|
|
8148
|
+
phase: 'runner.compute_billing_finalize_failed',
|
|
8149
|
+
ms: nowMs() - billingStartedAt,
|
|
8150
|
+
});
|
|
8071
8151
|
});
|
|
8072
|
-
|
|
8152
|
+
}
|
|
8073
8153
|
}
|
|
8074
8154
|
await signalParentPlayTerminal({
|
|
8075
8155
|
req,
|
|
8076
8156
|
status: aborted ? 'cancelled' : 'failed',
|
|
8077
|
-
error:
|
|
8157
|
+
error: terminalMessage,
|
|
8078
8158
|
}).catch(() => null);
|
|
8079
8159
|
recordRunnerPerfTrace({
|
|
8080
8160
|
req,
|
|
@@ -8088,7 +8168,9 @@ async function executeRunRequest(
|
|
|
8088
8168
|
await drainRunnerPerfTraces(req);
|
|
8089
8169
|
throw error;
|
|
8090
8170
|
} finally {
|
|
8091
|
-
|
|
8171
|
+
if (runtimeDeadlineTimer) {
|
|
8172
|
+
clearTimeout(runtimeDeadlineTimer);
|
|
8173
|
+
}
|
|
8092
8174
|
}
|
|
8093
8175
|
}
|
|
8094
8176
|
|
|
@@ -8180,10 +8262,12 @@ function runRequestFromWorkflowParams(
|
|
|
8180
8262
|
orgId: String(params.orgId ?? ''),
|
|
8181
8263
|
playName: String(params.playName ?? ''),
|
|
8182
8264
|
userEmail: typeof params.userEmail === 'string' ? params.userEmail : null,
|
|
8265
|
+
force: params.force === true,
|
|
8266
|
+
forceToolRefresh: params.forceToolRefresh === true,
|
|
8267
|
+
runAttempt: normalizeWorkerRunAttempt(params.runAttempt),
|
|
8183
8268
|
runtimeInput: isRecord(params.input)
|
|
8184
8269
|
? (params.input as Record<string, unknown>)
|
|
8185
8270
|
: {},
|
|
8186
|
-
force: params.forceToolRefresh === true,
|
|
8187
8271
|
inlineCsv: isInlineCsv(params.inlineCsv) ? params.inlineCsv : null,
|
|
8188
8272
|
inputFiles:
|
|
8189
8273
|
inputFile && inputStorageKey
|
|
@@ -8241,6 +8325,14 @@ function runRequestFromWorkflowParams(
|
|
|
8241
8325
|
params.coordinatorInternalToken.trim()
|
|
8242
8326
|
? params.coordinatorInternalToken.trim()
|
|
8243
8327
|
: null,
|
|
8328
|
+
runtimeTestFaultHeader:
|
|
8329
|
+
typeof params.runtimeTestFaultHeader === 'string' &&
|
|
8330
|
+
params.runtimeTestFaultHeader.trim()
|
|
8331
|
+
? params.runtimeTestFaultHeader.trim()
|
|
8332
|
+
: null,
|
|
8333
|
+
testPolicyOverrides: normalizeRuntimeTestPolicyOverrides(
|
|
8334
|
+
params.testPolicyOverrides,
|
|
8335
|
+
),
|
|
8244
8336
|
totalRows:
|
|
8245
8337
|
typeof params.totalRows === 'number' && Number.isFinite(params.totalRows)
|
|
8246
8338
|
? params.totalRows
|
|
@@ -8310,6 +8402,7 @@ async function persistResultDatasets(
|
|
|
8310
8402
|
rows: chunk.map((row) => ({ ...row })),
|
|
8311
8403
|
runId: req.runId,
|
|
8312
8404
|
inputOffset,
|
|
8405
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
8313
8406
|
userEmail: req.userEmail,
|
|
8314
8407
|
preloadedDbSessions: req.preloadedDbSessions ?? null,
|
|
8315
8408
|
});
|
|
@@ -8346,6 +8439,7 @@ async function persistResultDatasets(
|
|
|
8346
8439
|
rows: dataset.rows,
|
|
8347
8440
|
runId: req.runId,
|
|
8348
8441
|
inputOffset: 0,
|
|
8442
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
8349
8443
|
userEmail: req.userEmail,
|
|
8350
8444
|
});
|
|
8351
8445
|
}
|
|
@@ -8627,7 +8721,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
|
|
|
8627
8721
|
try {
|
|
8628
8722
|
const response = await fetchRuntimeApi(
|
|
8629
8723
|
req.baseUrl,
|
|
8630
|
-
'/api/v2/
|
|
8724
|
+
'/api/v2/internal/play-runtime',
|
|
8631
8725
|
{
|
|
8632
8726
|
method: 'POST',
|
|
8633
8727
|
headers: {
|