deepline 0.1.243 → 0.1.245

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +1 -1
  2. package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +5 -25
  3. package/dist/bundling-sources/sdk/src/release.ts +25 -26
  4. package/dist/bundling-sources/sdk/src/types.ts +1 -1
  5. package/dist/bundling-sources/shared_libs/play-runtime/backend.ts +2 -21
  6. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -6
  7. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +6 -13
  8. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +9 -26
  9. package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +1 -2
  10. package/dist/bundling-sources/shared_libs/play-runtime/map-row-identity.ts +1 -1
  11. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +191 -29
  12. package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +1 -1
  13. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +1 -36
  14. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +2 -3
  15. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +0 -7
  16. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +1 -12
  17. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +2 -3
  18. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +5 -15
  19. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +6 -21
  20. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +3 -4
  21. package/dist/bundling-sources/shared_libs/plays/artifact-kind-guard.ts +16 -29
  22. package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +4 -8
  23. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +21 -712
  24. package/dist/bundling-sources/shared_libs/plays/bundling/limits.ts +1 -18
  25. package/dist/cli/index.js +28 -71
  26. package/dist/cli/index.mjs +28 -71
  27. package/dist/{compiler-manifest-CYcwzSOJ.d.ts → compiler-manifest-DlE7dnRm.d.mts} +2 -8
  28. package/dist/{compiler-manifest-CYcwzSOJ.d.mts → compiler-manifest-DlE7dnRm.d.ts} +2 -8
  29. package/dist/index.d.mts +2 -2
  30. package/dist/index.d.ts +2 -2
  31. package/dist/index.js +23 -22
  32. package/dist/index.mjs +23 -22
  33. package/dist/plays/bundle-play-file.d.mts +7 -22
  34. package/dist/plays/bundle-play-file.d.ts +7 -22
  35. package/dist/plays/bundle-play-file.mjs +26 -614
  36. package/package.json +2 -2
  37. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +0 -4504
  38. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +0 -1386
  39. package/dist/bundling-sources/apps/play-runner-workers/src/durable-object-deploy-handoff.ts +0 -30
  40. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +0 -8385
  41. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md +0 -21
  42. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts +0 -251
  43. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/coordinator-progress.ts +0 -289
  44. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/csv-rows.ts +0 -85
  45. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +0 -23
  46. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/dataset-handles.ts +0 -528
  47. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +0 -229
  48. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/ledger-event-batches.ts +0 -81
  49. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/live-progress.ts +0 -22
  50. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +0 -920
  51. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts +0 -90
  52. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +0 -475
  53. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +0 -570
  54. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/result-dataset-persistence.ts +0 -50
  55. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/row-isolation.ts +0 -5
  56. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +0 -519
  57. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-batch.ts +0 -107
  58. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +0 -2541
  59. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-http-errors.ts +0 -7
  60. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +0 -227
  61. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +0 -130
  62. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +0 -39
  63. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/workflow-preview.ts +0 -26
  64. package/dist/bundling-sources/apps/play-runner-workers/src/runtime-api-retry.ts +0 -23
  65. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-instance-create.ts +0 -47
  66. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +0 -167
  67. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +0 -91
  68. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +0 -329
  69. package/dist/bundling-sources/sdk/src/worker-play-entry.ts +0 -240
  70. package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +0 -252
  71. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +0 -217
@@ -1,4504 +0,0 @@
1
- /**
2
- * Coordinator Worker entry — main module exported by wrangler.toml.
3
- *
4
- * Hosts:
5
- * - DynamicWorkflow (native Cloudflare Dynamic Workflows)
6
- * - PlayDedup DO (default export class)
7
- * - HTTP fetch handler for /api/v2/plays/run dispatch (Vercel forwards to here)
8
- * - WorkerLoader-backed dynamic play Workers
9
- */
10
-
11
- export { PlayDedup } from './dedup-do';
12
-
13
- import {
14
- WorkerEntrypoint,
15
- WorkflowEntrypoint,
16
- exports as workersExports,
17
- } from 'cloudflare:workers';
18
- import {
19
- createWorkerTelemetry,
20
- type TelemetryEvent,
21
- type TelemetryFields,
22
- type TelemetryScalar,
23
- } from '../../../shared_libs/observability/worker-telemetry';
24
- import { createSecretRedactionContext } from '../../../shared_libs/play-runtime/secret-redaction';
25
- import {
26
- _dispatcherBindingImpl as createDispatcherWorkflowBinding,
27
- dispatchWorkflow,
28
- DynamicWorkflowBinding,
29
- type WorkflowRunner,
30
- } from '@cloudflare/dynamic-workflows';
31
- import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
32
- import type { PlayRunInputPayload } from '../../../shared_libs/play-runtime/scheduler-backend';
33
- import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runtime/test-runtime-seams';
34
- import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
35
- import {
36
- DB_SESSION_DEFAULT_TTL_SECONDS,
37
- type PreloadedRuntimeDbSession,
38
- } from '../../../shared_libs/play-runtime/db-session';
39
- import {
40
- dbSessionPostgresUrlAad,
41
- encryptDbSessionPostgresUrl,
42
- } from '../../../shared_libs/play-runtime/db-session-crypto';
43
- import { planRuntimeDbSessionRequirements } from '../../../shared_libs/play-runtime/db-session-plan';
44
- import type { PlayRuntimeManifest } from '../../../shared_libs/plays/compiler-manifest';
45
- import {
46
- WORKSPACE_STORAGE_NOT_READY_CODE,
47
- WORKSPACE_STORAGE_NOT_READY_MESSAGE,
48
- normalizePlayRunFailure,
49
- } from '../../../shared_libs/play-runtime/run-failure';
50
- import type { PlayRunLedgerEvent } from '../../../shared_libs/play-runtime/run-ledger';
51
- import {
52
- COORDINATOR_INTERNAL_TOKEN_HEADER,
53
- COORDINATOR_RUN_SCOPE_HEADER,
54
- } from '../../../shared_libs/play-runtime/coordinator-headers';
55
- import {
56
- PLAY_RUNTIME_API_COMPAT_PATH,
57
- PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH,
58
- } from '../../../shared_libs/play-runtime/runtime-api-paths';
59
- import {
60
- decideWorkflowPlatformRetry,
61
- PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
62
- shouldPublishWorkflowRuntimeFailure,
63
- } from './workflow-retry';
64
- import {
65
- WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES,
66
- WORKFLOW_RETRY_PARAMS_MAX_BYTES,
67
- buildWorkflowRetryParams,
68
- jsonByteLength,
69
- workflowRetryParamsStorageKey,
70
- type WorkflowRetryParamsRef,
71
- } from './workflow-retry-state';
72
- import { createOrAttachWorkflowInstance } from './workflow-instance-create';
73
- import { sanitizeLiveLogLines } from './runtime/live-progress';
74
- import {
75
- DURABLE_OBJECT_DEPLOY_HANDOFF_RETRY_DELAYS_MS,
76
- durableObjectDeployHandoffMessage,
77
- sleepForDurableObjectDeployHandoffRetry,
78
- } from './durable-object-deploy-handoff';
79
-
80
- export { DynamicWorkflowBinding };
81
-
82
- const coordinatorTelemetry = createWorkerTelemetry({
83
- service: 'play-runtime',
84
- component: 'coordinator',
85
- });
86
- const coordinatorStorageRedactor = createSecretRedactionContext();
87
- const coordinatorPerfTelemetry = createWorkerTelemetry({
88
- service: 'play-runtime',
89
- component: 'coordinator',
90
- adapter: {
91
- emit(event: TelemetryEvent) {
92
- console.log(`[perf-trace] ${JSON.stringify(event)}`);
93
- },
94
- },
95
- });
96
-
97
- function coordinatorTelemetryFields(input: object): TelemetryFields {
98
- const fields: Record<string, TelemetryScalar> = {};
99
- for (const [key, value] of Object.entries(input)) {
100
- if (
101
- value === undefined ||
102
- value === null ||
103
- typeof value === 'string' ||
104
- typeof value === 'number' ||
105
- typeof value === 'boolean'
106
- ) {
107
- fields[key] = value;
108
- }
109
- }
110
- return fields;
111
- }
112
-
113
- function recordCoordinatorEvent(
114
- level: 'info' | 'warn' | 'error',
115
- event: string,
116
- tag: string,
117
- input: Record<string, unknown> = {},
118
- ): void {
119
- const telemetry = coordinatorTelemetry.child({
120
- context: {
121
- runId: typeof input.runId === 'string' ? input.runId : null,
122
- orgId: typeof input.orgId === 'string' ? input.orgId : null,
123
- playName: typeof input.playName === 'string' ? input.playName : null,
124
- requestId: typeof input.requestId === 'string' ? input.requestId : null,
125
- },
126
- });
127
- const fields = coordinatorTelemetryFields(input);
128
- if (level === 'error') {
129
- void telemetry.error(event, input.error ?? input.message, fields, { tag });
130
- } else {
131
- void telemetry[level](event, fields, { tag });
132
- }
133
- }
134
-
135
- export type PlayWorkflowParams = {
136
- runId: string;
137
- playId: string;
138
- playName: string;
139
- artifactStorageKey: string;
140
- artifactHash: string;
141
- graphHash: string;
142
- input: PlayRunInputPayload;
143
- force?: boolean;
144
- runAttempt?: number;
145
- inputFile?: {
146
- name?: string;
147
- r2Key?: string;
148
- storageKey?: string;
149
- path?: string;
150
- fileName?: string;
151
- logicalPath?: string;
152
- contentType?: string;
153
- bytes?: number;
154
- } | null;
155
- inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
156
- packagedFiles?: Array<{
157
- playPath: string;
158
- storageKey: string;
159
- contentType?: string;
160
- bytes?: number;
161
- inlineText?: string;
162
- }> | null;
163
- contractSnapshot?: unknown;
164
- executionPlan?: ExecutionPlan | null;
165
- forceToolRefresh?: boolean | null;
166
- preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
167
- preloadedDbSessionRef?: {
168
- runId: string;
169
- sessionCount: number;
170
- expiresAt: number;
171
- } | null;
172
- dynamicWorkerCode?: string | null;
173
- executorToken: string;
174
- baseUrl: string;
175
- integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
176
- orgId: string;
177
- userEmail: string;
178
- userId?: string | null;
179
- runtimeBackend: string;
180
- dedupBackend: string;
181
- totalRows?: number;
182
- coordinatorUrl?: string | null;
183
- coordinatorInternalToken?: string | null;
184
- runtimeTestFaultHeader?: string | null;
185
- testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
186
- submittedAt?: number | null;
187
- };
188
-
189
- type CoordinatorTiming = {
190
- phase: string;
191
- ms: number;
192
- graphHash?: string | null;
193
- extra?: Record<string, unknown>;
194
- };
195
-
196
- type CoordinatorPerfTracePayload = {
197
- ts: number;
198
- source: 'coordinator' | 'dynamic_worker';
199
- runId: string;
200
- phase: string;
201
- ms: number;
202
- graphHash?: string | null;
203
- [key: string]: unknown;
204
- };
205
-
206
- type CoordinatorPerfTraceInput = {
207
- runId: string;
208
- phase: string;
209
- ms: number;
210
- graphHash?: string | null;
211
- extra?: Record<string, unknown>;
212
- };
213
-
214
- type CoordinatorPerfTraceSink = (event: CoordinatorPerfTraceInput) => void;
215
-
216
- type CoordinatorTerminalState = {
217
- runId: string;
218
- status: 'completed' | 'failed' | 'cancelled';
219
- result?: unknown;
220
- error?: string | null;
221
- totalRows?: unknown;
222
- durationMs?: unknown;
223
- playName?: string | null;
224
- liveLogs?: string[] | null;
225
- liveNodeProgress?: unknown;
226
- completedAt?: number;
227
- };
228
-
229
- type CoordinatorRunEvent =
230
- | {
231
- seq?: number;
232
- runId: string;
233
- type: 'status';
234
- status: string;
235
- ts: number;
236
- logs?: string[];
237
- }
238
- | {
239
- seq?: number;
240
- runId: string;
241
- type: 'log';
242
- line: string;
243
- ts: number;
244
- }
245
- | {
246
- seq?: number;
247
- runId: string;
248
- type: 'progress';
249
- status: string;
250
- ts: number;
251
- logs?: string[];
252
- activeNodeId?: string | null;
253
- activeArtifactTableNamespace?: string | null;
254
- updatedAt?: number | null;
255
- liveNodeProgress?: unknown;
256
- }
257
- | {
258
- seq?: number;
259
- runId: string;
260
- type: 'terminal';
261
- status: 'completed' | 'failed' | 'cancelled';
262
- ts: number;
263
- result?: unknown;
264
- error?: string | null;
265
- totalRows?: unknown;
266
- durationMs?: unknown;
267
- playName?: string | null;
268
- liveLogs?: string[] | null;
269
- liveNodeProgress?: unknown;
270
- };
271
-
272
- type InlineWorkerRunResponse = {
273
- status?: 'completed' | 'failed';
274
- result?: unknown;
275
- outputRows?: number;
276
- durationMs?: number;
277
- parseMs?: number;
278
- timings?: Array<{
279
- phase?: unknown;
280
- ms?: unknown;
281
- extra?: unknown;
282
- }>;
283
- events?: Array<
284
- | { type: 'log'; message?: string; level?: string; ts?: number }
285
- | { type: 'result'; result?: unknown; outputRows?: number; ts?: number }
286
- | { type: 'error'; message?: string; stack?: string; ts?: number }
287
- | Record<string, unknown>
288
- >;
289
- error?: { message?: string; stack?: string | null };
290
- };
291
-
292
- function isRecord(value: unknown): value is Record<string, unknown> {
293
- return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
294
- }
295
-
296
- interface CoordinatorEnv {
297
- PLAY_WORKFLOW: Workflow<PlayWorkflowParams>;
298
- PLAY_DEDUP: DurableObjectNamespace;
299
- LOADER?: {
300
- get(
301
- id: string,
302
- load: () => Promise<{
303
- compatibilityDate: string;
304
- compatibilityFlags?: string[];
305
- mainModule: string;
306
- modules: Record<string, string>;
307
- env?: Record<string, unknown>;
308
- globalOutbound?: Fetcher | null;
309
- }>,
310
- ): {
311
- getEntrypoint(name?: string): Fetcher & WorkflowRunner;
312
- };
313
- };
314
- PLAYS_BUCKET: R2Bucket;
315
- DEEPLINE_API_BASE_URL: string;
316
- DEEPLINE_INTERNAL_TOKEN?: string;
317
- DEEPLINE_TAIL_LOG_TOKEN?: string;
318
- DEEPLINE_COORDINATOR_DEPLOY_MARKER?: string;
319
- CF_VERSION_METADATA?: WorkerVersionMetadata;
320
- VERCEL_PROTECTION_BYPASS_TOKEN?: string;
321
- DEEPLINE_PLAY_PREVIEW_SLUG?: string;
322
- /**
323
- * Service binding to the long-lived Play Harness Worker
324
- * (apps/play-harness-worker). Provides typed RPC access to leaf-level
325
- * helpers (runtime-API HTTP forwarder, Neon dataset IO, …) that we
326
- * deliberately keep OUT of every per-graphHash play bundle.
327
- *
328
- * Required in every Cloudflare play environment. The coordinator fails
329
- * submit/load before handing a per-play Worker an unwired harness binding.
330
- *
331
- * Wired in wrangler.toml as `[[services]] binding = "HARNESS"`.
332
- */
333
- HARNESS: import('../../play-harness-worker/src/rpc-types').PlayHarnessRpc;
334
- }
335
-
336
- const WORKFLOW_READ_ONLY_ACTIONS = new Set(['', 'result', 'status', 'tail']);
337
-
338
- function authorizeCoordinatorControlRequest(input: {
339
- request: Request;
340
- env: CoordinatorEnv;
341
- runId?: string;
342
- requireRunScope?: boolean;
343
- }): Response | null {
344
- const expectedToken = input.env.DEEPLINE_INTERNAL_TOKEN?.trim();
345
- if (!expectedToken) {
346
- return new Response('coordinator internal token is not configured', {
347
- status: 503,
348
- });
349
- }
350
- const actualToken =
351
- input.request.headers.get(COORDINATOR_INTERNAL_TOKEN_HEADER)?.trim() ?? '';
352
- if (actualToken !== expectedToken) {
353
- return new Response('unauthorized', { status: 401 });
354
- }
355
- if (input.requireRunScope) {
356
- const actualRunScope =
357
- input.request.headers.get(COORDINATOR_RUN_SCOPE_HEADER)?.trim() ?? '';
358
- if (!input.runId || actualRunScope !== input.runId) {
359
- return new Response('run scope mismatch', { status: 403 });
360
- }
361
- }
362
- return null;
363
- }
364
-
365
- function isWorkflowMutatingAction(action: string): boolean {
366
- return !WORKFLOW_READ_ONLY_ACTIONS.has(action);
367
- }
368
-
369
- /**
370
- * Best-effort disposal of an RPC stub. Workers' RPC machinery emits
371
- * "stub not disposed" warnings (and, under stress, contributes to V8
372
- * deopt failures in `wrangler dev` local mode) when these are GC'd
373
- * instead of explicitly released. Optional chaining keeps this safe
374
- * for non-stub values.
375
- */
376
- function disposeRpcStub(value: unknown): void {
377
- if (value == null) return;
378
- const disposable = value as { [Symbol.dispose]?: () => void };
379
- try {
380
- disposable[Symbol.dispose]?.();
381
- } catch {
382
- // intentional no-op — disposal is best-effort
383
- }
384
- }
385
-
386
- /**
387
- * Structured perf trace for the coordinator. Emits one [perf-trace] JSON
388
- * line that downstream tooling (tests/v2-plays/runner.ts --perf-summary)
389
- * scrapes from logs/coordinator.log / logs/cf-coordinator-tail.log.
390
- *
391
- * Workerd has no filesystem; this is stdout-only. Stays additive next to
392
- * the existing `[coordinator] ...` logs.
393
- */
394
- function buildCoordinatorPerfTracePayload(
395
- event: CoordinatorPerfTraceInput,
396
- ): CoordinatorPerfTracePayload | null {
397
- if (!event.runId || !event.phase) return null;
398
- return {
399
- ts: Date.now(),
400
- source: 'coordinator',
401
- runId: event.runId,
402
- phase: event.phase,
403
- ms: event.ms,
404
- ...(event.graphHash ? { graphHash: event.graphHash } : {}),
405
- ...(event.extra ?? {}),
406
- };
407
- }
408
-
409
- function logCoordinatorPerfTrace(payload: CoordinatorPerfTracePayload): void {
410
- void coordinatorPerfTelemetry
411
- .child({ context: { runId: payload.runId } })
412
- .info(payload.phase, coordinatorTelemetryFields(payload), {
413
- tag: '[perf-trace]',
414
- });
415
- }
416
-
417
- function recordCoordinatorPerfTrace(event: CoordinatorPerfTraceInput): void {
418
- const payload = buildCoordinatorPerfTracePayload(event);
419
- if (!payload) return;
420
- logCoordinatorPerfTrace(payload);
421
- }
422
-
423
- function safeOrigin(value: string | null | undefined): string | null {
424
- if (!value?.trim()) return null;
425
- try {
426
- return new URL(value).origin;
427
- } catch {
428
- return '<invalid>';
429
- }
430
- }
431
-
432
- async function appendCoordinatorPerfTrace(
433
- env: CoordinatorEnv,
434
- payload: CoordinatorPerfTracePayload,
435
- ): Promise<void> {
436
- const stub = env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(payload.runId));
437
- const response = await stub.fetch(
438
- 'https://deepline.dedup.internal/trace-add',
439
- {
440
- method: 'POST',
441
- headers: { 'content-type': 'application/json' },
442
- body: JSON.stringify(coordinatorStorageRedactor.redact(payload)),
443
- },
444
- );
445
- if (!response.ok) {
446
- throw new Error(
447
- `coordinator trace append failed ${response.status}: ${(
448
- await response.text().catch(() => '')
449
- ).slice(0, 200)}`,
450
- );
451
- }
452
- }
453
-
454
- function recordCoordinatorPerfTraceBuffered(
455
- env: CoordinatorEnv,
456
- ctx: ExecutionContext | null | undefined,
457
- event: CoordinatorPerfTraceInput,
458
- ): void {
459
- const payload = buildCoordinatorPerfTracePayload(event);
460
- if (!payload) return;
461
- logCoordinatorPerfTrace(payload);
462
- const append = appendCoordinatorPerfTrace(env, payload).catch((error) => {
463
- return coordinatorTelemetry
464
- .child({ context: { runId: payload.runId } })
465
- .warn(
466
- 'coordinator.perf_trace.buffer_failed',
467
- {
468
- phase: payload.phase,
469
- error: error instanceof Error ? error.message : String(error),
470
- },
471
- { tag: '[coordinator] failed to buffer perf trace' },
472
- );
473
- });
474
- if (typeof ctx?.waitUntil === 'function') {
475
- ctx.waitUntil(append);
476
- }
477
- }
478
-
479
- async function listCoordinatorPerfTrace(
480
- env: CoordinatorEnv,
481
- runId: string,
482
- ): Promise<CoordinatorPerfTracePayload[]> {
483
- const stub = env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(runId));
484
- const response = await stub.fetch(
485
- 'https://deepline.dedup.internal/trace-list',
486
- );
487
- if (!response.ok) {
488
- throw new Error(`coordinator trace list failed ${response.status}`);
489
- }
490
- const body = (await response.json().catch(() => ({}))) as {
491
- entries?: unknown;
492
- };
493
- if (!Array.isArray(body.entries)) return [];
494
- return body.entries.filter(
495
- (entry): entry is CoordinatorPerfTracePayload =>
496
- isRecord(entry) &&
497
- (entry.source === 'coordinator' || entry.source === 'dynamic_worker') &&
498
- typeof entry.runId === 'string' &&
499
- typeof entry.phase === 'string' &&
500
- typeof entry.ts === 'number' &&
501
- typeof entry.ms === 'number',
502
- );
503
- }
504
-
505
- async function writeCoordinatorTerminalState(
506
- env: CoordinatorEnv,
507
- state: CoordinatorTerminalState,
508
- ): Promise<void> {
509
- const stub = env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(state.runId));
510
- const response = await stub.fetch(
511
- 'https://deepline.dedup.internal/terminal-set',
512
- {
513
- method: 'POST',
514
- headers: { 'content-type': 'application/json' },
515
- body: JSON.stringify({
516
- ...state,
517
- completedAt: state.completedAt ?? Date.now(),
518
- }),
519
- },
520
- );
521
- if (!response.ok) {
522
- throw new Error(`coordinator terminal set failed ${response.status}`);
523
- }
524
- }
525
-
526
- async function appendCoordinatorRunEvent(
527
- env: CoordinatorEnv,
528
- event: CoordinatorRunEvent,
529
- ): Promise<void> {
530
- const stub = env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(event.runId));
531
- const response = await stub.fetch(
532
- 'https://deepline.dedup.internal/event-add',
533
- {
534
- method: 'POST',
535
- headers: { 'content-type': 'application/json' },
536
- body: JSON.stringify(event),
537
- },
538
- );
539
- if (!response.ok) {
540
- throw new Error(`coordinator event append failed ${response.status}`);
541
- }
542
- }
543
-
544
- async function listCoordinatorRunEvents(input: {
545
- env: CoordinatorEnv;
546
- runId: string;
547
- afterSeq: number;
548
- timeoutMs: number;
549
- }): Promise<{ events: CoordinatorRunEvent[]; latestSeq: number }> {
550
- const stub = input.env.PLAY_DEDUP.get(
551
- input.env.PLAY_DEDUP.idFromName(input.runId),
552
- );
553
- const response = await stub.fetch(
554
- `https://deepline.dedup.internal/event-list?afterSeq=${encodeURIComponent(
555
- String(Math.max(0, Math.floor(input.afterSeq))),
556
- )}&timeoutMs=${encodeURIComponent(
557
- String(Math.max(0, Math.floor(input.timeoutMs))),
558
- )}`,
559
- );
560
- if (!response.ok) {
561
- throw new Error(`coordinator event list failed ${response.status}`);
562
- }
563
- const body = (await response.json().catch(() => ({}))) as {
564
- events?: unknown;
565
- latestSeq?: unknown;
566
- };
567
- return {
568
- events: Array.isArray(body.events)
569
- ? body.events.filter(
570
- (event): event is CoordinatorRunEvent =>
571
- isRecord(event) &&
572
- typeof event.runId === 'string' &&
573
- event.runId === input.runId &&
574
- typeof event.type === 'string' &&
575
- typeof event.ts === 'number',
576
- )
577
- : [],
578
- latestSeq:
579
- typeof body.latestSeq === 'number' ? body.latestSeq : input.afterSeq,
580
- };
581
- }
582
-
583
- async function readCoordinatorTerminalState(
584
- env: CoordinatorEnv,
585
- runId: string,
586
- ): Promise<CoordinatorTerminalState | null> {
587
- const stub = env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(runId));
588
- const response = await stub.fetch(
589
- 'https://deepline.dedup.internal/terminal-get',
590
- );
591
- if (!response.ok) {
592
- throw new Error(`coordinator terminal get failed ${response.status}`);
593
- }
594
- const body = (await response.json().catch(() => ({}))) as {
595
- state?: unknown;
596
- };
597
- const state = body.state;
598
- if (!isRecord(state) || state.runId !== runId) return null;
599
- if (
600
- state.status !== 'completed' &&
601
- state.status !== 'failed' &&
602
- state.status !== 'cancelled'
603
- ) {
604
- return null;
605
- }
606
- return state as CoordinatorTerminalState;
607
- }
608
-
609
- function workflowEventType(name: string): string {
610
- const normalized = name
611
- .trim()
612
- .replace(/[^A-Za-z0-9_-]+/g, '_')
613
- .replace(/^_+|_+$/g, '')
614
- .slice(0, 100);
615
- return normalized || 'deepline_event';
616
- }
617
-
618
- type DynamicWorkflowMetadata = {
619
- runId?: string | null;
620
- graphHash: string;
621
- artifactStorageKey: string;
622
- artifactHash?: string | null;
623
- dynamicWorkerCode?: string | null;
624
- dynamicWorkerCodeHash?: string | null;
625
- integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
626
- packagedFiles?: Array<{
627
- playPath: string;
628
- storageKey: string;
629
- contentType?: string;
630
- bytes?: number;
631
- inlineText?: string;
632
- }> | null;
633
- };
634
-
635
- const SUBMIT_INITIAL_STATE_MAX_WAIT_MS = 0;
636
- const SUBMIT_INITIAL_STATE_POLL_MS = 50;
637
- const WORKFLOW_RETRY_STATE_TTL_MS = 60 * 60 * 1000;
638
- const WORKFLOW_SUBMIT_DUPLICATE_WAIT_MS = 55_000;
639
- const WORKFLOW_SUBMIT_DUPLICATE_POLL_MS = 250;
640
- const CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS = [
641
- 250, 750, 1500, 3000, 5000, 10000,
642
- ] as const;
643
- const COORDINATOR_WORKER_MEMORY_LIMIT_MESSAGE =
644
- 'Worker memory limit hit before the run could finish. For CSV enrich, use --output or --in-place for automatic batches; otherwise rerun smaller --rows ranges.';
645
-
646
- function normalizeIntegrationMode(
647
- value: unknown,
648
- ): 'live' | 'eval_stub' | 'fixture' | null {
649
- return value === 'live' || value === 'eval_stub' || value === 'fixture'
650
- ? value
651
- : null;
652
- }
653
-
654
- function buildDynamicWorkflowMetadata(
655
- params: PlayWorkflowParams,
656
- ): DynamicWorkflowMetadata {
657
- const dynamicWorkerCodeHash =
658
- typeof params.dynamicWorkerCode === 'string' &&
659
- params.dynamicWorkerCode.length > 0
660
- ? stableHash(params.dynamicWorkerCode)
661
- : null;
662
- return {
663
- runId: params.runId ?? null,
664
- graphHash: params.graphHash,
665
- artifactStorageKey: params.artifactStorageKey,
666
- artifactHash: params.artifactHash ?? null,
667
- dynamicWorkerCode: null,
668
- dynamicWorkerCodeHash,
669
- integrationMode: normalizeIntegrationMode(params.integrationMode),
670
- packagedFiles: normalizePackagedFiles(params.packagedFiles),
671
- };
672
- }
673
-
674
- function readWorkflowTraceContext(event: unknown): {
675
- runId: string;
676
- graphHash: string | null;
677
- instanceId: string | null;
678
- submittedAt: number | null;
679
- } {
680
- const record = isRecord(event) ? event : {};
681
- const payload = isRecord(record.payload) ? record.payload : {};
682
- const params = isRecord(payload.params) ? payload.params : payload;
683
- const metadata = isRecord(payload.__dispatcherMetadata)
684
- ? payload.__dispatcherMetadata
685
- : null;
686
- const runId =
687
- (typeof params?.runId === 'string' && params.runId) ||
688
- (typeof metadata?.runId === 'string' && metadata.runId) ||
689
- (typeof record.instanceId === 'string' && record.instanceId) ||
690
- 'unknown-workflow-run';
691
- const graphHash =
692
- (typeof params?.graphHash === 'string' && params.graphHash) ||
693
- (typeof metadata?.graphHash === 'string' && metadata.graphHash) ||
694
- null;
695
- return {
696
- runId,
697
- graphHash,
698
- instanceId:
699
- typeof record.instanceId === 'string' && record.instanceId
700
- ? record.instanceId
701
- : null,
702
- submittedAt:
703
- typeof params?.submittedAt === 'number' &&
704
- Number.isFinite(params.submittedAt)
705
- ? params.submittedAt
706
- : null,
707
- };
708
- }
709
-
710
- async function waitForSubmitInitialState(input: {
711
- instance: WorkflowInstance;
712
- runId: string;
713
- waitMs: number;
714
- }): Promise<Record<string, unknown> | null> {
715
- const waitMs = Math.max(
716
- 0,
717
- Math.min(Math.floor(input.waitMs), SUBMIT_INITIAL_STATE_MAX_WAIT_MS),
718
- );
719
- if (waitMs <= 0) return null;
720
- const startedAt = Date.now();
721
- let status = await input.instance.status();
722
- while (Date.now() - startedAt < waitMs) {
723
- const result = mapWorkflowResult(input.runId, status);
724
- if (
725
- result.status === 'completed' ||
726
- result.status === 'failed' ||
727
- result.status === 'cancelled'
728
- ) {
729
- return result as unknown as Record<string, unknown>;
730
- }
731
- await sleep(SUBMIT_INITIAL_STATE_POLL_MS);
732
- status = await input.instance.status();
733
- }
734
- const result = mapWorkflowResult(input.runId, status);
735
- return result.status === 'completed' ||
736
- result.status === 'failed' ||
737
- result.status === 'cancelled'
738
- ? (result as unknown as Record<string, unknown>)
739
- : null;
740
- }
741
-
742
- async function createDynamicWorkflowInstance(input: {
743
- env: CoordinatorEnv;
744
- id: string;
745
- params: PlayWorkflowParams;
746
- metadataParams?: PlayWorkflowParams;
747
- }): Promise<WorkflowInstance> {
748
- const workflow = createDispatcherWorkflowBinding(
749
- () => input.env.PLAY_WORKFLOW as unknown as Workflow,
750
- buildDynamicWorkflowMetadata(input.metadataParams ?? input.params),
751
- );
752
- return workflow.create({
753
- id: input.id,
754
- params: input.params,
755
- });
756
- }
757
-
758
- function runScopedDurableObject(
759
- env: CoordinatorEnv,
760
- runId: string,
761
- ): DurableObjectStub {
762
- return env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(runId));
763
- }
764
-
765
- /**
766
- * Address the rate-state Durable Object for a single `<orgId>:<provider>`
767
- * bucket. Keying the DO by the bucket id (not the run id) makes one
768
- * single-threaded instance own that bucket's request window across every
769
- * isolate of every run in the org — which is exactly what the distributed Rate
770
- * State Backend needs. Reuses the PlayDedup namespace (the DO already hosts the
771
- * token-bucket handlers) so no extra binding is required.
772
- */
773
- function rateBucketDurableObject(
774
- env: CoordinatorEnv,
775
- bucketId: string,
776
- ): DurableObjectStub {
777
- return env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(`rate:${bucketId}`));
778
- }
779
-
780
- async function callRateBucketControl<T>(
781
- env: CoordinatorEnv,
782
- bucketId: string,
783
- path: string,
784
- body: unknown,
785
- ): Promise<T> {
786
- const response = await rateBucketDurableObject(env, bucketId).fetch(
787
- `https://deepline.rate-state.internal${path}`,
788
- {
789
- method: 'POST',
790
- headers: { 'content-type': 'application/json' },
791
- body: JSON.stringify(body),
792
- },
793
- );
794
- if (!response.ok) {
795
- throw new Error(
796
- `rate state ${path} failed ${response.status}: ${(
797
- await response.text().catch(() => '')
798
- ).slice(0, 400)}`,
799
- );
800
- }
801
- return (await response.json()) as T;
802
- }
803
-
804
- async function callRunScopedControl<T>(
805
- env: CoordinatorEnv,
806
- runId: string,
807
- path: string,
808
- init?: RequestInit,
809
- ): Promise<T> {
810
- const maxAttempts = DURABLE_OBJECT_DEPLOY_HANDOFF_RETRY_DELAYS_MS.length + 1;
811
- for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
812
- let response: Response;
813
- try {
814
- response = await runScopedDurableObject(env, runId).fetch(
815
- `https://deepline.run-state.internal${path}`,
816
- {
817
- ...(init ?? {}),
818
- headers: {
819
- 'content-type': 'application/json',
820
- ...(init?.headers ?? {}),
821
- },
822
- },
823
- );
824
- } catch (error) {
825
- const handoffMessage = durableObjectDeployHandoffMessage(error);
826
- if (handoffMessage && attempt < maxAttempts) {
827
- recordCoordinatorEvent(
828
- 'warn',
829
- 'coordinator.run_state.handoff_retry',
830
- '[coordinator] run state DO deploy handoff retry',
831
- {
832
- runId,
833
- path,
834
- attempt,
835
- error: handoffMessage,
836
- },
837
- );
838
- await sleepForDurableObjectDeployHandoffRetry(attempt - 1);
839
- continue;
840
- }
841
- throw error;
842
- }
843
-
844
- if (response.ok) {
845
- return (await response.json()) as T;
846
- }
847
-
848
- const responseText = (await response.text().catch(() => '')).slice(0, 400);
849
- const handoffMessage = durableObjectDeployHandoffMessage(responseText);
850
- if (handoffMessage && attempt < maxAttempts) {
851
- recordCoordinatorEvent(
852
- 'warn',
853
- 'coordinator.run_state.handoff_retry',
854
- '[coordinator] run state DO deploy handoff retry',
855
- {
856
- runId,
857
- path,
858
- attempt,
859
- status: response.status,
860
- error: handoffMessage,
861
- },
862
- );
863
- await sleepForDurableObjectDeployHandoffRetry(attempt - 1);
864
- continue;
865
- }
866
-
867
- throw new Error(
868
- `run state ${path} failed ${response.status}: ${responseText}`,
869
- );
870
- }
871
- throw new Error(`run state ${path} failed after deploy handoff retries`);
872
- }
873
-
874
- async function recordWorkflowInstanceId(input: {
875
- env: CoordinatorEnv;
876
- runId: string;
877
- instanceId: string;
878
- }): Promise<void> {
879
- await callRunScopedControl<{ ok?: unknown }>(
880
- input.env,
881
- input.runId,
882
- '/workflow-instance-put',
883
- {
884
- method: 'POST',
885
- body: JSON.stringify({
886
- runId: input.runId,
887
- instanceId: input.instanceId,
888
- ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
889
- }),
890
- },
891
- );
892
- }
893
-
894
- async function resolveWorkflowInstanceIdForRun(
895
- env: CoordinatorEnv,
896
- runId: string,
897
- ): Promise<string> {
898
- const body = await callRunScopedControl<{ instanceId?: unknown }>(
899
- env,
900
- runId,
901
- `/workflow-instance-get?runId=${encodeURIComponent(runId)}`,
902
- ).catch(() => ({ instanceId: null }));
903
- return typeof body.instanceId === 'string' && body.instanceId
904
- ? body.instanceId
905
- : workflowInstanceId(runId);
906
- }
907
-
908
- async function claimWorkflowSubmit(input: {
909
- env: CoordinatorEnv;
910
- runId: string;
911
- }): Promise<{
912
- claimed: boolean;
913
- status: 'creating' | 'created' | null;
914
- instanceId: string | null;
915
- }> {
916
- const body = await callRunScopedControl<{
917
- claimed?: unknown;
918
- status?: unknown;
919
- instanceId?: unknown;
920
- }>(input.env, input.runId, '/workflow-submit-claim', {
921
- method: 'POST',
922
- body: JSON.stringify({
923
- runId: input.runId,
924
- ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
925
- }),
926
- });
927
- return {
928
- claimed: body.claimed === true,
929
- status:
930
- body.status === 'creating' || body.status === 'created'
931
- ? body.status
932
- : null,
933
- instanceId: typeof body.instanceId === 'string' ? body.instanceId : null,
934
- };
935
- }
936
-
937
- async function releaseWorkflowSubmitClaim(input: {
938
- env: CoordinatorEnv;
939
- runId: string;
940
- }): Promise<void> {
941
- await callRunScopedControl<{ ok?: unknown }>(
942
- input.env,
943
- input.runId,
944
- '/workflow-submit-release',
945
- {
946
- method: 'POST',
947
- body: JSON.stringify({ runId: input.runId }),
948
- },
949
- );
950
- }
951
-
952
- async function waitForWorkflowSubmitCreated(input: {
953
- env: CoordinatorEnv;
954
- runId: string;
955
- initialClaim: Awaited<ReturnType<typeof claimWorkflowSubmit>>;
956
- timeoutMs: number;
957
- pollMs: number;
958
- }): Promise<Awaited<ReturnType<typeof claimWorkflowSubmit>>> {
959
- let claim = input.initialClaim;
960
- const deadline = Date.now() + input.timeoutMs;
961
- while (claim.status !== 'created' || !claim.instanceId) {
962
- if (Date.now() >= deadline) {
963
- return claim;
964
- }
965
- await sleep(input.pollMs);
966
- claim = await claimWorkflowSubmit({
967
- env: input.env,
968
- runId: input.runId,
969
- });
970
- }
971
- return claim;
972
- }
973
-
974
- function assertEncryptedPreloadedDbSessions(
975
- sessions: PreloadedRuntimeDbSession[],
976
- ): void {
977
- for (const session of sessions) {
978
- if (session.session.postgresUrl) {
979
- throw new Error(
980
- 'Preloaded Runtime DB sessions must not carry raw Postgres URLs.',
981
- );
982
- }
983
- if (!session.session.encryptedPostgresUrl) {
984
- throw new Error(
985
- 'Preloaded Runtime DB sessions must carry encrypted Postgres URLs.',
986
- );
987
- }
988
- }
989
- }
990
-
991
- async function readWorkflowDbSessions(input: {
992
- env: CoordinatorEnv;
993
- ref: NonNullable<PlayWorkflowParams['preloadedDbSessionRef']>;
994
- }): Promise<PreloadedRuntimeDbSession[]> {
995
- const response = await runScopedDurableObject(
996
- input.env,
997
- input.ref.runId,
998
- ).fetch(
999
- `https://deepline.dedup.internal/db-sessions-get?runId=${encodeURIComponent(
1000
- input.ref.runId,
1001
- )}`,
1002
- );
1003
- if (!response.ok) {
1004
- throw new Error(
1005
- `workflow db session lookup failed ${response.status}: ${(
1006
- await response.text().catch(() => '')
1007
- ).slice(0, 400)}`,
1008
- );
1009
- }
1010
- const body = (await response.json().catch(() => ({}))) as {
1011
- sessions?: unknown;
1012
- };
1013
- if (!Array.isArray(body.sessions)) {
1014
- throw new Error('workflow db session lookup returned no sessions.');
1015
- }
1016
- const sessions = body.sessions as PreloadedRuntimeDbSession[];
1017
- assertEncryptedPreloadedDbSessions(sessions);
1018
- return sessions;
1019
- }
1020
-
1021
- async function readWorkflowDbSessionsWithRetry(input: {
1022
- env: CoordinatorEnv;
1023
- ref: NonNullable<PlayWorkflowParams['preloadedDbSessionRef']>;
1024
- }): Promise<PreloadedRuntimeDbSession[]> {
1025
- const delays = [25, 50, 100, 200] as const;
1026
- let lastError: unknown = null;
1027
- for (let attempt = 0; attempt <= delays.length; attempt += 1) {
1028
- try {
1029
- return await readWorkflowDbSessions(input);
1030
- } catch (error) {
1031
- lastError = error;
1032
- const message = error instanceof Error ? error.message : String(error);
1033
- if (
1034
- !message.includes('workflow db session lookup failed 404') ||
1035
- attempt >= delays.length
1036
- ) {
1037
- throw error;
1038
- }
1039
- await sleep(delays[attempt]);
1040
- }
1041
- }
1042
- throw lastError instanceof Error ? lastError : new Error(String(lastError));
1043
- }
1044
-
1045
- function externalizedWorkflowDbSessionParams(input: {
1046
- params: PlayWorkflowParams;
1047
- }): {
1048
- params: PlayWorkflowParams;
1049
- sessions: PreloadedRuntimeDbSession[];
1050
- ref: NonNullable<PlayWorkflowParams['preloadedDbSessionRef']> | null;
1051
- } {
1052
- const sessions = Array.isArray(input.params.preloadedDbSessions)
1053
- ? input.params.preloadedDbSessions
1054
- : [];
1055
- if (sessions.length === 0) {
1056
- return { params: input.params, sessions, ref: null };
1057
- }
1058
- const ref: NonNullable<PlayWorkflowParams['preloadedDbSessionRef']> = {
1059
- runId: input.params.runId,
1060
- sessionCount: sessions.length,
1061
- expiresAt: Date.now() + DB_SESSION_DEFAULT_TTL_SECONDS * 1000,
1062
- };
1063
- return {
1064
- params: {
1065
- ...input.params,
1066
- preloadedDbSessions: null,
1067
- preloadedDbSessionRef: ref,
1068
- },
1069
- sessions,
1070
- ref,
1071
- };
1072
- }
1073
-
1074
- function readPreloadedDbSessionRef(
1075
- value: unknown,
1076
- ): NonNullable<PlayWorkflowParams['preloadedDbSessionRef']> | null {
1077
- if (!isRecord(value)) return null;
1078
- const runId = typeof value.runId === 'string' ? value.runId : '';
1079
- const sessionCount =
1080
- typeof value.sessionCount === 'number' &&
1081
- Number.isFinite(value.sessionCount)
1082
- ? value.sessionCount
1083
- : 0;
1084
- const expiresAt =
1085
- typeof value.expiresAt === 'number' && Number.isFinite(value.expiresAt)
1086
- ? value.expiresAt
1087
- : 0;
1088
- if (!runId || sessionCount <= 0 || expiresAt <= 0) return null;
1089
- return { runId, sessionCount, expiresAt };
1090
- }
1091
-
1092
- async function hydrateWorkflowDbSessions(input: {
1093
- env: CoordinatorEnv;
1094
- event: unknown;
1095
- trace: CoordinatorPerfTraceSink;
1096
- }): Promise<unknown> {
1097
- if (!isRecord(input.event)) return input.event;
1098
- const payload = isRecord(input.event.payload) ? input.event.payload : null;
1099
- const envelopeParams = isRecord(payload?.params) ? payload.params : null;
1100
- const params = envelopeParams ?? payload;
1101
- if (!payload || !params || Array.isArray(params.preloadedDbSessions)) {
1102
- return input.event;
1103
- }
1104
- const ref = readPreloadedDbSessionRef(params.preloadedDbSessionRef);
1105
- if (!ref) return input.event;
1106
- const startedAt = Date.now();
1107
- const sessions = await readWorkflowDbSessionsWithRetry({
1108
- env: input.env,
1109
- ref,
1110
- });
1111
- input.trace({
1112
- runId: ref.runId,
1113
- phase: 'coordinator.workflow_db_sessions_hydrated',
1114
- ms: Date.now() - startedAt,
1115
- graphHash: typeof params.graphHash === 'string' ? params.graphHash : null,
1116
- extra: {
1117
- sessions: sessions.length,
1118
- refSessionCount: ref.sessionCount,
1119
- },
1120
- });
1121
- return {
1122
- ...input.event,
1123
- payload: envelopeParams
1124
- ? {
1125
- ...payload,
1126
- params: {
1127
- ...params,
1128
- preloadedDbSessions: sessions,
1129
- },
1130
- }
1131
- : {
1132
- ...payload,
1133
- preloadedDbSessions: sessions,
1134
- },
1135
- };
1136
- }
1137
-
1138
- async function hydrateWorkflowLaunchParams(input: {
1139
- env: CoordinatorEnv;
1140
- event: unknown;
1141
- trace: CoordinatorPerfTraceSink;
1142
- }): Promise<unknown> {
1143
- if (!isRecord(input.event)) return input.event;
1144
- const payload = isRecord(input.event.payload) ? input.event.payload : null;
1145
- const envelopeParams = isRecord(payload?.params) ? payload.params : null;
1146
- const params = envelopeParams ?? payload;
1147
- if (
1148
- !payload ||
1149
- !params ||
1150
- (!isRecord(params.workflowParams) && !isRecord(params.workflowParamsRef))
1151
- ) {
1152
- return input.event;
1153
- }
1154
- const startedAt = Date.now();
1155
- const hydrated = await hydrateWorkflowRetryParams({
1156
- env: input.env,
1157
- params: params.workflowParams,
1158
- paramsRef: params.workflowParamsRef,
1159
- });
1160
- if (!hydrated) return input.event;
1161
- const submittedAt =
1162
- typeof params.submittedAt === 'number' &&
1163
- Number.isFinite(params.submittedAt)
1164
- ? params.submittedAt
1165
- : hydrated.submittedAt;
1166
- const hydratedParams = {
1167
- ...hydrated,
1168
- submittedAt,
1169
- };
1170
- input.trace({
1171
- runId:
1172
- typeof hydratedParams.runId === 'string'
1173
- ? hydratedParams.runId
1174
- : 'unknown-workflow-run',
1175
- phase: 'coordinator.workflow_launch_params_hydrated',
1176
- ms: Date.now() - startedAt,
1177
- graphHash:
1178
- typeof hydratedParams.graphHash === 'string'
1179
- ? hydratedParams.graphHash
1180
- : null,
1181
- extra: {
1182
- paramsBytes: isRecord(params.workflowParamsRef)
1183
- ? typeof params.workflowParamsRef.bytes === 'number'
1184
- ? params.workflowParamsRef.bytes
1185
- : null
1186
- : jsonByteLength(params.workflowParams),
1187
- paramsStorage: isRecord(params.workflowParamsRef) ? 'r2' : 'inline',
1188
- },
1189
- });
1190
- return {
1191
- ...input.event,
1192
- payload: envelopeParams
1193
- ? {
1194
- ...payload,
1195
- params: hydratedParams,
1196
- }
1197
- : {
1198
- ...payload,
1199
- ...hydratedParams,
1200
- },
1201
- };
1202
- }
1203
-
1204
- function workflowCreateParams(
1205
- params: PlayWorkflowParams,
1206
- launchState: {
1207
- params?: PlayWorkflowParams;
1208
- paramsRef?: WorkflowRetryParamsRef;
1209
- },
1210
- ): PlayWorkflowParams {
1211
- const launchParams = launchState.params ?? params;
1212
- const envelope: Record<string, unknown> = {
1213
- runId: params.runId,
1214
- playName: params.playName,
1215
- graphHash: params.graphHash,
1216
- artifactStorageKey: params.artifactStorageKey,
1217
- artifactHash: params.artifactHash,
1218
- submittedAt: params.submittedAt,
1219
- };
1220
- if (launchState.paramsRef) {
1221
- envelope.workflowParamsRef = launchState.paramsRef;
1222
- } else {
1223
- envelope.workflowParams = launchParams;
1224
- }
1225
- return envelope as unknown as PlayWorkflowParams;
1226
- }
1227
-
1228
- function sleep(ms: number): Promise<void> {
1229
- return new Promise((resolve) => setTimeout(resolve, ms));
1230
- }
1231
-
1232
- function readWorkflowPayload(event: unknown): Record<string, unknown> | null {
1233
- if (!isRecord(event)) return null;
1234
- const payload = event.payload;
1235
- if (!isRecord(payload)) return null;
1236
- return isRecord(payload.params) ? payload.params : payload;
1237
- }
1238
-
1239
- function isCoordinatorWorkerMemoryLimitError(error: unknown): boolean {
1240
- const message = error instanceof Error ? error.message : String(error);
1241
- return message.toLowerCase().includes('worker exceeded memory limit');
1242
- }
1243
-
1244
- function normalizeCoordinatorWorkflowRuntimeFailure(
1245
- error: unknown,
1246
- ): ReturnType<typeof normalizePlayRunFailure> {
1247
- const failure = normalizePlayRunFailure(error);
1248
- if (!isCoordinatorWorkerMemoryLimitError(error)) {
1249
- return failure;
1250
- }
1251
- return {
1252
- ...failure,
1253
- code: 'WORKER_MEMORY_LIMIT_EXCEEDED',
1254
- message: COORDINATOR_WORKER_MEMORY_LIMIT_MESSAGE,
1255
- retryable: false,
1256
- cause: failure.message,
1257
- };
1258
- }
1259
-
1260
- async function markWorkflowRuntimeFailure(input: {
1261
- env: CoordinatorEnv;
1262
- event: unknown;
1263
- error: unknown;
1264
- }): Promise<void> {
1265
- const payload = readWorkflowPayload(input.event);
1266
- if (!payload) return;
1267
- const runId = typeof payload.runId === 'string' ? payload.runId : null;
1268
- const baseUrl = typeof payload.baseUrl === 'string' ? payload.baseUrl : null;
1269
- const executorToken =
1270
- typeof payload.executorToken === 'string' ? payload.executorToken : null;
1271
- if (!runId || !baseUrl || !executorToken) return;
1272
- const errorName =
1273
- input.error instanceof Error && input.error.name
1274
- ? input.error.name
1275
- : 'Error';
1276
- const errorMessage =
1277
- input.error instanceof Error ? input.error.message : String(input.error);
1278
- const errorStack =
1279
- input.error instanceof Error && typeof input.error.stack === 'string'
1280
- ? input.error.stack.split('\n').slice(0, 12).join('\n')
1281
- : null;
1282
- const failure = normalizeCoordinatorWorkflowRuntimeFailure(input.error);
1283
- const shouldPreserveRawError =
1284
- failure.code === 'RUN_FAILED' ||
1285
- (failure.cause !== undefined && failure.message === failure.cause);
1286
- const normalizedError = shouldPreserveRawError
1287
- ? `DynamicWorkflow runner failed: ${errorName}: ${errorMessage}${
1288
- errorStack ? `\n${errorStack}` : ''
1289
- }`
1290
- : `DynamicWorkflow runner failed: ${failure.message}`;
1291
- const headers = new Headers({
1292
- authorization: `Bearer ${executorToken}`,
1293
- 'content-type': 'application/json',
1294
- });
1295
- const bypass = input.env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim();
1296
- if (bypass) headers.set('x-vercel-protection-bypass', bypass);
1297
- const body = JSON.stringify({
1298
- action: 'append_run_events',
1299
- playId: runId,
1300
- events: [
1301
- {
1302
- type: 'run.failed',
1303
- runId,
1304
- source: 'coordinator',
1305
- occurredAt: Date.now(),
1306
- error: normalizedError,
1307
- } satisfies PlayRunLedgerEvent,
1308
- ],
1309
- });
1310
- const url = `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`;
1311
- const backoffMs = [200, 500, 1500];
1312
- let lastError: unknown = null;
1313
- for (let attempt = 0; attempt <= backoffMs.length; attempt += 1) {
1314
- const requestId = crypto.randomUUID();
1315
- try {
1316
- headers.set('x-deepline-request-id', requestId);
1317
- const response = await fetch(url, { method: 'POST', headers, body });
1318
- if (response.ok) {
1319
- if (attempt > 0) {
1320
- recordCoordinatorEvent(
1321
- 'info',
1322
- 'coordinator.runtime_api.retry',
1323
- '[coordinator.runtime_api.retry]',
1324
- {
1325
- action: 'append_run_events',
1326
- status: 'ok',
1327
- attempt,
1328
- runId,
1329
- requestId,
1330
- baseOrigin: safeOrigin(baseUrl),
1331
- },
1332
- );
1333
- }
1334
- return;
1335
- }
1336
- lastError = new Error(
1337
- `runtime API responded ${response.status}: ${(await response.text().catch(() => '')).slice(0, 400)}`,
1338
- );
1339
- if (
1340
- response.status >= 400 &&
1341
- response.status < 500 &&
1342
- response.status !== 408 &&
1343
- response.status !== 429
1344
- ) {
1345
- break;
1346
- }
1347
- } catch (error) {
1348
- lastError = error;
1349
- }
1350
- if (attempt < backoffMs.length) {
1351
- recordCoordinatorEvent(
1352
- 'warn',
1353
- 'coordinator.runtime_api.retry',
1354
- '[coordinator.runtime_api.retry]',
1355
- {
1356
- action: 'append_run_events',
1357
- status: 'retry',
1358
- attempt: attempt + 1,
1359
- retryDelayMs: backoffMs[attempt],
1360
- runId,
1361
- requestId,
1362
- baseOrigin: safeOrigin(baseUrl),
1363
- error:
1364
- lastError instanceof Error
1365
- ? lastError.message.slice(0, 300)
1366
- : String(lastError).slice(0, 300),
1367
- },
1368
- );
1369
- await new Promise((resolve) => setTimeout(resolve, backoffMs[attempt]));
1370
- }
1371
- }
1372
- recordCoordinatorEvent(
1373
- 'error',
1374
- 'coordinator.workflow_runtime_failure.mark_failed',
1375
- '[coordinator] failed to mark workflow runtime failure',
1376
- {
1377
- runId,
1378
- message:
1379
- lastError instanceof Error ? lastError.message : String(lastError),
1380
- },
1381
- );
1382
- }
1383
-
1384
- type StoredPlayArtifactPayload = {
1385
- artifact?: {
1386
- bundledCode?: string;
1387
- artifactKind?: string;
1388
- };
1389
- };
1390
-
1391
- const DYNAMIC_WORKER_COMPATIBILITY_DATE = '2026-05-01';
1392
-
1393
- async function persistWorkflowRetryState(input: {
1394
- env: CoordinatorEnv;
1395
- runId: string;
1396
- params: PlayWorkflowParams;
1397
- }): Promise<{
1398
- params?: PlayWorkflowParams;
1399
- paramsRef?: WorkflowRetryParamsRef;
1400
- }> {
1401
- const retryParams = buildWorkflowRetryParams(input.params);
1402
- const paramsBytes = jsonByteLength(retryParams);
1403
- if (paramsBytes > WORKFLOW_RETRY_PARAMS_MAX_BYTES) {
1404
- throw new Error(
1405
- `workflow retry params too large: ${paramsBytes} bytes exceeds ${WORKFLOW_RETRY_PARAMS_MAX_BYTES}. Pass large payloads as staged files or ctx.csv inputs instead of inline JSON.`,
1406
- );
1407
- }
1408
- let body: {
1409
- runId: string;
1410
- params?: PlayWorkflowParams;
1411
- paramsRef?: WorkflowRetryParamsRef;
1412
- paramsBytes: number;
1413
- ttlMs: number;
1414
- };
1415
- if (paramsBytes > WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES) {
1416
- const serialized = JSON.stringify(retryParams);
1417
- const hash = stableHash(serialized);
1418
- const storageKey = workflowRetryParamsStorageKey({
1419
- runId: input.runId,
1420
- hash,
1421
- });
1422
- const paramsRef: WorkflowRetryParamsRef = {
1423
- storageKind: 'r2',
1424
- storageKey,
1425
- bytes: paramsBytes,
1426
- hash,
1427
- expiresAt: Date.now() + WORKFLOW_RETRY_STATE_TTL_MS,
1428
- };
1429
- await input.env.PLAYS_BUCKET.put(storageKey, serialized, {
1430
- httpMetadata: { contentType: 'application/json' },
1431
- });
1432
- body = {
1433
- runId: input.runId,
1434
- paramsRef,
1435
- paramsBytes,
1436
- ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
1437
- };
1438
- await callRunScopedControl<{ ok?: unknown }>(
1439
- input.env,
1440
- input.runId,
1441
- '/run-retry-state-put',
1442
- {
1443
- method: 'POST',
1444
- body: JSON.stringify(body),
1445
- },
1446
- );
1447
- return { paramsRef };
1448
- } else {
1449
- body = {
1450
- runId: input.runId,
1451
- params: retryParams,
1452
- paramsBytes,
1453
- ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
1454
- };
1455
- }
1456
- await callRunScopedControl<{ ok?: unknown }>(
1457
- input.env,
1458
- input.runId,
1459
- '/run-retry-state-put',
1460
- {
1461
- method: 'POST',
1462
- body: JSON.stringify(body),
1463
- },
1464
- );
1465
- return { params: retryParams };
1466
- }
1467
-
1468
- async function persistWorkflowLaunchState(input: {
1469
- env: CoordinatorEnv;
1470
- runId: string;
1471
- params: PlayWorkflowParams;
1472
- sessions: PreloadedRuntimeDbSession[];
1473
- }): Promise<{
1474
- retryExpiresAt?: number;
1475
- dbSessionsExpiresAt?: number;
1476
- sessionCount?: number;
1477
- params?: PlayWorkflowParams;
1478
- paramsRef?: WorkflowRetryParamsRef;
1479
- }> {
1480
- if (input.sessions.length === 0) {
1481
- const state = await persistWorkflowRetryState({
1482
- env: input.env,
1483
- runId: input.runId,
1484
- params: input.params,
1485
- });
1486
- return state;
1487
- }
1488
- const retryParams = buildWorkflowRetryParams(input.params);
1489
- const paramsBytes = jsonByteLength(retryParams);
1490
- let body: {
1491
- runId: string;
1492
- params?: PlayWorkflowParams;
1493
- paramsRef?: WorkflowRetryParamsRef;
1494
- paramsBytes: number;
1495
- sessions: PreloadedRuntimeDbSession[];
1496
- retryTtlMs: number;
1497
- dbSessionsTtlMs: number;
1498
- };
1499
- if (paramsBytes > WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES) {
1500
- const serialized = JSON.stringify(retryParams);
1501
- const hash = stableHash(serialized);
1502
- const storageKey = workflowRetryParamsStorageKey({
1503
- runId: input.runId,
1504
- hash,
1505
- });
1506
- const paramsRef: WorkflowRetryParamsRef = {
1507
- storageKind: 'r2',
1508
- storageKey,
1509
- bytes: paramsBytes,
1510
- hash,
1511
- expiresAt: Date.now() + WORKFLOW_RETRY_STATE_TTL_MS,
1512
- };
1513
- await input.env.PLAYS_BUCKET.put(storageKey, serialized, {
1514
- httpMetadata: { contentType: 'application/json' },
1515
- });
1516
- body = {
1517
- runId: input.runId,
1518
- paramsRef,
1519
- paramsBytes,
1520
- sessions: input.sessions,
1521
- retryTtlMs: WORKFLOW_RETRY_STATE_TTL_MS,
1522
- dbSessionsTtlMs: DB_SESSION_DEFAULT_TTL_SECONDS * 1000,
1523
- };
1524
- } else {
1525
- body = {
1526
- runId: input.runId,
1527
- params: retryParams,
1528
- paramsBytes,
1529
- sessions: input.sessions,
1530
- retryTtlMs: WORKFLOW_RETRY_STATE_TTL_MS,
1531
- dbSessionsTtlMs: DB_SESSION_DEFAULT_TTL_SECONDS * 1000,
1532
- };
1533
- }
1534
- const response = await callRunScopedControl<{
1535
- ok?: unknown;
1536
- retryExpiresAt?: unknown;
1537
- dbSessionsExpiresAt?: unknown;
1538
- sessionCount?: unknown;
1539
- }>(input.env, input.runId, '/run-launch-state-put', {
1540
- method: 'POST',
1541
- body: JSON.stringify(body),
1542
- });
1543
- return {
1544
- retryExpiresAt:
1545
- typeof response.retryExpiresAt === 'number'
1546
- ? response.retryExpiresAt
1547
- : undefined,
1548
- dbSessionsExpiresAt:
1549
- typeof response.dbSessionsExpiresAt === 'number'
1550
- ? response.dbSessionsExpiresAt
1551
- : undefined,
1552
- sessionCount:
1553
- typeof response.sessionCount === 'number'
1554
- ? response.sessionCount
1555
- : undefined,
1556
- params: body.params,
1557
- paramsRef: body.paramsRef,
1558
- };
1559
- }
1560
-
1561
- function parseWorkflowRetryParamsRef(
1562
- paramsRef: unknown,
1563
- ): WorkflowRetryParamsRef | null {
1564
- if (!isRecord(paramsRef)) {
1565
- return null;
1566
- }
1567
- const storageKind = paramsRef.storageKind;
1568
- const storageKey = paramsRef.storageKey;
1569
- const expectedBytes = paramsRef.bytes;
1570
- const expectedHash = paramsRef.hash;
1571
- const expiresAt = paramsRef.expiresAt;
1572
- if (
1573
- storageKind !== 'r2' ||
1574
- typeof storageKey !== 'string' ||
1575
- !storageKey.startsWith('plays/workflow-retry-params/') ||
1576
- typeof expectedBytes !== 'number' ||
1577
- !Number.isFinite(expectedBytes) ||
1578
- typeof expectedHash !== 'string' ||
1579
- !expectedHash ||
1580
- typeof expiresAt !== 'number' ||
1581
- !Number.isFinite(expiresAt)
1582
- ) {
1583
- throw new Error('Invalid workflow retry params reference.');
1584
- }
1585
- return {
1586
- storageKind,
1587
- storageKey,
1588
- bytes: expectedBytes,
1589
- hash: expectedHash,
1590
- expiresAt,
1591
- };
1592
- }
1593
-
1594
- async function hydrateWorkflowRetryParams(input: {
1595
- env: CoordinatorEnv;
1596
- params: unknown;
1597
- paramsRef: unknown;
1598
- }): Promise<PlayWorkflowParams | null> {
1599
- if (isRecord(input.params)) {
1600
- return input.params as PlayWorkflowParams;
1601
- }
1602
- const paramsRef = parseWorkflowRetryParamsRef(input.paramsRef);
1603
- if (!paramsRef) return null;
1604
- const storageKey = paramsRef.storageKey;
1605
- const expectedBytes = paramsRef.bytes;
1606
- const expectedHash = paramsRef.hash;
1607
- if (paramsRef.expiresAt <= Date.now()) {
1608
- throw new Error('Workflow retry params reference expired.');
1609
- }
1610
- const object = await input.env.PLAYS_BUCKET.get(storageKey);
1611
- if (!object) {
1612
- throw new Error(`Workflow retry params missing from R2: ${storageKey}`);
1613
- }
1614
- const text = await object.text();
1615
- const actualBytes = new TextEncoder().encode(text).length;
1616
- if (actualBytes !== expectedBytes) {
1617
- throw new Error(
1618
- `Workflow retry params byte length mismatch: expected ${expectedBytes}, got ${actualBytes}.`,
1619
- );
1620
- }
1621
- const actualHash = stableHash(text);
1622
- if (actualHash !== expectedHash) {
1623
- throw new Error('Workflow retry params hash mismatch.');
1624
- }
1625
- const parsed = JSON.parse(text) as unknown;
1626
- return isRecord(parsed) ? (parsed as PlayWorkflowParams) : null;
1627
- }
1628
-
1629
- function workflowRetryStatePersistenceErrorResponse(input: {
1630
- runId: string;
1631
- error: unknown;
1632
- }): Response {
1633
- const message =
1634
- input.error instanceof Error ? input.error.message : String(input.error);
1635
- return Response.json(
1636
- {
1637
- error: {
1638
- code: 'WORKFLOW_RETRY_STATE_PERSISTENCE_FAILED',
1639
- message:
1640
- 'Failed to persist workflow retry state before dispatching the play run.',
1641
- phase: 'coordinator_retry_state_persistence',
1642
- runId: input.runId,
1643
- cause: message,
1644
- },
1645
- },
1646
- { status: 503 },
1647
- );
1648
- }
1649
-
1650
- async function claimWorkflowPlatformRetry(input: {
1651
- env: CoordinatorEnv;
1652
- runId: string;
1653
- }): Promise<{
1654
- claimed: boolean;
1655
- attempts: number;
1656
- params: PlayWorkflowParams | null;
1657
- paramsRef?: WorkflowRetryParamsRef;
1658
- }> {
1659
- const body = await callRunScopedControl<{
1660
- claimed?: unknown;
1661
- attempts?: unknown;
1662
- params?: unknown;
1663
- paramsRef?: unknown;
1664
- }>(input.env, input.runId, '/run-retry-claim', {
1665
- method: 'POST',
1666
- body: JSON.stringify({
1667
- runId: input.runId,
1668
- maxAttempts: PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
1669
- }),
1670
- });
1671
- const params = await hydrateWorkflowRetryParams({
1672
- env: input.env,
1673
- params: body.params,
1674
- paramsRef: body.paramsRef,
1675
- });
1676
- const paramsRef = parseWorkflowRetryParamsRef(body.paramsRef) ?? undefined;
1677
- return {
1678
- claimed: body.claimed === true,
1679
- attempts: typeof body.attempts === 'number' ? body.attempts : 0,
1680
- params,
1681
- paramsRef,
1682
- };
1683
- }
1684
-
1685
- function workflowRetryInstanceId(runId: string, attempt: number): string {
1686
- const suffix = `retry-${attempt}-${stableHash(`${runId}:${attempt}`).slice(0, 8)}`;
1687
- const base = workflowInstanceId(runId);
1688
- const maxBaseLength = Math.max(1, 100 - suffix.length - 1);
1689
- return `${base.slice(0, maxBaseLength)}-${suffix}`;
1690
- }
1691
-
1692
- function normalizeWorkflowRunAttempt(value: unknown): number {
1693
- return typeof value === 'number' && Number.isFinite(value)
1694
- ? Math.max(0, Math.floor(value))
1695
- : 0;
1696
- }
1697
-
1698
- async function restartWorkflowAfterPlatformReset(input: {
1699
- env: CoordinatorEnv;
1700
- ctx?: ExecutionContext;
1701
- runId: string;
1702
- oldInstance: WorkflowInstance;
1703
- error: string;
1704
- status: InstanceStatus;
1705
- }): Promise<{
1706
- retried: boolean;
1707
- result: Record<string, unknown> | null;
1708
- }> {
1709
- const decision = decideWorkflowPlatformRetry({
1710
- workflowStatus: String(input.status.status ?? ''),
1711
- error: input.error,
1712
- retryAttempts: 0,
1713
- });
1714
- if (decision.action !== 'retry') {
1715
- return { retried: false, result: null };
1716
- }
1717
- const claim = await claimWorkflowPlatformRetry({
1718
- env: input.env,
1719
- runId: input.runId,
1720
- }).catch((error) => {
1721
- recordCoordinatorEvent(
1722
- 'warn',
1723
- 'coordinator.workflow_platform_retry.claim_failed',
1724
- '[coordinator] workflow platform retry claim failed',
1725
- {
1726
- runId: input.runId,
1727
- error: error instanceof Error ? error.message : String(error),
1728
- },
1729
- );
1730
- return null;
1731
- });
1732
- if (!claim?.claimed || !claim.params) {
1733
- return { retried: false, result: null };
1734
- }
1735
- const retryInstanceId = workflowRetryInstanceId(input.runId, claim.attempts);
1736
- const retryStartedAt = Date.now();
1737
- let retryInstance: WorkflowInstance | null = null;
1738
- const retryParams = { ...claim.params, runAttempt: claim.attempts };
1739
- try {
1740
- retryInstance = await createDynamicWorkflowInstance({
1741
- env: input.env,
1742
- id: retryInstanceId,
1743
- params: workflowCreateParams(retryParams, {
1744
- params: retryParams,
1745
- paramsRef: claim.paramsRef,
1746
- }),
1747
- metadataParams: retryParams,
1748
- });
1749
- await recordWorkflowInstanceId({
1750
- env: input.env,
1751
- runId: input.runId,
1752
- instanceId: retryInstance.id,
1753
- });
1754
- input.ctx?.waitUntil(input.oldInstance.terminate().catch(() => undefined));
1755
- recordCoordinatorPerfTraceBuffered(input.env, input.ctx, {
1756
- runId: input.runId,
1757
- phase: 'coordinator.platform_deploy_retry',
1758
- ms: Date.now() - retryStartedAt,
1759
- graphHash: claim.params.graphHash ?? null,
1760
- extra: {
1761
- retryAttempt: claim.attempts,
1762
- retryInstanceId: retryInstance.id,
1763
- reason: decision.reason,
1764
- },
1765
- });
1766
- return {
1767
- retried: true,
1768
- result: {
1769
- runId: input.runId,
1770
- playName: claim.params.playName,
1771
- status: 'running',
1772
- result: null,
1773
- error: null,
1774
- retry: {
1775
- reason: decision.reason,
1776
- attempt: claim.attempts,
1777
- message: decision.message,
1778
- },
1779
- },
1780
- };
1781
- } finally {
1782
- disposeRpcStub(retryInstance);
1783
- }
1784
- }
1785
-
1786
- export class CoordinatorControl extends WorkerEntrypoint<
1787
- CoordinatorEnv,
1788
- undefined
1789
- > {
1790
- async signal(
1791
- runId: string,
1792
- body: Record<string, unknown>,
1793
- ): Promise<Record<string, unknown>> {
1794
- const response = await handleWorkflowRoute({
1795
- runId,
1796
- action: 'signal',
1797
- request: new Request(
1798
- `https://deepline.coordinator.internal/workflow/${encodeURIComponent(
1799
- runId,
1800
- )}/signal`,
1801
- {
1802
- method: 'POST',
1803
- headers: {
1804
- 'content-type': 'application/json',
1805
- 'x-deepline-request-id': crypto.randomUUID(),
1806
- },
1807
- body: JSON.stringify(body),
1808
- },
1809
- ),
1810
- env: this.env,
1811
- });
1812
- const text = await response.text().catch(() => '');
1813
- let parsed: Record<string, unknown> = {};
1814
- try {
1815
- parsed = text ? (JSON.parse(text) as Record<string, unknown>) : {};
1816
- } catch {
1817
- parsed = { error: text };
1818
- }
1819
- if (!response.ok) {
1820
- const error = isRecord(parsed.error) ? parsed.error : null;
1821
- const message =
1822
- (typeof error?.message === 'string' && error.message.trim()) ||
1823
- (typeof parsed.error === 'string' && parsed.error.trim()) ||
1824
- text.slice(0, 800) ||
1825
- `Coordinator signal failed with ${response.status}.`;
1826
- throw new Error(message);
1827
- }
1828
- return parsed;
1829
- }
1830
-
1831
- async recordPerfTrace(
1832
- runId: string,
1833
- payload: CoordinatorPerfTracePayload,
1834
- ): Promise<void> {
1835
- if (!runId || payload.runId !== runId) {
1836
- throw new Error('Trace runId mismatch.');
1837
- }
1838
- await appendCoordinatorPerfTrace(this.env, payload);
1839
- }
1840
-
1841
- async recordRunEvent(
1842
- runId: string,
1843
- event: CoordinatorRunEvent,
1844
- ): Promise<void> {
1845
- if (!runId || event.runId !== runId) {
1846
- throw new Error('Run event runId mismatch.');
1847
- }
1848
- await appendCoordinatorRunEvent(this.env, event);
1849
- }
1850
-
1851
- async readTerminalState(
1852
- runId: string,
1853
- ): Promise<CoordinatorTerminalState | null> {
1854
- if (!runId) {
1855
- throw new Error('runId is required.');
1856
- }
1857
- return await readCoordinatorTerminalState(this.env, runId);
1858
- }
1859
-
1860
- /**
1861
- * Distributed Rate State Backend acquire: lease up to `requested` request-
1862
- * window permits for `bucketId` (`<orgId>:<provider>`) from the per-bucket
1863
- * rate-state Durable Object. See CoordinatorRateStateBackend + dedup-do.ts.
1864
- */
1865
- async rateAcquire(input: {
1866
- bucketId: string;
1867
- rules: Array<{
1868
- ruleId: string;
1869
- requestsPerWindow: number;
1870
- windowMs: number;
1871
- maxConcurrency: number | null;
1872
- }>;
1873
- requested: number;
1874
- }): Promise<{ granted: number; waitMs: number }> {
1875
- if (!input.bucketId || !input.bucketId.trim()) {
1876
- throw new Error('bucketId is required.');
1877
- }
1878
- return await callRateBucketControl<{ granted: number; waitMs: number }>(
1879
- this.env,
1880
- input.bucketId,
1881
- '/rate-acquire',
1882
- input,
1883
- );
1884
- }
1885
-
1886
- async ratePenalize(input: {
1887
- bucketId: string;
1888
- cooldownMs: number;
1889
- }): Promise<void> {
1890
- if (!input.bucketId || !input.bucketId.trim()) {
1891
- throw new Error('bucketId is required.');
1892
- }
1893
- await callRateBucketControl<{ ok?: unknown }>(
1894
- this.env,
1895
- input.bucketId,
1896
- '/rate-penalize',
1897
- input,
1898
- );
1899
- }
1900
- }
1901
-
1902
- /**
1903
- * Custom DynamicWorkflow class — replaces createDynamicWorkflowEntrypoint
1904
- * so we can wrap the framework's runner.run(innerEvent, step) RPC in a
1905
- * try/catch and surface the underlying error. The library's helper hides
1906
- * that call inside dispatchWorkflow which we now invoke directly with
1907
- * extra instrumentation around the inner runner call.
1908
- */
1909
- export class DynamicWorkflow extends WorkflowEntrypoint<
1910
- CoordinatorEnv,
1911
- Record<string, unknown>
1912
- > {
1913
- async run(event: unknown, step: unknown): Promise<unknown> {
1914
- const trace: CoordinatorPerfTraceSink = (traceEvent) =>
1915
- recordCoordinatorPerfTraceBuffered(this.env, this.ctx, traceEvent);
1916
- const workflowEvent = event as {
1917
- payload?: unknown;
1918
- timestamp?: Date;
1919
- instanceId?: string;
1920
- };
1921
- const entryTrace = readWorkflowTraceContext(event);
1922
- trace({
1923
- runId: entryTrace.runId,
1924
- phase: 'coordinator.workflow_run_entry',
1925
- ms: 0,
1926
- graphHash: entryTrace.graphHash,
1927
- extra: {
1928
- instanceId: entryTrace.instanceId,
1929
- },
1930
- });
1931
- if (entryTrace.runId) {
1932
- this.ctx.waitUntil(
1933
- appendCoordinatorRunEvent(this.env, {
1934
- runId: entryTrace.runId,
1935
- type: 'status',
1936
- status: 'running',
1937
- ts: Date.now(),
1938
- }).catch((error) => {
1939
- recordCoordinatorEvent(
1940
- 'warn',
1941
- 'coordinator.workflow_entry.status_event_failed',
1942
- '[coordinator] workflow entry status event failed',
1943
- {
1944
- runId: entryTrace.runId,
1945
- error: error instanceof Error ? error.message : String(error),
1946
- },
1947
- );
1948
- }),
1949
- );
1950
- }
1951
- if (entryTrace.submittedAt !== null) {
1952
- trace({
1953
- runId: entryTrace.runId,
1954
- phase: 'coordinator.workflow_start_gap',
1955
- ms: Math.max(0, Date.now() - entryTrace.submittedAt),
1956
- graphHash: entryTrace.graphHash,
1957
- extra: {
1958
- instanceId: entryTrace.instanceId,
1959
- },
1960
- });
1961
- }
1962
- let dispatchedEvent = event;
1963
- dispatchedEvent = await hydrateWorkflowLaunchParams({
1964
- env: this.env,
1965
- event: dispatchedEvent,
1966
- trace,
1967
- });
1968
- try {
1969
- dispatchedEvent = await hydrateWorkflowDbSessions({
1970
- env: this.env,
1971
- event: dispatchedEvent,
1972
- trace,
1973
- });
1974
- } catch (hydrateError) {
1975
- // The workflow re-entered but cannot read its externalized DB sessions.
1976
- // A 410 (expired retention window) is known-permanent: the sessions are
1977
- // stored once at submit and never re-minted here, so the workflow can
1978
- // only keep failing. Publish a clean run.failed instead of letting the
1979
- // raw transport error escape uncaught — this call sits OUTSIDE the
1980
- // dispatchWorkflow try/catch that authors run.failed.
1981
- //
1982
- // A 404 that survived the read-retry ladder (`workflow db session
1983
- // lookup failed 404`) rethrows WITHOUT marking, like every other error:
1984
- // sessions are persisted (awaited) before the instance is created, so a
1985
- // 404 here is anomalous — most plausibly a framework re-entry against an
1986
- // ALREADY-TERMINAL run whose DO state the post-finish alarm cleaned up.
1987
- // Authoring run.failed there could flip a completed (billed + durable)
1988
- // run to failed via newest-terminal-wins. Only the 410 is safe to mark.
1989
- const hydrateMessage =
1990
- hydrateError instanceof Error
1991
- ? hydrateError.message
1992
- : String(hydrateError);
1993
- if (hydrateMessage.includes('workflow db session lookup failed 410')) {
1994
- await markWorkflowRuntimeFailure({
1995
- env: this.env,
1996
- event: dispatchedEvent,
1997
- error: hydrateError,
1998
- }).catch((markError) => {
1999
- recordCoordinatorEvent(
2000
- 'error',
2001
- 'coordinator.db_session_expired.failure_forward_failed',
2002
- '[coordinator] failed to forward expired-db-session run failure',
2003
- {
2004
- runId: entryTrace.runId,
2005
- message:
2006
- markError instanceof Error
2007
- ? markError.message
2008
- : String(markError),
2009
- },
2010
- );
2011
- });
2012
- if (entryTrace.runId) {
2013
- await writeCoordinatorTerminalState(this.env, {
2014
- runId: entryTrace.runId,
2015
- status: 'failed',
2016
- error:
2017
- normalizeCoordinatorWorkflowRuntimeFailure(hydrateError).message,
2018
- }).catch(() => undefined);
2019
- }
2020
- }
2021
- throw hydrateError;
2022
- }
2023
- const dispatchTrace = readWorkflowTraceContext(dispatchedEvent);
2024
- trace({
2025
- runId: dispatchTrace.runId,
2026
- phase: 'coordinator.dispatch_workflow_entry',
2027
- ms: 0,
2028
- graphHash: dispatchTrace.graphHash,
2029
- extra: {
2030
- instanceId: dispatchTrace.instanceId,
2031
- },
2032
- });
2033
-
2034
- const dispatchWorkflowStartedAt = Date.now();
2035
- try {
2036
- return await dispatchWorkflow(
2037
- { env: this.env, ctx: this.ctx },
2038
- dispatchedEvent as Parameters<typeof dispatchWorkflow>[1],
2039
- step as Parameters<typeof dispatchWorkflow>[2],
2040
- async ({ metadata, env }) => {
2041
- const graphHash = readMetadataString(metadata, 'graphHash');
2042
- const artifactStorageKey = readMetadataString(
2043
- metadata,
2044
- 'artifactStorageKey',
2045
- );
2046
- const runIdForTrace =
2047
- typeof (metadata as Record<string, unknown>).runId === 'string'
2048
- ? ((metadata as Record<string, unknown>).runId as string)
2049
- : graphHash;
2050
- const loaderStartedAt = Date.now();
2051
- trace({
2052
- runId: runIdForTrace,
2053
- phase: 'coordinator.loader_callback_entry',
2054
- ms: 0,
2055
- graphHash,
2056
- extra: { artifactStorageKey },
2057
- });
2058
- const stub = loadDynamicPlayWorkerSync(
2059
- env,
2060
- {
2061
- runId: runIdForTrace,
2062
- graphHash,
2063
- artifactStorageKey,
2064
- artifactHash:
2065
- typeof metadata.artifactHash === 'string'
2066
- ? metadata.artifactHash
2067
- : null,
2068
- dynamicWorkerCode:
2069
- typeof metadata.dynamicWorkerCode === 'string'
2070
- ? metadata.dynamicWorkerCode
2071
- : null,
2072
- dynamicWorkerCodeHash:
2073
- typeof metadata.dynamicWorkerCodeHash === 'string'
2074
- ? metadata.dynamicWorkerCodeHash
2075
- : null,
2076
- integrationMode: normalizeIntegrationMode(
2077
- metadata.integrationMode,
2078
- ),
2079
- packagedFiles: normalizePackagedFiles(metadata.packagedFiles),
2080
- },
2081
- trace,
2082
- );
2083
- const entrypointStartedAt = Date.now();
2084
- const entrypoint = stub.getEntrypoint(
2085
- 'TenantWorkflow',
2086
- ) as unknown as WorkflowRunner;
2087
- trace({
2088
- runId: runIdForTrace,
2089
- phase: 'coordinator.loader_get_entrypoint',
2090
- ms: Date.now() - entrypointStartedAt,
2091
- graphHash,
2092
- });
2093
- trace({
2094
- runId: runIdForTrace,
2095
- phase: 'coordinator.loader_compile',
2096
- ms: Date.now() - loaderStartedAt,
2097
- graphHash,
2098
- });
2099
- // Wrap the entrypoint so its run() failure surfaces here rather
2100
- // than disappearing into the framework's silent rpcMethod=run
2101
- // exception path.
2102
- return {
2103
- run: async (innerEvent: unknown, innerStep: unknown) => {
2104
- const innerStartedAt = Date.now();
2105
- trace({
2106
- runId: runIdForTrace,
2107
- phase: 'coordinator.runner_run_start',
2108
- ms: 0,
2109
- graphHash,
2110
- });
2111
- try {
2112
- const result = await (
2113
- entrypoint as unknown as {
2114
- run(e: unknown, s: unknown): Promise<unknown>;
2115
- }
2116
- ).run(innerEvent, innerStep);
2117
- const output = isRecord(result) ? result : null;
2118
- const terminalResult = output?.result ?? result;
2119
- const terminalStartedAt = Date.now();
2120
- try {
2121
- await writeCoordinatorTerminalState(env, {
2122
- runId: runIdForTrace,
2123
- status: 'completed',
2124
- result: terminalResult,
2125
- totalRows: output?.totalRows ?? output?.outputRows ?? null,
2126
- durationMs: output?.durationMs ?? null,
2127
- playName:
2128
- typeof output?.playName === 'string'
2129
- ? output.playName
2130
- : null,
2131
- liveLogs: sanitizeLiveLogLines(output?.liveLogs),
2132
- liveNodeProgress: output?.liveNodeProgress ?? null,
2133
- });
2134
- trace({
2135
- runId: runIdForTrace,
2136
- phase: 'coordinator.terminal_state_write',
2137
- ms: Date.now() - terminalStartedAt,
2138
- graphHash,
2139
- extra: { status: 'completed' },
2140
- });
2141
- } catch (terminalError) {
2142
- console.warn(
2143
- '[coordinator] completed terminal cache write failed; preserving completed run',
2144
- {
2145
- graphHash,
2146
- runId: runIdForTrace,
2147
- error:
2148
- terminalError instanceof Error
2149
- ? terminalError.message
2150
- : String(terminalError),
2151
- },
2152
- );
2153
- }
2154
- trace({
2155
- runId: runIdForTrace,
2156
- phase: 'coordinator.runner_run',
2157
- ms: Date.now() - innerStartedAt,
2158
- graphHash,
2159
- });
2160
- return result;
2161
- } catch (innerError) {
2162
- const failure =
2163
- normalizeCoordinatorWorkflowRuntimeFailure(innerError);
2164
- recordCoordinatorEvent(
2165
- 'error',
2166
- 'coordinator.dynamic_workflow.runner_failed',
2167
- '[coordinator] DynamicWorkflow runner.run threw',
2168
- {
2169
- graphHash,
2170
- message:
2171
- innerError instanceof Error
2172
- ? innerError.message
2173
- : String(innerError),
2174
- name: innerError instanceof Error ? innerError.name : null,
2175
- stack:
2176
- innerError instanceof Error &&
2177
- typeof innerError.stack === 'string'
2178
- ? innerError.stack.split('\n').slice(0, 12).join('\n')
2179
- : null,
2180
- },
2181
- );
2182
- if (
2183
- shouldPublishWorkflowRuntimeFailure({
2184
- error: innerError,
2185
- retryAttempts: 0,
2186
- })
2187
- ) {
2188
- await markWorkflowRuntimeFailure({
2189
- env,
2190
- event: innerEvent,
2191
- error: innerError,
2192
- }).catch((markError) => {
2193
- recordCoordinatorEvent(
2194
- 'error',
2195
- 'coordinator.dynamic_workflow.failure_forward_failed',
2196
- '[coordinator] failed to forward DynamicWorkflow runner error',
2197
- {
2198
- graphHash,
2199
- message:
2200
- markError instanceof Error
2201
- ? markError.message
2202
- : String(markError),
2203
- },
2204
- );
2205
- });
2206
- await writeCoordinatorTerminalState(env, {
2207
- runId: runIdForTrace,
2208
- status: 'failed',
2209
- error: failure.message,
2210
- }).catch(() => undefined);
2211
- } else {
2212
- recordCoordinatorEvent(
2213
- 'warn',
2214
- 'coordinator.dynamic_workflow.platform_reset_retrying',
2215
- '[coordinator] DynamicWorkflow platform reset will be retried; not publishing durable run.failed',
2216
- {
2217
- graphHash,
2218
- runId: runIdForTrace,
2219
- errorCode: failure.code,
2220
- retryLimit: PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
2221
- },
2222
- );
2223
- }
2224
- throw innerError;
2225
- }
2226
- },
2227
- };
2228
- },
2229
- );
2230
- } finally {
2231
- trace({
2232
- runId: dispatchTrace.runId,
2233
- phase: 'coordinator.dispatch_workflow_total',
2234
- ms: Date.now() - dispatchWorkflowStartedAt,
2235
- graphHash: dispatchTrace.graphHash,
2236
- extra: {
2237
- instanceId: dispatchTrace.instanceId,
2238
- },
2239
- });
2240
- }
2241
- }
2242
- }
2243
-
2244
- const coordinatorEntrypoint = {
2245
- /**
2246
- * HTTP entrypoint for the Vercel app to dispatch into. Routes:
2247
- * POST /workflow/{runId}/submit → PLAY_WORKFLOW.create({ id, params })
2248
- * GET /workflow/{runId}/tail → ordered live run events after a cursor
2249
- * POST /workflow/{runId}/cancel → Workflow instance terminate
2250
- * POST /workflow/{runId}/signal → integration_event
2251
- * GET /workflow/{runId}/result → terminal envelope
2252
- * GET /health → liveness
2253
- */
2254
- async fetch(
2255
- request: Request,
2256
- env: CoordinatorEnv,
2257
- ctx?: ExecutionContext,
2258
- ): Promise<Response> {
2259
- // Top-level guard: no coordinator route may let an exception escape to
2260
- // Cloudflare's default "Worker threw exception" HTML error page. Each
2261
- // route group below additionally wraps itself for precise phase/runId
2262
- // context; this outer catch is the backstop for URL parsing, route
2263
- // matching, and any future route added without its own envelope.
2264
- try {
2265
- return await coordinatorRouteFetch(request, env, ctx);
2266
- } catch (error) {
2267
- return coordinatorRouteErrorResponse({
2268
- logTag: '[coordinator.fetch.error]',
2269
- code: 'COORDINATOR_ROUTE_FAILED',
2270
- phase: 'coordinator.fetch',
2271
- runId: null,
2272
- error,
2273
- });
2274
- }
2275
- },
2276
- async tail(events: unknown[], env: CoordinatorEnv): Promise<void> {
2277
- await flushTailRunLogs(events, env);
2278
- },
2279
- async scheduled(
2280
- _controller: unknown,
2281
- _env: CoordinatorEnv,
2282
- _ctx?: ExecutionContext,
2283
- ): Promise<void> {},
2284
- };
2285
-
2286
- export default coordinatorEntrypoint;
2287
-
2288
- /**
2289
- * Route dispatcher for the coordinator fetch handler. Extracted as a standalone
2290
- * function so the outer {@link coordinatorEntrypoint.fetch} try/catch backstop
2291
- * is not coupled to `this` binding. Every route group wraps itself with a JSON
2292
- * envelope; the outer catch covers URL parsing / route matching escapes.
2293
- */
2294
- async function coordinatorRouteFetch(
2295
- request: Request,
2296
- env: CoordinatorEnv,
2297
- ctx?: ExecutionContext,
2298
- ): Promise<Response> {
2299
- const url = new URL(request.url);
2300
- if (url.pathname === '/health') {
2301
- const headers = new Headers();
2302
- headers.set(
2303
- 'x-deepline-runtime-deploy-version',
2304
- env.CF_VERSION_METADATA?.id ??
2305
- env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
2306
- '',
2307
- );
2308
- if (env.DEEPLINE_COORDINATOR_DEPLOY_MARKER) {
2309
- headers.set(
2310
- 'x-deepline-deploy-marker',
2311
- env.DEEPLINE_COORDINATOR_DEPLOY_MARKER,
2312
- );
2313
- }
2314
- return new Response('ok', { status: 200, headers });
2315
- }
2316
- if (url.pathname === '/internal-token/probe') {
2317
- const authError = authorizeCoordinatorControlRequest({ request, env });
2318
- if (authError) return authError;
2319
- return Response.json({
2320
- ok: true,
2321
- deployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
2322
- runtimeDeployVersion:
2323
- env.CF_VERSION_METADATA?.id ??
2324
- env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
2325
- null,
2326
- });
2327
- }
2328
- if (url.pathname === '/warmup/submit') {
2329
- const authError = authorizeCoordinatorControlRequest({ request, env });
2330
- if (authError) return authError;
2331
- try {
2332
- return await handleCoordinatorWarmup(request, env, ctx);
2333
- } catch (error) {
2334
- return coordinatorRouteErrorResponse({
2335
- logTag: '[coordinator.warmup.error]',
2336
- code: 'COORDINATOR_WARMUP_FAILED',
2337
- phase: 'coordinator.warmup',
2338
- runId: null,
2339
- error,
2340
- });
2341
- }
2342
- }
2343
- if (url.pathname === '/tail-log-token/probe') {
2344
- const authError = authorizeCoordinatorControlRequest({ request, env });
2345
- if (authError) return authError;
2346
- const expectedTailLogToken = env.DEEPLINE_TAIL_LOG_TOKEN?.trim();
2347
- if (!expectedTailLogToken) {
2348
- return Response.json(
2349
- { ok: false, error: 'tail log token is not configured' },
2350
- { status: 503 },
2351
- );
2352
- }
2353
- const actualTailLogToken =
2354
- request.headers.get('x-deepline-tail-log-token')?.trim() ?? '';
2355
- if (actualTailLogToken !== expectedTailLogToken) {
2356
- return Response.json(
2357
- { ok: false, error: 'tail log token mismatch' },
2358
- { status: 401 },
2359
- );
2360
- }
2361
- return Response.json({
2362
- ok: true,
2363
- deployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
2364
- runtimeDeployVersion:
2365
- env.CF_VERSION_METADATA?.id ??
2366
- env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
2367
- null,
2368
- });
2369
- }
2370
- if (url.pathname === '/harness/probe') {
2371
- const authError = authorizeCoordinatorControlRequest({ request, env });
2372
- if (authError) return authError;
2373
- try {
2374
- const harness = await env.HARNESS.ping();
2375
- return Response.json({
2376
- ok: true,
2377
- coordinatorDeployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
2378
- coordinatorRuntimeDeployVersion:
2379
- env.CF_VERSION_METADATA?.id ??
2380
- env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
2381
- null,
2382
- harnessDeployMarker: harness.deployMarker,
2383
- harnessRuntimeDeployVersion: harness.runtimeDeployVersion,
2384
- harnessTs: harness.ts,
2385
- });
2386
- } catch (error) {
2387
- return coordinatorRouteErrorResponse({
2388
- logTag: '[coordinator.harness_probe.error]',
2389
- code: 'COORDINATOR_HARNESS_PROBE_FAILED',
2390
- phase: 'coordinator.harness_probe',
2391
- runId: null,
2392
- error,
2393
- });
2394
- }
2395
- }
2396
- if (url.pathname === '/staged-files/put') {
2397
- const authError = authorizeCoordinatorControlRequest({ request, env });
2398
- if (authError) return authError;
2399
- try {
2400
- return await handleStagedFilePut(request, env);
2401
- } catch (error) {
2402
- return coordinatorRouteErrorResponse({
2403
- logTag: '[coordinator.staged_file_put.error]',
2404
- code: 'COORDINATOR_STAGED_FILE_PUT_FAILED',
2405
- phase: 'coordinator.staged_files_put',
2406
- runId: null,
2407
- error,
2408
- });
2409
- }
2410
- }
2411
- if (url.pathname === '/rate-acquire') {
2412
- const authError = authorizeCoordinatorControlRequest({ request, env });
2413
- if (authError) return authError;
2414
- try {
2415
- const body = (await request.json().catch(() => null)) as {
2416
- bucketId?: unknown;
2417
- rules?: unknown;
2418
- requested?: unknown;
2419
- } | null;
2420
- const bucketId = typeof body?.bucketId === 'string' ? body.bucketId : '';
2421
- if (!bucketId.trim()) {
2422
- return Response.json(
2423
- { error: 'bucketId is required.' },
2424
- { status: 400 },
2425
- );
2426
- }
2427
- const result = await callRateBucketControl<{
2428
- granted: number;
2429
- waitMs: number;
2430
- }>(env, bucketId, '/rate-acquire', body);
2431
- return Response.json(result);
2432
- } catch (error) {
2433
- return coordinatorRouteErrorResponse({
2434
- logTag: '[coordinator.rate_acquire.error]',
2435
- code: 'COORDINATOR_RATE_ROUTE_FAILED',
2436
- phase: 'coordinator.rate.acquire',
2437
- runId: null,
2438
- error,
2439
- });
2440
- }
2441
- }
2442
- if (url.pathname === '/rate-penalize') {
2443
- const authError = authorizeCoordinatorControlRequest({ request, env });
2444
- if (authError) return authError;
2445
- try {
2446
- const body = (await request.json().catch(() => null)) as {
2447
- bucketId?: unknown;
2448
- cooldownMs?: unknown;
2449
- } | null;
2450
- const bucketId = typeof body?.bucketId === 'string' ? body.bucketId : '';
2451
- if (!bucketId.trim()) {
2452
- return Response.json(
2453
- { error: 'bucketId is required.' },
2454
- { status: 400 },
2455
- );
2456
- }
2457
- await callRateBucketControl<{ ok?: unknown }>(
2458
- env,
2459
- bucketId,
2460
- '/rate-penalize',
2461
- body,
2462
- );
2463
- return Response.json({ ok: true });
2464
- } catch (error) {
2465
- return coordinatorRouteErrorResponse({
2466
- logTag: '[coordinator.rate_penalize.error]',
2467
- code: 'COORDINATOR_RATE_ROUTE_FAILED',
2468
- phase: 'coordinator.rate.penalize',
2469
- runId: null,
2470
- error,
2471
- });
2472
- }
2473
- }
2474
- // Workflow routes: /workflow/{runId}/{action}
2475
- const wfMatch = url.pathname.match(/^\/workflow\/([^/]+)(?:\/(.+))?$/);
2476
- if (wfMatch) {
2477
- const runId = decodeURIComponent(wfMatch[1]!);
2478
- const action = wfMatch[2] ?? '';
2479
- const authError = authorizeCoordinatorControlRequest({
2480
- request,
2481
- env,
2482
- runId,
2483
- requireRunScope: isWorkflowMutatingAction(action),
2484
- });
2485
- if (authError) return authError;
2486
- try {
2487
- return await handleWorkflowRoute({ runId, action, request, env, ctx });
2488
- } catch (error) {
2489
- return coordinatorWorkflowRouteErrorResponse({
2490
- runId,
2491
- action,
2492
- error,
2493
- });
2494
- }
2495
- }
2496
-
2497
- // Dedup routes: /dedup/{runId}/{action}
2498
- const dedupMatch = url.pathname.match(/^\/dedup\/([^/]+)(?:\/(.+))?$/);
2499
- if (dedupMatch) {
2500
- const runId = decodeURIComponent(dedupMatch[1]!);
2501
- const action = dedupMatch[2] ?? '';
2502
- const authError = authorizeCoordinatorControlRequest({
2503
- request,
2504
- env,
2505
- runId,
2506
- requireRunScope: request.method !== 'GET' && request.method !== 'HEAD',
2507
- });
2508
- if (authError) return authError;
2509
- try {
2510
- const doId = env.PLAY_DEDUP.idFromName(runId);
2511
- const stub = env.PLAY_DEDUP.get(doId);
2512
- const internalUrl = `https://internal/${action}`;
2513
- return await stub.fetch(internalUrl, {
2514
- method: request.method,
2515
- headers: request.headers,
2516
- body:
2517
- request.method === 'GET' || request.method === 'HEAD'
2518
- ? undefined
2519
- : request.body,
2520
- });
2521
- } catch (error) {
2522
- return coordinatorRouteErrorResponse({
2523
- logTag: '[coordinator.dedup_route.error]',
2524
- code: 'COORDINATOR_DEDUP_ROUTE_FAILED',
2525
- phase: `coordinator.dedup.${action || 'unknown'}`,
2526
- runId,
2527
- error,
2528
- });
2529
- }
2530
- }
2531
-
2532
- return new Response('not found', { status: 404 });
2533
- }
2534
-
2535
- const RUN_LOG_PREFIX_RE = /\[deepline-run:([^\]]+)\]\s*(.*)/;
2536
- const RUN_ID_RE = /\bplay\/[^/\s]+\/run\/[0-9a-zTt-]+/;
2537
-
2538
- async function flushTailRunLogs(
2539
- events: unknown[],
2540
- env: CoordinatorEnv,
2541
- ): Promise<void> {
2542
- const token = env.DEEPLINE_TAIL_LOG_TOKEN?.trim();
2543
- if (!token) {
2544
- return;
2545
- }
2546
- const grouped = new Map<string, string[]>();
2547
- for (const event of events) {
2548
- for (const line of extractTailLogLines(event)) {
2549
- const parsed = parseRunLogLine(line);
2550
- if (!parsed) {
2551
- continue;
2552
- }
2553
- const existing = grouped.get(parsed.runId) ?? [];
2554
- existing.push(parsed.line);
2555
- grouped.set(parsed.runId, existing);
2556
- }
2557
- }
2558
- await Promise.all(
2559
- [...grouped.entries()].map(async ([runId, lines]) => {
2560
- if (lines.length === 0) {
2561
- return;
2562
- }
2563
- await fetch(
2564
- `${env.DEEPLINE_API_BASE_URL}${PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH}`,
2565
- {
2566
- method: 'POST',
2567
- headers: {
2568
- 'content-type': 'application/json',
2569
- 'x-deepline-tail-log-token': token,
2570
- },
2571
- body: JSON.stringify({
2572
- runId,
2573
- lines: lines.slice(-100),
2574
- }),
2575
- },
2576
- ).catch(() => null);
2577
- }),
2578
- );
2579
- }
2580
-
2581
- function extractTailLogLines(event: unknown): string[] {
2582
- const lines: string[] = [];
2583
- const record =
2584
- event && typeof event === 'object' && !Array.isArray(event)
2585
- ? (event as Record<string, unknown>)
2586
- : {};
2587
- const logs = Array.isArray(record.logs) ? record.logs : [];
2588
- for (const log of logs) {
2589
- if (typeof log === 'string') {
2590
- lines.push(log);
2591
- continue;
2592
- }
2593
- if (!log || typeof log !== 'object' || Array.isArray(log)) {
2594
- continue;
2595
- }
2596
- const logRecord = log as Record<string, unknown>;
2597
- const message = logRecord.message;
2598
- if (Array.isArray(message)) {
2599
- lines.push(message.map(formatTailLogPart).join(' '));
2600
- } else if (typeof message === 'string') {
2601
- lines.push(message);
2602
- }
2603
- }
2604
- const exceptions = Array.isArray(record.exceptions) ? record.exceptions : [];
2605
- for (const exception of exceptions) {
2606
- lines.push(formatTailLogPart(exception));
2607
- }
2608
- return lines.map((line) => line.trim()).filter(Boolean);
2609
- }
2610
-
2611
- function formatTailLogPart(value: unknown): string {
2612
- if (typeof value === 'string') {
2613
- return value;
2614
- }
2615
- if (value instanceof Error) {
2616
- return value.stack ?? value.message;
2617
- }
2618
- try {
2619
- return JSON.stringify(value);
2620
- } catch {
2621
- return String(value);
2622
- }
2623
- }
2624
-
2625
- /**
2626
- * Recognize the tenant-Postgres degradation signatures that used to escape the
2627
- * coordinator submit path as an opaque 500 "internal error" with no repair
2628
- * instruction: revoked CONNECT / grant drift, connect stalls, and NeonDbError.
2629
- * Conservative on purpose — anything not matched stays a generic internal
2630
- * error (with its phase) so we do not mislabel unrelated failures.
2631
- */
2632
- function isTenantStorageDegradationError(error: unknown): boolean {
2633
- const message = error instanceof Error ? error.message : String(error);
2634
- return (
2635
- /permission denied (?:for|to grant) (?:schema|table|relation|database|sequence|role)\b/i.test(
2636
- message,
2637
- ) ||
2638
- /must be owner of (?:table|relation|sequence|schema|database|function|procedure|routine|type)\b/i.test(
2639
- message,
2640
- ) ||
2641
- /Runtime Postgres connection timed out/i.test(message) ||
2642
- /\bNeonDbError\b/i.test(message) ||
2643
- /timeout exceeded when trying to connect/i.test(message) ||
2644
- /connection terminated unexpectedly/i.test(message)
2645
- // Deliberately NOT matching bare socket/DNS codes (ECONNRESET, ETIMEDOUT,
2646
- // ECONNREFUSED, EAI_AGAIN): those are thrown by Cloudflare service bindings,
2647
- // the Workflow API, and Durable Object fetches too, not just tenant
2648
- // Postgres. Matching them here would mislabel a transient CF-infra blip as
2649
- // WORKSPACE_STORAGE_NOT_READY ("run db repair"). Every remaining pattern
2650
- // carries explicit Postgres/Neon context.
2651
- );
2652
- }
2653
-
2654
- function coordinatorWorkflowRouteErrorResponse(input: {
2655
- runId: string;
2656
- action: string;
2657
- error: unknown;
2658
- }): Response {
2659
- const message =
2660
- input.error instanceof Error ? input.error.message : String(input.error);
2661
- const phase = `coordinator.${input.action || 'unknown'}`;
2662
- recordCoordinatorEvent(
2663
- 'error',
2664
- 'coordinator.workflow_route.error',
2665
- '[coordinator.workflow_route.error]',
2666
- {
2667
- runId: input.runId,
2668
- action: input.action,
2669
- error: message,
2670
- },
2671
- );
2672
- if (
2673
- input.action === 'submit' &&
2674
- isTenantStorageDegradationError(input.error)
2675
- ) {
2676
- // De-redact: a tenant-storage failure gets the loud, actionable
2677
- // WORKSPACE_STORAGE_NOT_READY message + repair instruction instead of the
2678
- // generic "internal error". Same contract as the runtime route.
2679
- //
2680
- // Scoped to `submit` on purpose: the message states "nothing was billed",
2681
- // which is only true in the pre-run submit phase. The other workflow
2682
- // actions (status/result/tail/cancel/signal) run against an already-active,
2683
- // possibly-billing run, so surfacing "storage not ready, nothing billed,
2684
- // run db repair" there would be false and alarming on a healthy run.
2685
- recordCoordinatorEvent(
2686
- 'error',
2687
- 'coordinator.workflow_route.storage_not_ready',
2688
- '[coordinator.workflow_route.storage_not_ready]',
2689
- {
2690
- runId: input.runId,
2691
- action: input.action,
2692
- phase,
2693
- error: message,
2694
- },
2695
- );
2696
- return Response.json(
2697
- {
2698
- error: {
2699
- code: WORKSPACE_STORAGE_NOT_READY_CODE,
2700
- message: WORKSPACE_STORAGE_NOT_READY_MESSAGE,
2701
- phase,
2702
- runId: input.runId,
2703
- },
2704
- },
2705
- { status: 503 },
2706
- );
2707
- }
2708
- return Response.json(
2709
- {
2710
- error: {
2711
- code: 'COORDINATOR_WORKFLOW_ROUTE_FAILED',
2712
- message,
2713
- phase,
2714
- runId: input.runId,
2715
- },
2716
- },
2717
- { status: 500 },
2718
- );
2719
- }
2720
-
2721
- /**
2722
- * Generic JSON envelope for any thrown error escaping a coordinator fetch route.
2723
- * Mirrors {@link coordinatorWorkflowRouteErrorResponse} so that NO route group
2724
- * in the fetch handler can leak Cloudflare's raw "Worker threw exception" HTML
2725
- * page to clients/agents. Keep the envelope shape `{ error: { code, message,
2726
- * phase, runId } }` and the console.error logging convention identical.
2727
- */
2728
- function coordinatorRouteErrorResponse(input: {
2729
- logTag: string;
2730
- code: string;
2731
- phase: string;
2732
- runId: string | null;
2733
- error: unknown;
2734
- }): Response {
2735
- const message =
2736
- input.error instanceof Error ? input.error.message : String(input.error);
2737
- recordCoordinatorEvent('error', 'coordinator.route.error', input.logTag, {
2738
- runId: input.runId,
2739
- phase: input.phase,
2740
- error: message,
2741
- });
2742
- return Response.json(
2743
- {
2744
- error: {
2745
- code: input.code,
2746
- message,
2747
- phase: input.phase,
2748
- runId: input.runId,
2749
- },
2750
- },
2751
- { status: 500 },
2752
- );
2753
- }
2754
-
2755
- // Operator-diagnostic console lines that carry the [deepline-run:] prefix but
2756
- // are not user-facing run output. The console scrape fans run-prefixed lines
2757
- // back into the run's durable Run Log Stream ('system' channel), so harness/
2758
- // coordinator plumbing noise is filtered at ingestion, never at read time.
2759
- // User play log lines (runner-event echoes) intentionally pass through.
2760
- const OPERATOR_NOISE_LOG_PATTERNS: readonly RegExp[] = [
2761
- /\[perf-trace\]/,
2762
- /\[harness-probe\]/,
2763
- /TenantWorkflow\.run entered/,
2764
- /TenantWorkflow\.run threw/,
2765
- /failed to forward runner perf trace/,
2766
- /failed to forward TenantWorkflow\.run error/,
2767
- ];
2768
-
2769
- function parseRunLogLine(line: string): { runId: string; line: string } | null {
2770
- if (OPERATOR_NOISE_LOG_PATTERNS.some((pattern) => pattern.test(line))) {
2771
- return null;
2772
- }
2773
- const prefixed = line.match(RUN_LOG_PREFIX_RE);
2774
- if (prefixed?.[1]) {
2775
- const message = prefixed[2]?.trim() ?? '';
2776
- return {
2777
- runId: prefixed[1],
2778
- line: message || line,
2779
- };
2780
- }
2781
- const runId = line.match(RUN_ID_RE)?.[0];
2782
- if (!runId) {
2783
- return null;
2784
- }
2785
- return { runId, line };
2786
- }
2787
-
2788
- async function handleWorkflowRoute(input: {
2789
- runId: string;
2790
- action: string;
2791
- request: Request;
2792
- env: CoordinatorEnv;
2793
- ctx?: ExecutionContext;
2794
- }): Promise<Response> {
2795
- const { runId, action, request, env } = input;
2796
- const defaultInstanceId = workflowInstanceId(runId);
2797
- if (action === 'submit') {
2798
- const submitStartedAt = Date.now();
2799
- const coordinatorTimings: CoordinatorTiming[] = [];
2800
- let submittedRunId = runId;
2801
- const recordSubmitTiming = (timing: CoordinatorTiming): void => {
2802
- coordinatorTimings.push(timing);
2803
- recordCoordinatorPerfTraceBuffered(env, input.ctx, {
2804
- runId: submittedRunId,
2805
- phase: timing.phase,
2806
- ms: timing.ms,
2807
- graphHash: timing.graphHash ?? null,
2808
- extra: timing.extra,
2809
- });
2810
- };
2811
- const parseStartedAt = Date.now();
2812
- const params = (await request.json()) as PlayWorkflowParams;
2813
- submittedRunId = params.runId ?? runId;
2814
- recordSubmitTiming({
2815
- phase: 'coordinator.submit_received',
2816
- ms: Date.now() - submitStartedAt,
2817
- graphHash: params.graphHash ?? null,
2818
- extra: {
2819
- integrationMode: normalizeIntegrationMode(params.integrationMode),
2820
- hasDynamicWorkerCode: Boolean(params.dynamicWorkerCode),
2821
- dynamicWorkerBytes:
2822
- typeof params.dynamicWorkerCode === 'string'
2823
- ? params.dynamicWorkerCode.length
2824
- : 0,
2825
- },
2826
- });
2827
- recordSubmitTiming({
2828
- phase: 'coordinator.submit_parse_body',
2829
- ms: Date.now() - parseStartedAt,
2830
- graphHash: params.graphHash ?? null,
2831
- extra: {
2832
- integrationMode: normalizeIntegrationMode(params.integrationMode),
2833
- hasDynamicWorkerCode: Boolean(params.dynamicWorkerCode),
2834
- dynamicWorkerBytes:
2835
- typeof params.dynamicWorkerCode === 'string'
2836
- ? params.dynamicWorkerCode.length
2837
- : 0,
2838
- },
2839
- });
2840
- if (!env.LOADER) {
2841
- throw new Error(
2842
- 'Cloudflare Dynamic Workflows require a Worker Loader binding named LOADER.',
2843
- );
2844
- }
2845
- if (!env.HARNESS) {
2846
- throw new Error(
2847
- 'Cloudflare Dynamic Workflows require a service binding named HARNESS. ' +
2848
- 'Start apps/play-harness-worker before the coordinator or fix wrangler.toml services.',
2849
- );
2850
- }
2851
- const preloadedDbSessions = Array.isArray(params.preloadedDbSessions)
2852
- ? params.preloadedDbSessions
2853
- : [];
2854
- if (preloadedDbSessions.length > 0 && params.executorToken) {
2855
- const prewarmStartedAt = Date.now();
2856
- recordSubmitTiming({
2857
- phase: 'coordinator.harness_prewarm_postgres_start',
2858
- ms: 0,
2859
- graphHash: params.graphHash ?? null,
2860
- extra: { sessions: preloadedDbSessions.length },
2861
- });
2862
- const prewarmPromise = env.HARNESS.prewarmPostgresSessions({
2863
- executorToken: params.executorToken,
2864
- orgId: params.orgId,
2865
- playName: params.playName,
2866
- sessions: preloadedDbSessions,
2867
- })
2868
- .then((result) => {
2869
- recordSubmitTiming({
2870
- phase: 'coordinator.harness_prewarm_postgres',
2871
- ms: Date.now() - prewarmStartedAt,
2872
- graphHash: params.graphHash ?? null,
2873
- extra: {
2874
- status: 'ok',
2875
- sessions: result.sessions,
2876
- },
2877
- });
2878
- })
2879
- .catch((error: unknown) => {
2880
- const storageNotReady = isTenantStorageDegradationError(error);
2881
- recordSubmitTiming({
2882
- phase: 'coordinator.harness_prewarm_postgres',
2883
- ms: Date.now() - prewarmStartedAt,
2884
- graphHash: params.graphHash ?? null,
2885
- extra: {
2886
- status: 'failed',
2887
- sessions: preloadedDbSessions.length,
2888
- error: error instanceof Error ? error.message : String(error),
2889
- storageNotReady,
2890
- },
2891
- });
2892
- // The prewarm runs in waitUntil, so it cannot fail the already-issued
2893
- // submit. But its cause used to be fully redacted in coordinator
2894
- // logs. Log the tenant-storage classification loudly so the real
2895
- // cause (and the WORKSPACE_STORAGE_NOT_READY repair path) is visible.
2896
- if (storageNotReady) {
2897
- recordCoordinatorEvent(
2898
- 'error',
2899
- 'coordinator.harness_prewarm_postgres.storage_not_ready',
2900
- '[coordinator.harness_prewarm_postgres.storage_not_ready]',
2901
- {
2902
- runId: submittedRunId,
2903
- playName: params.playName,
2904
- code: WORKSPACE_STORAGE_NOT_READY_CODE,
2905
- error: error instanceof Error ? error.message : String(error),
2906
- },
2907
- );
2908
- }
2909
- });
2910
- input.ctx?.waitUntil(prewarmPromise);
2911
- }
2912
- const dbSessionExternalization = externalizedWorkflowDbSessionParams({
2913
- params,
2914
- });
2915
- const workflowParams = dbSessionExternalization.params;
2916
- workflowParams.submittedAt = Date.now();
2917
- let launchState: Awaited<ReturnType<typeof persistWorkflowLaunchState>> =
2918
- {};
2919
- try {
2920
- const retryStateStartedAt = Date.now();
2921
- launchState = await persistWorkflowLaunchState({
2922
- env,
2923
- runId: submittedRunId,
2924
- params: workflowParams,
2925
- sessions: dbSessionExternalization.sessions,
2926
- });
2927
- const persistedAt = Date.now();
2928
- if (dbSessionExternalization.sessions.length > 0) {
2929
- recordSubmitTiming({
2930
- phase: 'coordinator.workflow_db_sessions_externalized',
2931
- ms: persistedAt - retryStateStartedAt,
2932
- graphHash: params.graphHash ?? null,
2933
- extra: {
2934
- sessions:
2935
- launchState.sessionCount ??
2936
- dbSessionExternalization.sessions.length,
2937
- expiresAt:
2938
- launchState.dbSessionsExpiresAt ??
2939
- dbSessionExternalization.ref?.expiresAt,
2940
- combinedLaunchState: true,
2941
- },
2942
- });
2943
- }
2944
- recordSubmitTiming({
2945
- phase: 'coordinator.retry_state_persistence',
2946
- ms: persistedAt - retryStateStartedAt,
2947
- graphHash: params.graphHash ?? null,
2948
- extra: {
2949
- combinedLaunchState: dbSessionExternalization.sessions.length > 0,
2950
- },
2951
- });
2952
- } catch (error) {
2953
- const errorMessage =
2954
- error instanceof Error ? error.message : String(error);
2955
- recordCoordinatorEvent(
2956
- 'error',
2957
- 'coordinator.workflow_retry_state.persistence_failed',
2958
- '[coordinator] workflow retry state persistence failed',
2959
- {
2960
- code: 'WORKFLOW_RETRY_STATE_PERSISTENCE_FAILED',
2961
- runId: submittedRunId,
2962
- error: errorMessage,
2963
- },
2964
- );
2965
- recordSubmitTiming({
2966
- phase: 'coordinator.retry_state_persistence',
2967
- ms: 0,
2968
- graphHash: params.graphHash ?? null,
2969
- extra: {
2970
- status: 'failed',
2971
- error: errorMessage,
2972
- },
2973
- });
2974
- return workflowRetryStatePersistenceErrorResponse({
2975
- runId: submittedRunId,
2976
- error,
2977
- });
2978
- }
2979
- const createParams = workflowCreateParams(workflowParams, launchState);
2980
- let instance: WorkflowInstance | null = null;
2981
- try {
2982
- const submitClaimStartedAt = Date.now();
2983
- const submitClaim = await claimWorkflowSubmit({
2984
- env,
2985
- runId: submittedRunId,
2986
- });
2987
- recordSubmitTiming({
2988
- phase: 'coordinator.workflow_submit_claim',
2989
- ms: Date.now() - submitClaimStartedAt,
2990
- graphHash: params.graphHash ?? null,
2991
- extra: {
2992
- claimed: submitClaim.claimed,
2993
- status: submitClaim.status,
2994
- instanceId: submitClaim.instanceId ?? defaultInstanceId,
2995
- },
2996
- });
2997
- if (!submitClaim.claimed) {
2998
- const duplicateWaitStartedAt = Date.now();
2999
- const resolvedSubmitClaim = await waitForWorkflowSubmitCreated({
3000
- env,
3001
- runId: submittedRunId,
3002
- initialClaim: submitClaim,
3003
- timeoutMs: WORKFLOW_SUBMIT_DUPLICATE_WAIT_MS,
3004
- pollMs: WORKFLOW_SUBMIT_DUPLICATE_POLL_MS,
3005
- });
3006
- recordSubmitTiming({
3007
- phase: 'coordinator.workflow_submit_duplicate_wait',
3008
- ms: Date.now() - duplicateWaitStartedAt,
3009
- graphHash: params.graphHash ?? null,
3010
- extra: {
3011
- status: resolvedSubmitClaim.status,
3012
- instanceId: resolvedSubmitClaim.instanceId ?? null,
3013
- },
3014
- });
3015
- const totalMs = Date.now() - submitStartedAt;
3016
- if (
3017
- resolvedSubmitClaim.status !== 'created' ||
3018
- !resolvedSubmitClaim.instanceId
3019
- ) {
3020
- recordSubmitTiming({
3021
- phase: 'coordinator.submit_total',
3022
- ms: totalMs,
3023
- graphHash: params.graphHash ?? null,
3024
- extra: { duplicate: true, pending: true },
3025
- });
3026
- return Response.json(
3027
- {
3028
- runId,
3029
- status: 'pending',
3030
- workflowInstanceId: null,
3031
- instanceState: {
3032
- status: resolvedSubmitClaim.status ?? 'creating',
3033
- },
3034
- retryAfterMs: 250,
3035
- runtimeDeployVersion:
3036
- env.CF_VERSION_METADATA?.id ??
3037
- env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
3038
- null,
3039
- coordinatorTimings,
3040
- },
3041
- { status: 503, headers: { 'retry-after': '1' } },
3042
- );
3043
- }
3044
- recordSubmitTiming({
3045
- phase: 'coordinator.submit_total',
3046
- ms: totalMs,
3047
- graphHash: params.graphHash ?? null,
3048
- extra: { duplicate: true },
3049
- });
3050
- recordSubmitTiming({
3051
- phase: 'coordinator.submit_accepted',
3052
- ms: totalMs,
3053
- graphHash: params.graphHash ?? null,
3054
- extra: { duplicate: true },
3055
- });
3056
- return Response.json({
3057
- runId,
3058
- status: 'submitted',
3059
- workflowInstanceId: resolvedSubmitClaim.instanceId,
3060
- instanceState: { status: resolvedSubmitClaim.status },
3061
- runtimeDeployVersion:
3062
- env.CF_VERSION_METADATA?.id ??
3063
- env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
3064
- null,
3065
- coordinatorTimings,
3066
- });
3067
- }
3068
- const dispatchStartedAt = Date.now();
3069
- const createStartedAt = Date.now();
3070
- recordSubmitTiming({
3071
- phase: 'coordinator.workflow_create_start',
3072
- ms: 0,
3073
- graphHash: params.graphHash ?? null,
3074
- extra: { instanceId: defaultInstanceId },
3075
- });
3076
- const createResult = await createOrAttachWorkflowInstance({
3077
- create: () =>
3078
- createDynamicWorkflowInstance({
3079
- env,
3080
- id: defaultInstanceId,
3081
- params: createParams,
3082
- metadataParams: workflowParams,
3083
- }),
3084
- getExisting: () => env.PLAY_WORKFLOW.get(defaultInstanceId),
3085
- onCreateFailure: async () => {
3086
- await releaseWorkflowSubmitClaim({
3087
- env,
3088
- runId: submittedRunId,
3089
- });
3090
- },
3091
- });
3092
- instance = createResult.instance;
3093
- const workflowCreatedAt = Date.now();
3094
- recordSubmitTiming({
3095
- phase: 'coordinator.workflow_create',
3096
- ms: workflowCreatedAt - createStartedAt,
3097
- graphHash: params.graphHash ?? null,
3098
- extra: { instanceId: instance.id, startMode: createResult.startMode },
3099
- });
3100
- try {
3101
- await recordWorkflowInstanceId({
3102
- env,
3103
- runId: submittedRunId,
3104
- instanceId: instance.id,
3105
- });
3106
- } catch (error) {
3107
- recordCoordinatorEvent(
3108
- 'warn',
3109
- 'coordinator.workflow_instance_id.record_failed',
3110
- '[coordinator] workflow instance id record failed',
3111
- {
3112
- runId: submittedRunId,
3113
- instanceId: instance?.id ?? null,
3114
- error: error instanceof Error ? error.message : String(error),
3115
- },
3116
- );
3117
- await releaseWorkflowSubmitClaim({
3118
- env,
3119
- runId: submittedRunId,
3120
- }).catch((releaseError) => {
3121
- recordCoordinatorEvent(
3122
- 'warn',
3123
- 'coordinator.workflow_submit_claim.release_failed',
3124
- '[coordinator] workflow submit claim release after instance record failure failed',
3125
- {
3126
- runId: submittedRunId,
3127
- error:
3128
- releaseError instanceof Error
3129
- ? releaseError.message
3130
- : String(releaseError),
3131
- },
3132
- );
3133
- });
3134
- throw error;
3135
- }
3136
- recordSubmitTiming({
3137
- phase: 'coordinator.dispatch_workflow',
3138
- ms: Date.now() - dispatchStartedAt,
3139
- graphHash: params.graphHash ?? null,
3140
- extra: {
3141
- startMode: 'direct_workflow_create',
3142
- workflowCreateMode: createResult.startMode,
3143
- instanceIdRecord: 'recorded',
3144
- },
3145
- });
3146
- const initialWaitMsRaw = Number(
3147
- new URL(request.url).searchParams.get('initialWaitMs') ?? '0',
3148
- );
3149
- const initialStateStartedAt = Date.now();
3150
- const instanceState = await waitForSubmitInitialState({
3151
- instance,
3152
- runId: submittedRunId,
3153
- waitMs: Number.isFinite(initialWaitMsRaw) ? initialWaitMsRaw : 0,
3154
- });
3155
- recordSubmitTiming({
3156
- phase: 'coordinator.submit_initial_state',
3157
- ms: Date.now() - initialStateStartedAt,
3158
- graphHash: params.graphHash ?? null,
3159
- extra: {
3160
- waitMs: Number.isFinite(initialWaitMsRaw) ? initialWaitMsRaw : 0,
3161
- status:
3162
- typeof instanceState?.status === 'string'
3163
- ? instanceState.status
3164
- : null,
3165
- },
3166
- });
3167
- const totalMs = Date.now() - submitStartedAt;
3168
- recordSubmitTiming({
3169
- phase: 'coordinator.submit_total',
3170
- ms: totalMs,
3171
- graphHash: params.graphHash ?? null,
3172
- });
3173
- recordSubmitTiming({
3174
- phase: 'coordinator.submit_accepted',
3175
- ms: totalMs,
3176
- graphHash: params.graphHash ?? null,
3177
- });
3178
- return Response.json({
3179
- runId,
3180
- status: 'submitted',
3181
- workflowInstanceId: instance.id,
3182
- instanceState,
3183
- runtimeDeployVersion:
3184
- env.CF_VERSION_METADATA?.id ??
3185
- env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
3186
- null,
3187
- coordinatorTimings,
3188
- });
3189
- } finally {
3190
- disposeRpcStub(instance);
3191
- }
3192
- }
3193
-
3194
- if (action === 'tail') {
3195
- const url = new URL(request.url);
3196
- const waitMs = Math.min(
3197
- Math.max(Number(url.searchParams.get('waitMs') ?? '0'), 0),
3198
- 30_000,
3199
- );
3200
- const afterSeq = Math.max(
3201
- 0,
3202
- Math.floor(Number(url.searchParams.get('afterSeq') ?? '0')),
3203
- );
3204
- const includeTrace = url.searchParams.get('trace') === '1';
3205
- const statusStartedAt = Date.now();
3206
- const eventResult = await listCoordinatorRunEvents({
3207
- env,
3208
- runId,
3209
- afterSeq,
3210
- timeoutMs: waitMs,
3211
- }).catch(() => null);
3212
- const rawEvents = eventResult?.events ?? [];
3213
- const terminalEventIndex = rawEvents.findIndex(
3214
- (event) => event.type === 'terminal',
3215
- );
3216
- const events =
3217
- terminalEventIndex >= 0
3218
- ? rawEvents.slice(0, terminalEventIndex + 1)
3219
- : rawEvents;
3220
- const coordinatorTrace =
3221
- includeTrace && events.length
3222
- ? await listCoordinatorPerfTrace(env, runId).catch(() => [])
3223
- : [];
3224
- const terminalEvent = events.find(
3225
- (event): event is Extract<CoordinatorRunEvent, { type: 'terminal' }> =>
3226
- event.type === 'terminal',
3227
- );
3228
- if (terminalEvent) {
3229
- return Response.json({
3230
- runId,
3231
- ...(terminalEvent.playName ? { playName: terminalEvent.playName } : {}),
3232
- status: terminalEvent.status,
3233
- result: terminalEvent.result ?? null,
3234
- error: terminalEvent.error ?? null,
3235
- totalRows: terminalEvent.totalRows ?? null,
3236
- durationMs: terminalEvent.durationMs ?? null,
3237
- completedAt: terminalEvent.ts,
3238
- liveLogs: sanitizeLiveLogLines(terminalEvent.liveLogs),
3239
- liveNodeProgress: terminalEvent.liveNodeProgress ?? null,
3240
- events,
3241
- latestSeq: eventResult?.latestSeq ?? afterSeq,
3242
- wait: null,
3243
- coordinatorObserve: {
3244
- ms: Date.now() - statusStartedAt,
3245
- waitMs,
3246
- workflowStatus: 'terminal-event',
3247
- statusPolls: 0,
3248
- instanceId: null,
3249
- },
3250
- ...(includeTrace ? { coordinatorTrace } : {}),
3251
- });
3252
- }
3253
- return Response.json({
3254
- runId,
3255
- status: 'running',
3256
- events,
3257
- latestSeq: eventResult?.latestSeq ?? afterSeq,
3258
- wait: null,
3259
- coordinatorObserve: {
3260
- ms: Date.now() - statusStartedAt,
3261
- waitMs,
3262
- workflowStatus: eventResult?.events.length ? 'event' : 'event-timeout',
3263
- statusPolls: 0,
3264
- instanceId: null,
3265
- },
3266
- ...(includeTrace ? { coordinatorTrace } : {}),
3267
- });
3268
- }
3269
-
3270
- // get() throws if the instance doesn't exist (Workflows local-mode wipes
3271
- // state on wrangler dev reload, and superseded `--force` runs may target
3272
- // an instance that was never created). Treat that as a no-op cancel.
3273
- let instance: WorkflowInstance | null = null;
3274
- try {
3275
- const requestedInstanceId = new URL(request.url).searchParams
3276
- .get('instanceId')
3277
- ?.trim();
3278
- const instanceId =
3279
- requestedInstanceId &&
3280
- isWorkflowInstanceIdForRun(runId, requestedInstanceId)
3281
- ? requestedInstanceId
3282
- : await resolveWorkflowInstanceIdForRun(env, runId);
3283
- instance = await env.PLAY_WORKFLOW.get(instanceId);
3284
- } catch (error) {
3285
- const message = error instanceof Error ? error.message : String(error);
3286
- if (
3287
- !/not[ _]found|not_found|does not exist|no such instance|404/i.test(
3288
- message,
3289
- )
3290
- ) {
3291
- throw error;
3292
- }
3293
- }
3294
- try {
3295
- if (action === 'cancel') {
3296
- if (instance) {
3297
- try {
3298
- await instance.terminate();
3299
- } catch (error) {
3300
- const message =
3301
- error instanceof Error ? error.message : String(error);
3302
- // Tolerate four classes of error here:
3303
- // - already-terminal (complete / errored / terminated)
3304
- // - "Cannot terminate instance since its on a finite state"
3305
- // (the runtime's wording for "already finished")
3306
- // - "not implemented" (wrangler dev local mode doesn't support
3307
- // instance.terminate() yet — silently no-op there)
3308
- // - "not found" (instance never existed)
3309
- if (
3310
- !/complete|terminated|errored|finite state|cannot[ _]terminate|not[ _]implemented|not[ _]found|404/i.test(
3311
- message,
3312
- )
3313
- ) {
3314
- throw error;
3315
- }
3316
- }
3317
- }
3318
- // terminate() kills the dynamic worker before its run() wrapper can
3319
- // write terminal state (the only place completed/failed land), so
3320
- // without this write /tail reports 'running' forever and any
3321
- // start-stream watcher hangs after a cancel. Land the cancelled
3322
- // terminal state here — terminal-set appends a 'terminal' run event
3323
- // and wakes the dedup DO's long-poll waiters, which unblocks tails.
3324
- //
3325
- // Idempotency: first-wins from this side — if the run already went
3326
- // terminal (completed/failed/cancelled) we keep that state. The DO
3327
- // stores the cached terminal state under a single storage key
3328
- // (last-wins on raw writes), but the run-event log is append-only
3329
- // and /tail truncates at the FIRST terminal event, so a racing
3330
- // completed/failed write from a dying worker can at worst replace
3331
- // the cached key with another terminal status — it can never
3332
- // resurrect 'running'.
3333
- const existingTerminal = await readCoordinatorTerminalState(
3334
- env,
3335
- runId,
3336
- ).catch((error: unknown) => {
3337
- // Tolerated: better to risk a harmless terminal-over-terminal
3338
- // overwrite than to skip the cancelled write and hang watchers.
3339
- recordCoordinatorEvent(
3340
- 'warn',
3341
- 'coordinator.cancel.terminal_state_read_failed',
3342
- '[coordinator] terminal state read before cancel failed',
3343
- {
3344
- runId,
3345
- error: error instanceof Error ? error.message : String(error),
3346
- },
3347
- );
3348
- return null;
3349
- });
3350
- if (!existingTerminal) {
3351
- try {
3352
- await writeCoordinatorTerminalState(env, {
3353
- runId,
3354
- status: 'cancelled',
3355
- error: 'Run cancelled',
3356
- });
3357
- } catch (error) {
3358
- // Fail loudly: the workflow was terminated but watchers would
3359
- // hang on 'running' forever without the terminal event.
3360
- const message =
3361
- error instanceof Error ? error.message : String(error);
3362
- recordCoordinatorEvent(
3363
- 'error',
3364
- 'coordinator.cancel.terminal_state_write_failed',
3365
- '[coordinator] cancel terminal state write failed',
3366
- {
3367
- runId,
3368
- error: message,
3369
- },
3370
- );
3371
- return Response.json(
3372
- {
3373
- runId,
3374
- status: 'error',
3375
- error: `workflow terminated but cancelled terminal state write failed: ${message}`,
3376
- },
3377
- { status: 500 },
3378
- );
3379
- }
3380
- }
3381
- return Response.json({ runId, status: 'cancelled' });
3382
- }
3383
- if (action === 'signal') {
3384
- if (!instance) {
3385
- return new Response('not found', { status: 404 });
3386
- }
3387
- const body = (await request.json().catch(() => ({}))) as Record<
3388
- string,
3389
- unknown
3390
- >;
3391
- const eventKey =
3392
- typeof body.eventKey === 'string' && body.eventKey.trim()
3393
- ? body.eventKey.trim()
3394
- : typeof body.event_key === 'string' && body.event_key.trim()
3395
- ? body.event_key.trim()
3396
- : '';
3397
- const eventType =
3398
- typeof body.type === 'string' && body.type.trim()
3399
- ? body.type.trim()
3400
- : eventKey
3401
- ? `integration_event_${eventKey}`
3402
- : 'integration_event';
3403
- await instance.sendEvent({
3404
- type: workflowEventType(eventType),
3405
- payload: body,
3406
- });
3407
- return Response.json({
3408
- runId,
3409
- status: mapWorkflowStatus(await instance.status()),
3410
- });
3411
- }
3412
- if (action === 'result' || action === 'status' || action === '') {
3413
- const includeTrace =
3414
- new URL(request.url).searchParams.get('trace') === '1';
3415
- const statusStartedAt = Date.now();
3416
- const terminalState = await readCoordinatorTerminalState(
3417
- env,
3418
- runId,
3419
- ).catch(() => null);
3420
- if (terminalState) {
3421
- const coordinatorTrace = includeTrace
3422
- ? await listCoordinatorPerfTrace(env, runId).catch(() => [])
3423
- : [];
3424
- return Response.json({
3425
- runId,
3426
- ...(terminalState.playName
3427
- ? { playName: terminalState.playName }
3428
- : {}),
3429
- status: terminalState.status,
3430
- result: terminalState.result ?? null,
3431
- error: terminalState.error ?? null,
3432
- totalRows: terminalState.totalRows ?? null,
3433
- durationMs: terminalState.durationMs ?? null,
3434
- completedAt: terminalState.completedAt ?? null,
3435
- liveLogs: sanitizeLiveLogLines(terminalState.liveLogs),
3436
- liveNodeProgress: terminalState.liveNodeProgress ?? null,
3437
- wait: null,
3438
- coordinatorObserve: {
3439
- ms: Date.now() - statusStartedAt,
3440
- waitMs: 0,
3441
- workflowStatus: 'terminal-cache',
3442
- statusPolls: 0,
3443
- instanceId: instance?.id ?? null,
3444
- },
3445
- ...(includeTrace ? { coordinatorTrace } : {}),
3446
- });
3447
- }
3448
- if (!instance) {
3449
- return new Response('not found', { status: 404 });
3450
- }
3451
- const status = await instance.status();
3452
- const workflowError = readWorkflowError(status);
3453
- if (workflowError) {
3454
- const retry = await restartWorkflowAfterPlatformReset({
3455
- env,
3456
- ctx: input.ctx,
3457
- runId,
3458
- oldInstance: instance,
3459
- error: workflowError,
3460
- status,
3461
- });
3462
- if (retry.retried && retry.result) {
3463
- const coordinatorTrace = includeTrace
3464
- ? await listCoordinatorPerfTrace(env, runId).catch(() => [])
3465
- : [];
3466
- return Response.json({
3467
- ...retry.result,
3468
- coordinatorObserve: {
3469
- ms: Date.now() - statusStartedAt,
3470
- waitMs: 0,
3471
- workflowStatus: 'retrying',
3472
- statusPolls: 1,
3473
- instanceId: instance.id,
3474
- },
3475
- ...(includeTrace ? { coordinatorTrace } : {}),
3476
- });
3477
- }
3478
- }
3479
- const result = mapWorkflowResult(runId, status);
3480
- const observeMs = Date.now() - statusStartedAt;
3481
- // If we forced a permanent-error fail-fast (status='failed' even
3482
- // though the underlying instance is still 'running' mid-retry),
3483
- // terminate on the CF side once so the retry loop stops burning
3484
- // compute. Best-effort: tolerate races where the instance has
3485
- // already gone terminal between our status() and terminate().
3486
- if (
3487
- result.status === 'failed' &&
3488
- status.status !== 'errored' &&
3489
- status.status !== 'terminated' &&
3490
- status.status !== 'complete'
3491
- ) {
3492
- await instance.terminate().catch((error: unknown) => {
3493
- const message =
3494
- error instanceof Error ? error.message : String(error);
3495
- if (
3496
- !/complete|terminated|errored|finite state|cannot[ _]terminate|not[ _]implemented|not[ _]found|404/i.test(
3497
- message,
3498
- )
3499
- ) {
3500
- recordCoordinatorEvent(
3501
- 'warn',
3502
- 'coordinator.permanent_error.terminate_failed',
3503
- '[coordinator] terminate-after-permanent-error failed',
3504
- {
3505
- runId,
3506
- error: message,
3507
- },
3508
- );
3509
- }
3510
- });
3511
- }
3512
- const coordinatorTrace = includeTrace
3513
- ? await listCoordinatorPerfTrace(env, runId).catch(() => [])
3514
- : [];
3515
- return Response.json({
3516
- ...result,
3517
- coordinatorObserve: {
3518
- ms: observeMs,
3519
- waitMs: 0,
3520
- workflowStatus: status.status,
3521
- statusPolls: 1,
3522
- instanceId: instance.id,
3523
- },
3524
- ...(includeTrace ? { coordinatorTrace } : {}),
3525
- });
3526
- }
3527
- return new Response('not found', { status: 404 });
3528
- } finally {
3529
- disposeRpcStub(instance);
3530
- }
3531
- }
3532
-
3533
- function workflowInstanceId(runId: string): string {
3534
- const normalized = runId.toLowerCase().replace(/[^a-z0-9_-]+/g, '-');
3535
- if (normalized.length > 0 && normalized.length <= 100) {
3536
- return normalized;
3537
- }
3538
- return `run-${stableHash(runId)}`;
3539
- }
3540
-
3541
- function isWorkflowInstanceIdForRun(
3542
- runId: string,
3543
- instanceId: string,
3544
- ): boolean {
3545
- const canonical = workflowInstanceId(runId);
3546
- return (
3547
- instanceId === canonical || instanceId.startsWith(`${canonical}-retry-`)
3548
- );
3549
- }
3550
-
3551
- function stableHash(value: string): string {
3552
- let hash = 2166136261;
3553
- for (let index = 0; index < value.length; index += 1) {
3554
- hash ^= value.charCodeAt(index);
3555
- hash = Math.imul(hash, 16777619);
3556
- }
3557
- return (hash >>> 0).toString(36);
3558
- }
3559
-
3560
- const DYNAMIC_PLAY_WORKER_HARNESS_VERSION =
3561
- DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION;
3562
- const DYNAMIC_WORKER_BUNDLED_CODE_CACHE_MAX_ENTRIES = 64;
3563
- const dynamicWorkerBundledCodeCache = new Map<string, string>();
3564
-
3565
- function dynamicPlayWorkerCacheKey(input: {
3566
- env: CoordinatorEnv;
3567
- graphHash: string;
3568
- artifactIdentity: string;
3569
- }): string {
3570
- const deployMarker =
3571
- input.env.DEEPLINE_COORDINATOR_DEPLOY_MARKER?.trim() || 'local';
3572
- return [
3573
- 'play',
3574
- input.graphHash,
3575
- input.artifactIdentity,
3576
- `harness=${DYNAMIC_PLAY_WORKER_HARNESS_VERSION}`,
3577
- `deploy=${deployMarker}`,
3578
- ].join(':');
3579
- }
3580
-
3581
- function dynamicPlayWorkerArtifactIdentity(
3582
- metadata: Pick<
3583
- DynamicWorkflowMetadata,
3584
- | 'artifactHash'
3585
- | 'artifactStorageKey'
3586
- | 'dynamicWorkerCode'
3587
- | 'dynamicWorkerCodeHash'
3588
- >,
3589
- ): string {
3590
- const artifactHash = metadata.artifactHash?.trim();
3591
- const inlineCodeHash =
3592
- typeof metadata.dynamicWorkerCodeHash === 'string' &&
3593
- metadata.dynamicWorkerCodeHash.length > 0
3594
- ? metadata.dynamicWorkerCodeHash
3595
- : typeof metadata.dynamicWorkerCode === 'string' &&
3596
- metadata.dynamicWorkerCode.length > 0
3597
- ? stableHash(metadata.dynamicWorkerCode)
3598
- : null;
3599
- if (artifactHash && inlineCodeHash) {
3600
- return `${artifactHash}:inline=${inlineCodeHash}`;
3601
- }
3602
- if (artifactHash) {
3603
- return artifactHash;
3604
- }
3605
- if (inlineCodeHash) {
3606
- return `inline=${inlineCodeHash}`;
3607
- }
3608
- return stableHash(metadata.artifactStorageKey);
3609
- }
3610
-
3611
- function dynamicWorkerBundledCodeCacheKey(input: {
3612
- artifactStorageKey: string;
3613
- artifactHash?: string | null;
3614
- }): string {
3615
- return `${input.artifactHash?.trim() || 'no-hash'}:${input.artifactStorageKey}`;
3616
- }
3617
-
3618
- function readDynamicWorkerBundledCodeCache(key: string): string | null {
3619
- const cached = dynamicWorkerBundledCodeCache.get(key);
3620
- if (cached === undefined) return null;
3621
- dynamicWorkerBundledCodeCache.delete(key);
3622
- dynamicWorkerBundledCodeCache.set(key, cached);
3623
- return cached;
3624
- }
3625
-
3626
- function writeDynamicWorkerBundledCodeCache(key: string, code: string): void {
3627
- dynamicWorkerBundledCodeCache.set(key, code);
3628
- while (
3629
- dynamicWorkerBundledCodeCache.size >
3630
- DYNAMIC_WORKER_BUNDLED_CODE_CACHE_MAX_ENTRIES
3631
- ) {
3632
- const oldestKey = dynamicWorkerBundledCodeCache.keys().next().value;
3633
- if (typeof oldestKey !== 'string') break;
3634
- dynamicWorkerBundledCodeCache.delete(oldestKey);
3635
- }
3636
- }
3637
-
3638
- /**
3639
- * Synchronous wrapper around env.LOADER.get for use inside the
3640
- * createDynamicWorkflowEntrypoint loader callback. The framework's loader
3641
- * type allows async, but the official example
3642
- * (cloudflare/dynamic-workflows repo) uses sync; the microtask hop a
3643
- * top-level `await` introduces appears to interact badly with how the
3644
- * framework wires up the returned entrypoint stub for runner.run() RPC.
3645
- */
3646
- function loadDynamicPlayWorkerSync(
3647
- env: CoordinatorEnv,
3648
- metadata: DynamicWorkflowMetadata,
3649
- trace: CoordinatorPerfTraceSink = recordCoordinatorPerfTrace,
3650
- ): {
3651
- getEntrypoint(name?: string): Fetcher & WorkflowRunner;
3652
- } {
3653
- if (!env.LOADER) {
3654
- throw new Error(
3655
- 'Cloudflare Dynamic Workflows require a Worker Loader binding named LOADER.',
3656
- );
3657
- }
3658
- if (!env.HARNESS) {
3659
- throw new Error(
3660
- 'Cloudflare Dynamic Workflows require a service binding named HARNESS. ' +
3661
- 'Start apps/play-harness-worker before the coordinator or fix wrangler.toml services.',
3662
- );
3663
- }
3664
- const graphHash = metadata.graphHash.trim();
3665
- if (!graphHash) {
3666
- throw new Error('Dynamic play worker requires graphHash metadata.');
3667
- }
3668
- const artifactStorageKey = metadata.artifactStorageKey.trim();
3669
- if (!artifactStorageKey) {
3670
- throw new Error(
3671
- 'Dynamic play worker requires artifactStorageKey metadata.',
3672
- );
3673
- }
3674
- const artifactIdentity = dynamicPlayWorkerArtifactIdentity(metadata);
3675
- const workerCacheKey = dynamicPlayWorkerCacheKey({
3676
- env,
3677
- graphHash,
3678
- artifactIdentity,
3679
- });
3680
- const runIdForTrace = metadata.runId ?? graphHash;
3681
- const loaderGetStartedAt = Date.now();
3682
- const stub = env.LOADER.get(workerCacheKey, async () => {
3683
- const bundledCode = await loadDynamicWorkerBundledCode({
3684
- env,
3685
- metadata,
3686
- graphHash,
3687
- artifactStorageKey,
3688
- workerCacheKey,
3689
- runIdForTrace,
3690
- trace,
3691
- });
3692
- return {
3693
- compatibilityDate: DYNAMIC_WORKER_COMPATIBILITY_DATE,
3694
- compatibilityFlags: ['nodejs_compat'],
3695
- mainModule: 'index.js',
3696
- modules: { 'index.js': bundledCode },
3697
- env: {
3698
- // Service binding to the long-lived Play Harness Worker.
3699
- // Per-play code reaches this via `env.HARNESS.<method>(...)` —
3700
- // see sdk/src/plays/harness-stub.ts. This binding is required;
3701
- // miswired environments fail before user code starts.
3702
- HARNESS: env.HARNESS,
3703
- VERCEL_PROTECTION_BYPASS_TOKEN: env.VERCEL_PROTECTION_BYPASS_TOKEN,
3704
- // Runtime API bridge used by the play harness for status, tool
3705
- // execution, DB session, and artifact callbacks. This uses the
3706
- // long-lived HARNESS service binding, avoiding public callback HTTP
3707
- // without relying on dynamic-worker access to named exports.
3708
- ...makeRuntimeApiEnvBinding(env),
3709
- // In-process coordinator control bridge used by ctx.runPlay and
3710
- // parent terminal signals. This keeps scalar child plays inline with
3711
- // the parent instead of round-tripping through nested Workflow waits.
3712
- ...makeCoordinatorControlBinding(),
3713
- // NOTE: We intentionally do NOT pass `env.PLAYS_BUCKET` (an R2Bucket
3714
- // binding) through to the per-play Worker's env. Including a raw
3715
- // R2Bucket in the dynamically-loaded Worker's env makes Cloudflare
3716
- // Workflows fail at workflow init with
3717
- // "Could not serialize object of type 'R2Bucket'."
3718
- // because the framework persists workflow state through a JSDevalue
3719
- // path that R2Bucket bindings do not support. Per-play harness reads
3720
- // (CSVs, packaged imports) go through the long-lived HARNESS Worker.
3721
- },
3722
- };
3723
- });
3724
- trace({
3725
- runId: runIdForTrace,
3726
- phase: 'coordinator.loader_get_call',
3727
- ms: Date.now() - loaderGetStartedAt,
3728
- graphHash,
3729
- extra: { workerCacheKey },
3730
- });
3731
- return stub;
3732
- }
3733
-
3734
- async function loadDynamicPlayWorker(
3735
- env: CoordinatorEnv,
3736
- metadata: DynamicWorkflowMetadata,
3737
- trace: CoordinatorPerfTraceSink = recordCoordinatorPerfTrace,
3738
- ): Promise<{
3739
- getEntrypoint(name?: string): Fetcher & WorkflowRunner;
3740
- }> {
3741
- if (!env.LOADER) {
3742
- throw new Error(
3743
- 'Cloudflare Dynamic Workflows require a Worker Loader binding named LOADER.',
3744
- );
3745
- }
3746
- if (!env.HARNESS) {
3747
- throw new Error(
3748
- 'Cloudflare Dynamic Workflows require a service binding named HARNESS. ' +
3749
- 'Start apps/play-harness-worker before the coordinator or fix wrangler.toml services.',
3750
- );
3751
- }
3752
- const graphHash = metadata.graphHash.trim();
3753
- if (!graphHash) {
3754
- throw new Error('Dynamic play worker requires graphHash metadata.');
3755
- }
3756
- const artifactStorageKey = metadata.artifactStorageKey.trim();
3757
- if (!artifactStorageKey) {
3758
- throw new Error(
3759
- 'Dynamic play worker requires artifactStorageKey metadata.',
3760
- );
3761
- }
3762
- const artifactIdentity = dynamicPlayWorkerArtifactIdentity(metadata);
3763
- const workerCacheKey = dynamicPlayWorkerCacheKey({
3764
- env,
3765
- graphHash,
3766
- artifactIdentity,
3767
- });
3768
- const runIdForTrace = metadata.runId ?? graphHash;
3769
- const loaderGetStartedAt = Date.now();
3770
- const stub = env.LOADER.get(workerCacheKey, async () => {
3771
- const bundledCode = await loadDynamicWorkerBundledCode({
3772
- env,
3773
- metadata,
3774
- graphHash,
3775
- artifactStorageKey,
3776
- workerCacheKey,
3777
- runIdForTrace,
3778
- trace,
3779
- });
3780
- return {
3781
- compatibilityDate: DYNAMIC_WORKER_COMPATIBILITY_DATE,
3782
- compatibilityFlags: ['nodejs_compat'],
3783
- mainModule: 'index.js',
3784
- modules: { 'index.js': bundledCode },
3785
- env: {
3786
- // Mirror of the sync loader (above) — see that copy for the
3787
- // architectural rationale. The dynamic worker env is intentionally
3788
- // minimal; runtime callbacks use RUNTIME_API, file reads go through
3789
- // HARNESS, and child workflow control uses the COORDINATOR binding.
3790
- HARNESS: env.HARNESS,
3791
- VERCEL_PROTECTION_BYPASS_TOKEN: env.VERCEL_PROTECTION_BYPASS_TOKEN,
3792
- ...makeRuntimeApiEnvBinding(env),
3793
- ...makeCoordinatorControlBinding(),
3794
- },
3795
- };
3796
- });
3797
- trace({
3798
- runId: runIdForTrace,
3799
- phase: 'coordinator.loader_get_call',
3800
- ms: Date.now() - loaderGetStartedAt,
3801
- graphHash,
3802
- extra: { workerCacheKey },
3803
- });
3804
- return stub;
3805
- }
3806
-
3807
- async function loadDynamicWorkerBundledCode(input: {
3808
- env: CoordinatorEnv;
3809
- metadata: DynamicWorkflowMetadata;
3810
- graphHash: string;
3811
- artifactStorageKey: string;
3812
- workerCacheKey: string;
3813
- runIdForTrace: string;
3814
- trace: CoordinatorPerfTraceSink;
3815
- }): Promise<string> {
3816
- const callbackStartedAt = Date.now();
3817
- let codeSource: 'inline' | 'r2' | 'memory' = 'inline';
3818
- let r2Ms = 0;
3819
- const codeCacheKey = dynamicWorkerBundledCodeCacheKey({
3820
- artifactStorageKey: input.artifactStorageKey,
3821
- artifactHash: input.metadata.artifactHash,
3822
- });
3823
- let bundledCode = input.metadata.dynamicWorkerCode ?? null;
3824
- if (!bundledCode) {
3825
- bundledCode = readDynamicWorkerBundledCodeCache(codeCacheKey);
3826
- if (bundledCode) {
3827
- codeSource = 'memory';
3828
- }
3829
- }
3830
- if (!bundledCode) {
3831
- codeSource = 'r2';
3832
- const r2StartedAt = Date.now();
3833
- try {
3834
- const artifact = await loadStoredPlayArtifactFromR2(
3835
- input.env,
3836
- input.artifactStorageKey,
3837
- );
3838
- bundledCode = artifact?.artifact?.bundledCode ?? null;
3839
- if (typeof bundledCode === 'string' && bundledCode.length > 0) {
3840
- writeDynamicWorkerBundledCodeCache(codeCacheKey, bundledCode);
3841
- }
3842
- } finally {
3843
- r2Ms = Date.now() - r2StartedAt;
3844
- input.trace({
3845
- runId: input.runIdForTrace,
3846
- phase: 'coordinator.loader_code_r2_get',
3847
- ms: r2Ms,
3848
- graphHash: input.graphHash,
3849
- extra: { artifactStorageKey: input.artifactStorageKey },
3850
- });
3851
- }
3852
- }
3853
- if (typeof bundledCode !== 'string' || bundledCode.length === 0) {
3854
- throw new Error(
3855
- `Stored play artifact ${input.artifactStorageKey} does not contain bundledCode.`,
3856
- );
3857
- }
3858
- input.trace({
3859
- runId: input.runIdForTrace,
3860
- phase: 'coordinator.loader_code_callback',
3861
- ms: Date.now() - callbackStartedAt,
3862
- graphHash: input.graphHash,
3863
- extra: {
3864
- codeSource,
3865
- r2Ms,
3866
- workerCacheKey: input.workerCacheKey,
3867
- bundledCodeBytes: bundledCode.length,
3868
- },
3869
- });
3870
- return bundledCode;
3871
- }
3872
-
3873
- function normalizePackagedFiles(
3874
- value: unknown,
3875
- ): NonNullable<DynamicWorkflowMetadata['packagedFiles']> {
3876
- if (!Array.isArray(value)) return [];
3877
- return value
3878
- .filter((entry): entry is Record<string, unknown> =>
3879
- Boolean(entry && typeof entry === 'object' && !Array.isArray(entry)),
3880
- )
3881
- .map((entry) => ({
3882
- playPath: String(entry.playPath ?? '').replace(/^\.\//, ''),
3883
- storageKey: String(entry.storageKey ?? ''),
3884
- contentType:
3885
- typeof entry.contentType === 'string' ? entry.contentType : undefined,
3886
- bytes:
3887
- typeof entry.bytes === 'number' &&
3888
- Number.isSafeInteger(entry.bytes) &&
3889
- entry.bytes >= 0
3890
- ? entry.bytes
3891
- : undefined,
3892
- inlineText:
3893
- typeof entry.inlineText === 'string' ? entry.inlineText : undefined,
3894
- }))
3895
- .filter(
3896
- (entry) => entry.playPath.length > 0 && entry.storageKey.length > 0,
3897
- );
3898
- }
3899
-
3900
- const COORDINATOR_WARMUP_DYNAMIC_WORKER_CODE = `
3901
- import { WorkflowEntrypoint } from "cloudflare:workers";
3902
-
3903
- export class TenantWorkflow extends WorkflowEntrypoint {
3904
- async run(event) {
3905
- const payload = event && typeof event === "object" ? event.payload : null;
3906
- const runId = payload && typeof payload.runId === "string" ? payload.runId : "warmup";
3907
- const startedAt = Date.now();
3908
- if (this.env.RUNTIME_API) {
3909
- await this.env.RUNTIME_API.fetch(new Request("https://deepline.runtime.internal${PLAY_RUNTIME_API_COMPAT_PATH}", {
3910
- method: "POST",
3911
- headers: { "content-type": "application/json" },
3912
- body: "{}"
3913
- })).catch(() => null);
3914
- }
3915
- console.log("[perf-trace] " + JSON.stringify({
3916
- telemetrySchemaVersion: 1,
3917
- timestamp: new Date().toISOString(),
3918
- service: "play-runtime",
3919
- component: "coordinator-warmup",
3920
- event: "dynamic_worker.warmup_run",
3921
- tag: "[perf-trace]",
3922
- ts: Date.now(),
3923
- source: "dynamic_worker",
3924
- runId,
3925
- phase: "dynamic_worker.warmup_run",
3926
- ms: Date.now() - startedAt,
3927
- graphHash: "coordinator-warmup-v1",
3928
- telemetryKind: "log",
3929
- telemetryLevel: "info"
3930
- }));
3931
- return { ok: true, warmup: true };
3932
- }
3933
- }
3934
-
3935
- export default {
3936
- async fetch(request, env) {
3937
- const url = new URL(request.url);
3938
- if (request.method === "POST" && url.pathname === "/start") {
3939
- const body = await request.json().catch(() => null);
3940
- if (!body || typeof body.id !== "string" || !body.payload || typeof body.payload !== "object") {
3941
- return new Response("invalid workflow start body", { status: 400 });
3942
- }
3943
- const createStartedAt = Date.now();
3944
- const instance = await env.WORKFLOWS.create({ id: body.id, params: body.payload });
3945
- const workflowCreateMs = Date.now() - createStartedAt;
3946
- return Response.json({
3947
- id: instance.id,
3948
- status: "submitted",
3949
- timingsMs: { workflowCreate: workflowCreateMs }
3950
- });
3951
- }
3952
- if (request.method === "GET" && url.pathname === "/health") {
3953
- return new Response("ok", { status: 200 });
3954
- }
3955
- return new Response("not found", { status: 404 });
3956
- }
3957
- };
3958
- `;
3959
-
3960
- function isAllowedStagedFileStorageKey(key: string): boolean {
3961
- if (!key || key.length > 2_048) return false;
3962
- if (key.startsWith('/') || key.includes('..') || key.includes('\\')) {
3963
- return false;
3964
- }
3965
- return key.startsWith('plays/v2/orgs/') || key.includes('/plays/v2/orgs/');
3966
- }
3967
-
3968
- function decodeBase64ToUint8Array(value: string): Uint8Array {
3969
- const binary = atob(value);
3970
- const bytes = new Uint8Array(binary.length);
3971
- for (let index = 0; index < binary.length; index += 1) {
3972
- bytes[index] = binary.charCodeAt(index);
3973
- }
3974
- return bytes;
3975
- }
3976
-
3977
- async function handleStagedFilePut(
3978
- request: Request,
3979
- env: CoordinatorEnv,
3980
- ): Promise<Response> {
3981
- if (request.method !== 'POST') {
3982
- return new Response('method not allowed', { status: 405 });
3983
- }
3984
- const url = new URL(request.url);
3985
- const rawKey = url.searchParams.get('key')?.trim() ?? '';
3986
- if (rawKey) {
3987
- const key = rawKey;
3988
- const contentType =
3989
- url.searchParams.get('contentType')?.trim() || 'application/octet-stream';
3990
- const expectedBytes = Number(url.searchParams.get('bytes') ?? 'NaN');
3991
- if (!isAllowedStagedFileStorageKey(key)) {
3992
- return Response.json(
3993
- { error: 'invalid staged file key' },
3994
- { status: 400 },
3995
- );
3996
- }
3997
- if (!Number.isSafeInteger(expectedBytes) || expectedBytes < 0) {
3998
- return Response.json({ error: 'bytes is required' }, { status: 400 });
3999
- }
4000
- const existing = await headExistingStagedFile(env, key, expectedBytes);
4001
- if (existing.exists) {
4002
- recordCoordinatorEvent(
4003
- 'info',
4004
- 'coordinator.staged_file_put',
4005
- '[perf][coordinator.staged_file_put]',
4006
- {
4007
- key,
4008
- bytes: expectedBytes,
4009
- headMs: existing.ms,
4010
- putMs: 0,
4011
- ms: existing.ms,
4012
- transport: 'raw',
4013
- skipped: true,
4014
- },
4015
- );
4016
- return Response.json({
4017
- ok: true,
4018
- key,
4019
- bytes: expectedBytes,
4020
- existed: true,
4021
- timingsMs: { head: existing.ms, put: 0 },
4022
- });
4023
- }
4024
- const readStartedAt = Date.now();
4025
- const bytes = new Uint8Array(await request.arrayBuffer());
4026
- const readMs = Date.now() - readStartedAt;
4027
- if (bytes.byteLength !== expectedBytes) {
4028
- return Response.json(
4029
- { error: 'staged file byte length mismatch' },
4030
- { status: 400 },
4031
- );
4032
- }
4033
- const putStartedAt = Date.now();
4034
- await env.PLAYS_BUCKET.put(key, bytes, {
4035
- httpMetadata: { contentType },
4036
- });
4037
- const putMs = Date.now() - putStartedAt;
4038
- recordCoordinatorEvent(
4039
- 'info',
4040
- 'coordinator.staged_file_put',
4041
- '[perf][coordinator.staged_file_put]',
4042
- {
4043
- key,
4044
- bytes: bytes.byteLength,
4045
- readMs,
4046
- putMs,
4047
- ms: readMs + putMs,
4048
- transport: 'raw',
4049
- },
4050
- );
4051
- return Response.json({
4052
- ok: true,
4053
- key,
4054
- bytes: bytes.byteLength,
4055
- timingsMs: { read: readMs, put: putMs },
4056
- });
4057
- }
4058
- const body = (await request.json().catch(() => null)) as {
4059
- key?: unknown;
4060
- contentBase64?: unknown;
4061
- contentType?: unknown;
4062
- bytes?: unknown;
4063
- } | null;
4064
- const key = typeof body?.key === 'string' ? body.key.trim() : '';
4065
- const contentBase64 =
4066
- typeof body?.contentBase64 === 'string' ? body.contentBase64 : '';
4067
- const contentType =
4068
- typeof body?.contentType === 'string' && body.contentType.trim()
4069
- ? body.contentType.trim()
4070
- : 'application/octet-stream';
4071
- const expectedBytes = typeof body?.bytes === 'number' ? body.bytes : NaN;
4072
- if (!isAllowedStagedFileStorageKey(key)) {
4073
- return Response.json({ error: 'invalid staged file key' }, { status: 400 });
4074
- }
4075
- if (
4076
- !contentBase64 ||
4077
- !Number.isSafeInteger(expectedBytes) ||
4078
- expectedBytes < 0
4079
- ) {
4080
- return Response.json(
4081
- { error: 'contentBase64 and bytes are required' },
4082
- { status: 400 },
4083
- );
4084
- }
4085
- const existing = await headExistingStagedFile(env, key, expectedBytes);
4086
- if (existing.exists) {
4087
- recordCoordinatorEvent(
4088
- 'info',
4089
- 'coordinator.staged_file_put',
4090
- '[perf][coordinator.staged_file_put]',
4091
- {
4092
- key,
4093
- bytes: expectedBytes,
4094
- headMs: existing.ms,
4095
- putMs: 0,
4096
- ms: existing.ms,
4097
- transport: 'base64',
4098
- skipped: true,
4099
- },
4100
- );
4101
- return Response.json({
4102
- ok: true,
4103
- key,
4104
- bytes: expectedBytes,
4105
- existed: true,
4106
- timingsMs: { head: existing.ms, put: 0 },
4107
- });
4108
- }
4109
- const decodeStartedAt = Date.now();
4110
- const bytes = decodeBase64ToUint8Array(contentBase64);
4111
- const decodeMs = Date.now() - decodeStartedAt;
4112
- if (bytes.byteLength !== expectedBytes) {
4113
- return Response.json(
4114
- { error: 'staged file byte length mismatch' },
4115
- { status: 400 },
4116
- );
4117
- }
4118
- const putStartedAt = Date.now();
4119
- await env.PLAYS_BUCKET.put(key, bytes, {
4120
- httpMetadata: { contentType },
4121
- });
4122
- const putMs = Date.now() - putStartedAt;
4123
- recordCoordinatorEvent(
4124
- 'info',
4125
- 'coordinator.staged_file_put',
4126
- '[perf][coordinator.staged_file_put]',
4127
- {
4128
- key,
4129
- bytes: bytes.byteLength,
4130
- decodeMs,
4131
- putMs,
4132
- ms: decodeMs + putMs,
4133
- },
4134
- );
4135
- return Response.json({
4136
- ok: true,
4137
- key,
4138
- bytes: bytes.byteLength,
4139
- timingsMs: { decode: decodeMs, put: putMs },
4140
- });
4141
- }
4142
-
4143
- async function headExistingStagedFile(
4144
- env: CoordinatorEnv,
4145
- key: string,
4146
- expectedBytes: number,
4147
- ): Promise<{ exists: boolean; ms: number }> {
4148
- const startedAt = Date.now();
4149
- const object = await env.PLAYS_BUCKET.head(key).catch(() => null);
4150
- const ms = Date.now() - startedAt;
4151
- if (!object) {
4152
- return { exists: false, ms };
4153
- }
4154
- if (typeof object.size === 'number' && object.size !== expectedBytes) {
4155
- return { exists: false, ms };
4156
- }
4157
- return { exists: true, ms };
4158
- }
4159
-
4160
- async function handleCoordinatorWarmup(
4161
- request: Request,
4162
- env: CoordinatorEnv,
4163
- ctx?: ExecutionContext,
4164
- ): Promise<Response> {
4165
- if (request.method !== 'POST') {
4166
- return new Response('method not allowed', { status: 405 });
4167
- }
4168
- const warmupToken = env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim();
4169
- if (
4170
- warmupToken &&
4171
- request.headers.get('x-vercel-protection-bypass') !== warmupToken
4172
- ) {
4173
- return new Response('unauthorized', { status: 401 });
4174
- }
4175
- const url = new URL(request.url);
4176
- const label = url.searchParams.get('label')?.trim() || 'warmup';
4177
- const runId = `coordinator/warmup/${label}/${Date.now().toString(36)}`;
4178
- const artifactHash = `coordinator-warmup-${stableHash(COORDINATOR_WARMUP_DYNAMIC_WORKER_CODE)}`;
4179
- const artifactStorageKey = `coordinator-warmup/${artifactHash}.json`;
4180
- await env.PLAYS_BUCKET.put(
4181
- artifactStorageKey,
4182
- JSON.stringify({
4183
- artifact: {
4184
- artifactKind: 'esm_workers',
4185
- bundledCode: COORDINATOR_WARMUP_DYNAMIC_WORKER_CODE,
4186
- },
4187
- } satisfies StoredPlayArtifactPayload),
4188
- {
4189
- httpMetadata: {
4190
- contentType: 'application/json',
4191
- },
4192
- },
4193
- );
4194
- const params: PlayWorkflowParams = {
4195
- runId,
4196
- playId: runId,
4197
- playName: 'coordinator-warmup',
4198
- artifactStorageKey,
4199
- artifactHash,
4200
- graphHash: artifactHash,
4201
- input: {},
4202
- inputFile: null,
4203
- inlineCsv: null,
4204
- packagedFiles: null,
4205
- contractSnapshot: null,
4206
- executionPlan: null,
4207
- dynamicWorkerCode: null,
4208
- executorToken: 'coordinator-warmup',
4209
- baseUrl: env.DEEPLINE_API_BASE_URL,
4210
- orgId: 'coordinator-warmup',
4211
- userEmail: 'coordinator-warmup@deepline.local',
4212
- userId: 'coordinator-warmup',
4213
- runtimeBackend: 'cf_workflows_dynamic_worker',
4214
- dedupBackend: 'durable_object',
4215
- };
4216
- const startedAt = Date.now();
4217
- const response = await handleWorkflowRoute({
4218
- runId,
4219
- action: 'submit',
4220
- request: new Request(
4221
- `https://deepline.coordinator.internal/workflow/${encodeURIComponent(runId)}/submit`,
4222
- {
4223
- method: 'POST',
4224
- headers: {
4225
- 'content-type': 'application/json',
4226
- [COORDINATOR_INTERNAL_TOKEN_HEADER]:
4227
- env.DEEPLINE_INTERNAL_TOKEN?.trim() ?? '',
4228
- },
4229
- body: JSON.stringify(params),
4230
- },
4231
- ),
4232
- env,
4233
- });
4234
- const text = await response.text().catch(() => '');
4235
- const terminalWaitStartedAt = Date.now();
4236
- let terminalState: Record<string, unknown> | null = null;
4237
- if (response.ok) {
4238
- const instance = await env.PLAY_WORKFLOW.get(
4239
- await resolveWorkflowInstanceIdForRun(env, runId),
4240
- );
4241
- try {
4242
- while (Date.now() - terminalWaitStartedAt < 10_000) {
4243
- const status = await instance.status();
4244
- terminalState = mapWorkflowResult(runId, status);
4245
- const mappedStatus = String(terminalState.status ?? '');
4246
- if (
4247
- mappedStatus === 'completed' ||
4248
- mappedStatus === 'failed' ||
4249
- mappedStatus === 'cancelled'
4250
- ) {
4251
- break;
4252
- }
4253
- await new Promise((resolve) => setTimeout(resolve, 100));
4254
- }
4255
- } finally {
4256
- disposeRpcStub(instance);
4257
- }
4258
- }
4259
- recordCoordinatorPerfTrace({
4260
- runId,
4261
- phase: 'coordinator.warmup_terminal_wait',
4262
- ms: Date.now() - terminalWaitStartedAt,
4263
- graphHash: params.graphHash,
4264
- extra: { status: terminalState?.status ?? null, label },
4265
- });
4266
- recordCoordinatorPerfTrace({
4267
- runId,
4268
- phase: 'coordinator.warmup_submit_total',
4269
- ms: Date.now() - startedAt,
4270
- graphHash: params.graphHash,
4271
- extra: { status: response.status, label },
4272
- });
4273
- let body: unknown = null;
4274
- try {
4275
- body = text ? JSON.parse(text) : null;
4276
- } catch {
4277
- body = text;
4278
- }
4279
- const terminalStatus = String(terminalState?.status ?? '');
4280
- const warmupCompleted = response.ok && terminalStatus === 'completed';
4281
- const responseStatus = warmupCompleted
4282
- ? 200
4283
- : terminalStatus === 'running' || terminalStatus === 'sleeping'
4284
- ? 504
4285
- : response.ok
4286
- ? 500
4287
- : response.status;
4288
- return Response.json(
4289
- {
4290
- ok: warmupCompleted,
4291
- runId,
4292
- status: response.status,
4293
- body,
4294
- terminalState,
4295
- },
4296
- { status: responseStatus },
4297
- );
4298
- }
4299
-
4300
- /**
4301
- * Returns a structured-cloneable runtime API binding for the per-graphHash
4302
- * play Worker's `env.RUNTIME_API`. We intentionally pass the long-lived
4303
- * HARNESS WorkerEntrypoint service binding instead of a plain closure: the
4304
- * dynamic Worker env is serialized by Cloudflare Workflows, and service
4305
- * bindings are cloneable while closures are not. The per-play runtime accepts
4306
- * this binding via `runtimeApiCall(...)`, so callbacks still stay on
4307
- * service bindings and never fall back to public HTTP.
4308
- */
4309
- let loggedMissingCoordinatorControlExport = false;
4310
-
4311
- function makeRuntimeApiEnvBinding(env: CoordinatorEnv): {
4312
- RUNTIME_API: CoordinatorEnv['HARNESS'];
4313
- } {
4314
- return { RUNTIME_API: env.HARNESS };
4315
- }
4316
-
4317
- function makeCoordinatorControlBinding():
4318
- | {
4319
- COORDINATOR: {
4320
- signal(
4321
- runId: string,
4322
- body: Record<string, unknown>,
4323
- ): Promise<Record<string, unknown>>;
4324
- recordPerfTrace(
4325
- runId: string,
4326
- payload: CoordinatorPerfTracePayload,
4327
- ): Promise<void>;
4328
- recordRunEvent(
4329
- runId: string,
4330
- event: CoordinatorRunEvent,
4331
- ): Promise<void>;
4332
- readTerminalState(
4333
- runId: string,
4334
- ): Promise<CoordinatorTerminalState | null>;
4335
- rateAcquire(input: {
4336
- bucketId: string;
4337
- rules: Array<{
4338
- ruleId: string;
4339
- requestsPerWindow: number;
4340
- windowMs: number;
4341
- maxConcurrency: number | null;
4342
- }>;
4343
- requested: number;
4344
- }): Promise<{ granted: number; waitMs: number }>;
4345
- ratePenalize(input: {
4346
- bucketId: string;
4347
- cooldownMs: number;
4348
- }): Promise<void>;
4349
- };
4350
- }
4351
- | Record<string, never> {
4352
- const exports = workersExports as unknown as {
4353
- CoordinatorControl?: (init: { props: undefined }) => {
4354
- signal(
4355
- runId: string,
4356
- body: Record<string, unknown>,
4357
- ): Promise<Record<string, unknown>>;
4358
- recordPerfTrace(
4359
- runId: string,
4360
- payload: CoordinatorPerfTracePayload,
4361
- ): Promise<void>;
4362
- recordRunEvent(runId: string, event: CoordinatorRunEvent): Promise<void>;
4363
- readTerminalState(
4364
- runId: string,
4365
- ): Promise<CoordinatorTerminalState | null>;
4366
- rateAcquire(input: {
4367
- bucketId: string;
4368
- rules: Array<{
4369
- ruleId: string;
4370
- requestsPerWindow: number;
4371
- windowMs: number;
4372
- maxConcurrency: number | null;
4373
- }>;
4374
- requested: number;
4375
- }): Promise<{ granted: number; waitMs: number }>;
4376
- ratePenalize(input: {
4377
- bucketId: string;
4378
- cooldownMs: number;
4379
- }): Promise<void>;
4380
- };
4381
- };
4382
- const ctor = exports.CoordinatorControl;
4383
- if (typeof ctor !== 'function') {
4384
- if (!loggedMissingCoordinatorControlExport) {
4385
- loggedMissingCoordinatorControlExport = true;
4386
- recordCoordinatorEvent(
4387
- 'warn',
4388
- 'coordinator.control_export.missing',
4389
- '[coordinator] CoordinatorControl is not registered on cloudflare:workers exports; using public coordinator transport.',
4390
- );
4391
- }
4392
- return {};
4393
- }
4394
- return { COORDINATOR: ctor({ props: undefined }) };
4395
- }
4396
-
4397
- async function loadStoredPlayArtifactFromR2(
4398
- env: CoordinatorEnv,
4399
- storageKey: string,
4400
- ): Promise<StoredPlayArtifactPayload> {
4401
- const object = await env.PLAYS_BUCKET.get(storageKey);
4402
- if (!object) {
4403
- throw new Error(`Stored play artifact missing from R2: ${storageKey}`);
4404
- }
4405
- return JSON.parse(await object.text()) as StoredPlayArtifactPayload;
4406
- }
4407
-
4408
- function readMetadataString(
4409
- metadata: Record<string, unknown>,
4410
- key: keyof DynamicWorkflowMetadata,
4411
- ): string {
4412
- const value = metadata[key];
4413
- if (typeof value !== 'string' || !value.trim()) {
4414
- throw new Error(`Dynamic workflow metadata missing ${key}.`);
4415
- }
4416
- return value;
4417
- }
4418
-
4419
- function mapWorkflowResult(
4420
- runId: string,
4421
- status: InstanceStatus,
4422
- ): Record<string, unknown> {
4423
- const rawError = readWorkflowError(status);
4424
- const mapped = resolveTerminalStatus(status, rawError);
4425
- const error =
4426
- rawError ??
4427
- (mapped === 'failed'
4428
- ? `Cloudflare workflow reported ${String(status.status ?? 'failed')} without an error payload.`
4429
- : null);
4430
- const output =
4431
- status.output && typeof status.output === 'object'
4432
- ? (status.output as Record<string, unknown>)
4433
- : null;
4434
- return {
4435
- runId,
4436
- ...(typeof output?.playName === 'string' && output.playName.trim()
4437
- ? { playName: output.playName.trim() }
4438
- : {}),
4439
- status: mapped,
4440
- result: output?.result ?? status.output ?? null,
4441
- error,
4442
- totalRows: output?.totalRows ?? output?.outputRows ?? null,
4443
- durationMs: output?.durationMs ?? null,
4444
- liveLogs: sanitizeLiveLogLines(output?.liveLogs),
4445
- liveNodeProgress: output?.liveNodeProgress ?? null,
4446
- wait:
4447
- mapped === 'sleeping'
4448
- ? {
4449
- kind: 'integration_event_batch',
4450
- }
4451
- : null,
4452
- };
4453
- }
4454
-
4455
- function mapWorkflowStatus(
4456
- status: InstanceStatus,
4457
- ): 'running' | 'sleeping' | 'completed' | 'failed' | 'cancelled' {
4458
- if (status.status === 'complete') return 'completed';
4459
- if (status.status === 'errored') return 'failed';
4460
- if (status.status === 'terminated') return 'cancelled';
4461
- if (status.status === 'waiting' || status.status === 'paused')
4462
- return 'sleeping';
4463
- return 'running';
4464
- }
4465
-
4466
- /**
4467
- * CF Dynamic Workflows retries failed workflow bodies silently for ~3
4468
- * minutes before transitioning the instance to `errored`. During that
4469
- * retry window `status.status` stays `running`/`paused`/`waiting`, but
4470
- * **`status.error`** gets populated as soon as the first attempt fails —
4471
- * with CF's generic `"internal error; reference = <id>"` (they never
4472
- * surface the actual JS error message via this API). That's enough
4473
- * signal: any error presence on a non-terminal instance means at least
4474
- * one attempt has thrown, and the user is better served by failing fast
4475
- * than by waiting 3 minutes for CF's retry budget to exhaust.
4476
- *
4477
- * If the failure was actually transient and the next retry would have
4478
- * succeeded, the user can re-run. The previous "match on permanent-
4479
- * error patterns" approach didn't help because the error message we
4480
- * see from CF is never a JS error name — it's always the generic
4481
- * reference id.
4482
- */
4483
- function resolveTerminalStatus(
4484
- status: InstanceStatus,
4485
- error: string | null,
4486
- ): 'running' | 'sleeping' | 'completed' | 'failed' | 'cancelled' {
4487
- const mapped = mapWorkflowStatus(status);
4488
- if ((mapped === 'running' || mapped === 'sleeping') && error) {
4489
- return 'failed';
4490
- }
4491
- return mapped;
4492
- }
4493
-
4494
- function readWorkflowError(status: InstanceStatus): string | null {
4495
- const error = status.error as unknown;
4496
- if (!error) return null;
4497
- if (typeof error === 'string') return error.trim() || null;
4498
- if (typeof error === 'object' && 'message' in error) {
4499
- const message = String((error as { message?: unknown }).message ?? '');
4500
- return message.trim() || null;
4501
- }
4502
- const message = String(error);
4503
- return message.trim() || null;
4504
- }