deepline 0.1.243 → 0.1.244
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +5 -25
- package/dist/bundling-sources/sdk/src/release.ts +23 -21
- package/dist/bundling-sources/sdk/src/types.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/backend.ts +2 -21
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -6
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +6 -13
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +9 -26
- package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +1 -2
- package/dist/bundling-sources/shared_libs/play-runtime/map-row-identity.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +191 -29
- package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +1 -36
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +0 -7
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +1 -12
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +5 -15
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +6 -21
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +3 -4
- package/dist/bundling-sources/shared_libs/plays/artifact-kind-guard.ts +16 -29
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +4 -8
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +21 -712
- package/dist/bundling-sources/shared_libs/plays/bundling/limits.ts +1 -18
- package/dist/cli/index.js +29 -71
- package/dist/cli/index.mjs +29 -71
- package/dist/{compiler-manifest-CYcwzSOJ.d.ts → compiler-manifest-DlE7dnRm.d.mts} +2 -8
- package/dist/{compiler-manifest-CYcwzSOJ.d.mts → compiler-manifest-DlE7dnRm.d.ts} +2 -8
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +24 -22
- package/dist/index.mjs +24 -22
- package/dist/plays/bundle-play-file.d.mts +7 -22
- package/dist/plays/bundle-play-file.d.ts +7 -22
- package/dist/plays/bundle-play-file.mjs +26 -614
- package/package.json +2 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +0 -4504
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +0 -1386
- package/dist/bundling-sources/apps/play-runner-workers/src/durable-object-deploy-handoff.ts +0 -30
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +0 -8385
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md +0 -21
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts +0 -251
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/coordinator-progress.ts +0 -289
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/csv-rows.ts +0 -85
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +0 -23
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/dataset-handles.ts +0 -528
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +0 -229
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/ledger-event-batches.ts +0 -81
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/live-progress.ts +0 -22
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +0 -920
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts +0 -90
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +0 -475
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +0 -570
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/result-dataset-persistence.ts +0 -50
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/row-isolation.ts +0 -5
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +0 -519
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-batch.ts +0 -107
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +0 -2541
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-http-errors.ts +0 -7
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +0 -227
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +0 -130
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +0 -39
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/workflow-preview.ts +0 -26
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime-api-retry.ts +0 -23
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-instance-create.ts +0 -47
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +0 -167
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +0 -91
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +0 -329
- package/dist/bundling-sources/sdk/src/worker-play-entry.ts +0 -240
- package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +0 -252
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +0 -217
|
@@ -1,4504 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Coordinator Worker entry — main module exported by wrangler.toml.
|
|
3
|
-
*
|
|
4
|
-
* Hosts:
|
|
5
|
-
* - DynamicWorkflow (native Cloudflare Dynamic Workflows)
|
|
6
|
-
* - PlayDedup DO (default export class)
|
|
7
|
-
* - HTTP fetch handler for /api/v2/plays/run dispatch (Vercel forwards to here)
|
|
8
|
-
* - WorkerLoader-backed dynamic play Workers
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
export { PlayDedup } from './dedup-do';
|
|
12
|
-
|
|
13
|
-
import {
|
|
14
|
-
WorkerEntrypoint,
|
|
15
|
-
WorkflowEntrypoint,
|
|
16
|
-
exports as workersExports,
|
|
17
|
-
} from 'cloudflare:workers';
|
|
18
|
-
import {
|
|
19
|
-
createWorkerTelemetry,
|
|
20
|
-
type TelemetryEvent,
|
|
21
|
-
type TelemetryFields,
|
|
22
|
-
type TelemetryScalar,
|
|
23
|
-
} from '../../../shared_libs/observability/worker-telemetry';
|
|
24
|
-
import { createSecretRedactionContext } from '../../../shared_libs/play-runtime/secret-redaction';
|
|
25
|
-
import {
|
|
26
|
-
_dispatcherBindingImpl as createDispatcherWorkflowBinding,
|
|
27
|
-
dispatchWorkflow,
|
|
28
|
-
DynamicWorkflowBinding,
|
|
29
|
-
type WorkflowRunner,
|
|
30
|
-
} from '@cloudflare/dynamic-workflows';
|
|
31
|
-
import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
|
|
32
|
-
import type { PlayRunInputPayload } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
33
|
-
import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runtime/test-runtime-seams';
|
|
34
|
-
import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
|
|
35
|
-
import {
|
|
36
|
-
DB_SESSION_DEFAULT_TTL_SECONDS,
|
|
37
|
-
type PreloadedRuntimeDbSession,
|
|
38
|
-
} from '../../../shared_libs/play-runtime/db-session';
|
|
39
|
-
import {
|
|
40
|
-
dbSessionPostgresUrlAad,
|
|
41
|
-
encryptDbSessionPostgresUrl,
|
|
42
|
-
} from '../../../shared_libs/play-runtime/db-session-crypto';
|
|
43
|
-
import { planRuntimeDbSessionRequirements } from '../../../shared_libs/play-runtime/db-session-plan';
|
|
44
|
-
import type { PlayRuntimeManifest } from '../../../shared_libs/plays/compiler-manifest';
|
|
45
|
-
import {
|
|
46
|
-
WORKSPACE_STORAGE_NOT_READY_CODE,
|
|
47
|
-
WORKSPACE_STORAGE_NOT_READY_MESSAGE,
|
|
48
|
-
normalizePlayRunFailure,
|
|
49
|
-
} from '../../../shared_libs/play-runtime/run-failure';
|
|
50
|
-
import type { PlayRunLedgerEvent } from '../../../shared_libs/play-runtime/run-ledger';
|
|
51
|
-
import {
|
|
52
|
-
COORDINATOR_INTERNAL_TOKEN_HEADER,
|
|
53
|
-
COORDINATOR_RUN_SCOPE_HEADER,
|
|
54
|
-
} from '../../../shared_libs/play-runtime/coordinator-headers';
|
|
55
|
-
import {
|
|
56
|
-
PLAY_RUNTIME_API_COMPAT_PATH,
|
|
57
|
-
PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH,
|
|
58
|
-
} from '../../../shared_libs/play-runtime/runtime-api-paths';
|
|
59
|
-
import {
|
|
60
|
-
decideWorkflowPlatformRetry,
|
|
61
|
-
PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
|
|
62
|
-
shouldPublishWorkflowRuntimeFailure,
|
|
63
|
-
} from './workflow-retry';
|
|
64
|
-
import {
|
|
65
|
-
WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES,
|
|
66
|
-
WORKFLOW_RETRY_PARAMS_MAX_BYTES,
|
|
67
|
-
buildWorkflowRetryParams,
|
|
68
|
-
jsonByteLength,
|
|
69
|
-
workflowRetryParamsStorageKey,
|
|
70
|
-
type WorkflowRetryParamsRef,
|
|
71
|
-
} from './workflow-retry-state';
|
|
72
|
-
import { createOrAttachWorkflowInstance } from './workflow-instance-create';
|
|
73
|
-
import { sanitizeLiveLogLines } from './runtime/live-progress';
|
|
74
|
-
import {
|
|
75
|
-
DURABLE_OBJECT_DEPLOY_HANDOFF_RETRY_DELAYS_MS,
|
|
76
|
-
durableObjectDeployHandoffMessage,
|
|
77
|
-
sleepForDurableObjectDeployHandoffRetry,
|
|
78
|
-
} from './durable-object-deploy-handoff';
|
|
79
|
-
|
|
80
|
-
export { DynamicWorkflowBinding };
|
|
81
|
-
|
|
82
|
-
const coordinatorTelemetry = createWorkerTelemetry({
|
|
83
|
-
service: 'play-runtime',
|
|
84
|
-
component: 'coordinator',
|
|
85
|
-
});
|
|
86
|
-
const coordinatorStorageRedactor = createSecretRedactionContext();
|
|
87
|
-
const coordinatorPerfTelemetry = createWorkerTelemetry({
|
|
88
|
-
service: 'play-runtime',
|
|
89
|
-
component: 'coordinator',
|
|
90
|
-
adapter: {
|
|
91
|
-
emit(event: TelemetryEvent) {
|
|
92
|
-
console.log(`[perf-trace] ${JSON.stringify(event)}`);
|
|
93
|
-
},
|
|
94
|
-
},
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
function coordinatorTelemetryFields(input: object): TelemetryFields {
|
|
98
|
-
const fields: Record<string, TelemetryScalar> = {};
|
|
99
|
-
for (const [key, value] of Object.entries(input)) {
|
|
100
|
-
if (
|
|
101
|
-
value === undefined ||
|
|
102
|
-
value === null ||
|
|
103
|
-
typeof value === 'string' ||
|
|
104
|
-
typeof value === 'number' ||
|
|
105
|
-
typeof value === 'boolean'
|
|
106
|
-
) {
|
|
107
|
-
fields[key] = value;
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
return fields;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
function recordCoordinatorEvent(
|
|
114
|
-
level: 'info' | 'warn' | 'error',
|
|
115
|
-
event: string,
|
|
116
|
-
tag: string,
|
|
117
|
-
input: Record<string, unknown> = {},
|
|
118
|
-
): void {
|
|
119
|
-
const telemetry = coordinatorTelemetry.child({
|
|
120
|
-
context: {
|
|
121
|
-
runId: typeof input.runId === 'string' ? input.runId : null,
|
|
122
|
-
orgId: typeof input.orgId === 'string' ? input.orgId : null,
|
|
123
|
-
playName: typeof input.playName === 'string' ? input.playName : null,
|
|
124
|
-
requestId: typeof input.requestId === 'string' ? input.requestId : null,
|
|
125
|
-
},
|
|
126
|
-
});
|
|
127
|
-
const fields = coordinatorTelemetryFields(input);
|
|
128
|
-
if (level === 'error') {
|
|
129
|
-
void telemetry.error(event, input.error ?? input.message, fields, { tag });
|
|
130
|
-
} else {
|
|
131
|
-
void telemetry[level](event, fields, { tag });
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
export type PlayWorkflowParams = {
|
|
136
|
-
runId: string;
|
|
137
|
-
playId: string;
|
|
138
|
-
playName: string;
|
|
139
|
-
artifactStorageKey: string;
|
|
140
|
-
artifactHash: string;
|
|
141
|
-
graphHash: string;
|
|
142
|
-
input: PlayRunInputPayload;
|
|
143
|
-
force?: boolean;
|
|
144
|
-
runAttempt?: number;
|
|
145
|
-
inputFile?: {
|
|
146
|
-
name?: string;
|
|
147
|
-
r2Key?: string;
|
|
148
|
-
storageKey?: string;
|
|
149
|
-
path?: string;
|
|
150
|
-
fileName?: string;
|
|
151
|
-
logicalPath?: string;
|
|
152
|
-
contentType?: string;
|
|
153
|
-
bytes?: number;
|
|
154
|
-
} | null;
|
|
155
|
-
inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
|
|
156
|
-
packagedFiles?: Array<{
|
|
157
|
-
playPath: string;
|
|
158
|
-
storageKey: string;
|
|
159
|
-
contentType?: string;
|
|
160
|
-
bytes?: number;
|
|
161
|
-
inlineText?: string;
|
|
162
|
-
}> | null;
|
|
163
|
-
contractSnapshot?: unknown;
|
|
164
|
-
executionPlan?: ExecutionPlan | null;
|
|
165
|
-
forceToolRefresh?: boolean | null;
|
|
166
|
-
preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
|
|
167
|
-
preloadedDbSessionRef?: {
|
|
168
|
-
runId: string;
|
|
169
|
-
sessionCount: number;
|
|
170
|
-
expiresAt: number;
|
|
171
|
-
} | null;
|
|
172
|
-
dynamicWorkerCode?: string | null;
|
|
173
|
-
executorToken: string;
|
|
174
|
-
baseUrl: string;
|
|
175
|
-
integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
|
|
176
|
-
orgId: string;
|
|
177
|
-
userEmail: string;
|
|
178
|
-
userId?: string | null;
|
|
179
|
-
runtimeBackend: string;
|
|
180
|
-
dedupBackend: string;
|
|
181
|
-
totalRows?: number;
|
|
182
|
-
coordinatorUrl?: string | null;
|
|
183
|
-
coordinatorInternalToken?: string | null;
|
|
184
|
-
runtimeTestFaultHeader?: string | null;
|
|
185
|
-
testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
|
|
186
|
-
submittedAt?: number | null;
|
|
187
|
-
};
|
|
188
|
-
|
|
189
|
-
type CoordinatorTiming = {
|
|
190
|
-
phase: string;
|
|
191
|
-
ms: number;
|
|
192
|
-
graphHash?: string | null;
|
|
193
|
-
extra?: Record<string, unknown>;
|
|
194
|
-
};
|
|
195
|
-
|
|
196
|
-
type CoordinatorPerfTracePayload = {
|
|
197
|
-
ts: number;
|
|
198
|
-
source: 'coordinator' | 'dynamic_worker';
|
|
199
|
-
runId: string;
|
|
200
|
-
phase: string;
|
|
201
|
-
ms: number;
|
|
202
|
-
graphHash?: string | null;
|
|
203
|
-
[key: string]: unknown;
|
|
204
|
-
};
|
|
205
|
-
|
|
206
|
-
type CoordinatorPerfTraceInput = {
|
|
207
|
-
runId: string;
|
|
208
|
-
phase: string;
|
|
209
|
-
ms: number;
|
|
210
|
-
graphHash?: string | null;
|
|
211
|
-
extra?: Record<string, unknown>;
|
|
212
|
-
};
|
|
213
|
-
|
|
214
|
-
type CoordinatorPerfTraceSink = (event: CoordinatorPerfTraceInput) => void;
|
|
215
|
-
|
|
216
|
-
type CoordinatorTerminalState = {
|
|
217
|
-
runId: string;
|
|
218
|
-
status: 'completed' | 'failed' | 'cancelled';
|
|
219
|
-
result?: unknown;
|
|
220
|
-
error?: string | null;
|
|
221
|
-
totalRows?: unknown;
|
|
222
|
-
durationMs?: unknown;
|
|
223
|
-
playName?: string | null;
|
|
224
|
-
liveLogs?: string[] | null;
|
|
225
|
-
liveNodeProgress?: unknown;
|
|
226
|
-
completedAt?: number;
|
|
227
|
-
};
|
|
228
|
-
|
|
229
|
-
type CoordinatorRunEvent =
|
|
230
|
-
| {
|
|
231
|
-
seq?: number;
|
|
232
|
-
runId: string;
|
|
233
|
-
type: 'status';
|
|
234
|
-
status: string;
|
|
235
|
-
ts: number;
|
|
236
|
-
logs?: string[];
|
|
237
|
-
}
|
|
238
|
-
| {
|
|
239
|
-
seq?: number;
|
|
240
|
-
runId: string;
|
|
241
|
-
type: 'log';
|
|
242
|
-
line: string;
|
|
243
|
-
ts: number;
|
|
244
|
-
}
|
|
245
|
-
| {
|
|
246
|
-
seq?: number;
|
|
247
|
-
runId: string;
|
|
248
|
-
type: 'progress';
|
|
249
|
-
status: string;
|
|
250
|
-
ts: number;
|
|
251
|
-
logs?: string[];
|
|
252
|
-
activeNodeId?: string | null;
|
|
253
|
-
activeArtifactTableNamespace?: string | null;
|
|
254
|
-
updatedAt?: number | null;
|
|
255
|
-
liveNodeProgress?: unknown;
|
|
256
|
-
}
|
|
257
|
-
| {
|
|
258
|
-
seq?: number;
|
|
259
|
-
runId: string;
|
|
260
|
-
type: 'terminal';
|
|
261
|
-
status: 'completed' | 'failed' | 'cancelled';
|
|
262
|
-
ts: number;
|
|
263
|
-
result?: unknown;
|
|
264
|
-
error?: string | null;
|
|
265
|
-
totalRows?: unknown;
|
|
266
|
-
durationMs?: unknown;
|
|
267
|
-
playName?: string | null;
|
|
268
|
-
liveLogs?: string[] | null;
|
|
269
|
-
liveNodeProgress?: unknown;
|
|
270
|
-
};
|
|
271
|
-
|
|
272
|
-
type InlineWorkerRunResponse = {
|
|
273
|
-
status?: 'completed' | 'failed';
|
|
274
|
-
result?: unknown;
|
|
275
|
-
outputRows?: number;
|
|
276
|
-
durationMs?: number;
|
|
277
|
-
parseMs?: number;
|
|
278
|
-
timings?: Array<{
|
|
279
|
-
phase?: unknown;
|
|
280
|
-
ms?: unknown;
|
|
281
|
-
extra?: unknown;
|
|
282
|
-
}>;
|
|
283
|
-
events?: Array<
|
|
284
|
-
| { type: 'log'; message?: string; level?: string; ts?: number }
|
|
285
|
-
| { type: 'result'; result?: unknown; outputRows?: number; ts?: number }
|
|
286
|
-
| { type: 'error'; message?: string; stack?: string; ts?: number }
|
|
287
|
-
| Record<string, unknown>
|
|
288
|
-
>;
|
|
289
|
-
error?: { message?: string; stack?: string | null };
|
|
290
|
-
};
|
|
291
|
-
|
|
292
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
293
|
-
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
interface CoordinatorEnv {
|
|
297
|
-
PLAY_WORKFLOW: Workflow<PlayWorkflowParams>;
|
|
298
|
-
PLAY_DEDUP: DurableObjectNamespace;
|
|
299
|
-
LOADER?: {
|
|
300
|
-
get(
|
|
301
|
-
id: string,
|
|
302
|
-
load: () => Promise<{
|
|
303
|
-
compatibilityDate: string;
|
|
304
|
-
compatibilityFlags?: string[];
|
|
305
|
-
mainModule: string;
|
|
306
|
-
modules: Record<string, string>;
|
|
307
|
-
env?: Record<string, unknown>;
|
|
308
|
-
globalOutbound?: Fetcher | null;
|
|
309
|
-
}>,
|
|
310
|
-
): {
|
|
311
|
-
getEntrypoint(name?: string): Fetcher & WorkflowRunner;
|
|
312
|
-
};
|
|
313
|
-
};
|
|
314
|
-
PLAYS_BUCKET: R2Bucket;
|
|
315
|
-
DEEPLINE_API_BASE_URL: string;
|
|
316
|
-
DEEPLINE_INTERNAL_TOKEN?: string;
|
|
317
|
-
DEEPLINE_TAIL_LOG_TOKEN?: string;
|
|
318
|
-
DEEPLINE_COORDINATOR_DEPLOY_MARKER?: string;
|
|
319
|
-
CF_VERSION_METADATA?: WorkerVersionMetadata;
|
|
320
|
-
VERCEL_PROTECTION_BYPASS_TOKEN?: string;
|
|
321
|
-
DEEPLINE_PLAY_PREVIEW_SLUG?: string;
|
|
322
|
-
/**
|
|
323
|
-
* Service binding to the long-lived Play Harness Worker
|
|
324
|
-
* (apps/play-harness-worker). Provides typed RPC access to leaf-level
|
|
325
|
-
* helpers (runtime-API HTTP forwarder, Neon dataset IO, …) that we
|
|
326
|
-
* deliberately keep OUT of every per-graphHash play bundle.
|
|
327
|
-
*
|
|
328
|
-
* Required in every Cloudflare play environment. The coordinator fails
|
|
329
|
-
* submit/load before handing a per-play Worker an unwired harness binding.
|
|
330
|
-
*
|
|
331
|
-
* Wired in wrangler.toml as `[[services]] binding = "HARNESS"`.
|
|
332
|
-
*/
|
|
333
|
-
HARNESS: import('../../play-harness-worker/src/rpc-types').PlayHarnessRpc;
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
const WORKFLOW_READ_ONLY_ACTIONS = new Set(['', 'result', 'status', 'tail']);
|
|
337
|
-
|
|
338
|
-
function authorizeCoordinatorControlRequest(input: {
|
|
339
|
-
request: Request;
|
|
340
|
-
env: CoordinatorEnv;
|
|
341
|
-
runId?: string;
|
|
342
|
-
requireRunScope?: boolean;
|
|
343
|
-
}): Response | null {
|
|
344
|
-
const expectedToken = input.env.DEEPLINE_INTERNAL_TOKEN?.trim();
|
|
345
|
-
if (!expectedToken) {
|
|
346
|
-
return new Response('coordinator internal token is not configured', {
|
|
347
|
-
status: 503,
|
|
348
|
-
});
|
|
349
|
-
}
|
|
350
|
-
const actualToken =
|
|
351
|
-
input.request.headers.get(COORDINATOR_INTERNAL_TOKEN_HEADER)?.trim() ?? '';
|
|
352
|
-
if (actualToken !== expectedToken) {
|
|
353
|
-
return new Response('unauthorized', { status: 401 });
|
|
354
|
-
}
|
|
355
|
-
if (input.requireRunScope) {
|
|
356
|
-
const actualRunScope =
|
|
357
|
-
input.request.headers.get(COORDINATOR_RUN_SCOPE_HEADER)?.trim() ?? '';
|
|
358
|
-
if (!input.runId || actualRunScope !== input.runId) {
|
|
359
|
-
return new Response('run scope mismatch', { status: 403 });
|
|
360
|
-
}
|
|
361
|
-
}
|
|
362
|
-
return null;
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
function isWorkflowMutatingAction(action: string): boolean {
|
|
366
|
-
return !WORKFLOW_READ_ONLY_ACTIONS.has(action);
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
/**
|
|
370
|
-
* Best-effort disposal of an RPC stub. Workers' RPC machinery emits
|
|
371
|
-
* "stub not disposed" warnings (and, under stress, contributes to V8
|
|
372
|
-
* deopt failures in `wrangler dev` local mode) when these are GC'd
|
|
373
|
-
* instead of explicitly released. Optional chaining keeps this safe
|
|
374
|
-
* for non-stub values.
|
|
375
|
-
*/
|
|
376
|
-
function disposeRpcStub(value: unknown): void {
|
|
377
|
-
if (value == null) return;
|
|
378
|
-
const disposable = value as { [Symbol.dispose]?: () => void };
|
|
379
|
-
try {
|
|
380
|
-
disposable[Symbol.dispose]?.();
|
|
381
|
-
} catch {
|
|
382
|
-
// intentional no-op — disposal is best-effort
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
/**
|
|
387
|
-
* Structured perf trace for the coordinator. Emits one [perf-trace] JSON
|
|
388
|
-
* line that downstream tooling (tests/v2-plays/runner.ts --perf-summary)
|
|
389
|
-
* scrapes from logs/coordinator.log / logs/cf-coordinator-tail.log.
|
|
390
|
-
*
|
|
391
|
-
* Workerd has no filesystem; this is stdout-only. Stays additive next to
|
|
392
|
-
* the existing `[coordinator] ...` logs.
|
|
393
|
-
*/
|
|
394
|
-
function buildCoordinatorPerfTracePayload(
|
|
395
|
-
event: CoordinatorPerfTraceInput,
|
|
396
|
-
): CoordinatorPerfTracePayload | null {
|
|
397
|
-
if (!event.runId || !event.phase) return null;
|
|
398
|
-
return {
|
|
399
|
-
ts: Date.now(),
|
|
400
|
-
source: 'coordinator',
|
|
401
|
-
runId: event.runId,
|
|
402
|
-
phase: event.phase,
|
|
403
|
-
ms: event.ms,
|
|
404
|
-
...(event.graphHash ? { graphHash: event.graphHash } : {}),
|
|
405
|
-
...(event.extra ?? {}),
|
|
406
|
-
};
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
function logCoordinatorPerfTrace(payload: CoordinatorPerfTracePayload): void {
|
|
410
|
-
void coordinatorPerfTelemetry
|
|
411
|
-
.child({ context: { runId: payload.runId } })
|
|
412
|
-
.info(payload.phase, coordinatorTelemetryFields(payload), {
|
|
413
|
-
tag: '[perf-trace]',
|
|
414
|
-
});
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
function recordCoordinatorPerfTrace(event: CoordinatorPerfTraceInput): void {
|
|
418
|
-
const payload = buildCoordinatorPerfTracePayload(event);
|
|
419
|
-
if (!payload) return;
|
|
420
|
-
logCoordinatorPerfTrace(payload);
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
function safeOrigin(value: string | null | undefined): string | null {
|
|
424
|
-
if (!value?.trim()) return null;
|
|
425
|
-
try {
|
|
426
|
-
return new URL(value).origin;
|
|
427
|
-
} catch {
|
|
428
|
-
return '<invalid>';
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
async function appendCoordinatorPerfTrace(
|
|
433
|
-
env: CoordinatorEnv,
|
|
434
|
-
payload: CoordinatorPerfTracePayload,
|
|
435
|
-
): Promise<void> {
|
|
436
|
-
const stub = env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(payload.runId));
|
|
437
|
-
const response = await stub.fetch(
|
|
438
|
-
'https://deepline.dedup.internal/trace-add',
|
|
439
|
-
{
|
|
440
|
-
method: 'POST',
|
|
441
|
-
headers: { 'content-type': 'application/json' },
|
|
442
|
-
body: JSON.stringify(coordinatorStorageRedactor.redact(payload)),
|
|
443
|
-
},
|
|
444
|
-
);
|
|
445
|
-
if (!response.ok) {
|
|
446
|
-
throw new Error(
|
|
447
|
-
`coordinator trace append failed ${response.status}: ${(
|
|
448
|
-
await response.text().catch(() => '')
|
|
449
|
-
).slice(0, 200)}`,
|
|
450
|
-
);
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
function recordCoordinatorPerfTraceBuffered(
|
|
455
|
-
env: CoordinatorEnv,
|
|
456
|
-
ctx: ExecutionContext | null | undefined,
|
|
457
|
-
event: CoordinatorPerfTraceInput,
|
|
458
|
-
): void {
|
|
459
|
-
const payload = buildCoordinatorPerfTracePayload(event);
|
|
460
|
-
if (!payload) return;
|
|
461
|
-
logCoordinatorPerfTrace(payload);
|
|
462
|
-
const append = appendCoordinatorPerfTrace(env, payload).catch((error) => {
|
|
463
|
-
return coordinatorTelemetry
|
|
464
|
-
.child({ context: { runId: payload.runId } })
|
|
465
|
-
.warn(
|
|
466
|
-
'coordinator.perf_trace.buffer_failed',
|
|
467
|
-
{
|
|
468
|
-
phase: payload.phase,
|
|
469
|
-
error: error instanceof Error ? error.message : String(error),
|
|
470
|
-
},
|
|
471
|
-
{ tag: '[coordinator] failed to buffer perf trace' },
|
|
472
|
-
);
|
|
473
|
-
});
|
|
474
|
-
if (typeof ctx?.waitUntil === 'function') {
|
|
475
|
-
ctx.waitUntil(append);
|
|
476
|
-
}
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
async function listCoordinatorPerfTrace(
|
|
480
|
-
env: CoordinatorEnv,
|
|
481
|
-
runId: string,
|
|
482
|
-
): Promise<CoordinatorPerfTracePayload[]> {
|
|
483
|
-
const stub = env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(runId));
|
|
484
|
-
const response = await stub.fetch(
|
|
485
|
-
'https://deepline.dedup.internal/trace-list',
|
|
486
|
-
);
|
|
487
|
-
if (!response.ok) {
|
|
488
|
-
throw new Error(`coordinator trace list failed ${response.status}`);
|
|
489
|
-
}
|
|
490
|
-
const body = (await response.json().catch(() => ({}))) as {
|
|
491
|
-
entries?: unknown;
|
|
492
|
-
};
|
|
493
|
-
if (!Array.isArray(body.entries)) return [];
|
|
494
|
-
return body.entries.filter(
|
|
495
|
-
(entry): entry is CoordinatorPerfTracePayload =>
|
|
496
|
-
isRecord(entry) &&
|
|
497
|
-
(entry.source === 'coordinator' || entry.source === 'dynamic_worker') &&
|
|
498
|
-
typeof entry.runId === 'string' &&
|
|
499
|
-
typeof entry.phase === 'string' &&
|
|
500
|
-
typeof entry.ts === 'number' &&
|
|
501
|
-
typeof entry.ms === 'number',
|
|
502
|
-
);
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
async function writeCoordinatorTerminalState(
|
|
506
|
-
env: CoordinatorEnv,
|
|
507
|
-
state: CoordinatorTerminalState,
|
|
508
|
-
): Promise<void> {
|
|
509
|
-
const stub = env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(state.runId));
|
|
510
|
-
const response = await stub.fetch(
|
|
511
|
-
'https://deepline.dedup.internal/terminal-set',
|
|
512
|
-
{
|
|
513
|
-
method: 'POST',
|
|
514
|
-
headers: { 'content-type': 'application/json' },
|
|
515
|
-
body: JSON.stringify({
|
|
516
|
-
...state,
|
|
517
|
-
completedAt: state.completedAt ?? Date.now(),
|
|
518
|
-
}),
|
|
519
|
-
},
|
|
520
|
-
);
|
|
521
|
-
if (!response.ok) {
|
|
522
|
-
throw new Error(`coordinator terminal set failed ${response.status}`);
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
async function appendCoordinatorRunEvent(
|
|
527
|
-
env: CoordinatorEnv,
|
|
528
|
-
event: CoordinatorRunEvent,
|
|
529
|
-
): Promise<void> {
|
|
530
|
-
const stub = env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(event.runId));
|
|
531
|
-
const response = await stub.fetch(
|
|
532
|
-
'https://deepline.dedup.internal/event-add',
|
|
533
|
-
{
|
|
534
|
-
method: 'POST',
|
|
535
|
-
headers: { 'content-type': 'application/json' },
|
|
536
|
-
body: JSON.stringify(event),
|
|
537
|
-
},
|
|
538
|
-
);
|
|
539
|
-
if (!response.ok) {
|
|
540
|
-
throw new Error(`coordinator event append failed ${response.status}`);
|
|
541
|
-
}
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
async function listCoordinatorRunEvents(input: {
|
|
545
|
-
env: CoordinatorEnv;
|
|
546
|
-
runId: string;
|
|
547
|
-
afterSeq: number;
|
|
548
|
-
timeoutMs: number;
|
|
549
|
-
}): Promise<{ events: CoordinatorRunEvent[]; latestSeq: number }> {
|
|
550
|
-
const stub = input.env.PLAY_DEDUP.get(
|
|
551
|
-
input.env.PLAY_DEDUP.idFromName(input.runId),
|
|
552
|
-
);
|
|
553
|
-
const response = await stub.fetch(
|
|
554
|
-
`https://deepline.dedup.internal/event-list?afterSeq=${encodeURIComponent(
|
|
555
|
-
String(Math.max(0, Math.floor(input.afterSeq))),
|
|
556
|
-
)}&timeoutMs=${encodeURIComponent(
|
|
557
|
-
String(Math.max(0, Math.floor(input.timeoutMs))),
|
|
558
|
-
)}`,
|
|
559
|
-
);
|
|
560
|
-
if (!response.ok) {
|
|
561
|
-
throw new Error(`coordinator event list failed ${response.status}`);
|
|
562
|
-
}
|
|
563
|
-
const body = (await response.json().catch(() => ({}))) as {
|
|
564
|
-
events?: unknown;
|
|
565
|
-
latestSeq?: unknown;
|
|
566
|
-
};
|
|
567
|
-
return {
|
|
568
|
-
events: Array.isArray(body.events)
|
|
569
|
-
? body.events.filter(
|
|
570
|
-
(event): event is CoordinatorRunEvent =>
|
|
571
|
-
isRecord(event) &&
|
|
572
|
-
typeof event.runId === 'string' &&
|
|
573
|
-
event.runId === input.runId &&
|
|
574
|
-
typeof event.type === 'string' &&
|
|
575
|
-
typeof event.ts === 'number',
|
|
576
|
-
)
|
|
577
|
-
: [],
|
|
578
|
-
latestSeq:
|
|
579
|
-
typeof body.latestSeq === 'number' ? body.latestSeq : input.afterSeq,
|
|
580
|
-
};
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
async function readCoordinatorTerminalState(
|
|
584
|
-
env: CoordinatorEnv,
|
|
585
|
-
runId: string,
|
|
586
|
-
): Promise<CoordinatorTerminalState | null> {
|
|
587
|
-
const stub = env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(runId));
|
|
588
|
-
const response = await stub.fetch(
|
|
589
|
-
'https://deepline.dedup.internal/terminal-get',
|
|
590
|
-
);
|
|
591
|
-
if (!response.ok) {
|
|
592
|
-
throw new Error(`coordinator terminal get failed ${response.status}`);
|
|
593
|
-
}
|
|
594
|
-
const body = (await response.json().catch(() => ({}))) as {
|
|
595
|
-
state?: unknown;
|
|
596
|
-
};
|
|
597
|
-
const state = body.state;
|
|
598
|
-
if (!isRecord(state) || state.runId !== runId) return null;
|
|
599
|
-
if (
|
|
600
|
-
state.status !== 'completed' &&
|
|
601
|
-
state.status !== 'failed' &&
|
|
602
|
-
state.status !== 'cancelled'
|
|
603
|
-
) {
|
|
604
|
-
return null;
|
|
605
|
-
}
|
|
606
|
-
return state as CoordinatorTerminalState;
|
|
607
|
-
}
|
|
608
|
-
|
|
609
|
-
function workflowEventType(name: string): string {
|
|
610
|
-
const normalized = name
|
|
611
|
-
.trim()
|
|
612
|
-
.replace(/[^A-Za-z0-9_-]+/g, '_')
|
|
613
|
-
.replace(/^_+|_+$/g, '')
|
|
614
|
-
.slice(0, 100);
|
|
615
|
-
return normalized || 'deepline_event';
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
type DynamicWorkflowMetadata = {
|
|
619
|
-
runId?: string | null;
|
|
620
|
-
graphHash: string;
|
|
621
|
-
artifactStorageKey: string;
|
|
622
|
-
artifactHash?: string | null;
|
|
623
|
-
dynamicWorkerCode?: string | null;
|
|
624
|
-
dynamicWorkerCodeHash?: string | null;
|
|
625
|
-
integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
|
|
626
|
-
packagedFiles?: Array<{
|
|
627
|
-
playPath: string;
|
|
628
|
-
storageKey: string;
|
|
629
|
-
contentType?: string;
|
|
630
|
-
bytes?: number;
|
|
631
|
-
inlineText?: string;
|
|
632
|
-
}> | null;
|
|
633
|
-
};
|
|
634
|
-
|
|
635
|
-
const SUBMIT_INITIAL_STATE_MAX_WAIT_MS = 0;
|
|
636
|
-
const SUBMIT_INITIAL_STATE_POLL_MS = 50;
|
|
637
|
-
const WORKFLOW_RETRY_STATE_TTL_MS = 60 * 60 * 1000;
|
|
638
|
-
const WORKFLOW_SUBMIT_DUPLICATE_WAIT_MS = 55_000;
|
|
639
|
-
const WORKFLOW_SUBMIT_DUPLICATE_POLL_MS = 250;
|
|
640
|
-
const CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS = [
|
|
641
|
-
250, 750, 1500, 3000, 5000, 10000,
|
|
642
|
-
] as const;
|
|
643
|
-
const COORDINATOR_WORKER_MEMORY_LIMIT_MESSAGE =
|
|
644
|
-
'Worker memory limit hit before the run could finish. For CSV enrich, use --output or --in-place for automatic batches; otherwise rerun smaller --rows ranges.';
|
|
645
|
-
|
|
646
|
-
function normalizeIntegrationMode(
|
|
647
|
-
value: unknown,
|
|
648
|
-
): 'live' | 'eval_stub' | 'fixture' | null {
|
|
649
|
-
return value === 'live' || value === 'eval_stub' || value === 'fixture'
|
|
650
|
-
? value
|
|
651
|
-
: null;
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
function buildDynamicWorkflowMetadata(
|
|
655
|
-
params: PlayWorkflowParams,
|
|
656
|
-
): DynamicWorkflowMetadata {
|
|
657
|
-
const dynamicWorkerCodeHash =
|
|
658
|
-
typeof params.dynamicWorkerCode === 'string' &&
|
|
659
|
-
params.dynamicWorkerCode.length > 0
|
|
660
|
-
? stableHash(params.dynamicWorkerCode)
|
|
661
|
-
: null;
|
|
662
|
-
return {
|
|
663
|
-
runId: params.runId ?? null,
|
|
664
|
-
graphHash: params.graphHash,
|
|
665
|
-
artifactStorageKey: params.artifactStorageKey,
|
|
666
|
-
artifactHash: params.artifactHash ?? null,
|
|
667
|
-
dynamicWorkerCode: null,
|
|
668
|
-
dynamicWorkerCodeHash,
|
|
669
|
-
integrationMode: normalizeIntegrationMode(params.integrationMode),
|
|
670
|
-
packagedFiles: normalizePackagedFiles(params.packagedFiles),
|
|
671
|
-
};
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
function readWorkflowTraceContext(event: unknown): {
|
|
675
|
-
runId: string;
|
|
676
|
-
graphHash: string | null;
|
|
677
|
-
instanceId: string | null;
|
|
678
|
-
submittedAt: number | null;
|
|
679
|
-
} {
|
|
680
|
-
const record = isRecord(event) ? event : {};
|
|
681
|
-
const payload = isRecord(record.payload) ? record.payload : {};
|
|
682
|
-
const params = isRecord(payload.params) ? payload.params : payload;
|
|
683
|
-
const metadata = isRecord(payload.__dispatcherMetadata)
|
|
684
|
-
? payload.__dispatcherMetadata
|
|
685
|
-
: null;
|
|
686
|
-
const runId =
|
|
687
|
-
(typeof params?.runId === 'string' && params.runId) ||
|
|
688
|
-
(typeof metadata?.runId === 'string' && metadata.runId) ||
|
|
689
|
-
(typeof record.instanceId === 'string' && record.instanceId) ||
|
|
690
|
-
'unknown-workflow-run';
|
|
691
|
-
const graphHash =
|
|
692
|
-
(typeof params?.graphHash === 'string' && params.graphHash) ||
|
|
693
|
-
(typeof metadata?.graphHash === 'string' && metadata.graphHash) ||
|
|
694
|
-
null;
|
|
695
|
-
return {
|
|
696
|
-
runId,
|
|
697
|
-
graphHash,
|
|
698
|
-
instanceId:
|
|
699
|
-
typeof record.instanceId === 'string' && record.instanceId
|
|
700
|
-
? record.instanceId
|
|
701
|
-
: null,
|
|
702
|
-
submittedAt:
|
|
703
|
-
typeof params?.submittedAt === 'number' &&
|
|
704
|
-
Number.isFinite(params.submittedAt)
|
|
705
|
-
? params.submittedAt
|
|
706
|
-
: null,
|
|
707
|
-
};
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
async function waitForSubmitInitialState(input: {
|
|
711
|
-
instance: WorkflowInstance;
|
|
712
|
-
runId: string;
|
|
713
|
-
waitMs: number;
|
|
714
|
-
}): Promise<Record<string, unknown> | null> {
|
|
715
|
-
const waitMs = Math.max(
|
|
716
|
-
0,
|
|
717
|
-
Math.min(Math.floor(input.waitMs), SUBMIT_INITIAL_STATE_MAX_WAIT_MS),
|
|
718
|
-
);
|
|
719
|
-
if (waitMs <= 0) return null;
|
|
720
|
-
const startedAt = Date.now();
|
|
721
|
-
let status = await input.instance.status();
|
|
722
|
-
while (Date.now() - startedAt < waitMs) {
|
|
723
|
-
const result = mapWorkflowResult(input.runId, status);
|
|
724
|
-
if (
|
|
725
|
-
result.status === 'completed' ||
|
|
726
|
-
result.status === 'failed' ||
|
|
727
|
-
result.status === 'cancelled'
|
|
728
|
-
) {
|
|
729
|
-
return result as unknown as Record<string, unknown>;
|
|
730
|
-
}
|
|
731
|
-
await sleep(SUBMIT_INITIAL_STATE_POLL_MS);
|
|
732
|
-
status = await input.instance.status();
|
|
733
|
-
}
|
|
734
|
-
const result = mapWorkflowResult(input.runId, status);
|
|
735
|
-
return result.status === 'completed' ||
|
|
736
|
-
result.status === 'failed' ||
|
|
737
|
-
result.status === 'cancelled'
|
|
738
|
-
? (result as unknown as Record<string, unknown>)
|
|
739
|
-
: null;
|
|
740
|
-
}
|
|
741
|
-
|
|
742
|
-
async function createDynamicWorkflowInstance(input: {
|
|
743
|
-
env: CoordinatorEnv;
|
|
744
|
-
id: string;
|
|
745
|
-
params: PlayWorkflowParams;
|
|
746
|
-
metadataParams?: PlayWorkflowParams;
|
|
747
|
-
}): Promise<WorkflowInstance> {
|
|
748
|
-
const workflow = createDispatcherWorkflowBinding(
|
|
749
|
-
() => input.env.PLAY_WORKFLOW as unknown as Workflow,
|
|
750
|
-
buildDynamicWorkflowMetadata(input.metadataParams ?? input.params),
|
|
751
|
-
);
|
|
752
|
-
return workflow.create({
|
|
753
|
-
id: input.id,
|
|
754
|
-
params: input.params,
|
|
755
|
-
});
|
|
756
|
-
}
|
|
757
|
-
|
|
758
|
-
function runScopedDurableObject(
|
|
759
|
-
env: CoordinatorEnv,
|
|
760
|
-
runId: string,
|
|
761
|
-
): DurableObjectStub {
|
|
762
|
-
return env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(runId));
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
/**
|
|
766
|
-
* Address the rate-state Durable Object for a single `<orgId>:<provider>`
|
|
767
|
-
* bucket. Keying the DO by the bucket id (not the run id) makes one
|
|
768
|
-
* single-threaded instance own that bucket's request window across every
|
|
769
|
-
* isolate of every run in the org — which is exactly what the distributed Rate
|
|
770
|
-
* State Backend needs. Reuses the PlayDedup namespace (the DO already hosts the
|
|
771
|
-
* token-bucket handlers) so no extra binding is required.
|
|
772
|
-
*/
|
|
773
|
-
function rateBucketDurableObject(
|
|
774
|
-
env: CoordinatorEnv,
|
|
775
|
-
bucketId: string,
|
|
776
|
-
): DurableObjectStub {
|
|
777
|
-
return env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(`rate:${bucketId}`));
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
async function callRateBucketControl<T>(
|
|
781
|
-
env: CoordinatorEnv,
|
|
782
|
-
bucketId: string,
|
|
783
|
-
path: string,
|
|
784
|
-
body: unknown,
|
|
785
|
-
): Promise<T> {
|
|
786
|
-
const response = await rateBucketDurableObject(env, bucketId).fetch(
|
|
787
|
-
`https://deepline.rate-state.internal${path}`,
|
|
788
|
-
{
|
|
789
|
-
method: 'POST',
|
|
790
|
-
headers: { 'content-type': 'application/json' },
|
|
791
|
-
body: JSON.stringify(body),
|
|
792
|
-
},
|
|
793
|
-
);
|
|
794
|
-
if (!response.ok) {
|
|
795
|
-
throw new Error(
|
|
796
|
-
`rate state ${path} failed ${response.status}: ${(
|
|
797
|
-
await response.text().catch(() => '')
|
|
798
|
-
).slice(0, 400)}`,
|
|
799
|
-
);
|
|
800
|
-
}
|
|
801
|
-
return (await response.json()) as T;
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
async function callRunScopedControl<T>(
|
|
805
|
-
env: CoordinatorEnv,
|
|
806
|
-
runId: string,
|
|
807
|
-
path: string,
|
|
808
|
-
init?: RequestInit,
|
|
809
|
-
): Promise<T> {
|
|
810
|
-
const maxAttempts = DURABLE_OBJECT_DEPLOY_HANDOFF_RETRY_DELAYS_MS.length + 1;
|
|
811
|
-
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
812
|
-
let response: Response;
|
|
813
|
-
try {
|
|
814
|
-
response = await runScopedDurableObject(env, runId).fetch(
|
|
815
|
-
`https://deepline.run-state.internal${path}`,
|
|
816
|
-
{
|
|
817
|
-
...(init ?? {}),
|
|
818
|
-
headers: {
|
|
819
|
-
'content-type': 'application/json',
|
|
820
|
-
...(init?.headers ?? {}),
|
|
821
|
-
},
|
|
822
|
-
},
|
|
823
|
-
);
|
|
824
|
-
} catch (error) {
|
|
825
|
-
const handoffMessage = durableObjectDeployHandoffMessage(error);
|
|
826
|
-
if (handoffMessage && attempt < maxAttempts) {
|
|
827
|
-
recordCoordinatorEvent(
|
|
828
|
-
'warn',
|
|
829
|
-
'coordinator.run_state.handoff_retry',
|
|
830
|
-
'[coordinator] run state DO deploy handoff retry',
|
|
831
|
-
{
|
|
832
|
-
runId,
|
|
833
|
-
path,
|
|
834
|
-
attempt,
|
|
835
|
-
error: handoffMessage,
|
|
836
|
-
},
|
|
837
|
-
);
|
|
838
|
-
await sleepForDurableObjectDeployHandoffRetry(attempt - 1);
|
|
839
|
-
continue;
|
|
840
|
-
}
|
|
841
|
-
throw error;
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
if (response.ok) {
|
|
845
|
-
return (await response.json()) as T;
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
const responseText = (await response.text().catch(() => '')).slice(0, 400);
|
|
849
|
-
const handoffMessage = durableObjectDeployHandoffMessage(responseText);
|
|
850
|
-
if (handoffMessage && attempt < maxAttempts) {
|
|
851
|
-
recordCoordinatorEvent(
|
|
852
|
-
'warn',
|
|
853
|
-
'coordinator.run_state.handoff_retry',
|
|
854
|
-
'[coordinator] run state DO deploy handoff retry',
|
|
855
|
-
{
|
|
856
|
-
runId,
|
|
857
|
-
path,
|
|
858
|
-
attempt,
|
|
859
|
-
status: response.status,
|
|
860
|
-
error: handoffMessage,
|
|
861
|
-
},
|
|
862
|
-
);
|
|
863
|
-
await sleepForDurableObjectDeployHandoffRetry(attempt - 1);
|
|
864
|
-
continue;
|
|
865
|
-
}
|
|
866
|
-
|
|
867
|
-
throw new Error(
|
|
868
|
-
`run state ${path} failed ${response.status}: ${responseText}`,
|
|
869
|
-
);
|
|
870
|
-
}
|
|
871
|
-
throw new Error(`run state ${path} failed after deploy handoff retries`);
|
|
872
|
-
}
|
|
873
|
-
|
|
874
|
-
async function recordWorkflowInstanceId(input: {
|
|
875
|
-
env: CoordinatorEnv;
|
|
876
|
-
runId: string;
|
|
877
|
-
instanceId: string;
|
|
878
|
-
}): Promise<void> {
|
|
879
|
-
await callRunScopedControl<{ ok?: unknown }>(
|
|
880
|
-
input.env,
|
|
881
|
-
input.runId,
|
|
882
|
-
'/workflow-instance-put',
|
|
883
|
-
{
|
|
884
|
-
method: 'POST',
|
|
885
|
-
body: JSON.stringify({
|
|
886
|
-
runId: input.runId,
|
|
887
|
-
instanceId: input.instanceId,
|
|
888
|
-
ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
|
|
889
|
-
}),
|
|
890
|
-
},
|
|
891
|
-
);
|
|
892
|
-
}
|
|
893
|
-
|
|
894
|
-
async function resolveWorkflowInstanceIdForRun(
|
|
895
|
-
env: CoordinatorEnv,
|
|
896
|
-
runId: string,
|
|
897
|
-
): Promise<string> {
|
|
898
|
-
const body = await callRunScopedControl<{ instanceId?: unknown }>(
|
|
899
|
-
env,
|
|
900
|
-
runId,
|
|
901
|
-
`/workflow-instance-get?runId=${encodeURIComponent(runId)}`,
|
|
902
|
-
).catch(() => ({ instanceId: null }));
|
|
903
|
-
return typeof body.instanceId === 'string' && body.instanceId
|
|
904
|
-
? body.instanceId
|
|
905
|
-
: workflowInstanceId(runId);
|
|
906
|
-
}
|
|
907
|
-
|
|
908
|
-
async function claimWorkflowSubmit(input: {
|
|
909
|
-
env: CoordinatorEnv;
|
|
910
|
-
runId: string;
|
|
911
|
-
}): Promise<{
|
|
912
|
-
claimed: boolean;
|
|
913
|
-
status: 'creating' | 'created' | null;
|
|
914
|
-
instanceId: string | null;
|
|
915
|
-
}> {
|
|
916
|
-
const body = await callRunScopedControl<{
|
|
917
|
-
claimed?: unknown;
|
|
918
|
-
status?: unknown;
|
|
919
|
-
instanceId?: unknown;
|
|
920
|
-
}>(input.env, input.runId, '/workflow-submit-claim', {
|
|
921
|
-
method: 'POST',
|
|
922
|
-
body: JSON.stringify({
|
|
923
|
-
runId: input.runId,
|
|
924
|
-
ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
|
|
925
|
-
}),
|
|
926
|
-
});
|
|
927
|
-
return {
|
|
928
|
-
claimed: body.claimed === true,
|
|
929
|
-
status:
|
|
930
|
-
body.status === 'creating' || body.status === 'created'
|
|
931
|
-
? body.status
|
|
932
|
-
: null,
|
|
933
|
-
instanceId: typeof body.instanceId === 'string' ? body.instanceId : null,
|
|
934
|
-
};
|
|
935
|
-
}
|
|
936
|
-
|
|
937
|
-
async function releaseWorkflowSubmitClaim(input: {
|
|
938
|
-
env: CoordinatorEnv;
|
|
939
|
-
runId: string;
|
|
940
|
-
}): Promise<void> {
|
|
941
|
-
await callRunScopedControl<{ ok?: unknown }>(
|
|
942
|
-
input.env,
|
|
943
|
-
input.runId,
|
|
944
|
-
'/workflow-submit-release',
|
|
945
|
-
{
|
|
946
|
-
method: 'POST',
|
|
947
|
-
body: JSON.stringify({ runId: input.runId }),
|
|
948
|
-
},
|
|
949
|
-
);
|
|
950
|
-
}
|
|
951
|
-
|
|
952
|
-
async function waitForWorkflowSubmitCreated(input: {
|
|
953
|
-
env: CoordinatorEnv;
|
|
954
|
-
runId: string;
|
|
955
|
-
initialClaim: Awaited<ReturnType<typeof claimWorkflowSubmit>>;
|
|
956
|
-
timeoutMs: number;
|
|
957
|
-
pollMs: number;
|
|
958
|
-
}): Promise<Awaited<ReturnType<typeof claimWorkflowSubmit>>> {
|
|
959
|
-
let claim = input.initialClaim;
|
|
960
|
-
const deadline = Date.now() + input.timeoutMs;
|
|
961
|
-
while (claim.status !== 'created' || !claim.instanceId) {
|
|
962
|
-
if (Date.now() >= deadline) {
|
|
963
|
-
return claim;
|
|
964
|
-
}
|
|
965
|
-
await sleep(input.pollMs);
|
|
966
|
-
claim = await claimWorkflowSubmit({
|
|
967
|
-
env: input.env,
|
|
968
|
-
runId: input.runId,
|
|
969
|
-
});
|
|
970
|
-
}
|
|
971
|
-
return claim;
|
|
972
|
-
}
|
|
973
|
-
|
|
974
|
-
function assertEncryptedPreloadedDbSessions(
|
|
975
|
-
sessions: PreloadedRuntimeDbSession[],
|
|
976
|
-
): void {
|
|
977
|
-
for (const session of sessions) {
|
|
978
|
-
if (session.session.postgresUrl) {
|
|
979
|
-
throw new Error(
|
|
980
|
-
'Preloaded Runtime DB sessions must not carry raw Postgres URLs.',
|
|
981
|
-
);
|
|
982
|
-
}
|
|
983
|
-
if (!session.session.encryptedPostgresUrl) {
|
|
984
|
-
throw new Error(
|
|
985
|
-
'Preloaded Runtime DB sessions must carry encrypted Postgres URLs.',
|
|
986
|
-
);
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
|
-
}
|
|
990
|
-
|
|
991
|
-
async function readWorkflowDbSessions(input: {
|
|
992
|
-
env: CoordinatorEnv;
|
|
993
|
-
ref: NonNullable<PlayWorkflowParams['preloadedDbSessionRef']>;
|
|
994
|
-
}): Promise<PreloadedRuntimeDbSession[]> {
|
|
995
|
-
const response = await runScopedDurableObject(
|
|
996
|
-
input.env,
|
|
997
|
-
input.ref.runId,
|
|
998
|
-
).fetch(
|
|
999
|
-
`https://deepline.dedup.internal/db-sessions-get?runId=${encodeURIComponent(
|
|
1000
|
-
input.ref.runId,
|
|
1001
|
-
)}`,
|
|
1002
|
-
);
|
|
1003
|
-
if (!response.ok) {
|
|
1004
|
-
throw new Error(
|
|
1005
|
-
`workflow db session lookup failed ${response.status}: ${(
|
|
1006
|
-
await response.text().catch(() => '')
|
|
1007
|
-
).slice(0, 400)}`,
|
|
1008
|
-
);
|
|
1009
|
-
}
|
|
1010
|
-
const body = (await response.json().catch(() => ({}))) as {
|
|
1011
|
-
sessions?: unknown;
|
|
1012
|
-
};
|
|
1013
|
-
if (!Array.isArray(body.sessions)) {
|
|
1014
|
-
throw new Error('workflow db session lookup returned no sessions.');
|
|
1015
|
-
}
|
|
1016
|
-
const sessions = body.sessions as PreloadedRuntimeDbSession[];
|
|
1017
|
-
assertEncryptedPreloadedDbSessions(sessions);
|
|
1018
|
-
return sessions;
|
|
1019
|
-
}
|
|
1020
|
-
|
|
1021
|
-
async function readWorkflowDbSessionsWithRetry(input: {
|
|
1022
|
-
env: CoordinatorEnv;
|
|
1023
|
-
ref: NonNullable<PlayWorkflowParams['preloadedDbSessionRef']>;
|
|
1024
|
-
}): Promise<PreloadedRuntimeDbSession[]> {
|
|
1025
|
-
const delays = [25, 50, 100, 200] as const;
|
|
1026
|
-
let lastError: unknown = null;
|
|
1027
|
-
for (let attempt = 0; attempt <= delays.length; attempt += 1) {
|
|
1028
|
-
try {
|
|
1029
|
-
return await readWorkflowDbSessions(input);
|
|
1030
|
-
} catch (error) {
|
|
1031
|
-
lastError = error;
|
|
1032
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1033
|
-
if (
|
|
1034
|
-
!message.includes('workflow db session lookup failed 404') ||
|
|
1035
|
-
attempt >= delays.length
|
|
1036
|
-
) {
|
|
1037
|
-
throw error;
|
|
1038
|
-
}
|
|
1039
|
-
await sleep(delays[attempt]);
|
|
1040
|
-
}
|
|
1041
|
-
}
|
|
1042
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
1043
|
-
}
|
|
1044
|
-
|
|
1045
|
-
function externalizedWorkflowDbSessionParams(input: {
|
|
1046
|
-
params: PlayWorkflowParams;
|
|
1047
|
-
}): {
|
|
1048
|
-
params: PlayWorkflowParams;
|
|
1049
|
-
sessions: PreloadedRuntimeDbSession[];
|
|
1050
|
-
ref: NonNullable<PlayWorkflowParams['preloadedDbSessionRef']> | null;
|
|
1051
|
-
} {
|
|
1052
|
-
const sessions = Array.isArray(input.params.preloadedDbSessions)
|
|
1053
|
-
? input.params.preloadedDbSessions
|
|
1054
|
-
: [];
|
|
1055
|
-
if (sessions.length === 0) {
|
|
1056
|
-
return { params: input.params, sessions, ref: null };
|
|
1057
|
-
}
|
|
1058
|
-
const ref: NonNullable<PlayWorkflowParams['preloadedDbSessionRef']> = {
|
|
1059
|
-
runId: input.params.runId,
|
|
1060
|
-
sessionCount: sessions.length,
|
|
1061
|
-
expiresAt: Date.now() + DB_SESSION_DEFAULT_TTL_SECONDS * 1000,
|
|
1062
|
-
};
|
|
1063
|
-
return {
|
|
1064
|
-
params: {
|
|
1065
|
-
...input.params,
|
|
1066
|
-
preloadedDbSessions: null,
|
|
1067
|
-
preloadedDbSessionRef: ref,
|
|
1068
|
-
},
|
|
1069
|
-
sessions,
|
|
1070
|
-
ref,
|
|
1071
|
-
};
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
function readPreloadedDbSessionRef(
|
|
1075
|
-
value: unknown,
|
|
1076
|
-
): NonNullable<PlayWorkflowParams['preloadedDbSessionRef']> | null {
|
|
1077
|
-
if (!isRecord(value)) return null;
|
|
1078
|
-
const runId = typeof value.runId === 'string' ? value.runId : '';
|
|
1079
|
-
const sessionCount =
|
|
1080
|
-
typeof value.sessionCount === 'number' &&
|
|
1081
|
-
Number.isFinite(value.sessionCount)
|
|
1082
|
-
? value.sessionCount
|
|
1083
|
-
: 0;
|
|
1084
|
-
const expiresAt =
|
|
1085
|
-
typeof value.expiresAt === 'number' && Number.isFinite(value.expiresAt)
|
|
1086
|
-
? value.expiresAt
|
|
1087
|
-
: 0;
|
|
1088
|
-
if (!runId || sessionCount <= 0 || expiresAt <= 0) return null;
|
|
1089
|
-
return { runId, sessionCount, expiresAt };
|
|
1090
|
-
}
|
|
1091
|
-
|
|
1092
|
-
async function hydrateWorkflowDbSessions(input: {
|
|
1093
|
-
env: CoordinatorEnv;
|
|
1094
|
-
event: unknown;
|
|
1095
|
-
trace: CoordinatorPerfTraceSink;
|
|
1096
|
-
}): Promise<unknown> {
|
|
1097
|
-
if (!isRecord(input.event)) return input.event;
|
|
1098
|
-
const payload = isRecord(input.event.payload) ? input.event.payload : null;
|
|
1099
|
-
const envelopeParams = isRecord(payload?.params) ? payload.params : null;
|
|
1100
|
-
const params = envelopeParams ?? payload;
|
|
1101
|
-
if (!payload || !params || Array.isArray(params.preloadedDbSessions)) {
|
|
1102
|
-
return input.event;
|
|
1103
|
-
}
|
|
1104
|
-
const ref = readPreloadedDbSessionRef(params.preloadedDbSessionRef);
|
|
1105
|
-
if (!ref) return input.event;
|
|
1106
|
-
const startedAt = Date.now();
|
|
1107
|
-
const sessions = await readWorkflowDbSessionsWithRetry({
|
|
1108
|
-
env: input.env,
|
|
1109
|
-
ref,
|
|
1110
|
-
});
|
|
1111
|
-
input.trace({
|
|
1112
|
-
runId: ref.runId,
|
|
1113
|
-
phase: 'coordinator.workflow_db_sessions_hydrated',
|
|
1114
|
-
ms: Date.now() - startedAt,
|
|
1115
|
-
graphHash: typeof params.graphHash === 'string' ? params.graphHash : null,
|
|
1116
|
-
extra: {
|
|
1117
|
-
sessions: sessions.length,
|
|
1118
|
-
refSessionCount: ref.sessionCount,
|
|
1119
|
-
},
|
|
1120
|
-
});
|
|
1121
|
-
return {
|
|
1122
|
-
...input.event,
|
|
1123
|
-
payload: envelopeParams
|
|
1124
|
-
? {
|
|
1125
|
-
...payload,
|
|
1126
|
-
params: {
|
|
1127
|
-
...params,
|
|
1128
|
-
preloadedDbSessions: sessions,
|
|
1129
|
-
},
|
|
1130
|
-
}
|
|
1131
|
-
: {
|
|
1132
|
-
...payload,
|
|
1133
|
-
preloadedDbSessions: sessions,
|
|
1134
|
-
},
|
|
1135
|
-
};
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
async function hydrateWorkflowLaunchParams(input: {
|
|
1139
|
-
env: CoordinatorEnv;
|
|
1140
|
-
event: unknown;
|
|
1141
|
-
trace: CoordinatorPerfTraceSink;
|
|
1142
|
-
}): Promise<unknown> {
|
|
1143
|
-
if (!isRecord(input.event)) return input.event;
|
|
1144
|
-
const payload = isRecord(input.event.payload) ? input.event.payload : null;
|
|
1145
|
-
const envelopeParams = isRecord(payload?.params) ? payload.params : null;
|
|
1146
|
-
const params = envelopeParams ?? payload;
|
|
1147
|
-
if (
|
|
1148
|
-
!payload ||
|
|
1149
|
-
!params ||
|
|
1150
|
-
(!isRecord(params.workflowParams) && !isRecord(params.workflowParamsRef))
|
|
1151
|
-
) {
|
|
1152
|
-
return input.event;
|
|
1153
|
-
}
|
|
1154
|
-
const startedAt = Date.now();
|
|
1155
|
-
const hydrated = await hydrateWorkflowRetryParams({
|
|
1156
|
-
env: input.env,
|
|
1157
|
-
params: params.workflowParams,
|
|
1158
|
-
paramsRef: params.workflowParamsRef,
|
|
1159
|
-
});
|
|
1160
|
-
if (!hydrated) return input.event;
|
|
1161
|
-
const submittedAt =
|
|
1162
|
-
typeof params.submittedAt === 'number' &&
|
|
1163
|
-
Number.isFinite(params.submittedAt)
|
|
1164
|
-
? params.submittedAt
|
|
1165
|
-
: hydrated.submittedAt;
|
|
1166
|
-
const hydratedParams = {
|
|
1167
|
-
...hydrated,
|
|
1168
|
-
submittedAt,
|
|
1169
|
-
};
|
|
1170
|
-
input.trace({
|
|
1171
|
-
runId:
|
|
1172
|
-
typeof hydratedParams.runId === 'string'
|
|
1173
|
-
? hydratedParams.runId
|
|
1174
|
-
: 'unknown-workflow-run',
|
|
1175
|
-
phase: 'coordinator.workflow_launch_params_hydrated',
|
|
1176
|
-
ms: Date.now() - startedAt,
|
|
1177
|
-
graphHash:
|
|
1178
|
-
typeof hydratedParams.graphHash === 'string'
|
|
1179
|
-
? hydratedParams.graphHash
|
|
1180
|
-
: null,
|
|
1181
|
-
extra: {
|
|
1182
|
-
paramsBytes: isRecord(params.workflowParamsRef)
|
|
1183
|
-
? typeof params.workflowParamsRef.bytes === 'number'
|
|
1184
|
-
? params.workflowParamsRef.bytes
|
|
1185
|
-
: null
|
|
1186
|
-
: jsonByteLength(params.workflowParams),
|
|
1187
|
-
paramsStorage: isRecord(params.workflowParamsRef) ? 'r2' : 'inline',
|
|
1188
|
-
},
|
|
1189
|
-
});
|
|
1190
|
-
return {
|
|
1191
|
-
...input.event,
|
|
1192
|
-
payload: envelopeParams
|
|
1193
|
-
? {
|
|
1194
|
-
...payload,
|
|
1195
|
-
params: hydratedParams,
|
|
1196
|
-
}
|
|
1197
|
-
: {
|
|
1198
|
-
...payload,
|
|
1199
|
-
...hydratedParams,
|
|
1200
|
-
},
|
|
1201
|
-
};
|
|
1202
|
-
}
|
|
1203
|
-
|
|
1204
|
-
function workflowCreateParams(
|
|
1205
|
-
params: PlayWorkflowParams,
|
|
1206
|
-
launchState: {
|
|
1207
|
-
params?: PlayWorkflowParams;
|
|
1208
|
-
paramsRef?: WorkflowRetryParamsRef;
|
|
1209
|
-
},
|
|
1210
|
-
): PlayWorkflowParams {
|
|
1211
|
-
const launchParams = launchState.params ?? params;
|
|
1212
|
-
const envelope: Record<string, unknown> = {
|
|
1213
|
-
runId: params.runId,
|
|
1214
|
-
playName: params.playName,
|
|
1215
|
-
graphHash: params.graphHash,
|
|
1216
|
-
artifactStorageKey: params.artifactStorageKey,
|
|
1217
|
-
artifactHash: params.artifactHash,
|
|
1218
|
-
submittedAt: params.submittedAt,
|
|
1219
|
-
};
|
|
1220
|
-
if (launchState.paramsRef) {
|
|
1221
|
-
envelope.workflowParamsRef = launchState.paramsRef;
|
|
1222
|
-
} else {
|
|
1223
|
-
envelope.workflowParams = launchParams;
|
|
1224
|
-
}
|
|
1225
|
-
return envelope as unknown as PlayWorkflowParams;
|
|
1226
|
-
}
|
|
1227
|
-
|
|
1228
|
-
function sleep(ms: number): Promise<void> {
|
|
1229
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1230
|
-
}
|
|
1231
|
-
|
|
1232
|
-
function readWorkflowPayload(event: unknown): Record<string, unknown> | null {
|
|
1233
|
-
if (!isRecord(event)) return null;
|
|
1234
|
-
const payload = event.payload;
|
|
1235
|
-
if (!isRecord(payload)) return null;
|
|
1236
|
-
return isRecord(payload.params) ? payload.params : payload;
|
|
1237
|
-
}
|
|
1238
|
-
|
|
1239
|
-
function isCoordinatorWorkerMemoryLimitError(error: unknown): boolean {
|
|
1240
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1241
|
-
return message.toLowerCase().includes('worker exceeded memory limit');
|
|
1242
|
-
}
|
|
1243
|
-
|
|
1244
|
-
function normalizeCoordinatorWorkflowRuntimeFailure(
|
|
1245
|
-
error: unknown,
|
|
1246
|
-
): ReturnType<typeof normalizePlayRunFailure> {
|
|
1247
|
-
const failure = normalizePlayRunFailure(error);
|
|
1248
|
-
if (!isCoordinatorWorkerMemoryLimitError(error)) {
|
|
1249
|
-
return failure;
|
|
1250
|
-
}
|
|
1251
|
-
return {
|
|
1252
|
-
...failure,
|
|
1253
|
-
code: 'WORKER_MEMORY_LIMIT_EXCEEDED',
|
|
1254
|
-
message: COORDINATOR_WORKER_MEMORY_LIMIT_MESSAGE,
|
|
1255
|
-
retryable: false,
|
|
1256
|
-
cause: failure.message,
|
|
1257
|
-
};
|
|
1258
|
-
}
|
|
1259
|
-
|
|
1260
|
-
async function markWorkflowRuntimeFailure(input: {
|
|
1261
|
-
env: CoordinatorEnv;
|
|
1262
|
-
event: unknown;
|
|
1263
|
-
error: unknown;
|
|
1264
|
-
}): Promise<void> {
|
|
1265
|
-
const payload = readWorkflowPayload(input.event);
|
|
1266
|
-
if (!payload) return;
|
|
1267
|
-
const runId = typeof payload.runId === 'string' ? payload.runId : null;
|
|
1268
|
-
const baseUrl = typeof payload.baseUrl === 'string' ? payload.baseUrl : null;
|
|
1269
|
-
const executorToken =
|
|
1270
|
-
typeof payload.executorToken === 'string' ? payload.executorToken : null;
|
|
1271
|
-
if (!runId || !baseUrl || !executorToken) return;
|
|
1272
|
-
const errorName =
|
|
1273
|
-
input.error instanceof Error && input.error.name
|
|
1274
|
-
? input.error.name
|
|
1275
|
-
: 'Error';
|
|
1276
|
-
const errorMessage =
|
|
1277
|
-
input.error instanceof Error ? input.error.message : String(input.error);
|
|
1278
|
-
const errorStack =
|
|
1279
|
-
input.error instanceof Error && typeof input.error.stack === 'string'
|
|
1280
|
-
? input.error.stack.split('\n').slice(0, 12).join('\n')
|
|
1281
|
-
: null;
|
|
1282
|
-
const failure = normalizeCoordinatorWorkflowRuntimeFailure(input.error);
|
|
1283
|
-
const shouldPreserveRawError =
|
|
1284
|
-
failure.code === 'RUN_FAILED' ||
|
|
1285
|
-
(failure.cause !== undefined && failure.message === failure.cause);
|
|
1286
|
-
const normalizedError = shouldPreserveRawError
|
|
1287
|
-
? `DynamicWorkflow runner failed: ${errorName}: ${errorMessage}${
|
|
1288
|
-
errorStack ? `\n${errorStack}` : ''
|
|
1289
|
-
}`
|
|
1290
|
-
: `DynamicWorkflow runner failed: ${failure.message}`;
|
|
1291
|
-
const headers = new Headers({
|
|
1292
|
-
authorization: `Bearer ${executorToken}`,
|
|
1293
|
-
'content-type': 'application/json',
|
|
1294
|
-
});
|
|
1295
|
-
const bypass = input.env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim();
|
|
1296
|
-
if (bypass) headers.set('x-vercel-protection-bypass', bypass);
|
|
1297
|
-
const body = JSON.stringify({
|
|
1298
|
-
action: 'append_run_events',
|
|
1299
|
-
playId: runId,
|
|
1300
|
-
events: [
|
|
1301
|
-
{
|
|
1302
|
-
type: 'run.failed',
|
|
1303
|
-
runId,
|
|
1304
|
-
source: 'coordinator',
|
|
1305
|
-
occurredAt: Date.now(),
|
|
1306
|
-
error: normalizedError,
|
|
1307
|
-
} satisfies PlayRunLedgerEvent,
|
|
1308
|
-
],
|
|
1309
|
-
});
|
|
1310
|
-
const url = `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`;
|
|
1311
|
-
const backoffMs = [200, 500, 1500];
|
|
1312
|
-
let lastError: unknown = null;
|
|
1313
|
-
for (let attempt = 0; attempt <= backoffMs.length; attempt += 1) {
|
|
1314
|
-
const requestId = crypto.randomUUID();
|
|
1315
|
-
try {
|
|
1316
|
-
headers.set('x-deepline-request-id', requestId);
|
|
1317
|
-
const response = await fetch(url, { method: 'POST', headers, body });
|
|
1318
|
-
if (response.ok) {
|
|
1319
|
-
if (attempt > 0) {
|
|
1320
|
-
recordCoordinatorEvent(
|
|
1321
|
-
'info',
|
|
1322
|
-
'coordinator.runtime_api.retry',
|
|
1323
|
-
'[coordinator.runtime_api.retry]',
|
|
1324
|
-
{
|
|
1325
|
-
action: 'append_run_events',
|
|
1326
|
-
status: 'ok',
|
|
1327
|
-
attempt,
|
|
1328
|
-
runId,
|
|
1329
|
-
requestId,
|
|
1330
|
-
baseOrigin: safeOrigin(baseUrl),
|
|
1331
|
-
},
|
|
1332
|
-
);
|
|
1333
|
-
}
|
|
1334
|
-
return;
|
|
1335
|
-
}
|
|
1336
|
-
lastError = new Error(
|
|
1337
|
-
`runtime API responded ${response.status}: ${(await response.text().catch(() => '')).slice(0, 400)}`,
|
|
1338
|
-
);
|
|
1339
|
-
if (
|
|
1340
|
-
response.status >= 400 &&
|
|
1341
|
-
response.status < 500 &&
|
|
1342
|
-
response.status !== 408 &&
|
|
1343
|
-
response.status !== 429
|
|
1344
|
-
) {
|
|
1345
|
-
break;
|
|
1346
|
-
}
|
|
1347
|
-
} catch (error) {
|
|
1348
|
-
lastError = error;
|
|
1349
|
-
}
|
|
1350
|
-
if (attempt < backoffMs.length) {
|
|
1351
|
-
recordCoordinatorEvent(
|
|
1352
|
-
'warn',
|
|
1353
|
-
'coordinator.runtime_api.retry',
|
|
1354
|
-
'[coordinator.runtime_api.retry]',
|
|
1355
|
-
{
|
|
1356
|
-
action: 'append_run_events',
|
|
1357
|
-
status: 'retry',
|
|
1358
|
-
attempt: attempt + 1,
|
|
1359
|
-
retryDelayMs: backoffMs[attempt],
|
|
1360
|
-
runId,
|
|
1361
|
-
requestId,
|
|
1362
|
-
baseOrigin: safeOrigin(baseUrl),
|
|
1363
|
-
error:
|
|
1364
|
-
lastError instanceof Error
|
|
1365
|
-
? lastError.message.slice(0, 300)
|
|
1366
|
-
: String(lastError).slice(0, 300),
|
|
1367
|
-
},
|
|
1368
|
-
);
|
|
1369
|
-
await new Promise((resolve) => setTimeout(resolve, backoffMs[attempt]));
|
|
1370
|
-
}
|
|
1371
|
-
}
|
|
1372
|
-
recordCoordinatorEvent(
|
|
1373
|
-
'error',
|
|
1374
|
-
'coordinator.workflow_runtime_failure.mark_failed',
|
|
1375
|
-
'[coordinator] failed to mark workflow runtime failure',
|
|
1376
|
-
{
|
|
1377
|
-
runId,
|
|
1378
|
-
message:
|
|
1379
|
-
lastError instanceof Error ? lastError.message : String(lastError),
|
|
1380
|
-
},
|
|
1381
|
-
);
|
|
1382
|
-
}
|
|
1383
|
-
|
|
1384
|
-
type StoredPlayArtifactPayload = {
|
|
1385
|
-
artifact?: {
|
|
1386
|
-
bundledCode?: string;
|
|
1387
|
-
artifactKind?: string;
|
|
1388
|
-
};
|
|
1389
|
-
};
|
|
1390
|
-
|
|
1391
|
-
const DYNAMIC_WORKER_COMPATIBILITY_DATE = '2026-05-01';
|
|
1392
|
-
|
|
1393
|
-
async function persistWorkflowRetryState(input: {
|
|
1394
|
-
env: CoordinatorEnv;
|
|
1395
|
-
runId: string;
|
|
1396
|
-
params: PlayWorkflowParams;
|
|
1397
|
-
}): Promise<{
|
|
1398
|
-
params?: PlayWorkflowParams;
|
|
1399
|
-
paramsRef?: WorkflowRetryParamsRef;
|
|
1400
|
-
}> {
|
|
1401
|
-
const retryParams = buildWorkflowRetryParams(input.params);
|
|
1402
|
-
const paramsBytes = jsonByteLength(retryParams);
|
|
1403
|
-
if (paramsBytes > WORKFLOW_RETRY_PARAMS_MAX_BYTES) {
|
|
1404
|
-
throw new Error(
|
|
1405
|
-
`workflow retry params too large: ${paramsBytes} bytes exceeds ${WORKFLOW_RETRY_PARAMS_MAX_BYTES}. Pass large payloads as staged files or ctx.csv inputs instead of inline JSON.`,
|
|
1406
|
-
);
|
|
1407
|
-
}
|
|
1408
|
-
let body: {
|
|
1409
|
-
runId: string;
|
|
1410
|
-
params?: PlayWorkflowParams;
|
|
1411
|
-
paramsRef?: WorkflowRetryParamsRef;
|
|
1412
|
-
paramsBytes: number;
|
|
1413
|
-
ttlMs: number;
|
|
1414
|
-
};
|
|
1415
|
-
if (paramsBytes > WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES) {
|
|
1416
|
-
const serialized = JSON.stringify(retryParams);
|
|
1417
|
-
const hash = stableHash(serialized);
|
|
1418
|
-
const storageKey = workflowRetryParamsStorageKey({
|
|
1419
|
-
runId: input.runId,
|
|
1420
|
-
hash,
|
|
1421
|
-
});
|
|
1422
|
-
const paramsRef: WorkflowRetryParamsRef = {
|
|
1423
|
-
storageKind: 'r2',
|
|
1424
|
-
storageKey,
|
|
1425
|
-
bytes: paramsBytes,
|
|
1426
|
-
hash,
|
|
1427
|
-
expiresAt: Date.now() + WORKFLOW_RETRY_STATE_TTL_MS,
|
|
1428
|
-
};
|
|
1429
|
-
await input.env.PLAYS_BUCKET.put(storageKey, serialized, {
|
|
1430
|
-
httpMetadata: { contentType: 'application/json' },
|
|
1431
|
-
});
|
|
1432
|
-
body = {
|
|
1433
|
-
runId: input.runId,
|
|
1434
|
-
paramsRef,
|
|
1435
|
-
paramsBytes,
|
|
1436
|
-
ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
|
|
1437
|
-
};
|
|
1438
|
-
await callRunScopedControl<{ ok?: unknown }>(
|
|
1439
|
-
input.env,
|
|
1440
|
-
input.runId,
|
|
1441
|
-
'/run-retry-state-put',
|
|
1442
|
-
{
|
|
1443
|
-
method: 'POST',
|
|
1444
|
-
body: JSON.stringify(body),
|
|
1445
|
-
},
|
|
1446
|
-
);
|
|
1447
|
-
return { paramsRef };
|
|
1448
|
-
} else {
|
|
1449
|
-
body = {
|
|
1450
|
-
runId: input.runId,
|
|
1451
|
-
params: retryParams,
|
|
1452
|
-
paramsBytes,
|
|
1453
|
-
ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
|
|
1454
|
-
};
|
|
1455
|
-
}
|
|
1456
|
-
await callRunScopedControl<{ ok?: unknown }>(
|
|
1457
|
-
input.env,
|
|
1458
|
-
input.runId,
|
|
1459
|
-
'/run-retry-state-put',
|
|
1460
|
-
{
|
|
1461
|
-
method: 'POST',
|
|
1462
|
-
body: JSON.stringify(body),
|
|
1463
|
-
},
|
|
1464
|
-
);
|
|
1465
|
-
return { params: retryParams };
|
|
1466
|
-
}
|
|
1467
|
-
|
|
1468
|
-
async function persistWorkflowLaunchState(input: {
|
|
1469
|
-
env: CoordinatorEnv;
|
|
1470
|
-
runId: string;
|
|
1471
|
-
params: PlayWorkflowParams;
|
|
1472
|
-
sessions: PreloadedRuntimeDbSession[];
|
|
1473
|
-
}): Promise<{
|
|
1474
|
-
retryExpiresAt?: number;
|
|
1475
|
-
dbSessionsExpiresAt?: number;
|
|
1476
|
-
sessionCount?: number;
|
|
1477
|
-
params?: PlayWorkflowParams;
|
|
1478
|
-
paramsRef?: WorkflowRetryParamsRef;
|
|
1479
|
-
}> {
|
|
1480
|
-
if (input.sessions.length === 0) {
|
|
1481
|
-
const state = await persistWorkflowRetryState({
|
|
1482
|
-
env: input.env,
|
|
1483
|
-
runId: input.runId,
|
|
1484
|
-
params: input.params,
|
|
1485
|
-
});
|
|
1486
|
-
return state;
|
|
1487
|
-
}
|
|
1488
|
-
const retryParams = buildWorkflowRetryParams(input.params);
|
|
1489
|
-
const paramsBytes = jsonByteLength(retryParams);
|
|
1490
|
-
let body: {
|
|
1491
|
-
runId: string;
|
|
1492
|
-
params?: PlayWorkflowParams;
|
|
1493
|
-
paramsRef?: WorkflowRetryParamsRef;
|
|
1494
|
-
paramsBytes: number;
|
|
1495
|
-
sessions: PreloadedRuntimeDbSession[];
|
|
1496
|
-
retryTtlMs: number;
|
|
1497
|
-
dbSessionsTtlMs: number;
|
|
1498
|
-
};
|
|
1499
|
-
if (paramsBytes > WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES) {
|
|
1500
|
-
const serialized = JSON.stringify(retryParams);
|
|
1501
|
-
const hash = stableHash(serialized);
|
|
1502
|
-
const storageKey = workflowRetryParamsStorageKey({
|
|
1503
|
-
runId: input.runId,
|
|
1504
|
-
hash,
|
|
1505
|
-
});
|
|
1506
|
-
const paramsRef: WorkflowRetryParamsRef = {
|
|
1507
|
-
storageKind: 'r2',
|
|
1508
|
-
storageKey,
|
|
1509
|
-
bytes: paramsBytes,
|
|
1510
|
-
hash,
|
|
1511
|
-
expiresAt: Date.now() + WORKFLOW_RETRY_STATE_TTL_MS,
|
|
1512
|
-
};
|
|
1513
|
-
await input.env.PLAYS_BUCKET.put(storageKey, serialized, {
|
|
1514
|
-
httpMetadata: { contentType: 'application/json' },
|
|
1515
|
-
});
|
|
1516
|
-
body = {
|
|
1517
|
-
runId: input.runId,
|
|
1518
|
-
paramsRef,
|
|
1519
|
-
paramsBytes,
|
|
1520
|
-
sessions: input.sessions,
|
|
1521
|
-
retryTtlMs: WORKFLOW_RETRY_STATE_TTL_MS,
|
|
1522
|
-
dbSessionsTtlMs: DB_SESSION_DEFAULT_TTL_SECONDS * 1000,
|
|
1523
|
-
};
|
|
1524
|
-
} else {
|
|
1525
|
-
body = {
|
|
1526
|
-
runId: input.runId,
|
|
1527
|
-
params: retryParams,
|
|
1528
|
-
paramsBytes,
|
|
1529
|
-
sessions: input.sessions,
|
|
1530
|
-
retryTtlMs: WORKFLOW_RETRY_STATE_TTL_MS,
|
|
1531
|
-
dbSessionsTtlMs: DB_SESSION_DEFAULT_TTL_SECONDS * 1000,
|
|
1532
|
-
};
|
|
1533
|
-
}
|
|
1534
|
-
const response = await callRunScopedControl<{
|
|
1535
|
-
ok?: unknown;
|
|
1536
|
-
retryExpiresAt?: unknown;
|
|
1537
|
-
dbSessionsExpiresAt?: unknown;
|
|
1538
|
-
sessionCount?: unknown;
|
|
1539
|
-
}>(input.env, input.runId, '/run-launch-state-put', {
|
|
1540
|
-
method: 'POST',
|
|
1541
|
-
body: JSON.stringify(body),
|
|
1542
|
-
});
|
|
1543
|
-
return {
|
|
1544
|
-
retryExpiresAt:
|
|
1545
|
-
typeof response.retryExpiresAt === 'number'
|
|
1546
|
-
? response.retryExpiresAt
|
|
1547
|
-
: undefined,
|
|
1548
|
-
dbSessionsExpiresAt:
|
|
1549
|
-
typeof response.dbSessionsExpiresAt === 'number'
|
|
1550
|
-
? response.dbSessionsExpiresAt
|
|
1551
|
-
: undefined,
|
|
1552
|
-
sessionCount:
|
|
1553
|
-
typeof response.sessionCount === 'number'
|
|
1554
|
-
? response.sessionCount
|
|
1555
|
-
: undefined,
|
|
1556
|
-
params: body.params,
|
|
1557
|
-
paramsRef: body.paramsRef,
|
|
1558
|
-
};
|
|
1559
|
-
}
|
|
1560
|
-
|
|
1561
|
-
function parseWorkflowRetryParamsRef(
|
|
1562
|
-
paramsRef: unknown,
|
|
1563
|
-
): WorkflowRetryParamsRef | null {
|
|
1564
|
-
if (!isRecord(paramsRef)) {
|
|
1565
|
-
return null;
|
|
1566
|
-
}
|
|
1567
|
-
const storageKind = paramsRef.storageKind;
|
|
1568
|
-
const storageKey = paramsRef.storageKey;
|
|
1569
|
-
const expectedBytes = paramsRef.bytes;
|
|
1570
|
-
const expectedHash = paramsRef.hash;
|
|
1571
|
-
const expiresAt = paramsRef.expiresAt;
|
|
1572
|
-
if (
|
|
1573
|
-
storageKind !== 'r2' ||
|
|
1574
|
-
typeof storageKey !== 'string' ||
|
|
1575
|
-
!storageKey.startsWith('plays/workflow-retry-params/') ||
|
|
1576
|
-
typeof expectedBytes !== 'number' ||
|
|
1577
|
-
!Number.isFinite(expectedBytes) ||
|
|
1578
|
-
typeof expectedHash !== 'string' ||
|
|
1579
|
-
!expectedHash ||
|
|
1580
|
-
typeof expiresAt !== 'number' ||
|
|
1581
|
-
!Number.isFinite(expiresAt)
|
|
1582
|
-
) {
|
|
1583
|
-
throw new Error('Invalid workflow retry params reference.');
|
|
1584
|
-
}
|
|
1585
|
-
return {
|
|
1586
|
-
storageKind,
|
|
1587
|
-
storageKey,
|
|
1588
|
-
bytes: expectedBytes,
|
|
1589
|
-
hash: expectedHash,
|
|
1590
|
-
expiresAt,
|
|
1591
|
-
};
|
|
1592
|
-
}
|
|
1593
|
-
|
|
1594
|
-
async function hydrateWorkflowRetryParams(input: {
|
|
1595
|
-
env: CoordinatorEnv;
|
|
1596
|
-
params: unknown;
|
|
1597
|
-
paramsRef: unknown;
|
|
1598
|
-
}): Promise<PlayWorkflowParams | null> {
|
|
1599
|
-
if (isRecord(input.params)) {
|
|
1600
|
-
return input.params as PlayWorkflowParams;
|
|
1601
|
-
}
|
|
1602
|
-
const paramsRef = parseWorkflowRetryParamsRef(input.paramsRef);
|
|
1603
|
-
if (!paramsRef) return null;
|
|
1604
|
-
const storageKey = paramsRef.storageKey;
|
|
1605
|
-
const expectedBytes = paramsRef.bytes;
|
|
1606
|
-
const expectedHash = paramsRef.hash;
|
|
1607
|
-
if (paramsRef.expiresAt <= Date.now()) {
|
|
1608
|
-
throw new Error('Workflow retry params reference expired.');
|
|
1609
|
-
}
|
|
1610
|
-
const object = await input.env.PLAYS_BUCKET.get(storageKey);
|
|
1611
|
-
if (!object) {
|
|
1612
|
-
throw new Error(`Workflow retry params missing from R2: ${storageKey}`);
|
|
1613
|
-
}
|
|
1614
|
-
const text = await object.text();
|
|
1615
|
-
const actualBytes = new TextEncoder().encode(text).length;
|
|
1616
|
-
if (actualBytes !== expectedBytes) {
|
|
1617
|
-
throw new Error(
|
|
1618
|
-
`Workflow retry params byte length mismatch: expected ${expectedBytes}, got ${actualBytes}.`,
|
|
1619
|
-
);
|
|
1620
|
-
}
|
|
1621
|
-
const actualHash = stableHash(text);
|
|
1622
|
-
if (actualHash !== expectedHash) {
|
|
1623
|
-
throw new Error('Workflow retry params hash mismatch.');
|
|
1624
|
-
}
|
|
1625
|
-
const parsed = JSON.parse(text) as unknown;
|
|
1626
|
-
return isRecord(parsed) ? (parsed as PlayWorkflowParams) : null;
|
|
1627
|
-
}
|
|
1628
|
-
|
|
1629
|
-
function workflowRetryStatePersistenceErrorResponse(input: {
|
|
1630
|
-
runId: string;
|
|
1631
|
-
error: unknown;
|
|
1632
|
-
}): Response {
|
|
1633
|
-
const message =
|
|
1634
|
-
input.error instanceof Error ? input.error.message : String(input.error);
|
|
1635
|
-
return Response.json(
|
|
1636
|
-
{
|
|
1637
|
-
error: {
|
|
1638
|
-
code: 'WORKFLOW_RETRY_STATE_PERSISTENCE_FAILED',
|
|
1639
|
-
message:
|
|
1640
|
-
'Failed to persist workflow retry state before dispatching the play run.',
|
|
1641
|
-
phase: 'coordinator_retry_state_persistence',
|
|
1642
|
-
runId: input.runId,
|
|
1643
|
-
cause: message,
|
|
1644
|
-
},
|
|
1645
|
-
},
|
|
1646
|
-
{ status: 503 },
|
|
1647
|
-
);
|
|
1648
|
-
}
|
|
1649
|
-
|
|
1650
|
-
async function claimWorkflowPlatformRetry(input: {
|
|
1651
|
-
env: CoordinatorEnv;
|
|
1652
|
-
runId: string;
|
|
1653
|
-
}): Promise<{
|
|
1654
|
-
claimed: boolean;
|
|
1655
|
-
attempts: number;
|
|
1656
|
-
params: PlayWorkflowParams | null;
|
|
1657
|
-
paramsRef?: WorkflowRetryParamsRef;
|
|
1658
|
-
}> {
|
|
1659
|
-
const body = await callRunScopedControl<{
|
|
1660
|
-
claimed?: unknown;
|
|
1661
|
-
attempts?: unknown;
|
|
1662
|
-
params?: unknown;
|
|
1663
|
-
paramsRef?: unknown;
|
|
1664
|
-
}>(input.env, input.runId, '/run-retry-claim', {
|
|
1665
|
-
method: 'POST',
|
|
1666
|
-
body: JSON.stringify({
|
|
1667
|
-
runId: input.runId,
|
|
1668
|
-
maxAttempts: PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
|
|
1669
|
-
}),
|
|
1670
|
-
});
|
|
1671
|
-
const params = await hydrateWorkflowRetryParams({
|
|
1672
|
-
env: input.env,
|
|
1673
|
-
params: body.params,
|
|
1674
|
-
paramsRef: body.paramsRef,
|
|
1675
|
-
});
|
|
1676
|
-
const paramsRef = parseWorkflowRetryParamsRef(body.paramsRef) ?? undefined;
|
|
1677
|
-
return {
|
|
1678
|
-
claimed: body.claimed === true,
|
|
1679
|
-
attempts: typeof body.attempts === 'number' ? body.attempts : 0,
|
|
1680
|
-
params,
|
|
1681
|
-
paramsRef,
|
|
1682
|
-
};
|
|
1683
|
-
}
|
|
1684
|
-
|
|
1685
|
-
function workflowRetryInstanceId(runId: string, attempt: number): string {
|
|
1686
|
-
const suffix = `retry-${attempt}-${stableHash(`${runId}:${attempt}`).slice(0, 8)}`;
|
|
1687
|
-
const base = workflowInstanceId(runId);
|
|
1688
|
-
const maxBaseLength = Math.max(1, 100 - suffix.length - 1);
|
|
1689
|
-
return `${base.slice(0, maxBaseLength)}-${suffix}`;
|
|
1690
|
-
}
|
|
1691
|
-
|
|
1692
|
-
function normalizeWorkflowRunAttempt(value: unknown): number {
|
|
1693
|
-
return typeof value === 'number' && Number.isFinite(value)
|
|
1694
|
-
? Math.max(0, Math.floor(value))
|
|
1695
|
-
: 0;
|
|
1696
|
-
}
|
|
1697
|
-
|
|
1698
|
-
async function restartWorkflowAfterPlatformReset(input: {
|
|
1699
|
-
env: CoordinatorEnv;
|
|
1700
|
-
ctx?: ExecutionContext;
|
|
1701
|
-
runId: string;
|
|
1702
|
-
oldInstance: WorkflowInstance;
|
|
1703
|
-
error: string;
|
|
1704
|
-
status: InstanceStatus;
|
|
1705
|
-
}): Promise<{
|
|
1706
|
-
retried: boolean;
|
|
1707
|
-
result: Record<string, unknown> | null;
|
|
1708
|
-
}> {
|
|
1709
|
-
const decision = decideWorkflowPlatformRetry({
|
|
1710
|
-
workflowStatus: String(input.status.status ?? ''),
|
|
1711
|
-
error: input.error,
|
|
1712
|
-
retryAttempts: 0,
|
|
1713
|
-
});
|
|
1714
|
-
if (decision.action !== 'retry') {
|
|
1715
|
-
return { retried: false, result: null };
|
|
1716
|
-
}
|
|
1717
|
-
const claim = await claimWorkflowPlatformRetry({
|
|
1718
|
-
env: input.env,
|
|
1719
|
-
runId: input.runId,
|
|
1720
|
-
}).catch((error) => {
|
|
1721
|
-
recordCoordinatorEvent(
|
|
1722
|
-
'warn',
|
|
1723
|
-
'coordinator.workflow_platform_retry.claim_failed',
|
|
1724
|
-
'[coordinator] workflow platform retry claim failed',
|
|
1725
|
-
{
|
|
1726
|
-
runId: input.runId,
|
|
1727
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1728
|
-
},
|
|
1729
|
-
);
|
|
1730
|
-
return null;
|
|
1731
|
-
});
|
|
1732
|
-
if (!claim?.claimed || !claim.params) {
|
|
1733
|
-
return { retried: false, result: null };
|
|
1734
|
-
}
|
|
1735
|
-
const retryInstanceId = workflowRetryInstanceId(input.runId, claim.attempts);
|
|
1736
|
-
const retryStartedAt = Date.now();
|
|
1737
|
-
let retryInstance: WorkflowInstance | null = null;
|
|
1738
|
-
const retryParams = { ...claim.params, runAttempt: claim.attempts };
|
|
1739
|
-
try {
|
|
1740
|
-
retryInstance = await createDynamicWorkflowInstance({
|
|
1741
|
-
env: input.env,
|
|
1742
|
-
id: retryInstanceId,
|
|
1743
|
-
params: workflowCreateParams(retryParams, {
|
|
1744
|
-
params: retryParams,
|
|
1745
|
-
paramsRef: claim.paramsRef,
|
|
1746
|
-
}),
|
|
1747
|
-
metadataParams: retryParams,
|
|
1748
|
-
});
|
|
1749
|
-
await recordWorkflowInstanceId({
|
|
1750
|
-
env: input.env,
|
|
1751
|
-
runId: input.runId,
|
|
1752
|
-
instanceId: retryInstance.id,
|
|
1753
|
-
});
|
|
1754
|
-
input.ctx?.waitUntil(input.oldInstance.terminate().catch(() => undefined));
|
|
1755
|
-
recordCoordinatorPerfTraceBuffered(input.env, input.ctx, {
|
|
1756
|
-
runId: input.runId,
|
|
1757
|
-
phase: 'coordinator.platform_deploy_retry',
|
|
1758
|
-
ms: Date.now() - retryStartedAt,
|
|
1759
|
-
graphHash: claim.params.graphHash ?? null,
|
|
1760
|
-
extra: {
|
|
1761
|
-
retryAttempt: claim.attempts,
|
|
1762
|
-
retryInstanceId: retryInstance.id,
|
|
1763
|
-
reason: decision.reason,
|
|
1764
|
-
},
|
|
1765
|
-
});
|
|
1766
|
-
return {
|
|
1767
|
-
retried: true,
|
|
1768
|
-
result: {
|
|
1769
|
-
runId: input.runId,
|
|
1770
|
-
playName: claim.params.playName,
|
|
1771
|
-
status: 'running',
|
|
1772
|
-
result: null,
|
|
1773
|
-
error: null,
|
|
1774
|
-
retry: {
|
|
1775
|
-
reason: decision.reason,
|
|
1776
|
-
attempt: claim.attempts,
|
|
1777
|
-
message: decision.message,
|
|
1778
|
-
},
|
|
1779
|
-
},
|
|
1780
|
-
};
|
|
1781
|
-
} finally {
|
|
1782
|
-
disposeRpcStub(retryInstance);
|
|
1783
|
-
}
|
|
1784
|
-
}
|
|
1785
|
-
|
|
1786
|
-
export class CoordinatorControl extends WorkerEntrypoint<
|
|
1787
|
-
CoordinatorEnv,
|
|
1788
|
-
undefined
|
|
1789
|
-
> {
|
|
1790
|
-
async signal(
|
|
1791
|
-
runId: string,
|
|
1792
|
-
body: Record<string, unknown>,
|
|
1793
|
-
): Promise<Record<string, unknown>> {
|
|
1794
|
-
const response = await handleWorkflowRoute({
|
|
1795
|
-
runId,
|
|
1796
|
-
action: 'signal',
|
|
1797
|
-
request: new Request(
|
|
1798
|
-
`https://deepline.coordinator.internal/workflow/${encodeURIComponent(
|
|
1799
|
-
runId,
|
|
1800
|
-
)}/signal`,
|
|
1801
|
-
{
|
|
1802
|
-
method: 'POST',
|
|
1803
|
-
headers: {
|
|
1804
|
-
'content-type': 'application/json',
|
|
1805
|
-
'x-deepline-request-id': crypto.randomUUID(),
|
|
1806
|
-
},
|
|
1807
|
-
body: JSON.stringify(body),
|
|
1808
|
-
},
|
|
1809
|
-
),
|
|
1810
|
-
env: this.env,
|
|
1811
|
-
});
|
|
1812
|
-
const text = await response.text().catch(() => '');
|
|
1813
|
-
let parsed: Record<string, unknown> = {};
|
|
1814
|
-
try {
|
|
1815
|
-
parsed = text ? (JSON.parse(text) as Record<string, unknown>) : {};
|
|
1816
|
-
} catch {
|
|
1817
|
-
parsed = { error: text };
|
|
1818
|
-
}
|
|
1819
|
-
if (!response.ok) {
|
|
1820
|
-
const error = isRecord(parsed.error) ? parsed.error : null;
|
|
1821
|
-
const message =
|
|
1822
|
-
(typeof error?.message === 'string' && error.message.trim()) ||
|
|
1823
|
-
(typeof parsed.error === 'string' && parsed.error.trim()) ||
|
|
1824
|
-
text.slice(0, 800) ||
|
|
1825
|
-
`Coordinator signal failed with ${response.status}.`;
|
|
1826
|
-
throw new Error(message);
|
|
1827
|
-
}
|
|
1828
|
-
return parsed;
|
|
1829
|
-
}
|
|
1830
|
-
|
|
1831
|
-
async recordPerfTrace(
|
|
1832
|
-
runId: string,
|
|
1833
|
-
payload: CoordinatorPerfTracePayload,
|
|
1834
|
-
): Promise<void> {
|
|
1835
|
-
if (!runId || payload.runId !== runId) {
|
|
1836
|
-
throw new Error('Trace runId mismatch.');
|
|
1837
|
-
}
|
|
1838
|
-
await appendCoordinatorPerfTrace(this.env, payload);
|
|
1839
|
-
}
|
|
1840
|
-
|
|
1841
|
-
async recordRunEvent(
|
|
1842
|
-
runId: string,
|
|
1843
|
-
event: CoordinatorRunEvent,
|
|
1844
|
-
): Promise<void> {
|
|
1845
|
-
if (!runId || event.runId !== runId) {
|
|
1846
|
-
throw new Error('Run event runId mismatch.');
|
|
1847
|
-
}
|
|
1848
|
-
await appendCoordinatorRunEvent(this.env, event);
|
|
1849
|
-
}
|
|
1850
|
-
|
|
1851
|
-
async readTerminalState(
|
|
1852
|
-
runId: string,
|
|
1853
|
-
): Promise<CoordinatorTerminalState | null> {
|
|
1854
|
-
if (!runId) {
|
|
1855
|
-
throw new Error('runId is required.');
|
|
1856
|
-
}
|
|
1857
|
-
return await readCoordinatorTerminalState(this.env, runId);
|
|
1858
|
-
}
|
|
1859
|
-
|
|
1860
|
-
/**
|
|
1861
|
-
* Distributed Rate State Backend acquire: lease up to `requested` request-
|
|
1862
|
-
* window permits for `bucketId` (`<orgId>:<provider>`) from the per-bucket
|
|
1863
|
-
* rate-state Durable Object. See CoordinatorRateStateBackend + dedup-do.ts.
|
|
1864
|
-
*/
|
|
1865
|
-
async rateAcquire(input: {
|
|
1866
|
-
bucketId: string;
|
|
1867
|
-
rules: Array<{
|
|
1868
|
-
ruleId: string;
|
|
1869
|
-
requestsPerWindow: number;
|
|
1870
|
-
windowMs: number;
|
|
1871
|
-
maxConcurrency: number | null;
|
|
1872
|
-
}>;
|
|
1873
|
-
requested: number;
|
|
1874
|
-
}): Promise<{ granted: number; waitMs: number }> {
|
|
1875
|
-
if (!input.bucketId || !input.bucketId.trim()) {
|
|
1876
|
-
throw new Error('bucketId is required.');
|
|
1877
|
-
}
|
|
1878
|
-
return await callRateBucketControl<{ granted: number; waitMs: number }>(
|
|
1879
|
-
this.env,
|
|
1880
|
-
input.bucketId,
|
|
1881
|
-
'/rate-acquire',
|
|
1882
|
-
input,
|
|
1883
|
-
);
|
|
1884
|
-
}
|
|
1885
|
-
|
|
1886
|
-
async ratePenalize(input: {
|
|
1887
|
-
bucketId: string;
|
|
1888
|
-
cooldownMs: number;
|
|
1889
|
-
}): Promise<void> {
|
|
1890
|
-
if (!input.bucketId || !input.bucketId.trim()) {
|
|
1891
|
-
throw new Error('bucketId is required.');
|
|
1892
|
-
}
|
|
1893
|
-
await callRateBucketControl<{ ok?: unknown }>(
|
|
1894
|
-
this.env,
|
|
1895
|
-
input.bucketId,
|
|
1896
|
-
'/rate-penalize',
|
|
1897
|
-
input,
|
|
1898
|
-
);
|
|
1899
|
-
}
|
|
1900
|
-
}
|
|
1901
|
-
|
|
1902
|
-
/**
|
|
1903
|
-
* Custom DynamicWorkflow class — replaces createDynamicWorkflowEntrypoint
|
|
1904
|
-
* so we can wrap the framework's runner.run(innerEvent, step) RPC in a
|
|
1905
|
-
* try/catch and surface the underlying error. The library's helper hides
|
|
1906
|
-
* that call inside dispatchWorkflow which we now invoke directly with
|
|
1907
|
-
* extra instrumentation around the inner runner call.
|
|
1908
|
-
*/
|
|
1909
|
-
export class DynamicWorkflow extends WorkflowEntrypoint<
|
|
1910
|
-
CoordinatorEnv,
|
|
1911
|
-
Record<string, unknown>
|
|
1912
|
-
> {
|
|
1913
|
-
async run(event: unknown, step: unknown): Promise<unknown> {
|
|
1914
|
-
const trace: CoordinatorPerfTraceSink = (traceEvent) =>
|
|
1915
|
-
recordCoordinatorPerfTraceBuffered(this.env, this.ctx, traceEvent);
|
|
1916
|
-
const workflowEvent = event as {
|
|
1917
|
-
payload?: unknown;
|
|
1918
|
-
timestamp?: Date;
|
|
1919
|
-
instanceId?: string;
|
|
1920
|
-
};
|
|
1921
|
-
const entryTrace = readWorkflowTraceContext(event);
|
|
1922
|
-
trace({
|
|
1923
|
-
runId: entryTrace.runId,
|
|
1924
|
-
phase: 'coordinator.workflow_run_entry',
|
|
1925
|
-
ms: 0,
|
|
1926
|
-
graphHash: entryTrace.graphHash,
|
|
1927
|
-
extra: {
|
|
1928
|
-
instanceId: entryTrace.instanceId,
|
|
1929
|
-
},
|
|
1930
|
-
});
|
|
1931
|
-
if (entryTrace.runId) {
|
|
1932
|
-
this.ctx.waitUntil(
|
|
1933
|
-
appendCoordinatorRunEvent(this.env, {
|
|
1934
|
-
runId: entryTrace.runId,
|
|
1935
|
-
type: 'status',
|
|
1936
|
-
status: 'running',
|
|
1937
|
-
ts: Date.now(),
|
|
1938
|
-
}).catch((error) => {
|
|
1939
|
-
recordCoordinatorEvent(
|
|
1940
|
-
'warn',
|
|
1941
|
-
'coordinator.workflow_entry.status_event_failed',
|
|
1942
|
-
'[coordinator] workflow entry status event failed',
|
|
1943
|
-
{
|
|
1944
|
-
runId: entryTrace.runId,
|
|
1945
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1946
|
-
},
|
|
1947
|
-
);
|
|
1948
|
-
}),
|
|
1949
|
-
);
|
|
1950
|
-
}
|
|
1951
|
-
if (entryTrace.submittedAt !== null) {
|
|
1952
|
-
trace({
|
|
1953
|
-
runId: entryTrace.runId,
|
|
1954
|
-
phase: 'coordinator.workflow_start_gap',
|
|
1955
|
-
ms: Math.max(0, Date.now() - entryTrace.submittedAt),
|
|
1956
|
-
graphHash: entryTrace.graphHash,
|
|
1957
|
-
extra: {
|
|
1958
|
-
instanceId: entryTrace.instanceId,
|
|
1959
|
-
},
|
|
1960
|
-
});
|
|
1961
|
-
}
|
|
1962
|
-
let dispatchedEvent = event;
|
|
1963
|
-
dispatchedEvent = await hydrateWorkflowLaunchParams({
|
|
1964
|
-
env: this.env,
|
|
1965
|
-
event: dispatchedEvent,
|
|
1966
|
-
trace,
|
|
1967
|
-
});
|
|
1968
|
-
try {
|
|
1969
|
-
dispatchedEvent = await hydrateWorkflowDbSessions({
|
|
1970
|
-
env: this.env,
|
|
1971
|
-
event: dispatchedEvent,
|
|
1972
|
-
trace,
|
|
1973
|
-
});
|
|
1974
|
-
} catch (hydrateError) {
|
|
1975
|
-
// The workflow re-entered but cannot read its externalized DB sessions.
|
|
1976
|
-
// A 410 (expired retention window) is known-permanent: the sessions are
|
|
1977
|
-
// stored once at submit and never re-minted here, so the workflow can
|
|
1978
|
-
// only keep failing. Publish a clean run.failed instead of letting the
|
|
1979
|
-
// raw transport error escape uncaught — this call sits OUTSIDE the
|
|
1980
|
-
// dispatchWorkflow try/catch that authors run.failed.
|
|
1981
|
-
//
|
|
1982
|
-
// A 404 that survived the read-retry ladder (`workflow db session
|
|
1983
|
-
// lookup failed 404`) rethrows WITHOUT marking, like every other error:
|
|
1984
|
-
// sessions are persisted (awaited) before the instance is created, so a
|
|
1985
|
-
// 404 here is anomalous — most plausibly a framework re-entry against an
|
|
1986
|
-
// ALREADY-TERMINAL run whose DO state the post-finish alarm cleaned up.
|
|
1987
|
-
// Authoring run.failed there could flip a completed (billed + durable)
|
|
1988
|
-
// run to failed via newest-terminal-wins. Only the 410 is safe to mark.
|
|
1989
|
-
const hydrateMessage =
|
|
1990
|
-
hydrateError instanceof Error
|
|
1991
|
-
? hydrateError.message
|
|
1992
|
-
: String(hydrateError);
|
|
1993
|
-
if (hydrateMessage.includes('workflow db session lookup failed 410')) {
|
|
1994
|
-
await markWorkflowRuntimeFailure({
|
|
1995
|
-
env: this.env,
|
|
1996
|
-
event: dispatchedEvent,
|
|
1997
|
-
error: hydrateError,
|
|
1998
|
-
}).catch((markError) => {
|
|
1999
|
-
recordCoordinatorEvent(
|
|
2000
|
-
'error',
|
|
2001
|
-
'coordinator.db_session_expired.failure_forward_failed',
|
|
2002
|
-
'[coordinator] failed to forward expired-db-session run failure',
|
|
2003
|
-
{
|
|
2004
|
-
runId: entryTrace.runId,
|
|
2005
|
-
message:
|
|
2006
|
-
markError instanceof Error
|
|
2007
|
-
? markError.message
|
|
2008
|
-
: String(markError),
|
|
2009
|
-
},
|
|
2010
|
-
);
|
|
2011
|
-
});
|
|
2012
|
-
if (entryTrace.runId) {
|
|
2013
|
-
await writeCoordinatorTerminalState(this.env, {
|
|
2014
|
-
runId: entryTrace.runId,
|
|
2015
|
-
status: 'failed',
|
|
2016
|
-
error:
|
|
2017
|
-
normalizeCoordinatorWorkflowRuntimeFailure(hydrateError).message,
|
|
2018
|
-
}).catch(() => undefined);
|
|
2019
|
-
}
|
|
2020
|
-
}
|
|
2021
|
-
throw hydrateError;
|
|
2022
|
-
}
|
|
2023
|
-
const dispatchTrace = readWorkflowTraceContext(dispatchedEvent);
|
|
2024
|
-
trace({
|
|
2025
|
-
runId: dispatchTrace.runId,
|
|
2026
|
-
phase: 'coordinator.dispatch_workflow_entry',
|
|
2027
|
-
ms: 0,
|
|
2028
|
-
graphHash: dispatchTrace.graphHash,
|
|
2029
|
-
extra: {
|
|
2030
|
-
instanceId: dispatchTrace.instanceId,
|
|
2031
|
-
},
|
|
2032
|
-
});
|
|
2033
|
-
|
|
2034
|
-
const dispatchWorkflowStartedAt = Date.now();
|
|
2035
|
-
try {
|
|
2036
|
-
return await dispatchWorkflow(
|
|
2037
|
-
{ env: this.env, ctx: this.ctx },
|
|
2038
|
-
dispatchedEvent as Parameters<typeof dispatchWorkflow>[1],
|
|
2039
|
-
step as Parameters<typeof dispatchWorkflow>[2],
|
|
2040
|
-
async ({ metadata, env }) => {
|
|
2041
|
-
const graphHash = readMetadataString(metadata, 'graphHash');
|
|
2042
|
-
const artifactStorageKey = readMetadataString(
|
|
2043
|
-
metadata,
|
|
2044
|
-
'artifactStorageKey',
|
|
2045
|
-
);
|
|
2046
|
-
const runIdForTrace =
|
|
2047
|
-
typeof (metadata as Record<string, unknown>).runId === 'string'
|
|
2048
|
-
? ((metadata as Record<string, unknown>).runId as string)
|
|
2049
|
-
: graphHash;
|
|
2050
|
-
const loaderStartedAt = Date.now();
|
|
2051
|
-
trace({
|
|
2052
|
-
runId: runIdForTrace,
|
|
2053
|
-
phase: 'coordinator.loader_callback_entry',
|
|
2054
|
-
ms: 0,
|
|
2055
|
-
graphHash,
|
|
2056
|
-
extra: { artifactStorageKey },
|
|
2057
|
-
});
|
|
2058
|
-
const stub = loadDynamicPlayWorkerSync(
|
|
2059
|
-
env,
|
|
2060
|
-
{
|
|
2061
|
-
runId: runIdForTrace,
|
|
2062
|
-
graphHash,
|
|
2063
|
-
artifactStorageKey,
|
|
2064
|
-
artifactHash:
|
|
2065
|
-
typeof metadata.artifactHash === 'string'
|
|
2066
|
-
? metadata.artifactHash
|
|
2067
|
-
: null,
|
|
2068
|
-
dynamicWorkerCode:
|
|
2069
|
-
typeof metadata.dynamicWorkerCode === 'string'
|
|
2070
|
-
? metadata.dynamicWorkerCode
|
|
2071
|
-
: null,
|
|
2072
|
-
dynamicWorkerCodeHash:
|
|
2073
|
-
typeof metadata.dynamicWorkerCodeHash === 'string'
|
|
2074
|
-
? metadata.dynamicWorkerCodeHash
|
|
2075
|
-
: null,
|
|
2076
|
-
integrationMode: normalizeIntegrationMode(
|
|
2077
|
-
metadata.integrationMode,
|
|
2078
|
-
),
|
|
2079
|
-
packagedFiles: normalizePackagedFiles(metadata.packagedFiles),
|
|
2080
|
-
},
|
|
2081
|
-
trace,
|
|
2082
|
-
);
|
|
2083
|
-
const entrypointStartedAt = Date.now();
|
|
2084
|
-
const entrypoint = stub.getEntrypoint(
|
|
2085
|
-
'TenantWorkflow',
|
|
2086
|
-
) as unknown as WorkflowRunner;
|
|
2087
|
-
trace({
|
|
2088
|
-
runId: runIdForTrace,
|
|
2089
|
-
phase: 'coordinator.loader_get_entrypoint',
|
|
2090
|
-
ms: Date.now() - entrypointStartedAt,
|
|
2091
|
-
graphHash,
|
|
2092
|
-
});
|
|
2093
|
-
trace({
|
|
2094
|
-
runId: runIdForTrace,
|
|
2095
|
-
phase: 'coordinator.loader_compile',
|
|
2096
|
-
ms: Date.now() - loaderStartedAt,
|
|
2097
|
-
graphHash,
|
|
2098
|
-
});
|
|
2099
|
-
// Wrap the entrypoint so its run() failure surfaces here rather
|
|
2100
|
-
// than disappearing into the framework's silent rpcMethod=run
|
|
2101
|
-
// exception path.
|
|
2102
|
-
return {
|
|
2103
|
-
run: async (innerEvent: unknown, innerStep: unknown) => {
|
|
2104
|
-
const innerStartedAt = Date.now();
|
|
2105
|
-
trace({
|
|
2106
|
-
runId: runIdForTrace,
|
|
2107
|
-
phase: 'coordinator.runner_run_start',
|
|
2108
|
-
ms: 0,
|
|
2109
|
-
graphHash,
|
|
2110
|
-
});
|
|
2111
|
-
try {
|
|
2112
|
-
const result = await (
|
|
2113
|
-
entrypoint as unknown as {
|
|
2114
|
-
run(e: unknown, s: unknown): Promise<unknown>;
|
|
2115
|
-
}
|
|
2116
|
-
).run(innerEvent, innerStep);
|
|
2117
|
-
const output = isRecord(result) ? result : null;
|
|
2118
|
-
const terminalResult = output?.result ?? result;
|
|
2119
|
-
const terminalStartedAt = Date.now();
|
|
2120
|
-
try {
|
|
2121
|
-
await writeCoordinatorTerminalState(env, {
|
|
2122
|
-
runId: runIdForTrace,
|
|
2123
|
-
status: 'completed',
|
|
2124
|
-
result: terminalResult,
|
|
2125
|
-
totalRows: output?.totalRows ?? output?.outputRows ?? null,
|
|
2126
|
-
durationMs: output?.durationMs ?? null,
|
|
2127
|
-
playName:
|
|
2128
|
-
typeof output?.playName === 'string'
|
|
2129
|
-
? output.playName
|
|
2130
|
-
: null,
|
|
2131
|
-
liveLogs: sanitizeLiveLogLines(output?.liveLogs),
|
|
2132
|
-
liveNodeProgress: output?.liveNodeProgress ?? null,
|
|
2133
|
-
});
|
|
2134
|
-
trace({
|
|
2135
|
-
runId: runIdForTrace,
|
|
2136
|
-
phase: 'coordinator.terminal_state_write',
|
|
2137
|
-
ms: Date.now() - terminalStartedAt,
|
|
2138
|
-
graphHash,
|
|
2139
|
-
extra: { status: 'completed' },
|
|
2140
|
-
});
|
|
2141
|
-
} catch (terminalError) {
|
|
2142
|
-
console.warn(
|
|
2143
|
-
'[coordinator] completed terminal cache write failed; preserving completed run',
|
|
2144
|
-
{
|
|
2145
|
-
graphHash,
|
|
2146
|
-
runId: runIdForTrace,
|
|
2147
|
-
error:
|
|
2148
|
-
terminalError instanceof Error
|
|
2149
|
-
? terminalError.message
|
|
2150
|
-
: String(terminalError),
|
|
2151
|
-
},
|
|
2152
|
-
);
|
|
2153
|
-
}
|
|
2154
|
-
trace({
|
|
2155
|
-
runId: runIdForTrace,
|
|
2156
|
-
phase: 'coordinator.runner_run',
|
|
2157
|
-
ms: Date.now() - innerStartedAt,
|
|
2158
|
-
graphHash,
|
|
2159
|
-
});
|
|
2160
|
-
return result;
|
|
2161
|
-
} catch (innerError) {
|
|
2162
|
-
const failure =
|
|
2163
|
-
normalizeCoordinatorWorkflowRuntimeFailure(innerError);
|
|
2164
|
-
recordCoordinatorEvent(
|
|
2165
|
-
'error',
|
|
2166
|
-
'coordinator.dynamic_workflow.runner_failed',
|
|
2167
|
-
'[coordinator] DynamicWorkflow runner.run threw',
|
|
2168
|
-
{
|
|
2169
|
-
graphHash,
|
|
2170
|
-
message:
|
|
2171
|
-
innerError instanceof Error
|
|
2172
|
-
? innerError.message
|
|
2173
|
-
: String(innerError),
|
|
2174
|
-
name: innerError instanceof Error ? innerError.name : null,
|
|
2175
|
-
stack:
|
|
2176
|
-
innerError instanceof Error &&
|
|
2177
|
-
typeof innerError.stack === 'string'
|
|
2178
|
-
? innerError.stack.split('\n').slice(0, 12).join('\n')
|
|
2179
|
-
: null,
|
|
2180
|
-
},
|
|
2181
|
-
);
|
|
2182
|
-
if (
|
|
2183
|
-
shouldPublishWorkflowRuntimeFailure({
|
|
2184
|
-
error: innerError,
|
|
2185
|
-
retryAttempts: 0,
|
|
2186
|
-
})
|
|
2187
|
-
) {
|
|
2188
|
-
await markWorkflowRuntimeFailure({
|
|
2189
|
-
env,
|
|
2190
|
-
event: innerEvent,
|
|
2191
|
-
error: innerError,
|
|
2192
|
-
}).catch((markError) => {
|
|
2193
|
-
recordCoordinatorEvent(
|
|
2194
|
-
'error',
|
|
2195
|
-
'coordinator.dynamic_workflow.failure_forward_failed',
|
|
2196
|
-
'[coordinator] failed to forward DynamicWorkflow runner error',
|
|
2197
|
-
{
|
|
2198
|
-
graphHash,
|
|
2199
|
-
message:
|
|
2200
|
-
markError instanceof Error
|
|
2201
|
-
? markError.message
|
|
2202
|
-
: String(markError),
|
|
2203
|
-
},
|
|
2204
|
-
);
|
|
2205
|
-
});
|
|
2206
|
-
await writeCoordinatorTerminalState(env, {
|
|
2207
|
-
runId: runIdForTrace,
|
|
2208
|
-
status: 'failed',
|
|
2209
|
-
error: failure.message,
|
|
2210
|
-
}).catch(() => undefined);
|
|
2211
|
-
} else {
|
|
2212
|
-
recordCoordinatorEvent(
|
|
2213
|
-
'warn',
|
|
2214
|
-
'coordinator.dynamic_workflow.platform_reset_retrying',
|
|
2215
|
-
'[coordinator] DynamicWorkflow platform reset will be retried; not publishing durable run.failed',
|
|
2216
|
-
{
|
|
2217
|
-
graphHash,
|
|
2218
|
-
runId: runIdForTrace,
|
|
2219
|
-
errorCode: failure.code,
|
|
2220
|
-
retryLimit: PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
|
|
2221
|
-
},
|
|
2222
|
-
);
|
|
2223
|
-
}
|
|
2224
|
-
throw innerError;
|
|
2225
|
-
}
|
|
2226
|
-
},
|
|
2227
|
-
};
|
|
2228
|
-
},
|
|
2229
|
-
);
|
|
2230
|
-
} finally {
|
|
2231
|
-
trace({
|
|
2232
|
-
runId: dispatchTrace.runId,
|
|
2233
|
-
phase: 'coordinator.dispatch_workflow_total',
|
|
2234
|
-
ms: Date.now() - dispatchWorkflowStartedAt,
|
|
2235
|
-
graphHash: dispatchTrace.graphHash,
|
|
2236
|
-
extra: {
|
|
2237
|
-
instanceId: dispatchTrace.instanceId,
|
|
2238
|
-
},
|
|
2239
|
-
});
|
|
2240
|
-
}
|
|
2241
|
-
}
|
|
2242
|
-
}
|
|
2243
|
-
|
|
2244
|
-
const coordinatorEntrypoint = {
|
|
2245
|
-
/**
|
|
2246
|
-
* HTTP entrypoint for the Vercel app to dispatch into. Routes:
|
|
2247
|
-
* POST /workflow/{runId}/submit → PLAY_WORKFLOW.create({ id, params })
|
|
2248
|
-
* GET /workflow/{runId}/tail → ordered live run events after a cursor
|
|
2249
|
-
* POST /workflow/{runId}/cancel → Workflow instance terminate
|
|
2250
|
-
* POST /workflow/{runId}/signal → integration_event
|
|
2251
|
-
* GET /workflow/{runId}/result → terminal envelope
|
|
2252
|
-
* GET /health → liveness
|
|
2253
|
-
*/
|
|
2254
|
-
async fetch(
|
|
2255
|
-
request: Request,
|
|
2256
|
-
env: CoordinatorEnv,
|
|
2257
|
-
ctx?: ExecutionContext,
|
|
2258
|
-
): Promise<Response> {
|
|
2259
|
-
// Top-level guard: no coordinator route may let an exception escape to
|
|
2260
|
-
// Cloudflare's default "Worker threw exception" HTML error page. Each
|
|
2261
|
-
// route group below additionally wraps itself for precise phase/runId
|
|
2262
|
-
// context; this outer catch is the backstop for URL parsing, route
|
|
2263
|
-
// matching, and any future route added without its own envelope.
|
|
2264
|
-
try {
|
|
2265
|
-
return await coordinatorRouteFetch(request, env, ctx);
|
|
2266
|
-
} catch (error) {
|
|
2267
|
-
return coordinatorRouteErrorResponse({
|
|
2268
|
-
logTag: '[coordinator.fetch.error]',
|
|
2269
|
-
code: 'COORDINATOR_ROUTE_FAILED',
|
|
2270
|
-
phase: 'coordinator.fetch',
|
|
2271
|
-
runId: null,
|
|
2272
|
-
error,
|
|
2273
|
-
});
|
|
2274
|
-
}
|
|
2275
|
-
},
|
|
2276
|
-
async tail(events: unknown[], env: CoordinatorEnv): Promise<void> {
|
|
2277
|
-
await flushTailRunLogs(events, env);
|
|
2278
|
-
},
|
|
2279
|
-
async scheduled(
|
|
2280
|
-
_controller: unknown,
|
|
2281
|
-
_env: CoordinatorEnv,
|
|
2282
|
-
_ctx?: ExecutionContext,
|
|
2283
|
-
): Promise<void> {},
|
|
2284
|
-
};
|
|
2285
|
-
|
|
2286
|
-
export default coordinatorEntrypoint;
|
|
2287
|
-
|
|
2288
|
-
/**
|
|
2289
|
-
* Route dispatcher for the coordinator fetch handler. Extracted as a standalone
|
|
2290
|
-
* function so the outer {@link coordinatorEntrypoint.fetch} try/catch backstop
|
|
2291
|
-
* is not coupled to `this` binding. Every route group wraps itself with a JSON
|
|
2292
|
-
* envelope; the outer catch covers URL parsing / route matching escapes.
|
|
2293
|
-
*/
|
|
2294
|
-
async function coordinatorRouteFetch(
|
|
2295
|
-
request: Request,
|
|
2296
|
-
env: CoordinatorEnv,
|
|
2297
|
-
ctx?: ExecutionContext,
|
|
2298
|
-
): Promise<Response> {
|
|
2299
|
-
const url = new URL(request.url);
|
|
2300
|
-
if (url.pathname === '/health') {
|
|
2301
|
-
const headers = new Headers();
|
|
2302
|
-
headers.set(
|
|
2303
|
-
'x-deepline-runtime-deploy-version',
|
|
2304
|
-
env.CF_VERSION_METADATA?.id ??
|
|
2305
|
-
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
|
|
2306
|
-
'',
|
|
2307
|
-
);
|
|
2308
|
-
if (env.DEEPLINE_COORDINATOR_DEPLOY_MARKER) {
|
|
2309
|
-
headers.set(
|
|
2310
|
-
'x-deepline-deploy-marker',
|
|
2311
|
-
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER,
|
|
2312
|
-
);
|
|
2313
|
-
}
|
|
2314
|
-
return new Response('ok', { status: 200, headers });
|
|
2315
|
-
}
|
|
2316
|
-
if (url.pathname === '/internal-token/probe') {
|
|
2317
|
-
const authError = authorizeCoordinatorControlRequest({ request, env });
|
|
2318
|
-
if (authError) return authError;
|
|
2319
|
-
return Response.json({
|
|
2320
|
-
ok: true,
|
|
2321
|
-
deployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
|
|
2322
|
-
runtimeDeployVersion:
|
|
2323
|
-
env.CF_VERSION_METADATA?.id ??
|
|
2324
|
-
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
|
|
2325
|
-
null,
|
|
2326
|
-
});
|
|
2327
|
-
}
|
|
2328
|
-
if (url.pathname === '/warmup/submit') {
|
|
2329
|
-
const authError = authorizeCoordinatorControlRequest({ request, env });
|
|
2330
|
-
if (authError) return authError;
|
|
2331
|
-
try {
|
|
2332
|
-
return await handleCoordinatorWarmup(request, env, ctx);
|
|
2333
|
-
} catch (error) {
|
|
2334
|
-
return coordinatorRouteErrorResponse({
|
|
2335
|
-
logTag: '[coordinator.warmup.error]',
|
|
2336
|
-
code: 'COORDINATOR_WARMUP_FAILED',
|
|
2337
|
-
phase: 'coordinator.warmup',
|
|
2338
|
-
runId: null,
|
|
2339
|
-
error,
|
|
2340
|
-
});
|
|
2341
|
-
}
|
|
2342
|
-
}
|
|
2343
|
-
if (url.pathname === '/tail-log-token/probe') {
|
|
2344
|
-
const authError = authorizeCoordinatorControlRequest({ request, env });
|
|
2345
|
-
if (authError) return authError;
|
|
2346
|
-
const expectedTailLogToken = env.DEEPLINE_TAIL_LOG_TOKEN?.trim();
|
|
2347
|
-
if (!expectedTailLogToken) {
|
|
2348
|
-
return Response.json(
|
|
2349
|
-
{ ok: false, error: 'tail log token is not configured' },
|
|
2350
|
-
{ status: 503 },
|
|
2351
|
-
);
|
|
2352
|
-
}
|
|
2353
|
-
const actualTailLogToken =
|
|
2354
|
-
request.headers.get('x-deepline-tail-log-token')?.trim() ?? '';
|
|
2355
|
-
if (actualTailLogToken !== expectedTailLogToken) {
|
|
2356
|
-
return Response.json(
|
|
2357
|
-
{ ok: false, error: 'tail log token mismatch' },
|
|
2358
|
-
{ status: 401 },
|
|
2359
|
-
);
|
|
2360
|
-
}
|
|
2361
|
-
return Response.json({
|
|
2362
|
-
ok: true,
|
|
2363
|
-
deployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
|
|
2364
|
-
runtimeDeployVersion:
|
|
2365
|
-
env.CF_VERSION_METADATA?.id ??
|
|
2366
|
-
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
|
|
2367
|
-
null,
|
|
2368
|
-
});
|
|
2369
|
-
}
|
|
2370
|
-
if (url.pathname === '/harness/probe') {
|
|
2371
|
-
const authError = authorizeCoordinatorControlRequest({ request, env });
|
|
2372
|
-
if (authError) return authError;
|
|
2373
|
-
try {
|
|
2374
|
-
const harness = await env.HARNESS.ping();
|
|
2375
|
-
return Response.json({
|
|
2376
|
-
ok: true,
|
|
2377
|
-
coordinatorDeployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
|
|
2378
|
-
coordinatorRuntimeDeployVersion:
|
|
2379
|
-
env.CF_VERSION_METADATA?.id ??
|
|
2380
|
-
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
|
|
2381
|
-
null,
|
|
2382
|
-
harnessDeployMarker: harness.deployMarker,
|
|
2383
|
-
harnessRuntimeDeployVersion: harness.runtimeDeployVersion,
|
|
2384
|
-
harnessTs: harness.ts,
|
|
2385
|
-
});
|
|
2386
|
-
} catch (error) {
|
|
2387
|
-
return coordinatorRouteErrorResponse({
|
|
2388
|
-
logTag: '[coordinator.harness_probe.error]',
|
|
2389
|
-
code: 'COORDINATOR_HARNESS_PROBE_FAILED',
|
|
2390
|
-
phase: 'coordinator.harness_probe',
|
|
2391
|
-
runId: null,
|
|
2392
|
-
error,
|
|
2393
|
-
});
|
|
2394
|
-
}
|
|
2395
|
-
}
|
|
2396
|
-
if (url.pathname === '/staged-files/put') {
|
|
2397
|
-
const authError = authorizeCoordinatorControlRequest({ request, env });
|
|
2398
|
-
if (authError) return authError;
|
|
2399
|
-
try {
|
|
2400
|
-
return await handleStagedFilePut(request, env);
|
|
2401
|
-
} catch (error) {
|
|
2402
|
-
return coordinatorRouteErrorResponse({
|
|
2403
|
-
logTag: '[coordinator.staged_file_put.error]',
|
|
2404
|
-
code: 'COORDINATOR_STAGED_FILE_PUT_FAILED',
|
|
2405
|
-
phase: 'coordinator.staged_files_put',
|
|
2406
|
-
runId: null,
|
|
2407
|
-
error,
|
|
2408
|
-
});
|
|
2409
|
-
}
|
|
2410
|
-
}
|
|
2411
|
-
if (url.pathname === '/rate-acquire') {
|
|
2412
|
-
const authError = authorizeCoordinatorControlRequest({ request, env });
|
|
2413
|
-
if (authError) return authError;
|
|
2414
|
-
try {
|
|
2415
|
-
const body = (await request.json().catch(() => null)) as {
|
|
2416
|
-
bucketId?: unknown;
|
|
2417
|
-
rules?: unknown;
|
|
2418
|
-
requested?: unknown;
|
|
2419
|
-
} | null;
|
|
2420
|
-
const bucketId = typeof body?.bucketId === 'string' ? body.bucketId : '';
|
|
2421
|
-
if (!bucketId.trim()) {
|
|
2422
|
-
return Response.json(
|
|
2423
|
-
{ error: 'bucketId is required.' },
|
|
2424
|
-
{ status: 400 },
|
|
2425
|
-
);
|
|
2426
|
-
}
|
|
2427
|
-
const result = await callRateBucketControl<{
|
|
2428
|
-
granted: number;
|
|
2429
|
-
waitMs: number;
|
|
2430
|
-
}>(env, bucketId, '/rate-acquire', body);
|
|
2431
|
-
return Response.json(result);
|
|
2432
|
-
} catch (error) {
|
|
2433
|
-
return coordinatorRouteErrorResponse({
|
|
2434
|
-
logTag: '[coordinator.rate_acquire.error]',
|
|
2435
|
-
code: 'COORDINATOR_RATE_ROUTE_FAILED',
|
|
2436
|
-
phase: 'coordinator.rate.acquire',
|
|
2437
|
-
runId: null,
|
|
2438
|
-
error,
|
|
2439
|
-
});
|
|
2440
|
-
}
|
|
2441
|
-
}
|
|
2442
|
-
if (url.pathname === '/rate-penalize') {
|
|
2443
|
-
const authError = authorizeCoordinatorControlRequest({ request, env });
|
|
2444
|
-
if (authError) return authError;
|
|
2445
|
-
try {
|
|
2446
|
-
const body = (await request.json().catch(() => null)) as {
|
|
2447
|
-
bucketId?: unknown;
|
|
2448
|
-
cooldownMs?: unknown;
|
|
2449
|
-
} | null;
|
|
2450
|
-
const bucketId = typeof body?.bucketId === 'string' ? body.bucketId : '';
|
|
2451
|
-
if (!bucketId.trim()) {
|
|
2452
|
-
return Response.json(
|
|
2453
|
-
{ error: 'bucketId is required.' },
|
|
2454
|
-
{ status: 400 },
|
|
2455
|
-
);
|
|
2456
|
-
}
|
|
2457
|
-
await callRateBucketControl<{ ok?: unknown }>(
|
|
2458
|
-
env,
|
|
2459
|
-
bucketId,
|
|
2460
|
-
'/rate-penalize',
|
|
2461
|
-
body,
|
|
2462
|
-
);
|
|
2463
|
-
return Response.json({ ok: true });
|
|
2464
|
-
} catch (error) {
|
|
2465
|
-
return coordinatorRouteErrorResponse({
|
|
2466
|
-
logTag: '[coordinator.rate_penalize.error]',
|
|
2467
|
-
code: 'COORDINATOR_RATE_ROUTE_FAILED',
|
|
2468
|
-
phase: 'coordinator.rate.penalize',
|
|
2469
|
-
runId: null,
|
|
2470
|
-
error,
|
|
2471
|
-
});
|
|
2472
|
-
}
|
|
2473
|
-
}
|
|
2474
|
-
// Workflow routes: /workflow/{runId}/{action}
|
|
2475
|
-
const wfMatch = url.pathname.match(/^\/workflow\/([^/]+)(?:\/(.+))?$/);
|
|
2476
|
-
if (wfMatch) {
|
|
2477
|
-
const runId = decodeURIComponent(wfMatch[1]!);
|
|
2478
|
-
const action = wfMatch[2] ?? '';
|
|
2479
|
-
const authError = authorizeCoordinatorControlRequest({
|
|
2480
|
-
request,
|
|
2481
|
-
env,
|
|
2482
|
-
runId,
|
|
2483
|
-
requireRunScope: isWorkflowMutatingAction(action),
|
|
2484
|
-
});
|
|
2485
|
-
if (authError) return authError;
|
|
2486
|
-
try {
|
|
2487
|
-
return await handleWorkflowRoute({ runId, action, request, env, ctx });
|
|
2488
|
-
} catch (error) {
|
|
2489
|
-
return coordinatorWorkflowRouteErrorResponse({
|
|
2490
|
-
runId,
|
|
2491
|
-
action,
|
|
2492
|
-
error,
|
|
2493
|
-
});
|
|
2494
|
-
}
|
|
2495
|
-
}
|
|
2496
|
-
|
|
2497
|
-
// Dedup routes: /dedup/{runId}/{action}
|
|
2498
|
-
const dedupMatch = url.pathname.match(/^\/dedup\/([^/]+)(?:\/(.+))?$/);
|
|
2499
|
-
if (dedupMatch) {
|
|
2500
|
-
const runId = decodeURIComponent(dedupMatch[1]!);
|
|
2501
|
-
const action = dedupMatch[2] ?? '';
|
|
2502
|
-
const authError = authorizeCoordinatorControlRequest({
|
|
2503
|
-
request,
|
|
2504
|
-
env,
|
|
2505
|
-
runId,
|
|
2506
|
-
requireRunScope: request.method !== 'GET' && request.method !== 'HEAD',
|
|
2507
|
-
});
|
|
2508
|
-
if (authError) return authError;
|
|
2509
|
-
try {
|
|
2510
|
-
const doId = env.PLAY_DEDUP.idFromName(runId);
|
|
2511
|
-
const stub = env.PLAY_DEDUP.get(doId);
|
|
2512
|
-
const internalUrl = `https://internal/${action}`;
|
|
2513
|
-
return await stub.fetch(internalUrl, {
|
|
2514
|
-
method: request.method,
|
|
2515
|
-
headers: request.headers,
|
|
2516
|
-
body:
|
|
2517
|
-
request.method === 'GET' || request.method === 'HEAD'
|
|
2518
|
-
? undefined
|
|
2519
|
-
: request.body,
|
|
2520
|
-
});
|
|
2521
|
-
} catch (error) {
|
|
2522
|
-
return coordinatorRouteErrorResponse({
|
|
2523
|
-
logTag: '[coordinator.dedup_route.error]',
|
|
2524
|
-
code: 'COORDINATOR_DEDUP_ROUTE_FAILED',
|
|
2525
|
-
phase: `coordinator.dedup.${action || 'unknown'}`,
|
|
2526
|
-
runId,
|
|
2527
|
-
error,
|
|
2528
|
-
});
|
|
2529
|
-
}
|
|
2530
|
-
}
|
|
2531
|
-
|
|
2532
|
-
return new Response('not found', { status: 404 });
|
|
2533
|
-
}
|
|
2534
|
-
|
|
2535
|
-
const RUN_LOG_PREFIX_RE = /\[deepline-run:([^\]]+)\]\s*(.*)/;
|
|
2536
|
-
const RUN_ID_RE = /\bplay\/[^/\s]+\/run\/[0-9a-zTt-]+/;
|
|
2537
|
-
|
|
2538
|
-
async function flushTailRunLogs(
|
|
2539
|
-
events: unknown[],
|
|
2540
|
-
env: CoordinatorEnv,
|
|
2541
|
-
): Promise<void> {
|
|
2542
|
-
const token = env.DEEPLINE_TAIL_LOG_TOKEN?.trim();
|
|
2543
|
-
if (!token) {
|
|
2544
|
-
return;
|
|
2545
|
-
}
|
|
2546
|
-
const grouped = new Map<string, string[]>();
|
|
2547
|
-
for (const event of events) {
|
|
2548
|
-
for (const line of extractTailLogLines(event)) {
|
|
2549
|
-
const parsed = parseRunLogLine(line);
|
|
2550
|
-
if (!parsed) {
|
|
2551
|
-
continue;
|
|
2552
|
-
}
|
|
2553
|
-
const existing = grouped.get(parsed.runId) ?? [];
|
|
2554
|
-
existing.push(parsed.line);
|
|
2555
|
-
grouped.set(parsed.runId, existing);
|
|
2556
|
-
}
|
|
2557
|
-
}
|
|
2558
|
-
await Promise.all(
|
|
2559
|
-
[...grouped.entries()].map(async ([runId, lines]) => {
|
|
2560
|
-
if (lines.length === 0) {
|
|
2561
|
-
return;
|
|
2562
|
-
}
|
|
2563
|
-
await fetch(
|
|
2564
|
-
`${env.DEEPLINE_API_BASE_URL}${PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH}`,
|
|
2565
|
-
{
|
|
2566
|
-
method: 'POST',
|
|
2567
|
-
headers: {
|
|
2568
|
-
'content-type': 'application/json',
|
|
2569
|
-
'x-deepline-tail-log-token': token,
|
|
2570
|
-
},
|
|
2571
|
-
body: JSON.stringify({
|
|
2572
|
-
runId,
|
|
2573
|
-
lines: lines.slice(-100),
|
|
2574
|
-
}),
|
|
2575
|
-
},
|
|
2576
|
-
).catch(() => null);
|
|
2577
|
-
}),
|
|
2578
|
-
);
|
|
2579
|
-
}
|
|
2580
|
-
|
|
2581
|
-
function extractTailLogLines(event: unknown): string[] {
|
|
2582
|
-
const lines: string[] = [];
|
|
2583
|
-
const record =
|
|
2584
|
-
event && typeof event === 'object' && !Array.isArray(event)
|
|
2585
|
-
? (event as Record<string, unknown>)
|
|
2586
|
-
: {};
|
|
2587
|
-
const logs = Array.isArray(record.logs) ? record.logs : [];
|
|
2588
|
-
for (const log of logs) {
|
|
2589
|
-
if (typeof log === 'string') {
|
|
2590
|
-
lines.push(log);
|
|
2591
|
-
continue;
|
|
2592
|
-
}
|
|
2593
|
-
if (!log || typeof log !== 'object' || Array.isArray(log)) {
|
|
2594
|
-
continue;
|
|
2595
|
-
}
|
|
2596
|
-
const logRecord = log as Record<string, unknown>;
|
|
2597
|
-
const message = logRecord.message;
|
|
2598
|
-
if (Array.isArray(message)) {
|
|
2599
|
-
lines.push(message.map(formatTailLogPart).join(' '));
|
|
2600
|
-
} else if (typeof message === 'string') {
|
|
2601
|
-
lines.push(message);
|
|
2602
|
-
}
|
|
2603
|
-
}
|
|
2604
|
-
const exceptions = Array.isArray(record.exceptions) ? record.exceptions : [];
|
|
2605
|
-
for (const exception of exceptions) {
|
|
2606
|
-
lines.push(formatTailLogPart(exception));
|
|
2607
|
-
}
|
|
2608
|
-
return lines.map((line) => line.trim()).filter(Boolean);
|
|
2609
|
-
}
|
|
2610
|
-
|
|
2611
|
-
function formatTailLogPart(value: unknown): string {
|
|
2612
|
-
if (typeof value === 'string') {
|
|
2613
|
-
return value;
|
|
2614
|
-
}
|
|
2615
|
-
if (value instanceof Error) {
|
|
2616
|
-
return value.stack ?? value.message;
|
|
2617
|
-
}
|
|
2618
|
-
try {
|
|
2619
|
-
return JSON.stringify(value);
|
|
2620
|
-
} catch {
|
|
2621
|
-
return String(value);
|
|
2622
|
-
}
|
|
2623
|
-
}
|
|
2624
|
-
|
|
2625
|
-
/**
|
|
2626
|
-
* Recognize the tenant-Postgres degradation signatures that used to escape the
|
|
2627
|
-
* coordinator submit path as an opaque 500 "internal error" with no repair
|
|
2628
|
-
* instruction: revoked CONNECT / grant drift, connect stalls, and NeonDbError.
|
|
2629
|
-
* Conservative on purpose — anything not matched stays a generic internal
|
|
2630
|
-
* error (with its phase) so we do not mislabel unrelated failures.
|
|
2631
|
-
*/
|
|
2632
|
-
function isTenantStorageDegradationError(error: unknown): boolean {
|
|
2633
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
2634
|
-
return (
|
|
2635
|
-
/permission denied (?:for|to grant) (?:schema|table|relation|database|sequence|role)\b/i.test(
|
|
2636
|
-
message,
|
|
2637
|
-
) ||
|
|
2638
|
-
/must be owner of (?:table|relation|sequence|schema|database|function|procedure|routine|type)\b/i.test(
|
|
2639
|
-
message,
|
|
2640
|
-
) ||
|
|
2641
|
-
/Runtime Postgres connection timed out/i.test(message) ||
|
|
2642
|
-
/\bNeonDbError\b/i.test(message) ||
|
|
2643
|
-
/timeout exceeded when trying to connect/i.test(message) ||
|
|
2644
|
-
/connection terminated unexpectedly/i.test(message)
|
|
2645
|
-
// Deliberately NOT matching bare socket/DNS codes (ECONNRESET, ETIMEDOUT,
|
|
2646
|
-
// ECONNREFUSED, EAI_AGAIN): those are thrown by Cloudflare service bindings,
|
|
2647
|
-
// the Workflow API, and Durable Object fetches too, not just tenant
|
|
2648
|
-
// Postgres. Matching them here would mislabel a transient CF-infra blip as
|
|
2649
|
-
// WORKSPACE_STORAGE_NOT_READY ("run db repair"). Every remaining pattern
|
|
2650
|
-
// carries explicit Postgres/Neon context.
|
|
2651
|
-
);
|
|
2652
|
-
}
|
|
2653
|
-
|
|
2654
|
-
function coordinatorWorkflowRouteErrorResponse(input: {
|
|
2655
|
-
runId: string;
|
|
2656
|
-
action: string;
|
|
2657
|
-
error: unknown;
|
|
2658
|
-
}): Response {
|
|
2659
|
-
const message =
|
|
2660
|
-
input.error instanceof Error ? input.error.message : String(input.error);
|
|
2661
|
-
const phase = `coordinator.${input.action || 'unknown'}`;
|
|
2662
|
-
recordCoordinatorEvent(
|
|
2663
|
-
'error',
|
|
2664
|
-
'coordinator.workflow_route.error',
|
|
2665
|
-
'[coordinator.workflow_route.error]',
|
|
2666
|
-
{
|
|
2667
|
-
runId: input.runId,
|
|
2668
|
-
action: input.action,
|
|
2669
|
-
error: message,
|
|
2670
|
-
},
|
|
2671
|
-
);
|
|
2672
|
-
if (
|
|
2673
|
-
input.action === 'submit' &&
|
|
2674
|
-
isTenantStorageDegradationError(input.error)
|
|
2675
|
-
) {
|
|
2676
|
-
// De-redact: a tenant-storage failure gets the loud, actionable
|
|
2677
|
-
// WORKSPACE_STORAGE_NOT_READY message + repair instruction instead of the
|
|
2678
|
-
// generic "internal error". Same contract as the runtime route.
|
|
2679
|
-
//
|
|
2680
|
-
// Scoped to `submit` on purpose: the message states "nothing was billed",
|
|
2681
|
-
// which is only true in the pre-run submit phase. The other workflow
|
|
2682
|
-
// actions (status/result/tail/cancel/signal) run against an already-active,
|
|
2683
|
-
// possibly-billing run, so surfacing "storage not ready, nothing billed,
|
|
2684
|
-
// run db repair" there would be false and alarming on a healthy run.
|
|
2685
|
-
recordCoordinatorEvent(
|
|
2686
|
-
'error',
|
|
2687
|
-
'coordinator.workflow_route.storage_not_ready',
|
|
2688
|
-
'[coordinator.workflow_route.storage_not_ready]',
|
|
2689
|
-
{
|
|
2690
|
-
runId: input.runId,
|
|
2691
|
-
action: input.action,
|
|
2692
|
-
phase,
|
|
2693
|
-
error: message,
|
|
2694
|
-
},
|
|
2695
|
-
);
|
|
2696
|
-
return Response.json(
|
|
2697
|
-
{
|
|
2698
|
-
error: {
|
|
2699
|
-
code: WORKSPACE_STORAGE_NOT_READY_CODE,
|
|
2700
|
-
message: WORKSPACE_STORAGE_NOT_READY_MESSAGE,
|
|
2701
|
-
phase,
|
|
2702
|
-
runId: input.runId,
|
|
2703
|
-
},
|
|
2704
|
-
},
|
|
2705
|
-
{ status: 503 },
|
|
2706
|
-
);
|
|
2707
|
-
}
|
|
2708
|
-
return Response.json(
|
|
2709
|
-
{
|
|
2710
|
-
error: {
|
|
2711
|
-
code: 'COORDINATOR_WORKFLOW_ROUTE_FAILED',
|
|
2712
|
-
message,
|
|
2713
|
-
phase,
|
|
2714
|
-
runId: input.runId,
|
|
2715
|
-
},
|
|
2716
|
-
},
|
|
2717
|
-
{ status: 500 },
|
|
2718
|
-
);
|
|
2719
|
-
}
|
|
2720
|
-
|
|
2721
|
-
/**
|
|
2722
|
-
* Generic JSON envelope for any thrown error escaping a coordinator fetch route.
|
|
2723
|
-
* Mirrors {@link coordinatorWorkflowRouteErrorResponse} so that NO route group
|
|
2724
|
-
* in the fetch handler can leak Cloudflare's raw "Worker threw exception" HTML
|
|
2725
|
-
* page to clients/agents. Keep the envelope shape `{ error: { code, message,
|
|
2726
|
-
* phase, runId } }` and the console.error logging convention identical.
|
|
2727
|
-
*/
|
|
2728
|
-
function coordinatorRouteErrorResponse(input: {
|
|
2729
|
-
logTag: string;
|
|
2730
|
-
code: string;
|
|
2731
|
-
phase: string;
|
|
2732
|
-
runId: string | null;
|
|
2733
|
-
error: unknown;
|
|
2734
|
-
}): Response {
|
|
2735
|
-
const message =
|
|
2736
|
-
input.error instanceof Error ? input.error.message : String(input.error);
|
|
2737
|
-
recordCoordinatorEvent('error', 'coordinator.route.error', input.logTag, {
|
|
2738
|
-
runId: input.runId,
|
|
2739
|
-
phase: input.phase,
|
|
2740
|
-
error: message,
|
|
2741
|
-
});
|
|
2742
|
-
return Response.json(
|
|
2743
|
-
{
|
|
2744
|
-
error: {
|
|
2745
|
-
code: input.code,
|
|
2746
|
-
message,
|
|
2747
|
-
phase: input.phase,
|
|
2748
|
-
runId: input.runId,
|
|
2749
|
-
},
|
|
2750
|
-
},
|
|
2751
|
-
{ status: 500 },
|
|
2752
|
-
);
|
|
2753
|
-
}
|
|
2754
|
-
|
|
2755
|
-
// Operator-diagnostic console lines that carry the [deepline-run:] prefix but
|
|
2756
|
-
// are not user-facing run output. The console scrape fans run-prefixed lines
|
|
2757
|
-
// back into the run's durable Run Log Stream ('system' channel), so harness/
|
|
2758
|
-
// coordinator plumbing noise is filtered at ingestion, never at read time.
|
|
2759
|
-
// User play log lines (runner-event echoes) intentionally pass through.
|
|
2760
|
-
const OPERATOR_NOISE_LOG_PATTERNS: readonly RegExp[] = [
|
|
2761
|
-
/\[perf-trace\]/,
|
|
2762
|
-
/\[harness-probe\]/,
|
|
2763
|
-
/TenantWorkflow\.run entered/,
|
|
2764
|
-
/TenantWorkflow\.run threw/,
|
|
2765
|
-
/failed to forward runner perf trace/,
|
|
2766
|
-
/failed to forward TenantWorkflow\.run error/,
|
|
2767
|
-
];
|
|
2768
|
-
|
|
2769
|
-
function parseRunLogLine(line: string): { runId: string; line: string } | null {
|
|
2770
|
-
if (OPERATOR_NOISE_LOG_PATTERNS.some((pattern) => pattern.test(line))) {
|
|
2771
|
-
return null;
|
|
2772
|
-
}
|
|
2773
|
-
const prefixed = line.match(RUN_LOG_PREFIX_RE);
|
|
2774
|
-
if (prefixed?.[1]) {
|
|
2775
|
-
const message = prefixed[2]?.trim() ?? '';
|
|
2776
|
-
return {
|
|
2777
|
-
runId: prefixed[1],
|
|
2778
|
-
line: message || line,
|
|
2779
|
-
};
|
|
2780
|
-
}
|
|
2781
|
-
const runId = line.match(RUN_ID_RE)?.[0];
|
|
2782
|
-
if (!runId) {
|
|
2783
|
-
return null;
|
|
2784
|
-
}
|
|
2785
|
-
return { runId, line };
|
|
2786
|
-
}
|
|
2787
|
-
|
|
2788
|
-
async function handleWorkflowRoute(input: {
|
|
2789
|
-
runId: string;
|
|
2790
|
-
action: string;
|
|
2791
|
-
request: Request;
|
|
2792
|
-
env: CoordinatorEnv;
|
|
2793
|
-
ctx?: ExecutionContext;
|
|
2794
|
-
}): Promise<Response> {
|
|
2795
|
-
const { runId, action, request, env } = input;
|
|
2796
|
-
const defaultInstanceId = workflowInstanceId(runId);
|
|
2797
|
-
if (action === 'submit') {
|
|
2798
|
-
const submitStartedAt = Date.now();
|
|
2799
|
-
const coordinatorTimings: CoordinatorTiming[] = [];
|
|
2800
|
-
let submittedRunId = runId;
|
|
2801
|
-
const recordSubmitTiming = (timing: CoordinatorTiming): void => {
|
|
2802
|
-
coordinatorTimings.push(timing);
|
|
2803
|
-
recordCoordinatorPerfTraceBuffered(env, input.ctx, {
|
|
2804
|
-
runId: submittedRunId,
|
|
2805
|
-
phase: timing.phase,
|
|
2806
|
-
ms: timing.ms,
|
|
2807
|
-
graphHash: timing.graphHash ?? null,
|
|
2808
|
-
extra: timing.extra,
|
|
2809
|
-
});
|
|
2810
|
-
};
|
|
2811
|
-
const parseStartedAt = Date.now();
|
|
2812
|
-
const params = (await request.json()) as PlayWorkflowParams;
|
|
2813
|
-
submittedRunId = params.runId ?? runId;
|
|
2814
|
-
recordSubmitTiming({
|
|
2815
|
-
phase: 'coordinator.submit_received',
|
|
2816
|
-
ms: Date.now() - submitStartedAt,
|
|
2817
|
-
graphHash: params.graphHash ?? null,
|
|
2818
|
-
extra: {
|
|
2819
|
-
integrationMode: normalizeIntegrationMode(params.integrationMode),
|
|
2820
|
-
hasDynamicWorkerCode: Boolean(params.dynamicWorkerCode),
|
|
2821
|
-
dynamicWorkerBytes:
|
|
2822
|
-
typeof params.dynamicWorkerCode === 'string'
|
|
2823
|
-
? params.dynamicWorkerCode.length
|
|
2824
|
-
: 0,
|
|
2825
|
-
},
|
|
2826
|
-
});
|
|
2827
|
-
recordSubmitTiming({
|
|
2828
|
-
phase: 'coordinator.submit_parse_body',
|
|
2829
|
-
ms: Date.now() - parseStartedAt,
|
|
2830
|
-
graphHash: params.graphHash ?? null,
|
|
2831
|
-
extra: {
|
|
2832
|
-
integrationMode: normalizeIntegrationMode(params.integrationMode),
|
|
2833
|
-
hasDynamicWorkerCode: Boolean(params.dynamicWorkerCode),
|
|
2834
|
-
dynamicWorkerBytes:
|
|
2835
|
-
typeof params.dynamicWorkerCode === 'string'
|
|
2836
|
-
? params.dynamicWorkerCode.length
|
|
2837
|
-
: 0,
|
|
2838
|
-
},
|
|
2839
|
-
});
|
|
2840
|
-
if (!env.LOADER) {
|
|
2841
|
-
throw new Error(
|
|
2842
|
-
'Cloudflare Dynamic Workflows require a Worker Loader binding named LOADER.',
|
|
2843
|
-
);
|
|
2844
|
-
}
|
|
2845
|
-
if (!env.HARNESS) {
|
|
2846
|
-
throw new Error(
|
|
2847
|
-
'Cloudflare Dynamic Workflows require a service binding named HARNESS. ' +
|
|
2848
|
-
'Start apps/play-harness-worker before the coordinator or fix wrangler.toml services.',
|
|
2849
|
-
);
|
|
2850
|
-
}
|
|
2851
|
-
const preloadedDbSessions = Array.isArray(params.preloadedDbSessions)
|
|
2852
|
-
? params.preloadedDbSessions
|
|
2853
|
-
: [];
|
|
2854
|
-
if (preloadedDbSessions.length > 0 && params.executorToken) {
|
|
2855
|
-
const prewarmStartedAt = Date.now();
|
|
2856
|
-
recordSubmitTiming({
|
|
2857
|
-
phase: 'coordinator.harness_prewarm_postgres_start',
|
|
2858
|
-
ms: 0,
|
|
2859
|
-
graphHash: params.graphHash ?? null,
|
|
2860
|
-
extra: { sessions: preloadedDbSessions.length },
|
|
2861
|
-
});
|
|
2862
|
-
const prewarmPromise = env.HARNESS.prewarmPostgresSessions({
|
|
2863
|
-
executorToken: params.executorToken,
|
|
2864
|
-
orgId: params.orgId,
|
|
2865
|
-
playName: params.playName,
|
|
2866
|
-
sessions: preloadedDbSessions,
|
|
2867
|
-
})
|
|
2868
|
-
.then((result) => {
|
|
2869
|
-
recordSubmitTiming({
|
|
2870
|
-
phase: 'coordinator.harness_prewarm_postgres',
|
|
2871
|
-
ms: Date.now() - prewarmStartedAt,
|
|
2872
|
-
graphHash: params.graphHash ?? null,
|
|
2873
|
-
extra: {
|
|
2874
|
-
status: 'ok',
|
|
2875
|
-
sessions: result.sessions,
|
|
2876
|
-
},
|
|
2877
|
-
});
|
|
2878
|
-
})
|
|
2879
|
-
.catch((error: unknown) => {
|
|
2880
|
-
const storageNotReady = isTenantStorageDegradationError(error);
|
|
2881
|
-
recordSubmitTiming({
|
|
2882
|
-
phase: 'coordinator.harness_prewarm_postgres',
|
|
2883
|
-
ms: Date.now() - prewarmStartedAt,
|
|
2884
|
-
graphHash: params.graphHash ?? null,
|
|
2885
|
-
extra: {
|
|
2886
|
-
status: 'failed',
|
|
2887
|
-
sessions: preloadedDbSessions.length,
|
|
2888
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2889
|
-
storageNotReady,
|
|
2890
|
-
},
|
|
2891
|
-
});
|
|
2892
|
-
// The prewarm runs in waitUntil, so it cannot fail the already-issued
|
|
2893
|
-
// submit. But its cause used to be fully redacted in coordinator
|
|
2894
|
-
// logs. Log the tenant-storage classification loudly so the real
|
|
2895
|
-
// cause (and the WORKSPACE_STORAGE_NOT_READY repair path) is visible.
|
|
2896
|
-
if (storageNotReady) {
|
|
2897
|
-
recordCoordinatorEvent(
|
|
2898
|
-
'error',
|
|
2899
|
-
'coordinator.harness_prewarm_postgres.storage_not_ready',
|
|
2900
|
-
'[coordinator.harness_prewarm_postgres.storage_not_ready]',
|
|
2901
|
-
{
|
|
2902
|
-
runId: submittedRunId,
|
|
2903
|
-
playName: params.playName,
|
|
2904
|
-
code: WORKSPACE_STORAGE_NOT_READY_CODE,
|
|
2905
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2906
|
-
},
|
|
2907
|
-
);
|
|
2908
|
-
}
|
|
2909
|
-
});
|
|
2910
|
-
input.ctx?.waitUntil(prewarmPromise);
|
|
2911
|
-
}
|
|
2912
|
-
const dbSessionExternalization = externalizedWorkflowDbSessionParams({
|
|
2913
|
-
params,
|
|
2914
|
-
});
|
|
2915
|
-
const workflowParams = dbSessionExternalization.params;
|
|
2916
|
-
workflowParams.submittedAt = Date.now();
|
|
2917
|
-
let launchState: Awaited<ReturnType<typeof persistWorkflowLaunchState>> =
|
|
2918
|
-
{};
|
|
2919
|
-
try {
|
|
2920
|
-
const retryStateStartedAt = Date.now();
|
|
2921
|
-
launchState = await persistWorkflowLaunchState({
|
|
2922
|
-
env,
|
|
2923
|
-
runId: submittedRunId,
|
|
2924
|
-
params: workflowParams,
|
|
2925
|
-
sessions: dbSessionExternalization.sessions,
|
|
2926
|
-
});
|
|
2927
|
-
const persistedAt = Date.now();
|
|
2928
|
-
if (dbSessionExternalization.sessions.length > 0) {
|
|
2929
|
-
recordSubmitTiming({
|
|
2930
|
-
phase: 'coordinator.workflow_db_sessions_externalized',
|
|
2931
|
-
ms: persistedAt - retryStateStartedAt,
|
|
2932
|
-
graphHash: params.graphHash ?? null,
|
|
2933
|
-
extra: {
|
|
2934
|
-
sessions:
|
|
2935
|
-
launchState.sessionCount ??
|
|
2936
|
-
dbSessionExternalization.sessions.length,
|
|
2937
|
-
expiresAt:
|
|
2938
|
-
launchState.dbSessionsExpiresAt ??
|
|
2939
|
-
dbSessionExternalization.ref?.expiresAt,
|
|
2940
|
-
combinedLaunchState: true,
|
|
2941
|
-
},
|
|
2942
|
-
});
|
|
2943
|
-
}
|
|
2944
|
-
recordSubmitTiming({
|
|
2945
|
-
phase: 'coordinator.retry_state_persistence',
|
|
2946
|
-
ms: persistedAt - retryStateStartedAt,
|
|
2947
|
-
graphHash: params.graphHash ?? null,
|
|
2948
|
-
extra: {
|
|
2949
|
-
combinedLaunchState: dbSessionExternalization.sessions.length > 0,
|
|
2950
|
-
},
|
|
2951
|
-
});
|
|
2952
|
-
} catch (error) {
|
|
2953
|
-
const errorMessage =
|
|
2954
|
-
error instanceof Error ? error.message : String(error);
|
|
2955
|
-
recordCoordinatorEvent(
|
|
2956
|
-
'error',
|
|
2957
|
-
'coordinator.workflow_retry_state.persistence_failed',
|
|
2958
|
-
'[coordinator] workflow retry state persistence failed',
|
|
2959
|
-
{
|
|
2960
|
-
code: 'WORKFLOW_RETRY_STATE_PERSISTENCE_FAILED',
|
|
2961
|
-
runId: submittedRunId,
|
|
2962
|
-
error: errorMessage,
|
|
2963
|
-
},
|
|
2964
|
-
);
|
|
2965
|
-
recordSubmitTiming({
|
|
2966
|
-
phase: 'coordinator.retry_state_persistence',
|
|
2967
|
-
ms: 0,
|
|
2968
|
-
graphHash: params.graphHash ?? null,
|
|
2969
|
-
extra: {
|
|
2970
|
-
status: 'failed',
|
|
2971
|
-
error: errorMessage,
|
|
2972
|
-
},
|
|
2973
|
-
});
|
|
2974
|
-
return workflowRetryStatePersistenceErrorResponse({
|
|
2975
|
-
runId: submittedRunId,
|
|
2976
|
-
error,
|
|
2977
|
-
});
|
|
2978
|
-
}
|
|
2979
|
-
const createParams = workflowCreateParams(workflowParams, launchState);
|
|
2980
|
-
let instance: WorkflowInstance | null = null;
|
|
2981
|
-
try {
|
|
2982
|
-
const submitClaimStartedAt = Date.now();
|
|
2983
|
-
const submitClaim = await claimWorkflowSubmit({
|
|
2984
|
-
env,
|
|
2985
|
-
runId: submittedRunId,
|
|
2986
|
-
});
|
|
2987
|
-
recordSubmitTiming({
|
|
2988
|
-
phase: 'coordinator.workflow_submit_claim',
|
|
2989
|
-
ms: Date.now() - submitClaimStartedAt,
|
|
2990
|
-
graphHash: params.graphHash ?? null,
|
|
2991
|
-
extra: {
|
|
2992
|
-
claimed: submitClaim.claimed,
|
|
2993
|
-
status: submitClaim.status,
|
|
2994
|
-
instanceId: submitClaim.instanceId ?? defaultInstanceId,
|
|
2995
|
-
},
|
|
2996
|
-
});
|
|
2997
|
-
if (!submitClaim.claimed) {
|
|
2998
|
-
const duplicateWaitStartedAt = Date.now();
|
|
2999
|
-
const resolvedSubmitClaim = await waitForWorkflowSubmitCreated({
|
|
3000
|
-
env,
|
|
3001
|
-
runId: submittedRunId,
|
|
3002
|
-
initialClaim: submitClaim,
|
|
3003
|
-
timeoutMs: WORKFLOW_SUBMIT_DUPLICATE_WAIT_MS,
|
|
3004
|
-
pollMs: WORKFLOW_SUBMIT_DUPLICATE_POLL_MS,
|
|
3005
|
-
});
|
|
3006
|
-
recordSubmitTiming({
|
|
3007
|
-
phase: 'coordinator.workflow_submit_duplicate_wait',
|
|
3008
|
-
ms: Date.now() - duplicateWaitStartedAt,
|
|
3009
|
-
graphHash: params.graphHash ?? null,
|
|
3010
|
-
extra: {
|
|
3011
|
-
status: resolvedSubmitClaim.status,
|
|
3012
|
-
instanceId: resolvedSubmitClaim.instanceId ?? null,
|
|
3013
|
-
},
|
|
3014
|
-
});
|
|
3015
|
-
const totalMs = Date.now() - submitStartedAt;
|
|
3016
|
-
if (
|
|
3017
|
-
resolvedSubmitClaim.status !== 'created' ||
|
|
3018
|
-
!resolvedSubmitClaim.instanceId
|
|
3019
|
-
) {
|
|
3020
|
-
recordSubmitTiming({
|
|
3021
|
-
phase: 'coordinator.submit_total',
|
|
3022
|
-
ms: totalMs,
|
|
3023
|
-
graphHash: params.graphHash ?? null,
|
|
3024
|
-
extra: { duplicate: true, pending: true },
|
|
3025
|
-
});
|
|
3026
|
-
return Response.json(
|
|
3027
|
-
{
|
|
3028
|
-
runId,
|
|
3029
|
-
status: 'pending',
|
|
3030
|
-
workflowInstanceId: null,
|
|
3031
|
-
instanceState: {
|
|
3032
|
-
status: resolvedSubmitClaim.status ?? 'creating',
|
|
3033
|
-
},
|
|
3034
|
-
retryAfterMs: 250,
|
|
3035
|
-
runtimeDeployVersion:
|
|
3036
|
-
env.CF_VERSION_METADATA?.id ??
|
|
3037
|
-
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
|
|
3038
|
-
null,
|
|
3039
|
-
coordinatorTimings,
|
|
3040
|
-
},
|
|
3041
|
-
{ status: 503, headers: { 'retry-after': '1' } },
|
|
3042
|
-
);
|
|
3043
|
-
}
|
|
3044
|
-
recordSubmitTiming({
|
|
3045
|
-
phase: 'coordinator.submit_total',
|
|
3046
|
-
ms: totalMs,
|
|
3047
|
-
graphHash: params.graphHash ?? null,
|
|
3048
|
-
extra: { duplicate: true },
|
|
3049
|
-
});
|
|
3050
|
-
recordSubmitTiming({
|
|
3051
|
-
phase: 'coordinator.submit_accepted',
|
|
3052
|
-
ms: totalMs,
|
|
3053
|
-
graphHash: params.graphHash ?? null,
|
|
3054
|
-
extra: { duplicate: true },
|
|
3055
|
-
});
|
|
3056
|
-
return Response.json({
|
|
3057
|
-
runId,
|
|
3058
|
-
status: 'submitted',
|
|
3059
|
-
workflowInstanceId: resolvedSubmitClaim.instanceId,
|
|
3060
|
-
instanceState: { status: resolvedSubmitClaim.status },
|
|
3061
|
-
runtimeDeployVersion:
|
|
3062
|
-
env.CF_VERSION_METADATA?.id ??
|
|
3063
|
-
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
|
|
3064
|
-
null,
|
|
3065
|
-
coordinatorTimings,
|
|
3066
|
-
});
|
|
3067
|
-
}
|
|
3068
|
-
const dispatchStartedAt = Date.now();
|
|
3069
|
-
const createStartedAt = Date.now();
|
|
3070
|
-
recordSubmitTiming({
|
|
3071
|
-
phase: 'coordinator.workflow_create_start',
|
|
3072
|
-
ms: 0,
|
|
3073
|
-
graphHash: params.graphHash ?? null,
|
|
3074
|
-
extra: { instanceId: defaultInstanceId },
|
|
3075
|
-
});
|
|
3076
|
-
const createResult = await createOrAttachWorkflowInstance({
|
|
3077
|
-
create: () =>
|
|
3078
|
-
createDynamicWorkflowInstance({
|
|
3079
|
-
env,
|
|
3080
|
-
id: defaultInstanceId,
|
|
3081
|
-
params: createParams,
|
|
3082
|
-
metadataParams: workflowParams,
|
|
3083
|
-
}),
|
|
3084
|
-
getExisting: () => env.PLAY_WORKFLOW.get(defaultInstanceId),
|
|
3085
|
-
onCreateFailure: async () => {
|
|
3086
|
-
await releaseWorkflowSubmitClaim({
|
|
3087
|
-
env,
|
|
3088
|
-
runId: submittedRunId,
|
|
3089
|
-
});
|
|
3090
|
-
},
|
|
3091
|
-
});
|
|
3092
|
-
instance = createResult.instance;
|
|
3093
|
-
const workflowCreatedAt = Date.now();
|
|
3094
|
-
recordSubmitTiming({
|
|
3095
|
-
phase: 'coordinator.workflow_create',
|
|
3096
|
-
ms: workflowCreatedAt - createStartedAt,
|
|
3097
|
-
graphHash: params.graphHash ?? null,
|
|
3098
|
-
extra: { instanceId: instance.id, startMode: createResult.startMode },
|
|
3099
|
-
});
|
|
3100
|
-
try {
|
|
3101
|
-
await recordWorkflowInstanceId({
|
|
3102
|
-
env,
|
|
3103
|
-
runId: submittedRunId,
|
|
3104
|
-
instanceId: instance.id,
|
|
3105
|
-
});
|
|
3106
|
-
} catch (error) {
|
|
3107
|
-
recordCoordinatorEvent(
|
|
3108
|
-
'warn',
|
|
3109
|
-
'coordinator.workflow_instance_id.record_failed',
|
|
3110
|
-
'[coordinator] workflow instance id record failed',
|
|
3111
|
-
{
|
|
3112
|
-
runId: submittedRunId,
|
|
3113
|
-
instanceId: instance?.id ?? null,
|
|
3114
|
-
error: error instanceof Error ? error.message : String(error),
|
|
3115
|
-
},
|
|
3116
|
-
);
|
|
3117
|
-
await releaseWorkflowSubmitClaim({
|
|
3118
|
-
env,
|
|
3119
|
-
runId: submittedRunId,
|
|
3120
|
-
}).catch((releaseError) => {
|
|
3121
|
-
recordCoordinatorEvent(
|
|
3122
|
-
'warn',
|
|
3123
|
-
'coordinator.workflow_submit_claim.release_failed',
|
|
3124
|
-
'[coordinator] workflow submit claim release after instance record failure failed',
|
|
3125
|
-
{
|
|
3126
|
-
runId: submittedRunId,
|
|
3127
|
-
error:
|
|
3128
|
-
releaseError instanceof Error
|
|
3129
|
-
? releaseError.message
|
|
3130
|
-
: String(releaseError),
|
|
3131
|
-
},
|
|
3132
|
-
);
|
|
3133
|
-
});
|
|
3134
|
-
throw error;
|
|
3135
|
-
}
|
|
3136
|
-
recordSubmitTiming({
|
|
3137
|
-
phase: 'coordinator.dispatch_workflow',
|
|
3138
|
-
ms: Date.now() - dispatchStartedAt,
|
|
3139
|
-
graphHash: params.graphHash ?? null,
|
|
3140
|
-
extra: {
|
|
3141
|
-
startMode: 'direct_workflow_create',
|
|
3142
|
-
workflowCreateMode: createResult.startMode,
|
|
3143
|
-
instanceIdRecord: 'recorded',
|
|
3144
|
-
},
|
|
3145
|
-
});
|
|
3146
|
-
const initialWaitMsRaw = Number(
|
|
3147
|
-
new URL(request.url).searchParams.get('initialWaitMs') ?? '0',
|
|
3148
|
-
);
|
|
3149
|
-
const initialStateStartedAt = Date.now();
|
|
3150
|
-
const instanceState = await waitForSubmitInitialState({
|
|
3151
|
-
instance,
|
|
3152
|
-
runId: submittedRunId,
|
|
3153
|
-
waitMs: Number.isFinite(initialWaitMsRaw) ? initialWaitMsRaw : 0,
|
|
3154
|
-
});
|
|
3155
|
-
recordSubmitTiming({
|
|
3156
|
-
phase: 'coordinator.submit_initial_state',
|
|
3157
|
-
ms: Date.now() - initialStateStartedAt,
|
|
3158
|
-
graphHash: params.graphHash ?? null,
|
|
3159
|
-
extra: {
|
|
3160
|
-
waitMs: Number.isFinite(initialWaitMsRaw) ? initialWaitMsRaw : 0,
|
|
3161
|
-
status:
|
|
3162
|
-
typeof instanceState?.status === 'string'
|
|
3163
|
-
? instanceState.status
|
|
3164
|
-
: null,
|
|
3165
|
-
},
|
|
3166
|
-
});
|
|
3167
|
-
const totalMs = Date.now() - submitStartedAt;
|
|
3168
|
-
recordSubmitTiming({
|
|
3169
|
-
phase: 'coordinator.submit_total',
|
|
3170
|
-
ms: totalMs,
|
|
3171
|
-
graphHash: params.graphHash ?? null,
|
|
3172
|
-
});
|
|
3173
|
-
recordSubmitTiming({
|
|
3174
|
-
phase: 'coordinator.submit_accepted',
|
|
3175
|
-
ms: totalMs,
|
|
3176
|
-
graphHash: params.graphHash ?? null,
|
|
3177
|
-
});
|
|
3178
|
-
return Response.json({
|
|
3179
|
-
runId,
|
|
3180
|
-
status: 'submitted',
|
|
3181
|
-
workflowInstanceId: instance.id,
|
|
3182
|
-
instanceState,
|
|
3183
|
-
runtimeDeployVersion:
|
|
3184
|
-
env.CF_VERSION_METADATA?.id ??
|
|
3185
|
-
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
|
|
3186
|
-
null,
|
|
3187
|
-
coordinatorTimings,
|
|
3188
|
-
});
|
|
3189
|
-
} finally {
|
|
3190
|
-
disposeRpcStub(instance);
|
|
3191
|
-
}
|
|
3192
|
-
}
|
|
3193
|
-
|
|
3194
|
-
if (action === 'tail') {
|
|
3195
|
-
const url = new URL(request.url);
|
|
3196
|
-
const waitMs = Math.min(
|
|
3197
|
-
Math.max(Number(url.searchParams.get('waitMs') ?? '0'), 0),
|
|
3198
|
-
30_000,
|
|
3199
|
-
);
|
|
3200
|
-
const afterSeq = Math.max(
|
|
3201
|
-
0,
|
|
3202
|
-
Math.floor(Number(url.searchParams.get('afterSeq') ?? '0')),
|
|
3203
|
-
);
|
|
3204
|
-
const includeTrace = url.searchParams.get('trace') === '1';
|
|
3205
|
-
const statusStartedAt = Date.now();
|
|
3206
|
-
const eventResult = await listCoordinatorRunEvents({
|
|
3207
|
-
env,
|
|
3208
|
-
runId,
|
|
3209
|
-
afterSeq,
|
|
3210
|
-
timeoutMs: waitMs,
|
|
3211
|
-
}).catch(() => null);
|
|
3212
|
-
const rawEvents = eventResult?.events ?? [];
|
|
3213
|
-
const terminalEventIndex = rawEvents.findIndex(
|
|
3214
|
-
(event) => event.type === 'terminal',
|
|
3215
|
-
);
|
|
3216
|
-
const events =
|
|
3217
|
-
terminalEventIndex >= 0
|
|
3218
|
-
? rawEvents.slice(0, terminalEventIndex + 1)
|
|
3219
|
-
: rawEvents;
|
|
3220
|
-
const coordinatorTrace =
|
|
3221
|
-
includeTrace && events.length
|
|
3222
|
-
? await listCoordinatorPerfTrace(env, runId).catch(() => [])
|
|
3223
|
-
: [];
|
|
3224
|
-
const terminalEvent = events.find(
|
|
3225
|
-
(event): event is Extract<CoordinatorRunEvent, { type: 'terminal' }> =>
|
|
3226
|
-
event.type === 'terminal',
|
|
3227
|
-
);
|
|
3228
|
-
if (terminalEvent) {
|
|
3229
|
-
return Response.json({
|
|
3230
|
-
runId,
|
|
3231
|
-
...(terminalEvent.playName ? { playName: terminalEvent.playName } : {}),
|
|
3232
|
-
status: terminalEvent.status,
|
|
3233
|
-
result: terminalEvent.result ?? null,
|
|
3234
|
-
error: terminalEvent.error ?? null,
|
|
3235
|
-
totalRows: terminalEvent.totalRows ?? null,
|
|
3236
|
-
durationMs: terminalEvent.durationMs ?? null,
|
|
3237
|
-
completedAt: terminalEvent.ts,
|
|
3238
|
-
liveLogs: sanitizeLiveLogLines(terminalEvent.liveLogs),
|
|
3239
|
-
liveNodeProgress: terminalEvent.liveNodeProgress ?? null,
|
|
3240
|
-
events,
|
|
3241
|
-
latestSeq: eventResult?.latestSeq ?? afterSeq,
|
|
3242
|
-
wait: null,
|
|
3243
|
-
coordinatorObserve: {
|
|
3244
|
-
ms: Date.now() - statusStartedAt,
|
|
3245
|
-
waitMs,
|
|
3246
|
-
workflowStatus: 'terminal-event',
|
|
3247
|
-
statusPolls: 0,
|
|
3248
|
-
instanceId: null,
|
|
3249
|
-
},
|
|
3250
|
-
...(includeTrace ? { coordinatorTrace } : {}),
|
|
3251
|
-
});
|
|
3252
|
-
}
|
|
3253
|
-
return Response.json({
|
|
3254
|
-
runId,
|
|
3255
|
-
status: 'running',
|
|
3256
|
-
events,
|
|
3257
|
-
latestSeq: eventResult?.latestSeq ?? afterSeq,
|
|
3258
|
-
wait: null,
|
|
3259
|
-
coordinatorObserve: {
|
|
3260
|
-
ms: Date.now() - statusStartedAt,
|
|
3261
|
-
waitMs,
|
|
3262
|
-
workflowStatus: eventResult?.events.length ? 'event' : 'event-timeout',
|
|
3263
|
-
statusPolls: 0,
|
|
3264
|
-
instanceId: null,
|
|
3265
|
-
},
|
|
3266
|
-
...(includeTrace ? { coordinatorTrace } : {}),
|
|
3267
|
-
});
|
|
3268
|
-
}
|
|
3269
|
-
|
|
3270
|
-
// get() throws if the instance doesn't exist (Workflows local-mode wipes
|
|
3271
|
-
// state on wrangler dev reload, and superseded `--force` runs may target
|
|
3272
|
-
// an instance that was never created). Treat that as a no-op cancel.
|
|
3273
|
-
let instance: WorkflowInstance | null = null;
|
|
3274
|
-
try {
|
|
3275
|
-
const requestedInstanceId = new URL(request.url).searchParams
|
|
3276
|
-
.get('instanceId')
|
|
3277
|
-
?.trim();
|
|
3278
|
-
const instanceId =
|
|
3279
|
-
requestedInstanceId &&
|
|
3280
|
-
isWorkflowInstanceIdForRun(runId, requestedInstanceId)
|
|
3281
|
-
? requestedInstanceId
|
|
3282
|
-
: await resolveWorkflowInstanceIdForRun(env, runId);
|
|
3283
|
-
instance = await env.PLAY_WORKFLOW.get(instanceId);
|
|
3284
|
-
} catch (error) {
|
|
3285
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
3286
|
-
if (
|
|
3287
|
-
!/not[ _]found|not_found|does not exist|no such instance|404/i.test(
|
|
3288
|
-
message,
|
|
3289
|
-
)
|
|
3290
|
-
) {
|
|
3291
|
-
throw error;
|
|
3292
|
-
}
|
|
3293
|
-
}
|
|
3294
|
-
try {
|
|
3295
|
-
if (action === 'cancel') {
|
|
3296
|
-
if (instance) {
|
|
3297
|
-
try {
|
|
3298
|
-
await instance.terminate();
|
|
3299
|
-
} catch (error) {
|
|
3300
|
-
const message =
|
|
3301
|
-
error instanceof Error ? error.message : String(error);
|
|
3302
|
-
// Tolerate four classes of error here:
|
|
3303
|
-
// - already-terminal (complete / errored / terminated)
|
|
3304
|
-
// - "Cannot terminate instance since its on a finite state"
|
|
3305
|
-
// (the runtime's wording for "already finished")
|
|
3306
|
-
// - "not implemented" (wrangler dev local mode doesn't support
|
|
3307
|
-
// instance.terminate() yet — silently no-op there)
|
|
3308
|
-
// - "not found" (instance never existed)
|
|
3309
|
-
if (
|
|
3310
|
-
!/complete|terminated|errored|finite state|cannot[ _]terminate|not[ _]implemented|not[ _]found|404/i.test(
|
|
3311
|
-
message,
|
|
3312
|
-
)
|
|
3313
|
-
) {
|
|
3314
|
-
throw error;
|
|
3315
|
-
}
|
|
3316
|
-
}
|
|
3317
|
-
}
|
|
3318
|
-
// terminate() kills the dynamic worker before its run() wrapper can
|
|
3319
|
-
// write terminal state (the only place completed/failed land), so
|
|
3320
|
-
// without this write /tail reports 'running' forever and any
|
|
3321
|
-
// start-stream watcher hangs after a cancel. Land the cancelled
|
|
3322
|
-
// terminal state here — terminal-set appends a 'terminal' run event
|
|
3323
|
-
// and wakes the dedup DO's long-poll waiters, which unblocks tails.
|
|
3324
|
-
//
|
|
3325
|
-
// Idempotency: first-wins from this side — if the run already went
|
|
3326
|
-
// terminal (completed/failed/cancelled) we keep that state. The DO
|
|
3327
|
-
// stores the cached terminal state under a single storage key
|
|
3328
|
-
// (last-wins on raw writes), but the run-event log is append-only
|
|
3329
|
-
// and /tail truncates at the FIRST terminal event, so a racing
|
|
3330
|
-
// completed/failed write from a dying worker can at worst replace
|
|
3331
|
-
// the cached key with another terminal status — it can never
|
|
3332
|
-
// resurrect 'running'.
|
|
3333
|
-
const existingTerminal = await readCoordinatorTerminalState(
|
|
3334
|
-
env,
|
|
3335
|
-
runId,
|
|
3336
|
-
).catch((error: unknown) => {
|
|
3337
|
-
// Tolerated: better to risk a harmless terminal-over-terminal
|
|
3338
|
-
// overwrite than to skip the cancelled write and hang watchers.
|
|
3339
|
-
recordCoordinatorEvent(
|
|
3340
|
-
'warn',
|
|
3341
|
-
'coordinator.cancel.terminal_state_read_failed',
|
|
3342
|
-
'[coordinator] terminal state read before cancel failed',
|
|
3343
|
-
{
|
|
3344
|
-
runId,
|
|
3345
|
-
error: error instanceof Error ? error.message : String(error),
|
|
3346
|
-
},
|
|
3347
|
-
);
|
|
3348
|
-
return null;
|
|
3349
|
-
});
|
|
3350
|
-
if (!existingTerminal) {
|
|
3351
|
-
try {
|
|
3352
|
-
await writeCoordinatorTerminalState(env, {
|
|
3353
|
-
runId,
|
|
3354
|
-
status: 'cancelled',
|
|
3355
|
-
error: 'Run cancelled',
|
|
3356
|
-
});
|
|
3357
|
-
} catch (error) {
|
|
3358
|
-
// Fail loudly: the workflow was terminated but watchers would
|
|
3359
|
-
// hang on 'running' forever without the terminal event.
|
|
3360
|
-
const message =
|
|
3361
|
-
error instanceof Error ? error.message : String(error);
|
|
3362
|
-
recordCoordinatorEvent(
|
|
3363
|
-
'error',
|
|
3364
|
-
'coordinator.cancel.terminal_state_write_failed',
|
|
3365
|
-
'[coordinator] cancel terminal state write failed',
|
|
3366
|
-
{
|
|
3367
|
-
runId,
|
|
3368
|
-
error: message,
|
|
3369
|
-
},
|
|
3370
|
-
);
|
|
3371
|
-
return Response.json(
|
|
3372
|
-
{
|
|
3373
|
-
runId,
|
|
3374
|
-
status: 'error',
|
|
3375
|
-
error: `workflow terminated but cancelled terminal state write failed: ${message}`,
|
|
3376
|
-
},
|
|
3377
|
-
{ status: 500 },
|
|
3378
|
-
);
|
|
3379
|
-
}
|
|
3380
|
-
}
|
|
3381
|
-
return Response.json({ runId, status: 'cancelled' });
|
|
3382
|
-
}
|
|
3383
|
-
if (action === 'signal') {
|
|
3384
|
-
if (!instance) {
|
|
3385
|
-
return new Response('not found', { status: 404 });
|
|
3386
|
-
}
|
|
3387
|
-
const body = (await request.json().catch(() => ({}))) as Record<
|
|
3388
|
-
string,
|
|
3389
|
-
unknown
|
|
3390
|
-
>;
|
|
3391
|
-
const eventKey =
|
|
3392
|
-
typeof body.eventKey === 'string' && body.eventKey.trim()
|
|
3393
|
-
? body.eventKey.trim()
|
|
3394
|
-
: typeof body.event_key === 'string' && body.event_key.trim()
|
|
3395
|
-
? body.event_key.trim()
|
|
3396
|
-
: '';
|
|
3397
|
-
const eventType =
|
|
3398
|
-
typeof body.type === 'string' && body.type.trim()
|
|
3399
|
-
? body.type.trim()
|
|
3400
|
-
: eventKey
|
|
3401
|
-
? `integration_event_${eventKey}`
|
|
3402
|
-
: 'integration_event';
|
|
3403
|
-
await instance.sendEvent({
|
|
3404
|
-
type: workflowEventType(eventType),
|
|
3405
|
-
payload: body,
|
|
3406
|
-
});
|
|
3407
|
-
return Response.json({
|
|
3408
|
-
runId,
|
|
3409
|
-
status: mapWorkflowStatus(await instance.status()),
|
|
3410
|
-
});
|
|
3411
|
-
}
|
|
3412
|
-
if (action === 'result' || action === 'status' || action === '') {
|
|
3413
|
-
const includeTrace =
|
|
3414
|
-
new URL(request.url).searchParams.get('trace') === '1';
|
|
3415
|
-
const statusStartedAt = Date.now();
|
|
3416
|
-
const terminalState = await readCoordinatorTerminalState(
|
|
3417
|
-
env,
|
|
3418
|
-
runId,
|
|
3419
|
-
).catch(() => null);
|
|
3420
|
-
if (terminalState) {
|
|
3421
|
-
const coordinatorTrace = includeTrace
|
|
3422
|
-
? await listCoordinatorPerfTrace(env, runId).catch(() => [])
|
|
3423
|
-
: [];
|
|
3424
|
-
return Response.json({
|
|
3425
|
-
runId,
|
|
3426
|
-
...(terminalState.playName
|
|
3427
|
-
? { playName: terminalState.playName }
|
|
3428
|
-
: {}),
|
|
3429
|
-
status: terminalState.status,
|
|
3430
|
-
result: terminalState.result ?? null,
|
|
3431
|
-
error: terminalState.error ?? null,
|
|
3432
|
-
totalRows: terminalState.totalRows ?? null,
|
|
3433
|
-
durationMs: terminalState.durationMs ?? null,
|
|
3434
|
-
completedAt: terminalState.completedAt ?? null,
|
|
3435
|
-
liveLogs: sanitizeLiveLogLines(terminalState.liveLogs),
|
|
3436
|
-
liveNodeProgress: terminalState.liveNodeProgress ?? null,
|
|
3437
|
-
wait: null,
|
|
3438
|
-
coordinatorObserve: {
|
|
3439
|
-
ms: Date.now() - statusStartedAt,
|
|
3440
|
-
waitMs: 0,
|
|
3441
|
-
workflowStatus: 'terminal-cache',
|
|
3442
|
-
statusPolls: 0,
|
|
3443
|
-
instanceId: instance?.id ?? null,
|
|
3444
|
-
},
|
|
3445
|
-
...(includeTrace ? { coordinatorTrace } : {}),
|
|
3446
|
-
});
|
|
3447
|
-
}
|
|
3448
|
-
if (!instance) {
|
|
3449
|
-
return new Response('not found', { status: 404 });
|
|
3450
|
-
}
|
|
3451
|
-
const status = await instance.status();
|
|
3452
|
-
const workflowError = readWorkflowError(status);
|
|
3453
|
-
if (workflowError) {
|
|
3454
|
-
const retry = await restartWorkflowAfterPlatformReset({
|
|
3455
|
-
env,
|
|
3456
|
-
ctx: input.ctx,
|
|
3457
|
-
runId,
|
|
3458
|
-
oldInstance: instance,
|
|
3459
|
-
error: workflowError,
|
|
3460
|
-
status,
|
|
3461
|
-
});
|
|
3462
|
-
if (retry.retried && retry.result) {
|
|
3463
|
-
const coordinatorTrace = includeTrace
|
|
3464
|
-
? await listCoordinatorPerfTrace(env, runId).catch(() => [])
|
|
3465
|
-
: [];
|
|
3466
|
-
return Response.json({
|
|
3467
|
-
...retry.result,
|
|
3468
|
-
coordinatorObserve: {
|
|
3469
|
-
ms: Date.now() - statusStartedAt,
|
|
3470
|
-
waitMs: 0,
|
|
3471
|
-
workflowStatus: 'retrying',
|
|
3472
|
-
statusPolls: 1,
|
|
3473
|
-
instanceId: instance.id,
|
|
3474
|
-
},
|
|
3475
|
-
...(includeTrace ? { coordinatorTrace } : {}),
|
|
3476
|
-
});
|
|
3477
|
-
}
|
|
3478
|
-
}
|
|
3479
|
-
const result = mapWorkflowResult(runId, status);
|
|
3480
|
-
const observeMs = Date.now() - statusStartedAt;
|
|
3481
|
-
// If we forced a permanent-error fail-fast (status='failed' even
|
|
3482
|
-
// though the underlying instance is still 'running' mid-retry),
|
|
3483
|
-
// terminate on the CF side once so the retry loop stops burning
|
|
3484
|
-
// compute. Best-effort: tolerate races where the instance has
|
|
3485
|
-
// already gone terminal between our status() and terminate().
|
|
3486
|
-
if (
|
|
3487
|
-
result.status === 'failed' &&
|
|
3488
|
-
status.status !== 'errored' &&
|
|
3489
|
-
status.status !== 'terminated' &&
|
|
3490
|
-
status.status !== 'complete'
|
|
3491
|
-
) {
|
|
3492
|
-
await instance.terminate().catch((error: unknown) => {
|
|
3493
|
-
const message =
|
|
3494
|
-
error instanceof Error ? error.message : String(error);
|
|
3495
|
-
if (
|
|
3496
|
-
!/complete|terminated|errored|finite state|cannot[ _]terminate|not[ _]implemented|not[ _]found|404/i.test(
|
|
3497
|
-
message,
|
|
3498
|
-
)
|
|
3499
|
-
) {
|
|
3500
|
-
recordCoordinatorEvent(
|
|
3501
|
-
'warn',
|
|
3502
|
-
'coordinator.permanent_error.terminate_failed',
|
|
3503
|
-
'[coordinator] terminate-after-permanent-error failed',
|
|
3504
|
-
{
|
|
3505
|
-
runId,
|
|
3506
|
-
error: message,
|
|
3507
|
-
},
|
|
3508
|
-
);
|
|
3509
|
-
}
|
|
3510
|
-
});
|
|
3511
|
-
}
|
|
3512
|
-
const coordinatorTrace = includeTrace
|
|
3513
|
-
? await listCoordinatorPerfTrace(env, runId).catch(() => [])
|
|
3514
|
-
: [];
|
|
3515
|
-
return Response.json({
|
|
3516
|
-
...result,
|
|
3517
|
-
coordinatorObserve: {
|
|
3518
|
-
ms: observeMs,
|
|
3519
|
-
waitMs: 0,
|
|
3520
|
-
workflowStatus: status.status,
|
|
3521
|
-
statusPolls: 1,
|
|
3522
|
-
instanceId: instance.id,
|
|
3523
|
-
},
|
|
3524
|
-
...(includeTrace ? { coordinatorTrace } : {}),
|
|
3525
|
-
});
|
|
3526
|
-
}
|
|
3527
|
-
return new Response('not found', { status: 404 });
|
|
3528
|
-
} finally {
|
|
3529
|
-
disposeRpcStub(instance);
|
|
3530
|
-
}
|
|
3531
|
-
}
|
|
3532
|
-
|
|
3533
|
-
function workflowInstanceId(runId: string): string {
|
|
3534
|
-
const normalized = runId.toLowerCase().replace(/[^a-z0-9_-]+/g, '-');
|
|
3535
|
-
if (normalized.length > 0 && normalized.length <= 100) {
|
|
3536
|
-
return normalized;
|
|
3537
|
-
}
|
|
3538
|
-
return `run-${stableHash(runId)}`;
|
|
3539
|
-
}
|
|
3540
|
-
|
|
3541
|
-
function isWorkflowInstanceIdForRun(
|
|
3542
|
-
runId: string,
|
|
3543
|
-
instanceId: string,
|
|
3544
|
-
): boolean {
|
|
3545
|
-
const canonical = workflowInstanceId(runId);
|
|
3546
|
-
return (
|
|
3547
|
-
instanceId === canonical || instanceId.startsWith(`${canonical}-retry-`)
|
|
3548
|
-
);
|
|
3549
|
-
}
|
|
3550
|
-
|
|
3551
|
-
function stableHash(value: string): string {
|
|
3552
|
-
let hash = 2166136261;
|
|
3553
|
-
for (let index = 0; index < value.length; index += 1) {
|
|
3554
|
-
hash ^= value.charCodeAt(index);
|
|
3555
|
-
hash = Math.imul(hash, 16777619);
|
|
3556
|
-
}
|
|
3557
|
-
return (hash >>> 0).toString(36);
|
|
3558
|
-
}
|
|
3559
|
-
|
|
3560
|
-
const DYNAMIC_PLAY_WORKER_HARNESS_VERSION =
|
|
3561
|
-
DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION;
|
|
3562
|
-
const DYNAMIC_WORKER_BUNDLED_CODE_CACHE_MAX_ENTRIES = 64;
|
|
3563
|
-
const dynamicWorkerBundledCodeCache = new Map<string, string>();
|
|
3564
|
-
|
|
3565
|
-
function dynamicPlayWorkerCacheKey(input: {
|
|
3566
|
-
env: CoordinatorEnv;
|
|
3567
|
-
graphHash: string;
|
|
3568
|
-
artifactIdentity: string;
|
|
3569
|
-
}): string {
|
|
3570
|
-
const deployMarker =
|
|
3571
|
-
input.env.DEEPLINE_COORDINATOR_DEPLOY_MARKER?.trim() || 'local';
|
|
3572
|
-
return [
|
|
3573
|
-
'play',
|
|
3574
|
-
input.graphHash,
|
|
3575
|
-
input.artifactIdentity,
|
|
3576
|
-
`harness=${DYNAMIC_PLAY_WORKER_HARNESS_VERSION}`,
|
|
3577
|
-
`deploy=${deployMarker}`,
|
|
3578
|
-
].join(':');
|
|
3579
|
-
}
|
|
3580
|
-
|
|
3581
|
-
function dynamicPlayWorkerArtifactIdentity(
|
|
3582
|
-
metadata: Pick<
|
|
3583
|
-
DynamicWorkflowMetadata,
|
|
3584
|
-
| 'artifactHash'
|
|
3585
|
-
| 'artifactStorageKey'
|
|
3586
|
-
| 'dynamicWorkerCode'
|
|
3587
|
-
| 'dynamicWorkerCodeHash'
|
|
3588
|
-
>,
|
|
3589
|
-
): string {
|
|
3590
|
-
const artifactHash = metadata.artifactHash?.trim();
|
|
3591
|
-
const inlineCodeHash =
|
|
3592
|
-
typeof metadata.dynamicWorkerCodeHash === 'string' &&
|
|
3593
|
-
metadata.dynamicWorkerCodeHash.length > 0
|
|
3594
|
-
? metadata.dynamicWorkerCodeHash
|
|
3595
|
-
: typeof metadata.dynamicWorkerCode === 'string' &&
|
|
3596
|
-
metadata.dynamicWorkerCode.length > 0
|
|
3597
|
-
? stableHash(metadata.dynamicWorkerCode)
|
|
3598
|
-
: null;
|
|
3599
|
-
if (artifactHash && inlineCodeHash) {
|
|
3600
|
-
return `${artifactHash}:inline=${inlineCodeHash}`;
|
|
3601
|
-
}
|
|
3602
|
-
if (artifactHash) {
|
|
3603
|
-
return artifactHash;
|
|
3604
|
-
}
|
|
3605
|
-
if (inlineCodeHash) {
|
|
3606
|
-
return `inline=${inlineCodeHash}`;
|
|
3607
|
-
}
|
|
3608
|
-
return stableHash(metadata.artifactStorageKey);
|
|
3609
|
-
}
|
|
3610
|
-
|
|
3611
|
-
function dynamicWorkerBundledCodeCacheKey(input: {
|
|
3612
|
-
artifactStorageKey: string;
|
|
3613
|
-
artifactHash?: string | null;
|
|
3614
|
-
}): string {
|
|
3615
|
-
return `${input.artifactHash?.trim() || 'no-hash'}:${input.artifactStorageKey}`;
|
|
3616
|
-
}
|
|
3617
|
-
|
|
3618
|
-
function readDynamicWorkerBundledCodeCache(key: string): string | null {
|
|
3619
|
-
const cached = dynamicWorkerBundledCodeCache.get(key);
|
|
3620
|
-
if (cached === undefined) return null;
|
|
3621
|
-
dynamicWorkerBundledCodeCache.delete(key);
|
|
3622
|
-
dynamicWorkerBundledCodeCache.set(key, cached);
|
|
3623
|
-
return cached;
|
|
3624
|
-
}
|
|
3625
|
-
|
|
3626
|
-
function writeDynamicWorkerBundledCodeCache(key: string, code: string): void {
|
|
3627
|
-
dynamicWorkerBundledCodeCache.set(key, code);
|
|
3628
|
-
while (
|
|
3629
|
-
dynamicWorkerBundledCodeCache.size >
|
|
3630
|
-
DYNAMIC_WORKER_BUNDLED_CODE_CACHE_MAX_ENTRIES
|
|
3631
|
-
) {
|
|
3632
|
-
const oldestKey = dynamicWorkerBundledCodeCache.keys().next().value;
|
|
3633
|
-
if (typeof oldestKey !== 'string') break;
|
|
3634
|
-
dynamicWorkerBundledCodeCache.delete(oldestKey);
|
|
3635
|
-
}
|
|
3636
|
-
}
|
|
3637
|
-
|
|
3638
|
-
/**
|
|
3639
|
-
* Synchronous wrapper around env.LOADER.get for use inside the
|
|
3640
|
-
* createDynamicWorkflowEntrypoint loader callback. The framework's loader
|
|
3641
|
-
* type allows async, but the official example
|
|
3642
|
-
* (cloudflare/dynamic-workflows repo) uses sync; the microtask hop a
|
|
3643
|
-
* top-level `await` introduces appears to interact badly with how the
|
|
3644
|
-
* framework wires up the returned entrypoint stub for runner.run() RPC.
|
|
3645
|
-
*/
|
|
3646
|
-
function loadDynamicPlayWorkerSync(
|
|
3647
|
-
env: CoordinatorEnv,
|
|
3648
|
-
metadata: DynamicWorkflowMetadata,
|
|
3649
|
-
trace: CoordinatorPerfTraceSink = recordCoordinatorPerfTrace,
|
|
3650
|
-
): {
|
|
3651
|
-
getEntrypoint(name?: string): Fetcher & WorkflowRunner;
|
|
3652
|
-
} {
|
|
3653
|
-
if (!env.LOADER) {
|
|
3654
|
-
throw new Error(
|
|
3655
|
-
'Cloudflare Dynamic Workflows require a Worker Loader binding named LOADER.',
|
|
3656
|
-
);
|
|
3657
|
-
}
|
|
3658
|
-
if (!env.HARNESS) {
|
|
3659
|
-
throw new Error(
|
|
3660
|
-
'Cloudflare Dynamic Workflows require a service binding named HARNESS. ' +
|
|
3661
|
-
'Start apps/play-harness-worker before the coordinator or fix wrangler.toml services.',
|
|
3662
|
-
);
|
|
3663
|
-
}
|
|
3664
|
-
const graphHash = metadata.graphHash.trim();
|
|
3665
|
-
if (!graphHash) {
|
|
3666
|
-
throw new Error('Dynamic play worker requires graphHash metadata.');
|
|
3667
|
-
}
|
|
3668
|
-
const artifactStorageKey = metadata.artifactStorageKey.trim();
|
|
3669
|
-
if (!artifactStorageKey) {
|
|
3670
|
-
throw new Error(
|
|
3671
|
-
'Dynamic play worker requires artifactStorageKey metadata.',
|
|
3672
|
-
);
|
|
3673
|
-
}
|
|
3674
|
-
const artifactIdentity = dynamicPlayWorkerArtifactIdentity(metadata);
|
|
3675
|
-
const workerCacheKey = dynamicPlayWorkerCacheKey({
|
|
3676
|
-
env,
|
|
3677
|
-
graphHash,
|
|
3678
|
-
artifactIdentity,
|
|
3679
|
-
});
|
|
3680
|
-
const runIdForTrace = metadata.runId ?? graphHash;
|
|
3681
|
-
const loaderGetStartedAt = Date.now();
|
|
3682
|
-
const stub = env.LOADER.get(workerCacheKey, async () => {
|
|
3683
|
-
const bundledCode = await loadDynamicWorkerBundledCode({
|
|
3684
|
-
env,
|
|
3685
|
-
metadata,
|
|
3686
|
-
graphHash,
|
|
3687
|
-
artifactStorageKey,
|
|
3688
|
-
workerCacheKey,
|
|
3689
|
-
runIdForTrace,
|
|
3690
|
-
trace,
|
|
3691
|
-
});
|
|
3692
|
-
return {
|
|
3693
|
-
compatibilityDate: DYNAMIC_WORKER_COMPATIBILITY_DATE,
|
|
3694
|
-
compatibilityFlags: ['nodejs_compat'],
|
|
3695
|
-
mainModule: 'index.js',
|
|
3696
|
-
modules: { 'index.js': bundledCode },
|
|
3697
|
-
env: {
|
|
3698
|
-
// Service binding to the long-lived Play Harness Worker.
|
|
3699
|
-
// Per-play code reaches this via `env.HARNESS.<method>(...)` —
|
|
3700
|
-
// see sdk/src/plays/harness-stub.ts. This binding is required;
|
|
3701
|
-
// miswired environments fail before user code starts.
|
|
3702
|
-
HARNESS: env.HARNESS,
|
|
3703
|
-
VERCEL_PROTECTION_BYPASS_TOKEN: env.VERCEL_PROTECTION_BYPASS_TOKEN,
|
|
3704
|
-
// Runtime API bridge used by the play harness for status, tool
|
|
3705
|
-
// execution, DB session, and artifact callbacks. This uses the
|
|
3706
|
-
// long-lived HARNESS service binding, avoiding public callback HTTP
|
|
3707
|
-
// without relying on dynamic-worker access to named exports.
|
|
3708
|
-
...makeRuntimeApiEnvBinding(env),
|
|
3709
|
-
// In-process coordinator control bridge used by ctx.runPlay and
|
|
3710
|
-
// parent terminal signals. This keeps scalar child plays inline with
|
|
3711
|
-
// the parent instead of round-tripping through nested Workflow waits.
|
|
3712
|
-
...makeCoordinatorControlBinding(),
|
|
3713
|
-
// NOTE: We intentionally do NOT pass `env.PLAYS_BUCKET` (an R2Bucket
|
|
3714
|
-
// binding) through to the per-play Worker's env. Including a raw
|
|
3715
|
-
// R2Bucket in the dynamically-loaded Worker's env makes Cloudflare
|
|
3716
|
-
// Workflows fail at workflow init with
|
|
3717
|
-
// "Could not serialize object of type 'R2Bucket'."
|
|
3718
|
-
// because the framework persists workflow state through a JSDevalue
|
|
3719
|
-
// path that R2Bucket bindings do not support. Per-play harness reads
|
|
3720
|
-
// (CSVs, packaged imports) go through the long-lived HARNESS Worker.
|
|
3721
|
-
},
|
|
3722
|
-
};
|
|
3723
|
-
});
|
|
3724
|
-
trace({
|
|
3725
|
-
runId: runIdForTrace,
|
|
3726
|
-
phase: 'coordinator.loader_get_call',
|
|
3727
|
-
ms: Date.now() - loaderGetStartedAt,
|
|
3728
|
-
graphHash,
|
|
3729
|
-
extra: { workerCacheKey },
|
|
3730
|
-
});
|
|
3731
|
-
return stub;
|
|
3732
|
-
}
|
|
3733
|
-
|
|
3734
|
-
async function loadDynamicPlayWorker(
|
|
3735
|
-
env: CoordinatorEnv,
|
|
3736
|
-
metadata: DynamicWorkflowMetadata,
|
|
3737
|
-
trace: CoordinatorPerfTraceSink = recordCoordinatorPerfTrace,
|
|
3738
|
-
): Promise<{
|
|
3739
|
-
getEntrypoint(name?: string): Fetcher & WorkflowRunner;
|
|
3740
|
-
}> {
|
|
3741
|
-
if (!env.LOADER) {
|
|
3742
|
-
throw new Error(
|
|
3743
|
-
'Cloudflare Dynamic Workflows require a Worker Loader binding named LOADER.',
|
|
3744
|
-
);
|
|
3745
|
-
}
|
|
3746
|
-
if (!env.HARNESS) {
|
|
3747
|
-
throw new Error(
|
|
3748
|
-
'Cloudflare Dynamic Workflows require a service binding named HARNESS. ' +
|
|
3749
|
-
'Start apps/play-harness-worker before the coordinator or fix wrangler.toml services.',
|
|
3750
|
-
);
|
|
3751
|
-
}
|
|
3752
|
-
const graphHash = metadata.graphHash.trim();
|
|
3753
|
-
if (!graphHash) {
|
|
3754
|
-
throw new Error('Dynamic play worker requires graphHash metadata.');
|
|
3755
|
-
}
|
|
3756
|
-
const artifactStorageKey = metadata.artifactStorageKey.trim();
|
|
3757
|
-
if (!artifactStorageKey) {
|
|
3758
|
-
throw new Error(
|
|
3759
|
-
'Dynamic play worker requires artifactStorageKey metadata.',
|
|
3760
|
-
);
|
|
3761
|
-
}
|
|
3762
|
-
const artifactIdentity = dynamicPlayWorkerArtifactIdentity(metadata);
|
|
3763
|
-
const workerCacheKey = dynamicPlayWorkerCacheKey({
|
|
3764
|
-
env,
|
|
3765
|
-
graphHash,
|
|
3766
|
-
artifactIdentity,
|
|
3767
|
-
});
|
|
3768
|
-
const runIdForTrace = metadata.runId ?? graphHash;
|
|
3769
|
-
const loaderGetStartedAt = Date.now();
|
|
3770
|
-
const stub = env.LOADER.get(workerCacheKey, async () => {
|
|
3771
|
-
const bundledCode = await loadDynamicWorkerBundledCode({
|
|
3772
|
-
env,
|
|
3773
|
-
metadata,
|
|
3774
|
-
graphHash,
|
|
3775
|
-
artifactStorageKey,
|
|
3776
|
-
workerCacheKey,
|
|
3777
|
-
runIdForTrace,
|
|
3778
|
-
trace,
|
|
3779
|
-
});
|
|
3780
|
-
return {
|
|
3781
|
-
compatibilityDate: DYNAMIC_WORKER_COMPATIBILITY_DATE,
|
|
3782
|
-
compatibilityFlags: ['nodejs_compat'],
|
|
3783
|
-
mainModule: 'index.js',
|
|
3784
|
-
modules: { 'index.js': bundledCode },
|
|
3785
|
-
env: {
|
|
3786
|
-
// Mirror of the sync loader (above) — see that copy for the
|
|
3787
|
-
// architectural rationale. The dynamic worker env is intentionally
|
|
3788
|
-
// minimal; runtime callbacks use RUNTIME_API, file reads go through
|
|
3789
|
-
// HARNESS, and child workflow control uses the COORDINATOR binding.
|
|
3790
|
-
HARNESS: env.HARNESS,
|
|
3791
|
-
VERCEL_PROTECTION_BYPASS_TOKEN: env.VERCEL_PROTECTION_BYPASS_TOKEN,
|
|
3792
|
-
...makeRuntimeApiEnvBinding(env),
|
|
3793
|
-
...makeCoordinatorControlBinding(),
|
|
3794
|
-
},
|
|
3795
|
-
};
|
|
3796
|
-
});
|
|
3797
|
-
trace({
|
|
3798
|
-
runId: runIdForTrace,
|
|
3799
|
-
phase: 'coordinator.loader_get_call',
|
|
3800
|
-
ms: Date.now() - loaderGetStartedAt,
|
|
3801
|
-
graphHash,
|
|
3802
|
-
extra: { workerCacheKey },
|
|
3803
|
-
});
|
|
3804
|
-
return stub;
|
|
3805
|
-
}
|
|
3806
|
-
|
|
3807
|
-
async function loadDynamicWorkerBundledCode(input: {
|
|
3808
|
-
env: CoordinatorEnv;
|
|
3809
|
-
metadata: DynamicWorkflowMetadata;
|
|
3810
|
-
graphHash: string;
|
|
3811
|
-
artifactStorageKey: string;
|
|
3812
|
-
workerCacheKey: string;
|
|
3813
|
-
runIdForTrace: string;
|
|
3814
|
-
trace: CoordinatorPerfTraceSink;
|
|
3815
|
-
}): Promise<string> {
|
|
3816
|
-
const callbackStartedAt = Date.now();
|
|
3817
|
-
let codeSource: 'inline' | 'r2' | 'memory' = 'inline';
|
|
3818
|
-
let r2Ms = 0;
|
|
3819
|
-
const codeCacheKey = dynamicWorkerBundledCodeCacheKey({
|
|
3820
|
-
artifactStorageKey: input.artifactStorageKey,
|
|
3821
|
-
artifactHash: input.metadata.artifactHash,
|
|
3822
|
-
});
|
|
3823
|
-
let bundledCode = input.metadata.dynamicWorkerCode ?? null;
|
|
3824
|
-
if (!bundledCode) {
|
|
3825
|
-
bundledCode = readDynamicWorkerBundledCodeCache(codeCacheKey);
|
|
3826
|
-
if (bundledCode) {
|
|
3827
|
-
codeSource = 'memory';
|
|
3828
|
-
}
|
|
3829
|
-
}
|
|
3830
|
-
if (!bundledCode) {
|
|
3831
|
-
codeSource = 'r2';
|
|
3832
|
-
const r2StartedAt = Date.now();
|
|
3833
|
-
try {
|
|
3834
|
-
const artifact = await loadStoredPlayArtifactFromR2(
|
|
3835
|
-
input.env,
|
|
3836
|
-
input.artifactStorageKey,
|
|
3837
|
-
);
|
|
3838
|
-
bundledCode = artifact?.artifact?.bundledCode ?? null;
|
|
3839
|
-
if (typeof bundledCode === 'string' && bundledCode.length > 0) {
|
|
3840
|
-
writeDynamicWorkerBundledCodeCache(codeCacheKey, bundledCode);
|
|
3841
|
-
}
|
|
3842
|
-
} finally {
|
|
3843
|
-
r2Ms = Date.now() - r2StartedAt;
|
|
3844
|
-
input.trace({
|
|
3845
|
-
runId: input.runIdForTrace,
|
|
3846
|
-
phase: 'coordinator.loader_code_r2_get',
|
|
3847
|
-
ms: r2Ms,
|
|
3848
|
-
graphHash: input.graphHash,
|
|
3849
|
-
extra: { artifactStorageKey: input.artifactStorageKey },
|
|
3850
|
-
});
|
|
3851
|
-
}
|
|
3852
|
-
}
|
|
3853
|
-
if (typeof bundledCode !== 'string' || bundledCode.length === 0) {
|
|
3854
|
-
throw new Error(
|
|
3855
|
-
`Stored play artifact ${input.artifactStorageKey} does not contain bundledCode.`,
|
|
3856
|
-
);
|
|
3857
|
-
}
|
|
3858
|
-
input.trace({
|
|
3859
|
-
runId: input.runIdForTrace,
|
|
3860
|
-
phase: 'coordinator.loader_code_callback',
|
|
3861
|
-
ms: Date.now() - callbackStartedAt,
|
|
3862
|
-
graphHash: input.graphHash,
|
|
3863
|
-
extra: {
|
|
3864
|
-
codeSource,
|
|
3865
|
-
r2Ms,
|
|
3866
|
-
workerCacheKey: input.workerCacheKey,
|
|
3867
|
-
bundledCodeBytes: bundledCode.length,
|
|
3868
|
-
},
|
|
3869
|
-
});
|
|
3870
|
-
return bundledCode;
|
|
3871
|
-
}
|
|
3872
|
-
|
|
3873
|
-
function normalizePackagedFiles(
|
|
3874
|
-
value: unknown,
|
|
3875
|
-
): NonNullable<DynamicWorkflowMetadata['packagedFiles']> {
|
|
3876
|
-
if (!Array.isArray(value)) return [];
|
|
3877
|
-
return value
|
|
3878
|
-
.filter((entry): entry is Record<string, unknown> =>
|
|
3879
|
-
Boolean(entry && typeof entry === 'object' && !Array.isArray(entry)),
|
|
3880
|
-
)
|
|
3881
|
-
.map((entry) => ({
|
|
3882
|
-
playPath: String(entry.playPath ?? '').replace(/^\.\//, ''),
|
|
3883
|
-
storageKey: String(entry.storageKey ?? ''),
|
|
3884
|
-
contentType:
|
|
3885
|
-
typeof entry.contentType === 'string' ? entry.contentType : undefined,
|
|
3886
|
-
bytes:
|
|
3887
|
-
typeof entry.bytes === 'number' &&
|
|
3888
|
-
Number.isSafeInteger(entry.bytes) &&
|
|
3889
|
-
entry.bytes >= 0
|
|
3890
|
-
? entry.bytes
|
|
3891
|
-
: undefined,
|
|
3892
|
-
inlineText:
|
|
3893
|
-
typeof entry.inlineText === 'string' ? entry.inlineText : undefined,
|
|
3894
|
-
}))
|
|
3895
|
-
.filter(
|
|
3896
|
-
(entry) => entry.playPath.length > 0 && entry.storageKey.length > 0,
|
|
3897
|
-
);
|
|
3898
|
-
}
|
|
3899
|
-
|
|
3900
|
-
const COORDINATOR_WARMUP_DYNAMIC_WORKER_CODE = `
|
|
3901
|
-
import { WorkflowEntrypoint } from "cloudflare:workers";
|
|
3902
|
-
|
|
3903
|
-
export class TenantWorkflow extends WorkflowEntrypoint {
|
|
3904
|
-
async run(event) {
|
|
3905
|
-
const payload = event && typeof event === "object" ? event.payload : null;
|
|
3906
|
-
const runId = payload && typeof payload.runId === "string" ? payload.runId : "warmup";
|
|
3907
|
-
const startedAt = Date.now();
|
|
3908
|
-
if (this.env.RUNTIME_API) {
|
|
3909
|
-
await this.env.RUNTIME_API.fetch(new Request("https://deepline.runtime.internal${PLAY_RUNTIME_API_COMPAT_PATH}", {
|
|
3910
|
-
method: "POST",
|
|
3911
|
-
headers: { "content-type": "application/json" },
|
|
3912
|
-
body: "{}"
|
|
3913
|
-
})).catch(() => null);
|
|
3914
|
-
}
|
|
3915
|
-
console.log("[perf-trace] " + JSON.stringify({
|
|
3916
|
-
telemetrySchemaVersion: 1,
|
|
3917
|
-
timestamp: new Date().toISOString(),
|
|
3918
|
-
service: "play-runtime",
|
|
3919
|
-
component: "coordinator-warmup",
|
|
3920
|
-
event: "dynamic_worker.warmup_run",
|
|
3921
|
-
tag: "[perf-trace]",
|
|
3922
|
-
ts: Date.now(),
|
|
3923
|
-
source: "dynamic_worker",
|
|
3924
|
-
runId,
|
|
3925
|
-
phase: "dynamic_worker.warmup_run",
|
|
3926
|
-
ms: Date.now() - startedAt,
|
|
3927
|
-
graphHash: "coordinator-warmup-v1",
|
|
3928
|
-
telemetryKind: "log",
|
|
3929
|
-
telemetryLevel: "info"
|
|
3930
|
-
}));
|
|
3931
|
-
return { ok: true, warmup: true };
|
|
3932
|
-
}
|
|
3933
|
-
}
|
|
3934
|
-
|
|
3935
|
-
export default {
|
|
3936
|
-
async fetch(request, env) {
|
|
3937
|
-
const url = new URL(request.url);
|
|
3938
|
-
if (request.method === "POST" && url.pathname === "/start") {
|
|
3939
|
-
const body = await request.json().catch(() => null);
|
|
3940
|
-
if (!body || typeof body.id !== "string" || !body.payload || typeof body.payload !== "object") {
|
|
3941
|
-
return new Response("invalid workflow start body", { status: 400 });
|
|
3942
|
-
}
|
|
3943
|
-
const createStartedAt = Date.now();
|
|
3944
|
-
const instance = await env.WORKFLOWS.create({ id: body.id, params: body.payload });
|
|
3945
|
-
const workflowCreateMs = Date.now() - createStartedAt;
|
|
3946
|
-
return Response.json({
|
|
3947
|
-
id: instance.id,
|
|
3948
|
-
status: "submitted",
|
|
3949
|
-
timingsMs: { workflowCreate: workflowCreateMs }
|
|
3950
|
-
});
|
|
3951
|
-
}
|
|
3952
|
-
if (request.method === "GET" && url.pathname === "/health") {
|
|
3953
|
-
return new Response("ok", { status: 200 });
|
|
3954
|
-
}
|
|
3955
|
-
return new Response("not found", { status: 404 });
|
|
3956
|
-
}
|
|
3957
|
-
};
|
|
3958
|
-
`;
|
|
3959
|
-
|
|
3960
|
-
function isAllowedStagedFileStorageKey(key: string): boolean {
|
|
3961
|
-
if (!key || key.length > 2_048) return false;
|
|
3962
|
-
if (key.startsWith('/') || key.includes('..') || key.includes('\\')) {
|
|
3963
|
-
return false;
|
|
3964
|
-
}
|
|
3965
|
-
return key.startsWith('plays/v2/orgs/') || key.includes('/plays/v2/orgs/');
|
|
3966
|
-
}
|
|
3967
|
-
|
|
3968
|
-
function decodeBase64ToUint8Array(value: string): Uint8Array {
|
|
3969
|
-
const binary = atob(value);
|
|
3970
|
-
const bytes = new Uint8Array(binary.length);
|
|
3971
|
-
for (let index = 0; index < binary.length; index += 1) {
|
|
3972
|
-
bytes[index] = binary.charCodeAt(index);
|
|
3973
|
-
}
|
|
3974
|
-
return bytes;
|
|
3975
|
-
}
|
|
3976
|
-
|
|
3977
|
-
async function handleStagedFilePut(
|
|
3978
|
-
request: Request,
|
|
3979
|
-
env: CoordinatorEnv,
|
|
3980
|
-
): Promise<Response> {
|
|
3981
|
-
if (request.method !== 'POST') {
|
|
3982
|
-
return new Response('method not allowed', { status: 405 });
|
|
3983
|
-
}
|
|
3984
|
-
const url = new URL(request.url);
|
|
3985
|
-
const rawKey = url.searchParams.get('key')?.trim() ?? '';
|
|
3986
|
-
if (rawKey) {
|
|
3987
|
-
const key = rawKey;
|
|
3988
|
-
const contentType =
|
|
3989
|
-
url.searchParams.get('contentType')?.trim() || 'application/octet-stream';
|
|
3990
|
-
const expectedBytes = Number(url.searchParams.get('bytes') ?? 'NaN');
|
|
3991
|
-
if (!isAllowedStagedFileStorageKey(key)) {
|
|
3992
|
-
return Response.json(
|
|
3993
|
-
{ error: 'invalid staged file key' },
|
|
3994
|
-
{ status: 400 },
|
|
3995
|
-
);
|
|
3996
|
-
}
|
|
3997
|
-
if (!Number.isSafeInteger(expectedBytes) || expectedBytes < 0) {
|
|
3998
|
-
return Response.json({ error: 'bytes is required' }, { status: 400 });
|
|
3999
|
-
}
|
|
4000
|
-
const existing = await headExistingStagedFile(env, key, expectedBytes);
|
|
4001
|
-
if (existing.exists) {
|
|
4002
|
-
recordCoordinatorEvent(
|
|
4003
|
-
'info',
|
|
4004
|
-
'coordinator.staged_file_put',
|
|
4005
|
-
'[perf][coordinator.staged_file_put]',
|
|
4006
|
-
{
|
|
4007
|
-
key,
|
|
4008
|
-
bytes: expectedBytes,
|
|
4009
|
-
headMs: existing.ms,
|
|
4010
|
-
putMs: 0,
|
|
4011
|
-
ms: existing.ms,
|
|
4012
|
-
transport: 'raw',
|
|
4013
|
-
skipped: true,
|
|
4014
|
-
},
|
|
4015
|
-
);
|
|
4016
|
-
return Response.json({
|
|
4017
|
-
ok: true,
|
|
4018
|
-
key,
|
|
4019
|
-
bytes: expectedBytes,
|
|
4020
|
-
existed: true,
|
|
4021
|
-
timingsMs: { head: existing.ms, put: 0 },
|
|
4022
|
-
});
|
|
4023
|
-
}
|
|
4024
|
-
const readStartedAt = Date.now();
|
|
4025
|
-
const bytes = new Uint8Array(await request.arrayBuffer());
|
|
4026
|
-
const readMs = Date.now() - readStartedAt;
|
|
4027
|
-
if (bytes.byteLength !== expectedBytes) {
|
|
4028
|
-
return Response.json(
|
|
4029
|
-
{ error: 'staged file byte length mismatch' },
|
|
4030
|
-
{ status: 400 },
|
|
4031
|
-
);
|
|
4032
|
-
}
|
|
4033
|
-
const putStartedAt = Date.now();
|
|
4034
|
-
await env.PLAYS_BUCKET.put(key, bytes, {
|
|
4035
|
-
httpMetadata: { contentType },
|
|
4036
|
-
});
|
|
4037
|
-
const putMs = Date.now() - putStartedAt;
|
|
4038
|
-
recordCoordinatorEvent(
|
|
4039
|
-
'info',
|
|
4040
|
-
'coordinator.staged_file_put',
|
|
4041
|
-
'[perf][coordinator.staged_file_put]',
|
|
4042
|
-
{
|
|
4043
|
-
key,
|
|
4044
|
-
bytes: bytes.byteLength,
|
|
4045
|
-
readMs,
|
|
4046
|
-
putMs,
|
|
4047
|
-
ms: readMs + putMs,
|
|
4048
|
-
transport: 'raw',
|
|
4049
|
-
},
|
|
4050
|
-
);
|
|
4051
|
-
return Response.json({
|
|
4052
|
-
ok: true,
|
|
4053
|
-
key,
|
|
4054
|
-
bytes: bytes.byteLength,
|
|
4055
|
-
timingsMs: { read: readMs, put: putMs },
|
|
4056
|
-
});
|
|
4057
|
-
}
|
|
4058
|
-
const body = (await request.json().catch(() => null)) as {
|
|
4059
|
-
key?: unknown;
|
|
4060
|
-
contentBase64?: unknown;
|
|
4061
|
-
contentType?: unknown;
|
|
4062
|
-
bytes?: unknown;
|
|
4063
|
-
} | null;
|
|
4064
|
-
const key = typeof body?.key === 'string' ? body.key.trim() : '';
|
|
4065
|
-
const contentBase64 =
|
|
4066
|
-
typeof body?.contentBase64 === 'string' ? body.contentBase64 : '';
|
|
4067
|
-
const contentType =
|
|
4068
|
-
typeof body?.contentType === 'string' && body.contentType.trim()
|
|
4069
|
-
? body.contentType.trim()
|
|
4070
|
-
: 'application/octet-stream';
|
|
4071
|
-
const expectedBytes = typeof body?.bytes === 'number' ? body.bytes : NaN;
|
|
4072
|
-
if (!isAllowedStagedFileStorageKey(key)) {
|
|
4073
|
-
return Response.json({ error: 'invalid staged file key' }, { status: 400 });
|
|
4074
|
-
}
|
|
4075
|
-
if (
|
|
4076
|
-
!contentBase64 ||
|
|
4077
|
-
!Number.isSafeInteger(expectedBytes) ||
|
|
4078
|
-
expectedBytes < 0
|
|
4079
|
-
) {
|
|
4080
|
-
return Response.json(
|
|
4081
|
-
{ error: 'contentBase64 and bytes are required' },
|
|
4082
|
-
{ status: 400 },
|
|
4083
|
-
);
|
|
4084
|
-
}
|
|
4085
|
-
const existing = await headExistingStagedFile(env, key, expectedBytes);
|
|
4086
|
-
if (existing.exists) {
|
|
4087
|
-
recordCoordinatorEvent(
|
|
4088
|
-
'info',
|
|
4089
|
-
'coordinator.staged_file_put',
|
|
4090
|
-
'[perf][coordinator.staged_file_put]',
|
|
4091
|
-
{
|
|
4092
|
-
key,
|
|
4093
|
-
bytes: expectedBytes,
|
|
4094
|
-
headMs: existing.ms,
|
|
4095
|
-
putMs: 0,
|
|
4096
|
-
ms: existing.ms,
|
|
4097
|
-
transport: 'base64',
|
|
4098
|
-
skipped: true,
|
|
4099
|
-
},
|
|
4100
|
-
);
|
|
4101
|
-
return Response.json({
|
|
4102
|
-
ok: true,
|
|
4103
|
-
key,
|
|
4104
|
-
bytes: expectedBytes,
|
|
4105
|
-
existed: true,
|
|
4106
|
-
timingsMs: { head: existing.ms, put: 0 },
|
|
4107
|
-
});
|
|
4108
|
-
}
|
|
4109
|
-
const decodeStartedAt = Date.now();
|
|
4110
|
-
const bytes = decodeBase64ToUint8Array(contentBase64);
|
|
4111
|
-
const decodeMs = Date.now() - decodeStartedAt;
|
|
4112
|
-
if (bytes.byteLength !== expectedBytes) {
|
|
4113
|
-
return Response.json(
|
|
4114
|
-
{ error: 'staged file byte length mismatch' },
|
|
4115
|
-
{ status: 400 },
|
|
4116
|
-
);
|
|
4117
|
-
}
|
|
4118
|
-
const putStartedAt = Date.now();
|
|
4119
|
-
await env.PLAYS_BUCKET.put(key, bytes, {
|
|
4120
|
-
httpMetadata: { contentType },
|
|
4121
|
-
});
|
|
4122
|
-
const putMs = Date.now() - putStartedAt;
|
|
4123
|
-
recordCoordinatorEvent(
|
|
4124
|
-
'info',
|
|
4125
|
-
'coordinator.staged_file_put',
|
|
4126
|
-
'[perf][coordinator.staged_file_put]',
|
|
4127
|
-
{
|
|
4128
|
-
key,
|
|
4129
|
-
bytes: bytes.byteLength,
|
|
4130
|
-
decodeMs,
|
|
4131
|
-
putMs,
|
|
4132
|
-
ms: decodeMs + putMs,
|
|
4133
|
-
},
|
|
4134
|
-
);
|
|
4135
|
-
return Response.json({
|
|
4136
|
-
ok: true,
|
|
4137
|
-
key,
|
|
4138
|
-
bytes: bytes.byteLength,
|
|
4139
|
-
timingsMs: { decode: decodeMs, put: putMs },
|
|
4140
|
-
});
|
|
4141
|
-
}
|
|
4142
|
-
|
|
4143
|
-
async function headExistingStagedFile(
|
|
4144
|
-
env: CoordinatorEnv,
|
|
4145
|
-
key: string,
|
|
4146
|
-
expectedBytes: number,
|
|
4147
|
-
): Promise<{ exists: boolean; ms: number }> {
|
|
4148
|
-
const startedAt = Date.now();
|
|
4149
|
-
const object = await env.PLAYS_BUCKET.head(key).catch(() => null);
|
|
4150
|
-
const ms = Date.now() - startedAt;
|
|
4151
|
-
if (!object) {
|
|
4152
|
-
return { exists: false, ms };
|
|
4153
|
-
}
|
|
4154
|
-
if (typeof object.size === 'number' && object.size !== expectedBytes) {
|
|
4155
|
-
return { exists: false, ms };
|
|
4156
|
-
}
|
|
4157
|
-
return { exists: true, ms };
|
|
4158
|
-
}
|
|
4159
|
-
|
|
4160
|
-
async function handleCoordinatorWarmup(
|
|
4161
|
-
request: Request,
|
|
4162
|
-
env: CoordinatorEnv,
|
|
4163
|
-
ctx?: ExecutionContext,
|
|
4164
|
-
): Promise<Response> {
|
|
4165
|
-
if (request.method !== 'POST') {
|
|
4166
|
-
return new Response('method not allowed', { status: 405 });
|
|
4167
|
-
}
|
|
4168
|
-
const warmupToken = env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim();
|
|
4169
|
-
if (
|
|
4170
|
-
warmupToken &&
|
|
4171
|
-
request.headers.get('x-vercel-protection-bypass') !== warmupToken
|
|
4172
|
-
) {
|
|
4173
|
-
return new Response('unauthorized', { status: 401 });
|
|
4174
|
-
}
|
|
4175
|
-
const url = new URL(request.url);
|
|
4176
|
-
const label = url.searchParams.get('label')?.trim() || 'warmup';
|
|
4177
|
-
const runId = `coordinator/warmup/${label}/${Date.now().toString(36)}`;
|
|
4178
|
-
const artifactHash = `coordinator-warmup-${stableHash(COORDINATOR_WARMUP_DYNAMIC_WORKER_CODE)}`;
|
|
4179
|
-
const artifactStorageKey = `coordinator-warmup/${artifactHash}.json`;
|
|
4180
|
-
await env.PLAYS_BUCKET.put(
|
|
4181
|
-
artifactStorageKey,
|
|
4182
|
-
JSON.stringify({
|
|
4183
|
-
artifact: {
|
|
4184
|
-
artifactKind: 'esm_workers',
|
|
4185
|
-
bundledCode: COORDINATOR_WARMUP_DYNAMIC_WORKER_CODE,
|
|
4186
|
-
},
|
|
4187
|
-
} satisfies StoredPlayArtifactPayload),
|
|
4188
|
-
{
|
|
4189
|
-
httpMetadata: {
|
|
4190
|
-
contentType: 'application/json',
|
|
4191
|
-
},
|
|
4192
|
-
},
|
|
4193
|
-
);
|
|
4194
|
-
const params: PlayWorkflowParams = {
|
|
4195
|
-
runId,
|
|
4196
|
-
playId: runId,
|
|
4197
|
-
playName: 'coordinator-warmup',
|
|
4198
|
-
artifactStorageKey,
|
|
4199
|
-
artifactHash,
|
|
4200
|
-
graphHash: artifactHash,
|
|
4201
|
-
input: {},
|
|
4202
|
-
inputFile: null,
|
|
4203
|
-
inlineCsv: null,
|
|
4204
|
-
packagedFiles: null,
|
|
4205
|
-
contractSnapshot: null,
|
|
4206
|
-
executionPlan: null,
|
|
4207
|
-
dynamicWorkerCode: null,
|
|
4208
|
-
executorToken: 'coordinator-warmup',
|
|
4209
|
-
baseUrl: env.DEEPLINE_API_BASE_URL,
|
|
4210
|
-
orgId: 'coordinator-warmup',
|
|
4211
|
-
userEmail: 'coordinator-warmup@deepline.local',
|
|
4212
|
-
userId: 'coordinator-warmup',
|
|
4213
|
-
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
4214
|
-
dedupBackend: 'durable_object',
|
|
4215
|
-
};
|
|
4216
|
-
const startedAt = Date.now();
|
|
4217
|
-
const response = await handleWorkflowRoute({
|
|
4218
|
-
runId,
|
|
4219
|
-
action: 'submit',
|
|
4220
|
-
request: new Request(
|
|
4221
|
-
`https://deepline.coordinator.internal/workflow/${encodeURIComponent(runId)}/submit`,
|
|
4222
|
-
{
|
|
4223
|
-
method: 'POST',
|
|
4224
|
-
headers: {
|
|
4225
|
-
'content-type': 'application/json',
|
|
4226
|
-
[COORDINATOR_INTERNAL_TOKEN_HEADER]:
|
|
4227
|
-
env.DEEPLINE_INTERNAL_TOKEN?.trim() ?? '',
|
|
4228
|
-
},
|
|
4229
|
-
body: JSON.stringify(params),
|
|
4230
|
-
},
|
|
4231
|
-
),
|
|
4232
|
-
env,
|
|
4233
|
-
});
|
|
4234
|
-
const text = await response.text().catch(() => '');
|
|
4235
|
-
const terminalWaitStartedAt = Date.now();
|
|
4236
|
-
let terminalState: Record<string, unknown> | null = null;
|
|
4237
|
-
if (response.ok) {
|
|
4238
|
-
const instance = await env.PLAY_WORKFLOW.get(
|
|
4239
|
-
await resolveWorkflowInstanceIdForRun(env, runId),
|
|
4240
|
-
);
|
|
4241
|
-
try {
|
|
4242
|
-
while (Date.now() - terminalWaitStartedAt < 10_000) {
|
|
4243
|
-
const status = await instance.status();
|
|
4244
|
-
terminalState = mapWorkflowResult(runId, status);
|
|
4245
|
-
const mappedStatus = String(terminalState.status ?? '');
|
|
4246
|
-
if (
|
|
4247
|
-
mappedStatus === 'completed' ||
|
|
4248
|
-
mappedStatus === 'failed' ||
|
|
4249
|
-
mappedStatus === 'cancelled'
|
|
4250
|
-
) {
|
|
4251
|
-
break;
|
|
4252
|
-
}
|
|
4253
|
-
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
4254
|
-
}
|
|
4255
|
-
} finally {
|
|
4256
|
-
disposeRpcStub(instance);
|
|
4257
|
-
}
|
|
4258
|
-
}
|
|
4259
|
-
recordCoordinatorPerfTrace({
|
|
4260
|
-
runId,
|
|
4261
|
-
phase: 'coordinator.warmup_terminal_wait',
|
|
4262
|
-
ms: Date.now() - terminalWaitStartedAt,
|
|
4263
|
-
graphHash: params.graphHash,
|
|
4264
|
-
extra: { status: terminalState?.status ?? null, label },
|
|
4265
|
-
});
|
|
4266
|
-
recordCoordinatorPerfTrace({
|
|
4267
|
-
runId,
|
|
4268
|
-
phase: 'coordinator.warmup_submit_total',
|
|
4269
|
-
ms: Date.now() - startedAt,
|
|
4270
|
-
graphHash: params.graphHash,
|
|
4271
|
-
extra: { status: response.status, label },
|
|
4272
|
-
});
|
|
4273
|
-
let body: unknown = null;
|
|
4274
|
-
try {
|
|
4275
|
-
body = text ? JSON.parse(text) : null;
|
|
4276
|
-
} catch {
|
|
4277
|
-
body = text;
|
|
4278
|
-
}
|
|
4279
|
-
const terminalStatus = String(terminalState?.status ?? '');
|
|
4280
|
-
const warmupCompleted = response.ok && terminalStatus === 'completed';
|
|
4281
|
-
const responseStatus = warmupCompleted
|
|
4282
|
-
? 200
|
|
4283
|
-
: terminalStatus === 'running' || terminalStatus === 'sleeping'
|
|
4284
|
-
? 504
|
|
4285
|
-
: response.ok
|
|
4286
|
-
? 500
|
|
4287
|
-
: response.status;
|
|
4288
|
-
return Response.json(
|
|
4289
|
-
{
|
|
4290
|
-
ok: warmupCompleted,
|
|
4291
|
-
runId,
|
|
4292
|
-
status: response.status,
|
|
4293
|
-
body,
|
|
4294
|
-
terminalState,
|
|
4295
|
-
},
|
|
4296
|
-
{ status: responseStatus },
|
|
4297
|
-
);
|
|
4298
|
-
}
|
|
4299
|
-
|
|
4300
|
-
/**
|
|
4301
|
-
* Returns a structured-cloneable runtime API binding for the per-graphHash
|
|
4302
|
-
* play Worker's `env.RUNTIME_API`. We intentionally pass the long-lived
|
|
4303
|
-
* HARNESS WorkerEntrypoint service binding instead of a plain closure: the
|
|
4304
|
-
* dynamic Worker env is serialized by Cloudflare Workflows, and service
|
|
4305
|
-
* bindings are cloneable while closures are not. The per-play runtime accepts
|
|
4306
|
-
* this binding via `runtimeApiCall(...)`, so callbacks still stay on
|
|
4307
|
-
* service bindings and never fall back to public HTTP.
|
|
4308
|
-
*/
|
|
4309
|
-
let loggedMissingCoordinatorControlExport = false;
|
|
4310
|
-
|
|
4311
|
-
function makeRuntimeApiEnvBinding(env: CoordinatorEnv): {
|
|
4312
|
-
RUNTIME_API: CoordinatorEnv['HARNESS'];
|
|
4313
|
-
} {
|
|
4314
|
-
return { RUNTIME_API: env.HARNESS };
|
|
4315
|
-
}
|
|
4316
|
-
|
|
4317
|
-
function makeCoordinatorControlBinding():
|
|
4318
|
-
| {
|
|
4319
|
-
COORDINATOR: {
|
|
4320
|
-
signal(
|
|
4321
|
-
runId: string,
|
|
4322
|
-
body: Record<string, unknown>,
|
|
4323
|
-
): Promise<Record<string, unknown>>;
|
|
4324
|
-
recordPerfTrace(
|
|
4325
|
-
runId: string,
|
|
4326
|
-
payload: CoordinatorPerfTracePayload,
|
|
4327
|
-
): Promise<void>;
|
|
4328
|
-
recordRunEvent(
|
|
4329
|
-
runId: string,
|
|
4330
|
-
event: CoordinatorRunEvent,
|
|
4331
|
-
): Promise<void>;
|
|
4332
|
-
readTerminalState(
|
|
4333
|
-
runId: string,
|
|
4334
|
-
): Promise<CoordinatorTerminalState | null>;
|
|
4335
|
-
rateAcquire(input: {
|
|
4336
|
-
bucketId: string;
|
|
4337
|
-
rules: Array<{
|
|
4338
|
-
ruleId: string;
|
|
4339
|
-
requestsPerWindow: number;
|
|
4340
|
-
windowMs: number;
|
|
4341
|
-
maxConcurrency: number | null;
|
|
4342
|
-
}>;
|
|
4343
|
-
requested: number;
|
|
4344
|
-
}): Promise<{ granted: number; waitMs: number }>;
|
|
4345
|
-
ratePenalize(input: {
|
|
4346
|
-
bucketId: string;
|
|
4347
|
-
cooldownMs: number;
|
|
4348
|
-
}): Promise<void>;
|
|
4349
|
-
};
|
|
4350
|
-
}
|
|
4351
|
-
| Record<string, never> {
|
|
4352
|
-
const exports = workersExports as unknown as {
|
|
4353
|
-
CoordinatorControl?: (init: { props: undefined }) => {
|
|
4354
|
-
signal(
|
|
4355
|
-
runId: string,
|
|
4356
|
-
body: Record<string, unknown>,
|
|
4357
|
-
): Promise<Record<string, unknown>>;
|
|
4358
|
-
recordPerfTrace(
|
|
4359
|
-
runId: string,
|
|
4360
|
-
payload: CoordinatorPerfTracePayload,
|
|
4361
|
-
): Promise<void>;
|
|
4362
|
-
recordRunEvent(runId: string, event: CoordinatorRunEvent): Promise<void>;
|
|
4363
|
-
readTerminalState(
|
|
4364
|
-
runId: string,
|
|
4365
|
-
): Promise<CoordinatorTerminalState | null>;
|
|
4366
|
-
rateAcquire(input: {
|
|
4367
|
-
bucketId: string;
|
|
4368
|
-
rules: Array<{
|
|
4369
|
-
ruleId: string;
|
|
4370
|
-
requestsPerWindow: number;
|
|
4371
|
-
windowMs: number;
|
|
4372
|
-
maxConcurrency: number | null;
|
|
4373
|
-
}>;
|
|
4374
|
-
requested: number;
|
|
4375
|
-
}): Promise<{ granted: number; waitMs: number }>;
|
|
4376
|
-
ratePenalize(input: {
|
|
4377
|
-
bucketId: string;
|
|
4378
|
-
cooldownMs: number;
|
|
4379
|
-
}): Promise<void>;
|
|
4380
|
-
};
|
|
4381
|
-
};
|
|
4382
|
-
const ctor = exports.CoordinatorControl;
|
|
4383
|
-
if (typeof ctor !== 'function') {
|
|
4384
|
-
if (!loggedMissingCoordinatorControlExport) {
|
|
4385
|
-
loggedMissingCoordinatorControlExport = true;
|
|
4386
|
-
recordCoordinatorEvent(
|
|
4387
|
-
'warn',
|
|
4388
|
-
'coordinator.control_export.missing',
|
|
4389
|
-
'[coordinator] CoordinatorControl is not registered on cloudflare:workers exports; using public coordinator transport.',
|
|
4390
|
-
);
|
|
4391
|
-
}
|
|
4392
|
-
return {};
|
|
4393
|
-
}
|
|
4394
|
-
return { COORDINATOR: ctor({ props: undefined }) };
|
|
4395
|
-
}
|
|
4396
|
-
|
|
4397
|
-
async function loadStoredPlayArtifactFromR2(
|
|
4398
|
-
env: CoordinatorEnv,
|
|
4399
|
-
storageKey: string,
|
|
4400
|
-
): Promise<StoredPlayArtifactPayload> {
|
|
4401
|
-
const object = await env.PLAYS_BUCKET.get(storageKey);
|
|
4402
|
-
if (!object) {
|
|
4403
|
-
throw new Error(`Stored play artifact missing from R2: ${storageKey}`);
|
|
4404
|
-
}
|
|
4405
|
-
return JSON.parse(await object.text()) as StoredPlayArtifactPayload;
|
|
4406
|
-
}
|
|
4407
|
-
|
|
4408
|
-
function readMetadataString(
|
|
4409
|
-
metadata: Record<string, unknown>,
|
|
4410
|
-
key: keyof DynamicWorkflowMetadata,
|
|
4411
|
-
): string {
|
|
4412
|
-
const value = metadata[key];
|
|
4413
|
-
if (typeof value !== 'string' || !value.trim()) {
|
|
4414
|
-
throw new Error(`Dynamic workflow metadata missing ${key}.`);
|
|
4415
|
-
}
|
|
4416
|
-
return value;
|
|
4417
|
-
}
|
|
4418
|
-
|
|
4419
|
-
function mapWorkflowResult(
|
|
4420
|
-
runId: string,
|
|
4421
|
-
status: InstanceStatus,
|
|
4422
|
-
): Record<string, unknown> {
|
|
4423
|
-
const rawError = readWorkflowError(status);
|
|
4424
|
-
const mapped = resolveTerminalStatus(status, rawError);
|
|
4425
|
-
const error =
|
|
4426
|
-
rawError ??
|
|
4427
|
-
(mapped === 'failed'
|
|
4428
|
-
? `Cloudflare workflow reported ${String(status.status ?? 'failed')} without an error payload.`
|
|
4429
|
-
: null);
|
|
4430
|
-
const output =
|
|
4431
|
-
status.output && typeof status.output === 'object'
|
|
4432
|
-
? (status.output as Record<string, unknown>)
|
|
4433
|
-
: null;
|
|
4434
|
-
return {
|
|
4435
|
-
runId,
|
|
4436
|
-
...(typeof output?.playName === 'string' && output.playName.trim()
|
|
4437
|
-
? { playName: output.playName.trim() }
|
|
4438
|
-
: {}),
|
|
4439
|
-
status: mapped,
|
|
4440
|
-
result: output?.result ?? status.output ?? null,
|
|
4441
|
-
error,
|
|
4442
|
-
totalRows: output?.totalRows ?? output?.outputRows ?? null,
|
|
4443
|
-
durationMs: output?.durationMs ?? null,
|
|
4444
|
-
liveLogs: sanitizeLiveLogLines(output?.liveLogs),
|
|
4445
|
-
liveNodeProgress: output?.liveNodeProgress ?? null,
|
|
4446
|
-
wait:
|
|
4447
|
-
mapped === 'sleeping'
|
|
4448
|
-
? {
|
|
4449
|
-
kind: 'integration_event_batch',
|
|
4450
|
-
}
|
|
4451
|
-
: null,
|
|
4452
|
-
};
|
|
4453
|
-
}
|
|
4454
|
-
|
|
4455
|
-
function mapWorkflowStatus(
|
|
4456
|
-
status: InstanceStatus,
|
|
4457
|
-
): 'running' | 'sleeping' | 'completed' | 'failed' | 'cancelled' {
|
|
4458
|
-
if (status.status === 'complete') return 'completed';
|
|
4459
|
-
if (status.status === 'errored') return 'failed';
|
|
4460
|
-
if (status.status === 'terminated') return 'cancelled';
|
|
4461
|
-
if (status.status === 'waiting' || status.status === 'paused')
|
|
4462
|
-
return 'sleeping';
|
|
4463
|
-
return 'running';
|
|
4464
|
-
}
|
|
4465
|
-
|
|
4466
|
-
/**
|
|
4467
|
-
* CF Dynamic Workflows retries failed workflow bodies silently for ~3
|
|
4468
|
-
* minutes before transitioning the instance to `errored`. During that
|
|
4469
|
-
* retry window `status.status` stays `running`/`paused`/`waiting`, but
|
|
4470
|
-
* **`status.error`** gets populated as soon as the first attempt fails —
|
|
4471
|
-
* with CF's generic `"internal error; reference = <id>"` (they never
|
|
4472
|
-
* surface the actual JS error message via this API). That's enough
|
|
4473
|
-
* signal: any error presence on a non-terminal instance means at least
|
|
4474
|
-
* one attempt has thrown, and the user is better served by failing fast
|
|
4475
|
-
* than by waiting 3 minutes for CF's retry budget to exhaust.
|
|
4476
|
-
*
|
|
4477
|
-
* If the failure was actually transient and the next retry would have
|
|
4478
|
-
* succeeded, the user can re-run. The previous "match on permanent-
|
|
4479
|
-
* error patterns" approach didn't help because the error message we
|
|
4480
|
-
* see from CF is never a JS error name — it's always the generic
|
|
4481
|
-
* reference id.
|
|
4482
|
-
*/
|
|
4483
|
-
function resolveTerminalStatus(
|
|
4484
|
-
status: InstanceStatus,
|
|
4485
|
-
error: string | null,
|
|
4486
|
-
): 'running' | 'sleeping' | 'completed' | 'failed' | 'cancelled' {
|
|
4487
|
-
const mapped = mapWorkflowStatus(status);
|
|
4488
|
-
if ((mapped === 'running' || mapped === 'sleeping') && error) {
|
|
4489
|
-
return 'failed';
|
|
4490
|
-
}
|
|
4491
|
-
return mapped;
|
|
4492
|
-
}
|
|
4493
|
-
|
|
4494
|
-
function readWorkflowError(status: InstanceStatus): string | null {
|
|
4495
|
-
const error = status.error as unknown;
|
|
4496
|
-
if (!error) return null;
|
|
4497
|
-
if (typeof error === 'string') return error.trim() || null;
|
|
4498
|
-
if (typeof error === 'object' && 'message' in error) {
|
|
4499
|
-
const message = String((error as { message?: unknown }).message ?? '');
|
|
4500
|
-
return message.trim() || null;
|
|
4501
|
-
}
|
|
4502
|
-
const message = String(error);
|
|
4503
|
-
return message.trim() || null;
|
|
4504
|
-
}
|