deepline 0.1.228 → 0.1.229

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +76 -1842
  2. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +0 -113
  3. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +81 -968
  4. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +1 -50
  5. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  6. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +67 -15
  7. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +84 -79
  8. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +152 -688
  9. package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +3 -10
  10. package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -2
  11. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -47
  12. package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts +0 -2
  13. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +12 -200
  14. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +2 -17
  15. package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +6 -1
  16. package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +28 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/run-execution-scope.ts +16 -64
  18. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +0 -4
  19. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +37 -9
  20. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +0 -36
  21. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +3 -21
  22. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +9 -1
  23. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -0
  24. package/dist/cli/index.js +2 -2
  25. package/dist/cli/index.mjs +2 -2
  26. package/dist/{compiler-manifest-BhgZ23A4.d.ts → compiler-manifest-CYcwzSOJ.d.mts} +2 -0
  27. package/dist/{compiler-manifest-BhgZ23A4.d.mts → compiler-manifest-CYcwzSOJ.d.ts} +2 -0
  28. package/dist/index.d.mts +1 -1
  29. package/dist/index.d.ts +1 -1
  30. package/dist/index.js +2 -2
  31. package/dist/index.mjs +2 -2
  32. package/dist/plays/bundle-play-file.d.mts +2 -2
  33. package/dist/plays/bundle-play-file.d.ts +2 -2
  34. package/package.json +1 -1
  35. package/dist/bundling-sources/apps/play-runner-workers/src/child-manifest-resolver.ts +0 -108
  36. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +0 -374
  37. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +0 -203
  38. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +0 -14
  39. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +0 -121
@@ -29,11 +29,7 @@ import {
29
29
  type WorkflowRunner,
30
30
  } from '@cloudflare/dynamic-workflows';
31
31
  import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
32
- import { buildChildRunId as buildSharedChildRunId } from '../../../shared_libs/play-runtime/child-run-id';
33
- import type {
34
- PlayCallGovernanceSnapshot,
35
- PlayRunInputPayload,
36
- } from '../../../shared_libs/play-runtime/scheduler-backend';
32
+ import type { PlayRunInputPayload } from '../../../shared_libs/play-runtime/scheduler-backend';
37
33
  import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runtime/test-runtime-seams';
38
34
  import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
39
35
  import {
@@ -45,10 +41,7 @@ import {
45
41
  encryptDbSessionPostgresUrl,
46
42
  } from '../../../shared_libs/play-runtime/db-session-crypto';
47
43
  import { planRuntimeDbSessionRequirements } from '../../../shared_libs/play-runtime/db-session-plan';
48
- import type {
49
- PlayRuntimeManifest,
50
- PlayRuntimeManifestMap,
51
- } from '../../../shared_libs/plays/compiler-manifest';
44
+ import type { PlayRuntimeManifest } from '../../../shared_libs/plays/compiler-manifest';
52
45
  import {
53
46
  WORKSPACE_STORAGE_NOT_READY_CODE,
54
47
  WORKSPACE_STORAGE_NOT_READY_MESSAGE,
@@ -59,10 +52,6 @@ import {
59
52
  COORDINATOR_INTERNAL_TOKEN_HEADER,
60
53
  COORDINATOR_RUN_SCOPE_HEADER,
61
54
  } from '../../../shared_libs/play-runtime/coordinator-headers';
62
- import {
63
- PLAY_RUNTIME_CONTRACT,
64
- PLAY_RUNTIME_CONTRACT_HEADER,
65
- } from '../../../shared_libs/play-runtime/runtime-contract';
66
55
  import {
67
56
  PLAY_RUNTIME_API_COMPAT_PATH,
68
57
  PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH,
@@ -173,8 +162,6 @@ export type PlayWorkflowParams = {
173
162
  }> | null;
174
163
  contractSnapshot?: unknown;
175
164
  executionPlan?: ExecutionPlan | null;
176
- childPlayManifests?: PlayRuntimeManifestMap | null;
177
- playCallGovernance?: PlayCallGovernanceSnapshot | null;
178
165
  forceToolRefresh?: boolean | null;
179
166
  preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
180
167
  preloadedDbSessionRef?: {
@@ -185,8 +172,6 @@ export type PlayWorkflowParams = {
185
172
  dynamicWorkerCode?: string | null;
186
173
  executorToken: string;
187
174
  baseUrl: string;
188
- runtimeApiBaseUrl?: string | null;
189
- callbackUrl?: string | null;
190
175
  integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
191
176
  orgId: string;
192
177
  userEmail: string;
@@ -201,27 +186,6 @@ export type PlayWorkflowParams = {
201
186
  submittedAt?: number | null;
202
187
  };
203
188
 
204
- type InlineChildSubmitResult =
205
- | {
206
- workflowId: string;
207
- runId: string;
208
- status: 'completed';
209
- mode: 'inline_dynamic_worker';
210
- output: unknown;
211
- result: unknown;
212
- logs: string[];
213
- timings: CoordinatorTiming[];
214
- }
215
- | {
216
- workflowId: string;
217
- runId: string;
218
- status: 'failed';
219
- mode: 'inline_dynamic_worker';
220
- error: { message: string; stack?: string | null };
221
- logs: string[];
222
- timings: CoordinatorTiming[];
223
- };
224
-
225
189
  type CoordinatorTiming = {
226
190
  phase: string;
227
191
  ms: number;
@@ -262,12 +226,6 @@ type CoordinatorTerminalState = {
262
226
  completedAt?: number;
263
227
  };
264
228
 
265
- type CoordinatorChildTerminalState = {
266
- eventKey: string;
267
- data: unknown;
268
- storedAt: number;
269
- };
270
-
271
229
  type CoordinatorRunEvent =
272
230
  | {
273
231
  seq?: number;
@@ -648,67 +606,6 @@ async function readCoordinatorTerminalState(
648
606
  return state as CoordinatorTerminalState;
649
607
  }
650
608
 
651
- async function writeCoordinatorChildTerminalState(input: {
652
- env: CoordinatorEnv;
653
- parentRunId: string;
654
- eventKey: string;
655
- data: unknown;
656
- }): Promise<void> {
657
- const stub = input.env.PLAY_DEDUP.get(
658
- input.env.PLAY_DEDUP.idFromName(input.parentRunId),
659
- );
660
- const response = await stub.fetch(
661
- 'https://deepline.dedup.internal/child-terminal-set',
662
- {
663
- method: 'POST',
664
- headers: { 'content-type': 'application/json' },
665
- body: JSON.stringify({
666
- eventKey: input.eventKey,
667
- data: coordinatorStorageRedactor.redact(input.data),
668
- storedAt: Date.now(),
669
- }),
670
- },
671
- );
672
- if (!response.ok) {
673
- throw new Error(`coordinator child terminal set failed ${response.status}`);
674
- }
675
- }
676
-
677
- async function readCoordinatorChildTerminalState(input: {
678
- env: CoordinatorEnv;
679
- parentRunId: string;
680
- eventKey: string;
681
- timeoutMs?: number;
682
- }): Promise<CoordinatorChildTerminalState | null> {
683
- const stub = input.env.PLAY_DEDUP.get(
684
- input.env.PLAY_DEDUP.idFromName(input.parentRunId),
685
- );
686
- const endpoint =
687
- input.timeoutMs && input.timeoutMs > 0
688
- ? 'child-terminal-await'
689
- : 'child-terminal-get';
690
- const timeoutParam =
691
- input.timeoutMs && input.timeoutMs > 0
692
- ? `&timeoutMs=${encodeURIComponent(String(Math.floor(input.timeoutMs)))}`
693
- : '';
694
- const response = await stub.fetch(
695
- `https://deepline.dedup.internal/${endpoint}?eventKey=${encodeURIComponent(
696
- input.eventKey,
697
- )}${timeoutParam}`,
698
- );
699
- if (!response.ok) {
700
- throw new Error(
701
- `coordinator child terminal ${endpoint} failed ${response.status}`,
702
- );
703
- }
704
- const body = (await response.json().catch(() => ({}))) as {
705
- state?: unknown;
706
- };
707
- const state = body.state;
708
- if (!isRecord(state) || state.eventKey !== input.eventKey) return null;
709
- return state as CoordinatorChildTerminalState;
710
- }
711
-
712
609
  function workflowEventType(name: string): string {
713
610
  const normalized = name
714
611
  .trim()
@@ -1484,47 +1381,6 @@ async function markWorkflowRuntimeFailure(input: {
1484
1381
  );
1485
1382
  }
1486
1383
 
1487
- async function appendWorkflowChildCompletedRunEvent(input: {
1488
- env: CoordinatorEnv;
1489
- event: unknown;
1490
- result: unknown;
1491
- }): Promise<boolean> {
1492
- const payload = readWorkflowPayload(input.event);
1493
- if (!payload || !isRecord(payload.playCallGovernance)) return false;
1494
- const runId = typeof payload.runId === 'string' ? payload.runId : null;
1495
- const baseUrl = typeof payload.baseUrl === 'string' ? payload.baseUrl : null;
1496
- const executorToken =
1497
- typeof payload.executorToken === 'string' ? payload.executorToken : null;
1498
- if (!runId || !baseUrl || !executorToken) return false;
1499
- const headers = new Headers({
1500
- authorization: `Bearer ${executorToken}`,
1501
- 'content-type': 'application/json',
1502
- });
1503
- const bypass = input.env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim();
1504
- if (bypass) headers.set('x-vercel-protection-bypass', bypass);
1505
- const body = JSON.stringify({
1506
- action: 'append_run_events',
1507
- playId: runId,
1508
- events: [
1509
- {
1510
- type: 'run.completed',
1511
- runId,
1512
- source: 'worker',
1513
- occurredAt: Date.now(),
1514
- result: input.result,
1515
- } satisfies PlayRunLedgerEvent,
1516
- ],
1517
- });
1518
- const url = `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`;
1519
- const response = await fetch(url, { method: 'POST', headers, body });
1520
- if (!response.ok) {
1521
- throw new Error(
1522
- `runtime API responded ${response.status}: ${(await response.text().catch(() => '')).slice(0, 400)}`,
1523
- );
1524
- }
1525
- return true;
1526
- }
1527
-
1528
1384
  type StoredPlayArtifactPayload = {
1529
1385
  artifact?: {
1530
1386
  bundledCode?: string;
@@ -1835,1460 +1691,95 @@ function workflowRetryInstanceId(runId: string, attempt: number): string {
1835
1691
 
1836
1692
  function normalizeWorkflowRunAttempt(value: unknown): number {
1837
1693
  return typeof value === 'number' && Number.isFinite(value)
1838
- ? Math.max(0, Math.floor(value))
1839
- : 0;
1840
- }
1841
-
1842
- async function restartWorkflowAfterPlatformReset(input: {
1843
- env: CoordinatorEnv;
1844
- ctx?: ExecutionContext;
1845
- runId: string;
1846
- oldInstance: WorkflowInstance;
1847
- error: string;
1848
- status: InstanceStatus;
1849
- }): Promise<{
1850
- retried: boolean;
1851
- result: Record<string, unknown> | null;
1852
- }> {
1853
- const decision = decideWorkflowPlatformRetry({
1854
- workflowStatus: String(input.status.status ?? ''),
1855
- error: input.error,
1856
- retryAttempts: 0,
1857
- });
1858
- if (decision.action !== 'retry') {
1859
- return { retried: false, result: null };
1860
- }
1861
- const claim = await claimWorkflowPlatformRetry({
1862
- env: input.env,
1863
- runId: input.runId,
1864
- }).catch((error) => {
1865
- recordCoordinatorEvent(
1866
- 'warn',
1867
- 'coordinator.workflow_platform_retry.claim_failed',
1868
- '[coordinator] workflow platform retry claim failed',
1869
- {
1870
- runId: input.runId,
1871
- error: error instanceof Error ? error.message : String(error),
1872
- },
1873
- );
1874
- return null;
1875
- });
1876
- if (!claim?.claimed || !claim.params) {
1877
- return { retried: false, result: null };
1878
- }
1879
- const retryInstanceId = workflowRetryInstanceId(input.runId, claim.attempts);
1880
- const retryStartedAt = Date.now();
1881
- let retryInstance: WorkflowInstance | null = null;
1882
- const retryParams = { ...claim.params, runAttempt: claim.attempts };
1883
- try {
1884
- retryInstance = await createDynamicWorkflowInstance({
1885
- env: input.env,
1886
- id: retryInstanceId,
1887
- params: workflowCreateParams(retryParams, {
1888
- params: retryParams,
1889
- paramsRef: claim.paramsRef,
1890
- }),
1891
- metadataParams: retryParams,
1892
- });
1893
- await recordWorkflowInstanceId({
1894
- env: input.env,
1895
- runId: input.runId,
1896
- instanceId: retryInstance.id,
1897
- });
1898
- input.ctx?.waitUntil(input.oldInstance.terminate().catch(() => undefined));
1899
- recordCoordinatorPerfTraceBuffered(input.env, input.ctx, {
1900
- runId: input.runId,
1901
- phase: 'coordinator.platform_deploy_retry',
1902
- ms: Date.now() - retryStartedAt,
1903
- graphHash: claim.params.graphHash ?? null,
1904
- extra: {
1905
- retryAttempt: claim.attempts,
1906
- retryInstanceId: retryInstance.id,
1907
- reason: decision.reason,
1908
- },
1909
- });
1910
- return {
1911
- retried: true,
1912
- result: {
1913
- runId: input.runId,
1914
- playName: claim.params.playName,
1915
- status: 'running',
1916
- result: null,
1917
- error: null,
1918
- retry: {
1919
- reason: decision.reason,
1920
- attempt: claim.attempts,
1921
- message: decision.message,
1922
- },
1923
- },
1924
- };
1925
- } finally {
1926
- disposeRpcStub(retryInstance);
1927
- }
1928
- }
1929
-
1930
- async function markRegisteredChildRunFailed(input: {
1931
- env: CoordinatorEnv;
1932
- baseUrl: string;
1933
- childExecutorToken: string;
1934
- childRunId: string;
1935
- error: unknown;
1936
- }): Promise<void> {
1937
- const message =
1938
- input.error instanceof Error ? input.error.message : String(input.error);
1939
- const response = await input.env.HARNESS.runtimeApiCall({
1940
- contract: PLAY_RUNTIME_CONTRACT,
1941
- executorToken: input.childExecutorToken,
1942
- baseUrl: input.baseUrl,
1943
- path: PLAY_RUNTIME_API_COMPAT_PATH,
1944
- headers: {
1945
- 'x-deepline-request-id': crypto.randomUUID(),
1946
- [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
1947
- },
1948
- timeoutMs: 15_000,
1949
- body: {
1950
- action: 'append_run_events',
1951
- playId: input.childRunId,
1952
- events: [
1953
- {
1954
- type: 'run.failed',
1955
- runId: input.childRunId,
1956
- source: 'coordinator',
1957
- occurredAt: Date.now(),
1958
- error: `Child workflow submit failed: ${message}`,
1959
- } satisfies PlayRunLedgerEvent,
1960
- ],
1961
- },
1962
- });
1963
- if (response.status < 200 || response.status >= 300) {
1964
- const text = response.body ?? '';
1965
- throw new Error(
1966
- `Inline child run failure mark failed ${response.status}: ${text.slice(0, 800)}`,
1967
- );
1968
- }
1969
- }
1970
-
1971
- async function appendRegisteredChildRunTerminal(input: {
1972
- env: CoordinatorEnv;
1973
- baseUrl: string;
1974
- childExecutorToken: string;
1975
- childRunId: string;
1976
- status: 'completed' | 'failed';
1977
- result?: unknown;
1978
- error?: string | null;
1979
- }): Promise<void> {
1980
- const event =
1981
- input.status === 'completed'
1982
- ? ({
1983
- type: 'run.completed',
1984
- runId: input.childRunId,
1985
- source: 'worker',
1986
- occurredAt: Date.now(),
1987
- ...(input.result !== undefined ? { result: input.result } : {}),
1988
- } satisfies PlayRunLedgerEvent)
1989
- : ({
1990
- type: 'run.failed',
1991
- runId: input.childRunId,
1992
- source: 'worker',
1993
- occurredAt: Date.now(),
1994
- error: input.error ?? 'Inline child Worker failed.',
1995
- } satisfies PlayRunLedgerEvent);
1996
- const response = await input.env.HARNESS.runtimeApiCall({
1997
- contract: PLAY_RUNTIME_CONTRACT,
1998
- executorToken: input.childExecutorToken,
1999
- baseUrl: input.baseUrl,
2000
- path: PLAY_RUNTIME_API_COMPAT_PATH,
2001
- headers: {
2002
- 'x-deepline-request-id': crypto.randomUUID(),
2003
- [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
2004
- },
2005
- timeoutMs: 15_000,
2006
- body: {
2007
- action: 'append_run_events',
2008
- playId: input.childRunId,
2009
- events: [event],
2010
- },
2011
- });
2012
- if (response.status < 200 || response.status >= 300) {
2013
- const text = response.body ?? '';
2014
- throw new Error(
2015
- `Inline child terminal append failed ${response.status}: ${text.slice(0, 800)}`,
2016
- );
2017
- }
2018
- }
2019
-
2020
- type CoordinatorRuntimeApiTiming = {
2021
- phase: string;
2022
- ms: number;
2023
- bytes?: number;
2024
- };
2025
-
2026
- async function callRuntimeApiFromCoordinator(input: {
2027
- env: CoordinatorEnv;
2028
- baseUrl: string;
2029
- executorToken: string;
2030
- body: unknown;
2031
- }): Promise<{
2032
- status: number;
2033
- body: string;
2034
- timings: CoordinatorRuntimeApiTiming[];
2035
- }> {
2036
- const timings: CoordinatorRuntimeApiTiming[] = [];
2037
- const totalStartedAt = Date.now();
2038
- const recordTiming = (
2039
- phase: string,
2040
- startedAt: number,
2041
- extra?: { bytes?: number },
2042
- ): void => {
2043
- timings.push({
2044
- phase,
2045
- ms: Date.now() - startedAt,
2046
- ...(extra?.bytes !== undefined ? { bytes: extra.bytes } : {}),
2047
- });
2048
- };
2049
-
2050
- const buildStartedAt = Date.now();
2051
- const body = input.body ?? {};
2052
- const serializedBody = JSON.stringify(body);
2053
- recordTiming('coordinator.runtime_api.build_request', buildStartedAt, {
2054
- bytes: serializedBody.length,
2055
- });
2056
-
2057
- const fetchStartedAt = Date.now();
2058
- const requestId = crypto.randomUUID();
2059
- const action =
2060
- body && typeof body === 'object' && !Array.isArray(body)
2061
- ? String((body as { action?: unknown }).action ?? 'unknown')
2062
- : 'unknown';
2063
- const runId =
2064
- body && typeof body === 'object' && !Array.isArray(body)
2065
- ? String(
2066
- (body as { runId?: unknown; playId?: unknown }).runId ??
2067
- (body as { runId?: unknown; playId?: unknown }).playId ??
2068
- '',
2069
- ) || null
2070
- : null;
2071
- recordCoordinatorEvent(
2072
- 'info',
2073
- 'coordinator.runtime_api.breadcrumb',
2074
- '[coordinator.runtime_api.breadcrumb]',
2075
- {
2076
- phase: 'request',
2077
- action,
2078
- runId,
2079
- requestId,
2080
- baseOrigin: safeOrigin(input.baseUrl),
2081
- },
2082
- );
2083
- const response = await input.env.HARNESS.runtimeApiCall({
2084
- contract: PLAY_RUNTIME_CONTRACT,
2085
- executorToken: input.executorToken,
2086
- baseUrl: input.baseUrl,
2087
- path: PLAY_RUNTIME_API_COMPAT_PATH,
2088
- body,
2089
- headers: {
2090
- 'x-deepline-request-id': requestId,
2091
- [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
2092
- },
2093
- });
2094
- recordTiming('coordinator.runtime_api.fetch', fetchStartedAt);
2095
- recordCoordinatorEvent(
2096
- 'info',
2097
- 'coordinator.runtime_api.breadcrumb',
2098
- '[coordinator.runtime_api.breadcrumb]',
2099
- {
2100
- phase: 'response',
2101
- action,
2102
- runId,
2103
- requestId,
2104
- status: response.status,
2105
- ms: Date.now() - fetchStartedAt,
2106
- baseOrigin: safeOrigin(input.baseUrl),
2107
- },
2108
- );
2109
-
2110
- const bodyStartedAt = Date.now();
2111
- const responseBody = response.body;
2112
- recordTiming('coordinator.runtime_api.body', bodyStartedAt, {
2113
- bytes: responseBody.length,
2114
- });
2115
- recordTiming('coordinator.runtime_api.total', totalStartedAt);
2116
- return {
2117
- status: response.status,
2118
- body: responseBody,
2119
- timings,
2120
- };
2121
- }
2122
-
2123
- async function prepareInlineChildRunWithRuntime(input: {
2124
- env: CoordinatorEnv;
2125
- baseUrl: string;
2126
- parentExecutorToken: string;
2127
- parentRunId: string;
2128
- parentPlayName: string;
2129
- childRunId: string;
2130
- childPlayName: string;
2131
- manifest: PlayRuntimeManifest;
2132
- governance: PlayCallGovernanceSnapshot;
2133
- userEmail: string;
2134
- runtimeBackend?: string;
2135
- schedulerBackend?: string;
2136
- executionProfile?: string;
2137
- }): Promise<{
2138
- childToken: string;
2139
- preloadedDbSessions: PreloadedRuntimeDbSession[];
2140
- prepareTimings: unknown[];
2141
- transportTimings: unknown[];
2142
- }> {
2143
- const response = await callRuntimeApiFromCoordinator({
2144
- env: input.env,
2145
- baseUrl: input.baseUrl,
2146
- executorToken: input.parentExecutorToken,
2147
- body: {
2148
- action: 'prepare_inline_child_run',
2149
- parentRunId: input.parentRunId,
2150
- rootRunId: input.governance.rootRunId,
2151
- parentPlayName: input.parentPlayName,
2152
- childRunId: input.childRunId,
2153
- childPlayName: input.childPlayName,
2154
- workflowFamilyKey:
2155
- input.governance.rootRunId ??
2156
- input.governance.parentRunId ??
2157
- input.childRunId,
2158
- artifactStorageKey: input.manifest.artifactStorageKey,
2159
- artifactHash: input.manifest.artifactHash,
2160
- graphHash: input.manifest.graphHash,
2161
- runtimeBackend: input.runtimeBackend ?? 'workers_edge',
2162
- schedulerBackend: input.schedulerBackend ?? 'inline_child',
2163
- executionProfile: input.executionProfile ?? 'workers_edge',
2164
- ...(typeof input.manifest.maxCreditsPerRun === 'number'
2165
- ? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
2166
- : {}),
2167
- staticPipeline: input.manifest.staticPipeline ?? null,
2168
- source: 'published',
2169
- userEmail: input.userEmail,
2170
- },
2171
- });
2172
- const text = response.body;
2173
- let parsed: unknown = {};
2174
- try {
2175
- parsed = text ? JSON.parse(text) : {};
2176
- } catch {
2177
- parsed = {};
2178
- }
2179
- if (response.status < 200 || response.status >= 300) {
2180
- throw new Error(
2181
- `Inline child prepare failed ${response.status}: ${text.slice(0, 800)}`,
2182
- );
2183
- }
2184
- if (!isRecord(parsed) || typeof parsed.executorToken !== 'string') {
2185
- throw new Error('Inline child prepare response was missing executorToken.');
2186
- }
2187
- const preloadedDbSessions = Array.isArray(parsed.preloadedDbSessions)
2188
- ? (parsed.preloadedDbSessions as PreloadedRuntimeDbSession[])
2189
- : [];
2190
- const prepareTimings = Array.isArray(parsed.prepareTimings)
2191
- ? parsed.prepareTimings
2192
- : [];
2193
- return {
2194
- childToken: parsed.executorToken,
2195
- preloadedDbSessions,
2196
- prepareTimings,
2197
- transportTimings: response.timings,
2198
- };
2199
- }
2200
-
2201
- function buildChildRunId(input: {
2202
- playName: string;
2203
- parentRunId: string;
2204
- body: Record<string, unknown>;
2205
- }): string {
2206
- const { playName, parentRunId, body } = input;
2207
- const governance = isRecord(body.internalRunPlay)
2208
- ? body.internalRunPlay
2209
- : null;
2210
- // Adapt the coordinator's on-the-wire child-submit body to the shared
2211
- // derivation. Output stays byte-identical to the previous inline
2212
- // implementation (same fields → same digest → same run id), so child-submit
2213
- // idempotency is preserved. See shared_libs/play-runtime/child-run-id.ts.
2214
- return buildSharedChildRunId({
2215
- childPlayName: playName,
2216
- parentRunId,
2217
- explicitIdempotencyKey:
2218
- typeof body.childIdempotencyKey === 'string'
2219
- ? body.childIdempotencyKey
2220
- : null,
2221
- parentPlayName:
2222
- typeof governance?.parentPlayName === 'string'
2223
- ? governance.parentPlayName
2224
- : null,
2225
- key: typeof governance?.key === 'string' ? governance.key : null,
2226
- input: isRecord(body.input) || Array.isArray(body.input) ? body.input : {},
2227
- graphHash: isRecord(body.manifest)
2228
- ? typeof body.manifest.graphHash === 'string'
2229
- ? body.manifest.graphHash
2230
- : null
2231
- : null,
2232
- });
2233
- }
2234
-
2235
- function normalizeRuntimeBaseUrl(value: unknown): string | null {
2236
- if (typeof value !== 'string') return null;
2237
- const trimmed = value.trim();
2238
- if (!trimmed) return null;
2239
- let parsed: URL;
2240
- try {
2241
- parsed = new URL(trimmed);
2242
- } catch {
2243
- return null;
2244
- }
2245
- if (
2246
- (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') ||
2247
- parsed.username ||
2248
- parsed.password ||
2249
- parsed.search ||
2250
- parsed.hash ||
2251
- (parsed.pathname !== '/' && parsed.pathname !== '')
2252
- ) {
2253
- return null;
2254
- }
2255
- return parsed.toString().replace(/\/$/, '');
2256
- }
2257
-
2258
- const DEEPLINE_PRODUCTION_APP_HOST = 'code.deepline.com';
2259
- const DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE =
2260
- /^deepline-api(?:-[a-z0-9-]+)?-aero-team-0f650658\.vercel\.app$/i;
2261
- const DEEPLINE_VERCEL_PROJECT_HOST_RE =
2262
- /^deepline-[a-z0-9]+-aero-team-0f650658\.vercel\.app$/i;
2263
- const DEEPLINE_WORKERS_DEV_SUBDOMAINS = new Set(['chirag-d94']);
2264
-
2265
- function isDeeplineCoordinatorWorkerHost(hostname: string): boolean {
2266
- const labels = hostname.toLowerCase().split('.');
2267
- return (
2268
- labels.length === 4 &&
2269
- labels[0]?.startsWith('deepline-play-coordinator-') &&
2270
- labels[2] === 'workers' &&
2271
- labels[3] === 'dev' &&
2272
- DEEPLINE_WORKERS_DEV_SUBDOMAINS.has(labels[1] ?? '')
2273
- );
2274
- }
2275
-
2276
- function shouldCanonicalizeRuntimeApiBaseUrl(input: {
2277
- configuredBaseUrl: URL;
2278
- overrideBaseUrl: URL;
2279
- }): boolean {
2280
- if (input.overrideBaseUrl.host === input.configuredBaseUrl.host) {
2281
- return false;
2282
- }
2283
- const configuredHostname = input.configuredBaseUrl.hostname.toLowerCase();
2284
- const overrideHostname = input.overrideBaseUrl.hostname.toLowerCase();
2285
- if (isDeeplineCoordinatorWorkerHost(overrideHostname)) {
2286
- return true;
2287
- }
2288
- const overrideIsDeeplineVercelHost =
2289
- DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE.test(overrideHostname) ||
2290
- DEEPLINE_VERCEL_PROJECT_HOST_RE.test(overrideHostname);
2291
- if (!overrideIsDeeplineVercelHost) {
2292
- return false;
2293
- }
2294
- const configuredIsDeeplineVercelHost =
2295
- DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE.test(configuredHostname) ||
2296
- DEEPLINE_VERCEL_PROJECT_HOST_RE.test(configuredHostname);
2297
- return (
2298
- configuredHostname === DEEPLINE_PRODUCTION_APP_HOST ||
2299
- !configuredIsDeeplineVercelHost
2300
- );
2301
- }
2302
-
2303
- function resolveRuntimeApiBaseUrl(
2304
- env: CoordinatorEnv,
2305
- body: Record<string, unknown>,
2306
- ): string {
2307
- const configuredBaseUrl =
2308
- normalizeRuntimeBaseUrl(env.DEEPLINE_API_BASE_URL) ??
2309
- env.DEEPLINE_API_BASE_URL.replace(/\/$/, '');
2310
- const overrideBaseUrl =
2311
- normalizeRuntimeBaseUrl(body.runtimeApiBaseUrl) ??
2312
- normalizeRuntimeBaseUrl(body.baseUrl);
2313
- if (!overrideBaseUrl) {
2314
- return configuredBaseUrl;
2315
- }
2316
- if (
2317
- shouldCanonicalizeRuntimeApiBaseUrl({
2318
- configuredBaseUrl: new URL(configuredBaseUrl),
2319
- overrideBaseUrl: new URL(overrideBaseUrl),
2320
- })
2321
- ) {
2322
- return configuredBaseUrl;
2323
- }
2324
- return overrideBaseUrl;
2325
- }
2326
-
2327
- function resolveWorkflowCallbackBaseUrl(
2328
- env: CoordinatorEnv,
2329
- body: Record<string, unknown>,
2330
- ): string {
2331
- return (
2332
- normalizeRuntimeBaseUrl(body.callbackBaseUrl) ??
2333
- normalizeRuntimeBaseUrl(body.callbackUrl) ??
2334
- normalizeRuntimeBaseUrl(body.baseUrl) ??
2335
- env.DEEPLINE_API_BASE_URL.replace(/\/$/, '')
2336
- );
2337
- }
2338
-
2339
- function validateChildSubmitBody(input: {
2340
- parentRunId: string;
2341
- body: Record<string, unknown>;
2342
- }):
2343
- | {
2344
- ok: true;
2345
- manifest: PlayRuntimeManifest;
2346
- governance: PlayCallGovernanceSnapshot;
2347
- childPlayName: string;
2348
- orgId: string;
2349
- parentExecutorToken: string;
2350
- }
2351
- | { ok: false; status: number; error: Record<string, unknown> } {
2352
- const { parentRunId, body } = input;
2353
- const manifest = body.manifest as PlayRuntimeManifest | undefined;
2354
- const governance = body.internalRunPlay as
2355
- | PlayCallGovernanceSnapshot
2356
- | undefined;
2357
- const childPlayName =
2358
- typeof body.name === 'string' && body.name.trim()
2359
- ? body.name.trim()
2360
- : manifest?.playName?.trim();
2361
- if (
2362
- !manifest ||
2363
- !childPlayName ||
2364
- !manifest.artifactStorageKey ||
2365
- !manifest.artifactHash ||
2366
- !manifest.graphHash ||
2367
- !governance
2368
- ) {
2369
- return {
2370
- ok: false,
2371
- status: 400,
2372
- error: {
2373
- code: 'CHILD_MANIFEST_REQUIRED',
2374
- message: 'submit-child requires a trusted child manifest and lineage.',
2375
- phase: 'coordinator_child_submit',
2376
- parentRunId,
2377
- },
2378
- };
2379
- }
2380
- const orgId = typeof body.orgId === 'string' ? body.orgId : '';
2381
- if (!orgId) {
2382
- return {
2383
- ok: false,
2384
- status: 400,
2385
- error: {
2386
- code: 'CHILD_ORG_REQUIRED',
2387
- message: 'submit-child requires orgId from the parent runtime.',
2388
- phase: 'coordinator_child_submit',
2389
- parentRunId,
2390
- },
2391
- };
2392
- }
2393
- const parentExecutorToken =
2394
- typeof body.parentExecutorToken === 'string'
2395
- ? body.parentExecutorToken.trim()
2396
- : '';
2397
- if (!parentExecutorToken) {
2398
- return {
2399
- ok: false,
2400
- status: 400,
2401
- error: {
2402
- code: 'PARENT_EXECUTOR_TOKEN_REQUIRED',
2403
- message:
2404
- 'submit-child requires the parent executor token for origin-scoped child token minting.',
2405
- phase: 'coordinator_child_submit',
2406
- parentRunId,
2407
- },
2408
- };
2409
- }
2410
- return {
2411
- ok: true,
2412
- manifest,
2413
- governance,
2414
- childPlayName,
2415
- orgId,
2416
- parentExecutorToken,
2417
- };
2418
- }
2419
-
2420
- function buildChildWorkflowParams(input: {
2421
- env: CoordinatorEnv;
2422
- body: Record<string, unknown>;
2423
- manifest: PlayRuntimeManifest;
2424
- governance: PlayCallGovernanceSnapshot;
2425
- childRunId: string;
2426
- childPlayName: string;
2427
- childToken: string;
2428
- orgId: string;
2429
- coordinatorUrl: string | null;
2430
- runtimeBackend: string;
2431
- dynamicWorkerCode: string | null;
2432
- preloadedDbSessions: PreloadedRuntimeDbSession[] | null;
2433
- }): PlayWorkflowParams {
2434
- const {
2435
- env,
2436
- body,
2437
- manifest,
2438
- governance,
2439
- childRunId,
2440
- childPlayName,
2441
- childToken,
2442
- orgId,
2443
- coordinatorUrl,
2444
- runtimeBackend,
2445
- dynamicWorkerCode,
2446
- preloadedDbSessions,
2447
- } = input;
2448
- const baseUrl = resolveRuntimeApiBaseUrl(env, body);
2449
- const callbackUrl = resolveWorkflowCallbackBaseUrl(env, body);
2450
- return {
2451
- runId: childRunId,
2452
- playId: childRunId,
2453
- playName: childPlayName,
2454
- artifactStorageKey: manifest.artifactStorageKey,
2455
- artifactHash: manifest.artifactHash,
2456
- graphHash: manifest.graphHash,
2457
- input: isRecord(body.input) || Array.isArray(body.input) ? body.input : {},
2458
- contractSnapshot: {
2459
- source: 'published',
2460
- revisionVersion: null,
2461
- staticPipeline: manifest.staticPipeline,
2462
- billingLimit:
2463
- typeof manifest.maxCreditsPerRun === 'number'
2464
- ? { maxCreditsPerRun: manifest.maxCreditsPerRun }
2465
- : null,
2466
- sourceCode: manifest.sourceCode ?? '',
2467
- artifactMetadata: {
2468
- storageKey: manifest.artifactStorageKey,
2469
- artifactHash: manifest.artifactHash,
2470
- graphHash: manifest.graphHash,
2471
- },
2472
- codeFormat: 'cjs_module',
2473
- compatibility: {
2474
- apiVersion: 2,
2475
- runtimeBackend: 'workers_edge',
2476
- },
2477
- },
2478
- executionPlan: null,
2479
- childPlayManifests: isRecord(body.childPlayManifests)
2480
- ? (body.childPlayManifests as PlayRuntimeManifestMap)
2481
- : null,
2482
- playCallGovernance: governance,
2483
- preloadedDbSessions,
2484
- dynamicWorkerCode,
2485
- executorToken: childToken,
2486
- baseUrl: callbackUrl,
2487
- runtimeApiBaseUrl: baseUrl,
2488
- callbackUrl,
2489
- integrationMode: normalizeIntegrationMode(body.integrationMode),
2490
- force: body.force === true || body.forceToolRefresh === true,
2491
- forceToolRefresh: body.forceToolRefresh === true,
2492
- orgId,
2493
- userEmail: typeof body.userEmail === 'string' ? body.userEmail : '',
2494
- userId: typeof body.userId === 'string' ? body.userId : null,
2495
- runtimeBackend,
2496
- dedupBackend: 'in_memory',
2497
- coordinatorUrl,
2498
- coordinatorInternalToken: env.DEEPLINE_INTERNAL_TOKEN?.trim() || null,
2499
- };
2500
- }
2501
-
2502
- function runRequestFromPlayWorkflowParams(
2503
- params: PlayWorkflowParams,
2504
- ): Record<string, unknown> {
2505
- const inputFileName = String(
2506
- params.inputFile?.name ??
2507
- params.inputFile?.fileName ??
2508
- params.inputFile?.logicalPath ??
2509
- params.inputFile?.path ??
2510
- '',
2511
- );
2512
- const inputStorageKey =
2513
- params.inputFile?.r2Key ?? params.inputFile?.storageKey ?? null;
2514
- return {
2515
- runId: params.runId,
2516
- callbackUrl: params.callbackUrl ?? params.baseUrl,
2517
- executorToken: params.executorToken,
2518
- baseUrl: params.runtimeApiBaseUrl ?? params.baseUrl,
2519
- integrationMode: normalizeIntegrationMode(params.integrationMode),
2520
- orgId: params.orgId,
2521
- playName: params.playName,
2522
- graphHash: params.graphHash,
2523
- userEmail: params.userEmail,
2524
- runtimeInput: params.input,
2525
- runAttempt: normalizeWorkflowRunAttempt(params.runAttempt),
2526
- inlineCsv: params.inlineCsv ?? null,
2527
- inputFiles:
2528
- inputStorageKey && inputFileName
2529
- ? {
2530
- [inputFileName]: {
2531
- logicalPath:
2532
- params.inputFile?.logicalPath ??
2533
- params.inputFile?.path ??
2534
- inputFileName,
2535
- fileName: params.inputFile?.fileName ?? inputFileName,
2536
- storageKey: inputStorageKey,
2537
- contentType: params.inputFile?.contentType ?? null,
2538
- bytes:
2539
- typeof params.inputFile?.bytes === 'number'
2540
- ? params.inputFile.bytes
2541
- : null,
2542
- },
2543
- }
2544
- : null,
2545
- packagedFiles: params.packagedFiles ?? null,
2546
- partitionRange: null,
2547
- dedupBackend: params.dedupBackend,
2548
- resumeFromCheckpointR2Key: null,
2549
- contractSnapshot: params.contractSnapshot ?? null,
2550
- executionPlan: params.executionPlan ?? null,
2551
- childPlayManifests: params.childPlayManifests ?? null,
2552
- playCallGovernance: params.playCallGovernance ?? null,
2553
- force: params.force === true || params.forceToolRefresh === true,
2554
- forceToolRefresh: params.forceToolRefresh === true,
2555
- preloadedDbSessions: params.preloadedDbSessions ?? null,
2556
- runtimeTestFaultHeader: params.runtimeTestFaultHeader ?? null,
2557
- testPolicyOverrides: params.testPolicyOverrides ?? null,
2558
- coordinatorUrl: params.coordinatorUrl ?? null,
2559
- coordinatorInternalToken: params.coordinatorInternalToken ?? null,
2560
- totalRows: params.totalRows,
2561
- };
2562
- }
2563
-
2564
- async function readLegacyRunStream(response: Response): Promise<{
2565
- result: unknown;
2566
- outputRows: number | null;
2567
- logs: string[];
2568
- error: { message: string; stack: string | null } | null;
2569
- }> {
2570
- if (!response.body) {
2571
- throw new Error('Legacy child Worker returned an empty stream.');
2572
- }
2573
- const decoder = new TextDecoder();
2574
- const reader = response.body.getReader();
2575
- const logs: string[] = [];
2576
- let result: unknown = null;
2577
- let outputRows: number | null = null;
2578
- let error: { message: string; stack: string | null } | null = null;
2579
- let buffered = '';
2580
-
2581
- const consumeLine = (line: string): void => {
2582
- const trimmed = line.trim();
2583
- if (!trimmed) {
2584
- return;
2585
- }
2586
- let event: unknown;
2587
- try {
2588
- event = JSON.parse(trimmed);
2589
- } catch {
2590
- logs.push(trimmed);
2591
- return;
2592
- }
2593
- if (!event || typeof event !== 'object') {
2594
- return;
2595
- }
2596
- const record = event as Record<string, unknown>;
2597
- if (record.type === 'log' && typeof record.message === 'string') {
2598
- logs.push(record.message);
2599
- return;
2600
- }
2601
- if (record.type === 'result') {
2602
- result = record.result;
2603
- outputRows =
2604
- typeof record.outputRows === 'number' ? record.outputRows : null;
2605
- return;
2606
- }
2607
- if (record.type === 'error') {
2608
- error = {
2609
- message:
2610
- typeof record.message === 'string'
2611
- ? record.message
2612
- : 'Legacy child Worker failed.',
2613
- stack: typeof record.stack === 'string' ? record.stack : null,
2614
- };
2615
- }
2616
- };
2617
-
2618
- for (;;) {
2619
- const { value, done } = await reader.read();
2620
- if (value) {
2621
- buffered += decoder.decode(value, { stream: !done });
2622
- const lines = buffered.split('\n');
2623
- buffered = lines.pop() ?? '';
2624
- for (const line of lines) {
2625
- consumeLine(line);
2626
- }
2627
- }
2628
- if (done) {
2629
- buffered += decoder.decode();
2630
- consumeLine(buffered);
2631
- break;
2632
- }
2633
- }
2634
-
2635
- return { result, outputRows, logs, error };
2636
- }
2637
-
2638
- async function executeChildInline(input: {
2639
- env: CoordinatorEnv;
2640
- parentRunId: string;
2641
- body: Record<string, unknown>;
2642
- }): Promise<InlineChildSubmitResult> {
2643
- const startedAt = Date.now();
2644
- const validated = validateChildSubmitBody(input);
2645
- if (!validated.ok) {
2646
- throw new Error(String(validated.error.message ?? 'Invalid child submit.'));
2647
- }
2648
- const { manifest, governance, childPlayName, orgId, parentExecutorToken } =
2649
- validated;
2650
- const childRunId = buildChildRunId({
2651
- playName: childPlayName,
2652
- parentRunId: input.parentRunId,
2653
- body: input.body,
2654
- });
2655
- const timings: CoordinatorTiming[] = [];
2656
- const trace = (
2657
- phase: string,
2658
- phaseStartedAt: number,
2659
- extra?: Record<string, unknown>,
2660
- ): void => {
2661
- const timing: CoordinatorTiming = {
2662
- phase,
2663
- ms: Date.now() - phaseStartedAt,
2664
- graphHash: manifest.graphHash,
2665
- ...(extra ? { extra } : {}),
2666
- };
2667
- timings.push(timing);
2668
- recordCoordinatorPerfTrace({
2669
- runId: childRunId,
2670
- phase,
2671
- ms: timing.ms,
2672
- graphHash: manifest.graphHash,
2673
- extra: {
2674
- parentRunId: input.parentRunId,
2675
- mode: 'inline_dynamic_worker',
2676
- ...(extra ?? {}),
2677
- },
2678
- });
2679
- };
2680
- recordCoordinatorPerfTrace({
2681
- runId: childRunId,
2682
- phase: 'coordinator.inline_child_boundary',
2683
- ms: 0,
2684
- graphHash: manifest.graphHash,
2685
- extra: {
2686
- parentRunId: input.parentRunId,
2687
- childPlayName,
2688
- action: 'start',
2689
- mode: 'inline_dynamic_worker',
2690
- },
2691
- });
2692
-
2693
- const loaderStartedAt = Date.now();
2694
- const stub = loadDynamicPlayWorker(input.env, {
2695
- runId: childRunId,
2696
- graphHash: manifest.graphHash,
2697
- artifactStorageKey: manifest.artifactStorageKey,
2698
- artifactHash: manifest.artifactHash,
2699
- dynamicWorkerCode:
2700
- typeof manifest.bundledCode === 'string' ? manifest.bundledCode : null,
2701
- packagedFiles: null,
2702
- });
2703
- trace('coordinator.inline_child_loader_get', loaderStartedAt);
2704
-
2705
- const prepareStartedAt = Date.now();
2706
- const parentPlayName =
2707
- typeof input.body.parentPlayName === 'string' &&
2708
- input.body.parentPlayName.trim()
2709
- ? input.body.parentPlayName.trim()
2710
- : governance.parentPlayName;
2711
- const baseUrl = resolveRuntimeApiBaseUrl(input.env, input.body);
2712
- const { childToken, preloadedDbSessions, prepareTimings, transportTimings } =
2713
- await prepareInlineChildRunWithRuntime({
2714
- env: input.env,
2715
- baseUrl,
2716
- parentExecutorToken,
2717
- parentRunId: input.parentRunId,
2718
- parentPlayName,
2719
- childRunId,
2720
- childPlayName,
2721
- manifest,
2722
- governance,
2723
- userEmail:
2724
- typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
2725
- });
2726
- trace('coordinator.inline_child_prepare', prepareStartedAt, {
2727
- sessions: preloadedDbSessions.length,
2728
- prepareTimings,
2729
- transportTimings,
2730
- });
2731
-
2732
- const params = buildChildWorkflowParams({
2733
- env: input.env,
2734
- body: input.body,
2735
- manifest,
2736
- governance,
2737
- childRunId,
2738
- childPlayName,
2739
- childToken,
2740
- orgId,
2741
- coordinatorUrl: null,
2742
- runtimeBackend: 'cf_workflows_dynamic_worker_inline_child',
2743
- dynamicWorkerCode: null,
2744
- preloadedDbSessions:
2745
- preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
2746
- });
2747
-
2748
- let entrypoint: ReturnType<Awaited<typeof stub>['getEntrypoint']> | null =
2749
- null;
2750
- let response: Response | null = null;
2751
- try {
2752
- const awaitedStub = await stub;
2753
- const entrypointStartedAt = Date.now();
2754
- entrypoint = awaitedStub.getEntrypoint();
2755
- trace('coordinator.inline_child_get_entrypoint', entrypointStartedAt);
2756
- const fetchStartedAt = Date.now();
2757
- const inlineResponse = await entrypoint.fetch(
2758
- new Request('https://deepline.dynamic.internal/run-inline', {
2759
- method: 'POST',
2760
- headers: { 'content-type': 'application/json' },
2761
- body: JSON.stringify(runRequestFromPlayWorkflowParams(params)),
2762
- }),
2763
- );
2764
- if (!inlineResponse) {
2765
- throw new Error('Inline child Worker returned no response.');
2766
- }
2767
- let workerResponse = inlineResponse as Response;
2768
- response = workerResponse;
2769
- trace('coordinator.inline_child_worker_fetch', fetchStartedAt, {
2770
- status: workerResponse.status,
2771
- endpoint: '/run-inline',
2772
- });
2773
- let usedLegacyRunStream = false;
2774
- if (workerResponse.status === 404) {
2775
- disposeRpcStub(workerResponse);
2776
- const legacyFetchStartedAt = Date.now();
2777
- const legacyResponse = await entrypoint.fetch(
2778
- new Request('https://deepline.dynamic.internal/run', {
2779
- method: 'POST',
2780
- headers: { 'content-type': 'application/json' },
2781
- body: JSON.stringify(runRequestFromPlayWorkflowParams(params)),
2782
- }),
2783
- );
2784
- if (!legacyResponse) {
2785
- throw new Error('Legacy inline child Worker returned no response.');
2786
- }
2787
- workerResponse = legacyResponse as Response;
2788
- response = workerResponse;
2789
- usedLegacyRunStream = true;
2790
- trace('coordinator.inline_child_worker_fetch', legacyFetchStartedAt, {
2791
- status: workerResponse.status,
2792
- endpoint: '/run',
2793
- compatibility: 'legacy_stream',
2794
- });
2795
- }
2796
- if (!workerResponse.ok) {
2797
- const text = await workerResponse.text().catch(() => '');
2798
- throw new Error(
2799
- `Inline child Worker failed ${workerResponse.status}: ${text.slice(0, 800)}`,
2800
- );
2801
- }
2802
- const responseStartedAt = Date.now();
2803
- const parsed: InlineWorkerRunResponse = usedLegacyRunStream
2804
- ? await readLegacyRunStream(workerResponse).then((legacy) => ({
2805
- status: legacy.error ? 'failed' : 'completed',
2806
- result: legacy.result,
2807
- outputRows: legacy.outputRows ?? undefined,
2808
- events: legacy.logs.map((message) => ({
2809
- type: 'log',
2810
- message,
2811
- })),
2812
- error: legacy.error ?? undefined,
2813
- }))
2814
- : ((await workerResponse.json()) as InlineWorkerRunResponse);
2815
- const logs = (parsed.events ?? []).flatMap((event) => {
2816
- if (
2817
- event &&
2818
- typeof event === 'object' &&
2819
- event.type === 'log' &&
2820
- typeof event.message === 'string'
2821
- ) {
2822
- return [event.message];
2823
- }
2824
- return [];
2825
- });
2826
- trace('coordinator.inline_child_response', responseStartedAt, {
2827
- status: parsed.status ?? null,
2828
- endpoint: usedLegacyRunStream ? '/run' : '/run-inline',
2829
- logCount: logs.length,
2830
- outputRows:
2831
- typeof parsed.outputRows === 'number' ? parsed.outputRows : null,
2832
- durationMs:
2833
- typeof parsed.durationMs === 'number' ? parsed.durationMs : null,
2834
- });
2835
- for (const timing of parsed.timings ?? []) {
2836
- if (
2837
- !timing ||
2838
- typeof timing !== 'object' ||
2839
- typeof timing.phase !== 'string' ||
2840
- typeof timing.ms !== 'number' ||
2841
- !Number.isFinite(timing.ms)
2842
- ) {
2843
- continue;
2844
- }
2845
- recordCoordinatorPerfTrace({
2846
- runId: childRunId,
2847
- phase: `dynamic_worker.${timing.phase}`,
2848
- ms: Math.max(0, Math.round(timing.ms)),
2849
- graphHash: manifest.graphHash,
2850
- extra: {
2851
- parentRunId: input.parentRunId,
2852
- mode: 'inline_dynamic_worker',
2853
- ...(isRecord(timing.extra) ? timing.extra : {}),
2854
- },
2855
- });
2856
- timings.push({
2857
- phase: `dynamic_worker.${timing.phase}`,
2858
- ms: Math.max(0, Math.round(timing.ms)),
2859
- graphHash: manifest.graphHash,
2860
- ...(isRecord(timing.extra) ? { extra: timing.extra } : {}),
2861
- });
2862
- }
2863
- trace('coordinator.inline_child_total', startedAt);
2864
- if (parsed.status === 'failed' || parsed.error) {
2865
- const error = {
2866
- message:
2867
- typeof parsed.error?.message === 'string'
2868
- ? parsed.error.message
2869
- : 'Inline child Worker failed.',
2870
- stack:
2871
- typeof parsed.error?.stack === 'string' ? parsed.error.stack : null,
2872
- };
2873
- recordCoordinatorPerfTrace({
2874
- runId: childRunId,
2875
- phase: 'coordinator.inline_child_boundary',
2876
- ms: Date.now() - startedAt,
2877
- graphHash: manifest.graphHash,
2878
- extra: {
2879
- parentRunId: input.parentRunId,
2880
- childPlayName,
2881
- action: 'failed',
2882
- mode: 'inline_dynamic_worker',
2883
- error: error.message,
2884
- },
2885
- });
2886
- const terminalAppendStartedAt = Date.now();
2887
- await appendRegisteredChildRunTerminal({
2888
- env: input.env,
2889
- baseUrl,
2890
- childExecutorToken: childToken,
2891
- childRunId,
2892
- status: 'failed',
2893
- error: error.message,
2894
- });
2895
- trace(
2896
- 'coordinator.inline_child_terminal_ledger_append',
2897
- terminalAppendStartedAt,
2898
- { status: 'failed' },
2899
- );
2900
- return {
2901
- workflowId: childRunId,
2902
- runId: childRunId,
2903
- status: 'failed',
2904
- mode: 'inline_dynamic_worker',
2905
- error,
2906
- logs,
2907
- timings,
2908
- };
2909
- }
2910
- recordCoordinatorPerfTrace({
2911
- runId: childRunId,
2912
- phase: 'coordinator.inline_child_boundary',
2913
- ms: Date.now() - startedAt,
2914
- graphHash: manifest.graphHash,
2915
- extra: {
2916
- parentRunId: input.parentRunId,
2917
- childPlayName,
2918
- action: 'completed',
2919
- mode: 'inline_dynamic_worker',
2920
- },
2921
- });
2922
- const terminalAppendStartedAt = Date.now();
2923
- await appendRegisteredChildRunTerminal({
2924
- env: input.env,
2925
- baseUrl,
2926
- childExecutorToken: childToken,
2927
- childRunId,
2928
- status: 'completed',
2929
- result: parsed.result,
2930
- });
2931
- trace(
2932
- 'coordinator.inline_child_terminal_ledger_append',
2933
- terminalAppendStartedAt,
2934
- { status: 'completed' },
2935
- );
2936
- return {
2937
- workflowId: childRunId,
2938
- runId: childRunId,
2939
- status: 'completed',
2940
- mode: 'inline_dynamic_worker',
2941
- result: parsed.result,
2942
- output: parsed.result,
2943
- logs,
2944
- timings,
2945
- };
2946
- } finally {
2947
- disposeRpcStub(response);
2948
- disposeRpcStub(entrypoint);
2949
- }
2950
- }
2951
-
2952
- function isRetryableChildWorkflowSubmitError(error: unknown): boolean {
2953
- const message = error instanceof Error ? error.message : String(error);
2954
- return /timed out|timeout|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|internal error/i.test(
2955
- message,
2956
- );
2957
- }
2958
-
2959
- function isRetryableChildWorkflowSubmitResponse(
2960
- status: number,
2961
- body: string,
2962
- ): boolean {
2963
- if (
2964
- status === 408 ||
2965
- status === 429 ||
2966
- status === 502 ||
2967
- status === 503 ||
2968
- status === 504 ||
2969
- status === 530
2970
- ) {
2971
- return true;
2972
- }
2973
- return (
2974
- status === 500 &&
2975
- /COORDINATOR_WORKFLOW_ROUTE_FAILED|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|UND_ERR_CONNECT_TIMEOUT|internal error/i.test(
2976
- body,
2977
- )
2978
- );
1694
+ ? Math.max(0, Math.floor(value))
1695
+ : 0;
2979
1696
  }
2980
1697
 
2981
- async function submitChildWorkflowThroughCoordinator(input: {
1698
+ async function restartWorkflowAfterPlatformReset(input: {
2982
1699
  env: CoordinatorEnv;
2983
- parentRunId: string;
2984
- body: Record<string, unknown>;
2985
- coordinatorUrl: string | null;
1700
+ ctx?: ExecutionContext;
1701
+ runId: string;
1702
+ oldInstance: WorkflowInstance;
1703
+ error: string;
1704
+ status: InstanceStatus;
2986
1705
  }): Promise<{
2987
- response: Response;
2988
- responseText: string;
2989
- childRunId: string;
2990
- childPlayName: string;
2991
- startedAt: number;
2992
- timings: CoordinatorTiming[];
1706
+ retried: boolean;
1707
+ result: Record<string, unknown> | null;
2993
1708
  }> {
2994
- const startedAt = Date.now();
2995
- const timings: CoordinatorTiming[] = [];
2996
- const trace = (
2997
- phase: string,
2998
- phaseStartedAt: number,
2999
- graphHash?: string | null,
3000
- extra?: Record<string, unknown>,
3001
- ): void => {
3002
- const timing: CoordinatorTiming = {
3003
- phase,
3004
- ms: Date.now() - phaseStartedAt,
3005
- ...(graphHash ? { graphHash } : {}),
3006
- ...(extra ? { extra } : {}),
3007
- };
3008
- timings.push(timing);
3009
- recordCoordinatorPerfTrace({
3010
- runId: input.parentRunId,
3011
- phase,
3012
- ms: timing.ms,
3013
- graphHash: graphHash ?? undefined,
3014
- extra,
3015
- });
3016
- };
3017
- const validated = validateChildSubmitBody({
3018
- parentRunId: input.parentRunId,
3019
- body: input.body,
1709
+ const decision = decideWorkflowPlatformRetry({
1710
+ workflowStatus: String(input.status.status ?? ''),
1711
+ error: input.error,
1712
+ retryAttempts: 0,
3020
1713
  });
3021
- if (!validated.ok) {
3022
- return {
3023
- response: Response.json(
3024
- { error: validated.error },
3025
- { status: validated.status },
3026
- ),
3027
- responseText: '',
3028
- childRunId: '',
3029
- childPlayName: '',
3030
- startedAt,
3031
- timings,
3032
- };
1714
+ if (decision.action !== 'retry') {
1715
+ return { retried: false, result: null };
3033
1716
  }
3034
- const { manifest, governance, childPlayName, orgId, parentExecutorToken } =
3035
- validated;
3036
- const childRunId = buildChildRunId({
3037
- playName: childPlayName,
3038
- parentRunId: input.parentRunId,
3039
- body: input.body,
3040
- });
3041
- const baseUrl = resolveRuntimeApiBaseUrl(input.env, input.body);
3042
-
3043
- const prepareStartedAt = Date.now();
3044
- const { childToken, preloadedDbSessions, prepareTimings, transportTimings } =
3045
- await prepareInlineChildRunWithRuntime({
3046
- env: input.env,
3047
- baseUrl,
3048
- parentExecutorToken,
3049
- parentRunId: input.parentRunId,
3050
- parentPlayName:
3051
- typeof input.body.parentPlayName === 'string' &&
3052
- input.body.parentPlayName.trim()
3053
- ? input.body.parentPlayName.trim()
3054
- : governance.parentPlayName,
3055
- childRunId,
3056
- childPlayName,
3057
- manifest,
3058
- governance,
3059
- userEmail:
3060
- typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
3061
- runtimeBackend: 'cf_workflows_dynamic_worker',
3062
- schedulerBackend: 'cf_workflows',
3063
- executionProfile: 'workers_edge',
3064
- });
3065
- trace(
3066
- 'coordinator.child_submit_prepare',
3067
- prepareStartedAt,
3068
- manifest.graphHash,
3069
- {
3070
- childRunId,
3071
- childPlayName,
3072
- sessions: preloadedDbSessions.length,
3073
- prepareTimings,
3074
- transportTimings,
3075
- },
3076
- );
3077
-
3078
- const params = buildChildWorkflowParams({
1717
+ const claim = await claimWorkflowPlatformRetry({
3079
1718
  env: input.env,
3080
- body: input.body,
3081
- manifest,
3082
- governance,
3083
- childRunId,
3084
- childPlayName,
3085
- childToken,
3086
- orgId,
3087
- coordinatorUrl: input.coordinatorUrl,
3088
- runtimeBackend: 'cf_workflows_dynamic_worker',
3089
- dynamicWorkerCode:
3090
- typeof manifest.bundledCode === 'string' ? manifest.bundledCode : null,
3091
- preloadedDbSessions:
3092
- preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
3093
- });
3094
-
3095
- const workflowSubmitStartedAt = Date.now();
3096
- let response: Response | null = null;
3097
- let responseText = '';
3098
- let lastSubmitError: unknown = null;
3099
- for (
3100
- let attempt = 0;
3101
- attempt <= CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS.length;
3102
- attempt += 1
3103
- ) {
3104
- try {
3105
- response = await handleWorkflowRoute({
3106
- runId: childRunId,
3107
- action: 'submit',
3108
- request: new Request(
3109
- `https://deepline.coordinator.internal/workflow/${encodeURIComponent(
3110
- childRunId,
3111
- )}/submit`,
3112
- {
3113
- method: 'POST',
3114
- headers: { 'content-type': 'application/json' },
3115
- body: JSON.stringify(params),
3116
- },
3117
- ),
3118
- env: input.env,
3119
- });
3120
- } catch (error) {
3121
- lastSubmitError = error;
3122
- trace(
3123
- 'coordinator.child_submit_workflow_attempt',
3124
- workflowSubmitStartedAt,
3125
- manifest.graphHash,
3126
- {
3127
- childRunId,
3128
- attempt: attempt + 1,
3129
- status: 'threw',
3130
- error: error instanceof Error ? error.message : String(error),
3131
- },
3132
- );
3133
- if (
3134
- attempt >= CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS.length ||
3135
- !isRetryableChildWorkflowSubmitError(error)
3136
- ) {
3137
- await markRegisteredChildRunFailed({
3138
- env: input.env,
3139
- baseUrl,
3140
- childExecutorToken: childToken,
3141
- childRunId,
3142
- error,
3143
- }).catch((markError) => {
3144
- recordCoordinatorEvent(
3145
- 'error',
3146
- 'coordinator.child_workflow_submit.failure_mark_failed',
3147
- '[coordinator] child workflow submit failure mark failed',
3148
- {
3149
- childRunId,
3150
- error:
3151
- markError instanceof Error
3152
- ? markError.message
3153
- : String(markError),
3154
- },
3155
- );
3156
- });
3157
- throw error;
3158
- }
3159
- await sleep(CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS[attempt] ?? 0);
3160
- continue;
3161
- }
3162
-
3163
- responseText = await response.text().catch(() => '');
3164
- trace(
3165
- 'coordinator.child_submit_workflow_attempt',
3166
- workflowSubmitStartedAt,
3167
- manifest.graphHash,
1719
+ runId: input.runId,
1720
+ }).catch((error) => {
1721
+ recordCoordinatorEvent(
1722
+ 'warn',
1723
+ 'coordinator.workflow_platform_retry.claim_failed',
1724
+ '[coordinator] workflow platform retry claim failed',
3168
1725
  {
3169
- childRunId,
3170
- attempt: attempt + 1,
3171
- status: response.status,
1726
+ runId: input.runId,
1727
+ error: error instanceof Error ? error.message : String(error),
3172
1728
  },
3173
1729
  );
3174
- if (
3175
- response.ok ||
3176
- attempt >= CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS.length ||
3177
- !isRetryableChildWorkflowSubmitResponse(response.status, responseText)
3178
- ) {
3179
- break;
3180
- }
3181
- lastSubmitError = `workflow submit returned ${response.status}: ${responseText.slice(0, 800)}`;
3182
- await sleep(CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS[attempt] ?? 0);
3183
- }
3184
- if (!response) {
3185
- throw lastSubmitError instanceof Error
3186
- ? lastSubmitError
3187
- : new Error(String(lastSubmitError));
1730
+ return null;
1731
+ });
1732
+ if (!claim?.claimed || !claim.params) {
1733
+ return { retried: false, result: null };
3188
1734
  }
3189
- trace(
3190
- 'coordinator.child_submit_workflow',
3191
- workflowSubmitStartedAt,
3192
- manifest.graphHash,
3193
- { childRunId, status: response.status },
3194
- );
3195
- if (!response.ok) {
3196
- await markRegisteredChildRunFailed({
1735
+ const retryInstanceId = workflowRetryInstanceId(input.runId, claim.attempts);
1736
+ const retryStartedAt = Date.now();
1737
+ let retryInstance: WorkflowInstance | null = null;
1738
+ const retryParams = { ...claim.params, runAttempt: claim.attempts };
1739
+ try {
1740
+ retryInstance = await createDynamicWorkflowInstance({
3197
1741
  env: input.env,
3198
- baseUrl,
3199
- childExecutorToken: childToken,
3200
- childRunId,
3201
- error: `workflow submit returned ${response.status}: ${responseText.slice(0, 800)}`,
3202
- }).catch((markError) => {
3203
- recordCoordinatorEvent(
3204
- 'error',
3205
- 'coordinator.child_workflow_submit.failure_mark_failed',
3206
- '[coordinator] child workflow submit failure mark failed',
3207
- {
3208
- childRunId,
3209
- error:
3210
- markError instanceof Error ? markError.message : String(markError),
3211
- },
3212
- );
1742
+ id: retryInstanceId,
1743
+ params: workflowCreateParams(retryParams, {
1744
+ params: retryParams,
1745
+ paramsRef: claim.paramsRef,
1746
+ }),
1747
+ metadataParams: retryParams,
3213
1748
  });
3214
- }
3215
- return {
3216
- response,
3217
- responseText,
3218
- childRunId,
3219
- childPlayName,
3220
- startedAt,
3221
- timings,
3222
- };
3223
- }
3224
-
3225
- /**
3226
- * In-process Fetcher handed to each per-graphHash play Worker as
3227
- * `env.RUNTIME_API`. Runs in the coordinator's isolate. Forwards runtime
3228
- * callbacks to DEEPLINE_API_BASE_URL: in dev (the only mode — deployed CF
3229
- * coordinator + local app) that is the cloudflared tunnel URL exposing the
3230
- * laptop's app; in prod it is the deployed app URL. There is no
3231
- * direct-to-localhost path (the local-workerd dev mode was removed).
3232
- *
3233
- * Has to be a `WorkerEntrypoint` (not a plain closure) because closures
3234
- * containing captured state aren't structured-cloneable, and Cloudflare
3235
- * Workflows serializes the dynamic Worker's `env` map when it persists
3236
- * workflow state. WorkerEntrypoint stubs ARE cloneable.
3237
- *
3238
- * Path allowlist: only `/api/v2/plays/*` and `/api/v2/integrations/*`.
3239
- * Anything else is a sandbox-escape attempt and gets a loud 403 — the
3240
- * coordinator must NOT proxy a play's request to internal admin routes
3241
- * even if the play tries to construct such a URL.
3242
- */
3243
- export class RuntimeApi extends WorkerEntrypoint<CoordinatorEnv, undefined> {
3244
- async fetch(request: Request): Promise<Response> {
3245
- const incoming = new URL(request.url);
3246
- const allowed =
3247
- incoming.pathname.startsWith('/api/v2/plays/') ||
3248
- incoming.pathname.startsWith('/api/v2/integrations/');
3249
- if (!allowed) {
3250
- return new Response(
3251
- JSON.stringify({
3252
- error:
3253
- 'RUNTIME_API binding only forwards /api/v2/plays/* and /api/v2/integrations/* paths',
3254
- attemptedPath: incoming.pathname,
3255
- }),
3256
- { status: 403, headers: { 'content-type': 'application/json' } },
3257
- );
3258
- }
3259
- const configuredApiBaseUrl =
3260
- typeof this.env.DEEPLINE_API_BASE_URL === 'string' &&
3261
- this.env.DEEPLINE_API_BASE_URL.trim()
3262
- ? this.env.DEEPLINE_API_BASE_URL.trim()
3263
- : 'https://code.deepline.com';
3264
- const target = new URL(
3265
- incoming.pathname + incoming.search,
3266
- configuredApiBaseUrl,
3267
- );
3268
- const forwarded = new Request(target.toString(), request);
3269
- const bypassToken = this.env.VERCEL_PROTECTION_BYPASS_TOKEN;
3270
- if (typeof bypassToken === 'string' && bypassToken) {
3271
- forwarded.headers.set('x-vercel-protection-bypass', bypassToken);
3272
- }
3273
- const res = await fetch(forwarded);
3274
- if (!res.ok) {
3275
- const body = await res
3276
- .clone()
3277
- .text()
3278
- .catch(() => '');
3279
- recordCoordinatorEvent(
3280
- 'error',
3281
- 'coordinator.runtime_api.proxy_failed',
3282
- '[RUNTIME_API] request failed',
3283
- {
3284
- path: incoming.pathname,
3285
- status: res.status,
3286
- targetOrigin: target.origin,
3287
- body: body.slice(0, 500),
1749
+ await recordWorkflowInstanceId({
1750
+ env: input.env,
1751
+ runId: input.runId,
1752
+ instanceId: retryInstance.id,
1753
+ });
1754
+ input.ctx?.waitUntil(input.oldInstance.terminate().catch(() => undefined));
1755
+ recordCoordinatorPerfTraceBuffered(input.env, input.ctx, {
1756
+ runId: input.runId,
1757
+ phase: 'coordinator.platform_deploy_retry',
1758
+ ms: Date.now() - retryStartedAt,
1759
+ graphHash: claim.params.graphHash ?? null,
1760
+ extra: {
1761
+ retryAttempt: claim.attempts,
1762
+ retryInstanceId: retryInstance.id,
1763
+ reason: decision.reason,
1764
+ },
1765
+ });
1766
+ return {
1767
+ retried: true,
1768
+ result: {
1769
+ runId: input.runId,
1770
+ playName: claim.params.playName,
1771
+ status: 'running',
1772
+ result: null,
1773
+ error: null,
1774
+ retry: {
1775
+ reason: decision.reason,
1776
+ attempt: claim.attempts,
1777
+ message: decision.message,
3288
1778
  },
3289
- );
3290
- }
3291
- return res;
1779
+ },
1780
+ };
1781
+ } finally {
1782
+ disposeRpcStub(retryInstance);
3292
1783
  }
3293
1784
  }
3294
1785
 
@@ -3296,60 +1787,6 @@ export class CoordinatorControl extends WorkerEntrypoint<
3296
1787
  CoordinatorEnv,
3297
1788
  undefined
3298
1789
  > {
3299
- async submitChild(
3300
- parentRunId: string,
3301
- body: Record<string, unknown>,
3302
- ): Promise<InlineChildSubmitResult> {
3303
- return await executeChildInline({
3304
- env: this.env,
3305
- parentRunId,
3306
- body,
3307
- });
3308
- }
3309
-
3310
- async submitWorkflowChild(
3311
- parentRunId: string,
3312
- body: Record<string, unknown>,
3313
- ): Promise<{
3314
- workflowId?: string;
3315
- runId?: string;
3316
- status?: string;
3317
- mode?: string;
3318
- timings?: CoordinatorTiming[];
3319
- coordinator?: unknown;
3320
- error?: unknown;
3321
- }> {
3322
- const { response, responseText, childRunId, timings } =
3323
- await submitChildWorkflowThroughCoordinator({
3324
- env: this.env,
3325
- parentRunId,
3326
- body,
3327
- coordinatorUrl: null,
3328
- });
3329
- let parsed: unknown = {};
3330
- try {
3331
- parsed = responseText ? JSON.parse(responseText) : {};
3332
- } catch {
3333
- parsed = { error: responseText };
3334
- }
3335
- if (!response.ok) {
3336
- return {
3337
- runId: childRunId || undefined,
3338
- workflowId: childRunId || undefined,
3339
- status: 'failed',
3340
- error: isRecord(parsed) ? (parsed.error ?? parsed) : parsed,
3341
- };
3342
- }
3343
- return {
3344
- workflowId: childRunId,
3345
- runId: childRunId,
3346
- status: 'started',
3347
- mode: 'workflow_rpc',
3348
- coordinator: parsed,
3349
- timings,
3350
- };
3351
- }
3352
-
3353
1790
  async signal(
3354
1791
  runId: string,
3355
1792
  body: Record<string, unknown>,
@@ -3420,25 +1857,6 @@ export class CoordinatorControl extends WorkerEntrypoint<
3420
1857
  return await readCoordinatorTerminalState(this.env, runId);
3421
1858
  }
3422
1859
 
3423
- async readChildTerminalState(
3424
- parentRunId: string,
3425
- eventKey: string,
3426
- timeoutMs?: number,
3427
- ): Promise<CoordinatorChildTerminalState | null> {
3428
- if (!parentRunId || !eventKey) {
3429
- throw new Error('parentRunId and eventKey are required.');
3430
- }
3431
- return await readCoordinatorChildTerminalState({
3432
- env: this.env,
3433
- parentRunId,
3434
- eventKey,
3435
- timeoutMs:
3436
- typeof timeoutMs === 'number' && Number.isFinite(timeoutMs)
3437
- ? Math.max(0, Math.min(Math.floor(timeoutMs), 30_000))
3438
- : undefined,
3439
- });
3440
- }
3441
-
3442
1860
  /**
3443
1861
  * Distributed Rate State Backend acquire: lease up to `requested` request-
3444
1862
  * window permits for `bucketId` (`<orgId>:<provider>`) from the per-bucket
@@ -3699,37 +2117,6 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
3699
2117
  const output = isRecord(result) ? result : null;
3700
2118
  const terminalResult = output?.result ?? result;
3701
2119
  const terminalStartedAt = Date.now();
3702
- const childTerminalStartedAt = Date.now();
3703
- try {
3704
- const childTerminalPersisted =
3705
- await appendWorkflowChildCompletedRunEvent({
3706
- env,
3707
- event: innerEvent,
3708
- result: terminalResult,
3709
- });
3710
- if (childTerminalPersisted) {
3711
- trace({
3712
- runId: runIdForTrace,
3713
- phase:
3714
- 'coordinator.child_terminal_ledger_append_completed',
3715
- ms: Date.now() - childTerminalStartedAt,
3716
- graphHash,
3717
- });
3718
- }
3719
- } catch (childTerminalError) {
3720
- recordCoordinatorEvent(
3721
- 'warn',
3722
- 'coordinator.child_terminal.ledger_append_failed',
3723
- '[coordinator] child terminal ledger append failed; preserving coordinator terminal cache',
3724
- {
3725
- runId: runIdForTrace,
3726
- message:
3727
- childTerminalError instanceof Error
3728
- ? childTerminalError.message
3729
- : String(childTerminalError),
3730
- },
3731
- );
3732
- }
3733
2120
  try {
3734
2121
  await writeCoordinatorTerminalState(env, {
3735
2122
  runId: runIdForTrace,
@@ -4804,113 +3191,6 @@ async function handleWorkflowRoute(input: {
4804
3191
  }
4805
3192
  }
4806
3193
 
4807
- if (action === 'submit-child') {
4808
- const startedAt = Date.now();
4809
- try {
4810
- const body = (await request.json().catch(() => null)) as Record<
4811
- string,
4812
- unknown
4813
- > | null;
4814
- if (!body || typeof body !== 'object') {
4815
- return Response.json(
4816
- {
4817
- error: {
4818
- code: 'CHILD_SUBMIT_BODY_REQUIRED',
4819
- message: 'submit-child requires a JSON body.',
4820
- phase: 'coordinator_child_submit',
4821
- parentRunId: runId,
4822
- },
4823
- },
4824
- { status: 400 },
4825
- );
4826
- }
4827
- const {
4828
- response: submitResponse,
4829
- responseText,
4830
- childRunId,
4831
- childPlayName,
4832
- } = await submitChildWorkflowThroughCoordinator({
4833
- env,
4834
- parentRunId: runId,
4835
- body,
4836
- coordinatorUrl: new URL(request.url).origin,
4837
- });
4838
- recordCoordinatorPerfTrace({
4839
- runId,
4840
- phase: 'coordinator.child_submit',
4841
- ms: Date.now() - startedAt,
4842
- extra: { status: submitResponse.status, childRunId },
4843
- });
4844
- recordCoordinatorEvent(
4845
- 'info',
4846
- 'play.runtime.span',
4847
- '[play.runtime.span]',
4848
- {
4849
- event: 'play.runtime.span',
4850
- phase: 'coordinator_child_submit',
4851
- runId,
4852
- parentRunId: runId,
4853
- childRunId,
4854
- playName: childPlayName,
4855
- ms: Date.now() - startedAt,
4856
- status: submitResponse.ok ? 'ok' : 'failed',
4857
- ...(submitResponse.ok
4858
- ? {}
4859
- : { errorCode: 'COORDINATOR_CHILD_SUBMIT_FAILED' }),
4860
- },
4861
- );
4862
- if (!submitResponse.ok) {
4863
- return new Response(responseText, {
4864
- status: submitResponse.status,
4865
- headers: {
4866
- 'content-type':
4867
- submitResponse.headers.get('content-type') ?? 'application/json',
4868
- 'cache-control': 'no-store',
4869
- },
4870
- });
4871
- }
4872
- return Response.json(
4873
- {
4874
- workflowId: childRunId,
4875
- runId: childRunId,
4876
- status: 'started',
4877
- coordinator: JSON.parse(responseText || '{}'),
4878
- },
4879
- {
4880
- headers: {
4881
- 'cache-control': 'no-store',
4882
- },
4883
- },
4884
- );
4885
- } catch (error) {
4886
- const message = error instanceof Error ? error.message : String(error);
4887
- recordCoordinatorEvent(
4888
- 'error',
4889
- 'coordinator.child_submit.error',
4890
- '[coordinator.child_submit.error]',
4891
- {
4892
- runId,
4893
- ms: Date.now() - startedAt,
4894
- error: message,
4895
- },
4896
- );
4897
- return Response.json(
4898
- {
4899
- error: {
4900
- code: 'COORDINATOR_CHILD_SUBMIT_FAILED',
4901
- message,
4902
- phase: 'coordinator_child_submit',
4903
- parentRunId: runId,
4904
- },
4905
- },
4906
- {
4907
- status: 500,
4908
- headers: { 'cache-control': 'no-store' },
4909
- },
4910
- );
4911
- }
4912
- }
4913
-
4914
3194
  if (action === 'tail') {
4915
3195
  const url = new URL(request.url);
4916
3196
  const waitMs = Math.min(
@@ -5120,25 +3400,6 @@ async function handleWorkflowRoute(input: {
5120
3400
  : eventKey
5121
3401
  ? `integration_event_${eventKey}`
5122
3402
  : 'integration_event';
5123
- if (body.signal === 'integration_event' && eventKey) {
5124
- await writeCoordinatorChildTerminalState({
5125
- env,
5126
- parentRunId: runId,
5127
- eventKey,
5128
- data: body.data ?? body,
5129
- }).catch((error: unknown) => {
5130
- recordCoordinatorEvent(
5131
- 'warn',
5132
- 'coordinator.child_terminal.cache_write_failed',
5133
- '[coordinator] child terminal cache write failed',
5134
- {
5135
- runId,
5136
- eventKey,
5137
- error: error instanceof Error ? error.message : String(error),
5138
- },
5139
- );
5140
- });
5141
- }
5142
3403
  await instance.sendEvent({
5143
3404
  type: workflowEventType(eventType),
5144
3405
  payload: body,
@@ -5943,7 +4204,6 @@ async function handleCoordinatorWarmup(
5943
4204
  packagedFiles: null,
5944
4205
  contractSnapshot: null,
5945
4206
  executionPlan: null,
5946
- playCallGovernance: null,
5947
4207
  dynamicWorkerCode: null,
5948
4208
  executorToken: 'coordinator-warmup',
5949
4209
  baseUrl: env.DEEPLINE_API_BASE_URL,
@@ -6057,14 +4317,6 @@ function makeRuntimeApiEnvBinding(env: CoordinatorEnv): {
6057
4317
  function makeCoordinatorControlBinding():
6058
4318
  | {
6059
4319
  COORDINATOR: {
6060
- submitChild(
6061
- parentRunId: string,
6062
- body: Record<string, unknown>,
6063
- ): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
6064
- submitWorkflowChild(
6065
- parentRunId: string,
6066
- body: Record<string, unknown>,
6067
- ): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
6068
4320
  signal(
6069
4321
  runId: string,
6070
4322
  body: Record<string, unknown>,
@@ -6080,11 +4332,6 @@ function makeCoordinatorControlBinding():
6080
4332
  readTerminalState(
6081
4333
  runId: string,
6082
4334
  ): Promise<CoordinatorTerminalState | null>;
6083
- readChildTerminalState(
6084
- parentRunId: string,
6085
- eventKey: string,
6086
- timeoutMs?: number,
6087
- ): Promise<CoordinatorChildTerminalState | null>;
6088
4335
  rateAcquire(input: {
6089
4336
  bucketId: string;
6090
4337
  rules: Array<{
@@ -6104,14 +4351,6 @@ function makeCoordinatorControlBinding():
6104
4351
  | Record<string, never> {
6105
4352
  const exports = workersExports as unknown as {
6106
4353
  CoordinatorControl?: (init: { props: undefined }) => {
6107
- submitChild(
6108
- parentRunId: string,
6109
- body: Record<string, unknown>,
6110
- ): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
6111
- submitWorkflowChild(
6112
- parentRunId: string,
6113
- body: Record<string, unknown>,
6114
- ): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
6115
4354
  signal(
6116
4355
  runId: string,
6117
4356
  body: Record<string, unknown>,
@@ -6124,11 +4363,6 @@ function makeCoordinatorControlBinding():
6124
4363
  readTerminalState(
6125
4364
  runId: string,
6126
4365
  ): Promise<CoordinatorTerminalState | null>;
6127
- readChildTerminalState(
6128
- parentRunId: string,
6129
- eventKey: string,
6130
- timeoutMs?: number,
6131
- ): Promise<CoordinatorChildTerminalState | null>;
6132
4366
  rateAcquire(input: {
6133
4367
  bucketId: string;
6134
4368
  rules: Array<{