deepline 0.1.228 → 0.1.229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +76 -1842
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +0 -113
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +81 -968
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +1 -50
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +67 -15
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +84 -79
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +152 -688
- package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +3 -10
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -47
- package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts +0 -2
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +12 -200
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +2 -17
- package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +28 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-execution-scope.ts +16 -64
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +0 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +37 -9
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +0 -36
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +3 -21
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +9 -1
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -0
- package/dist/cli/index.js +2 -2
- package/dist/cli/index.mjs +2 -2
- package/dist/{compiler-manifest-BhgZ23A4.d.ts → compiler-manifest-CYcwzSOJ.d.mts} +2 -0
- package/dist/{compiler-manifest-BhgZ23A4.d.mts → compiler-manifest-CYcwzSOJ.d.ts} +2 -0
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/package.json +1 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/child-manifest-resolver.ts +0 -108
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +0 -374
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +0 -203
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +0 -14
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +0 -121
|
@@ -29,11 +29,7 @@ import {
|
|
|
29
29
|
type WorkflowRunner,
|
|
30
30
|
} from '@cloudflare/dynamic-workflows';
|
|
31
31
|
import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
|
|
32
|
-
import {
|
|
33
|
-
import type {
|
|
34
|
-
PlayCallGovernanceSnapshot,
|
|
35
|
-
PlayRunInputPayload,
|
|
36
|
-
} from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
32
|
+
import type { PlayRunInputPayload } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
37
33
|
import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runtime/test-runtime-seams';
|
|
38
34
|
import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
|
|
39
35
|
import {
|
|
@@ -45,10 +41,7 @@ import {
|
|
|
45
41
|
encryptDbSessionPostgresUrl,
|
|
46
42
|
} from '../../../shared_libs/play-runtime/db-session-crypto';
|
|
47
43
|
import { planRuntimeDbSessionRequirements } from '../../../shared_libs/play-runtime/db-session-plan';
|
|
48
|
-
import type {
|
|
49
|
-
PlayRuntimeManifest,
|
|
50
|
-
PlayRuntimeManifestMap,
|
|
51
|
-
} from '../../../shared_libs/plays/compiler-manifest';
|
|
44
|
+
import type { PlayRuntimeManifest } from '../../../shared_libs/plays/compiler-manifest';
|
|
52
45
|
import {
|
|
53
46
|
WORKSPACE_STORAGE_NOT_READY_CODE,
|
|
54
47
|
WORKSPACE_STORAGE_NOT_READY_MESSAGE,
|
|
@@ -59,10 +52,6 @@ import {
|
|
|
59
52
|
COORDINATOR_INTERNAL_TOKEN_HEADER,
|
|
60
53
|
COORDINATOR_RUN_SCOPE_HEADER,
|
|
61
54
|
} from '../../../shared_libs/play-runtime/coordinator-headers';
|
|
62
|
-
import {
|
|
63
|
-
PLAY_RUNTIME_CONTRACT,
|
|
64
|
-
PLAY_RUNTIME_CONTRACT_HEADER,
|
|
65
|
-
} from '../../../shared_libs/play-runtime/runtime-contract';
|
|
66
55
|
import {
|
|
67
56
|
PLAY_RUNTIME_API_COMPAT_PATH,
|
|
68
57
|
PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH,
|
|
@@ -173,8 +162,6 @@ export type PlayWorkflowParams = {
|
|
|
173
162
|
}> | null;
|
|
174
163
|
contractSnapshot?: unknown;
|
|
175
164
|
executionPlan?: ExecutionPlan | null;
|
|
176
|
-
childPlayManifests?: PlayRuntimeManifestMap | null;
|
|
177
|
-
playCallGovernance?: PlayCallGovernanceSnapshot | null;
|
|
178
165
|
forceToolRefresh?: boolean | null;
|
|
179
166
|
preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
|
|
180
167
|
preloadedDbSessionRef?: {
|
|
@@ -185,8 +172,6 @@ export type PlayWorkflowParams = {
|
|
|
185
172
|
dynamicWorkerCode?: string | null;
|
|
186
173
|
executorToken: string;
|
|
187
174
|
baseUrl: string;
|
|
188
|
-
runtimeApiBaseUrl?: string | null;
|
|
189
|
-
callbackUrl?: string | null;
|
|
190
175
|
integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
|
|
191
176
|
orgId: string;
|
|
192
177
|
userEmail: string;
|
|
@@ -201,27 +186,6 @@ export type PlayWorkflowParams = {
|
|
|
201
186
|
submittedAt?: number | null;
|
|
202
187
|
};
|
|
203
188
|
|
|
204
|
-
type InlineChildSubmitResult =
|
|
205
|
-
| {
|
|
206
|
-
workflowId: string;
|
|
207
|
-
runId: string;
|
|
208
|
-
status: 'completed';
|
|
209
|
-
mode: 'inline_dynamic_worker';
|
|
210
|
-
output: unknown;
|
|
211
|
-
result: unknown;
|
|
212
|
-
logs: string[];
|
|
213
|
-
timings: CoordinatorTiming[];
|
|
214
|
-
}
|
|
215
|
-
| {
|
|
216
|
-
workflowId: string;
|
|
217
|
-
runId: string;
|
|
218
|
-
status: 'failed';
|
|
219
|
-
mode: 'inline_dynamic_worker';
|
|
220
|
-
error: { message: string; stack?: string | null };
|
|
221
|
-
logs: string[];
|
|
222
|
-
timings: CoordinatorTiming[];
|
|
223
|
-
};
|
|
224
|
-
|
|
225
189
|
type CoordinatorTiming = {
|
|
226
190
|
phase: string;
|
|
227
191
|
ms: number;
|
|
@@ -262,12 +226,6 @@ type CoordinatorTerminalState = {
|
|
|
262
226
|
completedAt?: number;
|
|
263
227
|
};
|
|
264
228
|
|
|
265
|
-
type CoordinatorChildTerminalState = {
|
|
266
|
-
eventKey: string;
|
|
267
|
-
data: unknown;
|
|
268
|
-
storedAt: number;
|
|
269
|
-
};
|
|
270
|
-
|
|
271
229
|
type CoordinatorRunEvent =
|
|
272
230
|
| {
|
|
273
231
|
seq?: number;
|
|
@@ -648,67 +606,6 @@ async function readCoordinatorTerminalState(
|
|
|
648
606
|
return state as CoordinatorTerminalState;
|
|
649
607
|
}
|
|
650
608
|
|
|
651
|
-
async function writeCoordinatorChildTerminalState(input: {
|
|
652
|
-
env: CoordinatorEnv;
|
|
653
|
-
parentRunId: string;
|
|
654
|
-
eventKey: string;
|
|
655
|
-
data: unknown;
|
|
656
|
-
}): Promise<void> {
|
|
657
|
-
const stub = input.env.PLAY_DEDUP.get(
|
|
658
|
-
input.env.PLAY_DEDUP.idFromName(input.parentRunId),
|
|
659
|
-
);
|
|
660
|
-
const response = await stub.fetch(
|
|
661
|
-
'https://deepline.dedup.internal/child-terminal-set',
|
|
662
|
-
{
|
|
663
|
-
method: 'POST',
|
|
664
|
-
headers: { 'content-type': 'application/json' },
|
|
665
|
-
body: JSON.stringify({
|
|
666
|
-
eventKey: input.eventKey,
|
|
667
|
-
data: coordinatorStorageRedactor.redact(input.data),
|
|
668
|
-
storedAt: Date.now(),
|
|
669
|
-
}),
|
|
670
|
-
},
|
|
671
|
-
);
|
|
672
|
-
if (!response.ok) {
|
|
673
|
-
throw new Error(`coordinator child terminal set failed ${response.status}`);
|
|
674
|
-
}
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
async function readCoordinatorChildTerminalState(input: {
|
|
678
|
-
env: CoordinatorEnv;
|
|
679
|
-
parentRunId: string;
|
|
680
|
-
eventKey: string;
|
|
681
|
-
timeoutMs?: number;
|
|
682
|
-
}): Promise<CoordinatorChildTerminalState | null> {
|
|
683
|
-
const stub = input.env.PLAY_DEDUP.get(
|
|
684
|
-
input.env.PLAY_DEDUP.idFromName(input.parentRunId),
|
|
685
|
-
);
|
|
686
|
-
const endpoint =
|
|
687
|
-
input.timeoutMs && input.timeoutMs > 0
|
|
688
|
-
? 'child-terminal-await'
|
|
689
|
-
: 'child-terminal-get';
|
|
690
|
-
const timeoutParam =
|
|
691
|
-
input.timeoutMs && input.timeoutMs > 0
|
|
692
|
-
? `&timeoutMs=${encodeURIComponent(String(Math.floor(input.timeoutMs)))}`
|
|
693
|
-
: '';
|
|
694
|
-
const response = await stub.fetch(
|
|
695
|
-
`https://deepline.dedup.internal/${endpoint}?eventKey=${encodeURIComponent(
|
|
696
|
-
input.eventKey,
|
|
697
|
-
)}${timeoutParam}`,
|
|
698
|
-
);
|
|
699
|
-
if (!response.ok) {
|
|
700
|
-
throw new Error(
|
|
701
|
-
`coordinator child terminal ${endpoint} failed ${response.status}`,
|
|
702
|
-
);
|
|
703
|
-
}
|
|
704
|
-
const body = (await response.json().catch(() => ({}))) as {
|
|
705
|
-
state?: unknown;
|
|
706
|
-
};
|
|
707
|
-
const state = body.state;
|
|
708
|
-
if (!isRecord(state) || state.eventKey !== input.eventKey) return null;
|
|
709
|
-
return state as CoordinatorChildTerminalState;
|
|
710
|
-
}
|
|
711
|
-
|
|
712
609
|
function workflowEventType(name: string): string {
|
|
713
610
|
const normalized = name
|
|
714
611
|
.trim()
|
|
@@ -1484,47 +1381,6 @@ async function markWorkflowRuntimeFailure(input: {
|
|
|
1484
1381
|
);
|
|
1485
1382
|
}
|
|
1486
1383
|
|
|
1487
|
-
async function appendWorkflowChildCompletedRunEvent(input: {
|
|
1488
|
-
env: CoordinatorEnv;
|
|
1489
|
-
event: unknown;
|
|
1490
|
-
result: unknown;
|
|
1491
|
-
}): Promise<boolean> {
|
|
1492
|
-
const payload = readWorkflowPayload(input.event);
|
|
1493
|
-
if (!payload || !isRecord(payload.playCallGovernance)) return false;
|
|
1494
|
-
const runId = typeof payload.runId === 'string' ? payload.runId : null;
|
|
1495
|
-
const baseUrl = typeof payload.baseUrl === 'string' ? payload.baseUrl : null;
|
|
1496
|
-
const executorToken =
|
|
1497
|
-
typeof payload.executorToken === 'string' ? payload.executorToken : null;
|
|
1498
|
-
if (!runId || !baseUrl || !executorToken) return false;
|
|
1499
|
-
const headers = new Headers({
|
|
1500
|
-
authorization: `Bearer ${executorToken}`,
|
|
1501
|
-
'content-type': 'application/json',
|
|
1502
|
-
});
|
|
1503
|
-
const bypass = input.env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim();
|
|
1504
|
-
if (bypass) headers.set('x-vercel-protection-bypass', bypass);
|
|
1505
|
-
const body = JSON.stringify({
|
|
1506
|
-
action: 'append_run_events',
|
|
1507
|
-
playId: runId,
|
|
1508
|
-
events: [
|
|
1509
|
-
{
|
|
1510
|
-
type: 'run.completed',
|
|
1511
|
-
runId,
|
|
1512
|
-
source: 'worker',
|
|
1513
|
-
occurredAt: Date.now(),
|
|
1514
|
-
result: input.result,
|
|
1515
|
-
} satisfies PlayRunLedgerEvent,
|
|
1516
|
-
],
|
|
1517
|
-
});
|
|
1518
|
-
const url = `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`;
|
|
1519
|
-
const response = await fetch(url, { method: 'POST', headers, body });
|
|
1520
|
-
if (!response.ok) {
|
|
1521
|
-
throw new Error(
|
|
1522
|
-
`runtime API responded ${response.status}: ${(await response.text().catch(() => '')).slice(0, 400)}`,
|
|
1523
|
-
);
|
|
1524
|
-
}
|
|
1525
|
-
return true;
|
|
1526
|
-
}
|
|
1527
|
-
|
|
1528
1384
|
type StoredPlayArtifactPayload = {
|
|
1529
1385
|
artifact?: {
|
|
1530
1386
|
bundledCode?: string;
|
|
@@ -1835,1460 +1691,95 @@ function workflowRetryInstanceId(runId: string, attempt: number): string {
|
|
|
1835
1691
|
|
|
1836
1692
|
function normalizeWorkflowRunAttempt(value: unknown): number {
|
|
1837
1693
|
return typeof value === 'number' && Number.isFinite(value)
|
|
1838
|
-
? Math.max(0, Math.floor(value))
|
|
1839
|
-
: 0;
|
|
1840
|
-
}
|
|
1841
|
-
|
|
1842
|
-
async function restartWorkflowAfterPlatformReset(input: {
|
|
1843
|
-
env: CoordinatorEnv;
|
|
1844
|
-
ctx?: ExecutionContext;
|
|
1845
|
-
runId: string;
|
|
1846
|
-
oldInstance: WorkflowInstance;
|
|
1847
|
-
error: string;
|
|
1848
|
-
status: InstanceStatus;
|
|
1849
|
-
}): Promise<{
|
|
1850
|
-
retried: boolean;
|
|
1851
|
-
result: Record<string, unknown> | null;
|
|
1852
|
-
}> {
|
|
1853
|
-
const decision = decideWorkflowPlatformRetry({
|
|
1854
|
-
workflowStatus: String(input.status.status ?? ''),
|
|
1855
|
-
error: input.error,
|
|
1856
|
-
retryAttempts: 0,
|
|
1857
|
-
});
|
|
1858
|
-
if (decision.action !== 'retry') {
|
|
1859
|
-
return { retried: false, result: null };
|
|
1860
|
-
}
|
|
1861
|
-
const claim = await claimWorkflowPlatformRetry({
|
|
1862
|
-
env: input.env,
|
|
1863
|
-
runId: input.runId,
|
|
1864
|
-
}).catch((error) => {
|
|
1865
|
-
recordCoordinatorEvent(
|
|
1866
|
-
'warn',
|
|
1867
|
-
'coordinator.workflow_platform_retry.claim_failed',
|
|
1868
|
-
'[coordinator] workflow platform retry claim failed',
|
|
1869
|
-
{
|
|
1870
|
-
runId: input.runId,
|
|
1871
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1872
|
-
},
|
|
1873
|
-
);
|
|
1874
|
-
return null;
|
|
1875
|
-
});
|
|
1876
|
-
if (!claim?.claimed || !claim.params) {
|
|
1877
|
-
return { retried: false, result: null };
|
|
1878
|
-
}
|
|
1879
|
-
const retryInstanceId = workflowRetryInstanceId(input.runId, claim.attempts);
|
|
1880
|
-
const retryStartedAt = Date.now();
|
|
1881
|
-
let retryInstance: WorkflowInstance | null = null;
|
|
1882
|
-
const retryParams = { ...claim.params, runAttempt: claim.attempts };
|
|
1883
|
-
try {
|
|
1884
|
-
retryInstance = await createDynamicWorkflowInstance({
|
|
1885
|
-
env: input.env,
|
|
1886
|
-
id: retryInstanceId,
|
|
1887
|
-
params: workflowCreateParams(retryParams, {
|
|
1888
|
-
params: retryParams,
|
|
1889
|
-
paramsRef: claim.paramsRef,
|
|
1890
|
-
}),
|
|
1891
|
-
metadataParams: retryParams,
|
|
1892
|
-
});
|
|
1893
|
-
await recordWorkflowInstanceId({
|
|
1894
|
-
env: input.env,
|
|
1895
|
-
runId: input.runId,
|
|
1896
|
-
instanceId: retryInstance.id,
|
|
1897
|
-
});
|
|
1898
|
-
input.ctx?.waitUntil(input.oldInstance.terminate().catch(() => undefined));
|
|
1899
|
-
recordCoordinatorPerfTraceBuffered(input.env, input.ctx, {
|
|
1900
|
-
runId: input.runId,
|
|
1901
|
-
phase: 'coordinator.platform_deploy_retry',
|
|
1902
|
-
ms: Date.now() - retryStartedAt,
|
|
1903
|
-
graphHash: claim.params.graphHash ?? null,
|
|
1904
|
-
extra: {
|
|
1905
|
-
retryAttempt: claim.attempts,
|
|
1906
|
-
retryInstanceId: retryInstance.id,
|
|
1907
|
-
reason: decision.reason,
|
|
1908
|
-
},
|
|
1909
|
-
});
|
|
1910
|
-
return {
|
|
1911
|
-
retried: true,
|
|
1912
|
-
result: {
|
|
1913
|
-
runId: input.runId,
|
|
1914
|
-
playName: claim.params.playName,
|
|
1915
|
-
status: 'running',
|
|
1916
|
-
result: null,
|
|
1917
|
-
error: null,
|
|
1918
|
-
retry: {
|
|
1919
|
-
reason: decision.reason,
|
|
1920
|
-
attempt: claim.attempts,
|
|
1921
|
-
message: decision.message,
|
|
1922
|
-
},
|
|
1923
|
-
},
|
|
1924
|
-
};
|
|
1925
|
-
} finally {
|
|
1926
|
-
disposeRpcStub(retryInstance);
|
|
1927
|
-
}
|
|
1928
|
-
}
|
|
1929
|
-
|
|
1930
|
-
async function markRegisteredChildRunFailed(input: {
|
|
1931
|
-
env: CoordinatorEnv;
|
|
1932
|
-
baseUrl: string;
|
|
1933
|
-
childExecutorToken: string;
|
|
1934
|
-
childRunId: string;
|
|
1935
|
-
error: unknown;
|
|
1936
|
-
}): Promise<void> {
|
|
1937
|
-
const message =
|
|
1938
|
-
input.error instanceof Error ? input.error.message : String(input.error);
|
|
1939
|
-
const response = await input.env.HARNESS.runtimeApiCall({
|
|
1940
|
-
contract: PLAY_RUNTIME_CONTRACT,
|
|
1941
|
-
executorToken: input.childExecutorToken,
|
|
1942
|
-
baseUrl: input.baseUrl,
|
|
1943
|
-
path: PLAY_RUNTIME_API_COMPAT_PATH,
|
|
1944
|
-
headers: {
|
|
1945
|
-
'x-deepline-request-id': crypto.randomUUID(),
|
|
1946
|
-
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
1947
|
-
},
|
|
1948
|
-
timeoutMs: 15_000,
|
|
1949
|
-
body: {
|
|
1950
|
-
action: 'append_run_events',
|
|
1951
|
-
playId: input.childRunId,
|
|
1952
|
-
events: [
|
|
1953
|
-
{
|
|
1954
|
-
type: 'run.failed',
|
|
1955
|
-
runId: input.childRunId,
|
|
1956
|
-
source: 'coordinator',
|
|
1957
|
-
occurredAt: Date.now(),
|
|
1958
|
-
error: `Child workflow submit failed: ${message}`,
|
|
1959
|
-
} satisfies PlayRunLedgerEvent,
|
|
1960
|
-
],
|
|
1961
|
-
},
|
|
1962
|
-
});
|
|
1963
|
-
if (response.status < 200 || response.status >= 300) {
|
|
1964
|
-
const text = response.body ?? '';
|
|
1965
|
-
throw new Error(
|
|
1966
|
-
`Inline child run failure mark failed ${response.status}: ${text.slice(0, 800)}`,
|
|
1967
|
-
);
|
|
1968
|
-
}
|
|
1969
|
-
}
|
|
1970
|
-
|
|
1971
|
-
async function appendRegisteredChildRunTerminal(input: {
|
|
1972
|
-
env: CoordinatorEnv;
|
|
1973
|
-
baseUrl: string;
|
|
1974
|
-
childExecutorToken: string;
|
|
1975
|
-
childRunId: string;
|
|
1976
|
-
status: 'completed' | 'failed';
|
|
1977
|
-
result?: unknown;
|
|
1978
|
-
error?: string | null;
|
|
1979
|
-
}): Promise<void> {
|
|
1980
|
-
const event =
|
|
1981
|
-
input.status === 'completed'
|
|
1982
|
-
? ({
|
|
1983
|
-
type: 'run.completed',
|
|
1984
|
-
runId: input.childRunId,
|
|
1985
|
-
source: 'worker',
|
|
1986
|
-
occurredAt: Date.now(),
|
|
1987
|
-
...(input.result !== undefined ? { result: input.result } : {}),
|
|
1988
|
-
} satisfies PlayRunLedgerEvent)
|
|
1989
|
-
: ({
|
|
1990
|
-
type: 'run.failed',
|
|
1991
|
-
runId: input.childRunId,
|
|
1992
|
-
source: 'worker',
|
|
1993
|
-
occurredAt: Date.now(),
|
|
1994
|
-
error: input.error ?? 'Inline child Worker failed.',
|
|
1995
|
-
} satisfies PlayRunLedgerEvent);
|
|
1996
|
-
const response = await input.env.HARNESS.runtimeApiCall({
|
|
1997
|
-
contract: PLAY_RUNTIME_CONTRACT,
|
|
1998
|
-
executorToken: input.childExecutorToken,
|
|
1999
|
-
baseUrl: input.baseUrl,
|
|
2000
|
-
path: PLAY_RUNTIME_API_COMPAT_PATH,
|
|
2001
|
-
headers: {
|
|
2002
|
-
'x-deepline-request-id': crypto.randomUUID(),
|
|
2003
|
-
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
2004
|
-
},
|
|
2005
|
-
timeoutMs: 15_000,
|
|
2006
|
-
body: {
|
|
2007
|
-
action: 'append_run_events',
|
|
2008
|
-
playId: input.childRunId,
|
|
2009
|
-
events: [event],
|
|
2010
|
-
},
|
|
2011
|
-
});
|
|
2012
|
-
if (response.status < 200 || response.status >= 300) {
|
|
2013
|
-
const text = response.body ?? '';
|
|
2014
|
-
throw new Error(
|
|
2015
|
-
`Inline child terminal append failed ${response.status}: ${text.slice(0, 800)}`,
|
|
2016
|
-
);
|
|
2017
|
-
}
|
|
2018
|
-
}
|
|
2019
|
-
|
|
2020
|
-
type CoordinatorRuntimeApiTiming = {
|
|
2021
|
-
phase: string;
|
|
2022
|
-
ms: number;
|
|
2023
|
-
bytes?: number;
|
|
2024
|
-
};
|
|
2025
|
-
|
|
2026
|
-
async function callRuntimeApiFromCoordinator(input: {
|
|
2027
|
-
env: CoordinatorEnv;
|
|
2028
|
-
baseUrl: string;
|
|
2029
|
-
executorToken: string;
|
|
2030
|
-
body: unknown;
|
|
2031
|
-
}): Promise<{
|
|
2032
|
-
status: number;
|
|
2033
|
-
body: string;
|
|
2034
|
-
timings: CoordinatorRuntimeApiTiming[];
|
|
2035
|
-
}> {
|
|
2036
|
-
const timings: CoordinatorRuntimeApiTiming[] = [];
|
|
2037
|
-
const totalStartedAt = Date.now();
|
|
2038
|
-
const recordTiming = (
|
|
2039
|
-
phase: string,
|
|
2040
|
-
startedAt: number,
|
|
2041
|
-
extra?: { bytes?: number },
|
|
2042
|
-
): void => {
|
|
2043
|
-
timings.push({
|
|
2044
|
-
phase,
|
|
2045
|
-
ms: Date.now() - startedAt,
|
|
2046
|
-
...(extra?.bytes !== undefined ? { bytes: extra.bytes } : {}),
|
|
2047
|
-
});
|
|
2048
|
-
};
|
|
2049
|
-
|
|
2050
|
-
const buildStartedAt = Date.now();
|
|
2051
|
-
const body = input.body ?? {};
|
|
2052
|
-
const serializedBody = JSON.stringify(body);
|
|
2053
|
-
recordTiming('coordinator.runtime_api.build_request', buildStartedAt, {
|
|
2054
|
-
bytes: serializedBody.length,
|
|
2055
|
-
});
|
|
2056
|
-
|
|
2057
|
-
const fetchStartedAt = Date.now();
|
|
2058
|
-
const requestId = crypto.randomUUID();
|
|
2059
|
-
const action =
|
|
2060
|
-
body && typeof body === 'object' && !Array.isArray(body)
|
|
2061
|
-
? String((body as { action?: unknown }).action ?? 'unknown')
|
|
2062
|
-
: 'unknown';
|
|
2063
|
-
const runId =
|
|
2064
|
-
body && typeof body === 'object' && !Array.isArray(body)
|
|
2065
|
-
? String(
|
|
2066
|
-
(body as { runId?: unknown; playId?: unknown }).runId ??
|
|
2067
|
-
(body as { runId?: unknown; playId?: unknown }).playId ??
|
|
2068
|
-
'',
|
|
2069
|
-
) || null
|
|
2070
|
-
: null;
|
|
2071
|
-
recordCoordinatorEvent(
|
|
2072
|
-
'info',
|
|
2073
|
-
'coordinator.runtime_api.breadcrumb',
|
|
2074
|
-
'[coordinator.runtime_api.breadcrumb]',
|
|
2075
|
-
{
|
|
2076
|
-
phase: 'request',
|
|
2077
|
-
action,
|
|
2078
|
-
runId,
|
|
2079
|
-
requestId,
|
|
2080
|
-
baseOrigin: safeOrigin(input.baseUrl),
|
|
2081
|
-
},
|
|
2082
|
-
);
|
|
2083
|
-
const response = await input.env.HARNESS.runtimeApiCall({
|
|
2084
|
-
contract: PLAY_RUNTIME_CONTRACT,
|
|
2085
|
-
executorToken: input.executorToken,
|
|
2086
|
-
baseUrl: input.baseUrl,
|
|
2087
|
-
path: PLAY_RUNTIME_API_COMPAT_PATH,
|
|
2088
|
-
body,
|
|
2089
|
-
headers: {
|
|
2090
|
-
'x-deepline-request-id': requestId,
|
|
2091
|
-
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
2092
|
-
},
|
|
2093
|
-
});
|
|
2094
|
-
recordTiming('coordinator.runtime_api.fetch', fetchStartedAt);
|
|
2095
|
-
recordCoordinatorEvent(
|
|
2096
|
-
'info',
|
|
2097
|
-
'coordinator.runtime_api.breadcrumb',
|
|
2098
|
-
'[coordinator.runtime_api.breadcrumb]',
|
|
2099
|
-
{
|
|
2100
|
-
phase: 'response',
|
|
2101
|
-
action,
|
|
2102
|
-
runId,
|
|
2103
|
-
requestId,
|
|
2104
|
-
status: response.status,
|
|
2105
|
-
ms: Date.now() - fetchStartedAt,
|
|
2106
|
-
baseOrigin: safeOrigin(input.baseUrl),
|
|
2107
|
-
},
|
|
2108
|
-
);
|
|
2109
|
-
|
|
2110
|
-
const bodyStartedAt = Date.now();
|
|
2111
|
-
const responseBody = response.body;
|
|
2112
|
-
recordTiming('coordinator.runtime_api.body', bodyStartedAt, {
|
|
2113
|
-
bytes: responseBody.length,
|
|
2114
|
-
});
|
|
2115
|
-
recordTiming('coordinator.runtime_api.total', totalStartedAt);
|
|
2116
|
-
return {
|
|
2117
|
-
status: response.status,
|
|
2118
|
-
body: responseBody,
|
|
2119
|
-
timings,
|
|
2120
|
-
};
|
|
2121
|
-
}
|
|
2122
|
-
|
|
2123
|
-
async function prepareInlineChildRunWithRuntime(input: {
|
|
2124
|
-
env: CoordinatorEnv;
|
|
2125
|
-
baseUrl: string;
|
|
2126
|
-
parentExecutorToken: string;
|
|
2127
|
-
parentRunId: string;
|
|
2128
|
-
parentPlayName: string;
|
|
2129
|
-
childRunId: string;
|
|
2130
|
-
childPlayName: string;
|
|
2131
|
-
manifest: PlayRuntimeManifest;
|
|
2132
|
-
governance: PlayCallGovernanceSnapshot;
|
|
2133
|
-
userEmail: string;
|
|
2134
|
-
runtimeBackend?: string;
|
|
2135
|
-
schedulerBackend?: string;
|
|
2136
|
-
executionProfile?: string;
|
|
2137
|
-
}): Promise<{
|
|
2138
|
-
childToken: string;
|
|
2139
|
-
preloadedDbSessions: PreloadedRuntimeDbSession[];
|
|
2140
|
-
prepareTimings: unknown[];
|
|
2141
|
-
transportTimings: unknown[];
|
|
2142
|
-
}> {
|
|
2143
|
-
const response = await callRuntimeApiFromCoordinator({
|
|
2144
|
-
env: input.env,
|
|
2145
|
-
baseUrl: input.baseUrl,
|
|
2146
|
-
executorToken: input.parentExecutorToken,
|
|
2147
|
-
body: {
|
|
2148
|
-
action: 'prepare_inline_child_run',
|
|
2149
|
-
parentRunId: input.parentRunId,
|
|
2150
|
-
rootRunId: input.governance.rootRunId,
|
|
2151
|
-
parentPlayName: input.parentPlayName,
|
|
2152
|
-
childRunId: input.childRunId,
|
|
2153
|
-
childPlayName: input.childPlayName,
|
|
2154
|
-
workflowFamilyKey:
|
|
2155
|
-
input.governance.rootRunId ??
|
|
2156
|
-
input.governance.parentRunId ??
|
|
2157
|
-
input.childRunId,
|
|
2158
|
-
artifactStorageKey: input.manifest.artifactStorageKey,
|
|
2159
|
-
artifactHash: input.manifest.artifactHash,
|
|
2160
|
-
graphHash: input.manifest.graphHash,
|
|
2161
|
-
runtimeBackend: input.runtimeBackend ?? 'workers_edge',
|
|
2162
|
-
schedulerBackend: input.schedulerBackend ?? 'inline_child',
|
|
2163
|
-
executionProfile: input.executionProfile ?? 'workers_edge',
|
|
2164
|
-
...(typeof input.manifest.maxCreditsPerRun === 'number'
|
|
2165
|
-
? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
|
|
2166
|
-
: {}),
|
|
2167
|
-
staticPipeline: input.manifest.staticPipeline ?? null,
|
|
2168
|
-
source: 'published',
|
|
2169
|
-
userEmail: input.userEmail,
|
|
2170
|
-
},
|
|
2171
|
-
});
|
|
2172
|
-
const text = response.body;
|
|
2173
|
-
let parsed: unknown = {};
|
|
2174
|
-
try {
|
|
2175
|
-
parsed = text ? JSON.parse(text) : {};
|
|
2176
|
-
} catch {
|
|
2177
|
-
parsed = {};
|
|
2178
|
-
}
|
|
2179
|
-
if (response.status < 200 || response.status >= 300) {
|
|
2180
|
-
throw new Error(
|
|
2181
|
-
`Inline child prepare failed ${response.status}: ${text.slice(0, 800)}`,
|
|
2182
|
-
);
|
|
2183
|
-
}
|
|
2184
|
-
if (!isRecord(parsed) || typeof parsed.executorToken !== 'string') {
|
|
2185
|
-
throw new Error('Inline child prepare response was missing executorToken.');
|
|
2186
|
-
}
|
|
2187
|
-
const preloadedDbSessions = Array.isArray(parsed.preloadedDbSessions)
|
|
2188
|
-
? (parsed.preloadedDbSessions as PreloadedRuntimeDbSession[])
|
|
2189
|
-
: [];
|
|
2190
|
-
const prepareTimings = Array.isArray(parsed.prepareTimings)
|
|
2191
|
-
? parsed.prepareTimings
|
|
2192
|
-
: [];
|
|
2193
|
-
return {
|
|
2194
|
-
childToken: parsed.executorToken,
|
|
2195
|
-
preloadedDbSessions,
|
|
2196
|
-
prepareTimings,
|
|
2197
|
-
transportTimings: response.timings,
|
|
2198
|
-
};
|
|
2199
|
-
}
|
|
2200
|
-
|
|
2201
|
-
function buildChildRunId(input: {
|
|
2202
|
-
playName: string;
|
|
2203
|
-
parentRunId: string;
|
|
2204
|
-
body: Record<string, unknown>;
|
|
2205
|
-
}): string {
|
|
2206
|
-
const { playName, parentRunId, body } = input;
|
|
2207
|
-
const governance = isRecord(body.internalRunPlay)
|
|
2208
|
-
? body.internalRunPlay
|
|
2209
|
-
: null;
|
|
2210
|
-
// Adapt the coordinator's on-the-wire child-submit body to the shared
|
|
2211
|
-
// derivation. Output stays byte-identical to the previous inline
|
|
2212
|
-
// implementation (same fields → same digest → same run id), so child-submit
|
|
2213
|
-
// idempotency is preserved. See shared_libs/play-runtime/child-run-id.ts.
|
|
2214
|
-
return buildSharedChildRunId({
|
|
2215
|
-
childPlayName: playName,
|
|
2216
|
-
parentRunId,
|
|
2217
|
-
explicitIdempotencyKey:
|
|
2218
|
-
typeof body.childIdempotencyKey === 'string'
|
|
2219
|
-
? body.childIdempotencyKey
|
|
2220
|
-
: null,
|
|
2221
|
-
parentPlayName:
|
|
2222
|
-
typeof governance?.parentPlayName === 'string'
|
|
2223
|
-
? governance.parentPlayName
|
|
2224
|
-
: null,
|
|
2225
|
-
key: typeof governance?.key === 'string' ? governance.key : null,
|
|
2226
|
-
input: isRecord(body.input) || Array.isArray(body.input) ? body.input : {},
|
|
2227
|
-
graphHash: isRecord(body.manifest)
|
|
2228
|
-
? typeof body.manifest.graphHash === 'string'
|
|
2229
|
-
? body.manifest.graphHash
|
|
2230
|
-
: null
|
|
2231
|
-
: null,
|
|
2232
|
-
});
|
|
2233
|
-
}
|
|
2234
|
-
|
|
2235
|
-
function normalizeRuntimeBaseUrl(value: unknown): string | null {
|
|
2236
|
-
if (typeof value !== 'string') return null;
|
|
2237
|
-
const trimmed = value.trim();
|
|
2238
|
-
if (!trimmed) return null;
|
|
2239
|
-
let parsed: URL;
|
|
2240
|
-
try {
|
|
2241
|
-
parsed = new URL(trimmed);
|
|
2242
|
-
} catch {
|
|
2243
|
-
return null;
|
|
2244
|
-
}
|
|
2245
|
-
if (
|
|
2246
|
-
(parsed.protocol !== 'http:' && parsed.protocol !== 'https:') ||
|
|
2247
|
-
parsed.username ||
|
|
2248
|
-
parsed.password ||
|
|
2249
|
-
parsed.search ||
|
|
2250
|
-
parsed.hash ||
|
|
2251
|
-
(parsed.pathname !== '/' && parsed.pathname !== '')
|
|
2252
|
-
) {
|
|
2253
|
-
return null;
|
|
2254
|
-
}
|
|
2255
|
-
return parsed.toString().replace(/\/$/, '');
|
|
2256
|
-
}
|
|
2257
|
-
|
|
2258
|
-
const DEEPLINE_PRODUCTION_APP_HOST = 'code.deepline.com';
|
|
2259
|
-
const DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE =
|
|
2260
|
-
/^deepline-api(?:-[a-z0-9-]+)?-aero-team-0f650658\.vercel\.app$/i;
|
|
2261
|
-
const DEEPLINE_VERCEL_PROJECT_HOST_RE =
|
|
2262
|
-
/^deepline-[a-z0-9]+-aero-team-0f650658\.vercel\.app$/i;
|
|
2263
|
-
const DEEPLINE_WORKERS_DEV_SUBDOMAINS = new Set(['chirag-d94']);
|
|
2264
|
-
|
|
2265
|
-
function isDeeplineCoordinatorWorkerHost(hostname: string): boolean {
|
|
2266
|
-
const labels = hostname.toLowerCase().split('.');
|
|
2267
|
-
return (
|
|
2268
|
-
labels.length === 4 &&
|
|
2269
|
-
labels[0]?.startsWith('deepline-play-coordinator-') &&
|
|
2270
|
-
labels[2] === 'workers' &&
|
|
2271
|
-
labels[3] === 'dev' &&
|
|
2272
|
-
DEEPLINE_WORKERS_DEV_SUBDOMAINS.has(labels[1] ?? '')
|
|
2273
|
-
);
|
|
2274
|
-
}
|
|
2275
|
-
|
|
2276
|
-
function shouldCanonicalizeRuntimeApiBaseUrl(input: {
|
|
2277
|
-
configuredBaseUrl: URL;
|
|
2278
|
-
overrideBaseUrl: URL;
|
|
2279
|
-
}): boolean {
|
|
2280
|
-
if (input.overrideBaseUrl.host === input.configuredBaseUrl.host) {
|
|
2281
|
-
return false;
|
|
2282
|
-
}
|
|
2283
|
-
const configuredHostname = input.configuredBaseUrl.hostname.toLowerCase();
|
|
2284
|
-
const overrideHostname = input.overrideBaseUrl.hostname.toLowerCase();
|
|
2285
|
-
if (isDeeplineCoordinatorWorkerHost(overrideHostname)) {
|
|
2286
|
-
return true;
|
|
2287
|
-
}
|
|
2288
|
-
const overrideIsDeeplineVercelHost =
|
|
2289
|
-
DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE.test(overrideHostname) ||
|
|
2290
|
-
DEEPLINE_VERCEL_PROJECT_HOST_RE.test(overrideHostname);
|
|
2291
|
-
if (!overrideIsDeeplineVercelHost) {
|
|
2292
|
-
return false;
|
|
2293
|
-
}
|
|
2294
|
-
const configuredIsDeeplineVercelHost =
|
|
2295
|
-
DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE.test(configuredHostname) ||
|
|
2296
|
-
DEEPLINE_VERCEL_PROJECT_HOST_RE.test(configuredHostname);
|
|
2297
|
-
return (
|
|
2298
|
-
configuredHostname === DEEPLINE_PRODUCTION_APP_HOST ||
|
|
2299
|
-
!configuredIsDeeplineVercelHost
|
|
2300
|
-
);
|
|
2301
|
-
}
|
|
2302
|
-
|
|
2303
|
-
function resolveRuntimeApiBaseUrl(
|
|
2304
|
-
env: CoordinatorEnv,
|
|
2305
|
-
body: Record<string, unknown>,
|
|
2306
|
-
): string {
|
|
2307
|
-
const configuredBaseUrl =
|
|
2308
|
-
normalizeRuntimeBaseUrl(env.DEEPLINE_API_BASE_URL) ??
|
|
2309
|
-
env.DEEPLINE_API_BASE_URL.replace(/\/$/, '');
|
|
2310
|
-
const overrideBaseUrl =
|
|
2311
|
-
normalizeRuntimeBaseUrl(body.runtimeApiBaseUrl) ??
|
|
2312
|
-
normalizeRuntimeBaseUrl(body.baseUrl);
|
|
2313
|
-
if (!overrideBaseUrl) {
|
|
2314
|
-
return configuredBaseUrl;
|
|
2315
|
-
}
|
|
2316
|
-
if (
|
|
2317
|
-
shouldCanonicalizeRuntimeApiBaseUrl({
|
|
2318
|
-
configuredBaseUrl: new URL(configuredBaseUrl),
|
|
2319
|
-
overrideBaseUrl: new URL(overrideBaseUrl),
|
|
2320
|
-
})
|
|
2321
|
-
) {
|
|
2322
|
-
return configuredBaseUrl;
|
|
2323
|
-
}
|
|
2324
|
-
return overrideBaseUrl;
|
|
2325
|
-
}
|
|
2326
|
-
|
|
2327
|
-
function resolveWorkflowCallbackBaseUrl(
|
|
2328
|
-
env: CoordinatorEnv,
|
|
2329
|
-
body: Record<string, unknown>,
|
|
2330
|
-
): string {
|
|
2331
|
-
return (
|
|
2332
|
-
normalizeRuntimeBaseUrl(body.callbackBaseUrl) ??
|
|
2333
|
-
normalizeRuntimeBaseUrl(body.callbackUrl) ??
|
|
2334
|
-
normalizeRuntimeBaseUrl(body.baseUrl) ??
|
|
2335
|
-
env.DEEPLINE_API_BASE_URL.replace(/\/$/, '')
|
|
2336
|
-
);
|
|
2337
|
-
}
|
|
2338
|
-
|
|
2339
|
-
function validateChildSubmitBody(input: {
|
|
2340
|
-
parentRunId: string;
|
|
2341
|
-
body: Record<string, unknown>;
|
|
2342
|
-
}):
|
|
2343
|
-
| {
|
|
2344
|
-
ok: true;
|
|
2345
|
-
manifest: PlayRuntimeManifest;
|
|
2346
|
-
governance: PlayCallGovernanceSnapshot;
|
|
2347
|
-
childPlayName: string;
|
|
2348
|
-
orgId: string;
|
|
2349
|
-
parentExecutorToken: string;
|
|
2350
|
-
}
|
|
2351
|
-
| { ok: false; status: number; error: Record<string, unknown> } {
|
|
2352
|
-
const { parentRunId, body } = input;
|
|
2353
|
-
const manifest = body.manifest as PlayRuntimeManifest | undefined;
|
|
2354
|
-
const governance = body.internalRunPlay as
|
|
2355
|
-
| PlayCallGovernanceSnapshot
|
|
2356
|
-
| undefined;
|
|
2357
|
-
const childPlayName =
|
|
2358
|
-
typeof body.name === 'string' && body.name.trim()
|
|
2359
|
-
? body.name.trim()
|
|
2360
|
-
: manifest?.playName?.trim();
|
|
2361
|
-
if (
|
|
2362
|
-
!manifest ||
|
|
2363
|
-
!childPlayName ||
|
|
2364
|
-
!manifest.artifactStorageKey ||
|
|
2365
|
-
!manifest.artifactHash ||
|
|
2366
|
-
!manifest.graphHash ||
|
|
2367
|
-
!governance
|
|
2368
|
-
) {
|
|
2369
|
-
return {
|
|
2370
|
-
ok: false,
|
|
2371
|
-
status: 400,
|
|
2372
|
-
error: {
|
|
2373
|
-
code: 'CHILD_MANIFEST_REQUIRED',
|
|
2374
|
-
message: 'submit-child requires a trusted child manifest and lineage.',
|
|
2375
|
-
phase: 'coordinator_child_submit',
|
|
2376
|
-
parentRunId,
|
|
2377
|
-
},
|
|
2378
|
-
};
|
|
2379
|
-
}
|
|
2380
|
-
const orgId = typeof body.orgId === 'string' ? body.orgId : '';
|
|
2381
|
-
if (!orgId) {
|
|
2382
|
-
return {
|
|
2383
|
-
ok: false,
|
|
2384
|
-
status: 400,
|
|
2385
|
-
error: {
|
|
2386
|
-
code: 'CHILD_ORG_REQUIRED',
|
|
2387
|
-
message: 'submit-child requires orgId from the parent runtime.',
|
|
2388
|
-
phase: 'coordinator_child_submit',
|
|
2389
|
-
parentRunId,
|
|
2390
|
-
},
|
|
2391
|
-
};
|
|
2392
|
-
}
|
|
2393
|
-
const parentExecutorToken =
|
|
2394
|
-
typeof body.parentExecutorToken === 'string'
|
|
2395
|
-
? body.parentExecutorToken.trim()
|
|
2396
|
-
: '';
|
|
2397
|
-
if (!parentExecutorToken) {
|
|
2398
|
-
return {
|
|
2399
|
-
ok: false,
|
|
2400
|
-
status: 400,
|
|
2401
|
-
error: {
|
|
2402
|
-
code: 'PARENT_EXECUTOR_TOKEN_REQUIRED',
|
|
2403
|
-
message:
|
|
2404
|
-
'submit-child requires the parent executor token for origin-scoped child token minting.',
|
|
2405
|
-
phase: 'coordinator_child_submit',
|
|
2406
|
-
parentRunId,
|
|
2407
|
-
},
|
|
2408
|
-
};
|
|
2409
|
-
}
|
|
2410
|
-
return {
|
|
2411
|
-
ok: true,
|
|
2412
|
-
manifest,
|
|
2413
|
-
governance,
|
|
2414
|
-
childPlayName,
|
|
2415
|
-
orgId,
|
|
2416
|
-
parentExecutorToken,
|
|
2417
|
-
};
|
|
2418
|
-
}
|
|
2419
|
-
|
|
2420
|
-
function buildChildWorkflowParams(input: {
|
|
2421
|
-
env: CoordinatorEnv;
|
|
2422
|
-
body: Record<string, unknown>;
|
|
2423
|
-
manifest: PlayRuntimeManifest;
|
|
2424
|
-
governance: PlayCallGovernanceSnapshot;
|
|
2425
|
-
childRunId: string;
|
|
2426
|
-
childPlayName: string;
|
|
2427
|
-
childToken: string;
|
|
2428
|
-
orgId: string;
|
|
2429
|
-
coordinatorUrl: string | null;
|
|
2430
|
-
runtimeBackend: string;
|
|
2431
|
-
dynamicWorkerCode: string | null;
|
|
2432
|
-
preloadedDbSessions: PreloadedRuntimeDbSession[] | null;
|
|
2433
|
-
}): PlayWorkflowParams {
|
|
2434
|
-
const {
|
|
2435
|
-
env,
|
|
2436
|
-
body,
|
|
2437
|
-
manifest,
|
|
2438
|
-
governance,
|
|
2439
|
-
childRunId,
|
|
2440
|
-
childPlayName,
|
|
2441
|
-
childToken,
|
|
2442
|
-
orgId,
|
|
2443
|
-
coordinatorUrl,
|
|
2444
|
-
runtimeBackend,
|
|
2445
|
-
dynamicWorkerCode,
|
|
2446
|
-
preloadedDbSessions,
|
|
2447
|
-
} = input;
|
|
2448
|
-
const baseUrl = resolveRuntimeApiBaseUrl(env, body);
|
|
2449
|
-
const callbackUrl = resolveWorkflowCallbackBaseUrl(env, body);
|
|
2450
|
-
return {
|
|
2451
|
-
runId: childRunId,
|
|
2452
|
-
playId: childRunId,
|
|
2453
|
-
playName: childPlayName,
|
|
2454
|
-
artifactStorageKey: manifest.artifactStorageKey,
|
|
2455
|
-
artifactHash: manifest.artifactHash,
|
|
2456
|
-
graphHash: manifest.graphHash,
|
|
2457
|
-
input: isRecord(body.input) || Array.isArray(body.input) ? body.input : {},
|
|
2458
|
-
contractSnapshot: {
|
|
2459
|
-
source: 'published',
|
|
2460
|
-
revisionVersion: null,
|
|
2461
|
-
staticPipeline: manifest.staticPipeline,
|
|
2462
|
-
billingLimit:
|
|
2463
|
-
typeof manifest.maxCreditsPerRun === 'number'
|
|
2464
|
-
? { maxCreditsPerRun: manifest.maxCreditsPerRun }
|
|
2465
|
-
: null,
|
|
2466
|
-
sourceCode: manifest.sourceCode ?? '',
|
|
2467
|
-
artifactMetadata: {
|
|
2468
|
-
storageKey: manifest.artifactStorageKey,
|
|
2469
|
-
artifactHash: manifest.artifactHash,
|
|
2470
|
-
graphHash: manifest.graphHash,
|
|
2471
|
-
},
|
|
2472
|
-
codeFormat: 'cjs_module',
|
|
2473
|
-
compatibility: {
|
|
2474
|
-
apiVersion: 2,
|
|
2475
|
-
runtimeBackend: 'workers_edge',
|
|
2476
|
-
},
|
|
2477
|
-
},
|
|
2478
|
-
executionPlan: null,
|
|
2479
|
-
childPlayManifests: isRecord(body.childPlayManifests)
|
|
2480
|
-
? (body.childPlayManifests as PlayRuntimeManifestMap)
|
|
2481
|
-
: null,
|
|
2482
|
-
playCallGovernance: governance,
|
|
2483
|
-
preloadedDbSessions,
|
|
2484
|
-
dynamicWorkerCode,
|
|
2485
|
-
executorToken: childToken,
|
|
2486
|
-
baseUrl: callbackUrl,
|
|
2487
|
-
runtimeApiBaseUrl: baseUrl,
|
|
2488
|
-
callbackUrl,
|
|
2489
|
-
integrationMode: normalizeIntegrationMode(body.integrationMode),
|
|
2490
|
-
force: body.force === true || body.forceToolRefresh === true,
|
|
2491
|
-
forceToolRefresh: body.forceToolRefresh === true,
|
|
2492
|
-
orgId,
|
|
2493
|
-
userEmail: typeof body.userEmail === 'string' ? body.userEmail : '',
|
|
2494
|
-
userId: typeof body.userId === 'string' ? body.userId : null,
|
|
2495
|
-
runtimeBackend,
|
|
2496
|
-
dedupBackend: 'in_memory',
|
|
2497
|
-
coordinatorUrl,
|
|
2498
|
-
coordinatorInternalToken: env.DEEPLINE_INTERNAL_TOKEN?.trim() || null,
|
|
2499
|
-
};
|
|
2500
|
-
}
|
|
2501
|
-
|
|
2502
|
-
function runRequestFromPlayWorkflowParams(
|
|
2503
|
-
params: PlayWorkflowParams,
|
|
2504
|
-
): Record<string, unknown> {
|
|
2505
|
-
const inputFileName = String(
|
|
2506
|
-
params.inputFile?.name ??
|
|
2507
|
-
params.inputFile?.fileName ??
|
|
2508
|
-
params.inputFile?.logicalPath ??
|
|
2509
|
-
params.inputFile?.path ??
|
|
2510
|
-
'',
|
|
2511
|
-
);
|
|
2512
|
-
const inputStorageKey =
|
|
2513
|
-
params.inputFile?.r2Key ?? params.inputFile?.storageKey ?? null;
|
|
2514
|
-
return {
|
|
2515
|
-
runId: params.runId,
|
|
2516
|
-
callbackUrl: params.callbackUrl ?? params.baseUrl,
|
|
2517
|
-
executorToken: params.executorToken,
|
|
2518
|
-
baseUrl: params.runtimeApiBaseUrl ?? params.baseUrl,
|
|
2519
|
-
integrationMode: normalizeIntegrationMode(params.integrationMode),
|
|
2520
|
-
orgId: params.orgId,
|
|
2521
|
-
playName: params.playName,
|
|
2522
|
-
graphHash: params.graphHash,
|
|
2523
|
-
userEmail: params.userEmail,
|
|
2524
|
-
runtimeInput: params.input,
|
|
2525
|
-
runAttempt: normalizeWorkflowRunAttempt(params.runAttempt),
|
|
2526
|
-
inlineCsv: params.inlineCsv ?? null,
|
|
2527
|
-
inputFiles:
|
|
2528
|
-
inputStorageKey && inputFileName
|
|
2529
|
-
? {
|
|
2530
|
-
[inputFileName]: {
|
|
2531
|
-
logicalPath:
|
|
2532
|
-
params.inputFile?.logicalPath ??
|
|
2533
|
-
params.inputFile?.path ??
|
|
2534
|
-
inputFileName,
|
|
2535
|
-
fileName: params.inputFile?.fileName ?? inputFileName,
|
|
2536
|
-
storageKey: inputStorageKey,
|
|
2537
|
-
contentType: params.inputFile?.contentType ?? null,
|
|
2538
|
-
bytes:
|
|
2539
|
-
typeof params.inputFile?.bytes === 'number'
|
|
2540
|
-
? params.inputFile.bytes
|
|
2541
|
-
: null,
|
|
2542
|
-
},
|
|
2543
|
-
}
|
|
2544
|
-
: null,
|
|
2545
|
-
packagedFiles: params.packagedFiles ?? null,
|
|
2546
|
-
partitionRange: null,
|
|
2547
|
-
dedupBackend: params.dedupBackend,
|
|
2548
|
-
resumeFromCheckpointR2Key: null,
|
|
2549
|
-
contractSnapshot: params.contractSnapshot ?? null,
|
|
2550
|
-
executionPlan: params.executionPlan ?? null,
|
|
2551
|
-
childPlayManifests: params.childPlayManifests ?? null,
|
|
2552
|
-
playCallGovernance: params.playCallGovernance ?? null,
|
|
2553
|
-
force: params.force === true || params.forceToolRefresh === true,
|
|
2554
|
-
forceToolRefresh: params.forceToolRefresh === true,
|
|
2555
|
-
preloadedDbSessions: params.preloadedDbSessions ?? null,
|
|
2556
|
-
runtimeTestFaultHeader: params.runtimeTestFaultHeader ?? null,
|
|
2557
|
-
testPolicyOverrides: params.testPolicyOverrides ?? null,
|
|
2558
|
-
coordinatorUrl: params.coordinatorUrl ?? null,
|
|
2559
|
-
coordinatorInternalToken: params.coordinatorInternalToken ?? null,
|
|
2560
|
-
totalRows: params.totalRows,
|
|
2561
|
-
};
|
|
2562
|
-
}
|
|
2563
|
-
|
|
2564
|
-
async function readLegacyRunStream(response: Response): Promise<{
|
|
2565
|
-
result: unknown;
|
|
2566
|
-
outputRows: number | null;
|
|
2567
|
-
logs: string[];
|
|
2568
|
-
error: { message: string; stack: string | null } | null;
|
|
2569
|
-
}> {
|
|
2570
|
-
if (!response.body) {
|
|
2571
|
-
throw new Error('Legacy child Worker returned an empty stream.');
|
|
2572
|
-
}
|
|
2573
|
-
const decoder = new TextDecoder();
|
|
2574
|
-
const reader = response.body.getReader();
|
|
2575
|
-
const logs: string[] = [];
|
|
2576
|
-
let result: unknown = null;
|
|
2577
|
-
let outputRows: number | null = null;
|
|
2578
|
-
let error: { message: string; stack: string | null } | null = null;
|
|
2579
|
-
let buffered = '';
|
|
2580
|
-
|
|
2581
|
-
const consumeLine = (line: string): void => {
|
|
2582
|
-
const trimmed = line.trim();
|
|
2583
|
-
if (!trimmed) {
|
|
2584
|
-
return;
|
|
2585
|
-
}
|
|
2586
|
-
let event: unknown;
|
|
2587
|
-
try {
|
|
2588
|
-
event = JSON.parse(trimmed);
|
|
2589
|
-
} catch {
|
|
2590
|
-
logs.push(trimmed);
|
|
2591
|
-
return;
|
|
2592
|
-
}
|
|
2593
|
-
if (!event || typeof event !== 'object') {
|
|
2594
|
-
return;
|
|
2595
|
-
}
|
|
2596
|
-
const record = event as Record<string, unknown>;
|
|
2597
|
-
if (record.type === 'log' && typeof record.message === 'string') {
|
|
2598
|
-
logs.push(record.message);
|
|
2599
|
-
return;
|
|
2600
|
-
}
|
|
2601
|
-
if (record.type === 'result') {
|
|
2602
|
-
result = record.result;
|
|
2603
|
-
outputRows =
|
|
2604
|
-
typeof record.outputRows === 'number' ? record.outputRows : null;
|
|
2605
|
-
return;
|
|
2606
|
-
}
|
|
2607
|
-
if (record.type === 'error') {
|
|
2608
|
-
error = {
|
|
2609
|
-
message:
|
|
2610
|
-
typeof record.message === 'string'
|
|
2611
|
-
? record.message
|
|
2612
|
-
: 'Legacy child Worker failed.',
|
|
2613
|
-
stack: typeof record.stack === 'string' ? record.stack : null,
|
|
2614
|
-
};
|
|
2615
|
-
}
|
|
2616
|
-
};
|
|
2617
|
-
|
|
2618
|
-
for (;;) {
|
|
2619
|
-
const { value, done } = await reader.read();
|
|
2620
|
-
if (value) {
|
|
2621
|
-
buffered += decoder.decode(value, { stream: !done });
|
|
2622
|
-
const lines = buffered.split('\n');
|
|
2623
|
-
buffered = lines.pop() ?? '';
|
|
2624
|
-
for (const line of lines) {
|
|
2625
|
-
consumeLine(line);
|
|
2626
|
-
}
|
|
2627
|
-
}
|
|
2628
|
-
if (done) {
|
|
2629
|
-
buffered += decoder.decode();
|
|
2630
|
-
consumeLine(buffered);
|
|
2631
|
-
break;
|
|
2632
|
-
}
|
|
2633
|
-
}
|
|
2634
|
-
|
|
2635
|
-
return { result, outputRows, logs, error };
|
|
2636
|
-
}
|
|
2637
|
-
|
|
2638
|
-
async function executeChildInline(input: {
|
|
2639
|
-
env: CoordinatorEnv;
|
|
2640
|
-
parentRunId: string;
|
|
2641
|
-
body: Record<string, unknown>;
|
|
2642
|
-
}): Promise<InlineChildSubmitResult> {
|
|
2643
|
-
const startedAt = Date.now();
|
|
2644
|
-
const validated = validateChildSubmitBody(input);
|
|
2645
|
-
if (!validated.ok) {
|
|
2646
|
-
throw new Error(String(validated.error.message ?? 'Invalid child submit.'));
|
|
2647
|
-
}
|
|
2648
|
-
const { manifest, governance, childPlayName, orgId, parentExecutorToken } =
|
|
2649
|
-
validated;
|
|
2650
|
-
const childRunId = buildChildRunId({
|
|
2651
|
-
playName: childPlayName,
|
|
2652
|
-
parentRunId: input.parentRunId,
|
|
2653
|
-
body: input.body,
|
|
2654
|
-
});
|
|
2655
|
-
const timings: CoordinatorTiming[] = [];
|
|
2656
|
-
const trace = (
|
|
2657
|
-
phase: string,
|
|
2658
|
-
phaseStartedAt: number,
|
|
2659
|
-
extra?: Record<string, unknown>,
|
|
2660
|
-
): void => {
|
|
2661
|
-
const timing: CoordinatorTiming = {
|
|
2662
|
-
phase,
|
|
2663
|
-
ms: Date.now() - phaseStartedAt,
|
|
2664
|
-
graphHash: manifest.graphHash,
|
|
2665
|
-
...(extra ? { extra } : {}),
|
|
2666
|
-
};
|
|
2667
|
-
timings.push(timing);
|
|
2668
|
-
recordCoordinatorPerfTrace({
|
|
2669
|
-
runId: childRunId,
|
|
2670
|
-
phase,
|
|
2671
|
-
ms: timing.ms,
|
|
2672
|
-
graphHash: manifest.graphHash,
|
|
2673
|
-
extra: {
|
|
2674
|
-
parentRunId: input.parentRunId,
|
|
2675
|
-
mode: 'inline_dynamic_worker',
|
|
2676
|
-
...(extra ?? {}),
|
|
2677
|
-
},
|
|
2678
|
-
});
|
|
2679
|
-
};
|
|
2680
|
-
recordCoordinatorPerfTrace({
|
|
2681
|
-
runId: childRunId,
|
|
2682
|
-
phase: 'coordinator.inline_child_boundary',
|
|
2683
|
-
ms: 0,
|
|
2684
|
-
graphHash: manifest.graphHash,
|
|
2685
|
-
extra: {
|
|
2686
|
-
parentRunId: input.parentRunId,
|
|
2687
|
-
childPlayName,
|
|
2688
|
-
action: 'start',
|
|
2689
|
-
mode: 'inline_dynamic_worker',
|
|
2690
|
-
},
|
|
2691
|
-
});
|
|
2692
|
-
|
|
2693
|
-
const loaderStartedAt = Date.now();
|
|
2694
|
-
const stub = loadDynamicPlayWorker(input.env, {
|
|
2695
|
-
runId: childRunId,
|
|
2696
|
-
graphHash: manifest.graphHash,
|
|
2697
|
-
artifactStorageKey: manifest.artifactStorageKey,
|
|
2698
|
-
artifactHash: manifest.artifactHash,
|
|
2699
|
-
dynamicWorkerCode:
|
|
2700
|
-
typeof manifest.bundledCode === 'string' ? manifest.bundledCode : null,
|
|
2701
|
-
packagedFiles: null,
|
|
2702
|
-
});
|
|
2703
|
-
trace('coordinator.inline_child_loader_get', loaderStartedAt);
|
|
2704
|
-
|
|
2705
|
-
const prepareStartedAt = Date.now();
|
|
2706
|
-
const parentPlayName =
|
|
2707
|
-
typeof input.body.parentPlayName === 'string' &&
|
|
2708
|
-
input.body.parentPlayName.trim()
|
|
2709
|
-
? input.body.parentPlayName.trim()
|
|
2710
|
-
: governance.parentPlayName;
|
|
2711
|
-
const baseUrl = resolveRuntimeApiBaseUrl(input.env, input.body);
|
|
2712
|
-
const { childToken, preloadedDbSessions, prepareTimings, transportTimings } =
|
|
2713
|
-
await prepareInlineChildRunWithRuntime({
|
|
2714
|
-
env: input.env,
|
|
2715
|
-
baseUrl,
|
|
2716
|
-
parentExecutorToken,
|
|
2717
|
-
parentRunId: input.parentRunId,
|
|
2718
|
-
parentPlayName,
|
|
2719
|
-
childRunId,
|
|
2720
|
-
childPlayName,
|
|
2721
|
-
manifest,
|
|
2722
|
-
governance,
|
|
2723
|
-
userEmail:
|
|
2724
|
-
typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
|
|
2725
|
-
});
|
|
2726
|
-
trace('coordinator.inline_child_prepare', prepareStartedAt, {
|
|
2727
|
-
sessions: preloadedDbSessions.length,
|
|
2728
|
-
prepareTimings,
|
|
2729
|
-
transportTimings,
|
|
2730
|
-
});
|
|
2731
|
-
|
|
2732
|
-
const params = buildChildWorkflowParams({
|
|
2733
|
-
env: input.env,
|
|
2734
|
-
body: input.body,
|
|
2735
|
-
manifest,
|
|
2736
|
-
governance,
|
|
2737
|
-
childRunId,
|
|
2738
|
-
childPlayName,
|
|
2739
|
-
childToken,
|
|
2740
|
-
orgId,
|
|
2741
|
-
coordinatorUrl: null,
|
|
2742
|
-
runtimeBackend: 'cf_workflows_dynamic_worker_inline_child',
|
|
2743
|
-
dynamicWorkerCode: null,
|
|
2744
|
-
preloadedDbSessions:
|
|
2745
|
-
preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
|
|
2746
|
-
});
|
|
2747
|
-
|
|
2748
|
-
let entrypoint: ReturnType<Awaited<typeof stub>['getEntrypoint']> | null =
|
|
2749
|
-
null;
|
|
2750
|
-
let response: Response | null = null;
|
|
2751
|
-
try {
|
|
2752
|
-
const awaitedStub = await stub;
|
|
2753
|
-
const entrypointStartedAt = Date.now();
|
|
2754
|
-
entrypoint = awaitedStub.getEntrypoint();
|
|
2755
|
-
trace('coordinator.inline_child_get_entrypoint', entrypointStartedAt);
|
|
2756
|
-
const fetchStartedAt = Date.now();
|
|
2757
|
-
const inlineResponse = await entrypoint.fetch(
|
|
2758
|
-
new Request('https://deepline.dynamic.internal/run-inline', {
|
|
2759
|
-
method: 'POST',
|
|
2760
|
-
headers: { 'content-type': 'application/json' },
|
|
2761
|
-
body: JSON.stringify(runRequestFromPlayWorkflowParams(params)),
|
|
2762
|
-
}),
|
|
2763
|
-
);
|
|
2764
|
-
if (!inlineResponse) {
|
|
2765
|
-
throw new Error('Inline child Worker returned no response.');
|
|
2766
|
-
}
|
|
2767
|
-
let workerResponse = inlineResponse as Response;
|
|
2768
|
-
response = workerResponse;
|
|
2769
|
-
trace('coordinator.inline_child_worker_fetch', fetchStartedAt, {
|
|
2770
|
-
status: workerResponse.status,
|
|
2771
|
-
endpoint: '/run-inline',
|
|
2772
|
-
});
|
|
2773
|
-
let usedLegacyRunStream = false;
|
|
2774
|
-
if (workerResponse.status === 404) {
|
|
2775
|
-
disposeRpcStub(workerResponse);
|
|
2776
|
-
const legacyFetchStartedAt = Date.now();
|
|
2777
|
-
const legacyResponse = await entrypoint.fetch(
|
|
2778
|
-
new Request('https://deepline.dynamic.internal/run', {
|
|
2779
|
-
method: 'POST',
|
|
2780
|
-
headers: { 'content-type': 'application/json' },
|
|
2781
|
-
body: JSON.stringify(runRequestFromPlayWorkflowParams(params)),
|
|
2782
|
-
}),
|
|
2783
|
-
);
|
|
2784
|
-
if (!legacyResponse) {
|
|
2785
|
-
throw new Error('Legacy inline child Worker returned no response.');
|
|
2786
|
-
}
|
|
2787
|
-
workerResponse = legacyResponse as Response;
|
|
2788
|
-
response = workerResponse;
|
|
2789
|
-
usedLegacyRunStream = true;
|
|
2790
|
-
trace('coordinator.inline_child_worker_fetch', legacyFetchStartedAt, {
|
|
2791
|
-
status: workerResponse.status,
|
|
2792
|
-
endpoint: '/run',
|
|
2793
|
-
compatibility: 'legacy_stream',
|
|
2794
|
-
});
|
|
2795
|
-
}
|
|
2796
|
-
if (!workerResponse.ok) {
|
|
2797
|
-
const text = await workerResponse.text().catch(() => '');
|
|
2798
|
-
throw new Error(
|
|
2799
|
-
`Inline child Worker failed ${workerResponse.status}: ${text.slice(0, 800)}`,
|
|
2800
|
-
);
|
|
2801
|
-
}
|
|
2802
|
-
const responseStartedAt = Date.now();
|
|
2803
|
-
const parsed: InlineWorkerRunResponse = usedLegacyRunStream
|
|
2804
|
-
? await readLegacyRunStream(workerResponse).then((legacy) => ({
|
|
2805
|
-
status: legacy.error ? 'failed' : 'completed',
|
|
2806
|
-
result: legacy.result,
|
|
2807
|
-
outputRows: legacy.outputRows ?? undefined,
|
|
2808
|
-
events: legacy.logs.map((message) => ({
|
|
2809
|
-
type: 'log',
|
|
2810
|
-
message,
|
|
2811
|
-
})),
|
|
2812
|
-
error: legacy.error ?? undefined,
|
|
2813
|
-
}))
|
|
2814
|
-
: ((await workerResponse.json()) as InlineWorkerRunResponse);
|
|
2815
|
-
const logs = (parsed.events ?? []).flatMap((event) => {
|
|
2816
|
-
if (
|
|
2817
|
-
event &&
|
|
2818
|
-
typeof event === 'object' &&
|
|
2819
|
-
event.type === 'log' &&
|
|
2820
|
-
typeof event.message === 'string'
|
|
2821
|
-
) {
|
|
2822
|
-
return [event.message];
|
|
2823
|
-
}
|
|
2824
|
-
return [];
|
|
2825
|
-
});
|
|
2826
|
-
trace('coordinator.inline_child_response', responseStartedAt, {
|
|
2827
|
-
status: parsed.status ?? null,
|
|
2828
|
-
endpoint: usedLegacyRunStream ? '/run' : '/run-inline',
|
|
2829
|
-
logCount: logs.length,
|
|
2830
|
-
outputRows:
|
|
2831
|
-
typeof parsed.outputRows === 'number' ? parsed.outputRows : null,
|
|
2832
|
-
durationMs:
|
|
2833
|
-
typeof parsed.durationMs === 'number' ? parsed.durationMs : null,
|
|
2834
|
-
});
|
|
2835
|
-
for (const timing of parsed.timings ?? []) {
|
|
2836
|
-
if (
|
|
2837
|
-
!timing ||
|
|
2838
|
-
typeof timing !== 'object' ||
|
|
2839
|
-
typeof timing.phase !== 'string' ||
|
|
2840
|
-
typeof timing.ms !== 'number' ||
|
|
2841
|
-
!Number.isFinite(timing.ms)
|
|
2842
|
-
) {
|
|
2843
|
-
continue;
|
|
2844
|
-
}
|
|
2845
|
-
recordCoordinatorPerfTrace({
|
|
2846
|
-
runId: childRunId,
|
|
2847
|
-
phase: `dynamic_worker.${timing.phase}`,
|
|
2848
|
-
ms: Math.max(0, Math.round(timing.ms)),
|
|
2849
|
-
graphHash: manifest.graphHash,
|
|
2850
|
-
extra: {
|
|
2851
|
-
parentRunId: input.parentRunId,
|
|
2852
|
-
mode: 'inline_dynamic_worker',
|
|
2853
|
-
...(isRecord(timing.extra) ? timing.extra : {}),
|
|
2854
|
-
},
|
|
2855
|
-
});
|
|
2856
|
-
timings.push({
|
|
2857
|
-
phase: `dynamic_worker.${timing.phase}`,
|
|
2858
|
-
ms: Math.max(0, Math.round(timing.ms)),
|
|
2859
|
-
graphHash: manifest.graphHash,
|
|
2860
|
-
...(isRecord(timing.extra) ? { extra: timing.extra } : {}),
|
|
2861
|
-
});
|
|
2862
|
-
}
|
|
2863
|
-
trace('coordinator.inline_child_total', startedAt);
|
|
2864
|
-
if (parsed.status === 'failed' || parsed.error) {
|
|
2865
|
-
const error = {
|
|
2866
|
-
message:
|
|
2867
|
-
typeof parsed.error?.message === 'string'
|
|
2868
|
-
? parsed.error.message
|
|
2869
|
-
: 'Inline child Worker failed.',
|
|
2870
|
-
stack:
|
|
2871
|
-
typeof parsed.error?.stack === 'string' ? parsed.error.stack : null,
|
|
2872
|
-
};
|
|
2873
|
-
recordCoordinatorPerfTrace({
|
|
2874
|
-
runId: childRunId,
|
|
2875
|
-
phase: 'coordinator.inline_child_boundary',
|
|
2876
|
-
ms: Date.now() - startedAt,
|
|
2877
|
-
graphHash: manifest.graphHash,
|
|
2878
|
-
extra: {
|
|
2879
|
-
parentRunId: input.parentRunId,
|
|
2880
|
-
childPlayName,
|
|
2881
|
-
action: 'failed',
|
|
2882
|
-
mode: 'inline_dynamic_worker',
|
|
2883
|
-
error: error.message,
|
|
2884
|
-
},
|
|
2885
|
-
});
|
|
2886
|
-
const terminalAppendStartedAt = Date.now();
|
|
2887
|
-
await appendRegisteredChildRunTerminal({
|
|
2888
|
-
env: input.env,
|
|
2889
|
-
baseUrl,
|
|
2890
|
-
childExecutorToken: childToken,
|
|
2891
|
-
childRunId,
|
|
2892
|
-
status: 'failed',
|
|
2893
|
-
error: error.message,
|
|
2894
|
-
});
|
|
2895
|
-
trace(
|
|
2896
|
-
'coordinator.inline_child_terminal_ledger_append',
|
|
2897
|
-
terminalAppendStartedAt,
|
|
2898
|
-
{ status: 'failed' },
|
|
2899
|
-
);
|
|
2900
|
-
return {
|
|
2901
|
-
workflowId: childRunId,
|
|
2902
|
-
runId: childRunId,
|
|
2903
|
-
status: 'failed',
|
|
2904
|
-
mode: 'inline_dynamic_worker',
|
|
2905
|
-
error,
|
|
2906
|
-
logs,
|
|
2907
|
-
timings,
|
|
2908
|
-
};
|
|
2909
|
-
}
|
|
2910
|
-
recordCoordinatorPerfTrace({
|
|
2911
|
-
runId: childRunId,
|
|
2912
|
-
phase: 'coordinator.inline_child_boundary',
|
|
2913
|
-
ms: Date.now() - startedAt,
|
|
2914
|
-
graphHash: manifest.graphHash,
|
|
2915
|
-
extra: {
|
|
2916
|
-
parentRunId: input.parentRunId,
|
|
2917
|
-
childPlayName,
|
|
2918
|
-
action: 'completed',
|
|
2919
|
-
mode: 'inline_dynamic_worker',
|
|
2920
|
-
},
|
|
2921
|
-
});
|
|
2922
|
-
const terminalAppendStartedAt = Date.now();
|
|
2923
|
-
await appendRegisteredChildRunTerminal({
|
|
2924
|
-
env: input.env,
|
|
2925
|
-
baseUrl,
|
|
2926
|
-
childExecutorToken: childToken,
|
|
2927
|
-
childRunId,
|
|
2928
|
-
status: 'completed',
|
|
2929
|
-
result: parsed.result,
|
|
2930
|
-
});
|
|
2931
|
-
trace(
|
|
2932
|
-
'coordinator.inline_child_terminal_ledger_append',
|
|
2933
|
-
terminalAppendStartedAt,
|
|
2934
|
-
{ status: 'completed' },
|
|
2935
|
-
);
|
|
2936
|
-
return {
|
|
2937
|
-
workflowId: childRunId,
|
|
2938
|
-
runId: childRunId,
|
|
2939
|
-
status: 'completed',
|
|
2940
|
-
mode: 'inline_dynamic_worker',
|
|
2941
|
-
result: parsed.result,
|
|
2942
|
-
output: parsed.result,
|
|
2943
|
-
logs,
|
|
2944
|
-
timings,
|
|
2945
|
-
};
|
|
2946
|
-
} finally {
|
|
2947
|
-
disposeRpcStub(response);
|
|
2948
|
-
disposeRpcStub(entrypoint);
|
|
2949
|
-
}
|
|
2950
|
-
}
|
|
2951
|
-
|
|
2952
|
-
function isRetryableChildWorkflowSubmitError(error: unknown): boolean {
|
|
2953
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
2954
|
-
return /timed out|timeout|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|internal error/i.test(
|
|
2955
|
-
message,
|
|
2956
|
-
);
|
|
2957
|
-
}
|
|
2958
|
-
|
|
2959
|
-
function isRetryableChildWorkflowSubmitResponse(
|
|
2960
|
-
status: number,
|
|
2961
|
-
body: string,
|
|
2962
|
-
): boolean {
|
|
2963
|
-
if (
|
|
2964
|
-
status === 408 ||
|
|
2965
|
-
status === 429 ||
|
|
2966
|
-
status === 502 ||
|
|
2967
|
-
status === 503 ||
|
|
2968
|
-
status === 504 ||
|
|
2969
|
-
status === 530
|
|
2970
|
-
) {
|
|
2971
|
-
return true;
|
|
2972
|
-
}
|
|
2973
|
-
return (
|
|
2974
|
-
status === 500 &&
|
|
2975
|
-
/COORDINATOR_WORKFLOW_ROUTE_FAILED|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|UND_ERR_CONNECT_TIMEOUT|internal error/i.test(
|
|
2976
|
-
body,
|
|
2977
|
-
)
|
|
2978
|
-
);
|
|
1694
|
+
? Math.max(0, Math.floor(value))
|
|
1695
|
+
: 0;
|
|
2979
1696
|
}
|
|
2980
1697
|
|
|
2981
|
-
async function
|
|
1698
|
+
async function restartWorkflowAfterPlatformReset(input: {
|
|
2982
1699
|
env: CoordinatorEnv;
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
1700
|
+
ctx?: ExecutionContext;
|
|
1701
|
+
runId: string;
|
|
1702
|
+
oldInstance: WorkflowInstance;
|
|
1703
|
+
error: string;
|
|
1704
|
+
status: InstanceStatus;
|
|
2986
1705
|
}): Promise<{
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
childRunId: string;
|
|
2990
|
-
childPlayName: string;
|
|
2991
|
-
startedAt: number;
|
|
2992
|
-
timings: CoordinatorTiming[];
|
|
1706
|
+
retried: boolean;
|
|
1707
|
+
result: Record<string, unknown> | null;
|
|
2993
1708
|
}> {
|
|
2994
|
-
const
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
phaseStartedAt: number,
|
|
2999
|
-
graphHash?: string | null,
|
|
3000
|
-
extra?: Record<string, unknown>,
|
|
3001
|
-
): void => {
|
|
3002
|
-
const timing: CoordinatorTiming = {
|
|
3003
|
-
phase,
|
|
3004
|
-
ms: Date.now() - phaseStartedAt,
|
|
3005
|
-
...(graphHash ? { graphHash } : {}),
|
|
3006
|
-
...(extra ? { extra } : {}),
|
|
3007
|
-
};
|
|
3008
|
-
timings.push(timing);
|
|
3009
|
-
recordCoordinatorPerfTrace({
|
|
3010
|
-
runId: input.parentRunId,
|
|
3011
|
-
phase,
|
|
3012
|
-
ms: timing.ms,
|
|
3013
|
-
graphHash: graphHash ?? undefined,
|
|
3014
|
-
extra,
|
|
3015
|
-
});
|
|
3016
|
-
};
|
|
3017
|
-
const validated = validateChildSubmitBody({
|
|
3018
|
-
parentRunId: input.parentRunId,
|
|
3019
|
-
body: input.body,
|
|
1709
|
+
const decision = decideWorkflowPlatformRetry({
|
|
1710
|
+
workflowStatus: String(input.status.status ?? ''),
|
|
1711
|
+
error: input.error,
|
|
1712
|
+
retryAttempts: 0,
|
|
3020
1713
|
});
|
|
3021
|
-
if (
|
|
3022
|
-
return {
|
|
3023
|
-
response: Response.json(
|
|
3024
|
-
{ error: validated.error },
|
|
3025
|
-
{ status: validated.status },
|
|
3026
|
-
),
|
|
3027
|
-
responseText: '',
|
|
3028
|
-
childRunId: '',
|
|
3029
|
-
childPlayName: '',
|
|
3030
|
-
startedAt,
|
|
3031
|
-
timings,
|
|
3032
|
-
};
|
|
1714
|
+
if (decision.action !== 'retry') {
|
|
1715
|
+
return { retried: false, result: null };
|
|
3033
1716
|
}
|
|
3034
|
-
const
|
|
3035
|
-
validated;
|
|
3036
|
-
const childRunId = buildChildRunId({
|
|
3037
|
-
playName: childPlayName,
|
|
3038
|
-
parentRunId: input.parentRunId,
|
|
3039
|
-
body: input.body,
|
|
3040
|
-
});
|
|
3041
|
-
const baseUrl = resolveRuntimeApiBaseUrl(input.env, input.body);
|
|
3042
|
-
|
|
3043
|
-
const prepareStartedAt = Date.now();
|
|
3044
|
-
const { childToken, preloadedDbSessions, prepareTimings, transportTimings } =
|
|
3045
|
-
await prepareInlineChildRunWithRuntime({
|
|
3046
|
-
env: input.env,
|
|
3047
|
-
baseUrl,
|
|
3048
|
-
parentExecutorToken,
|
|
3049
|
-
parentRunId: input.parentRunId,
|
|
3050
|
-
parentPlayName:
|
|
3051
|
-
typeof input.body.parentPlayName === 'string' &&
|
|
3052
|
-
input.body.parentPlayName.trim()
|
|
3053
|
-
? input.body.parentPlayName.trim()
|
|
3054
|
-
: governance.parentPlayName,
|
|
3055
|
-
childRunId,
|
|
3056
|
-
childPlayName,
|
|
3057
|
-
manifest,
|
|
3058
|
-
governance,
|
|
3059
|
-
userEmail:
|
|
3060
|
-
typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
|
|
3061
|
-
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
3062
|
-
schedulerBackend: 'cf_workflows',
|
|
3063
|
-
executionProfile: 'workers_edge',
|
|
3064
|
-
});
|
|
3065
|
-
trace(
|
|
3066
|
-
'coordinator.child_submit_prepare',
|
|
3067
|
-
prepareStartedAt,
|
|
3068
|
-
manifest.graphHash,
|
|
3069
|
-
{
|
|
3070
|
-
childRunId,
|
|
3071
|
-
childPlayName,
|
|
3072
|
-
sessions: preloadedDbSessions.length,
|
|
3073
|
-
prepareTimings,
|
|
3074
|
-
transportTimings,
|
|
3075
|
-
},
|
|
3076
|
-
);
|
|
3077
|
-
|
|
3078
|
-
const params = buildChildWorkflowParams({
|
|
1717
|
+
const claim = await claimWorkflowPlatformRetry({
|
|
3079
1718
|
env: input.env,
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
orgId,
|
|
3087
|
-
coordinatorUrl: input.coordinatorUrl,
|
|
3088
|
-
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
3089
|
-
dynamicWorkerCode:
|
|
3090
|
-
typeof manifest.bundledCode === 'string' ? manifest.bundledCode : null,
|
|
3091
|
-
preloadedDbSessions:
|
|
3092
|
-
preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
|
|
3093
|
-
});
|
|
3094
|
-
|
|
3095
|
-
const workflowSubmitStartedAt = Date.now();
|
|
3096
|
-
let response: Response | null = null;
|
|
3097
|
-
let responseText = '';
|
|
3098
|
-
let lastSubmitError: unknown = null;
|
|
3099
|
-
for (
|
|
3100
|
-
let attempt = 0;
|
|
3101
|
-
attempt <= CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS.length;
|
|
3102
|
-
attempt += 1
|
|
3103
|
-
) {
|
|
3104
|
-
try {
|
|
3105
|
-
response = await handleWorkflowRoute({
|
|
3106
|
-
runId: childRunId,
|
|
3107
|
-
action: 'submit',
|
|
3108
|
-
request: new Request(
|
|
3109
|
-
`https://deepline.coordinator.internal/workflow/${encodeURIComponent(
|
|
3110
|
-
childRunId,
|
|
3111
|
-
)}/submit`,
|
|
3112
|
-
{
|
|
3113
|
-
method: 'POST',
|
|
3114
|
-
headers: { 'content-type': 'application/json' },
|
|
3115
|
-
body: JSON.stringify(params),
|
|
3116
|
-
},
|
|
3117
|
-
),
|
|
3118
|
-
env: input.env,
|
|
3119
|
-
});
|
|
3120
|
-
} catch (error) {
|
|
3121
|
-
lastSubmitError = error;
|
|
3122
|
-
trace(
|
|
3123
|
-
'coordinator.child_submit_workflow_attempt',
|
|
3124
|
-
workflowSubmitStartedAt,
|
|
3125
|
-
manifest.graphHash,
|
|
3126
|
-
{
|
|
3127
|
-
childRunId,
|
|
3128
|
-
attempt: attempt + 1,
|
|
3129
|
-
status: 'threw',
|
|
3130
|
-
error: error instanceof Error ? error.message : String(error),
|
|
3131
|
-
},
|
|
3132
|
-
);
|
|
3133
|
-
if (
|
|
3134
|
-
attempt >= CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS.length ||
|
|
3135
|
-
!isRetryableChildWorkflowSubmitError(error)
|
|
3136
|
-
) {
|
|
3137
|
-
await markRegisteredChildRunFailed({
|
|
3138
|
-
env: input.env,
|
|
3139
|
-
baseUrl,
|
|
3140
|
-
childExecutorToken: childToken,
|
|
3141
|
-
childRunId,
|
|
3142
|
-
error,
|
|
3143
|
-
}).catch((markError) => {
|
|
3144
|
-
recordCoordinatorEvent(
|
|
3145
|
-
'error',
|
|
3146
|
-
'coordinator.child_workflow_submit.failure_mark_failed',
|
|
3147
|
-
'[coordinator] child workflow submit failure mark failed',
|
|
3148
|
-
{
|
|
3149
|
-
childRunId,
|
|
3150
|
-
error:
|
|
3151
|
-
markError instanceof Error
|
|
3152
|
-
? markError.message
|
|
3153
|
-
: String(markError),
|
|
3154
|
-
},
|
|
3155
|
-
);
|
|
3156
|
-
});
|
|
3157
|
-
throw error;
|
|
3158
|
-
}
|
|
3159
|
-
await sleep(CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS[attempt] ?? 0);
|
|
3160
|
-
continue;
|
|
3161
|
-
}
|
|
3162
|
-
|
|
3163
|
-
responseText = await response.text().catch(() => '');
|
|
3164
|
-
trace(
|
|
3165
|
-
'coordinator.child_submit_workflow_attempt',
|
|
3166
|
-
workflowSubmitStartedAt,
|
|
3167
|
-
manifest.graphHash,
|
|
1719
|
+
runId: input.runId,
|
|
1720
|
+
}).catch((error) => {
|
|
1721
|
+
recordCoordinatorEvent(
|
|
1722
|
+
'warn',
|
|
1723
|
+
'coordinator.workflow_platform_retry.claim_failed',
|
|
1724
|
+
'[coordinator] workflow platform retry claim failed',
|
|
3168
1725
|
{
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
status: response.status,
|
|
1726
|
+
runId: input.runId,
|
|
1727
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3172
1728
|
},
|
|
3173
1729
|
);
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
) {
|
|
3179
|
-
break;
|
|
3180
|
-
}
|
|
3181
|
-
lastSubmitError = `workflow submit returned ${response.status}: ${responseText.slice(0, 800)}`;
|
|
3182
|
-
await sleep(CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS[attempt] ?? 0);
|
|
3183
|
-
}
|
|
3184
|
-
if (!response) {
|
|
3185
|
-
throw lastSubmitError instanceof Error
|
|
3186
|
-
? lastSubmitError
|
|
3187
|
-
: new Error(String(lastSubmitError));
|
|
1730
|
+
return null;
|
|
1731
|
+
});
|
|
1732
|
+
if (!claim?.claimed || !claim.params) {
|
|
1733
|
+
return { retried: false, result: null };
|
|
3188
1734
|
}
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
if (!response.ok) {
|
|
3196
|
-
await markRegisteredChildRunFailed({
|
|
1735
|
+
const retryInstanceId = workflowRetryInstanceId(input.runId, claim.attempts);
|
|
1736
|
+
const retryStartedAt = Date.now();
|
|
1737
|
+
let retryInstance: WorkflowInstance | null = null;
|
|
1738
|
+
const retryParams = { ...claim.params, runAttempt: claim.attempts };
|
|
1739
|
+
try {
|
|
1740
|
+
retryInstance = await createDynamicWorkflowInstance({
|
|
3197
1741
|
env: input.env,
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
'error',
|
|
3205
|
-
'coordinator.child_workflow_submit.failure_mark_failed',
|
|
3206
|
-
'[coordinator] child workflow submit failure mark failed',
|
|
3207
|
-
{
|
|
3208
|
-
childRunId,
|
|
3209
|
-
error:
|
|
3210
|
-
markError instanceof Error ? markError.message : String(markError),
|
|
3211
|
-
},
|
|
3212
|
-
);
|
|
1742
|
+
id: retryInstanceId,
|
|
1743
|
+
params: workflowCreateParams(retryParams, {
|
|
1744
|
+
params: retryParams,
|
|
1745
|
+
paramsRef: claim.paramsRef,
|
|
1746
|
+
}),
|
|
1747
|
+
metadataParams: retryParams,
|
|
3213
1748
|
});
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
export class RuntimeApi extends WorkerEntrypoint<CoordinatorEnv, undefined> {
|
|
3244
|
-
async fetch(request: Request): Promise<Response> {
|
|
3245
|
-
const incoming = new URL(request.url);
|
|
3246
|
-
const allowed =
|
|
3247
|
-
incoming.pathname.startsWith('/api/v2/plays/') ||
|
|
3248
|
-
incoming.pathname.startsWith('/api/v2/integrations/');
|
|
3249
|
-
if (!allowed) {
|
|
3250
|
-
return new Response(
|
|
3251
|
-
JSON.stringify({
|
|
3252
|
-
error:
|
|
3253
|
-
'RUNTIME_API binding only forwards /api/v2/plays/* and /api/v2/integrations/* paths',
|
|
3254
|
-
attemptedPath: incoming.pathname,
|
|
3255
|
-
}),
|
|
3256
|
-
{ status: 403, headers: { 'content-type': 'application/json' } },
|
|
3257
|
-
);
|
|
3258
|
-
}
|
|
3259
|
-
const configuredApiBaseUrl =
|
|
3260
|
-
typeof this.env.DEEPLINE_API_BASE_URL === 'string' &&
|
|
3261
|
-
this.env.DEEPLINE_API_BASE_URL.trim()
|
|
3262
|
-
? this.env.DEEPLINE_API_BASE_URL.trim()
|
|
3263
|
-
: 'https://code.deepline.com';
|
|
3264
|
-
const target = new URL(
|
|
3265
|
-
incoming.pathname + incoming.search,
|
|
3266
|
-
configuredApiBaseUrl,
|
|
3267
|
-
);
|
|
3268
|
-
const forwarded = new Request(target.toString(), request);
|
|
3269
|
-
const bypassToken = this.env.VERCEL_PROTECTION_BYPASS_TOKEN;
|
|
3270
|
-
if (typeof bypassToken === 'string' && bypassToken) {
|
|
3271
|
-
forwarded.headers.set('x-vercel-protection-bypass', bypassToken);
|
|
3272
|
-
}
|
|
3273
|
-
const res = await fetch(forwarded);
|
|
3274
|
-
if (!res.ok) {
|
|
3275
|
-
const body = await res
|
|
3276
|
-
.clone()
|
|
3277
|
-
.text()
|
|
3278
|
-
.catch(() => '');
|
|
3279
|
-
recordCoordinatorEvent(
|
|
3280
|
-
'error',
|
|
3281
|
-
'coordinator.runtime_api.proxy_failed',
|
|
3282
|
-
'[RUNTIME_API] request failed',
|
|
3283
|
-
{
|
|
3284
|
-
path: incoming.pathname,
|
|
3285
|
-
status: res.status,
|
|
3286
|
-
targetOrigin: target.origin,
|
|
3287
|
-
body: body.slice(0, 500),
|
|
1749
|
+
await recordWorkflowInstanceId({
|
|
1750
|
+
env: input.env,
|
|
1751
|
+
runId: input.runId,
|
|
1752
|
+
instanceId: retryInstance.id,
|
|
1753
|
+
});
|
|
1754
|
+
input.ctx?.waitUntil(input.oldInstance.terminate().catch(() => undefined));
|
|
1755
|
+
recordCoordinatorPerfTraceBuffered(input.env, input.ctx, {
|
|
1756
|
+
runId: input.runId,
|
|
1757
|
+
phase: 'coordinator.platform_deploy_retry',
|
|
1758
|
+
ms: Date.now() - retryStartedAt,
|
|
1759
|
+
graphHash: claim.params.graphHash ?? null,
|
|
1760
|
+
extra: {
|
|
1761
|
+
retryAttempt: claim.attempts,
|
|
1762
|
+
retryInstanceId: retryInstance.id,
|
|
1763
|
+
reason: decision.reason,
|
|
1764
|
+
},
|
|
1765
|
+
});
|
|
1766
|
+
return {
|
|
1767
|
+
retried: true,
|
|
1768
|
+
result: {
|
|
1769
|
+
runId: input.runId,
|
|
1770
|
+
playName: claim.params.playName,
|
|
1771
|
+
status: 'running',
|
|
1772
|
+
result: null,
|
|
1773
|
+
error: null,
|
|
1774
|
+
retry: {
|
|
1775
|
+
reason: decision.reason,
|
|
1776
|
+
attempt: claim.attempts,
|
|
1777
|
+
message: decision.message,
|
|
3288
1778
|
},
|
|
3289
|
-
|
|
3290
|
-
}
|
|
3291
|
-
|
|
1779
|
+
},
|
|
1780
|
+
};
|
|
1781
|
+
} finally {
|
|
1782
|
+
disposeRpcStub(retryInstance);
|
|
3292
1783
|
}
|
|
3293
1784
|
}
|
|
3294
1785
|
|
|
@@ -3296,60 +1787,6 @@ export class CoordinatorControl extends WorkerEntrypoint<
|
|
|
3296
1787
|
CoordinatorEnv,
|
|
3297
1788
|
undefined
|
|
3298
1789
|
> {
|
|
3299
|
-
async submitChild(
|
|
3300
|
-
parentRunId: string,
|
|
3301
|
-
body: Record<string, unknown>,
|
|
3302
|
-
): Promise<InlineChildSubmitResult> {
|
|
3303
|
-
return await executeChildInline({
|
|
3304
|
-
env: this.env,
|
|
3305
|
-
parentRunId,
|
|
3306
|
-
body,
|
|
3307
|
-
});
|
|
3308
|
-
}
|
|
3309
|
-
|
|
3310
|
-
async submitWorkflowChild(
|
|
3311
|
-
parentRunId: string,
|
|
3312
|
-
body: Record<string, unknown>,
|
|
3313
|
-
): Promise<{
|
|
3314
|
-
workflowId?: string;
|
|
3315
|
-
runId?: string;
|
|
3316
|
-
status?: string;
|
|
3317
|
-
mode?: string;
|
|
3318
|
-
timings?: CoordinatorTiming[];
|
|
3319
|
-
coordinator?: unknown;
|
|
3320
|
-
error?: unknown;
|
|
3321
|
-
}> {
|
|
3322
|
-
const { response, responseText, childRunId, timings } =
|
|
3323
|
-
await submitChildWorkflowThroughCoordinator({
|
|
3324
|
-
env: this.env,
|
|
3325
|
-
parentRunId,
|
|
3326
|
-
body,
|
|
3327
|
-
coordinatorUrl: null,
|
|
3328
|
-
});
|
|
3329
|
-
let parsed: unknown = {};
|
|
3330
|
-
try {
|
|
3331
|
-
parsed = responseText ? JSON.parse(responseText) : {};
|
|
3332
|
-
} catch {
|
|
3333
|
-
parsed = { error: responseText };
|
|
3334
|
-
}
|
|
3335
|
-
if (!response.ok) {
|
|
3336
|
-
return {
|
|
3337
|
-
runId: childRunId || undefined,
|
|
3338
|
-
workflowId: childRunId || undefined,
|
|
3339
|
-
status: 'failed',
|
|
3340
|
-
error: isRecord(parsed) ? (parsed.error ?? parsed) : parsed,
|
|
3341
|
-
};
|
|
3342
|
-
}
|
|
3343
|
-
return {
|
|
3344
|
-
workflowId: childRunId,
|
|
3345
|
-
runId: childRunId,
|
|
3346
|
-
status: 'started',
|
|
3347
|
-
mode: 'workflow_rpc',
|
|
3348
|
-
coordinator: parsed,
|
|
3349
|
-
timings,
|
|
3350
|
-
};
|
|
3351
|
-
}
|
|
3352
|
-
|
|
3353
1790
|
async signal(
|
|
3354
1791
|
runId: string,
|
|
3355
1792
|
body: Record<string, unknown>,
|
|
@@ -3420,25 +1857,6 @@ export class CoordinatorControl extends WorkerEntrypoint<
|
|
|
3420
1857
|
return await readCoordinatorTerminalState(this.env, runId);
|
|
3421
1858
|
}
|
|
3422
1859
|
|
|
3423
|
-
async readChildTerminalState(
|
|
3424
|
-
parentRunId: string,
|
|
3425
|
-
eventKey: string,
|
|
3426
|
-
timeoutMs?: number,
|
|
3427
|
-
): Promise<CoordinatorChildTerminalState | null> {
|
|
3428
|
-
if (!parentRunId || !eventKey) {
|
|
3429
|
-
throw new Error('parentRunId and eventKey are required.');
|
|
3430
|
-
}
|
|
3431
|
-
return await readCoordinatorChildTerminalState({
|
|
3432
|
-
env: this.env,
|
|
3433
|
-
parentRunId,
|
|
3434
|
-
eventKey,
|
|
3435
|
-
timeoutMs:
|
|
3436
|
-
typeof timeoutMs === 'number' && Number.isFinite(timeoutMs)
|
|
3437
|
-
? Math.max(0, Math.min(Math.floor(timeoutMs), 30_000))
|
|
3438
|
-
: undefined,
|
|
3439
|
-
});
|
|
3440
|
-
}
|
|
3441
|
-
|
|
3442
1860
|
/**
|
|
3443
1861
|
* Distributed Rate State Backend acquire: lease up to `requested` request-
|
|
3444
1862
|
* window permits for `bucketId` (`<orgId>:<provider>`) from the per-bucket
|
|
@@ -3699,37 +2117,6 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
|
|
|
3699
2117
|
const output = isRecord(result) ? result : null;
|
|
3700
2118
|
const terminalResult = output?.result ?? result;
|
|
3701
2119
|
const terminalStartedAt = Date.now();
|
|
3702
|
-
const childTerminalStartedAt = Date.now();
|
|
3703
|
-
try {
|
|
3704
|
-
const childTerminalPersisted =
|
|
3705
|
-
await appendWorkflowChildCompletedRunEvent({
|
|
3706
|
-
env,
|
|
3707
|
-
event: innerEvent,
|
|
3708
|
-
result: terminalResult,
|
|
3709
|
-
});
|
|
3710
|
-
if (childTerminalPersisted) {
|
|
3711
|
-
trace({
|
|
3712
|
-
runId: runIdForTrace,
|
|
3713
|
-
phase:
|
|
3714
|
-
'coordinator.child_terminal_ledger_append_completed',
|
|
3715
|
-
ms: Date.now() - childTerminalStartedAt,
|
|
3716
|
-
graphHash,
|
|
3717
|
-
});
|
|
3718
|
-
}
|
|
3719
|
-
} catch (childTerminalError) {
|
|
3720
|
-
recordCoordinatorEvent(
|
|
3721
|
-
'warn',
|
|
3722
|
-
'coordinator.child_terminal.ledger_append_failed',
|
|
3723
|
-
'[coordinator] child terminal ledger append failed; preserving coordinator terminal cache',
|
|
3724
|
-
{
|
|
3725
|
-
runId: runIdForTrace,
|
|
3726
|
-
message:
|
|
3727
|
-
childTerminalError instanceof Error
|
|
3728
|
-
? childTerminalError.message
|
|
3729
|
-
: String(childTerminalError),
|
|
3730
|
-
},
|
|
3731
|
-
);
|
|
3732
|
-
}
|
|
3733
2120
|
try {
|
|
3734
2121
|
await writeCoordinatorTerminalState(env, {
|
|
3735
2122
|
runId: runIdForTrace,
|
|
@@ -4804,113 +3191,6 @@ async function handleWorkflowRoute(input: {
|
|
|
4804
3191
|
}
|
|
4805
3192
|
}
|
|
4806
3193
|
|
|
4807
|
-
if (action === 'submit-child') {
|
|
4808
|
-
const startedAt = Date.now();
|
|
4809
|
-
try {
|
|
4810
|
-
const body = (await request.json().catch(() => null)) as Record<
|
|
4811
|
-
string,
|
|
4812
|
-
unknown
|
|
4813
|
-
> | null;
|
|
4814
|
-
if (!body || typeof body !== 'object') {
|
|
4815
|
-
return Response.json(
|
|
4816
|
-
{
|
|
4817
|
-
error: {
|
|
4818
|
-
code: 'CHILD_SUBMIT_BODY_REQUIRED',
|
|
4819
|
-
message: 'submit-child requires a JSON body.',
|
|
4820
|
-
phase: 'coordinator_child_submit',
|
|
4821
|
-
parentRunId: runId,
|
|
4822
|
-
},
|
|
4823
|
-
},
|
|
4824
|
-
{ status: 400 },
|
|
4825
|
-
);
|
|
4826
|
-
}
|
|
4827
|
-
const {
|
|
4828
|
-
response: submitResponse,
|
|
4829
|
-
responseText,
|
|
4830
|
-
childRunId,
|
|
4831
|
-
childPlayName,
|
|
4832
|
-
} = await submitChildWorkflowThroughCoordinator({
|
|
4833
|
-
env,
|
|
4834
|
-
parentRunId: runId,
|
|
4835
|
-
body,
|
|
4836
|
-
coordinatorUrl: new URL(request.url).origin,
|
|
4837
|
-
});
|
|
4838
|
-
recordCoordinatorPerfTrace({
|
|
4839
|
-
runId,
|
|
4840
|
-
phase: 'coordinator.child_submit',
|
|
4841
|
-
ms: Date.now() - startedAt,
|
|
4842
|
-
extra: { status: submitResponse.status, childRunId },
|
|
4843
|
-
});
|
|
4844
|
-
recordCoordinatorEvent(
|
|
4845
|
-
'info',
|
|
4846
|
-
'play.runtime.span',
|
|
4847
|
-
'[play.runtime.span]',
|
|
4848
|
-
{
|
|
4849
|
-
event: 'play.runtime.span',
|
|
4850
|
-
phase: 'coordinator_child_submit',
|
|
4851
|
-
runId,
|
|
4852
|
-
parentRunId: runId,
|
|
4853
|
-
childRunId,
|
|
4854
|
-
playName: childPlayName,
|
|
4855
|
-
ms: Date.now() - startedAt,
|
|
4856
|
-
status: submitResponse.ok ? 'ok' : 'failed',
|
|
4857
|
-
...(submitResponse.ok
|
|
4858
|
-
? {}
|
|
4859
|
-
: { errorCode: 'COORDINATOR_CHILD_SUBMIT_FAILED' }),
|
|
4860
|
-
},
|
|
4861
|
-
);
|
|
4862
|
-
if (!submitResponse.ok) {
|
|
4863
|
-
return new Response(responseText, {
|
|
4864
|
-
status: submitResponse.status,
|
|
4865
|
-
headers: {
|
|
4866
|
-
'content-type':
|
|
4867
|
-
submitResponse.headers.get('content-type') ?? 'application/json',
|
|
4868
|
-
'cache-control': 'no-store',
|
|
4869
|
-
},
|
|
4870
|
-
});
|
|
4871
|
-
}
|
|
4872
|
-
return Response.json(
|
|
4873
|
-
{
|
|
4874
|
-
workflowId: childRunId,
|
|
4875
|
-
runId: childRunId,
|
|
4876
|
-
status: 'started',
|
|
4877
|
-
coordinator: JSON.parse(responseText || '{}'),
|
|
4878
|
-
},
|
|
4879
|
-
{
|
|
4880
|
-
headers: {
|
|
4881
|
-
'cache-control': 'no-store',
|
|
4882
|
-
},
|
|
4883
|
-
},
|
|
4884
|
-
);
|
|
4885
|
-
} catch (error) {
|
|
4886
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
4887
|
-
recordCoordinatorEvent(
|
|
4888
|
-
'error',
|
|
4889
|
-
'coordinator.child_submit.error',
|
|
4890
|
-
'[coordinator.child_submit.error]',
|
|
4891
|
-
{
|
|
4892
|
-
runId,
|
|
4893
|
-
ms: Date.now() - startedAt,
|
|
4894
|
-
error: message,
|
|
4895
|
-
},
|
|
4896
|
-
);
|
|
4897
|
-
return Response.json(
|
|
4898
|
-
{
|
|
4899
|
-
error: {
|
|
4900
|
-
code: 'COORDINATOR_CHILD_SUBMIT_FAILED',
|
|
4901
|
-
message,
|
|
4902
|
-
phase: 'coordinator_child_submit',
|
|
4903
|
-
parentRunId: runId,
|
|
4904
|
-
},
|
|
4905
|
-
},
|
|
4906
|
-
{
|
|
4907
|
-
status: 500,
|
|
4908
|
-
headers: { 'cache-control': 'no-store' },
|
|
4909
|
-
},
|
|
4910
|
-
);
|
|
4911
|
-
}
|
|
4912
|
-
}
|
|
4913
|
-
|
|
4914
3194
|
if (action === 'tail') {
|
|
4915
3195
|
const url = new URL(request.url);
|
|
4916
3196
|
const waitMs = Math.min(
|
|
@@ -5120,25 +3400,6 @@ async function handleWorkflowRoute(input: {
|
|
|
5120
3400
|
: eventKey
|
|
5121
3401
|
? `integration_event_${eventKey}`
|
|
5122
3402
|
: 'integration_event';
|
|
5123
|
-
if (body.signal === 'integration_event' && eventKey) {
|
|
5124
|
-
await writeCoordinatorChildTerminalState({
|
|
5125
|
-
env,
|
|
5126
|
-
parentRunId: runId,
|
|
5127
|
-
eventKey,
|
|
5128
|
-
data: body.data ?? body,
|
|
5129
|
-
}).catch((error: unknown) => {
|
|
5130
|
-
recordCoordinatorEvent(
|
|
5131
|
-
'warn',
|
|
5132
|
-
'coordinator.child_terminal.cache_write_failed',
|
|
5133
|
-
'[coordinator] child terminal cache write failed',
|
|
5134
|
-
{
|
|
5135
|
-
runId,
|
|
5136
|
-
eventKey,
|
|
5137
|
-
error: error instanceof Error ? error.message : String(error),
|
|
5138
|
-
},
|
|
5139
|
-
);
|
|
5140
|
-
});
|
|
5141
|
-
}
|
|
5142
3403
|
await instance.sendEvent({
|
|
5143
3404
|
type: workflowEventType(eventType),
|
|
5144
3405
|
payload: body,
|
|
@@ -5943,7 +4204,6 @@ async function handleCoordinatorWarmup(
|
|
|
5943
4204
|
packagedFiles: null,
|
|
5944
4205
|
contractSnapshot: null,
|
|
5945
4206
|
executionPlan: null,
|
|
5946
|
-
playCallGovernance: null,
|
|
5947
4207
|
dynamicWorkerCode: null,
|
|
5948
4208
|
executorToken: 'coordinator-warmup',
|
|
5949
4209
|
baseUrl: env.DEEPLINE_API_BASE_URL,
|
|
@@ -6057,14 +4317,6 @@ function makeRuntimeApiEnvBinding(env: CoordinatorEnv): {
|
|
|
6057
4317
|
function makeCoordinatorControlBinding():
|
|
6058
4318
|
| {
|
|
6059
4319
|
COORDINATOR: {
|
|
6060
|
-
submitChild(
|
|
6061
|
-
parentRunId: string,
|
|
6062
|
-
body: Record<string, unknown>,
|
|
6063
|
-
): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
|
|
6064
|
-
submitWorkflowChild(
|
|
6065
|
-
parentRunId: string,
|
|
6066
|
-
body: Record<string, unknown>,
|
|
6067
|
-
): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
|
|
6068
4320
|
signal(
|
|
6069
4321
|
runId: string,
|
|
6070
4322
|
body: Record<string, unknown>,
|
|
@@ -6080,11 +4332,6 @@ function makeCoordinatorControlBinding():
|
|
|
6080
4332
|
readTerminalState(
|
|
6081
4333
|
runId: string,
|
|
6082
4334
|
): Promise<CoordinatorTerminalState | null>;
|
|
6083
|
-
readChildTerminalState(
|
|
6084
|
-
parentRunId: string,
|
|
6085
|
-
eventKey: string,
|
|
6086
|
-
timeoutMs?: number,
|
|
6087
|
-
): Promise<CoordinatorChildTerminalState | null>;
|
|
6088
4335
|
rateAcquire(input: {
|
|
6089
4336
|
bucketId: string;
|
|
6090
4337
|
rules: Array<{
|
|
@@ -6104,14 +4351,6 @@ function makeCoordinatorControlBinding():
|
|
|
6104
4351
|
| Record<string, never> {
|
|
6105
4352
|
const exports = workersExports as unknown as {
|
|
6106
4353
|
CoordinatorControl?: (init: { props: undefined }) => {
|
|
6107
|
-
submitChild(
|
|
6108
|
-
parentRunId: string,
|
|
6109
|
-
body: Record<string, unknown>,
|
|
6110
|
-
): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
|
|
6111
|
-
submitWorkflowChild(
|
|
6112
|
-
parentRunId: string,
|
|
6113
|
-
body: Record<string, unknown>,
|
|
6114
|
-
): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
|
|
6115
4354
|
signal(
|
|
6116
4355
|
runId: string,
|
|
6117
4356
|
body: Record<string, unknown>,
|
|
@@ -6124,11 +4363,6 @@ function makeCoordinatorControlBinding():
|
|
|
6124
4363
|
readTerminalState(
|
|
6125
4364
|
runId: string,
|
|
6126
4365
|
): Promise<CoordinatorTerminalState | null>;
|
|
6127
|
-
readChildTerminalState(
|
|
6128
|
-
parentRunId: string,
|
|
6129
|
-
eventKey: string,
|
|
6130
|
-
timeoutMs?: number,
|
|
6131
|
-
): Promise<CoordinatorChildTerminalState | null>;
|
|
6132
4366
|
rateAcquire(input: {
|
|
6133
4367
|
bucketId: string;
|
|
6134
4368
|
rules: Array<{
|