deepline 0.1.193 → 0.1.195
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +38 -351
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +92 -3
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +27 -7
- package/dist/bundling-sources/sdk/src/client.ts +61 -1
- package/dist/bundling-sources/sdk/src/index.ts +2 -0
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +11 -0
- package/dist/cli/index.js +309 -35
- package/dist/cli/index.mjs +303 -29
- package/dist/index.d.mts +39 -1
- package/dist/index.d.ts +39 -1
- package/dist/index.js +25 -2
- package/dist/index.mjs +25 -2
- package/package.json +1 -1
|
@@ -31,21 +31,13 @@ import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runti
|
|
|
31
31
|
import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
|
|
32
32
|
import {
|
|
33
33
|
DB_SESSION_DEFAULT_TTL_SECONDS,
|
|
34
|
-
createDbSessionResponseSchema,
|
|
35
|
-
type CreateDbSessionResponse,
|
|
36
34
|
type PreloadedRuntimeDbSession,
|
|
37
35
|
} from '../../../shared_libs/play-runtime/db-session';
|
|
38
36
|
import {
|
|
39
37
|
dbSessionPostgresUrlAad,
|
|
40
|
-
decryptDbSessionPostgresUrlWithPrivateKey,
|
|
41
38
|
encryptDbSessionPostgresUrl,
|
|
42
|
-
generateDbSessionPostgresUrlDecryptionKey,
|
|
43
|
-
type PostgresUrlDecryptionKey,
|
|
44
39
|
} from '../../../shared_libs/play-runtime/db-session-crypto';
|
|
45
|
-
import {
|
|
46
|
-
planRuntimeDbSessionRequirements,
|
|
47
|
-
type RuntimeDbSessionRequirement,
|
|
48
|
-
} from '../../../shared_libs/play-runtime/db-session-plan';
|
|
40
|
+
import { planRuntimeDbSessionRequirements } from '../../../shared_libs/play-runtime/db-session-plan';
|
|
49
41
|
import type {
|
|
50
42
|
PlayRuntimeManifest,
|
|
51
43
|
PlayRuntimeManifestMap,
|
|
@@ -66,7 +58,6 @@ import {
|
|
|
66
58
|
} from '../../../shared_libs/play-runtime/runtime-contract';
|
|
67
59
|
import {
|
|
68
60
|
PLAY_RUNTIME_API_COMPAT_PATH,
|
|
69
|
-
PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
|
|
70
61
|
PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH,
|
|
71
62
|
} from '../../../shared_libs/play-runtime/runtime-api-paths';
|
|
72
63
|
import {
|
|
@@ -1836,286 +1827,6 @@ async function restartWorkflowAfterPlatformReset(input: {
|
|
|
1836
1827
|
}
|
|
1837
1828
|
}
|
|
1838
1829
|
|
|
1839
|
-
async function mintChildWorkflowExecutorToken(input: {
|
|
1840
|
-
env: CoordinatorEnv;
|
|
1841
|
-
baseUrl: string;
|
|
1842
|
-
parentExecutorToken: string;
|
|
1843
|
-
parentRunId: string;
|
|
1844
|
-
parentPlayName: string;
|
|
1845
|
-
childRunId: string;
|
|
1846
|
-
childPlayName: string;
|
|
1847
|
-
maxCreditsPerRun?: number | null;
|
|
1848
|
-
}): Promise<string> {
|
|
1849
|
-
const response = await input.env.HARNESS.runtimeApiCall({
|
|
1850
|
-
contract: PLAY_RUNTIME_CONTRACT,
|
|
1851
|
-
executorToken: input.parentExecutorToken,
|
|
1852
|
-
baseUrl: input.baseUrl,
|
|
1853
|
-
path: PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
|
|
1854
|
-
headers: {
|
|
1855
|
-
'x-deepline-request-id': crypto.randomUUID(),
|
|
1856
|
-
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
1857
|
-
},
|
|
1858
|
-
timeoutMs: 15_000,
|
|
1859
|
-
body: {
|
|
1860
|
-
parentRunId: input.parentRunId,
|
|
1861
|
-
parentPlayName: input.parentPlayName,
|
|
1862
|
-
childRunId: input.childRunId,
|
|
1863
|
-
childPlayName: input.childPlayName,
|
|
1864
|
-
maxCreditsPerRun: input.maxCreditsPerRun ?? null,
|
|
1865
|
-
},
|
|
1866
|
-
});
|
|
1867
|
-
const text = response.body;
|
|
1868
|
-
let parsed: Record<string, unknown> = {};
|
|
1869
|
-
try {
|
|
1870
|
-
parsed = text ? (JSON.parse(text) as Record<string, unknown>) : {};
|
|
1871
|
-
} catch {
|
|
1872
|
-
parsed = {};
|
|
1873
|
-
}
|
|
1874
|
-
if (response.status < 200 || response.status >= 300) {
|
|
1875
|
-
const error = isRecord(parsed.error) ? parsed.error : null;
|
|
1876
|
-
const message =
|
|
1877
|
-
(typeof error?.message === 'string' && error.message.trim()) ||
|
|
1878
|
-
(typeof parsed.error === 'string' && parsed.error.trim()) ||
|
|
1879
|
-
text.slice(0, 800) ||
|
|
1880
|
-
`Origin child executor token mint failed with ${response.status}.`;
|
|
1881
|
-
throw new Error(message);
|
|
1882
|
-
}
|
|
1883
|
-
const executorToken = parsed.executorToken;
|
|
1884
|
-
if (typeof executorToken !== 'string' || !executorToken.trim()) {
|
|
1885
|
-
throw new Error(
|
|
1886
|
-
'Origin child executor token response was missing executorToken.',
|
|
1887
|
-
);
|
|
1888
|
-
}
|
|
1889
|
-
return executorToken;
|
|
1890
|
-
}
|
|
1891
|
-
|
|
1892
|
-
function stripDbSessionUrl(
|
|
1893
|
-
session: CreateDbSessionResponse,
|
|
1894
|
-
): Omit<CreateDbSessionResponse, 'postgresUrl' | 'encryptedPostgresUrl'> {
|
|
1895
|
-
const {
|
|
1896
|
-
postgresUrl: _postgresUrl,
|
|
1897
|
-
encryptedPostgresUrl: _encryptedPostgresUrl,
|
|
1898
|
-
...sessionWithoutUrl
|
|
1899
|
-
} = session;
|
|
1900
|
-
void _postgresUrl;
|
|
1901
|
-
void _encryptedPostgresUrl;
|
|
1902
|
-
return sessionWithoutUrl;
|
|
1903
|
-
}
|
|
1904
|
-
|
|
1905
|
-
function assertChildDbSessionScope(input: {
|
|
1906
|
-
session: CreateDbSessionResponse;
|
|
1907
|
-
requirement: RuntimeDbSessionRequirement;
|
|
1908
|
-
orgId: string;
|
|
1909
|
-
childPlayName: string;
|
|
1910
|
-
}): void {
|
|
1911
|
-
const { session, requirement, orgId, childPlayName } = input;
|
|
1912
|
-
if (
|
|
1913
|
-
session.playName !== childPlayName ||
|
|
1914
|
-
session.target.orgId !== orgId ||
|
|
1915
|
-
session.target.tableNamespace !== requirement.tableNamespace ||
|
|
1916
|
-
session.target.logicalTable !== requirement.logicalTable
|
|
1917
|
-
) {
|
|
1918
|
-
throw new Error(
|
|
1919
|
-
'Runtime API returned a DB session outside the child play scope.',
|
|
1920
|
-
);
|
|
1921
|
-
}
|
|
1922
|
-
const expectedOperations = [...requirement.operations].sort().join(',');
|
|
1923
|
-
const actualOperations = [...session.operations].sort().join(',');
|
|
1924
|
-
if (actualOperations !== expectedOperations) {
|
|
1925
|
-
throw new Error(
|
|
1926
|
-
'Runtime API returned a DB session with unexpected operations.',
|
|
1927
|
-
);
|
|
1928
|
-
}
|
|
1929
|
-
}
|
|
1930
|
-
|
|
1931
|
-
async function reencryptChildDbSessionForExecutor(input: {
|
|
1932
|
-
session: CreateDbSessionResponse;
|
|
1933
|
-
decryptionKey: PostgresUrlDecryptionKey;
|
|
1934
|
-
childExecutorToken: string;
|
|
1935
|
-
}): Promise<CreateDbSessionResponse> {
|
|
1936
|
-
const sessionWithoutUrl = stripDbSessionUrl(input.session);
|
|
1937
|
-
if (!input.session.encryptedPostgresUrl) {
|
|
1938
|
-
throw new Error('Runtime API returned an unencrypted DB session URL.');
|
|
1939
|
-
}
|
|
1940
|
-
const postgresUrl = await decryptDbSessionPostgresUrlWithPrivateKey({
|
|
1941
|
-
encrypted: input.session.encryptedPostgresUrl,
|
|
1942
|
-
privateKey: input.decryptionKey.privateKey,
|
|
1943
|
-
aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
|
|
1944
|
-
});
|
|
1945
|
-
return {
|
|
1946
|
-
...sessionWithoutUrl,
|
|
1947
|
-
encryptedPostgresUrl: await encryptDbSessionPostgresUrl({
|
|
1948
|
-
postgresUrl,
|
|
1949
|
-
secret: input.childExecutorToken,
|
|
1950
|
-
aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
|
|
1951
|
-
}),
|
|
1952
|
-
};
|
|
1953
|
-
}
|
|
1954
|
-
|
|
1955
|
-
async function createChildRuntimeDbSession(input: {
|
|
1956
|
-
env: CoordinatorEnv;
|
|
1957
|
-
baseUrl: string;
|
|
1958
|
-
childExecutorToken: string;
|
|
1959
|
-
childPlayName: string;
|
|
1960
|
-
requirement: RuntimeDbSessionRequirement;
|
|
1961
|
-
userEmail: string;
|
|
1962
|
-
orgId: string;
|
|
1963
|
-
}): Promise<CreateDbSessionResponse> {
|
|
1964
|
-
const decryptionKey = await generateDbSessionPostgresUrlDecryptionKey();
|
|
1965
|
-
const response = await input.env.HARNESS.runtimeApiCall({
|
|
1966
|
-
contract: PLAY_RUNTIME_CONTRACT,
|
|
1967
|
-
executorToken: input.childExecutorToken,
|
|
1968
|
-
baseUrl: input.baseUrl,
|
|
1969
|
-
path: PLAY_RUNTIME_API_COMPAT_PATH,
|
|
1970
|
-
headers: {
|
|
1971
|
-
'x-deepline-request-id': crypto.randomUUID(),
|
|
1972
|
-
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
1973
|
-
},
|
|
1974
|
-
timeoutMs: 15_000,
|
|
1975
|
-
body: {
|
|
1976
|
-
action: 'create_db_session',
|
|
1977
|
-
playName: input.childPlayName,
|
|
1978
|
-
target: {
|
|
1979
|
-
tableNamespace: input.requirement.tableNamespace,
|
|
1980
|
-
logicalTable: input.requirement.logicalTable,
|
|
1981
|
-
},
|
|
1982
|
-
operations: input.requirement.operations,
|
|
1983
|
-
limits: input.requirement.limits,
|
|
1984
|
-
sheetContract: input.requirement.sheetContract ?? null,
|
|
1985
|
-
ttlSeconds: DB_SESSION_DEFAULT_TTL_SECONDS,
|
|
1986
|
-
userEmail: input.userEmail,
|
|
1987
|
-
postgresUrlEncryption: decryptionKey.request,
|
|
1988
|
-
},
|
|
1989
|
-
});
|
|
1990
|
-
const text = response.body;
|
|
1991
|
-
let parsed: unknown = {};
|
|
1992
|
-
try {
|
|
1993
|
-
parsed = text ? JSON.parse(text) : {};
|
|
1994
|
-
} catch {
|
|
1995
|
-
parsed = {};
|
|
1996
|
-
}
|
|
1997
|
-
if (response.status < 200 || response.status >= 300) {
|
|
1998
|
-
const error =
|
|
1999
|
-
isRecord(parsed) && isRecord(parsed.error) ? parsed.error : {};
|
|
2000
|
-
const message =
|
|
2001
|
-
(typeof error.message === 'string' && error.message.trim()) ||
|
|
2002
|
-
(isRecord(parsed) &&
|
|
2003
|
-
typeof parsed.error === 'string' &&
|
|
2004
|
-
parsed.error.trim()
|
|
2005
|
-
? parsed.error.trim()
|
|
2006
|
-
: '') ||
|
|
2007
|
-
text.slice(0, 800) ||
|
|
2008
|
-
`Origin DB session mint failed with ${response.status}.`;
|
|
2009
|
-
throw new Error(message);
|
|
2010
|
-
}
|
|
2011
|
-
const session = createDbSessionResponseSchema.parse(parsed);
|
|
2012
|
-
assertChildDbSessionScope({
|
|
2013
|
-
session,
|
|
2014
|
-
requirement: input.requirement,
|
|
2015
|
-
orgId: input.orgId,
|
|
2016
|
-
childPlayName: input.childPlayName,
|
|
2017
|
-
});
|
|
2018
|
-
return await reencryptChildDbSessionForExecutor({
|
|
2019
|
-
session,
|
|
2020
|
-
decryptionKey,
|
|
2021
|
-
childExecutorToken: input.childExecutorToken,
|
|
2022
|
-
});
|
|
2023
|
-
}
|
|
2024
|
-
|
|
2025
|
-
async function preloadChildRuntimeDbSessions(input: {
|
|
2026
|
-
env: CoordinatorEnv;
|
|
2027
|
-
baseUrl: string;
|
|
2028
|
-
childExecutorToken: string;
|
|
2029
|
-
childRunId: string;
|
|
2030
|
-
childPlayName: string;
|
|
2031
|
-
manifest: PlayRuntimeManifest;
|
|
2032
|
-
orgId: string;
|
|
2033
|
-
userEmail: string;
|
|
2034
|
-
}): Promise<PreloadedRuntimeDbSession[]> {
|
|
2035
|
-
const startedAt = Date.now();
|
|
2036
|
-
const requirements = planRuntimeDbSessionRequirements(
|
|
2037
|
-
input.manifest.staticPipeline ?? null,
|
|
2038
|
-
);
|
|
2039
|
-
const sessions = await Promise.all(
|
|
2040
|
-
requirements.map(async (requirement) => ({
|
|
2041
|
-
tableNamespace: requirement.tableNamespace,
|
|
2042
|
-
logicalTable: requirement.logicalTable,
|
|
2043
|
-
operations: requirement.operations,
|
|
2044
|
-
...(requirement.limits ? { limits: requirement.limits } : {}),
|
|
2045
|
-
session: await createChildRuntimeDbSession({
|
|
2046
|
-
env: input.env,
|
|
2047
|
-
baseUrl: input.baseUrl,
|
|
2048
|
-
childExecutorToken: input.childExecutorToken,
|
|
2049
|
-
childPlayName: input.childPlayName,
|
|
2050
|
-
requirement,
|
|
2051
|
-
userEmail: input.userEmail,
|
|
2052
|
-
orgId: input.orgId,
|
|
2053
|
-
}),
|
|
2054
|
-
})),
|
|
2055
|
-
);
|
|
2056
|
-
recordCoordinatorPerfTrace({
|
|
2057
|
-
runId: input.childRunId,
|
|
2058
|
-
phase: 'coordinator.child_db_session_preload',
|
|
2059
|
-
ms: Date.now() - startedAt,
|
|
2060
|
-
graphHash: input.manifest.graphHash,
|
|
2061
|
-
extra: { sessions: sessions.length },
|
|
2062
|
-
});
|
|
2063
|
-
return sessions;
|
|
2064
|
-
}
|
|
2065
|
-
|
|
2066
|
-
async function registerChildRunWithRuntime(input: {
|
|
2067
|
-
env: CoordinatorEnv;
|
|
2068
|
-
baseUrl: string;
|
|
2069
|
-
childExecutorToken: string;
|
|
2070
|
-
childRunId: string;
|
|
2071
|
-
childPlayName: string;
|
|
2072
|
-
manifest: PlayRuntimeManifest;
|
|
2073
|
-
governance: PlayCallGovernanceSnapshot;
|
|
2074
|
-
runtimeBackend: string;
|
|
2075
|
-
schedulerBackend: string;
|
|
2076
|
-
executionProfile: string;
|
|
2077
|
-
}): Promise<void> {
|
|
2078
|
-
const response = await input.env.HARNESS.runtimeApiCall({
|
|
2079
|
-
contract: PLAY_RUNTIME_CONTRACT,
|
|
2080
|
-
executorToken: input.childExecutorToken,
|
|
2081
|
-
baseUrl: input.baseUrl,
|
|
2082
|
-
path: PLAY_RUNTIME_API_COMPAT_PATH,
|
|
2083
|
-
headers: {
|
|
2084
|
-
'x-deepline-request-id': crypto.randomUUID(),
|
|
2085
|
-
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
2086
|
-
},
|
|
2087
|
-
timeoutMs: 15_000,
|
|
2088
|
-
body: {
|
|
2089
|
-
action: 'start_inline_child_run',
|
|
2090
|
-
playName: input.childPlayName,
|
|
2091
|
-
runId: input.childRunId,
|
|
2092
|
-
parentRunId: input.governance.parentRunId,
|
|
2093
|
-
rootRunId: input.governance.rootRunId,
|
|
2094
|
-
workflowFamilyKey:
|
|
2095
|
-
input.governance.rootRunId ??
|
|
2096
|
-
input.governance.parentRunId ??
|
|
2097
|
-
input.childRunId,
|
|
2098
|
-
artifactStorageKey: input.manifest.artifactStorageKey,
|
|
2099
|
-
artifactHash: input.manifest.artifactHash,
|
|
2100
|
-
graphHash: input.manifest.graphHash,
|
|
2101
|
-
runtimeBackend: input.runtimeBackend,
|
|
2102
|
-
schedulerBackend: input.schedulerBackend,
|
|
2103
|
-
executionProfile: input.executionProfile,
|
|
2104
|
-
...(typeof input.manifest.maxCreditsPerRun === 'number'
|
|
2105
|
-
? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
|
|
2106
|
-
: {}),
|
|
2107
|
-
staticPipeline: input.manifest.staticPipeline ?? null,
|
|
2108
|
-
source: 'published',
|
|
2109
|
-
},
|
|
2110
|
-
});
|
|
2111
|
-
if (response.status < 200 || response.status >= 300) {
|
|
2112
|
-
const text = response.body ?? '';
|
|
2113
|
-
throw new Error(
|
|
2114
|
-
`Inline child run registration failed ${response.status}: ${text.slice(0, 800)}`,
|
|
2115
|
-
);
|
|
2116
|
-
}
|
|
2117
|
-
}
|
|
2118
|
-
|
|
2119
1830
|
async function markRegisteredChildRunFailed(input: {
|
|
2120
1831
|
env: CoordinatorEnv;
|
|
2121
1832
|
baseUrl: string;
|
|
@@ -2310,6 +2021,9 @@ async function prepareInlineChildRunWithRuntime(input: {
|
|
|
2310
2021
|
manifest: PlayRuntimeManifest;
|
|
2311
2022
|
governance: PlayCallGovernanceSnapshot;
|
|
2312
2023
|
userEmail: string;
|
|
2024
|
+
runtimeBackend?: string;
|
|
2025
|
+
schedulerBackend?: string;
|
|
2026
|
+
executionProfile?: string;
|
|
2313
2027
|
}): Promise<{
|
|
2314
2028
|
childToken: string;
|
|
2315
2029
|
preloadedDbSessions: PreloadedRuntimeDbSession[];
|
|
@@ -2334,9 +2048,9 @@ async function prepareInlineChildRunWithRuntime(input: {
|
|
|
2334
2048
|
artifactStorageKey: input.manifest.artifactStorageKey,
|
|
2335
2049
|
artifactHash: input.manifest.artifactHash,
|
|
2336
2050
|
graphHash: input.manifest.graphHash,
|
|
2337
|
-
runtimeBackend: 'workers_edge',
|
|
2338
|
-
schedulerBackend: 'inline_child',
|
|
2339
|
-
executionProfile: 'workers_edge',
|
|
2051
|
+
runtimeBackend: input.runtimeBackend ?? 'workers_edge',
|
|
2052
|
+
schedulerBackend: input.schedulerBackend ?? 'inline_child',
|
|
2053
|
+
executionProfile: input.executionProfile ?? 'workers_edge',
|
|
2340
2054
|
...(typeof input.manifest.maxCreditsPerRun === 'number'
|
|
2341
2055
|
? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
|
|
2342
2056
|
: {}),
|
|
@@ -3218,43 +2932,39 @@ async function submitChildWorkflowThroughCoordinator(input: {
|
|
|
3218
2932
|
});
|
|
3219
2933
|
const baseUrl = resolveRuntimeApiBaseUrl(input.env, input.body);
|
|
3220
2934
|
|
|
3221
|
-
const
|
|
3222
|
-
const childToken
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
env: input.env,
|
|
3244
|
-
baseUrl,
|
|
3245
|
-
childExecutorToken: childToken,
|
|
3246
|
-
childRunId,
|
|
3247
|
-
childPlayName,
|
|
3248
|
-
manifest,
|
|
3249
|
-
orgId,
|
|
3250
|
-
userEmail:
|
|
3251
|
-
typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
|
|
3252
|
-
});
|
|
2935
|
+
const prepareStartedAt = Date.now();
|
|
2936
|
+
const { childToken, preloadedDbSessions, prepareTimings, transportTimings } =
|
|
2937
|
+
await prepareInlineChildRunWithRuntime({
|
|
2938
|
+
env: input.env,
|
|
2939
|
+
baseUrl,
|
|
2940
|
+
parentExecutorToken,
|
|
2941
|
+
parentRunId: input.parentRunId,
|
|
2942
|
+
parentPlayName:
|
|
2943
|
+
typeof input.body.parentPlayName === 'string' &&
|
|
2944
|
+
input.body.parentPlayName.trim()
|
|
2945
|
+
? input.body.parentPlayName.trim()
|
|
2946
|
+
: governance.parentPlayName,
|
|
2947
|
+
childRunId,
|
|
2948
|
+
childPlayName,
|
|
2949
|
+
manifest,
|
|
2950
|
+
governance,
|
|
2951
|
+
userEmail:
|
|
2952
|
+
typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
|
|
2953
|
+
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
2954
|
+
schedulerBackend: 'cf_workflows',
|
|
2955
|
+
executionProfile: 'workers_edge',
|
|
2956
|
+
});
|
|
3253
2957
|
trace(
|
|
3254
|
-
'coordinator.
|
|
3255
|
-
|
|
2958
|
+
'coordinator.child_submit_prepare',
|
|
2959
|
+
prepareStartedAt,
|
|
3256
2960
|
manifest.graphHash,
|
|
3257
|
-
{
|
|
2961
|
+
{
|
|
2962
|
+
childRunId,
|
|
2963
|
+
childPlayName,
|
|
2964
|
+
sessions: preloadedDbSessions.length,
|
|
2965
|
+
prepareTimings,
|
|
2966
|
+
transportTimings,
|
|
2967
|
+
},
|
|
3258
2968
|
);
|
|
3259
2969
|
|
|
3260
2970
|
const params = buildChildWorkflowParams({
|
|
@@ -3274,29 +2984,6 @@ async function submitChildWorkflowThroughCoordinator(input: {
|
|
|
3274
2984
|
preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
|
|
3275
2985
|
});
|
|
3276
2986
|
|
|
3277
|
-
const registerStartedAt = Date.now();
|
|
3278
|
-
await registerChildRunWithRuntime({
|
|
3279
|
-
env: input.env,
|
|
3280
|
-
baseUrl,
|
|
3281
|
-
childExecutorToken: childToken,
|
|
3282
|
-
childRunId,
|
|
3283
|
-
childPlayName,
|
|
3284
|
-
manifest,
|
|
3285
|
-
governance,
|
|
3286
|
-
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
3287
|
-
schedulerBackend: 'cf_workflows',
|
|
3288
|
-
executionProfile: 'workers_edge',
|
|
3289
|
-
});
|
|
3290
|
-
trace(
|
|
3291
|
-
'coordinator.child_submit_register_run',
|
|
3292
|
-
registerStartedAt,
|
|
3293
|
-
manifest.graphHash,
|
|
3294
|
-
{
|
|
3295
|
-
childRunId,
|
|
3296
|
-
childPlayName,
|
|
3297
|
-
},
|
|
3298
|
-
);
|
|
3299
|
-
|
|
3300
2987
|
const workflowSubmitStartedAt = Date.now();
|
|
3301
2988
|
let response: Response | null = null;
|
|
3302
2989
|
let responseText = '';
|
|
@@ -1323,6 +1323,14 @@ function redactSecretsFromLogString(value: string): string {
|
|
|
1323
1323
|
);
|
|
1324
1324
|
}
|
|
1325
1325
|
|
|
1326
|
+
function safeRuntimeApiOrigin(value: string): string {
|
|
1327
|
+
try {
|
|
1328
|
+
return new URL(value).origin;
|
|
1329
|
+
} catch {
|
|
1330
|
+
return '<invalid>';
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1326
1334
|
async function postRuntimeApi<T>(
|
|
1327
1335
|
baseUrl: string,
|
|
1328
1336
|
executorToken: string,
|
|
@@ -1336,6 +1344,9 @@ async function postRuntimeApi<T>(
|
|
|
1336
1344
|
// Routes through the in-process RUNTIME_API service binding. Missing binding
|
|
1337
1345
|
// is an infra error in workers_edge, not a reason to fall back to public HTTP.
|
|
1338
1346
|
const serializedBody = JSON.stringify(body);
|
|
1347
|
+
const requestDescription = describeRuntimeApiBody(body);
|
|
1348
|
+
const maxAttempts = RUNTIME_API_RETRY_DELAYS_MS.length + 1;
|
|
1349
|
+
const baseOrigin = safeRuntimeApiOrigin(baseUrl);
|
|
1339
1350
|
let lastError: unknown = null;
|
|
1340
1351
|
for (
|
|
1341
1352
|
let attempt = 0;
|
|
@@ -1343,6 +1354,7 @@ async function postRuntimeApi<T>(
|
|
|
1343
1354
|
attempt += 1
|
|
1344
1355
|
) {
|
|
1345
1356
|
let res: Response;
|
|
1357
|
+
const requestId = makeRequestId();
|
|
1346
1358
|
try {
|
|
1347
1359
|
res = await fetchRuntimeApi(
|
|
1348
1360
|
baseUrl,
|
|
@@ -1352,7 +1364,7 @@ async function postRuntimeApi<T>(
|
|
|
1352
1364
|
headers: {
|
|
1353
1365
|
'content-type': 'application/json',
|
|
1354
1366
|
authorization: `Bearer ${executorToken}`,
|
|
1355
|
-
'x-deepline-request-id':
|
|
1367
|
+
'x-deepline-request-id': requestId,
|
|
1356
1368
|
...(options.runtimeTestFaultHeader?.trim()
|
|
1357
1369
|
? {
|
|
1358
1370
|
[PLAY_RUNTIME_TEST_FAULT_HEADER]:
|
|
@@ -1373,8 +1385,25 @@ async function postRuntimeApi<T>(
|
|
|
1373
1385
|
attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
|
|
1374
1386
|
!isRetryableWorkerRuntimeApiError(error)
|
|
1375
1387
|
) {
|
|
1388
|
+
console.warn('[play-harness.runtime_api.exhausted]', {
|
|
1389
|
+
request: requestDescription,
|
|
1390
|
+
attempt: attempt + 1,
|
|
1391
|
+
maxAttempts,
|
|
1392
|
+
requestId,
|
|
1393
|
+
baseOrigin,
|
|
1394
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1395
|
+
});
|
|
1376
1396
|
throw error;
|
|
1377
1397
|
}
|
|
1398
|
+
console.warn('[play-harness.runtime_api.retry]', {
|
|
1399
|
+
request: requestDescription,
|
|
1400
|
+
attempt: attempt + 1,
|
|
1401
|
+
maxAttempts,
|
|
1402
|
+
requestId,
|
|
1403
|
+
baseOrigin,
|
|
1404
|
+
retryInMs: RUNTIME_API_RETRY_DELAYS_MS[attempt] ?? 0,
|
|
1405
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1406
|
+
});
|
|
1378
1407
|
await sleepRuntimeApiRetry(attempt);
|
|
1379
1408
|
continue;
|
|
1380
1409
|
}
|
|
@@ -1390,8 +1419,27 @@ async function postRuntimeApi<T>(
|
|
|
1390
1419
|
attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
|
|
1391
1420
|
!isRetryableRuntimeApiResponse(res.status, text)
|
|
1392
1421
|
) {
|
|
1422
|
+
console.warn('[play-harness.runtime_api.exhausted]', {
|
|
1423
|
+
request: requestDescription,
|
|
1424
|
+
attempt: attempt + 1,
|
|
1425
|
+
maxAttempts,
|
|
1426
|
+
requestId,
|
|
1427
|
+
baseOrigin,
|
|
1428
|
+
status: res.status,
|
|
1429
|
+
body: redacted,
|
|
1430
|
+
});
|
|
1393
1431
|
throw lastError;
|
|
1394
1432
|
}
|
|
1433
|
+
console.warn('[play-harness.runtime_api.retry]', {
|
|
1434
|
+
request: requestDescription,
|
|
1435
|
+
attempt: attempt + 1,
|
|
1436
|
+
maxAttempts,
|
|
1437
|
+
requestId,
|
|
1438
|
+
baseOrigin,
|
|
1439
|
+
status: res.status,
|
|
1440
|
+
retryInMs: RUNTIME_API_RETRY_DELAYS_MS[attempt] ?? 0,
|
|
1441
|
+
body: redacted,
|
|
1442
|
+
});
|
|
1395
1443
|
await sleepRuntimeApiRetry(attempt);
|
|
1396
1444
|
}
|
|
1397
1445
|
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
@@ -1628,6 +1676,42 @@ function workerChildRevisionFingerprint(input: {
|
|
|
1628
1676
|
);
|
|
1629
1677
|
}
|
|
1630
1678
|
|
|
1679
|
+
type RuntimeResolvedPlayResponse = {
|
|
1680
|
+
manifest?: PlayRuntimeManifest | null;
|
|
1681
|
+
};
|
|
1682
|
+
|
|
1683
|
+
async function resolveAuthorizedWorkerChildManifest(input: {
|
|
1684
|
+
req: RunRequest;
|
|
1685
|
+
playRef: string;
|
|
1686
|
+
cache: Map<string, PlayRuntimeManifest>;
|
|
1687
|
+
}): Promise<PlayRuntimeManifest | null> {
|
|
1688
|
+
const cached = input.cache.get(input.playRef);
|
|
1689
|
+
if (cached) return cached;
|
|
1690
|
+
const preloaded = input.req.childPlayManifests?.[input.playRef];
|
|
1691
|
+
if (preloaded) {
|
|
1692
|
+
input.cache.set(input.playRef, preloaded);
|
|
1693
|
+
return preloaded;
|
|
1694
|
+
}
|
|
1695
|
+
const response = await postRuntimeApi<RuntimeResolvedPlayResponse>(
|
|
1696
|
+
input.req.baseUrl,
|
|
1697
|
+
input.req.executorToken,
|
|
1698
|
+
{
|
|
1699
|
+
action: 'resolve_play',
|
|
1700
|
+
playRef: input.playRef,
|
|
1701
|
+
artifactKind: 'esm_workers',
|
|
1702
|
+
},
|
|
1703
|
+
{
|
|
1704
|
+
runtimeTestFaultHeader: input.req.runtimeTestFaultHeader,
|
|
1705
|
+
timeoutMsOverride: 30_000,
|
|
1706
|
+
},
|
|
1707
|
+
);
|
|
1708
|
+
const manifest = response.manifest ?? null;
|
|
1709
|
+
if (manifest) {
|
|
1710
|
+
input.cache.set(input.playRef, manifest);
|
|
1711
|
+
}
|
|
1712
|
+
return manifest;
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1631
1715
|
async function childPlayEventKey(input: {
|
|
1632
1716
|
key: string;
|
|
1633
1717
|
workflowId: string;
|
|
@@ -4269,6 +4353,7 @@ function createMinimalWorkerCtx(
|
|
|
4269
4353
|
// lineage snapshot are owned by the Governor (createGovernorForRun above).
|
|
4270
4354
|
// The worker keeps only substrate mechanism here.
|
|
4271
4355
|
const stepCallCounts: Record<string, number> = {};
|
|
4356
|
+
const childManifestCache = new Map<string, PlayRuntimeManifest>();
|
|
4272
4357
|
const secretRedactor = createSecretRedactionContext();
|
|
4273
4358
|
|
|
4274
4359
|
const resolveSecretAuth = async (auth?: SecretAuth) => {
|
|
@@ -6790,7 +6875,11 @@ function createMinimalWorkerCtx(
|
|
|
6790
6875
|
if (!resolvedName) {
|
|
6791
6876
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
6792
6877
|
}
|
|
6793
|
-
const childManifest =
|
|
6878
|
+
const childManifest = await resolveAuthorizedWorkerChildManifest({
|
|
6879
|
+
req,
|
|
6880
|
+
playRef: resolvedName,
|
|
6881
|
+
cache: childManifestCache,
|
|
6882
|
+
});
|
|
6794
6883
|
const receiptKey = buildDurableCtxCallCacheKey({
|
|
6795
6884
|
orgId: req.orgId,
|
|
6796
6885
|
playId: req.playName,
|
|
@@ -6831,7 +6920,7 @@ function createMinimalWorkerCtx(
|
|
|
6831
6920
|
});
|
|
6832
6921
|
if (!childManifest) {
|
|
6833
6922
|
throw new Error(
|
|
6834
|
-
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}:
|
|
6923
|
+
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: not callable.`,
|
|
6835
6924
|
);
|
|
6836
6925
|
}
|
|
6837
6926
|
const childIsDatasetBacked = childPipelineUsesCtxDataset(
|
|
@@ -15,6 +15,31 @@ export type WorkflowRetryDecision =
|
|
|
15
15
|
message: string | null;
|
|
16
16
|
};
|
|
17
17
|
|
|
18
|
+
function workflowRetryReasonForFailure(failure: {
|
|
19
|
+
code: string;
|
|
20
|
+
cause?: string;
|
|
21
|
+
}): string {
|
|
22
|
+
if (failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED') {
|
|
23
|
+
return 'cloudflare_worker_subrequest_limit';
|
|
24
|
+
}
|
|
25
|
+
if (
|
|
26
|
+
failure.cause &&
|
|
27
|
+
/runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i.test(
|
|
28
|
+
failure.cause,
|
|
29
|
+
)
|
|
30
|
+
) {
|
|
31
|
+
return 'vercel_runtime_api_deployment_missing';
|
|
32
|
+
}
|
|
33
|
+
return 'cloudflare_durable_object_reset';
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function workflowRetryExhaustedReasonForFailure(failure: {
|
|
37
|
+
code: string;
|
|
38
|
+
cause?: string;
|
|
39
|
+
}): string {
|
|
40
|
+
return `${workflowRetryReasonForFailure(failure)}_retry_exhausted`;
|
|
41
|
+
}
|
|
42
|
+
|
|
18
43
|
export function decideWorkflowPlatformRetry(input: {
|
|
19
44
|
workflowStatus: string;
|
|
20
45
|
error: string | null;
|
|
@@ -33,10 +58,7 @@ export function decideWorkflowPlatformRetry(input: {
|
|
|
33
58
|
) {
|
|
34
59
|
return {
|
|
35
60
|
action: 'retry',
|
|
36
|
-
reason:
|
|
37
|
-
failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED'
|
|
38
|
-
? 'cloudflare_worker_subrequest_limit'
|
|
39
|
-
: 'cloudflare_durable_object_reset',
|
|
61
|
+
reason: workflowRetryReasonForFailure(failure),
|
|
40
62
|
nextAttempt: input.retryAttempts + 1,
|
|
41
63
|
message: failure.message,
|
|
42
64
|
};
|
|
@@ -44,9 +66,7 @@ export function decideWorkflowPlatformRetry(input: {
|
|
|
44
66
|
return {
|
|
45
67
|
action: 'fail',
|
|
46
68
|
reason: retryablePlatformReset
|
|
47
|
-
? failure
|
|
48
|
-
? 'cloudflare_worker_subrequest_limit_retry_exhausted'
|
|
49
|
-
: 'cloudflare_durable_object_reset_retry_exhausted'
|
|
69
|
+
? workflowRetryExhaustedReasonForFailure(failure)
|
|
50
70
|
: null,
|
|
51
71
|
message: failure.message,
|
|
52
72
|
};
|