deepline 0.1.193 → 0.1.195

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -31,21 +31,13 @@ import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runti
31
31
  import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
32
32
  import {
33
33
  DB_SESSION_DEFAULT_TTL_SECONDS,
34
- createDbSessionResponseSchema,
35
- type CreateDbSessionResponse,
36
34
  type PreloadedRuntimeDbSession,
37
35
  } from '../../../shared_libs/play-runtime/db-session';
38
36
  import {
39
37
  dbSessionPostgresUrlAad,
40
- decryptDbSessionPostgresUrlWithPrivateKey,
41
38
  encryptDbSessionPostgresUrl,
42
- generateDbSessionPostgresUrlDecryptionKey,
43
- type PostgresUrlDecryptionKey,
44
39
  } from '../../../shared_libs/play-runtime/db-session-crypto';
45
- import {
46
- planRuntimeDbSessionRequirements,
47
- type RuntimeDbSessionRequirement,
48
- } from '../../../shared_libs/play-runtime/db-session-plan';
40
+ import { planRuntimeDbSessionRequirements } from '../../../shared_libs/play-runtime/db-session-plan';
49
41
  import type {
50
42
  PlayRuntimeManifest,
51
43
  PlayRuntimeManifestMap,
@@ -66,7 +58,6 @@ import {
66
58
  } from '../../../shared_libs/play-runtime/runtime-contract';
67
59
  import {
68
60
  PLAY_RUNTIME_API_COMPAT_PATH,
69
- PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
70
61
  PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH,
71
62
  } from '../../../shared_libs/play-runtime/runtime-api-paths';
72
63
  import {
@@ -1836,286 +1827,6 @@ async function restartWorkflowAfterPlatformReset(input: {
1836
1827
  }
1837
1828
  }
1838
1829
 
1839
- async function mintChildWorkflowExecutorToken(input: {
1840
- env: CoordinatorEnv;
1841
- baseUrl: string;
1842
- parentExecutorToken: string;
1843
- parentRunId: string;
1844
- parentPlayName: string;
1845
- childRunId: string;
1846
- childPlayName: string;
1847
- maxCreditsPerRun?: number | null;
1848
- }): Promise<string> {
1849
- const response = await input.env.HARNESS.runtimeApiCall({
1850
- contract: PLAY_RUNTIME_CONTRACT,
1851
- executorToken: input.parentExecutorToken,
1852
- baseUrl: input.baseUrl,
1853
- path: PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
1854
- headers: {
1855
- 'x-deepline-request-id': crypto.randomUUID(),
1856
- [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
1857
- },
1858
- timeoutMs: 15_000,
1859
- body: {
1860
- parentRunId: input.parentRunId,
1861
- parentPlayName: input.parentPlayName,
1862
- childRunId: input.childRunId,
1863
- childPlayName: input.childPlayName,
1864
- maxCreditsPerRun: input.maxCreditsPerRun ?? null,
1865
- },
1866
- });
1867
- const text = response.body;
1868
- let parsed: Record<string, unknown> = {};
1869
- try {
1870
- parsed = text ? (JSON.parse(text) as Record<string, unknown>) : {};
1871
- } catch {
1872
- parsed = {};
1873
- }
1874
- if (response.status < 200 || response.status >= 300) {
1875
- const error = isRecord(parsed.error) ? parsed.error : null;
1876
- const message =
1877
- (typeof error?.message === 'string' && error.message.trim()) ||
1878
- (typeof parsed.error === 'string' && parsed.error.trim()) ||
1879
- text.slice(0, 800) ||
1880
- `Origin child executor token mint failed with ${response.status}.`;
1881
- throw new Error(message);
1882
- }
1883
- const executorToken = parsed.executorToken;
1884
- if (typeof executorToken !== 'string' || !executorToken.trim()) {
1885
- throw new Error(
1886
- 'Origin child executor token response was missing executorToken.',
1887
- );
1888
- }
1889
- return executorToken;
1890
- }
1891
-
1892
- function stripDbSessionUrl(
1893
- session: CreateDbSessionResponse,
1894
- ): Omit<CreateDbSessionResponse, 'postgresUrl' | 'encryptedPostgresUrl'> {
1895
- const {
1896
- postgresUrl: _postgresUrl,
1897
- encryptedPostgresUrl: _encryptedPostgresUrl,
1898
- ...sessionWithoutUrl
1899
- } = session;
1900
- void _postgresUrl;
1901
- void _encryptedPostgresUrl;
1902
- return sessionWithoutUrl;
1903
- }
1904
-
1905
- function assertChildDbSessionScope(input: {
1906
- session: CreateDbSessionResponse;
1907
- requirement: RuntimeDbSessionRequirement;
1908
- orgId: string;
1909
- childPlayName: string;
1910
- }): void {
1911
- const { session, requirement, orgId, childPlayName } = input;
1912
- if (
1913
- session.playName !== childPlayName ||
1914
- session.target.orgId !== orgId ||
1915
- session.target.tableNamespace !== requirement.tableNamespace ||
1916
- session.target.logicalTable !== requirement.logicalTable
1917
- ) {
1918
- throw new Error(
1919
- 'Runtime API returned a DB session outside the child play scope.',
1920
- );
1921
- }
1922
- const expectedOperations = [...requirement.operations].sort().join(',');
1923
- const actualOperations = [...session.operations].sort().join(',');
1924
- if (actualOperations !== expectedOperations) {
1925
- throw new Error(
1926
- 'Runtime API returned a DB session with unexpected operations.',
1927
- );
1928
- }
1929
- }
1930
-
1931
- async function reencryptChildDbSessionForExecutor(input: {
1932
- session: CreateDbSessionResponse;
1933
- decryptionKey: PostgresUrlDecryptionKey;
1934
- childExecutorToken: string;
1935
- }): Promise<CreateDbSessionResponse> {
1936
- const sessionWithoutUrl = stripDbSessionUrl(input.session);
1937
- if (!input.session.encryptedPostgresUrl) {
1938
- throw new Error('Runtime API returned an unencrypted DB session URL.');
1939
- }
1940
- const postgresUrl = await decryptDbSessionPostgresUrlWithPrivateKey({
1941
- encrypted: input.session.encryptedPostgresUrl,
1942
- privateKey: input.decryptionKey.privateKey,
1943
- aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
1944
- });
1945
- return {
1946
- ...sessionWithoutUrl,
1947
- encryptedPostgresUrl: await encryptDbSessionPostgresUrl({
1948
- postgresUrl,
1949
- secret: input.childExecutorToken,
1950
- aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
1951
- }),
1952
- };
1953
- }
1954
-
1955
- async function createChildRuntimeDbSession(input: {
1956
- env: CoordinatorEnv;
1957
- baseUrl: string;
1958
- childExecutorToken: string;
1959
- childPlayName: string;
1960
- requirement: RuntimeDbSessionRequirement;
1961
- userEmail: string;
1962
- orgId: string;
1963
- }): Promise<CreateDbSessionResponse> {
1964
- const decryptionKey = await generateDbSessionPostgresUrlDecryptionKey();
1965
- const response = await input.env.HARNESS.runtimeApiCall({
1966
- contract: PLAY_RUNTIME_CONTRACT,
1967
- executorToken: input.childExecutorToken,
1968
- baseUrl: input.baseUrl,
1969
- path: PLAY_RUNTIME_API_COMPAT_PATH,
1970
- headers: {
1971
- 'x-deepline-request-id': crypto.randomUUID(),
1972
- [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
1973
- },
1974
- timeoutMs: 15_000,
1975
- body: {
1976
- action: 'create_db_session',
1977
- playName: input.childPlayName,
1978
- target: {
1979
- tableNamespace: input.requirement.tableNamespace,
1980
- logicalTable: input.requirement.logicalTable,
1981
- },
1982
- operations: input.requirement.operations,
1983
- limits: input.requirement.limits,
1984
- sheetContract: input.requirement.sheetContract ?? null,
1985
- ttlSeconds: DB_SESSION_DEFAULT_TTL_SECONDS,
1986
- userEmail: input.userEmail,
1987
- postgresUrlEncryption: decryptionKey.request,
1988
- },
1989
- });
1990
- const text = response.body;
1991
- let parsed: unknown = {};
1992
- try {
1993
- parsed = text ? JSON.parse(text) : {};
1994
- } catch {
1995
- parsed = {};
1996
- }
1997
- if (response.status < 200 || response.status >= 300) {
1998
- const error =
1999
- isRecord(parsed) && isRecord(parsed.error) ? parsed.error : {};
2000
- const message =
2001
- (typeof error.message === 'string' && error.message.trim()) ||
2002
- (isRecord(parsed) &&
2003
- typeof parsed.error === 'string' &&
2004
- parsed.error.trim()
2005
- ? parsed.error.trim()
2006
- : '') ||
2007
- text.slice(0, 800) ||
2008
- `Origin DB session mint failed with ${response.status}.`;
2009
- throw new Error(message);
2010
- }
2011
- const session = createDbSessionResponseSchema.parse(parsed);
2012
- assertChildDbSessionScope({
2013
- session,
2014
- requirement: input.requirement,
2015
- orgId: input.orgId,
2016
- childPlayName: input.childPlayName,
2017
- });
2018
- return await reencryptChildDbSessionForExecutor({
2019
- session,
2020
- decryptionKey,
2021
- childExecutorToken: input.childExecutorToken,
2022
- });
2023
- }
2024
-
2025
- async function preloadChildRuntimeDbSessions(input: {
2026
- env: CoordinatorEnv;
2027
- baseUrl: string;
2028
- childExecutorToken: string;
2029
- childRunId: string;
2030
- childPlayName: string;
2031
- manifest: PlayRuntimeManifest;
2032
- orgId: string;
2033
- userEmail: string;
2034
- }): Promise<PreloadedRuntimeDbSession[]> {
2035
- const startedAt = Date.now();
2036
- const requirements = planRuntimeDbSessionRequirements(
2037
- input.manifest.staticPipeline ?? null,
2038
- );
2039
- const sessions = await Promise.all(
2040
- requirements.map(async (requirement) => ({
2041
- tableNamespace: requirement.tableNamespace,
2042
- logicalTable: requirement.logicalTable,
2043
- operations: requirement.operations,
2044
- ...(requirement.limits ? { limits: requirement.limits } : {}),
2045
- session: await createChildRuntimeDbSession({
2046
- env: input.env,
2047
- baseUrl: input.baseUrl,
2048
- childExecutorToken: input.childExecutorToken,
2049
- childPlayName: input.childPlayName,
2050
- requirement,
2051
- userEmail: input.userEmail,
2052
- orgId: input.orgId,
2053
- }),
2054
- })),
2055
- );
2056
- recordCoordinatorPerfTrace({
2057
- runId: input.childRunId,
2058
- phase: 'coordinator.child_db_session_preload',
2059
- ms: Date.now() - startedAt,
2060
- graphHash: input.manifest.graphHash,
2061
- extra: { sessions: sessions.length },
2062
- });
2063
- return sessions;
2064
- }
2065
-
2066
- async function registerChildRunWithRuntime(input: {
2067
- env: CoordinatorEnv;
2068
- baseUrl: string;
2069
- childExecutorToken: string;
2070
- childRunId: string;
2071
- childPlayName: string;
2072
- manifest: PlayRuntimeManifest;
2073
- governance: PlayCallGovernanceSnapshot;
2074
- runtimeBackend: string;
2075
- schedulerBackend: string;
2076
- executionProfile: string;
2077
- }): Promise<void> {
2078
- const response = await input.env.HARNESS.runtimeApiCall({
2079
- contract: PLAY_RUNTIME_CONTRACT,
2080
- executorToken: input.childExecutorToken,
2081
- baseUrl: input.baseUrl,
2082
- path: PLAY_RUNTIME_API_COMPAT_PATH,
2083
- headers: {
2084
- 'x-deepline-request-id': crypto.randomUUID(),
2085
- [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
2086
- },
2087
- timeoutMs: 15_000,
2088
- body: {
2089
- action: 'start_inline_child_run',
2090
- playName: input.childPlayName,
2091
- runId: input.childRunId,
2092
- parentRunId: input.governance.parentRunId,
2093
- rootRunId: input.governance.rootRunId,
2094
- workflowFamilyKey:
2095
- input.governance.rootRunId ??
2096
- input.governance.parentRunId ??
2097
- input.childRunId,
2098
- artifactStorageKey: input.manifest.artifactStorageKey,
2099
- artifactHash: input.manifest.artifactHash,
2100
- graphHash: input.manifest.graphHash,
2101
- runtimeBackend: input.runtimeBackend,
2102
- schedulerBackend: input.schedulerBackend,
2103
- executionProfile: input.executionProfile,
2104
- ...(typeof input.manifest.maxCreditsPerRun === 'number'
2105
- ? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
2106
- : {}),
2107
- staticPipeline: input.manifest.staticPipeline ?? null,
2108
- source: 'published',
2109
- },
2110
- });
2111
- if (response.status < 200 || response.status >= 300) {
2112
- const text = response.body ?? '';
2113
- throw new Error(
2114
- `Inline child run registration failed ${response.status}: ${text.slice(0, 800)}`,
2115
- );
2116
- }
2117
- }
2118
-
2119
1830
  async function markRegisteredChildRunFailed(input: {
2120
1831
  env: CoordinatorEnv;
2121
1832
  baseUrl: string;
@@ -2310,6 +2021,9 @@ async function prepareInlineChildRunWithRuntime(input: {
2310
2021
  manifest: PlayRuntimeManifest;
2311
2022
  governance: PlayCallGovernanceSnapshot;
2312
2023
  userEmail: string;
2024
+ runtimeBackend?: string;
2025
+ schedulerBackend?: string;
2026
+ executionProfile?: string;
2313
2027
  }): Promise<{
2314
2028
  childToken: string;
2315
2029
  preloadedDbSessions: PreloadedRuntimeDbSession[];
@@ -2334,9 +2048,9 @@ async function prepareInlineChildRunWithRuntime(input: {
2334
2048
  artifactStorageKey: input.manifest.artifactStorageKey,
2335
2049
  artifactHash: input.manifest.artifactHash,
2336
2050
  graphHash: input.manifest.graphHash,
2337
- runtimeBackend: 'workers_edge',
2338
- schedulerBackend: 'inline_child',
2339
- executionProfile: 'workers_edge',
2051
+ runtimeBackend: input.runtimeBackend ?? 'workers_edge',
2052
+ schedulerBackend: input.schedulerBackend ?? 'inline_child',
2053
+ executionProfile: input.executionProfile ?? 'workers_edge',
2340
2054
  ...(typeof input.manifest.maxCreditsPerRun === 'number'
2341
2055
  ? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
2342
2056
  : {}),
@@ -3218,43 +2932,39 @@ async function submitChildWorkflowThroughCoordinator(input: {
3218
2932
  });
3219
2933
  const baseUrl = resolveRuntimeApiBaseUrl(input.env, input.body);
3220
2934
 
3221
- const tokenStartedAt = Date.now();
3222
- const childToken = await mintChildWorkflowExecutorToken({
3223
- env: input.env,
3224
- baseUrl,
3225
- parentExecutorToken,
3226
- parentRunId: input.parentRunId,
3227
- parentPlayName:
3228
- typeof input.body.parentPlayName === 'string' &&
3229
- input.body.parentPlayName.trim()
3230
- ? input.body.parentPlayName.trim()
3231
- : governance.parentPlayName,
3232
- childRunId,
3233
- childPlayName,
3234
- maxCreditsPerRun: manifest.maxCreditsPerRun ?? null,
3235
- });
3236
- trace('coordinator.child_submit_token', tokenStartedAt, manifest.graphHash, {
3237
- childRunId,
3238
- childPlayName,
3239
- });
3240
-
3241
- const dbSessionStartedAt = Date.now();
3242
- const preloadedDbSessions = await preloadChildRuntimeDbSessions({
3243
- env: input.env,
3244
- baseUrl,
3245
- childExecutorToken: childToken,
3246
- childRunId,
3247
- childPlayName,
3248
- manifest,
3249
- orgId,
3250
- userEmail:
3251
- typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
3252
- });
2935
+ const prepareStartedAt = Date.now();
2936
+ const { childToken, preloadedDbSessions, prepareTimings, transportTimings } =
2937
+ await prepareInlineChildRunWithRuntime({
2938
+ env: input.env,
2939
+ baseUrl,
2940
+ parentExecutorToken,
2941
+ parentRunId: input.parentRunId,
2942
+ parentPlayName:
2943
+ typeof input.body.parentPlayName === 'string' &&
2944
+ input.body.parentPlayName.trim()
2945
+ ? input.body.parentPlayName.trim()
2946
+ : governance.parentPlayName,
2947
+ childRunId,
2948
+ childPlayName,
2949
+ manifest,
2950
+ governance,
2951
+ userEmail:
2952
+ typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
2953
+ runtimeBackend: 'cf_workflows_dynamic_worker',
2954
+ schedulerBackend: 'cf_workflows',
2955
+ executionProfile: 'workers_edge',
2956
+ });
3253
2957
  trace(
3254
- 'coordinator.child_submit_db_session_preload',
3255
- dbSessionStartedAt,
2958
+ 'coordinator.child_submit_prepare',
2959
+ prepareStartedAt,
3256
2960
  manifest.graphHash,
3257
- { childRunId, sessions: preloadedDbSessions.length },
2961
+ {
2962
+ childRunId,
2963
+ childPlayName,
2964
+ sessions: preloadedDbSessions.length,
2965
+ prepareTimings,
2966
+ transportTimings,
2967
+ },
3258
2968
  );
3259
2969
 
3260
2970
  const params = buildChildWorkflowParams({
@@ -3274,29 +2984,6 @@ async function submitChildWorkflowThroughCoordinator(input: {
3274
2984
  preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
3275
2985
  });
3276
2986
 
3277
- const registerStartedAt = Date.now();
3278
- await registerChildRunWithRuntime({
3279
- env: input.env,
3280
- baseUrl,
3281
- childExecutorToken: childToken,
3282
- childRunId,
3283
- childPlayName,
3284
- manifest,
3285
- governance,
3286
- runtimeBackend: 'cf_workflows_dynamic_worker',
3287
- schedulerBackend: 'cf_workflows',
3288
- executionProfile: 'workers_edge',
3289
- });
3290
- trace(
3291
- 'coordinator.child_submit_register_run',
3292
- registerStartedAt,
3293
- manifest.graphHash,
3294
- {
3295
- childRunId,
3296
- childPlayName,
3297
- },
3298
- );
3299
-
3300
2987
  const workflowSubmitStartedAt = Date.now();
3301
2988
  let response: Response | null = null;
3302
2989
  let responseText = '';
@@ -1323,6 +1323,14 @@ function redactSecretsFromLogString(value: string): string {
1323
1323
  );
1324
1324
  }
1325
1325
 
1326
+ function safeRuntimeApiOrigin(value: string): string {
1327
+ try {
1328
+ return new URL(value).origin;
1329
+ } catch {
1330
+ return '<invalid>';
1331
+ }
1332
+ }
1333
+
1326
1334
  async function postRuntimeApi<T>(
1327
1335
  baseUrl: string,
1328
1336
  executorToken: string,
@@ -1336,6 +1344,9 @@ async function postRuntimeApi<T>(
1336
1344
  // Routes through the in-process RUNTIME_API service binding. Missing binding
1337
1345
  // is an infra error in workers_edge, not a reason to fall back to public HTTP.
1338
1346
  const serializedBody = JSON.stringify(body);
1347
+ const requestDescription = describeRuntimeApiBody(body);
1348
+ const maxAttempts = RUNTIME_API_RETRY_DELAYS_MS.length + 1;
1349
+ const baseOrigin = safeRuntimeApiOrigin(baseUrl);
1339
1350
  let lastError: unknown = null;
1340
1351
  for (
1341
1352
  let attempt = 0;
@@ -1343,6 +1354,7 @@ async function postRuntimeApi<T>(
1343
1354
  attempt += 1
1344
1355
  ) {
1345
1356
  let res: Response;
1357
+ const requestId = makeRequestId();
1346
1358
  try {
1347
1359
  res = await fetchRuntimeApi(
1348
1360
  baseUrl,
@@ -1352,7 +1364,7 @@ async function postRuntimeApi<T>(
1352
1364
  headers: {
1353
1365
  'content-type': 'application/json',
1354
1366
  authorization: `Bearer ${executorToken}`,
1355
- 'x-deepline-request-id': makeRequestId(),
1367
+ 'x-deepline-request-id': requestId,
1356
1368
  ...(options.runtimeTestFaultHeader?.trim()
1357
1369
  ? {
1358
1370
  [PLAY_RUNTIME_TEST_FAULT_HEADER]:
@@ -1373,8 +1385,25 @@ async function postRuntimeApi<T>(
1373
1385
  attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
1374
1386
  !isRetryableWorkerRuntimeApiError(error)
1375
1387
  ) {
1388
+ console.warn('[play-harness.runtime_api.exhausted]', {
1389
+ request: requestDescription,
1390
+ attempt: attempt + 1,
1391
+ maxAttempts,
1392
+ requestId,
1393
+ baseOrigin,
1394
+ error: error instanceof Error ? error.message : String(error),
1395
+ });
1376
1396
  throw error;
1377
1397
  }
1398
+ console.warn('[play-harness.runtime_api.retry]', {
1399
+ request: requestDescription,
1400
+ attempt: attempt + 1,
1401
+ maxAttempts,
1402
+ requestId,
1403
+ baseOrigin,
1404
+ retryInMs: RUNTIME_API_RETRY_DELAYS_MS[attempt] ?? 0,
1405
+ error: error instanceof Error ? error.message : String(error),
1406
+ });
1378
1407
  await sleepRuntimeApiRetry(attempt);
1379
1408
  continue;
1380
1409
  }
@@ -1390,8 +1419,27 @@ async function postRuntimeApi<T>(
1390
1419
  attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
1391
1420
  !isRetryableRuntimeApiResponse(res.status, text)
1392
1421
  ) {
1422
+ console.warn('[play-harness.runtime_api.exhausted]', {
1423
+ request: requestDescription,
1424
+ attempt: attempt + 1,
1425
+ maxAttempts,
1426
+ requestId,
1427
+ baseOrigin,
1428
+ status: res.status,
1429
+ body: redacted,
1430
+ });
1393
1431
  throw lastError;
1394
1432
  }
1433
+ console.warn('[play-harness.runtime_api.retry]', {
1434
+ request: requestDescription,
1435
+ attempt: attempt + 1,
1436
+ maxAttempts,
1437
+ requestId,
1438
+ baseOrigin,
1439
+ status: res.status,
1440
+ retryInMs: RUNTIME_API_RETRY_DELAYS_MS[attempt] ?? 0,
1441
+ body: redacted,
1442
+ });
1395
1443
  await sleepRuntimeApiRetry(attempt);
1396
1444
  }
1397
1445
  throw lastError instanceof Error ? lastError : new Error(String(lastError));
@@ -1628,6 +1676,42 @@ function workerChildRevisionFingerprint(input: {
1628
1676
  );
1629
1677
  }
1630
1678
 
1679
+ type RuntimeResolvedPlayResponse = {
1680
+ manifest?: PlayRuntimeManifest | null;
1681
+ };
1682
+
1683
+ async function resolveAuthorizedWorkerChildManifest(input: {
1684
+ req: RunRequest;
1685
+ playRef: string;
1686
+ cache: Map<string, PlayRuntimeManifest>;
1687
+ }): Promise<PlayRuntimeManifest | null> {
1688
+ const cached = input.cache.get(input.playRef);
1689
+ if (cached) return cached;
1690
+ const preloaded = input.req.childPlayManifests?.[input.playRef];
1691
+ if (preloaded) {
1692
+ input.cache.set(input.playRef, preloaded);
1693
+ return preloaded;
1694
+ }
1695
+ const response = await postRuntimeApi<RuntimeResolvedPlayResponse>(
1696
+ input.req.baseUrl,
1697
+ input.req.executorToken,
1698
+ {
1699
+ action: 'resolve_play',
1700
+ playRef: input.playRef,
1701
+ artifactKind: 'esm_workers',
1702
+ },
1703
+ {
1704
+ runtimeTestFaultHeader: input.req.runtimeTestFaultHeader,
1705
+ timeoutMsOverride: 30_000,
1706
+ },
1707
+ );
1708
+ const manifest = response.manifest ?? null;
1709
+ if (manifest) {
1710
+ input.cache.set(input.playRef, manifest);
1711
+ }
1712
+ return manifest;
1713
+ }
1714
+
1631
1715
  async function childPlayEventKey(input: {
1632
1716
  key: string;
1633
1717
  workflowId: string;
@@ -4269,6 +4353,7 @@ function createMinimalWorkerCtx(
4269
4353
  // lineage snapshot are owned by the Governor (createGovernorForRun above).
4270
4354
  // The worker keeps only substrate mechanism here.
4271
4355
  const stepCallCounts: Record<string, number> = {};
4356
+ const childManifestCache = new Map<string, PlayRuntimeManifest>();
4272
4357
  const secretRedactor = createSecretRedactionContext();
4273
4358
 
4274
4359
  const resolveSecretAuth = async (auth?: SecretAuth) => {
@@ -6790,7 +6875,11 @@ function createMinimalWorkerCtx(
6790
6875
  if (!resolvedName) {
6791
6876
  throw new Error('ctx.runPlay(...) requires a resolvable play name.');
6792
6877
  }
6793
- const childManifest = req.childPlayManifests?.[resolvedName];
6878
+ const childManifest = await resolveAuthorizedWorkerChildManifest({
6879
+ req,
6880
+ playRef: resolvedName,
6881
+ cache: childManifestCache,
6882
+ });
6794
6883
  const receiptKey = buildDurableCtxCallCacheKey({
6795
6884
  orgId: req.orgId,
6796
6885
  playId: req.playName,
@@ -6831,7 +6920,7 @@ function createMinimalWorkerCtx(
6831
6920
  });
6832
6921
  if (!childManifest) {
6833
6922
  throw new Error(
6834
- `ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
6923
+ `ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: not callable.`,
6835
6924
  );
6836
6925
  }
6837
6926
  const childIsDatasetBacked = childPipelineUsesCtxDataset(
@@ -15,6 +15,31 @@ export type WorkflowRetryDecision =
15
15
  message: string | null;
16
16
  };
17
17
 
18
+ function workflowRetryReasonForFailure(failure: {
19
+ code: string;
20
+ cause?: string;
21
+ }): string {
22
+ if (failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED') {
23
+ return 'cloudflare_worker_subrequest_limit';
24
+ }
25
+ if (
26
+ failure.cause &&
27
+ /runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i.test(
28
+ failure.cause,
29
+ )
30
+ ) {
31
+ return 'vercel_runtime_api_deployment_missing';
32
+ }
33
+ return 'cloudflare_durable_object_reset';
34
+ }
35
+
36
+ function workflowRetryExhaustedReasonForFailure(failure: {
37
+ code: string;
38
+ cause?: string;
39
+ }): string {
40
+ return `${workflowRetryReasonForFailure(failure)}_retry_exhausted`;
41
+ }
42
+
18
43
  export function decideWorkflowPlatformRetry(input: {
19
44
  workflowStatus: string;
20
45
  error: string | null;
@@ -33,10 +58,7 @@ export function decideWorkflowPlatformRetry(input: {
33
58
  ) {
34
59
  return {
35
60
  action: 'retry',
36
- reason:
37
- failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED'
38
- ? 'cloudflare_worker_subrequest_limit'
39
- : 'cloudflare_durable_object_reset',
61
+ reason: workflowRetryReasonForFailure(failure),
40
62
  nextAttempt: input.retryAttempts + 1,
41
63
  message: failure.message,
42
64
  };
@@ -44,9 +66,7 @@ export function decideWorkflowPlatformRetry(input: {
44
66
  return {
45
67
  action: 'fail',
46
68
  reason: retryablePlatformReset
47
- ? failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED'
48
- ? 'cloudflare_worker_subrequest_limit_retry_exhausted'
49
- : 'cloudflare_durable_object_reset_retry_exhausted'
69
+ ? workflowRetryExhaustedReasonForFailure(failure)
50
70
  : null,
51
71
  message: failure.message,
52
72
  };