deepline 0.1.194 → 0.1.195

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -31,21 +31,13 @@ import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runti
31
31
  import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
32
32
  import {
33
33
  DB_SESSION_DEFAULT_TTL_SECONDS,
34
- createDbSessionResponseSchema,
35
- type CreateDbSessionResponse,
36
34
  type PreloadedRuntimeDbSession,
37
35
  } from '../../../shared_libs/play-runtime/db-session';
38
36
  import {
39
37
  dbSessionPostgresUrlAad,
40
- decryptDbSessionPostgresUrlWithPrivateKey,
41
38
  encryptDbSessionPostgresUrl,
42
- generateDbSessionPostgresUrlDecryptionKey,
43
- type PostgresUrlDecryptionKey,
44
39
  } from '../../../shared_libs/play-runtime/db-session-crypto';
45
- import {
46
- planRuntimeDbSessionRequirements,
47
- type RuntimeDbSessionRequirement,
48
- } from '../../../shared_libs/play-runtime/db-session-plan';
40
+ import { planRuntimeDbSessionRequirements } from '../../../shared_libs/play-runtime/db-session-plan';
49
41
  import type {
50
42
  PlayRuntimeManifest,
51
43
  PlayRuntimeManifestMap,
@@ -66,7 +58,6 @@ import {
66
58
  } from '../../../shared_libs/play-runtime/runtime-contract';
67
59
  import {
68
60
  PLAY_RUNTIME_API_COMPAT_PATH,
69
- PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
70
61
  PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH,
71
62
  } from '../../../shared_libs/play-runtime/runtime-api-paths';
72
63
  import {
@@ -1836,286 +1827,6 @@ async function restartWorkflowAfterPlatformReset(input: {
1836
1827
  }
1837
1828
  }
1838
1829
 
1839
- async function mintChildWorkflowExecutorToken(input: {
1840
- env: CoordinatorEnv;
1841
- baseUrl: string;
1842
- parentExecutorToken: string;
1843
- parentRunId: string;
1844
- parentPlayName: string;
1845
- childRunId: string;
1846
- childPlayName: string;
1847
- maxCreditsPerRun?: number | null;
1848
- }): Promise<string> {
1849
- const response = await input.env.HARNESS.runtimeApiCall({
1850
- contract: PLAY_RUNTIME_CONTRACT,
1851
- executorToken: input.parentExecutorToken,
1852
- baseUrl: input.baseUrl,
1853
- path: PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
1854
- headers: {
1855
- 'x-deepline-request-id': crypto.randomUUID(),
1856
- [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
1857
- },
1858
- timeoutMs: 15_000,
1859
- body: {
1860
- parentRunId: input.parentRunId,
1861
- parentPlayName: input.parentPlayName,
1862
- childRunId: input.childRunId,
1863
- childPlayName: input.childPlayName,
1864
- maxCreditsPerRun: input.maxCreditsPerRun ?? null,
1865
- },
1866
- });
1867
- const text = response.body;
1868
- let parsed: Record<string, unknown> = {};
1869
- try {
1870
- parsed = text ? (JSON.parse(text) as Record<string, unknown>) : {};
1871
- } catch {
1872
- parsed = {};
1873
- }
1874
- if (response.status < 200 || response.status >= 300) {
1875
- const error = isRecord(parsed.error) ? parsed.error : null;
1876
- const message =
1877
- (typeof error?.message === 'string' && error.message.trim()) ||
1878
- (typeof parsed.error === 'string' && parsed.error.trim()) ||
1879
- text.slice(0, 800) ||
1880
- `Origin child executor token mint failed with ${response.status}.`;
1881
- throw new Error(message);
1882
- }
1883
- const executorToken = parsed.executorToken;
1884
- if (typeof executorToken !== 'string' || !executorToken.trim()) {
1885
- throw new Error(
1886
- 'Origin child executor token response was missing executorToken.',
1887
- );
1888
- }
1889
- return executorToken;
1890
- }
1891
-
1892
- function stripDbSessionUrl(
1893
- session: CreateDbSessionResponse,
1894
- ): Omit<CreateDbSessionResponse, 'postgresUrl' | 'encryptedPostgresUrl'> {
1895
- const {
1896
- postgresUrl: _postgresUrl,
1897
- encryptedPostgresUrl: _encryptedPostgresUrl,
1898
- ...sessionWithoutUrl
1899
- } = session;
1900
- void _postgresUrl;
1901
- void _encryptedPostgresUrl;
1902
- return sessionWithoutUrl;
1903
- }
1904
-
1905
- function assertChildDbSessionScope(input: {
1906
- session: CreateDbSessionResponse;
1907
- requirement: RuntimeDbSessionRequirement;
1908
- orgId: string;
1909
- childPlayName: string;
1910
- }): void {
1911
- const { session, requirement, orgId, childPlayName } = input;
1912
- if (
1913
- session.playName !== childPlayName ||
1914
- session.target.orgId !== orgId ||
1915
- session.target.tableNamespace !== requirement.tableNamespace ||
1916
- session.target.logicalTable !== requirement.logicalTable
1917
- ) {
1918
- throw new Error(
1919
- 'Runtime API returned a DB session outside the child play scope.',
1920
- );
1921
- }
1922
- const expectedOperations = [...requirement.operations].sort().join(',');
1923
- const actualOperations = [...session.operations].sort().join(',');
1924
- if (actualOperations !== expectedOperations) {
1925
- throw new Error(
1926
- 'Runtime API returned a DB session with unexpected operations.',
1927
- );
1928
- }
1929
- }
1930
-
1931
- async function reencryptChildDbSessionForExecutor(input: {
1932
- session: CreateDbSessionResponse;
1933
- decryptionKey: PostgresUrlDecryptionKey;
1934
- childExecutorToken: string;
1935
- }): Promise<CreateDbSessionResponse> {
1936
- const sessionWithoutUrl = stripDbSessionUrl(input.session);
1937
- if (!input.session.encryptedPostgresUrl) {
1938
- throw new Error('Runtime API returned an unencrypted DB session URL.');
1939
- }
1940
- const postgresUrl = await decryptDbSessionPostgresUrlWithPrivateKey({
1941
- encrypted: input.session.encryptedPostgresUrl,
1942
- privateKey: input.decryptionKey.privateKey,
1943
- aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
1944
- });
1945
- return {
1946
- ...sessionWithoutUrl,
1947
- encryptedPostgresUrl: await encryptDbSessionPostgresUrl({
1948
- postgresUrl,
1949
- secret: input.childExecutorToken,
1950
- aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
1951
- }),
1952
- };
1953
- }
1954
-
1955
- async function createChildRuntimeDbSession(input: {
1956
- env: CoordinatorEnv;
1957
- baseUrl: string;
1958
- childExecutorToken: string;
1959
- childPlayName: string;
1960
- requirement: RuntimeDbSessionRequirement;
1961
- userEmail: string;
1962
- orgId: string;
1963
- }): Promise<CreateDbSessionResponse> {
1964
- const decryptionKey = await generateDbSessionPostgresUrlDecryptionKey();
1965
- const response = await input.env.HARNESS.runtimeApiCall({
1966
- contract: PLAY_RUNTIME_CONTRACT,
1967
- executorToken: input.childExecutorToken,
1968
- baseUrl: input.baseUrl,
1969
- path: PLAY_RUNTIME_API_COMPAT_PATH,
1970
- headers: {
1971
- 'x-deepline-request-id': crypto.randomUUID(),
1972
- [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
1973
- },
1974
- timeoutMs: 15_000,
1975
- body: {
1976
- action: 'create_db_session',
1977
- playName: input.childPlayName,
1978
- target: {
1979
- tableNamespace: input.requirement.tableNamespace,
1980
- logicalTable: input.requirement.logicalTable,
1981
- },
1982
- operations: input.requirement.operations,
1983
- limits: input.requirement.limits,
1984
- sheetContract: input.requirement.sheetContract ?? null,
1985
- ttlSeconds: DB_SESSION_DEFAULT_TTL_SECONDS,
1986
- userEmail: input.userEmail,
1987
- postgresUrlEncryption: decryptionKey.request,
1988
- },
1989
- });
1990
- const text = response.body;
1991
- let parsed: unknown = {};
1992
- try {
1993
- parsed = text ? JSON.parse(text) : {};
1994
- } catch {
1995
- parsed = {};
1996
- }
1997
- if (response.status < 200 || response.status >= 300) {
1998
- const error =
1999
- isRecord(parsed) && isRecord(parsed.error) ? parsed.error : {};
2000
- const message =
2001
- (typeof error.message === 'string' && error.message.trim()) ||
2002
- (isRecord(parsed) &&
2003
- typeof parsed.error === 'string' &&
2004
- parsed.error.trim()
2005
- ? parsed.error.trim()
2006
- : '') ||
2007
- text.slice(0, 800) ||
2008
- `Origin DB session mint failed with ${response.status}.`;
2009
- throw new Error(message);
2010
- }
2011
- const session = createDbSessionResponseSchema.parse(parsed);
2012
- assertChildDbSessionScope({
2013
- session,
2014
- requirement: input.requirement,
2015
- orgId: input.orgId,
2016
- childPlayName: input.childPlayName,
2017
- });
2018
- return await reencryptChildDbSessionForExecutor({
2019
- session,
2020
- decryptionKey,
2021
- childExecutorToken: input.childExecutorToken,
2022
- });
2023
- }
2024
-
2025
- async function preloadChildRuntimeDbSessions(input: {
2026
- env: CoordinatorEnv;
2027
- baseUrl: string;
2028
- childExecutorToken: string;
2029
- childRunId: string;
2030
- childPlayName: string;
2031
- manifest: PlayRuntimeManifest;
2032
- orgId: string;
2033
- userEmail: string;
2034
- }): Promise<PreloadedRuntimeDbSession[]> {
2035
- const startedAt = Date.now();
2036
- const requirements = planRuntimeDbSessionRequirements(
2037
- input.manifest.staticPipeline ?? null,
2038
- );
2039
- const sessions = await Promise.all(
2040
- requirements.map(async (requirement) => ({
2041
- tableNamespace: requirement.tableNamespace,
2042
- logicalTable: requirement.logicalTable,
2043
- operations: requirement.operations,
2044
- ...(requirement.limits ? { limits: requirement.limits } : {}),
2045
- session: await createChildRuntimeDbSession({
2046
- env: input.env,
2047
- baseUrl: input.baseUrl,
2048
- childExecutorToken: input.childExecutorToken,
2049
- childPlayName: input.childPlayName,
2050
- requirement,
2051
- userEmail: input.userEmail,
2052
- orgId: input.orgId,
2053
- }),
2054
- })),
2055
- );
2056
- recordCoordinatorPerfTrace({
2057
- runId: input.childRunId,
2058
- phase: 'coordinator.child_db_session_preload',
2059
- ms: Date.now() - startedAt,
2060
- graphHash: input.manifest.graphHash,
2061
- extra: { sessions: sessions.length },
2062
- });
2063
- return sessions;
2064
- }
2065
-
2066
- async function registerChildRunWithRuntime(input: {
2067
- env: CoordinatorEnv;
2068
- baseUrl: string;
2069
- childExecutorToken: string;
2070
- childRunId: string;
2071
- childPlayName: string;
2072
- manifest: PlayRuntimeManifest;
2073
- governance: PlayCallGovernanceSnapshot;
2074
- runtimeBackend: string;
2075
- schedulerBackend: string;
2076
- executionProfile: string;
2077
- }): Promise<void> {
2078
- const response = await input.env.HARNESS.runtimeApiCall({
2079
- contract: PLAY_RUNTIME_CONTRACT,
2080
- executorToken: input.childExecutorToken,
2081
- baseUrl: input.baseUrl,
2082
- path: PLAY_RUNTIME_API_COMPAT_PATH,
2083
- headers: {
2084
- 'x-deepline-request-id': crypto.randomUUID(),
2085
- [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
2086
- },
2087
- timeoutMs: 15_000,
2088
- body: {
2089
- action: 'start_inline_child_run',
2090
- playName: input.childPlayName,
2091
- runId: input.childRunId,
2092
- parentRunId: input.governance.parentRunId,
2093
- rootRunId: input.governance.rootRunId,
2094
- workflowFamilyKey:
2095
- input.governance.rootRunId ??
2096
- input.governance.parentRunId ??
2097
- input.childRunId,
2098
- artifactStorageKey: input.manifest.artifactStorageKey,
2099
- artifactHash: input.manifest.artifactHash,
2100
- graphHash: input.manifest.graphHash,
2101
- runtimeBackend: input.runtimeBackend,
2102
- schedulerBackend: input.schedulerBackend,
2103
- executionProfile: input.executionProfile,
2104
- ...(typeof input.manifest.maxCreditsPerRun === 'number'
2105
- ? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
2106
- : {}),
2107
- staticPipeline: input.manifest.staticPipeline ?? null,
2108
- source: 'published',
2109
- },
2110
- });
2111
- if (response.status < 200 || response.status >= 300) {
2112
- const text = response.body ?? '';
2113
- throw new Error(
2114
- `Inline child run registration failed ${response.status}: ${text.slice(0, 800)}`,
2115
- );
2116
- }
2117
- }
2118
-
2119
1830
  async function markRegisteredChildRunFailed(input: {
2120
1831
  env: CoordinatorEnv;
2121
1832
  baseUrl: string;
@@ -2310,6 +2021,9 @@ async function prepareInlineChildRunWithRuntime(input: {
2310
2021
  manifest: PlayRuntimeManifest;
2311
2022
  governance: PlayCallGovernanceSnapshot;
2312
2023
  userEmail: string;
2024
+ runtimeBackend?: string;
2025
+ schedulerBackend?: string;
2026
+ executionProfile?: string;
2313
2027
  }): Promise<{
2314
2028
  childToken: string;
2315
2029
  preloadedDbSessions: PreloadedRuntimeDbSession[];
@@ -2334,9 +2048,9 @@ async function prepareInlineChildRunWithRuntime(input: {
2334
2048
  artifactStorageKey: input.manifest.artifactStorageKey,
2335
2049
  artifactHash: input.manifest.artifactHash,
2336
2050
  graphHash: input.manifest.graphHash,
2337
- runtimeBackend: 'workers_edge',
2338
- schedulerBackend: 'inline_child',
2339
- executionProfile: 'workers_edge',
2051
+ runtimeBackend: input.runtimeBackend ?? 'workers_edge',
2052
+ schedulerBackend: input.schedulerBackend ?? 'inline_child',
2053
+ executionProfile: input.executionProfile ?? 'workers_edge',
2340
2054
  ...(typeof input.manifest.maxCreditsPerRun === 'number'
2341
2055
  ? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
2342
2056
  : {}),
@@ -3218,43 +2932,39 @@ async function submitChildWorkflowThroughCoordinator(input: {
3218
2932
  });
3219
2933
  const baseUrl = resolveRuntimeApiBaseUrl(input.env, input.body);
3220
2934
 
3221
- const tokenStartedAt = Date.now();
3222
- const childToken = await mintChildWorkflowExecutorToken({
3223
- env: input.env,
3224
- baseUrl,
3225
- parentExecutorToken,
3226
- parentRunId: input.parentRunId,
3227
- parentPlayName:
3228
- typeof input.body.parentPlayName === 'string' &&
3229
- input.body.parentPlayName.trim()
3230
- ? input.body.parentPlayName.trim()
3231
- : governance.parentPlayName,
3232
- childRunId,
3233
- childPlayName,
3234
- maxCreditsPerRun: manifest.maxCreditsPerRun ?? null,
3235
- });
3236
- trace('coordinator.child_submit_token', tokenStartedAt, manifest.graphHash, {
3237
- childRunId,
3238
- childPlayName,
3239
- });
3240
-
3241
- const dbSessionStartedAt = Date.now();
3242
- const preloadedDbSessions = await preloadChildRuntimeDbSessions({
3243
- env: input.env,
3244
- baseUrl,
3245
- childExecutorToken: childToken,
3246
- childRunId,
3247
- childPlayName,
3248
- manifest,
3249
- orgId,
3250
- userEmail:
3251
- typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
3252
- });
2935
+ const prepareStartedAt = Date.now();
2936
+ const { childToken, preloadedDbSessions, prepareTimings, transportTimings } =
2937
+ await prepareInlineChildRunWithRuntime({
2938
+ env: input.env,
2939
+ baseUrl,
2940
+ parentExecutorToken,
2941
+ parentRunId: input.parentRunId,
2942
+ parentPlayName:
2943
+ typeof input.body.parentPlayName === 'string' &&
2944
+ input.body.parentPlayName.trim()
2945
+ ? input.body.parentPlayName.trim()
2946
+ : governance.parentPlayName,
2947
+ childRunId,
2948
+ childPlayName,
2949
+ manifest,
2950
+ governance,
2951
+ userEmail:
2952
+ typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
2953
+ runtimeBackend: 'cf_workflows_dynamic_worker',
2954
+ schedulerBackend: 'cf_workflows',
2955
+ executionProfile: 'workers_edge',
2956
+ });
3253
2957
  trace(
3254
- 'coordinator.child_submit_db_session_preload',
3255
- dbSessionStartedAt,
2958
+ 'coordinator.child_submit_prepare',
2959
+ prepareStartedAt,
3256
2960
  manifest.graphHash,
3257
- { childRunId, sessions: preloadedDbSessions.length },
2961
+ {
2962
+ childRunId,
2963
+ childPlayName,
2964
+ sessions: preloadedDbSessions.length,
2965
+ prepareTimings,
2966
+ transportTimings,
2967
+ },
3258
2968
  );
3259
2969
 
3260
2970
  const params = buildChildWorkflowParams({
@@ -3274,29 +2984,6 @@ async function submitChildWorkflowThroughCoordinator(input: {
3274
2984
  preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
3275
2985
  });
3276
2986
 
3277
- const registerStartedAt = Date.now();
3278
- await registerChildRunWithRuntime({
3279
- env: input.env,
3280
- baseUrl,
3281
- childExecutorToken: childToken,
3282
- childRunId,
3283
- childPlayName,
3284
- manifest,
3285
- governance,
3286
- runtimeBackend: 'cf_workflows_dynamic_worker',
3287
- schedulerBackend: 'cf_workflows',
3288
- executionProfile: 'workers_edge',
3289
- });
3290
- trace(
3291
- 'coordinator.child_submit_register_run',
3292
- registerStartedAt,
3293
- manifest.graphHash,
3294
- {
3295
- childRunId,
3296
- childPlayName,
3297
- },
3298
- );
3299
-
3300
2987
  const workflowSubmitStartedAt = Date.now();
3301
2988
  let response: Response | null = null;
3302
2989
  let responseText = '';
@@ -1323,6 +1323,14 @@ function redactSecretsFromLogString(value: string): string {
1323
1323
  );
1324
1324
  }
1325
1325
 
1326
+ function safeRuntimeApiOrigin(value: string): string {
1327
+ try {
1328
+ return new URL(value).origin;
1329
+ } catch {
1330
+ return '<invalid>';
1331
+ }
1332
+ }
1333
+
1326
1334
  async function postRuntimeApi<T>(
1327
1335
  baseUrl: string,
1328
1336
  executorToken: string,
@@ -1336,6 +1344,9 @@ async function postRuntimeApi<T>(
1336
1344
  // Routes through the in-process RUNTIME_API service binding. Missing binding
1337
1345
  // is an infra error in workers_edge, not a reason to fall back to public HTTP.
1338
1346
  const serializedBody = JSON.stringify(body);
1347
+ const requestDescription = describeRuntimeApiBody(body);
1348
+ const maxAttempts = RUNTIME_API_RETRY_DELAYS_MS.length + 1;
1349
+ const baseOrigin = safeRuntimeApiOrigin(baseUrl);
1339
1350
  let lastError: unknown = null;
1340
1351
  for (
1341
1352
  let attempt = 0;
@@ -1343,6 +1354,7 @@ async function postRuntimeApi<T>(
1343
1354
  attempt += 1
1344
1355
  ) {
1345
1356
  let res: Response;
1357
+ const requestId = makeRequestId();
1346
1358
  try {
1347
1359
  res = await fetchRuntimeApi(
1348
1360
  baseUrl,
@@ -1352,7 +1364,7 @@ async function postRuntimeApi<T>(
1352
1364
  headers: {
1353
1365
  'content-type': 'application/json',
1354
1366
  authorization: `Bearer ${executorToken}`,
1355
- 'x-deepline-request-id': makeRequestId(),
1367
+ 'x-deepline-request-id': requestId,
1356
1368
  ...(options.runtimeTestFaultHeader?.trim()
1357
1369
  ? {
1358
1370
  [PLAY_RUNTIME_TEST_FAULT_HEADER]:
@@ -1373,8 +1385,25 @@ async function postRuntimeApi<T>(
1373
1385
  attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
1374
1386
  !isRetryableWorkerRuntimeApiError(error)
1375
1387
  ) {
1388
+ console.warn('[play-harness.runtime_api.exhausted]', {
1389
+ request: requestDescription,
1390
+ attempt: attempt + 1,
1391
+ maxAttempts,
1392
+ requestId,
1393
+ baseOrigin,
1394
+ error: error instanceof Error ? error.message : String(error),
1395
+ });
1376
1396
  throw error;
1377
1397
  }
1398
+ console.warn('[play-harness.runtime_api.retry]', {
1399
+ request: requestDescription,
1400
+ attempt: attempt + 1,
1401
+ maxAttempts,
1402
+ requestId,
1403
+ baseOrigin,
1404
+ retryInMs: RUNTIME_API_RETRY_DELAYS_MS[attempt] ?? 0,
1405
+ error: error instanceof Error ? error.message : String(error),
1406
+ });
1378
1407
  await sleepRuntimeApiRetry(attempt);
1379
1408
  continue;
1380
1409
  }
@@ -1390,8 +1419,27 @@ async function postRuntimeApi<T>(
1390
1419
  attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
1391
1420
  !isRetryableRuntimeApiResponse(res.status, text)
1392
1421
  ) {
1422
+ console.warn('[play-harness.runtime_api.exhausted]', {
1423
+ request: requestDescription,
1424
+ attempt: attempt + 1,
1425
+ maxAttempts,
1426
+ requestId,
1427
+ baseOrigin,
1428
+ status: res.status,
1429
+ body: redacted,
1430
+ });
1393
1431
  throw lastError;
1394
1432
  }
1433
+ console.warn('[play-harness.runtime_api.retry]', {
1434
+ request: requestDescription,
1435
+ attempt: attempt + 1,
1436
+ maxAttempts,
1437
+ requestId,
1438
+ baseOrigin,
1439
+ status: res.status,
1440
+ retryInMs: RUNTIME_API_RETRY_DELAYS_MS[attempt] ?? 0,
1441
+ body: redacted,
1442
+ });
1395
1443
  await sleepRuntimeApiRetry(attempt);
1396
1444
  }
1397
1445
  throw lastError instanceof Error ? lastError : new Error(String(lastError));
@@ -15,6 +15,31 @@ export type WorkflowRetryDecision =
15
15
  message: string | null;
16
16
  };
17
17
 
18
+ function workflowRetryReasonForFailure(failure: {
19
+ code: string;
20
+ cause?: string;
21
+ }): string {
22
+ if (failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED') {
23
+ return 'cloudflare_worker_subrequest_limit';
24
+ }
25
+ if (
26
+ failure.cause &&
27
+ /runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i.test(
28
+ failure.cause,
29
+ )
30
+ ) {
31
+ return 'vercel_runtime_api_deployment_missing';
32
+ }
33
+ return 'cloudflare_durable_object_reset';
34
+ }
35
+
36
+ function workflowRetryExhaustedReasonForFailure(failure: {
37
+ code: string;
38
+ cause?: string;
39
+ }): string {
40
+ return `${workflowRetryReasonForFailure(failure)}_retry_exhausted`;
41
+ }
42
+
18
43
  export function decideWorkflowPlatformRetry(input: {
19
44
  workflowStatus: string;
20
45
  error: string | null;
@@ -33,10 +58,7 @@ export function decideWorkflowPlatformRetry(input: {
33
58
  ) {
34
59
  return {
35
60
  action: 'retry',
36
- reason:
37
- failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED'
38
- ? 'cloudflare_worker_subrequest_limit'
39
- : 'cloudflare_durable_object_reset',
61
+ reason: workflowRetryReasonForFailure(failure),
40
62
  nextAttempt: input.retryAttempts + 1,
41
63
  message: failure.message,
42
64
  };
@@ -44,9 +66,7 @@ export function decideWorkflowPlatformRetry(input: {
44
66
  return {
45
67
  action: 'fail',
46
68
  reason: retryablePlatformReset
47
- ? failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED'
48
- ? 'cloudflare_worker_subrequest_limit_retry_exhausted'
49
- : 'cloudflare_durable_object_reset_retry_exhausted'
69
+ ? workflowRetryExhaustedReasonForFailure(failure)
50
70
  : null,
51
71
  message: failure.message,
52
72
  };
@@ -494,6 +494,29 @@ export type BillingInvoicesResult = {
494
494
  entries: BillingInvoiceEntry[];
495
495
  };
496
496
 
497
+ /** Saved payment method details returned by one-click billing top-up. */
498
+ export type BillingPaymentMethodSummary = {
499
+ brand?: string | null;
500
+ last4?: string | null;
501
+ exp_month?: number | null;
502
+ exp_year?: number | null;
503
+ label?: string | null;
504
+ };
505
+
506
+ /**
507
+ * Result of `POST /api/v2/billing/top-up`: a saved-card charge and purchased
508
+ * Deepline credit grant for the active workspace.
509
+ */
510
+ export type BillingTopUpResult = {
511
+ ok: true;
512
+ credits: number;
513
+ amount_cents: number;
514
+ currency: string;
515
+ balance: number;
516
+ stripe_payment_intent_id: string;
517
+ payment_method?: BillingPaymentMethodSummary | null;
518
+ };
519
+
497
520
  /** One published plan from `GET /api/v2/billing/catalog/current`. */
498
521
  export type BillingPlanEntry = {
499
522
  plan_family_id: string;
@@ -553,6 +576,11 @@ export type BillingPlansResult = {
553
576
  * @sdkReference client 030 client.billing
554
577
  */
555
578
  export type BillingNamespace = {
579
+ /** Charge the saved payment method and add Deepline credits to the active workspace. */
580
+ topUp: (options: {
581
+ credits: number;
582
+ idempotencyKey?: string;
583
+ }) => Promise<BillingTopUpResult>;
556
584
  /** Published plans plus the plan you are on ("what plans exist and what am I on"). */
557
585
  plans: () => Promise<BillingPlansResult>;
558
586
  subscription: {
@@ -701,7 +729,10 @@ const STAGE_LEGACY_MULTIPART_MAX_BYTES = 4_000_000;
701
729
 
702
730
  const STAGE_DIRECT_UPLOAD_MAX_ATTEMPTS = 3;
703
731
 
704
- function stagedUploadIdentity(logicalPath: string, contentHash: string): string {
732
+ function stagedUploadIdentity(
733
+ logicalPath: string,
734
+ contentHash: string,
735
+ ): string {
705
736
  return `${contentHash}:${logicalPath}`;
706
737
  }
707
738
 
@@ -1021,6 +1052,7 @@ export class DeeplineClient {
1021
1052
  stopAll: (options) => this.stopAllRuns(options),
1022
1053
  };
1023
1054
  this.billing = {
1055
+ topUp: (options) => this.topUpBillingBalance(options),
1024
1056
  plans: () => this.getBillingPlans(),
1025
1057
  subscription: {
1026
1058
  status: () => this.getBillingSubscriptionStatus(),
@@ -3251,6 +3283,34 @@ export class DeeplineClient {
3251
3283
  return this.http.get<BillingPlansResult>('/api/v2/billing/catalog/current');
3252
3284
  }
3253
3285
 
3286
+ /**
3287
+ * Charge the saved payment method and add Deepline credits to the active
3288
+ * workspace. Prefer `client.billing.topUp(...)`.
3289
+ *
3290
+ * @throws {@link DeeplineError} with `statusCode: 409` when checkout is
3291
+ * required because the workspace has no saved payment method or the card
3292
+ * requires confirmation.
3293
+ */
3294
+ async topUpBillingBalance(options: {
3295
+ credits: number;
3296
+ idempotencyKey?: string;
3297
+ }): Promise<BillingTopUpResult> {
3298
+ return this.http.post<BillingTopUpResult>(
3299
+ '/api/v2/billing/top-up',
3300
+ {
3301
+ credits: options.credits,
3302
+ ...(options.idempotencyKey
3303
+ ? { idempotency_key: options.idempotencyKey }
3304
+ : {}),
3305
+ },
3306
+ undefined,
3307
+ // The idempotency key makes a retry at the product layer safe, but the
3308
+ // CLI should not hide ambiguous local delivery failures behind implicit
3309
+ // transport retries for a payment mutation.
3310
+ { maxRetries: 0, exactUrlOnly: true },
3311
+ );
3312
+ }
3313
+
3254
3314
  /**
3255
3315
  * Subscription state for the active workspace: active plan, whether a
3256
3316
  * Stripe subscription backs it, renewal/cancellation facts, and remaining
@@ -62,8 +62,10 @@ export type {
62
62
  BillingInvoiceEntry,
63
63
  BillingInvoicesResult,
64
64
  BillingNamespace,
65
+ BillingPaymentMethodSummary,
65
66
  BillingSubscriptionCancelResult,
66
67
  BillingSubscriptionStatus,
68
+ BillingTopUpResult,
67
69
  PlayStatus,
68
70
  PlaySheetRow,
69
71
  PlaySheetRowsResult,