deepline 0.1.194 → 0.1.196
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +38 -351
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +49 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +27 -7
- package/dist/bundling-sources/sdk/src/client.ts +61 -1
- package/dist/bundling-sources/sdk/src/index.ts +2 -0
- package/dist/bundling-sources/sdk/src/release.ts +5 -4
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +11 -0
- package/dist/cli/index.js +309 -35
- package/dist/cli/index.mjs +303 -29
- package/dist/index.d.mts +39 -1
- package/dist/index.d.ts +39 -1
- package/dist/index.js +25 -2
- package/dist/index.mjs +25 -2
- package/package.json +1 -1
|
@@ -31,21 +31,13 @@ import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runti
|
|
|
31
31
|
import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
|
|
32
32
|
import {
|
|
33
33
|
DB_SESSION_DEFAULT_TTL_SECONDS,
|
|
34
|
-
createDbSessionResponseSchema,
|
|
35
|
-
type CreateDbSessionResponse,
|
|
36
34
|
type PreloadedRuntimeDbSession,
|
|
37
35
|
} from '../../../shared_libs/play-runtime/db-session';
|
|
38
36
|
import {
|
|
39
37
|
dbSessionPostgresUrlAad,
|
|
40
|
-
decryptDbSessionPostgresUrlWithPrivateKey,
|
|
41
38
|
encryptDbSessionPostgresUrl,
|
|
42
|
-
generateDbSessionPostgresUrlDecryptionKey,
|
|
43
|
-
type PostgresUrlDecryptionKey,
|
|
44
39
|
} from '../../../shared_libs/play-runtime/db-session-crypto';
|
|
45
|
-
import {
|
|
46
|
-
planRuntimeDbSessionRequirements,
|
|
47
|
-
type RuntimeDbSessionRequirement,
|
|
48
|
-
} from '../../../shared_libs/play-runtime/db-session-plan';
|
|
40
|
+
import { planRuntimeDbSessionRequirements } from '../../../shared_libs/play-runtime/db-session-plan';
|
|
49
41
|
import type {
|
|
50
42
|
PlayRuntimeManifest,
|
|
51
43
|
PlayRuntimeManifestMap,
|
|
@@ -66,7 +58,6 @@ import {
|
|
|
66
58
|
} from '../../../shared_libs/play-runtime/runtime-contract';
|
|
67
59
|
import {
|
|
68
60
|
PLAY_RUNTIME_API_COMPAT_PATH,
|
|
69
|
-
PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
|
|
70
61
|
PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH,
|
|
71
62
|
} from '../../../shared_libs/play-runtime/runtime-api-paths';
|
|
72
63
|
import {
|
|
@@ -1836,286 +1827,6 @@ async function restartWorkflowAfterPlatformReset(input: {
|
|
|
1836
1827
|
}
|
|
1837
1828
|
}
|
|
1838
1829
|
|
|
1839
|
-
async function mintChildWorkflowExecutorToken(input: {
|
|
1840
|
-
env: CoordinatorEnv;
|
|
1841
|
-
baseUrl: string;
|
|
1842
|
-
parentExecutorToken: string;
|
|
1843
|
-
parentRunId: string;
|
|
1844
|
-
parentPlayName: string;
|
|
1845
|
-
childRunId: string;
|
|
1846
|
-
childPlayName: string;
|
|
1847
|
-
maxCreditsPerRun?: number | null;
|
|
1848
|
-
}): Promise<string> {
|
|
1849
|
-
const response = await input.env.HARNESS.runtimeApiCall({
|
|
1850
|
-
contract: PLAY_RUNTIME_CONTRACT,
|
|
1851
|
-
executorToken: input.parentExecutorToken,
|
|
1852
|
-
baseUrl: input.baseUrl,
|
|
1853
|
-
path: PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
|
|
1854
|
-
headers: {
|
|
1855
|
-
'x-deepline-request-id': crypto.randomUUID(),
|
|
1856
|
-
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
1857
|
-
},
|
|
1858
|
-
timeoutMs: 15_000,
|
|
1859
|
-
body: {
|
|
1860
|
-
parentRunId: input.parentRunId,
|
|
1861
|
-
parentPlayName: input.parentPlayName,
|
|
1862
|
-
childRunId: input.childRunId,
|
|
1863
|
-
childPlayName: input.childPlayName,
|
|
1864
|
-
maxCreditsPerRun: input.maxCreditsPerRun ?? null,
|
|
1865
|
-
},
|
|
1866
|
-
});
|
|
1867
|
-
const text = response.body;
|
|
1868
|
-
let parsed: Record<string, unknown> = {};
|
|
1869
|
-
try {
|
|
1870
|
-
parsed = text ? (JSON.parse(text) as Record<string, unknown>) : {};
|
|
1871
|
-
} catch {
|
|
1872
|
-
parsed = {};
|
|
1873
|
-
}
|
|
1874
|
-
if (response.status < 200 || response.status >= 300) {
|
|
1875
|
-
const error = isRecord(parsed.error) ? parsed.error : null;
|
|
1876
|
-
const message =
|
|
1877
|
-
(typeof error?.message === 'string' && error.message.trim()) ||
|
|
1878
|
-
(typeof parsed.error === 'string' && parsed.error.trim()) ||
|
|
1879
|
-
text.slice(0, 800) ||
|
|
1880
|
-
`Origin child executor token mint failed with ${response.status}.`;
|
|
1881
|
-
throw new Error(message);
|
|
1882
|
-
}
|
|
1883
|
-
const executorToken = parsed.executorToken;
|
|
1884
|
-
if (typeof executorToken !== 'string' || !executorToken.trim()) {
|
|
1885
|
-
throw new Error(
|
|
1886
|
-
'Origin child executor token response was missing executorToken.',
|
|
1887
|
-
);
|
|
1888
|
-
}
|
|
1889
|
-
return executorToken;
|
|
1890
|
-
}
|
|
1891
|
-
|
|
1892
|
-
function stripDbSessionUrl(
|
|
1893
|
-
session: CreateDbSessionResponse,
|
|
1894
|
-
): Omit<CreateDbSessionResponse, 'postgresUrl' | 'encryptedPostgresUrl'> {
|
|
1895
|
-
const {
|
|
1896
|
-
postgresUrl: _postgresUrl,
|
|
1897
|
-
encryptedPostgresUrl: _encryptedPostgresUrl,
|
|
1898
|
-
...sessionWithoutUrl
|
|
1899
|
-
} = session;
|
|
1900
|
-
void _postgresUrl;
|
|
1901
|
-
void _encryptedPostgresUrl;
|
|
1902
|
-
return sessionWithoutUrl;
|
|
1903
|
-
}
|
|
1904
|
-
|
|
1905
|
-
function assertChildDbSessionScope(input: {
|
|
1906
|
-
session: CreateDbSessionResponse;
|
|
1907
|
-
requirement: RuntimeDbSessionRequirement;
|
|
1908
|
-
orgId: string;
|
|
1909
|
-
childPlayName: string;
|
|
1910
|
-
}): void {
|
|
1911
|
-
const { session, requirement, orgId, childPlayName } = input;
|
|
1912
|
-
if (
|
|
1913
|
-
session.playName !== childPlayName ||
|
|
1914
|
-
session.target.orgId !== orgId ||
|
|
1915
|
-
session.target.tableNamespace !== requirement.tableNamespace ||
|
|
1916
|
-
session.target.logicalTable !== requirement.logicalTable
|
|
1917
|
-
) {
|
|
1918
|
-
throw new Error(
|
|
1919
|
-
'Runtime API returned a DB session outside the child play scope.',
|
|
1920
|
-
);
|
|
1921
|
-
}
|
|
1922
|
-
const expectedOperations = [...requirement.operations].sort().join(',');
|
|
1923
|
-
const actualOperations = [...session.operations].sort().join(',');
|
|
1924
|
-
if (actualOperations !== expectedOperations) {
|
|
1925
|
-
throw new Error(
|
|
1926
|
-
'Runtime API returned a DB session with unexpected operations.',
|
|
1927
|
-
);
|
|
1928
|
-
}
|
|
1929
|
-
}
|
|
1930
|
-
|
|
1931
|
-
async function reencryptChildDbSessionForExecutor(input: {
|
|
1932
|
-
session: CreateDbSessionResponse;
|
|
1933
|
-
decryptionKey: PostgresUrlDecryptionKey;
|
|
1934
|
-
childExecutorToken: string;
|
|
1935
|
-
}): Promise<CreateDbSessionResponse> {
|
|
1936
|
-
const sessionWithoutUrl = stripDbSessionUrl(input.session);
|
|
1937
|
-
if (!input.session.encryptedPostgresUrl) {
|
|
1938
|
-
throw new Error('Runtime API returned an unencrypted DB session URL.');
|
|
1939
|
-
}
|
|
1940
|
-
const postgresUrl = await decryptDbSessionPostgresUrlWithPrivateKey({
|
|
1941
|
-
encrypted: input.session.encryptedPostgresUrl,
|
|
1942
|
-
privateKey: input.decryptionKey.privateKey,
|
|
1943
|
-
aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
|
|
1944
|
-
});
|
|
1945
|
-
return {
|
|
1946
|
-
...sessionWithoutUrl,
|
|
1947
|
-
encryptedPostgresUrl: await encryptDbSessionPostgresUrl({
|
|
1948
|
-
postgresUrl,
|
|
1949
|
-
secret: input.childExecutorToken,
|
|
1950
|
-
aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
|
|
1951
|
-
}),
|
|
1952
|
-
};
|
|
1953
|
-
}
|
|
1954
|
-
|
|
1955
|
-
async function createChildRuntimeDbSession(input: {
|
|
1956
|
-
env: CoordinatorEnv;
|
|
1957
|
-
baseUrl: string;
|
|
1958
|
-
childExecutorToken: string;
|
|
1959
|
-
childPlayName: string;
|
|
1960
|
-
requirement: RuntimeDbSessionRequirement;
|
|
1961
|
-
userEmail: string;
|
|
1962
|
-
orgId: string;
|
|
1963
|
-
}): Promise<CreateDbSessionResponse> {
|
|
1964
|
-
const decryptionKey = await generateDbSessionPostgresUrlDecryptionKey();
|
|
1965
|
-
const response = await input.env.HARNESS.runtimeApiCall({
|
|
1966
|
-
contract: PLAY_RUNTIME_CONTRACT,
|
|
1967
|
-
executorToken: input.childExecutorToken,
|
|
1968
|
-
baseUrl: input.baseUrl,
|
|
1969
|
-
path: PLAY_RUNTIME_API_COMPAT_PATH,
|
|
1970
|
-
headers: {
|
|
1971
|
-
'x-deepline-request-id': crypto.randomUUID(),
|
|
1972
|
-
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
1973
|
-
},
|
|
1974
|
-
timeoutMs: 15_000,
|
|
1975
|
-
body: {
|
|
1976
|
-
action: 'create_db_session',
|
|
1977
|
-
playName: input.childPlayName,
|
|
1978
|
-
target: {
|
|
1979
|
-
tableNamespace: input.requirement.tableNamespace,
|
|
1980
|
-
logicalTable: input.requirement.logicalTable,
|
|
1981
|
-
},
|
|
1982
|
-
operations: input.requirement.operations,
|
|
1983
|
-
limits: input.requirement.limits,
|
|
1984
|
-
sheetContract: input.requirement.sheetContract ?? null,
|
|
1985
|
-
ttlSeconds: DB_SESSION_DEFAULT_TTL_SECONDS,
|
|
1986
|
-
userEmail: input.userEmail,
|
|
1987
|
-
postgresUrlEncryption: decryptionKey.request,
|
|
1988
|
-
},
|
|
1989
|
-
});
|
|
1990
|
-
const text = response.body;
|
|
1991
|
-
let parsed: unknown = {};
|
|
1992
|
-
try {
|
|
1993
|
-
parsed = text ? JSON.parse(text) : {};
|
|
1994
|
-
} catch {
|
|
1995
|
-
parsed = {};
|
|
1996
|
-
}
|
|
1997
|
-
if (response.status < 200 || response.status >= 300) {
|
|
1998
|
-
const error =
|
|
1999
|
-
isRecord(parsed) && isRecord(parsed.error) ? parsed.error : {};
|
|
2000
|
-
const message =
|
|
2001
|
-
(typeof error.message === 'string' && error.message.trim()) ||
|
|
2002
|
-
(isRecord(parsed) &&
|
|
2003
|
-
typeof parsed.error === 'string' &&
|
|
2004
|
-
parsed.error.trim()
|
|
2005
|
-
? parsed.error.trim()
|
|
2006
|
-
: '') ||
|
|
2007
|
-
text.slice(0, 800) ||
|
|
2008
|
-
`Origin DB session mint failed with ${response.status}.`;
|
|
2009
|
-
throw new Error(message);
|
|
2010
|
-
}
|
|
2011
|
-
const session = createDbSessionResponseSchema.parse(parsed);
|
|
2012
|
-
assertChildDbSessionScope({
|
|
2013
|
-
session,
|
|
2014
|
-
requirement: input.requirement,
|
|
2015
|
-
orgId: input.orgId,
|
|
2016
|
-
childPlayName: input.childPlayName,
|
|
2017
|
-
});
|
|
2018
|
-
return await reencryptChildDbSessionForExecutor({
|
|
2019
|
-
session,
|
|
2020
|
-
decryptionKey,
|
|
2021
|
-
childExecutorToken: input.childExecutorToken,
|
|
2022
|
-
});
|
|
2023
|
-
}
|
|
2024
|
-
|
|
2025
|
-
async function preloadChildRuntimeDbSessions(input: {
|
|
2026
|
-
env: CoordinatorEnv;
|
|
2027
|
-
baseUrl: string;
|
|
2028
|
-
childExecutorToken: string;
|
|
2029
|
-
childRunId: string;
|
|
2030
|
-
childPlayName: string;
|
|
2031
|
-
manifest: PlayRuntimeManifest;
|
|
2032
|
-
orgId: string;
|
|
2033
|
-
userEmail: string;
|
|
2034
|
-
}): Promise<PreloadedRuntimeDbSession[]> {
|
|
2035
|
-
const startedAt = Date.now();
|
|
2036
|
-
const requirements = planRuntimeDbSessionRequirements(
|
|
2037
|
-
input.manifest.staticPipeline ?? null,
|
|
2038
|
-
);
|
|
2039
|
-
const sessions = await Promise.all(
|
|
2040
|
-
requirements.map(async (requirement) => ({
|
|
2041
|
-
tableNamespace: requirement.tableNamespace,
|
|
2042
|
-
logicalTable: requirement.logicalTable,
|
|
2043
|
-
operations: requirement.operations,
|
|
2044
|
-
...(requirement.limits ? { limits: requirement.limits } : {}),
|
|
2045
|
-
session: await createChildRuntimeDbSession({
|
|
2046
|
-
env: input.env,
|
|
2047
|
-
baseUrl: input.baseUrl,
|
|
2048
|
-
childExecutorToken: input.childExecutorToken,
|
|
2049
|
-
childPlayName: input.childPlayName,
|
|
2050
|
-
requirement,
|
|
2051
|
-
userEmail: input.userEmail,
|
|
2052
|
-
orgId: input.orgId,
|
|
2053
|
-
}),
|
|
2054
|
-
})),
|
|
2055
|
-
);
|
|
2056
|
-
recordCoordinatorPerfTrace({
|
|
2057
|
-
runId: input.childRunId,
|
|
2058
|
-
phase: 'coordinator.child_db_session_preload',
|
|
2059
|
-
ms: Date.now() - startedAt,
|
|
2060
|
-
graphHash: input.manifest.graphHash,
|
|
2061
|
-
extra: { sessions: sessions.length },
|
|
2062
|
-
});
|
|
2063
|
-
return sessions;
|
|
2064
|
-
}
|
|
2065
|
-
|
|
2066
|
-
async function registerChildRunWithRuntime(input: {
|
|
2067
|
-
env: CoordinatorEnv;
|
|
2068
|
-
baseUrl: string;
|
|
2069
|
-
childExecutorToken: string;
|
|
2070
|
-
childRunId: string;
|
|
2071
|
-
childPlayName: string;
|
|
2072
|
-
manifest: PlayRuntimeManifest;
|
|
2073
|
-
governance: PlayCallGovernanceSnapshot;
|
|
2074
|
-
runtimeBackend: string;
|
|
2075
|
-
schedulerBackend: string;
|
|
2076
|
-
executionProfile: string;
|
|
2077
|
-
}): Promise<void> {
|
|
2078
|
-
const response = await input.env.HARNESS.runtimeApiCall({
|
|
2079
|
-
contract: PLAY_RUNTIME_CONTRACT,
|
|
2080
|
-
executorToken: input.childExecutorToken,
|
|
2081
|
-
baseUrl: input.baseUrl,
|
|
2082
|
-
path: PLAY_RUNTIME_API_COMPAT_PATH,
|
|
2083
|
-
headers: {
|
|
2084
|
-
'x-deepline-request-id': crypto.randomUUID(),
|
|
2085
|
-
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
2086
|
-
},
|
|
2087
|
-
timeoutMs: 15_000,
|
|
2088
|
-
body: {
|
|
2089
|
-
action: 'start_inline_child_run',
|
|
2090
|
-
playName: input.childPlayName,
|
|
2091
|
-
runId: input.childRunId,
|
|
2092
|
-
parentRunId: input.governance.parentRunId,
|
|
2093
|
-
rootRunId: input.governance.rootRunId,
|
|
2094
|
-
workflowFamilyKey:
|
|
2095
|
-
input.governance.rootRunId ??
|
|
2096
|
-
input.governance.parentRunId ??
|
|
2097
|
-
input.childRunId,
|
|
2098
|
-
artifactStorageKey: input.manifest.artifactStorageKey,
|
|
2099
|
-
artifactHash: input.manifest.artifactHash,
|
|
2100
|
-
graphHash: input.manifest.graphHash,
|
|
2101
|
-
runtimeBackend: input.runtimeBackend,
|
|
2102
|
-
schedulerBackend: input.schedulerBackend,
|
|
2103
|
-
executionProfile: input.executionProfile,
|
|
2104
|
-
...(typeof input.manifest.maxCreditsPerRun === 'number'
|
|
2105
|
-
? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
|
|
2106
|
-
: {}),
|
|
2107
|
-
staticPipeline: input.manifest.staticPipeline ?? null,
|
|
2108
|
-
source: 'published',
|
|
2109
|
-
},
|
|
2110
|
-
});
|
|
2111
|
-
if (response.status < 200 || response.status >= 300) {
|
|
2112
|
-
const text = response.body ?? '';
|
|
2113
|
-
throw new Error(
|
|
2114
|
-
`Inline child run registration failed ${response.status}: ${text.slice(0, 800)}`,
|
|
2115
|
-
);
|
|
2116
|
-
}
|
|
2117
|
-
}
|
|
2118
|
-
|
|
2119
1830
|
async function markRegisteredChildRunFailed(input: {
|
|
2120
1831
|
env: CoordinatorEnv;
|
|
2121
1832
|
baseUrl: string;
|
|
@@ -2310,6 +2021,9 @@ async function prepareInlineChildRunWithRuntime(input: {
|
|
|
2310
2021
|
manifest: PlayRuntimeManifest;
|
|
2311
2022
|
governance: PlayCallGovernanceSnapshot;
|
|
2312
2023
|
userEmail: string;
|
|
2024
|
+
runtimeBackend?: string;
|
|
2025
|
+
schedulerBackend?: string;
|
|
2026
|
+
executionProfile?: string;
|
|
2313
2027
|
}): Promise<{
|
|
2314
2028
|
childToken: string;
|
|
2315
2029
|
preloadedDbSessions: PreloadedRuntimeDbSession[];
|
|
@@ -2334,9 +2048,9 @@ async function prepareInlineChildRunWithRuntime(input: {
|
|
|
2334
2048
|
artifactStorageKey: input.manifest.artifactStorageKey,
|
|
2335
2049
|
artifactHash: input.manifest.artifactHash,
|
|
2336
2050
|
graphHash: input.manifest.graphHash,
|
|
2337
|
-
runtimeBackend: 'workers_edge',
|
|
2338
|
-
schedulerBackend: 'inline_child',
|
|
2339
|
-
executionProfile: 'workers_edge',
|
|
2051
|
+
runtimeBackend: input.runtimeBackend ?? 'workers_edge',
|
|
2052
|
+
schedulerBackend: input.schedulerBackend ?? 'inline_child',
|
|
2053
|
+
executionProfile: input.executionProfile ?? 'workers_edge',
|
|
2340
2054
|
...(typeof input.manifest.maxCreditsPerRun === 'number'
|
|
2341
2055
|
? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
|
|
2342
2056
|
: {}),
|
|
@@ -3218,43 +2932,39 @@ async function submitChildWorkflowThroughCoordinator(input: {
|
|
|
3218
2932
|
});
|
|
3219
2933
|
const baseUrl = resolveRuntimeApiBaseUrl(input.env, input.body);
|
|
3220
2934
|
|
|
3221
|
-
const
|
|
3222
|
-
const childToken
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
env: input.env,
|
|
3244
|
-
baseUrl,
|
|
3245
|
-
childExecutorToken: childToken,
|
|
3246
|
-
childRunId,
|
|
3247
|
-
childPlayName,
|
|
3248
|
-
manifest,
|
|
3249
|
-
orgId,
|
|
3250
|
-
userEmail:
|
|
3251
|
-
typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
|
|
3252
|
-
});
|
|
2935
|
+
const prepareStartedAt = Date.now();
|
|
2936
|
+
const { childToken, preloadedDbSessions, prepareTimings, transportTimings } =
|
|
2937
|
+
await prepareInlineChildRunWithRuntime({
|
|
2938
|
+
env: input.env,
|
|
2939
|
+
baseUrl,
|
|
2940
|
+
parentExecutorToken,
|
|
2941
|
+
parentRunId: input.parentRunId,
|
|
2942
|
+
parentPlayName:
|
|
2943
|
+
typeof input.body.parentPlayName === 'string' &&
|
|
2944
|
+
input.body.parentPlayName.trim()
|
|
2945
|
+
? input.body.parentPlayName.trim()
|
|
2946
|
+
: governance.parentPlayName,
|
|
2947
|
+
childRunId,
|
|
2948
|
+
childPlayName,
|
|
2949
|
+
manifest,
|
|
2950
|
+
governance,
|
|
2951
|
+
userEmail:
|
|
2952
|
+
typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
|
|
2953
|
+
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
2954
|
+
schedulerBackend: 'cf_workflows',
|
|
2955
|
+
executionProfile: 'workers_edge',
|
|
2956
|
+
});
|
|
3253
2957
|
trace(
|
|
3254
|
-
'coordinator.
|
|
3255
|
-
|
|
2958
|
+
'coordinator.child_submit_prepare',
|
|
2959
|
+
prepareStartedAt,
|
|
3256
2960
|
manifest.graphHash,
|
|
3257
|
-
{
|
|
2961
|
+
{
|
|
2962
|
+
childRunId,
|
|
2963
|
+
childPlayName,
|
|
2964
|
+
sessions: preloadedDbSessions.length,
|
|
2965
|
+
prepareTimings,
|
|
2966
|
+
transportTimings,
|
|
2967
|
+
},
|
|
3258
2968
|
);
|
|
3259
2969
|
|
|
3260
2970
|
const params = buildChildWorkflowParams({
|
|
@@ -3274,29 +2984,6 @@ async function submitChildWorkflowThroughCoordinator(input: {
|
|
|
3274
2984
|
preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
|
|
3275
2985
|
});
|
|
3276
2986
|
|
|
3277
|
-
const registerStartedAt = Date.now();
|
|
3278
|
-
await registerChildRunWithRuntime({
|
|
3279
|
-
env: input.env,
|
|
3280
|
-
baseUrl,
|
|
3281
|
-
childExecutorToken: childToken,
|
|
3282
|
-
childRunId,
|
|
3283
|
-
childPlayName,
|
|
3284
|
-
manifest,
|
|
3285
|
-
governance,
|
|
3286
|
-
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
3287
|
-
schedulerBackend: 'cf_workflows',
|
|
3288
|
-
executionProfile: 'workers_edge',
|
|
3289
|
-
});
|
|
3290
|
-
trace(
|
|
3291
|
-
'coordinator.child_submit_register_run',
|
|
3292
|
-
registerStartedAt,
|
|
3293
|
-
manifest.graphHash,
|
|
3294
|
-
{
|
|
3295
|
-
childRunId,
|
|
3296
|
-
childPlayName,
|
|
3297
|
-
},
|
|
3298
|
-
);
|
|
3299
|
-
|
|
3300
2987
|
const workflowSubmitStartedAt = Date.now();
|
|
3301
2988
|
let response: Response | null = null;
|
|
3302
2989
|
let responseText = '';
|
|
@@ -1323,6 +1323,14 @@ function redactSecretsFromLogString(value: string): string {
|
|
|
1323
1323
|
);
|
|
1324
1324
|
}
|
|
1325
1325
|
|
|
1326
|
+
function safeRuntimeApiOrigin(value: string): string {
|
|
1327
|
+
try {
|
|
1328
|
+
return new URL(value).origin;
|
|
1329
|
+
} catch {
|
|
1330
|
+
return '<invalid>';
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1326
1334
|
async function postRuntimeApi<T>(
|
|
1327
1335
|
baseUrl: string,
|
|
1328
1336
|
executorToken: string,
|
|
@@ -1336,6 +1344,9 @@ async function postRuntimeApi<T>(
|
|
|
1336
1344
|
// Routes through the in-process RUNTIME_API service binding. Missing binding
|
|
1337
1345
|
// is an infra error in workers_edge, not a reason to fall back to public HTTP.
|
|
1338
1346
|
const serializedBody = JSON.stringify(body);
|
|
1347
|
+
const requestDescription = describeRuntimeApiBody(body);
|
|
1348
|
+
const maxAttempts = RUNTIME_API_RETRY_DELAYS_MS.length + 1;
|
|
1349
|
+
const baseOrigin = safeRuntimeApiOrigin(baseUrl);
|
|
1339
1350
|
let lastError: unknown = null;
|
|
1340
1351
|
for (
|
|
1341
1352
|
let attempt = 0;
|
|
@@ -1343,6 +1354,7 @@ async function postRuntimeApi<T>(
|
|
|
1343
1354
|
attempt += 1
|
|
1344
1355
|
) {
|
|
1345
1356
|
let res: Response;
|
|
1357
|
+
const requestId = makeRequestId();
|
|
1346
1358
|
try {
|
|
1347
1359
|
res = await fetchRuntimeApi(
|
|
1348
1360
|
baseUrl,
|
|
@@ -1352,7 +1364,7 @@ async function postRuntimeApi<T>(
|
|
|
1352
1364
|
headers: {
|
|
1353
1365
|
'content-type': 'application/json',
|
|
1354
1366
|
authorization: `Bearer ${executorToken}`,
|
|
1355
|
-
'x-deepline-request-id':
|
|
1367
|
+
'x-deepline-request-id': requestId,
|
|
1356
1368
|
...(options.runtimeTestFaultHeader?.trim()
|
|
1357
1369
|
? {
|
|
1358
1370
|
[PLAY_RUNTIME_TEST_FAULT_HEADER]:
|
|
@@ -1373,8 +1385,25 @@ async function postRuntimeApi<T>(
|
|
|
1373
1385
|
attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
|
|
1374
1386
|
!isRetryableWorkerRuntimeApiError(error)
|
|
1375
1387
|
) {
|
|
1388
|
+
console.warn('[play-harness.runtime_api.exhausted]', {
|
|
1389
|
+
request: requestDescription,
|
|
1390
|
+
attempt: attempt + 1,
|
|
1391
|
+
maxAttempts,
|
|
1392
|
+
requestId,
|
|
1393
|
+
baseOrigin,
|
|
1394
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1395
|
+
});
|
|
1376
1396
|
throw error;
|
|
1377
1397
|
}
|
|
1398
|
+
console.warn('[play-harness.runtime_api.retry]', {
|
|
1399
|
+
request: requestDescription,
|
|
1400
|
+
attempt: attempt + 1,
|
|
1401
|
+
maxAttempts,
|
|
1402
|
+
requestId,
|
|
1403
|
+
baseOrigin,
|
|
1404
|
+
retryInMs: RUNTIME_API_RETRY_DELAYS_MS[attempt] ?? 0,
|
|
1405
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1406
|
+
});
|
|
1378
1407
|
await sleepRuntimeApiRetry(attempt);
|
|
1379
1408
|
continue;
|
|
1380
1409
|
}
|
|
@@ -1390,8 +1419,27 @@ async function postRuntimeApi<T>(
|
|
|
1390
1419
|
attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
|
|
1391
1420
|
!isRetryableRuntimeApiResponse(res.status, text)
|
|
1392
1421
|
) {
|
|
1422
|
+
console.warn('[play-harness.runtime_api.exhausted]', {
|
|
1423
|
+
request: requestDescription,
|
|
1424
|
+
attempt: attempt + 1,
|
|
1425
|
+
maxAttempts,
|
|
1426
|
+
requestId,
|
|
1427
|
+
baseOrigin,
|
|
1428
|
+
status: res.status,
|
|
1429
|
+
body: redacted,
|
|
1430
|
+
});
|
|
1393
1431
|
throw lastError;
|
|
1394
1432
|
}
|
|
1433
|
+
console.warn('[play-harness.runtime_api.retry]', {
|
|
1434
|
+
request: requestDescription,
|
|
1435
|
+
attempt: attempt + 1,
|
|
1436
|
+
maxAttempts,
|
|
1437
|
+
requestId,
|
|
1438
|
+
baseOrigin,
|
|
1439
|
+
status: res.status,
|
|
1440
|
+
retryInMs: RUNTIME_API_RETRY_DELAYS_MS[attempt] ?? 0,
|
|
1441
|
+
body: redacted,
|
|
1442
|
+
});
|
|
1395
1443
|
await sleepRuntimeApiRetry(attempt);
|
|
1396
1444
|
}
|
|
1397
1445
|
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
@@ -15,6 +15,31 @@ export type WorkflowRetryDecision =
|
|
|
15
15
|
message: string | null;
|
|
16
16
|
};
|
|
17
17
|
|
|
18
|
+
function workflowRetryReasonForFailure(failure: {
|
|
19
|
+
code: string;
|
|
20
|
+
cause?: string;
|
|
21
|
+
}): string {
|
|
22
|
+
if (failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED') {
|
|
23
|
+
return 'cloudflare_worker_subrequest_limit';
|
|
24
|
+
}
|
|
25
|
+
if (
|
|
26
|
+
failure.cause &&
|
|
27
|
+
/runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i.test(
|
|
28
|
+
failure.cause,
|
|
29
|
+
)
|
|
30
|
+
) {
|
|
31
|
+
return 'vercel_runtime_api_deployment_missing';
|
|
32
|
+
}
|
|
33
|
+
return 'cloudflare_durable_object_reset';
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function workflowRetryExhaustedReasonForFailure(failure: {
|
|
37
|
+
code: string;
|
|
38
|
+
cause?: string;
|
|
39
|
+
}): string {
|
|
40
|
+
return `${workflowRetryReasonForFailure(failure)}_retry_exhausted`;
|
|
41
|
+
}
|
|
42
|
+
|
|
18
43
|
export function decideWorkflowPlatformRetry(input: {
|
|
19
44
|
workflowStatus: string;
|
|
20
45
|
error: string | null;
|
|
@@ -33,10 +58,7 @@ export function decideWorkflowPlatformRetry(input: {
|
|
|
33
58
|
) {
|
|
34
59
|
return {
|
|
35
60
|
action: 'retry',
|
|
36
|
-
reason:
|
|
37
|
-
failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED'
|
|
38
|
-
? 'cloudflare_worker_subrequest_limit'
|
|
39
|
-
: 'cloudflare_durable_object_reset',
|
|
61
|
+
reason: workflowRetryReasonForFailure(failure),
|
|
40
62
|
nextAttempt: input.retryAttempts + 1,
|
|
41
63
|
message: failure.message,
|
|
42
64
|
};
|
|
@@ -44,9 +66,7 @@ export function decideWorkflowPlatformRetry(input: {
|
|
|
44
66
|
return {
|
|
45
67
|
action: 'fail',
|
|
46
68
|
reason: retryablePlatformReset
|
|
47
|
-
? failure
|
|
48
|
-
? 'cloudflare_worker_subrequest_limit_retry_exhausted'
|
|
49
|
-
: 'cloudflare_durable_object_reset_retry_exhausted'
|
|
69
|
+
? workflowRetryExhaustedReasonForFailure(failure)
|
|
50
70
|
: null,
|
|
51
71
|
message: failure.message,
|
|
52
72
|
};
|
|
@@ -494,6 +494,29 @@ export type BillingInvoicesResult = {
|
|
|
494
494
|
entries: BillingInvoiceEntry[];
|
|
495
495
|
};
|
|
496
496
|
|
|
497
|
+
/** Saved payment method details returned by one-click billing top-up. */
|
|
498
|
+
export type BillingPaymentMethodSummary = {
|
|
499
|
+
brand?: string | null;
|
|
500
|
+
last4?: string | null;
|
|
501
|
+
exp_month?: number | null;
|
|
502
|
+
exp_year?: number | null;
|
|
503
|
+
label?: string | null;
|
|
504
|
+
};
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* Result of `POST /api/v2/billing/top-up`: a saved-card charge and purchased
|
|
508
|
+
* Deepline credit grant for the active workspace.
|
|
509
|
+
*/
|
|
510
|
+
export type BillingTopUpResult = {
|
|
511
|
+
ok: true;
|
|
512
|
+
credits: number;
|
|
513
|
+
amount_cents: number;
|
|
514
|
+
currency: string;
|
|
515
|
+
balance: number;
|
|
516
|
+
stripe_payment_intent_id: string;
|
|
517
|
+
payment_method?: BillingPaymentMethodSummary | null;
|
|
518
|
+
};
|
|
519
|
+
|
|
497
520
|
/** One published plan from `GET /api/v2/billing/catalog/current`. */
|
|
498
521
|
export type BillingPlanEntry = {
|
|
499
522
|
plan_family_id: string;
|
|
@@ -553,6 +576,11 @@ export type BillingPlansResult = {
|
|
|
553
576
|
* @sdkReference client 030 client.billing
|
|
554
577
|
*/
|
|
555
578
|
export type BillingNamespace = {
|
|
579
|
+
/** Charge the saved payment method and add Deepline credits to the active workspace. */
|
|
580
|
+
topUp: (options: {
|
|
581
|
+
credits: number;
|
|
582
|
+
idempotencyKey?: string;
|
|
583
|
+
}) => Promise<BillingTopUpResult>;
|
|
556
584
|
/** Published plans plus the plan you are on ("what plans exist and what am I on"). */
|
|
557
585
|
plans: () => Promise<BillingPlansResult>;
|
|
558
586
|
subscription: {
|
|
@@ -701,7 +729,10 @@ const STAGE_LEGACY_MULTIPART_MAX_BYTES = 4_000_000;
|
|
|
701
729
|
|
|
702
730
|
const STAGE_DIRECT_UPLOAD_MAX_ATTEMPTS = 3;
|
|
703
731
|
|
|
704
|
-
function stagedUploadIdentity(
|
|
732
|
+
function stagedUploadIdentity(
|
|
733
|
+
logicalPath: string,
|
|
734
|
+
contentHash: string,
|
|
735
|
+
): string {
|
|
705
736
|
return `${contentHash}:${logicalPath}`;
|
|
706
737
|
}
|
|
707
738
|
|
|
@@ -1021,6 +1052,7 @@ export class DeeplineClient {
|
|
|
1021
1052
|
stopAll: (options) => this.stopAllRuns(options),
|
|
1022
1053
|
};
|
|
1023
1054
|
this.billing = {
|
|
1055
|
+
topUp: (options) => this.topUpBillingBalance(options),
|
|
1024
1056
|
plans: () => this.getBillingPlans(),
|
|
1025
1057
|
subscription: {
|
|
1026
1058
|
status: () => this.getBillingSubscriptionStatus(),
|
|
@@ -3251,6 +3283,34 @@ export class DeeplineClient {
|
|
|
3251
3283
|
return this.http.get<BillingPlansResult>('/api/v2/billing/catalog/current');
|
|
3252
3284
|
}
|
|
3253
3285
|
|
|
3286
|
+
/**
|
|
3287
|
+
* Charge the saved payment method and add Deepline credits to the active
|
|
3288
|
+
* workspace. Prefer `client.billing.topUp(...)`.
|
|
3289
|
+
*
|
|
3290
|
+
* @throws {@link DeeplineError} with `statusCode: 409` when checkout is
|
|
3291
|
+
* required because the workspace has no saved payment method or the card
|
|
3292
|
+
* requires confirmation.
|
|
3293
|
+
*/
|
|
3294
|
+
async topUpBillingBalance(options: {
|
|
3295
|
+
credits: number;
|
|
3296
|
+
idempotencyKey?: string;
|
|
3297
|
+
}): Promise<BillingTopUpResult> {
|
|
3298
|
+
return this.http.post<BillingTopUpResult>(
|
|
3299
|
+
'/api/v2/billing/top-up',
|
|
3300
|
+
{
|
|
3301
|
+
credits: options.credits,
|
|
3302
|
+
...(options.idempotencyKey
|
|
3303
|
+
? { idempotency_key: options.idempotencyKey }
|
|
3304
|
+
: {}),
|
|
3305
|
+
},
|
|
3306
|
+
undefined,
|
|
3307
|
+
// The idempotency key makes a retry at the product layer safe, but the
|
|
3308
|
+
// CLI should not hide ambiguous local delivery failures behind implicit
|
|
3309
|
+
// transport retries for a payment mutation.
|
|
3310
|
+
{ maxRetries: 0, exactUrlOnly: true },
|
|
3311
|
+
);
|
|
3312
|
+
}
|
|
3313
|
+
|
|
3254
3314
|
/**
|
|
3255
3315
|
* Subscription state for the active workspace: active plan, whether a
|
|
3256
3316
|
* Stripe subscription backs it, renewal/cancellation facts, and remaining
|
|
@@ -62,8 +62,10 @@ export type {
|
|
|
62
62
|
BillingInvoiceEntry,
|
|
63
63
|
BillingInvoicesResult,
|
|
64
64
|
BillingNamespace,
|
|
65
|
+
BillingPaymentMethodSummary,
|
|
65
66
|
BillingSubscriptionCancelResult,
|
|
66
67
|
BillingSubscriptionStatus,
|
|
68
|
+
BillingTopUpResult,
|
|
67
69
|
PlayStatus,
|
|
68
70
|
PlaySheetRow,
|
|
69
71
|
PlaySheetRowsResult,
|