deepline 0.1.192 → 0.1.194
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +3 -18
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +23 -13
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +77 -37
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime-api-retry.ts +23 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +7 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +10 -3
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +8 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +14 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -1
- package/dist/cli/index.js +2 -2
- package/dist/cli/index.mjs +2 -2
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/package.json +1 -1
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
* is a coordinator-side safety net for missed signals.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
+
import { isRetryableWorkerRuntimeApiError } from './runtime-api-retry';
|
|
16
|
+
|
|
15
17
|
export type WorkflowStepLike = {
|
|
16
18
|
waitForEvent: (
|
|
17
19
|
name: string,
|
|
@@ -77,12 +79,6 @@ const CHILD_TERMINAL_DURABLE_VERIFY_GRACE_MS = 5_000;
|
|
|
77
79
|
const CHILD_TERMINAL_DURABLE_VERIFY_POLL_MS = 250;
|
|
78
80
|
const CHILD_TERMINAL_DURABLE_VERIFY_READ_TIMEOUT_MS = 3_000;
|
|
79
81
|
const CHILD_TERMINAL_DURABLE_VERIFY_MIN_READ_TIMEOUT_MS = 2_000;
|
|
80
|
-
const CHILD_TERMINAL_PERMANENT_RUNTIME_API_STATUS_RE =
|
|
81
|
-
/runtime API (400|401|403|404):/i;
|
|
82
|
-
const CHILD_TERMINAL_RETRYABLE_RUNTIME_API_STATUS_RE =
|
|
83
|
-
/runtime API (408|429|500|502|503|504|530):/i;
|
|
84
|
-
const CHILD_TERMINAL_RETRYABLE_ERROR_RE =
|
|
85
|
-
/timed out|timeout|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT/i;
|
|
86
82
|
|
|
87
83
|
export interface AwaitChildTerminalInput {
|
|
88
84
|
parentRunId: string;
|
|
@@ -140,17 +136,6 @@ function terminalPayloadFromState(
|
|
|
140
136
|
return isRecord(state?.data) ? state.data : null;
|
|
141
137
|
}
|
|
142
138
|
|
|
143
|
-
function isRetryableChildTerminalSnapshotError(error: unknown): boolean {
|
|
144
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
145
|
-
if (CHILD_TERMINAL_PERMANENT_RUNTIME_API_STATUS_RE.test(message)) {
|
|
146
|
-
return false;
|
|
147
|
-
}
|
|
148
|
-
return (
|
|
149
|
-
CHILD_TERMINAL_RETRYABLE_RUNTIME_API_STATUS_RE.test(message) ||
|
|
150
|
-
CHILD_TERMINAL_RETRYABLE_ERROR_RE.test(message)
|
|
151
|
-
);
|
|
152
|
-
}
|
|
153
|
-
|
|
154
139
|
function sleep(ms: number): Promise<void> {
|
|
155
140
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
156
141
|
}
|
|
@@ -259,7 +244,7 @@ async function readVerifiedChildTerminalPayload(input: {
|
|
|
259
244
|
if (payload) return { payload, lastState, lastError };
|
|
260
245
|
} catch (error) {
|
|
261
246
|
lastError = error;
|
|
262
|
-
if (!
|
|
247
|
+
if (!isRetryableWorkerRuntimeApiError(error)) {
|
|
263
248
|
throw error;
|
|
264
249
|
}
|
|
265
250
|
}
|
|
@@ -23,7 +23,10 @@ import {
|
|
|
23
23
|
} from '@cloudflare/dynamic-workflows';
|
|
24
24
|
import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
|
|
25
25
|
import { buildChildRunId as buildSharedChildRunId } from '../../../shared_libs/play-runtime/child-run-id';
|
|
26
|
-
import type {
|
|
26
|
+
import type {
|
|
27
|
+
PlayCallGovernanceSnapshot,
|
|
28
|
+
PlayRunInputPayload,
|
|
29
|
+
} from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
27
30
|
import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runtime/test-runtime-seams';
|
|
28
31
|
import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
|
|
29
32
|
import {
|
|
@@ -96,7 +99,7 @@ export type PlayWorkflowParams = {
|
|
|
96
99
|
artifactStorageKey: string;
|
|
97
100
|
artifactHash: string;
|
|
98
101
|
graphHash: string;
|
|
99
|
-
input:
|
|
102
|
+
input: PlayRunInputPayload;
|
|
100
103
|
force?: boolean;
|
|
101
104
|
runAttempt?: number;
|
|
102
105
|
inputFile?: {
|
|
@@ -2396,7 +2399,7 @@ function buildChildRunId(input: {
|
|
|
2396
2399
|
? governance.parentPlayName
|
|
2397
2400
|
: null,
|
|
2398
2401
|
key: typeof governance?.key === 'string' ? governance.key : null,
|
|
2399
|
-
input: isRecord(body.input) ? body.input : {},
|
|
2402
|
+
input: isRecord(body.input) || Array.isArray(body.input) ? body.input : {},
|
|
2400
2403
|
graphHash: isRecord(body.manifest)
|
|
2401
2404
|
? typeof body.manifest.graphHash === 'string'
|
|
2402
2405
|
? body.manifest.graphHash
|
|
@@ -2455,24 +2458,31 @@ function shouldCanonicalizeRuntimeApiBaseUrl(input: {
|
|
|
2455
2458
|
}
|
|
2456
2459
|
const configuredHostname = input.configuredBaseUrl.hostname.toLowerCase();
|
|
2457
2460
|
const overrideHostname = input.overrideBaseUrl.hostname.toLowerCase();
|
|
2458
|
-
if (
|
|
2461
|
+
if (isDeeplineCoordinatorWorkerHost(overrideHostname)) {
|
|
2459
2462
|
return true;
|
|
2460
2463
|
}
|
|
2461
|
-
|
|
2462
|
-
(
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
return true;
|
|
2464
|
+
const overrideIsDeeplineVercelHost =
|
|
2465
|
+
DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE.test(overrideHostname) ||
|
|
2466
|
+
DEEPLINE_VERCEL_PROJECT_HOST_RE.test(overrideHostname);
|
|
2467
|
+
if (!overrideIsDeeplineVercelHost) {
|
|
2468
|
+
return false;
|
|
2467
2469
|
}
|
|
2468
|
-
|
|
2470
|
+
const configuredIsDeeplineVercelHost =
|
|
2471
|
+
DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE.test(configuredHostname) ||
|
|
2472
|
+
DEEPLINE_VERCEL_PROJECT_HOST_RE.test(configuredHostname);
|
|
2473
|
+
return (
|
|
2474
|
+
configuredHostname === DEEPLINE_PRODUCTION_APP_HOST ||
|
|
2475
|
+
!configuredIsDeeplineVercelHost
|
|
2476
|
+
);
|
|
2469
2477
|
}
|
|
2470
2478
|
|
|
2471
2479
|
function resolveRuntimeApiBaseUrl(
|
|
2472
2480
|
env: CoordinatorEnv,
|
|
2473
2481
|
body: Record<string, unknown>,
|
|
2474
2482
|
): string {
|
|
2475
|
-
const configuredBaseUrl =
|
|
2483
|
+
const configuredBaseUrl =
|
|
2484
|
+
normalizeRuntimeBaseUrl(env.DEEPLINE_API_BASE_URL) ??
|
|
2485
|
+
env.DEEPLINE_API_BASE_URL.replace(/\/$/, '');
|
|
2476
2486
|
const overrideBaseUrl =
|
|
2477
2487
|
normalizeRuntimeBaseUrl(body.runtimeApiBaseUrl) ??
|
|
2478
2488
|
normalizeRuntimeBaseUrl(body.baseUrl);
|
|
@@ -2620,7 +2630,7 @@ function buildChildWorkflowParams(input: {
|
|
|
2620
2630
|
artifactStorageKey: manifest.artifactStorageKey,
|
|
2621
2631
|
artifactHash: manifest.artifactHash,
|
|
2622
2632
|
graphHash: manifest.graphHash,
|
|
2623
|
-
input: isRecord(body.input) ? body.input : {},
|
|
2633
|
+
input: isRecord(body.input) || Array.isArray(body.input) ? body.input : {},
|
|
2624
2634
|
contractSnapshot: {
|
|
2625
2635
|
source: 'published',
|
|
2626
2636
|
revisionVersion: null,
|
|
@@ -65,6 +65,7 @@ import {
|
|
|
65
65
|
type WorkflowStepLike,
|
|
66
66
|
} from './child-play-await';
|
|
67
67
|
import { submitChildPlayThroughCoordinator } from './child-play-submit';
|
|
68
|
+
import { isRetryableWorkerRuntimeApiError } from './runtime-api-retry';
|
|
68
69
|
import {
|
|
69
70
|
attachToolResultListDataset,
|
|
70
71
|
createToolExecuteResult,
|
|
@@ -307,6 +308,7 @@ import {
|
|
|
307
308
|
type PreviousCell,
|
|
308
309
|
} from '../../../shared_libs/play-runtime/cell-staleness';
|
|
309
310
|
import type { PlayRowUpdate } from '../../../shared_libs/play-runtime/ctx-types';
|
|
311
|
+
import type { PlayRunInputPayload } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
310
312
|
|
|
311
313
|
// The play's default export. The bundler injects this — see bundle-play-file.ts.
|
|
312
314
|
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
@@ -323,7 +325,7 @@ type RunRequest = {
|
|
|
323
325
|
playName: string;
|
|
324
326
|
graphHash?: string | null;
|
|
325
327
|
userEmail: string | null;
|
|
326
|
-
runtimeInput:
|
|
328
|
+
runtimeInput: PlayRunInputPayload;
|
|
327
329
|
/** Start a fresh run graph and recompute runtime-sheet rows. */
|
|
328
330
|
force?: boolean;
|
|
329
331
|
/** Explicit durable tool receipt refresh that can re-execute completed provider calls. */
|
|
@@ -648,8 +650,15 @@ type WorkerEnv = {
|
|
|
648
650
|
|
|
649
651
|
let cachedRuntimeApiBinding: WorkerEnv['RUNTIME_API'] | null = null;
|
|
650
652
|
let cachedRuntimeApiVercelBypassToken: string | null = null;
|
|
653
|
+
|
|
654
|
+
function runtimeApiBindingFromEnv(
|
|
655
|
+
env: WorkerEnv,
|
|
656
|
+
): WorkerEnv['RUNTIME_API'] | null {
|
|
657
|
+
return env.RUNTIME_API ?? env.HARNESS ?? null;
|
|
658
|
+
}
|
|
659
|
+
|
|
651
660
|
function captureRuntimeApiBinding(env: WorkerEnv): void {
|
|
652
|
-
cachedRuntimeApiBinding = env
|
|
661
|
+
cachedRuntimeApiBinding = runtimeApiBindingFromEnv(env);
|
|
653
662
|
cachedRuntimeApiVercelBypassToken =
|
|
654
663
|
typeof env.VERCEL_PROTECTION_BYPASS_TOKEN === 'string' &&
|
|
655
664
|
env.VERCEL_PROTECTION_BYPASS_TOKEN.trim()
|
|
@@ -831,7 +840,9 @@ async function fetchRuntimeApi(
|
|
|
831
840
|
signal: controller.signal,
|
|
832
841
|
};
|
|
833
842
|
if (!cachedRuntimeApiBinding) {
|
|
834
|
-
throw new Error(
|
|
843
|
+
throw new Error(
|
|
844
|
+
'[play-harness] RUNTIME_API/HARNESS service binding is required',
|
|
845
|
+
);
|
|
835
846
|
}
|
|
836
847
|
if (abortSignal?.aborted) {
|
|
837
848
|
throw abortError();
|
|
@@ -1360,7 +1371,7 @@ async function postRuntimeApi<T>(
|
|
|
1360
1371
|
lastError = error;
|
|
1361
1372
|
if (
|
|
1362
1373
|
attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
|
|
1363
|
-
!
|
|
1374
|
+
!isRetryableWorkerRuntimeApiError(error)
|
|
1364
1375
|
) {
|
|
1365
1376
|
throw error;
|
|
1366
1377
|
}
|
|
@@ -1386,15 +1397,6 @@ async function postRuntimeApi<T>(
|
|
|
1386
1397
|
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
1387
1398
|
}
|
|
1388
1399
|
|
|
1389
|
-
const RETRYABLE_RUNTIME_API_ERROR_RE =
|
|
1390
|
-
/timeout|timed out|fetch failed|ECONNRESET|ECONNREFUSED|DeploymentNotFound|requested deployment .*exist|Authentication infrastructure/i;
|
|
1391
|
-
|
|
1392
|
-
function isRetryableRuntimeApiError(error: unknown): boolean {
|
|
1393
|
-
return RETRYABLE_RUNTIME_API_ERROR_RE.test(
|
|
1394
|
-
error instanceof Error ? error.message : String(error),
|
|
1395
|
-
);
|
|
1396
|
-
}
|
|
1397
|
-
|
|
1398
1400
|
function isRetryableRuntimePersistenceError(error: unknown): boolean {
|
|
1399
1401
|
const message = error instanceof Error ? error.message : String(error);
|
|
1400
1402
|
return /deadlock detected|Runtime Postgres connection timed out|could not serialize access|tuple concurrently updated/i.test(
|
|
@@ -1403,17 +1405,18 @@ function isRetryableRuntimePersistenceError(error: unknown): boolean {
|
|
|
1403
1405
|
}
|
|
1404
1406
|
|
|
1405
1407
|
function isRetryableRuntimeApiResponse(status: number, body: string): boolean {
|
|
1406
|
-
|
|
1408
|
+
return (
|
|
1407
1409
|
status === 408 ||
|
|
1408
1410
|
status === 429 ||
|
|
1409
|
-
status
|
|
1410
|
-
status ===
|
|
1411
|
-
status ===
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1411
|
+
(status >= 502 && status <= 504) ||
|
|
1412
|
+
status === 530 ||
|
|
1413
|
+
(status === 404 &&
|
|
1414
|
+
/DEPLOYMENT_NOT_FOUND|DeploymentNotFound|deployment .*not found|requested deployment .*exist/i.test(
|
|
1415
|
+
body,
|
|
1416
|
+
)) ||
|
|
1417
|
+
(status === 500 &&
|
|
1418
|
+
isRetryableWorkerRuntimeApiError(`runtime API ${status}: ${body}`))
|
|
1419
|
+
);
|
|
1417
1420
|
}
|
|
1418
1421
|
|
|
1419
1422
|
async function sleepRuntimeApiRetry(attempt: number): Promise<void> {
|
|
@@ -1625,6 +1628,42 @@ function workerChildRevisionFingerprint(input: {
|
|
|
1625
1628
|
);
|
|
1626
1629
|
}
|
|
1627
1630
|
|
|
1631
|
+
type RuntimeResolvedPlayResponse = {
|
|
1632
|
+
manifest?: PlayRuntimeManifest | null;
|
|
1633
|
+
};
|
|
1634
|
+
|
|
1635
|
+
async function resolveAuthorizedWorkerChildManifest(input: {
|
|
1636
|
+
req: RunRequest;
|
|
1637
|
+
playRef: string;
|
|
1638
|
+
cache: Map<string, PlayRuntimeManifest>;
|
|
1639
|
+
}): Promise<PlayRuntimeManifest | null> {
|
|
1640
|
+
const cached = input.cache.get(input.playRef);
|
|
1641
|
+
if (cached) return cached;
|
|
1642
|
+
const preloaded = input.req.childPlayManifests?.[input.playRef];
|
|
1643
|
+
if (preloaded) {
|
|
1644
|
+
input.cache.set(input.playRef, preloaded);
|
|
1645
|
+
return preloaded;
|
|
1646
|
+
}
|
|
1647
|
+
const response = await postRuntimeApi<RuntimeResolvedPlayResponse>(
|
|
1648
|
+
input.req.baseUrl,
|
|
1649
|
+
input.req.executorToken,
|
|
1650
|
+
{
|
|
1651
|
+
action: 'resolve_play',
|
|
1652
|
+
playRef: input.playRef,
|
|
1653
|
+
artifactKind: 'esm_workers',
|
|
1654
|
+
},
|
|
1655
|
+
{
|
|
1656
|
+
runtimeTestFaultHeader: input.req.runtimeTestFaultHeader,
|
|
1657
|
+
timeoutMsOverride: 30_000,
|
|
1658
|
+
},
|
|
1659
|
+
);
|
|
1660
|
+
const manifest = response.manifest ?? null;
|
|
1661
|
+
if (manifest) {
|
|
1662
|
+
input.cache.set(input.playRef, manifest);
|
|
1663
|
+
}
|
|
1664
|
+
return manifest;
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1628
1667
|
async function childPlayEventKey(input: {
|
|
1629
1668
|
key: string;
|
|
1630
1669
|
workflowId: string;
|
|
@@ -2073,7 +2112,7 @@ async function callToolDirect(
|
|
|
2073
2112
|
);
|
|
2074
2113
|
if (
|
|
2075
2114
|
transportAttempt >= TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS ||
|
|
2076
|
-
!
|
|
2115
|
+
!isRetryableWorkerRuntimeApiError(error)
|
|
2077
2116
|
) {
|
|
2078
2117
|
throw lastError;
|
|
2079
2118
|
}
|
|
@@ -2357,7 +2396,10 @@ function isRuntimeReceiptPersistenceError(error: unknown): boolean {
|
|
|
2357
2396
|
}
|
|
2358
2397
|
|
|
2359
2398
|
function isRunFatalWorkerRowError(error: unknown): boolean {
|
|
2399
|
+
const failure = normalizePlayRunFailure(error);
|
|
2360
2400
|
return (
|
|
2401
|
+
failure.code === 'PLATFORM_DEPLOY_INTERRUPTED' ||
|
|
2402
|
+
failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED' ||
|
|
2361
2403
|
isCoordinatorProgressAuthorizationError(error) ||
|
|
2362
2404
|
isRowIsolationExemptError(error) ||
|
|
2363
2405
|
isRuntimeReceiptPersistenceError(error) ||
|
|
@@ -4263,6 +4305,7 @@ function createMinimalWorkerCtx(
|
|
|
4263
4305
|
// lineage snapshot are owned by the Governor (createGovernorForRun above).
|
|
4264
4306
|
// The worker keeps only substrate mechanism here.
|
|
4265
4307
|
const stepCallCounts: Record<string, number> = {};
|
|
4308
|
+
const childManifestCache = new Map<string, PlayRuntimeManifest>();
|
|
4266
4309
|
const secretRedactor = createSecretRedactionContext();
|
|
4267
4310
|
|
|
4268
4311
|
const resolveSecretAuth = async (auth?: SecretAuth) => {
|
|
@@ -6784,7 +6827,11 @@ function createMinimalWorkerCtx(
|
|
|
6784
6827
|
if (!resolvedName) {
|
|
6785
6828
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
6786
6829
|
}
|
|
6787
|
-
const childManifest =
|
|
6830
|
+
const childManifest = await resolveAuthorizedWorkerChildManifest({
|
|
6831
|
+
req,
|
|
6832
|
+
playRef: resolvedName,
|
|
6833
|
+
cache: childManifestCache,
|
|
6834
|
+
});
|
|
6788
6835
|
const receiptKey = buildDurableCtxCallCacheKey({
|
|
6789
6836
|
orgId: req.orgId,
|
|
6790
6837
|
playId: req.playName,
|
|
@@ -6825,7 +6872,7 @@ function createMinimalWorkerCtx(
|
|
|
6825
6872
|
});
|
|
6826
6873
|
if (!childManifest) {
|
|
6827
6874
|
throw new Error(
|
|
6828
|
-
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}:
|
|
6875
|
+
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: not callable.`,
|
|
6829
6876
|
);
|
|
6830
6877
|
}
|
|
6831
6878
|
const childIsDatasetBacked = childPipelineUsesCtxDataset(
|
|
@@ -6888,7 +6935,7 @@ function createMinimalWorkerCtx(
|
|
|
6888
6935
|
childIdempotencyKey: receiptKey,
|
|
6889
6936
|
input: isRecord(input) ? input : {},
|
|
6890
6937
|
orgId: req.orgId,
|
|
6891
|
-
|
|
6938
|
+
callbackUrl: req.callbackUrl,
|
|
6892
6939
|
baseUrl: req.baseUrl,
|
|
6893
6940
|
integrationMode: req.integrationMode ?? null,
|
|
6894
6941
|
parentExecutorToken: req.executorToken,
|
|
@@ -7378,10 +7425,7 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
|
|
|
7378
7425
|
await probeHarnessOnce(env, runPrefix);
|
|
7379
7426
|
const ctx = createMinimalWorkerCtx(req, emit, env);
|
|
7380
7427
|
const result = await (
|
|
7381
|
-
playFn as (
|
|
7382
|
-
ctx: unknown,
|
|
7383
|
-
input: Record<string, unknown>,
|
|
7384
|
-
) => Promise<unknown>
|
|
7428
|
+
playFn as (ctx: unknown, input: PlayRunInputPayload) => Promise<unknown>
|
|
7385
7429
|
)(ctx, req.runtimeInput);
|
|
7386
7430
|
emit({
|
|
7387
7431
|
type: 'result',
|
|
@@ -7910,10 +7954,7 @@ async function executeRunRequest(
|
|
|
7910
7954
|
try {
|
|
7911
7955
|
const playStartedAt = nowMs();
|
|
7912
7956
|
const result = await (
|
|
7913
|
-
playFn as (
|
|
7914
|
-
ctx: unknown,
|
|
7915
|
-
input: Record<string, unknown>,
|
|
7916
|
-
) => Promise<unknown>
|
|
7957
|
+
playFn as (ctx: unknown, input: PlayRunInputPayload) => Promise<unknown>
|
|
7917
7958
|
)(ctx, req.runtimeInput);
|
|
7918
7959
|
recordRunnerPerfTrace({
|
|
7919
7960
|
req,
|
|
@@ -8312,9 +8353,8 @@ function runRequestFromWorkflowParams(
|
|
|
8312
8353
|
force: params.force === true,
|
|
8313
8354
|
forceToolRefresh: params.forceToolRefresh === true,
|
|
8314
8355
|
runAttempt: normalizeWorkerRunAttempt(params.runAttempt),
|
|
8315
|
-
runtimeInput:
|
|
8316
|
-
|
|
8317
|
-
: {},
|
|
8356
|
+
runtimeInput:
|
|
8357
|
+
isRecord(params.input) || Array.isArray(params.input) ? params.input : {},
|
|
8318
8358
|
inlineCsv: isInlineCsv(params.inlineCsv) ? params.inlineCsv : null,
|
|
8319
8359
|
inputFiles:
|
|
8320
8360
|
inputFile && inputStorageKey
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
const PERMANENT_RUNTIME_API_STATUS_RE = /runtime API 40[0134]:/i;
|
|
2
|
+
const RETRYABLE_RUNTIME_API_ERROR_RE =
|
|
3
|
+
/runtime API (408|429|500|50[2-4]|530):|time(?:d out|out)|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|The deployment could not be found on Vercel|Authentication infrastructure/i;
|
|
4
|
+
|
|
5
|
+
function isVercelDeploymentMissingRuntimeApi404(message: string): boolean {
|
|
6
|
+
return (
|
|
7
|
+
/runtime API 404:/i.test(message) &&
|
|
8
|
+
/DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|The deployment could not be found on Vercel/i.test(
|
|
9
|
+
message,
|
|
10
|
+
)
|
|
11
|
+
);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function isRetryableWorkerRuntimeApiError(error: unknown): boolean {
|
|
15
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
16
|
+
if (
|
|
17
|
+
PERMANENT_RUNTIME_API_STATUS_RE.test(message) &&
|
|
18
|
+
!isVercelDeploymentMissingRuntimeApi404(message)
|
|
19
|
+
) {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
return RETRYABLE_RUNTIME_API_ERROR_RE.test(message);
|
|
23
|
+
}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
|
|
2
|
-
import type {
|
|
2
|
+
import type {
|
|
3
|
+
PlayCallGovernanceSnapshot,
|
|
4
|
+
PlayRunInputPayload,
|
|
5
|
+
} from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
3
6
|
import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
|
|
4
7
|
import type {
|
|
5
8
|
PlayRuntimeManifest,
|
|
@@ -33,7 +36,7 @@ export type WorkflowRetryPlayParams = {
|
|
|
33
36
|
artifactStorageKey: string;
|
|
34
37
|
artifactHash: string;
|
|
35
38
|
graphHash: string;
|
|
36
|
-
input:
|
|
39
|
+
input: PlayRunInputPayload;
|
|
37
40
|
inputFile?: {
|
|
38
41
|
name?: string;
|
|
39
42
|
r2Key?: string;
|
|
@@ -65,6 +68,8 @@ export type WorkflowRetryPlayParams = {
|
|
|
65
68
|
dynamicWorkerCode?: string | null;
|
|
66
69
|
executorToken: string;
|
|
67
70
|
baseUrl: string;
|
|
71
|
+
runtimeApiBaseUrl?: string | null;
|
|
72
|
+
callbackUrl?: string | null;
|
|
68
73
|
orgId: string;
|
|
69
74
|
userEmail: string;
|
|
70
75
|
userId?: string | null;
|
|
@@ -24,14 +24,19 @@ export function decideWorkflowPlatformRetry(input: {
|
|
|
24
24
|
return { action: 'fail', reason: null, message: null };
|
|
25
25
|
}
|
|
26
26
|
const failure = normalizePlayRunFailure(input.error);
|
|
27
|
-
const retryablePlatformReset =
|
|
27
|
+
const retryablePlatformReset =
|
|
28
|
+
failure.code === 'PLATFORM_DEPLOY_INTERRUPTED' ||
|
|
29
|
+
failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED';
|
|
28
30
|
if (
|
|
29
31
|
retryablePlatformReset &&
|
|
30
32
|
input.retryAttempts < PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT
|
|
31
33
|
) {
|
|
32
34
|
return {
|
|
33
35
|
action: 'retry',
|
|
34
|
-
reason:
|
|
36
|
+
reason:
|
|
37
|
+
failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED'
|
|
38
|
+
? 'cloudflare_worker_subrequest_limit'
|
|
39
|
+
: 'cloudflare_durable_object_reset',
|
|
35
40
|
nextAttempt: input.retryAttempts + 1,
|
|
36
41
|
message: failure.message,
|
|
37
42
|
};
|
|
@@ -39,7 +44,9 @@ export function decideWorkflowPlatformRetry(input: {
|
|
|
39
44
|
return {
|
|
40
45
|
action: 'fail',
|
|
41
46
|
reason: retryablePlatformReset
|
|
42
|
-
? '
|
|
47
|
+
? failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED'
|
|
48
|
+
? 'cloudflare_worker_subrequest_limit_retry_exhausted'
|
|
49
|
+
: 'cloudflare_durable_object_reset_retry_exhausted'
|
|
43
50
|
: null,
|
|
44
51
|
message: failure.message,
|
|
45
52
|
};
|
|
@@ -105,10 +105,10 @@ export const SDK_RELEASE = {
|
|
|
105
105
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
106
106
|
// fields shipped in 0.1.153.
|
|
107
107
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
108
|
-
version: '0.1.
|
|
108
|
+
version: '0.1.194',
|
|
109
109
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
110
110
|
supportPolicy: {
|
|
111
|
-
latest: '0.1.
|
|
111
|
+
latest: '0.1.194',
|
|
112
112
|
minimumSupported: '0.1.53',
|
|
113
113
|
deprecatedBelow: '0.1.53',
|
|
114
114
|
commandMinimumSupported: [
|
|
@@ -49,7 +49,7 @@ export interface ChildRunIdInputs {
|
|
|
49
49
|
*/
|
|
50
50
|
runPlaySemanticKey?: string | null;
|
|
51
51
|
/** Child input payload (semantic-fallback input). */
|
|
52
|
-
input?:
|
|
52
|
+
input?: unknown;
|
|
53
53
|
/**
|
|
54
54
|
* Compiled child graph hash. Only present on workers_edge; `null` in-process.
|
|
55
55
|
*/
|
|
@@ -89,11 +89,16 @@ export function buildChildRunId(inputs: ChildRunIdInputs): string {
|
|
|
89
89
|
stableStringify({
|
|
90
90
|
parentRunId: inputs.parentRunId,
|
|
91
91
|
parentPlayName:
|
|
92
|
-
typeof inputs.parentPlayName === 'string'
|
|
92
|
+
typeof inputs.parentPlayName === 'string'
|
|
93
|
+
? inputs.parentPlayName
|
|
94
|
+
: null,
|
|
93
95
|
childPlayName: inputs.childPlayName,
|
|
94
96
|
key: typeof inputs.key === 'string' ? inputs.key : null,
|
|
95
97
|
...(runPlaySemanticKey ? { runPlaySemanticKey } : {}),
|
|
96
|
-
input:
|
|
98
|
+
input:
|
|
99
|
+
isRecord(inputs.input) || Array.isArray(inputs.input)
|
|
100
|
+
? inputs.input
|
|
101
|
+
: {},
|
|
97
102
|
graphHash: typeof inputs.graphHash === 'string' ? inputs.graphHash : null,
|
|
98
103
|
});
|
|
99
104
|
const digest = sha256Hex(semanticKey).slice(0, 24);
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
const CLOUDFLARE_DURABLE_OBJECT_RESET_RE =
|
|
2
|
-
/Durable Object.*(?:code was updated|storage caused object)/;
|
|
2
|
+
/Durable Object.*(?:code (?:was|has been) updated|storage caused object)/;
|
|
3
|
+
const CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE =
|
|
4
|
+
/Too many subrequests by single Worker invocation/i;
|
|
3
5
|
|
|
4
6
|
export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
|
|
5
7
|
'Run interrupted by a platform deploy. Deepline retries this automatically when possible; if this error is still visible, re-run the same command.';
|
|
8
|
+
export const PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE =
|
|
9
|
+
'Worker subrequest limit hit. Deepline retries automatically; if still visible, re-run.';
|
|
6
10
|
|
|
7
11
|
export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
|
|
8
12
|
'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
|
|
@@ -64,6 +68,15 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
|
|
|
64
68
|
cause,
|
|
65
69
|
};
|
|
66
70
|
}
|
|
71
|
+
if (CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE.test(cause)) {
|
|
72
|
+
return {
|
|
73
|
+
code: 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED',
|
|
74
|
+
phase: 'runtime',
|
|
75
|
+
message: PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE,
|
|
76
|
+
retryable: true,
|
|
77
|
+
cause,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
67
80
|
const playDepthBudgetMatch = cause.match(
|
|
68
81
|
/Play execution playDepth budget exceeded \((\d+)\/(\d+)\)\./,
|
|
69
82
|
);
|
|
@@ -36,6 +36,8 @@ export const PLAY_SCHEDULER_BACKENDS = {
|
|
|
36
36
|
export type PlaySchedulerBackendId =
|
|
37
37
|
(typeof PLAY_SCHEDULER_BACKENDS)[keyof typeof PLAY_SCHEDULER_BACKENDS];
|
|
38
38
|
|
|
39
|
+
export type PlayRunInputPayload = Record<string, unknown> | unknown[];
|
|
40
|
+
|
|
39
41
|
export type PlayCallGovernanceSnapshot = {
|
|
40
42
|
rootRunId: string;
|
|
41
43
|
parentRunId: string;
|
|
@@ -83,7 +85,7 @@ export type PlaySchedulerSubmitInput = {
|
|
|
83
85
|
runtimeArtifact?: unknown;
|
|
84
86
|
artifactHash: string;
|
|
85
87
|
graphHash: string;
|
|
86
|
-
input:
|
|
88
|
+
input: PlayRunInputPayload;
|
|
87
89
|
/** Start a fresh run graph and recompute runtime-sheet rows. */
|
|
88
90
|
force?: boolean;
|
|
89
91
|
/** Explicit cache bypass for completed ctx.tools.execute receipts. */
|
package/dist/cli/index.js
CHANGED
|
@@ -623,10 +623,10 @@ var SDK_RELEASE = {
|
|
|
623
623
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
624
624
|
// fields shipped in 0.1.153.
|
|
625
625
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
626
|
-
version: "0.1.
|
|
626
|
+
version: "0.1.194",
|
|
627
627
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
628
628
|
supportPolicy: {
|
|
629
|
-
latest: "0.1.
|
|
629
|
+
latest: "0.1.194",
|
|
630
630
|
minimumSupported: "0.1.53",
|
|
631
631
|
deprecatedBelow: "0.1.53",
|
|
632
632
|
commandMinimumSupported: [
|
package/dist/cli/index.mjs
CHANGED
|
@@ -608,10 +608,10 @@ var SDK_RELEASE = {
|
|
|
608
608
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
609
609
|
// fields shipped in 0.1.153.
|
|
610
610
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
611
|
-
version: "0.1.
|
|
611
|
+
version: "0.1.194",
|
|
612
612
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
613
613
|
supportPolicy: {
|
|
614
|
-
latest: "0.1.
|
|
614
|
+
latest: "0.1.194",
|
|
615
615
|
minimumSupported: "0.1.53",
|
|
616
616
|
deprecatedBelow: "0.1.53",
|
|
617
617
|
commandMinimumSupported: [
|
package/dist/index.js
CHANGED
|
@@ -422,10 +422,10 @@ var SDK_RELEASE = {
|
|
|
422
422
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
423
423
|
// fields shipped in 0.1.153.
|
|
424
424
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
425
|
-
version: "0.1.
|
|
425
|
+
version: "0.1.194",
|
|
426
426
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
427
427
|
supportPolicy: {
|
|
428
|
-
latest: "0.1.
|
|
428
|
+
latest: "0.1.194",
|
|
429
429
|
minimumSupported: "0.1.53",
|
|
430
430
|
deprecatedBelow: "0.1.53",
|
|
431
431
|
commandMinimumSupported: [
|
package/dist/index.mjs
CHANGED
|
@@ -352,10 +352,10 @@ var SDK_RELEASE = {
|
|
|
352
352
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
353
353
|
// fields shipped in 0.1.153.
|
|
354
354
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
355
|
-
version: "0.1.
|
|
355
|
+
version: "0.1.194",
|
|
356
356
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
357
357
|
supportPolicy: {
|
|
358
|
-
latest: "0.1.
|
|
358
|
+
latest: "0.1.194",
|
|
359
359
|
minimumSupported: "0.1.53",
|
|
360
360
|
deprecatedBelow: "0.1.53",
|
|
361
361
|
commandMinimumSupported: [
|