deepline 0.3.23 → 0.3.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/http.ts +3 -20
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/skills-version.ts +107 -0
- package/dist/bundling-sources/sdk/src/types.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +83 -7
- package/dist/bundling-sources/shared_libs/play-runtime/db-session-plan.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/projection.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-modal-fallback.ts +101 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +61 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +74 -1
- package/dist/bundling-sources/shared_libs/plays/tool-category-descriptions.ts +5 -0
- package/dist/cli/index.js +678 -857
- package/dist/cli/index.mjs +748 -927
- package/dist/index.d.mts +18 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.js +78 -22
- package/dist/index.mjs +84 -28
- package/dist/install-integrity.json +1 -0
- package/package.json +1 -1
|
@@ -18,9 +18,6 @@
|
|
|
18
18
|
*
|
|
19
19
|
* @module
|
|
20
20
|
*/
|
|
21
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
22
|
-
import { homedir } from 'node:os';
|
|
23
|
-
import { join } from 'node:path';
|
|
24
21
|
import type { ResolvedConfig } from './types.js';
|
|
25
22
|
import {
|
|
26
23
|
AuthError,
|
|
@@ -36,8 +33,8 @@ import {
|
|
|
36
33
|
} from '../../shared_libs/tool-execution-error.js';
|
|
37
34
|
import { SDK_API_CONTRACT, SDK_VERSION } from './version.js';
|
|
38
35
|
import type { LiveEventEnvelope } from './types.js';
|
|
39
|
-
import { baseUrlSlug, sdkCliStateDirPath } from './config.js';
|
|
40
36
|
import { detectAgentRuntime, isCoworkLikeSandbox } from './agent-runtime.js';
|
|
37
|
+
import { readSdkSkillsLocalVersion } from './skills-version.js';
|
|
41
38
|
import {
|
|
42
39
|
ABSURD_RELEASE_OVERRIDE_HEADER,
|
|
43
40
|
COORDINATOR_INTERNAL_TOKEN_HEADER,
|
|
@@ -201,23 +198,9 @@ export class HttpClient {
|
|
|
201
198
|
);
|
|
202
199
|
if (explicit) return explicit;
|
|
203
200
|
try {
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
'skills-version',
|
|
201
|
+
return this.cleanDiagnosticHeader(
|
|
202
|
+
readSdkSkillsLocalVersion(this.config.baseUrl),
|
|
207
203
|
);
|
|
208
|
-
const legacyVersionPath = join(
|
|
209
|
-
process.env.HOME?.trim() || homedir(),
|
|
210
|
-
'.local',
|
|
211
|
-
'deepline',
|
|
212
|
-
baseUrlSlug(this.config.baseUrl),
|
|
213
|
-
'sdk-skills',
|
|
214
|
-
'.version',
|
|
215
|
-
);
|
|
216
|
-
const resolvedPath = existsSync(versionPath)
|
|
217
|
-
? versionPath
|
|
218
|
-
: legacyVersionPath;
|
|
219
|
-
if (!existsSync(resolvedPath)) return null;
|
|
220
|
-
return this.cleanDiagnosticHeader(readFileSync(resolvedPath, 'utf-8'));
|
|
221
204
|
} catch {
|
|
222
205
|
return null;
|
|
223
206
|
}
|
|
@@ -192,7 +192,7 @@ export const SDK_RELEASE = {
|
|
|
192
192
|
// 0.3.0 introduces raw-v2: complete scrubbed provider responses are
|
|
193
193
|
// available at toolResponse.rawV2 while toolResponse.raw and all declared
|
|
194
194
|
// getters keep their established compatibility behavior.
|
|
195
|
-
version: '0.3.
|
|
195
|
+
version: '0.3.25',
|
|
196
196
|
updateSummary:
|
|
197
197
|
'New raw-v2 tool responses preserve complete scrubbed provider envelopes at toolResponse.rawV2, including JSON:API included resources, links, cursors, and totals. Existing toolResponse.raw and declared getters keep working unchanged.',
|
|
198
198
|
contracts: {
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { detectAgentRuntime } from './agent-runtime.js';
|
|
4
|
+
import { sdkCliStateDirPath } from './config.js';
|
|
5
|
+
|
|
6
|
+
function activePluginSkillsDir(): string {
|
|
7
|
+
const pluginMode = process.env.DEEPLINE_PLUGIN_MODE?.trim().toLowerCase();
|
|
8
|
+
if (
|
|
9
|
+
pluginMode !== 'true' &&
|
|
10
|
+
pluginMode !== '1' &&
|
|
11
|
+
pluginMode !== 'yes' &&
|
|
12
|
+
pluginMode !== 'on'
|
|
13
|
+
) {
|
|
14
|
+
return '';
|
|
15
|
+
}
|
|
16
|
+
const dir = process.env.DEEPLINE_PLUGIN_SKILLS_DIR?.trim() ?? '';
|
|
17
|
+
return dir && existsSync(dir) ? dir : '';
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function hasActivePluginSkills(): boolean {
|
|
21
|
+
return Boolean(activePluginSkillsDir());
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function readPluginSkillsVersion(): string {
|
|
25
|
+
const dir = activePluginSkillsDir();
|
|
26
|
+
if (!dir) return '';
|
|
27
|
+
try {
|
|
28
|
+
return readFileSync(join(dir, '.version'), 'utf-8').trim();
|
|
29
|
+
} catch {
|
|
30
|
+
return '';
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function sdkSkillsVersionPath(
|
|
35
|
+
baseUrl: string,
|
|
36
|
+
agents: readonly string[] = [],
|
|
37
|
+
): string {
|
|
38
|
+
const suffix = agents.length > 0 ? `-${agents.join('-')}` : '';
|
|
39
|
+
return join(sdkCliStateDirPath(baseUrl), `skills${suffix}-version`);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function legacySdkSkillsVersionPath(baseUrl: string): string {
|
|
43
|
+
return join(dirname(sdkCliStateDirPath(baseUrl)), 'sdk-skills', '.version');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function resolveAutoSyncSkillAgents(): string[] {
|
|
47
|
+
switch (detectAgentRuntime()) {
|
|
48
|
+
case 'codex':
|
|
49
|
+
return ['codex'];
|
|
50
|
+
case 'claude_code':
|
|
51
|
+
return ['claude-code'];
|
|
52
|
+
case 'cursor':
|
|
53
|
+
return ['cursor'];
|
|
54
|
+
case 'gemini':
|
|
55
|
+
return ['gemini-cli'];
|
|
56
|
+
case 'antigravity':
|
|
57
|
+
return ['antigravity'];
|
|
58
|
+
default:
|
|
59
|
+
return [];
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function readSdkSkillsLocalVersion(baseUrl: string): string {
|
|
64
|
+
const pluginVersion = readPluginSkillsVersion();
|
|
65
|
+
if (pluginVersion) return pluginVersion;
|
|
66
|
+
|
|
67
|
+
const agents = resolveAutoSyncSkillAgents();
|
|
68
|
+
const scopedPath = sdkSkillsVersionPath(baseUrl, agents);
|
|
69
|
+
if (agents.length > 0 && existsSync(scopedPath)) {
|
|
70
|
+
try {
|
|
71
|
+
return readFileSync(scopedPath, 'utf-8').trim();
|
|
72
|
+
} catch {
|
|
73
|
+
return '';
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
// Legacy clients wrote a host-wide version after updating only their active
|
|
77
|
+
// agent. A detected agent without its own marker must therefore re-check
|
|
78
|
+
// instead of assuming another agent's legacy install applies to it.
|
|
79
|
+
if (agents.length > 0) {
|
|
80
|
+
const legacyPath = legacySdkSkillsVersionPath(baseUrl);
|
|
81
|
+
if (!existsSync(legacyPath)) return '';
|
|
82
|
+
try {
|
|
83
|
+
return readFileSync(legacyPath, 'utf-8').trim();
|
|
84
|
+
} catch {
|
|
85
|
+
return '';
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
const path = existsSync(sdkSkillsVersionPath(baseUrl))
|
|
89
|
+
? sdkSkillsVersionPath(baseUrl)
|
|
90
|
+
: legacySdkSkillsVersionPath(baseUrl);
|
|
91
|
+
if (!existsSync(path)) return '';
|
|
92
|
+
try {
|
|
93
|
+
return readFileSync(path, 'utf-8').trim();
|
|
94
|
+
} catch {
|
|
95
|
+
return '';
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function writeSdkSkillsLocalVersion(
|
|
100
|
+
baseUrl: string,
|
|
101
|
+
version: string,
|
|
102
|
+
agents: readonly string[],
|
|
103
|
+
): void {
|
|
104
|
+
const path = sdkSkillsVersionPath(baseUrl, agents);
|
|
105
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
106
|
+
writeFileSync(path, `${version}\n`, 'utf-8');
|
|
107
|
+
}
|
|
@@ -1280,6 +1280,17 @@ export interface PlayCheckResult {
|
|
|
1280
1280
|
valid: boolean;
|
|
1281
1281
|
errors: string[];
|
|
1282
1282
|
warnings?: string[];
|
|
1283
|
+
/**
|
|
1284
|
+
* Effective sandbox limits selected for this Play when authoring-contract
|
|
1285
|
+
* preflight succeeded. A valid modern check includes the default 30-minute
|
|
1286
|
+
* timeout when the author did not declare `runtime`.
|
|
1287
|
+
*/
|
|
1288
|
+
runtimeLimit?: {
|
|
1289
|
+
timeoutSeconds: number;
|
|
1290
|
+
memoryGiB: number;
|
|
1291
|
+
cpu: number;
|
|
1292
|
+
diskGiB: number;
|
|
1293
|
+
} | null;
|
|
1283
1294
|
staticPipeline?: Record<string, unknown> | null;
|
|
1284
1295
|
toolGetterHints?: PlayCheckToolGetterHint[];
|
|
1285
1296
|
/**
|
|
@@ -1343,6 +1354,13 @@ export interface PlayCheckResult {
|
|
|
1343
1354
|
limitBytes: number;
|
|
1344
1355
|
withinLimit: boolean;
|
|
1345
1356
|
};
|
|
1357
|
+
/** Present when this Play declares a cron binding. Advisory only; publish reserves capacity. */
|
|
1358
|
+
activeScheduledPlays?: {
|
|
1359
|
+
used: number;
|
|
1360
|
+
limit: number;
|
|
1361
|
+
remaining: number;
|
|
1362
|
+
approachingLimit: boolean;
|
|
1363
|
+
};
|
|
1346
1364
|
};
|
|
1347
1365
|
}
|
|
1348
1366
|
|
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
import { vercelProtectionBypassHeaders } from '@shared_libs/play-runtime/vercel-protection';
|
|
53
53
|
import type { RuntimeReceiptAction } from '@shared_libs/play-runtime/runtime-actions';
|
|
54
54
|
import { RUNTIME_CAPACITY_POLICY } from '@shared_libs/play-runtime/runtime-capacity-policy';
|
|
55
|
+
import { RUNTIME_RELIABILITY_POLICY } from '@shared_libs/play-runtime/runtime-reliability-policy';
|
|
55
56
|
import {
|
|
56
57
|
DEFAULT_RUNTIME_TRAFFIC_POLICY,
|
|
57
58
|
isRuntimeTrafficPolicy,
|
|
@@ -258,6 +259,10 @@ function applyRetryJitter(delayMs: number): number {
|
|
|
258
259
|
}
|
|
259
260
|
const APP_RUNTIME_API_DEFAULT_REQUEST_TIMEOUT_MS =
|
|
260
261
|
RUNTIME_CAPACITY_POLICY.receiptGateway.requestTimeoutMs;
|
|
262
|
+
const SIGNED_R2_FETCH_HEADERS_TIMEOUT_MS =
|
|
263
|
+
RUNTIME_RELIABILITY_POLICY.egress.fetchHeadersTimeoutMs;
|
|
264
|
+
const SIGNED_R2_FETCH_BODY_TIMEOUT_MS =
|
|
265
|
+
RUNTIME_RELIABILITY_POLICY.egress.fetchBodyTimeoutMs;
|
|
261
266
|
const APP_RUNTIME_RECEIPT_RETRY_TELEMETRY_TAG =
|
|
262
267
|
'[perf][worker.receipt_api.transport]';
|
|
263
268
|
const RUN_STATUS_LEDGER_SNAPSHOT_CACHE_LIMIT = 1_000;
|
|
@@ -1338,6 +1343,19 @@ type SignedR2ReadUrlResponse = {
|
|
|
1338
1343
|
expiresAt: string;
|
|
1339
1344
|
};
|
|
1340
1345
|
|
|
1346
|
+
class SignedR2FetchTimeoutError extends Error {
|
|
1347
|
+
constructor(
|
|
1348
|
+
kind: 'artifact' | 'staged_file',
|
|
1349
|
+
phase: 'headers' | 'body',
|
|
1350
|
+
timeoutMs: number,
|
|
1351
|
+
) {
|
|
1352
|
+
super(
|
|
1353
|
+
`Signed R2 ${kind} fetch exceeded its ${phase} deadline after ${timeoutMs}ms.`,
|
|
1354
|
+
);
|
|
1355
|
+
this.name = 'SignedR2FetchTimeoutError';
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1341
1359
|
function logSignedR2FetchPerf(input: {
|
|
1342
1360
|
kind: 'artifact' | 'staged_file';
|
|
1343
1361
|
storageKey: string;
|
|
@@ -1352,18 +1370,67 @@ function logSignedR2FetchPerf(input: {
|
|
|
1352
1370
|
});
|
|
1353
1371
|
}
|
|
1354
1372
|
|
|
1373
|
+
async function runSignedR2FetchPhase<T>(input: {
|
|
1374
|
+
controller: AbortController;
|
|
1375
|
+
kind: 'artifact' | 'staged_file';
|
|
1376
|
+
phase: 'headers' | 'body';
|
|
1377
|
+
run: () => Promise<T>;
|
|
1378
|
+
timeoutMs: number;
|
|
1379
|
+
}): Promise<T> {
|
|
1380
|
+
let timeout: ReturnType<typeof setTimeout> | null = null;
|
|
1381
|
+
const timeoutPromise = new Promise<never>((_resolve, reject) => {
|
|
1382
|
+
timeout = setTimeout(() => {
|
|
1383
|
+
const error = new SignedR2FetchTimeoutError(
|
|
1384
|
+
input.kind,
|
|
1385
|
+
input.phase,
|
|
1386
|
+
input.timeoutMs,
|
|
1387
|
+
);
|
|
1388
|
+
input.controller.abort(error);
|
|
1389
|
+
reject(error);
|
|
1390
|
+
}, input.timeoutMs);
|
|
1391
|
+
});
|
|
1392
|
+
try {
|
|
1393
|
+
return await Promise.race([input.run(), timeoutPromise]);
|
|
1394
|
+
} finally {
|
|
1395
|
+
if (timeout) clearTimeout(timeout);
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
async function fetchSignedR2Response(input: {
|
|
1400
|
+
kind: 'artifact' | 'staged_file';
|
|
1401
|
+
signed: SignedR2ReadUrlResponse;
|
|
1402
|
+
}): Promise<{ controller: AbortController; response: Response }> {
|
|
1403
|
+
const controller = new AbortController();
|
|
1404
|
+
const response = await runSignedR2FetchPhase({
|
|
1405
|
+
controller,
|
|
1406
|
+
kind: input.kind,
|
|
1407
|
+
phase: 'headers',
|
|
1408
|
+
timeoutMs: SIGNED_R2_FETCH_HEADERS_TIMEOUT_MS,
|
|
1409
|
+
run: () => fetch(input.signed.url, { signal: controller.signal }),
|
|
1410
|
+
});
|
|
1411
|
+
return { controller, response };
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1355
1414
|
async function fetchSignedR2Buffer(input: {
|
|
1356
1415
|
kind: 'artifact' | 'staged_file';
|
|
1357
1416
|
signed: SignedR2ReadUrlResponse;
|
|
1358
1417
|
}): Promise<Buffer> {
|
|
1359
1418
|
const startedAt = Date.now();
|
|
1360
|
-
const response = await
|
|
1419
|
+
const { controller, response } = await fetchSignedR2Response(input);
|
|
1361
1420
|
if (!response.ok) {
|
|
1362
1421
|
throw new Error(
|
|
1363
1422
|
`Signed R2 ${input.kind} fetch failed for ${input.signed.storageKey} with status ${response.status}: ${await response.text()}`,
|
|
1364
1423
|
);
|
|
1365
1424
|
}
|
|
1366
|
-
const buffer = Buffer.from(
|
|
1425
|
+
const buffer = Buffer.from(
|
|
1426
|
+
await runSignedR2FetchPhase({
|
|
1427
|
+
controller,
|
|
1428
|
+
kind: input.kind,
|
|
1429
|
+
phase: 'body',
|
|
1430
|
+
timeoutMs: SIGNED_R2_FETCH_BODY_TIMEOUT_MS,
|
|
1431
|
+
run: () => response.arrayBuffer(),
|
|
1432
|
+
}),
|
|
1433
|
+
);
|
|
1367
1434
|
logSignedR2FetchPerf({
|
|
1368
1435
|
kind: input.kind,
|
|
1369
1436
|
storageKey: input.signed.storageKey,
|
|
@@ -1379,7 +1446,7 @@ async function fetchSignedR2ToFile(input: {
|
|
|
1379
1446
|
targetPath: string;
|
|
1380
1447
|
}): Promise<void> {
|
|
1381
1448
|
const startedAt = Date.now();
|
|
1382
|
-
const response = await
|
|
1449
|
+
const { controller, response } = await fetchSignedR2Response(input);
|
|
1383
1450
|
if (!response.ok) {
|
|
1384
1451
|
throw new Error(
|
|
1385
1452
|
`Signed R2 ${input.kind} fetch failed for ${input.signed.storageKey} with status ${response.status}: ${await response.text()}`,
|
|
@@ -1390,10 +1457,19 @@ async function fetchSignedR2ToFile(input: {
|
|
|
1390
1457
|
`Signed R2 ${input.kind} fetch returned an empty response body for ${input.signed.storageKey}.`,
|
|
1391
1458
|
);
|
|
1392
1459
|
}
|
|
1393
|
-
await
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1460
|
+
await runSignedR2FetchPhase({
|
|
1461
|
+
controller,
|
|
1462
|
+
kind: input.kind,
|
|
1463
|
+
phase: 'body',
|
|
1464
|
+
timeoutMs: SIGNED_R2_FETCH_BODY_TIMEOUT_MS,
|
|
1465
|
+
run: async () =>
|
|
1466
|
+
await pipeline(
|
|
1467
|
+
Readable.fromWeb(
|
|
1468
|
+
response.body as Parameters<typeof Readable.fromWeb>[0],
|
|
1469
|
+
),
|
|
1470
|
+
createWriteStream(input.targetPath),
|
|
1471
|
+
),
|
|
1472
|
+
});
|
|
1397
1473
|
const written = await stat(input.targetPath);
|
|
1398
1474
|
logSignedR2FetchPerf({
|
|
1399
1475
|
kind: input.kind,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
2
|
+
getTopLevelPipelineSubsteps,
|
|
3
3
|
type PlaySheetContract,
|
|
4
4
|
type PlayStaticPipeline,
|
|
5
5
|
type PlayStaticSubstep,
|
|
@@ -70,7 +70,7 @@ function datasetSessionContracts(
|
|
|
70
70
|
}
|
|
71
71
|
const rootContract = pipeline.sheetContract ?? null;
|
|
72
72
|
return currentExecutionScopeSubsteps(
|
|
73
|
-
|
|
73
|
+
getTopLevelPipelineSubsteps(pipeline),
|
|
74
74
|
).flatMap<DatasetSessionContract>((substep): DatasetSessionContract[] => {
|
|
75
75
|
if (substep.type !== 'dataset') return [];
|
|
76
76
|
const rawNamespace = (substep.tableNamespace ?? substep.field ?? '').trim();
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
//
|
|
3
3
|
// "Projection" is the step that turns a provider response into the value a
|
|
4
4
|
// target (email, email_status, phone, ...) resolves to. Historically this was
|
|
5
|
-
// reimplemented in three runtimes (the V2 tool-result runtime, the
|
|
5
|
+
// reimplemented in three runtimes (the V2 tool-result runtime, the portable
|
|
6
6
|
// waterfall runtime, and the emitted V1-enrich play) with divergent precedence
|
|
7
7
|
// — a latent drift bug class. This module is the one authoritative
|
|
8
8
|
// implementation: callers supply a `ProjectionLookup` that knows how to walk
|
|
@@ -29,7 +29,7 @@ export type ProjectionHit = { value: unknown; path: string } | null;
|
|
|
29
29
|
* The only seam-crossing dependency. Each runtime supplies an adapter that
|
|
30
30
|
* resolves a list of candidate paths against its own payload shape and returns
|
|
31
31
|
* the first meaningful hit (or null). This absorbs the input-shape difference
|
|
32
|
-
* (V2 `{toolResponse:{raw}}` envelope vs
|
|
32
|
+
* (V2 `{toolResponse:{raw}}` envelope vs portable raw payload vs the enrich
|
|
33
33
|
* play's pre-projected getters); the interpreter itself is payload-agnostic.
|
|
34
34
|
*/
|
|
35
35
|
export type ProjectionLookup = (paths: readonly string[]) => ProjectionHit;
|
|
@@ -18,6 +18,7 @@ const RUNTIME_SANDBOX_START_FAILED_RE = /\bRUNTIME_SANDBOX_START_FAILED\b/i;
|
|
|
18
18
|
const RUNTIME_SANDBOX_OOM_RE =
|
|
19
19
|
/\bRUNTIME_SANDBOX_OOM\b|(?:javascript heap out of memory|fatal error:.*(?:heap|allocation).*memory)/i;
|
|
20
20
|
const RUNTIME_SANDBOX_KILLED_RE = /\bRUNTIME_SANDBOX_KILLED\b/i;
|
|
21
|
+
const MODAL_PAYLOAD_UPLOAD_TIMEOUT_RE = /\bMODAL_PAYLOAD_UPLOAD_TIMEOUT\b/i;
|
|
21
22
|
const OUTPUT_TOO_LARGE_RE = /\b(?:OUTPUT_TOO_LARGE|OutputTooLarge)\b/;
|
|
22
23
|
|
|
23
24
|
export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
|
|
@@ -48,6 +49,8 @@ export const RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_MESSAGE =
|
|
|
48
49
|
// carries: nothing ran, so no provider call can already exist.
|
|
49
50
|
export const RUNTIME_SANDBOX_START_FAILED_MESSAGE =
|
|
50
51
|
'The execution sandbox never finished starting, so this play never began running. Re-run the same command; if this keeps happening, contact Deepline support with the run ID.';
|
|
52
|
+
export const MODAL_PAYLOAD_UPLOAD_TIMEOUT_MESSAGE =
|
|
53
|
+
'The execution sandbox payload upload timed out before this play began. Re-run the same command; if this keeps happening, contact Deepline support with the run ID.';
|
|
51
54
|
|
|
52
55
|
export const WORKSPACE_STORAGE_NOT_READY_CODE = 'WORKSPACE_STORAGE_NOT_READY';
|
|
53
56
|
|
|
@@ -371,6 +374,15 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
|
|
|
371
374
|
...(causes.length > 0 ? { causes } : {}),
|
|
372
375
|
};
|
|
373
376
|
}
|
|
377
|
+
if (MODAL_PAYLOAD_UPLOAD_TIMEOUT_RE.test(rawCause)) {
|
|
378
|
+
return {
|
|
379
|
+
code: 'MODAL_PAYLOAD_UPLOAD_TIMEOUT',
|
|
380
|
+
phase: 'infrastructure',
|
|
381
|
+
message: MODAL_PAYLOAD_UPLOAD_TIMEOUT_MESSAGE,
|
|
382
|
+
retryable: true,
|
|
383
|
+
cause,
|
|
384
|
+
};
|
|
385
|
+
}
|
|
374
386
|
if (RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_RE.test(rawCause)) {
|
|
375
387
|
return {
|
|
376
388
|
code: 'RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE',
|
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts
CHANGED
|
@@ -417,6 +417,14 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
417
417
|
attempt,
|
|
418
418
|
error: message,
|
|
419
419
|
});
|
|
420
|
+
const immediateFallbackReason =
|
|
421
|
+
resolveDaytonaSandboxAcquisitionUnavailableReason([message]);
|
|
422
|
+
if (immediateFallbackReason === 'daytona_total_cpu_limit_exceeded') {
|
|
423
|
+
throw new DaytonaSandboxAcquisitionUnavailableError(
|
|
424
|
+
immediateFallbackReason,
|
|
425
|
+
`Daytona sandbox create rejected by the organization CPU limit on provider attempt ${attempt}: ${message}`,
|
|
426
|
+
);
|
|
427
|
+
}
|
|
420
428
|
if (isDaytonaSandboxStartTimeout(message)) {
|
|
421
429
|
const deleted = await reconcileAndDeleteTimedOutDaytonaSandbox({
|
|
422
430
|
daytona: input.daytona,
|
|
@@ -11,10 +11,14 @@ import type {
|
|
|
11
11
|
import { daytonaPlayRunnerBackend } from './daytona';
|
|
12
12
|
import { DaytonaSandboxAcquisitionUnavailableError } from './daytona-lifecycle';
|
|
13
13
|
import { modalPlayRunnerBackend } from './modal';
|
|
14
|
+
import { isPlayRunRecoveryError } from '@shared_libs/play-runtime/play-run-recovery-policy';
|
|
14
15
|
import {
|
|
15
16
|
canTransitionRuntimeSandboxPlacement,
|
|
17
|
+
isRuntimeSandboxCanaryProduction,
|
|
16
18
|
resolveRuntimeSandboxPlacementPolicy,
|
|
17
19
|
RUNTIME_SANDBOX_PLACEMENT_POLICIES,
|
|
20
|
+
selectRuntimeSandboxCanary,
|
|
21
|
+
type RuntimeSandboxCanarySelection,
|
|
18
22
|
type RuntimeSandboxPlacementPolicyId,
|
|
19
23
|
type RuntimeSandboxPlacementFailureReason,
|
|
20
24
|
type RuntimeSandboxPlacementPolicy,
|
|
@@ -41,6 +45,54 @@ type RuntimeSandboxProviderAdapter = {
|
|
|
41
45
|
) => RuntimeSandboxPlacementFailureReason | null;
|
|
42
46
|
};
|
|
43
47
|
|
|
48
|
+
function selectionForRun(
|
|
49
|
+
policy: RuntimeSandboxPlacementPolicy,
|
|
50
|
+
runId?: string | null,
|
|
51
|
+
runtimeSchedulerSchema?: string | null,
|
|
52
|
+
): RuntimeSandboxCanarySelection {
|
|
53
|
+
return selectRuntimeSandboxCanary({
|
|
54
|
+
policy,
|
|
55
|
+
runId,
|
|
56
|
+
production: isRuntimeSandboxCanaryProduction({
|
|
57
|
+
nodeEnv: process.env.NODE_ENV,
|
|
58
|
+
runtimeSchedulerSchema,
|
|
59
|
+
}),
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function logCanaryOutcome(input: {
|
|
64
|
+
config: PlayRunnerExecutionConfig;
|
|
65
|
+
policy: RuntimeSandboxPlacementPolicy;
|
|
66
|
+
selection: RuntimeSandboxCanarySelection;
|
|
67
|
+
outcome: string;
|
|
68
|
+
error?: unknown;
|
|
69
|
+
}): void {
|
|
70
|
+
const recoveryReason = isPlayRunRecoveryError(input.error)
|
|
71
|
+
? input.error.decision.reason
|
|
72
|
+
: null;
|
|
73
|
+
console.warn(
|
|
74
|
+
'[play-runner.sandbox_provider_canary]',
|
|
75
|
+
JSON.stringify({
|
|
76
|
+
runId: input.config.context.runId ?? null,
|
|
77
|
+
workflowId: input.config.context.workflowId ?? null,
|
|
78
|
+
runAttempt: input.config.context.runAttempt ?? null,
|
|
79
|
+
policyId: input.policy.id,
|
|
80
|
+
provider: input.selection.canary,
|
|
81
|
+
bucket: input.selection.bucket,
|
|
82
|
+
basisPoints: input.selection.basisPoints,
|
|
83
|
+
runtimeEnvironment: 'production',
|
|
84
|
+
outcome: input.outcome,
|
|
85
|
+
phase: 'startup',
|
|
86
|
+
// The Modal Adapter also performs scheduler-owned fencing, staging, and
|
|
87
|
+
// readiness work. Page only for its existing typed provider decisions;
|
|
88
|
+
// an unclassified setup failure remains observable without blaming Modal.
|
|
89
|
+
providerHealthFailure: recoveryReason?.startsWith('modal_') ?? false,
|
|
90
|
+
errorName: input.error instanceof Error ? input.error.name : null,
|
|
91
|
+
recoveryReason,
|
|
92
|
+
}),
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
|
|
44
96
|
function classifyDaytonaAcquisitionFailure(
|
|
45
97
|
error: unknown,
|
|
46
98
|
): RuntimeSandboxPlacementFailureReason | null {
|
|
@@ -70,18 +122,23 @@ export function createRuntimeSandboxPlacementBackend(input: {
|
|
|
70
122
|
Partial<Record<RuntimeSandboxProvider, RuntimeSandboxProviderAdapter>>
|
|
71
123
|
>;
|
|
72
124
|
}): PlayRunnerBackend {
|
|
73
|
-
const primaryProvider = input.policy.providers[0];
|
|
74
|
-
const primaryAdapter = input.adapters[primaryProvider];
|
|
75
|
-
if (!primaryAdapter) {
|
|
76
|
-
throw new Error(
|
|
77
|
-
`Runtime sandbox placement policy ${input.policy.id} has no Adapter for ${primaryProvider}.`,
|
|
78
|
-
);
|
|
79
|
-
}
|
|
80
125
|
return {
|
|
81
126
|
async prepare(
|
|
82
127
|
prepareInput: PlayRunnerPrepareInput,
|
|
83
128
|
callbacks?: PlayRunnerCallbacks,
|
|
84
129
|
): Promise<PlayRunnerPreparedExecution> {
|
|
130
|
+
const selection = selectionForRun(
|
|
131
|
+
input.policy,
|
|
132
|
+
prepareInput.context.runId,
|
|
133
|
+
prepareInput.context.runtimeSchedulerSchema,
|
|
134
|
+
);
|
|
135
|
+
const primaryProvider = selection.canary ?? input.policy.providers[0];
|
|
136
|
+
const primaryAdapter = input.adapters[primaryProvider];
|
|
137
|
+
if (!primaryAdapter) {
|
|
138
|
+
throw new Error(
|
|
139
|
+
`Runtime sandbox placement policy ${input.policy.id} has no Adapter for ${primaryProvider}.`,
|
|
140
|
+
);
|
|
141
|
+
}
|
|
85
142
|
const providerPrepared = await primaryAdapter.backend.prepare?.(
|
|
86
143
|
prepareInput,
|
|
87
144
|
callbacks,
|
|
@@ -105,7 +162,15 @@ export function createRuntimeSandboxPlacementBackend(input: {
|
|
|
105
162
|
isPrepared(prepared) && prepared.policyId === input.policy.id
|
|
106
163
|
? prepared
|
|
107
164
|
: null;
|
|
108
|
-
|
|
165
|
+
const selection = selectionForRun(
|
|
166
|
+
input.policy,
|
|
167
|
+
config.context.runId,
|
|
168
|
+
config.context.runtimeSchedulerSchema,
|
|
169
|
+
);
|
|
170
|
+
const providers = selection.canary
|
|
171
|
+
? ([selection.canary] as const)
|
|
172
|
+
: input.policy.providers;
|
|
173
|
+
for (const [index, provider] of providers.entries()) {
|
|
109
174
|
const adapter = input.adapters[provider];
|
|
110
175
|
if (!adapter) {
|
|
111
176
|
throw new Error(
|
|
@@ -113,15 +178,34 @@ export function createRuntimeSandboxPlacementBackend(input: {
|
|
|
113
178
|
);
|
|
114
179
|
}
|
|
115
180
|
try {
|
|
116
|
-
|
|
181
|
+
const result = await adapter.backend.execute(
|
|
117
182
|
config,
|
|
118
183
|
callbacks,
|
|
119
184
|
index === 0 && policyPrepared?.provider === provider
|
|
120
185
|
? policyPrepared.providerPrepared
|
|
121
186
|
: undefined,
|
|
122
187
|
);
|
|
188
|
+
if (selection.canary) {
|
|
189
|
+
logCanaryOutcome({
|
|
190
|
+
config,
|
|
191
|
+
policy: input.policy,
|
|
192
|
+
selection,
|
|
193
|
+
outcome: result.status,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
return result;
|
|
123
197
|
} catch (error) {
|
|
124
|
-
|
|
198
|
+
if (selection.canary) {
|
|
199
|
+
logCanaryOutcome({
|
|
200
|
+
config,
|
|
201
|
+
policy: input.policy,
|
|
202
|
+
selection,
|
|
203
|
+
outcome: 'error',
|
|
204
|
+
error,
|
|
205
|
+
});
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
const nextProvider = providers[index + 1];
|
|
125
209
|
const reason = adapter.classifyAcquisitionFailure(error);
|
|
126
210
|
if (
|
|
127
211
|
!nextProvider ||
|
|
@@ -199,6 +283,11 @@ export function createDaytonaModalFallbackBackend(
|
|
|
199
283
|
export const daytonaModalFallbackPlayRunnerBackend =
|
|
200
284
|
createDaytonaModalFallbackBackend();
|
|
201
285
|
|
|
286
|
+
export const daytonaModalCanaryPlayRunnerBackend =
|
|
287
|
+
createRuntimeSandboxPlacementBackendForPolicy(
|
|
288
|
+
RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaModalCanaryV1,
|
|
289
|
+
);
|
|
290
|
+
|
|
202
291
|
export const daytonaOnlyPlayRunnerBackend =
|
|
203
292
|
createRuntimeSandboxPlacementBackendForPolicy(
|
|
204
293
|
RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaOnlyV1,
|
|
@@ -216,6 +305,8 @@ const DEFAULT_POLICY_BACKENDS: Readonly<
|
|
|
216
305
|
daytonaOnlyPlayRunnerBackend,
|
|
217
306
|
[RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaThenModalV1]:
|
|
218
307
|
daytonaModalFallbackPlayRunnerBackend,
|
|
308
|
+
[RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaModalCanaryV1]:
|
|
309
|
+
daytonaModalCanaryPlayRunnerBackend,
|
|
219
310
|
[RUNTIME_SANDBOX_PLACEMENT_POLICIES.modalOnlyV1]: modalOnlyPlayRunnerBackend,
|
|
220
311
|
};
|
|
221
312
|
|