deepline 0.3.44 → 0.3.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +2 -3
- package/dist/bundling-sources/sdk/src/errors.ts +2 -2
- package/dist/bundling-sources/sdk/src/http.ts +1 -1
- package/dist/bundling-sources/sdk/src/play.ts +7 -10
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +3 -3
- package/dist/bundling-sources/sdk/src/release.ts +3 -3
- package/dist/bundling-sources/sdk/src/version.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-data-plane/contracts.ts +25 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/sheet-contract.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +19 -19
- package/dist/bundling-sources/shared_libs/play-runtime/backend.ts +5 -6
- package/dist/bundling-sources/shared_libs/play-runtime/cell-staleness.ts +1 -66
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1551 -276
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +14 -3
- package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +4 -3
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +3 -4
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/email-status.ts +1 -275
- package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/extractor-targets.ts +1 -106
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +1 -2
- package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +13 -4
- package/dist/bundling-sources/shared_libs/play-runtime/live-events.ts +1 -5
- package/dist/bundling-sources/shared_libs/play-runtime/map-row-identity.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/map-row-outcome.ts +2 -1
- package/dist/bundling-sources/shared_libs/play-runtime/modal-runtime-config.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/projection.ts +5 -5
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +4 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-modal-fallback.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +8 -8
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +11 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +26 -8
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +7 -7
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +23 -11
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts +13 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runner-events.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +58 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +6 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-constraints.ts +1 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-env.ts +55 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +418 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/index.ts +452 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +304 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/testkit.ts +83 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runtime-limits.ts +1 -109
- package/dist/bundling-sources/shared_libs/play-runtime/step-program-dataset-builder.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/always-fresh-adapter.ts +50 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/contract.ts +135 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +291 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts +335 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +904 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +522 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/testkit.ts +165 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts +6 -5
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result-types.ts +1 -206
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +10 -7
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +8 -3
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +4 -4
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +3 -6
- package/dist/bundling-sources/shared_libs/plays/cell-staleness.ts +66 -0
- package/dist/bundling-sources/shared_libs/plays/compiler-manifest.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +2 -2
- package/dist/bundling-sources/shared_libs/plays/core.ts +466 -0
- package/dist/bundling-sources/shared_libs/plays/email-status.ts +287 -0
- package/dist/bundling-sources/shared_libs/plays/extractor-targets.ts +106 -0
- package/dist/bundling-sources/shared_libs/plays/row-identity.ts +1 -3
- package/dist/bundling-sources/shared_libs/plays/runtime-constraints.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/runtime-validation.ts +4 -2
- package/dist/bundling-sources/shared_libs/plays/sandbox-runtime-limits.ts +109 -0
- package/dist/bundling-sources/shared_libs/plays/tool-execution-error.ts +624 -0
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +206 -0
- package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +1 -1
- package/dist/cli/index.js +174 -168
- package/dist/cli/index.mjs +182 -169
- package/dist/{compiler-manifest-DwYe2C2S.d.mts → compiler-manifest-BuoqasqI.d.mts} +585 -540
- package/dist/{compiler-manifest-DwYe2C2S.d.ts → compiler-manifest-BuoqasqI.d.ts} +585 -540
- package/dist/index.d.mts +9 -2
- package/dist/index.d.ts +9 -2
- package/dist/index.js +47 -40
- package/dist/index.mjs +47 -40
- package/dist/install-integrity.json +24 -3
- package/dist/plays/bundle-play-file.d.mts +4 -61
- package/dist/plays/bundle-play-file.d.ts +4 -61
- package/dist/plays/bundle-play-file.mjs +157 -155
- package/package.json +9 -6
- /package/dist/bundling-sources/shared_libs/{play-runtime → plays}/tool-response-contract.ts +0 -0
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts
CHANGED
|
@@ -2,13 +2,13 @@ import { type Daytona } from '@daytonaio/sdk';
|
|
|
2
2
|
import type {
|
|
3
3
|
PlayRunnerExecutionConfig,
|
|
4
4
|
PlayRunnerResult,
|
|
5
|
-
} from '
|
|
6
|
-
import { isIsolatedRuntimeSchedulerSchema } from '
|
|
7
|
-
import { PLAY_RUNNER_TIMEOUT_SECONDS } from '
|
|
5
|
+
} from '../../protocol';
|
|
6
|
+
import { isIsolatedRuntimeSchedulerSchema } from '../../runtime-scheduler-topology';
|
|
7
|
+
import { PLAY_RUNNER_TIMEOUT_SECONDS } from '../../runtime-constants';
|
|
8
8
|
import {
|
|
9
9
|
STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
|
|
10
10
|
validatePlaySandboxRuntimeLimits,
|
|
11
|
-
} from '
|
|
11
|
+
} from '../../../plays/sandbox-runtime-limits';
|
|
12
12
|
import type { PlayRunnerRuntimeLifecycleEvent } from '../types';
|
|
13
13
|
import { DAYTONA_PLAY_RUNNER_LABEL_SOURCE } from './daytona-labels';
|
|
14
14
|
|
|
@@ -7,11 +7,11 @@ import type {
|
|
|
7
7
|
import type {
|
|
8
8
|
PlayRunnerExecutionConfig,
|
|
9
9
|
PlayRunnerResult,
|
|
10
|
-
} from '
|
|
10
|
+
} from '../../protocol';
|
|
11
11
|
import { daytonaPlayRunnerBackend } from './daytona';
|
|
12
12
|
import { DaytonaSandboxAcquisitionUnavailableError } from './daytona-lifecycle';
|
|
13
13
|
import { modalPlayRunnerBackend } from './modal';
|
|
14
|
-
import { isPlayRunRecoveryError } from '
|
|
14
|
+
import { isPlayRunRecoveryError } from '../../play-run-recovery-policy';
|
|
15
15
|
import {
|
|
16
16
|
canTransitionRuntimeSandboxPlacement,
|
|
17
17
|
isRuntimeSandboxCanaryProduction,
|
|
@@ -23,7 +23,7 @@ import {
|
|
|
23
23
|
type RuntimeSandboxPlacementFailureReason,
|
|
24
24
|
type RuntimeSandboxPlacementPolicy,
|
|
25
25
|
type RuntimeSandboxProvider,
|
|
26
|
-
} from '
|
|
26
|
+
} from '../../runtime-sandbox-placement-policy';
|
|
27
27
|
|
|
28
28
|
type DaytonaFallbackPreparedExecution = PlayRunnerPreparedExecution & {
|
|
29
29
|
kind: 'runtime_sandbox_placement';
|
|
@@ -4,33 +4,33 @@ import { gzipSync } from 'node:zlib';
|
|
|
4
4
|
import {
|
|
5
5
|
RUNNER_TERMINAL_CHECKPOINT_DISPOSITION,
|
|
6
6
|
type PlayRunnerExecutionConfig,
|
|
7
|
-
} from '
|
|
7
|
+
} from '../../protocol';
|
|
8
8
|
import {
|
|
9
9
|
PLAY_RUNTIME_CONTRACT,
|
|
10
10
|
PLAY_RUNTIME_CONTRACT_HEADER,
|
|
11
|
-
} from '
|
|
11
|
+
} from '../../runtime-contract';
|
|
12
12
|
import {
|
|
13
13
|
PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
14
14
|
PLAY_RUNNER_STARTUP_GRACE_SECONDS,
|
|
15
15
|
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
|
|
16
|
-
} from '
|
|
16
|
+
} from '../../runtime-constants';
|
|
17
17
|
import {
|
|
18
18
|
RUNTIME_RELIABILITY_ENV,
|
|
19
19
|
RUNTIME_RELIABILITY_SEV_OVERRIDE_ENV,
|
|
20
|
-
} from '
|
|
20
|
+
} from '../../runtime-reliability-policy';
|
|
21
21
|
import {
|
|
22
22
|
STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
|
|
23
23
|
validatePlaySandboxRuntimeLimits,
|
|
24
|
-
} from '
|
|
24
|
+
} from '../../../plays/sandbox-runtime-limits';
|
|
25
25
|
import {
|
|
26
26
|
RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES,
|
|
27
27
|
RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES,
|
|
28
|
-
} from '
|
|
28
|
+
} from '../../output-size-limits';
|
|
29
29
|
import {
|
|
30
30
|
RUNTIME_SANDBOX_KILLED_MESSAGE,
|
|
31
31
|
RUNTIME_SANDBOX_OOM_MESSAGE,
|
|
32
|
-
} from '
|
|
33
|
-
import { compactPlayArtifactForRuntimeTransport } from '
|
|
32
|
+
} from '../../run-failure';
|
|
33
|
+
import { compactPlayArtifactForRuntimeTransport } from '../../../plays/artifact-transport';
|
|
34
34
|
import type {
|
|
35
35
|
DaytonaExecutionContext,
|
|
36
36
|
DaytonaSandbox,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { DaytonaSandbox } from './daytona-lifecycle';
|
|
2
|
-
import { RUNTIME_RELIABILITY_POLICY } from '
|
|
2
|
+
import { RUNTIME_RELIABILITY_POLICY } from '../../runtime-reliability-policy';
|
|
3
3
|
|
|
4
4
|
export type DetachedDaytonaStartupDiagnostic = {
|
|
5
5
|
schedulerReadinessObserved: boolean;
|
|
@@ -358,6 +358,14 @@ export function prepareDetachedDaytonaRunner(input: {
|
|
|
358
358
|
sessionId: string;
|
|
359
359
|
readiness: DetachedRunnerReadinessPort;
|
|
360
360
|
cancellation?: Promise<never>;
|
|
361
|
+
/** Invoked immediately after Daytona accepts the detached command. */
|
|
362
|
+
onCommandAccepted?: (
|
|
363
|
+
state: Extract<DetachedDaytonaRunnerState, { phase: 'command_accepted' }>,
|
|
364
|
+
) => Promise<void> | void;
|
|
365
|
+
/** Invoked after scheduler-owned liveness is confirmed. */
|
|
366
|
+
onReady?: (
|
|
367
|
+
state: Extract<DetachedDaytonaRunnerState, { phase: 'ready' }>,
|
|
368
|
+
) => Promise<void> | void;
|
|
361
369
|
}): DetachedDaytonaRunnerSupervisor {
|
|
362
370
|
const cancellation = input.cancellation ?? new Promise<never>(() => {});
|
|
363
371
|
const sessionCreation = input.sandbox.process
|
|
@@ -409,6 +417,7 @@ export function prepareDetachedDaytonaRunner(input: {
|
|
|
409
417
|
cmdId,
|
|
410
418
|
baselineHeartbeatAt: baseline.heartbeatAt,
|
|
411
419
|
};
|
|
420
|
+
await input.onCommandAccepted?.(state);
|
|
412
421
|
await Promise.race([
|
|
413
422
|
confirmDetachedDaytonaRunnerReady({
|
|
414
423
|
sandbox: input.sandbox,
|
|
@@ -422,6 +431,7 @@ export function prepareDetachedDaytonaRunner(input: {
|
|
|
422
431
|
cancellation,
|
|
423
432
|
]);
|
|
424
433
|
state = { ...state, phase: 'ready' };
|
|
434
|
+
await input.onReady?.(state);
|
|
425
435
|
return state;
|
|
426
436
|
},
|
|
427
437
|
};
|
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
import {
|
|
9
9
|
isRuntimeSandboxCapacityLimitError,
|
|
10
10
|
RuntimeResourceFenceLostError,
|
|
11
|
+
SandboxRunnerPreCodeDeferralError,
|
|
11
12
|
} from '../types';
|
|
12
13
|
import { buildPlayRunnerBundle } from '../bundle';
|
|
13
14
|
import { findPlayRunnerResult, parsePlayRunnerEvents } from '../runner-events';
|
|
@@ -15,15 +16,15 @@ import type {
|
|
|
15
16
|
PlayRunnerExecutionConfig,
|
|
16
17
|
PlayRunnerRuntimeTiming,
|
|
17
18
|
PlayRunnerResult,
|
|
18
|
-
} from '
|
|
19
|
+
} from '../../protocol';
|
|
19
20
|
import {
|
|
20
21
|
PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
21
22
|
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
|
|
22
|
-
} from '
|
|
23
|
+
} from '../../runtime-constants';
|
|
23
24
|
import {
|
|
24
25
|
loadDaytonaRequiredConfig,
|
|
25
26
|
loadDaytonaRunnerPathsConfig,
|
|
26
|
-
} from '
|
|
27
|
+
} from '../../daytona-runtime-config';
|
|
27
28
|
import {
|
|
28
29
|
DAYTONA_CANCELLED_ERROR,
|
|
29
30
|
DaytonaSandboxAcquisitionUnavailableError,
|
|
@@ -41,7 +42,7 @@ import {
|
|
|
41
42
|
DaytonaRunnerInitializationError,
|
|
42
43
|
prepareDetachedDaytonaRunner,
|
|
43
44
|
} from './daytona-session-execution';
|
|
44
|
-
import { RUNTIME_RELIABILITY_POLICY } from '
|
|
45
|
+
import { RUNTIME_RELIABILITY_POLICY } from '../../runtime-reliability-policy';
|
|
45
46
|
|
|
46
47
|
const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS =
|
|
47
48
|
RUNTIME_RELIABILITY_POLICY.sandbox.daytonaCommandRecoveryTimeoutMs;
|
|
@@ -1197,11 +1198,29 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1197
1198
|
// Session allocation is independent of payload staging. Start it
|
|
1198
1199
|
// now so Daytona control-plane latency overlaps compression/upload.
|
|
1199
1200
|
const sessionId = `deepline-play-${randomUUID()}`;
|
|
1201
|
+
const runnerAttempt = Math.max(
|
|
1202
|
+
0,
|
|
1203
|
+
Math.floor(config.context.runAttempt ?? 0),
|
|
1204
|
+
);
|
|
1200
1205
|
const detachedRunner = prepareDetachedDaytonaRunner({
|
|
1201
1206
|
sandbox,
|
|
1202
1207
|
sessionId,
|
|
1203
1208
|
readiness: readRunnerReadiness,
|
|
1204
1209
|
cancellation: cancellationPromise,
|
|
1210
|
+
onCommandAccepted: async () => {
|
|
1211
|
+
await callbacks?.onRunnerCommandAccepted?.({
|
|
1212
|
+
provider: 'daytona',
|
|
1213
|
+
sandboxId: sandbox.id,
|
|
1214
|
+
});
|
|
1215
|
+
},
|
|
1216
|
+
onReady: async (ready) => {
|
|
1217
|
+
await callbacks?.onDetachedRunnerReady?.({
|
|
1218
|
+
provider: 'daytona',
|
|
1219
|
+
sandboxId: sandbox.id,
|
|
1220
|
+
boundaryId: `detached-runner:${push.runId}:${runnerAttempt}`,
|
|
1221
|
+
baselineHeartbeatAt: ready.baselineHeartbeatAt,
|
|
1222
|
+
});
|
|
1223
|
+
},
|
|
1205
1224
|
});
|
|
1206
1225
|
|
|
1207
1226
|
const uploadStartedAt = Date.now();
|
|
@@ -1310,10 +1329,6 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1310
1329
|
// classification below decide fresh-sandbox retry vs loud failure.
|
|
1311
1330
|
throw error;
|
|
1312
1331
|
}
|
|
1313
|
-
const runnerAttempt = Math.max(
|
|
1314
|
-
0,
|
|
1315
|
-
Math.floor(config.context.runAttempt ?? 0),
|
|
1316
|
-
);
|
|
1317
1332
|
emitDaytonaStage(callbacks, config.context, 'execute:detached', {
|
|
1318
1333
|
sandboxId: sandbox.id,
|
|
1319
1334
|
sessionId: start.sessionId,
|
|
@@ -1435,6 +1450,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1435
1450
|
if (isRuntimeSandboxCapacityLimitError(error)) {
|
|
1436
1451
|
throw error;
|
|
1437
1452
|
}
|
|
1453
|
+
if (error instanceof SandboxRunnerPreCodeDeferralError) {
|
|
1454
|
+
throw error;
|
|
1455
|
+
}
|
|
1438
1456
|
if (error instanceof DaytonaSandboxAcquisitionUnavailableError) {
|
|
1439
1457
|
throw error;
|
|
1440
1458
|
}
|
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts
CHANGED
|
@@ -6,7 +6,7 @@ import type { SpawnOptionsWithoutStdio } from 'node:child_process';
|
|
|
6
6
|
import {
|
|
7
7
|
setSpanAttributes,
|
|
8
8
|
withActiveSpan,
|
|
9
|
-
} from '
|
|
9
|
+
} from '../../../observability/tracing';
|
|
10
10
|
import type { PlayRunnerBackend } from '../types';
|
|
11
11
|
import { buildPlayRunnerBundle } from '../bundle';
|
|
12
12
|
import {
|
|
@@ -14,25 +14,25 @@ import {
|
|
|
14
14
|
parsePlayRunnerEventLine,
|
|
15
15
|
parsePlayRunnerEvents,
|
|
16
16
|
} from '../runner-events';
|
|
17
|
-
import type { PlayRunnerExecutionConfig } from '
|
|
18
|
-
import { resolveDaytonaSandboxComputeItem } from '
|
|
17
|
+
import type { PlayRunnerExecutionConfig } from '../../protocol';
|
|
18
|
+
import { resolveDaytonaSandboxComputeItem } from '../../worker-api-types';
|
|
19
19
|
import {
|
|
20
20
|
recordComputeBillingItemViaAppRuntime,
|
|
21
21
|
upsertComputeBillingSessionViaAppRuntime,
|
|
22
22
|
type WorkerRuntimeApiContext,
|
|
23
|
-
} from '
|
|
23
|
+
} from '../../app-runtime-api';
|
|
24
24
|
import {
|
|
25
25
|
PLAY_RUNNER_STARTUP_GRACE_SECONDS,
|
|
26
26
|
PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
27
|
-
} from '
|
|
27
|
+
} from '../../runtime-constants';
|
|
28
28
|
import {
|
|
29
29
|
STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
|
|
30
30
|
validatePlaySandboxRuntimeLimits,
|
|
31
|
-
} from '
|
|
31
|
+
} from '../../../plays/sandbox-runtime-limits';
|
|
32
32
|
import {
|
|
33
33
|
runtimeLimitExceededCause,
|
|
34
34
|
runtimeLimitExceededFailure,
|
|
35
|
-
} from '
|
|
35
|
+
} from '../../run-failure';
|
|
36
36
|
import {
|
|
37
37
|
buildDaytonaRuntimeWatchdogSource,
|
|
38
38
|
DAYTONA_RUNTIME_WATCHDOG_EXIT_CODE,
|
|
@@ -3,24 +3,25 @@ import type { PlayRunnerBackend, PlayRunnerCallbacks } from '../types';
|
|
|
3
3
|
import {
|
|
4
4
|
isRuntimeSandboxCapacityLimitError,
|
|
5
5
|
RuntimeResourceFenceLostError,
|
|
6
|
+
SandboxRunnerPreCodeDeferralError,
|
|
6
7
|
} from '../types';
|
|
7
8
|
import { buildPlayRunnerBundle } from '../bundle';
|
|
8
9
|
import type {
|
|
9
10
|
PlayRunnerExecutionConfig,
|
|
10
11
|
PlayRunnerResult,
|
|
11
12
|
PlayRunnerRuntimeTiming,
|
|
12
|
-
} from '
|
|
13
|
+
} from '../../protocol';
|
|
13
14
|
import {
|
|
14
15
|
PLAY_RUNNER_STARTUP_GRACE_SECONDS,
|
|
15
16
|
PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
16
|
-
} from '
|
|
17
|
+
} from '../../runtime-constants';
|
|
17
18
|
import {
|
|
18
19
|
MODAL_SANDBOX_CPU_CORES,
|
|
19
20
|
MODAL_SANDBOX_MEMORY_MIB,
|
|
20
21
|
loadModalClientConfig,
|
|
21
22
|
loadModalRequiredConfig,
|
|
22
|
-
} from '
|
|
23
|
-
import type { PlaySandboxRuntimeLimits } from '
|
|
23
|
+
} from '../../modal-runtime-config';
|
|
24
|
+
import type { PlaySandboxRuntimeLimits } from '../../../plays/sandbox-runtime-limits';
|
|
24
25
|
import { validateDaytonaExecutionContext } from './daytona-lifecycle';
|
|
25
26
|
import { stageRunnerPayload } from './daytona-payload-transport';
|
|
26
27
|
import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
|
|
@@ -28,9 +29,9 @@ import {
|
|
|
28
29
|
isPlayRunRecoveryError,
|
|
29
30
|
recoveryForOpenModalCircuit,
|
|
30
31
|
recoveryForModalSandboxCreateFailure,
|
|
31
|
-
} from '
|
|
32
|
-
import { shouldInjectModalSandboxCreateResourceExhausted } from '
|
|
33
|
-
import { RUNTIME_RELIABILITY_POLICY } from '
|
|
32
|
+
} from '../../play-run-recovery-policy';
|
|
33
|
+
import { shouldInjectModalSandboxCreateResourceExhausted } from '../../test-runtime-seams';
|
|
34
|
+
import { RUNTIME_RELIABILITY_POLICY } from '../../runtime-reliability-policy';
|
|
34
35
|
|
|
35
36
|
const MODAL_RUNNER_READY_TIMEOUT_MS =
|
|
36
37
|
RUNTIME_RELIABILITY_POLICY.sandbox.runnerReadyTimeoutMs;
|
|
@@ -618,6 +619,10 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
618
619
|
cancellationPromise,
|
|
619
620
|
]);
|
|
620
621
|
runtimeTiming.modalExecuteMs = Date.now() - executeStartedAt;
|
|
622
|
+
await callbacks?.onRunnerCommandAccepted?.({
|
|
623
|
+
provider: 'modal',
|
|
624
|
+
sandboxId: sandbox.sandboxId,
|
|
625
|
+
});
|
|
621
626
|
// A readiness observation after the command is accepted is ambiguous:
|
|
622
627
|
// the detached runner may already have received its durable park and
|
|
623
628
|
// started customer code while this worker cannot read the scheduler.
|
|
@@ -627,16 +632,22 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
627
632
|
baselineHeartbeatAt,
|
|
628
633
|
cancellation: cancellationPromise,
|
|
629
634
|
});
|
|
635
|
+
const runnerAttempt = Math.max(
|
|
636
|
+
0,
|
|
637
|
+
Math.floor(config.context.runAttempt ?? 0),
|
|
638
|
+
);
|
|
639
|
+
await callbacks?.onDetachedRunnerReady?.({
|
|
640
|
+
provider: 'modal',
|
|
641
|
+
sandboxId: sandbox.sandboxId,
|
|
642
|
+
boundaryId: `detached-runner:${push.runId}:${runnerAttempt}`,
|
|
643
|
+
baselineHeartbeatAt,
|
|
644
|
+
});
|
|
630
645
|
if (callbacks?.cancellationSignal?.aborted) {
|
|
631
646
|
throw new Error('Modal play runner cancelled');
|
|
632
647
|
}
|
|
633
648
|
|
|
634
649
|
detached = true;
|
|
635
650
|
sandbox.detach();
|
|
636
|
-
const runnerAttempt = Math.max(
|
|
637
|
-
0,
|
|
638
|
-
Math.floor(config.context.runAttempt ?? 0),
|
|
639
|
-
);
|
|
640
651
|
emitModalStage(config.context, 'execute:detached', {
|
|
641
652
|
sandboxId: sandbox.sandboxId,
|
|
642
653
|
runnerAttempt,
|
|
@@ -716,6 +727,7 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
716
727
|
if (
|
|
717
728
|
error === runtimeResourceRegistrationError ||
|
|
718
729
|
error instanceof RuntimeResourceFenceLostError ||
|
|
730
|
+
error instanceof SandboxRunnerPreCodeDeferralError ||
|
|
719
731
|
isRuntimeSandboxCapacityLimitError(error) ||
|
|
720
732
|
isPlayRunRecoveryError(error)
|
|
721
733
|
) {
|
|
@@ -5,7 +5,7 @@ import { createRequire } from 'node:module';
|
|
|
5
5
|
import {
|
|
6
6
|
setSpanAttributes,
|
|
7
7
|
withActiveSpan,
|
|
8
|
-
} from '
|
|
8
|
+
} from '../../observability/tracing';
|
|
9
9
|
|
|
10
10
|
const runtimeRequire = createRequire(import.meta.url);
|
|
11
11
|
const esbuild = runtimeRequire('esbuild') as {
|
|
@@ -71,6 +71,12 @@ async function listFiles(dir: string): Promise<string[]> {
|
|
|
71
71
|
const entries = await readdir(dir, { withFileTypes: true });
|
|
72
72
|
const values = await Promise.all(
|
|
73
73
|
entries.map(async (entry) => {
|
|
74
|
+
if (
|
|
75
|
+
entry.isDirectory() &&
|
|
76
|
+
['.turbo', 'dist', 'node_modules'].includes(entry.name)
|
|
77
|
+
) {
|
|
78
|
+
return [];
|
|
79
|
+
}
|
|
74
80
|
const fullPath = join(dir, entry.name);
|
|
75
81
|
if (entry.isDirectory()) {
|
|
76
82
|
return await listFiles(fullPath);
|
|
@@ -112,7 +118,7 @@ function createRunnerBoundaryPlugin(projectRoot: string) {
|
|
|
112
118
|
|
|
113
119
|
function rejectionForImport(args: EsbuildOnResolveArgs): string | null {
|
|
114
120
|
if (args.path.startsWith('@/') || args.path.startsWith('@convex/')) {
|
|
115
|
-
return `play-runner bundle cannot import ${args.path}; move shared runtime code to
|
|
121
|
+
return `play-runner bundle cannot import ${args.path}; move shared runtime code to a package or call the app over the runtime API`;
|
|
116
122
|
}
|
|
117
123
|
|
|
118
124
|
if (!args.path.startsWith('.') || !args.importer) {
|
|
@@ -195,6 +201,11 @@ export async function buildPlayRunnerBundle(): Promise<string> {
|
|
|
195
201
|
|
|
196
202
|
const sourceRoots = [
|
|
197
203
|
resolve(process.cwd(), 'apps', 'play-runner', 'src'),
|
|
204
|
+
resolve(process.cwd(), 'packages', 'observability'),
|
|
205
|
+
resolve(process.cwd(), 'packages', 'play-data-plane'),
|
|
206
|
+
resolve(process.cwd(), 'packages', 'play-runtime'),
|
|
207
|
+
resolve(process.cwd(), 'packages', 'plays'),
|
|
208
|
+
resolve(process.cwd(), 'packages', 'security'),
|
|
198
209
|
resolve(process.cwd(), 'shared_libs'),
|
|
199
210
|
];
|
|
200
211
|
const sourceHash = await hashRunnerSources(sourceRoots);
|
|
@@ -13,13 +13,13 @@ import {
|
|
|
13
13
|
PLAY_RUNTIME_BACKENDS,
|
|
14
14
|
normalizePlayRuntimeBackend,
|
|
15
15
|
type PlayRuntimeBackendId,
|
|
16
|
-
} from '
|
|
16
|
+
} from '../backend';
|
|
17
17
|
import {
|
|
18
18
|
isRuntimeSandboxCanaryProduction,
|
|
19
19
|
RUNTIME_SANDBOX_PLACEMENT_POLICIES,
|
|
20
20
|
runtimeSandboxPlacementPolicyForBackend,
|
|
21
21
|
type RuntimeSandboxPlacementPolicyId,
|
|
22
|
-
} from '
|
|
22
|
+
} from '../runtime-sandbox-placement-policy';
|
|
23
23
|
|
|
24
24
|
export function resolveRuntimeSandboxPlacementBackend(
|
|
25
25
|
policyId: RuntimeSandboxPlacementPolicyId | string,
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
deleteModalSandboxById,
|
|
7
7
|
readDetachedModalRuntimeCompletion,
|
|
8
8
|
} from './backends/modal';
|
|
9
|
-
import type { DetachedRunnerSuspension } from '
|
|
9
|
+
import type { DetachedRunnerSuspension } from '../suspension';
|
|
10
10
|
|
|
11
11
|
export type RuntimeSandboxProviderId = 'daytona' | 'modal';
|
|
12
12
|
|
|
@@ -2,12 +2,12 @@ import type {
|
|
|
2
2
|
PlayCheckpoint,
|
|
3
3
|
PlayExecutionEvent,
|
|
4
4
|
PlayRowUpdate,
|
|
5
|
-
} from '
|
|
5
|
+
} from '../ctx-types';
|
|
6
6
|
import type {
|
|
7
7
|
PlayRunnerExecutionConfig,
|
|
8
8
|
PlayRunnerLogEvent,
|
|
9
9
|
PlayRunnerResult,
|
|
10
|
-
} from '
|
|
10
|
+
} from '../protocol';
|
|
11
11
|
|
|
12
12
|
export type PlayRunnerRuntimeResource = {
|
|
13
13
|
kind: 'daytona_sandbox' | 'modal_sandbox';
|
|
@@ -79,6 +79,26 @@ export type PlayRunnerRuntimeLifecycleEvent = {
|
|
|
79
79
|
errorClass?: 'timeout' | 'capacity' | 'rate_limit' | 'other';
|
|
80
80
|
};
|
|
81
81
|
|
|
82
|
+
/**
|
|
83
|
+
* The provider has accepted the detached runner command. This is deliberately
|
|
84
|
+
* smaller than a runner payload: the scheduler needs only the physical
|
|
85
|
+
* resource identity to cross the no-replay boundary.
|
|
86
|
+
*/
|
|
87
|
+
export type PlayRunnerCommandAcceptance = {
|
|
88
|
+
provider: 'daytona' | 'modal';
|
|
89
|
+
sandboxId: string;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Scheduler-owned liveness was observed after a detached command was
|
|
94
|
+
* accepted. A backend must emit this only immediately before it returns a
|
|
95
|
+
* detached suspension.
|
|
96
|
+
*/
|
|
97
|
+
export type PlayRunnerDetachedRunnerReady = PlayRunnerCommandAcceptance & {
|
|
98
|
+
boundaryId: string;
|
|
99
|
+
baselineHeartbeatAt: string | null;
|
|
100
|
+
};
|
|
101
|
+
|
|
82
102
|
export class RuntimeResourceFenceLostError extends Error {
|
|
83
103
|
constructor(message: string) {
|
|
84
104
|
super(message);
|
|
@@ -94,6 +114,27 @@ export class RuntimeResourceFenceLostError extends Error {
|
|
|
94
114
|
export const RUNTIME_SANDBOX_CAPACITY_LIMIT_ERROR_NAME =
|
|
95
115
|
'RuntimeSandboxCapacityLimitError';
|
|
96
116
|
|
|
117
|
+
/**
|
|
118
|
+
* A scheduler admission decision reached the legacy runner protocol before a
|
|
119
|
+
* provider create. Backends rethrow it verbatim so the Sandbox Runs Adapter can
|
|
120
|
+
* preserve a retry-safe `pre_code_deferred` outcome instead of manufacturing a
|
|
121
|
+
* runner failure.
|
|
122
|
+
*/
|
|
123
|
+
export class SandboxRunnerPreCodeDeferralError extends Error {
|
|
124
|
+
constructor(
|
|
125
|
+
readonly reason:
|
|
126
|
+
| 'capacity'
|
|
127
|
+
| 'provider_unavailable'
|
|
128
|
+
| 'rate_limited'
|
|
129
|
+
| 'startup_timeout',
|
|
130
|
+
) {
|
|
131
|
+
super(
|
|
132
|
+
`Sandbox Runs deferred fresh execution before provider create: ${reason}.`,
|
|
133
|
+
);
|
|
134
|
+
this.name = 'SandboxRunnerPreCodeDeferralError';
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
97
138
|
export function isRuntimeSandboxCapacityLimitError(
|
|
98
139
|
error: unknown,
|
|
99
140
|
): error is Error {
|
|
@@ -155,6 +196,21 @@ export interface PlayRunnerCallbacks {
|
|
|
155
196
|
onRuntimeLifecycleEvent?: (
|
|
156
197
|
event: PlayRunnerRuntimeLifecycleEvent,
|
|
157
198
|
) => Promise<void>;
|
|
199
|
+
/**
|
|
200
|
+
* Marks the command acceptance no-replay boundary. This is separate from a
|
|
201
|
+
* resource acquisition because a recorded sandbox can still fail staging
|
|
202
|
+
* before customer code is accepted.
|
|
203
|
+
*/
|
|
204
|
+
onRunnerCommandAccepted?: (
|
|
205
|
+
acceptance: PlayRunnerCommandAcceptance,
|
|
206
|
+
) => void | Promise<void>;
|
|
207
|
+
/**
|
|
208
|
+
* Marks the scheduler-owned readiness that makes a detached suspension safe
|
|
209
|
+
* to park. It is never a provider heartbeat or a runner payload callback.
|
|
210
|
+
*/
|
|
211
|
+
onDetachedRunnerReady?: (
|
|
212
|
+
readiness: PlayRunnerDetachedRunnerReady,
|
|
213
|
+
) => void | Promise<void>;
|
|
158
214
|
/**
|
|
159
215
|
* Scheduler-owned readiness read for a detached runner attempt. Daytona uses
|
|
160
216
|
* this direct control-plane port before parking; it is never serialized into
|
|
@@ -4,7 +4,7 @@ import type { PlayRowUpdate } from './ctx-types';
|
|
|
4
4
|
import type { PlaySheetContract } from '../plays/static-pipeline';
|
|
5
5
|
import type { PlayRunLedgerEvent } from './run-ledger';
|
|
6
6
|
import type { WorkReceiptFailureKind } from './work-receipts';
|
|
7
|
-
import type { ToolExecutionFailureV1 } from '../tool-execution-error';
|
|
7
|
+
import type { ToolExecutionFailureV1 } from '../plays/tool-execution-error';
|
|
8
8
|
import type { DocflowObservationUpsert } from './docflow-observation';
|
|
9
9
|
|
|
10
10
|
export type RuntimeAppendRunEventsAction = {
|
|
@@ -8,7 +8,10 @@ import {
|
|
|
8
8
|
type RuntimePoolClient,
|
|
9
9
|
type RuntimeOneShotQueryClient,
|
|
10
10
|
} from './runtime-pg-driver';
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
createDeferredPlayDataset,
|
|
13
|
+
type PlayDataset,
|
|
14
|
+
} from '../plays/dataset';
|
|
12
15
|
import type {
|
|
13
16
|
PlayStaticPipeline,
|
|
14
17
|
PlaySheetContract,
|
|
@@ -59,7 +62,7 @@ import {
|
|
|
59
62
|
workReceiptFailureKindFromCode,
|
|
60
63
|
type WorkReceiptFailureKind,
|
|
61
64
|
} from './work-receipts';
|
|
62
|
-
import type { ToolExecutionFailureV1 } from '../tool-execution-error';
|
|
65
|
+
import type { ToolExecutionFailureV1 } from '../plays/tool-execution-error';
|
|
63
66
|
import {
|
|
64
67
|
sanitizePostgresJsonValue,
|
|
65
68
|
stringifyPostgresJson,
|
|
@@ -92,7 +95,7 @@ import {
|
|
|
92
95
|
mapRowOutcomeRuntimeFields,
|
|
93
96
|
resolveMapRowOutcomeKey,
|
|
94
97
|
} from './map-row-outcome';
|
|
95
|
-
import { DEEPLINE_CELL_META_FIELD } from '
|
|
98
|
+
import { DEEPLINE_CELL_META_FIELD } from '../plays/cell-staleness';
|
|
96
99
|
import {
|
|
97
100
|
PLAY_RUNTIME_CONTRACT,
|
|
98
101
|
PLAY_RUNTIME_CONTRACT_HEADER,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { MAX_PLAY_SANDBOX_RUNTIME_LIMITS } from '
|
|
1
|
+
import { MAX_PLAY_SANDBOX_RUNTIME_LIMITS } from '../plays/sandbox-runtime-limits';
|
|
2
2
|
|
|
3
3
|
/** Maximum active user-code runtime for a standard play, in seconds. */
|
|
4
4
|
export const STANDARD_PLAY_RUNTIME_LIMIT_SECONDS = 30 * 60;
|
|
@@ -1,2 +1 @@
|
|
|
1
|
-
export
|
|
2
|
-
'No run_javascript tool in plays. Just write regular code.';
|
|
1
|
+
export * from '../plays/runtime-constraints';
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
export type DeeplineRuntimeEnv = 'prod' | 'dev';
|
|
2
|
+
|
|
3
|
+
function normalizeEnvToken(value: string): DeeplineRuntimeEnv | null {
|
|
4
|
+
const token = value.trim().toLowerCase();
|
|
5
|
+
if (!token) return null;
|
|
6
|
+
|
|
7
|
+
if (
|
|
8
|
+
token === 'prod' ||
|
|
9
|
+
token === 'production' ||
|
|
10
|
+
token === 'live' ||
|
|
11
|
+
token.startsWith('prod-')
|
|
12
|
+
) {
|
|
13
|
+
return 'prod';
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
if (
|
|
17
|
+
token === 'dev' ||
|
|
18
|
+
token === 'development' ||
|
|
19
|
+
token === 'local' ||
|
|
20
|
+
token === 'preview' ||
|
|
21
|
+
token === 'staging' ||
|
|
22
|
+
token === 'stage' ||
|
|
23
|
+
token === 'test' ||
|
|
24
|
+
token.startsWith('dev-')
|
|
25
|
+
) {
|
|
26
|
+
return 'dev';
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function getRuntimeEnv(): DeeplineRuntimeEnv {
|
|
33
|
+
const explicit = normalizeEnvToken(process.env.DEEPLINE_ENV ?? '');
|
|
34
|
+
if (explicit) return explicit;
|
|
35
|
+
|
|
36
|
+
const convexDeployment = String(process.env.CONVEX_DEPLOYMENT ?? '')
|
|
37
|
+
.trim()
|
|
38
|
+
.toLowerCase();
|
|
39
|
+
if (convexDeployment.startsWith('prod:')) return 'prod';
|
|
40
|
+
if (convexDeployment.startsWith('dev:')) return 'dev';
|
|
41
|
+
|
|
42
|
+
const vercelEnv = normalizeEnvToken(process.env.VERCEL_ENV ?? '');
|
|
43
|
+
if (vercelEnv) return vercelEnv;
|
|
44
|
+
|
|
45
|
+
return process.env.NODE_ENV === 'production' ? 'prod' : 'dev';
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function getRuntimeEnvInfo() {
|
|
49
|
+
return {
|
|
50
|
+
env: getRuntimeEnv(),
|
|
51
|
+
deepline_env: process.env.DEEPLINE_ENV ?? null,
|
|
52
|
+
convex_deployment: process.env.CONVEX_DEPLOYMENT ?? null,
|
|
53
|
+
node_env: process.env.NODE_ENV ?? null,
|
|
54
|
+
};
|
|
55
|
+
}
|