deepline 0.1.214 → 0.1.215
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-manifest-resolver.ts +107 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +85 -10
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-runtime-proxy.ts +34 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +50 -36
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +12 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +4 -0
- package/dist/bundling-sources/sdk/src/client.ts +1 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +21 -3
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +6 -6
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +14 -3
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/profiles.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +32 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +5 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +66 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +18 -55
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +57 -25
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +17 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +11 -1
- package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +10 -0
- package/dist/cli/index.js +35 -12
- package/dist/cli/index.mjs +35 -12
- package/dist/index.d.mts +2 -3
- package/dist/index.d.ts +2 -3
- package/dist/index.js +7 -6
- package/dist/index.mjs +7 -6
- package/package.json +1 -1
|
@@ -38,7 +38,7 @@ import {
|
|
|
38
38
|
PLAY_RUNTIME_CONTRACT,
|
|
39
39
|
PLAY_RUNTIME_CONTRACT_HEADER,
|
|
40
40
|
} from '@shared_libs/play-runtime/runtime-contract';
|
|
41
|
-
import {
|
|
41
|
+
import { PLAY_RUNTIME_API_CURRENT_PATH } from '@shared_libs/play-runtime/runtime-api-paths';
|
|
42
42
|
import { PLAY_RUNTIME_TEST_FAULT_HEADER } from '@shared_libs/play-runtime/test-runtime-seams';
|
|
43
43
|
import { vercelProtectionBypassHeaders } from '@shared_libs/play-runtime/vercel-protection';
|
|
44
44
|
|
|
@@ -569,6 +569,19 @@ function summarizeAppRuntimeErrorBody(body: string): string {
|
|
|
569
569
|
return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
|
|
570
570
|
}
|
|
571
571
|
|
|
572
|
+
function appRuntimeErrorCode(response: Response, body: string): string | null {
|
|
573
|
+
const header = response.headers.get('x-deepline-error-code')?.trim();
|
|
574
|
+
if (header) return header;
|
|
575
|
+
try {
|
|
576
|
+
const parsed = JSON.parse(body) as { code?: unknown };
|
|
577
|
+
return typeof parsed?.code === 'string' && parsed.code.trim()
|
|
578
|
+
? parsed.code.trim()
|
|
579
|
+
: null;
|
|
580
|
+
} catch {
|
|
581
|
+
return null;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
|
|
572
585
|
export function isTransientAppRuntimeFailureBody(body: string): boolean {
|
|
573
586
|
return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated|RUNTIME_SCHEDULER_SATURATED|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open|OptimisticConcurrencyControlFailure|changed while this mutation was being run|Documents read from or written to the .* table changed while this mutation/i.test(
|
|
574
587
|
body,
|
|
@@ -775,7 +788,7 @@ export class AppRuntimeApiTransportError extends Error {
|
|
|
775
788
|
|
|
776
789
|
function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
|
|
777
790
|
const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
|
|
778
|
-
return `${baseUrl}${
|
|
791
|
+
return `${baseUrl}${PLAY_RUNTIME_API_CURRENT_PATH}`;
|
|
779
792
|
}
|
|
780
793
|
|
|
781
794
|
async function postAppRuntimeApi<TResponse>(
|
|
@@ -892,8 +905,13 @@ async function postAppRuntimeApi<TResponse>(
|
|
|
892
905
|
);
|
|
893
906
|
continue;
|
|
894
907
|
}
|
|
908
|
+
const code = appRuntimeErrorCode(response, responseText);
|
|
909
|
+
const requestId = response.headers.get('x-deepline-request-id')?.trim();
|
|
895
910
|
throw new Error(
|
|
896
|
-
`App runtime API ${body.action} failed with status ${response.status}
|
|
911
|
+
`App runtime API ${body.action} failed with status ${response.status}` +
|
|
912
|
+
`${code ? ` code=${code}` : ''}` +
|
|
913
|
+
`${requestId ? ` request_id=${requestId}` : ''}: ` +
|
|
914
|
+
summarizeAppRuntimeErrorBody(responseText),
|
|
897
915
|
);
|
|
898
916
|
}
|
|
899
917
|
|
|
@@ -48,8 +48,8 @@ import {
|
|
|
48
48
|
SYNTHETIC_RUN_HEADER,
|
|
49
49
|
} from './coordinator-headers';
|
|
50
50
|
import {
|
|
51
|
-
|
|
52
|
-
|
|
51
|
+
PLAY_RUNTIME_API_CURRENT_PATH,
|
|
52
|
+
PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_CURRENT_PATH,
|
|
53
53
|
} from './runtime-api-paths';
|
|
54
54
|
import {
|
|
55
55
|
createRootRunExecutionScope,
|
|
@@ -1702,7 +1702,7 @@ export class PlayContextImpl {
|
|
|
1702
1702
|
| null
|
|
1703
1703
|
> {
|
|
1704
1704
|
const response = await fetch(
|
|
1705
|
-
`${this.runtimeApiBaseUrl()}${
|
|
1705
|
+
`${this.runtimeApiBaseUrl()}${PLAY_RUNTIME_API_CURRENT_PATH}`,
|
|
1706
1706
|
{
|
|
1707
1707
|
method: 'POST',
|
|
1708
1708
|
headers: await this.runtimeApiHeaders(),
|
|
@@ -1817,7 +1817,7 @@ export class PlayContextImpl {
|
|
|
1817
1817
|
this.#options.runId
|
|
1818
1818
|
) {
|
|
1819
1819
|
const response = await fetch(
|
|
1820
|
-
`${this.#options.baseUrl.replace(/\/$/, '')}${
|
|
1820
|
+
`${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_CURRENT_PATH}`,
|
|
1821
1821
|
{
|
|
1822
1822
|
method: 'POST',
|
|
1823
1823
|
headers: {
|
|
@@ -6707,7 +6707,7 @@ export class PlayContextImpl {
|
|
|
6707
6707
|
: inlineChildGovernor!.snapshot();
|
|
6708
6708
|
const childPlaySlot = launchThroughScheduler
|
|
6709
6709
|
? await this.governor.acquireChildSubmitSlot()
|
|
6710
|
-
:
|
|
6710
|
+
: await this.governor.acquireInlineChildSlot();
|
|
6711
6711
|
const rowStore = rowContext.getStore();
|
|
6712
6712
|
const producer = {
|
|
6713
6713
|
kind: 'play' as const,
|
|
@@ -6964,7 +6964,7 @@ export class PlayContextImpl {
|
|
|
6964
6964
|
return this.#options.executorToken;
|
|
6965
6965
|
}
|
|
6966
6966
|
const response = await fetch(
|
|
6967
|
-
`${this.#options.baseUrl.replace(/\/$/, '')}${
|
|
6967
|
+
`${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_CURRENT_PATH}`,
|
|
6968
6968
|
{
|
|
6969
6969
|
method: 'POST',
|
|
6970
6970
|
headers: {
|
|
@@ -429,7 +429,11 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
429
429
|
receipt: RuntimeStepReceipt,
|
|
430
430
|
source: DurableReceiptRecoverySource = 'cache',
|
|
431
431
|
): Promise<T> => {
|
|
432
|
-
input.log(
|
|
432
|
+
input.log(
|
|
433
|
+
source === 'owner'
|
|
434
|
+
? `ctx.${input.operation}(${input.id}): executed and converged through immutable publication`
|
|
435
|
+
: `ctx.${input.operation}(${input.id}): reused completed work`,
|
|
436
|
+
);
|
|
433
437
|
if (receipt.output === undefined) {
|
|
434
438
|
return receipt.output as T;
|
|
435
439
|
}
|
|
@@ -489,8 +493,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
489
493
|
const afterLock = await input.store.get(input.receiptKey);
|
|
490
494
|
if (
|
|
491
495
|
input.force !== true &&
|
|
492
|
-
(afterLock?.status === 'completed' ||
|
|
493
|
-
afterLock?.status === 'skipped')
|
|
496
|
+
(afterLock?.status === 'completed' || afterLock?.status === 'skipped')
|
|
494
497
|
) {
|
|
495
498
|
await input.store.releaseExecutionLock({
|
|
496
499
|
receiptKey: input.receiptKey,
|
|
@@ -566,6 +569,10 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
566
569
|
completed?.status === 'completed' ||
|
|
567
570
|
completed?.status === 'skipped'
|
|
568
571
|
) {
|
|
572
|
+
// Completion is a compact acknowledgement. This execution already
|
|
573
|
+
// owns the just-produced value; only a later receipt read needs the
|
|
574
|
+
// persisted payload for replay.
|
|
575
|
+
if (completed.output === undefined) return result;
|
|
569
576
|
return await recoverCompletedReceipt(completed, 'owner');
|
|
570
577
|
}
|
|
571
578
|
const winner = await input.store.get(input.receiptKey);
|
|
@@ -828,6 +835,10 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
828
835
|
(completed.status === 'completed' || completed.status === 'skipped')
|
|
829
836
|
) {
|
|
830
837
|
logPhase('total', lifecycleStartedAt);
|
|
838
|
+
// Completion endpoints acknowledge ownership and status only. The caller
|
|
839
|
+
// already owns the live value; replay reads are the only path that needs
|
|
840
|
+
// the receipt payload returned over the transport.
|
|
841
|
+
if (completed.output === undefined) return result;
|
|
831
842
|
return await recoverCompletedReceipt(completed, 'owner');
|
|
832
843
|
}
|
|
833
844
|
if (input.store.canPersistCompletion) {
|
|
@@ -106,6 +106,13 @@ export interface PlayExecutionGovernor {
|
|
|
106
106
|
acquireRowSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
|
|
107
107
|
/** Block until a child-play submit slot is free. Released after submit, not terminal. */
|
|
108
108
|
acquireChildSubmitSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
|
|
109
|
+
/**
|
|
110
|
+
* Block until an in-process child-composition slot is free. Unlike a
|
|
111
|
+
* scheduled child's submit slot, this lease is held through child terminal
|
|
112
|
+
* execution so a large row fan-out cannot materialize thousands of child
|
|
113
|
+
* contexts and receipt closures at once.
|
|
114
|
+
*/
|
|
115
|
+
acquireInlineChildSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
|
|
109
116
|
/**
|
|
110
117
|
* Block until a global tool-concurrency slot AND the per-(org,provider) pacer
|
|
111
118
|
* permit are free, then charge the tool-call budget and return a lease. Order:
|
|
@@ -397,6 +404,7 @@ export function createPlayExecutionGovernor(
|
|
|
397
404
|
|
|
398
405
|
acquireRowSlot: (opts) => rowSlots.acquire(opts?.signal),
|
|
399
406
|
acquireChildSubmitSlot: (opts) => childPlaySlots.acquire(opts?.signal),
|
|
407
|
+
acquireInlineChildSlot: (opts) => childPlaySlots.acquire(opts?.signal),
|
|
400
408
|
|
|
401
409
|
async acquireToolSlot(toolId, opts) {
|
|
402
410
|
// 1. global tool-concurrency slot.
|
|
@@ -612,6 +620,17 @@ function createInlineChildGovernor(
|
|
|
612
620
|
policy: root.policy,
|
|
613
621
|
acquireRowSlot: (opts) => root.acquireRowSlot(opts),
|
|
614
622
|
acquireChildSubmitSlot: (opts) => root.acquireChildSubmitSlot(opts),
|
|
623
|
+
async acquireInlineChildSlot(opts) {
|
|
624
|
+
if (opts?.signal?.aborted) {
|
|
625
|
+
throw opts.signal.reason instanceof Error
|
|
626
|
+
? opts.signal.reason
|
|
627
|
+
: new Error('Slot acquire aborted.');
|
|
628
|
+
}
|
|
629
|
+
// This governor already runs inside a root inline-child lease. Descendant
|
|
630
|
+
// composition shares that admitted subtree slot; reacquiring the root
|
|
631
|
+
// semaphore here can deadlock when every admitted parent awaits a child.
|
|
632
|
+
return { release: () => {} };
|
|
633
|
+
},
|
|
615
634
|
acquireToolSlot: (toolId, opts) => root.acquireToolSlot(toolId, opts),
|
|
616
635
|
suggestedParallelism: (toolId, fallback) =>
|
|
617
636
|
root.suggestedParallelism(toolId, fallback),
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
PLAY_RUNTIME_PROVIDERS,
|
|
7
7
|
PLAY_RUNTIME_PROVIDER_IDS,
|
|
8
8
|
defaultPlayRuntimeProvider,
|
|
9
|
+
resolveEnabledPlayRuntimeProvider,
|
|
9
10
|
resolvePlayRuntimeProvider,
|
|
10
11
|
type PlayRuntimeProviderId,
|
|
11
12
|
} from './providers';
|
|
@@ -50,3 +51,24 @@ export function resolveExecutionProfile(
|
|
|
50
51
|
throw error;
|
|
51
52
|
}
|
|
52
53
|
}
|
|
54
|
+
|
|
55
|
+
export function resolveEnabledExecutionProfile(
|
|
56
|
+
override?: string | null,
|
|
57
|
+
): PlayExecutionProfile {
|
|
58
|
+
try {
|
|
59
|
+
return resolveEnabledPlayRuntimeProvider(override);
|
|
60
|
+
} catch (error) {
|
|
61
|
+
if (
|
|
62
|
+
override?.trim() &&
|
|
63
|
+
error instanceof Error &&
|
|
64
|
+
/Unsupported play runtime provider/.test(error.message)
|
|
65
|
+
) {
|
|
66
|
+
throw new Error(
|
|
67
|
+
`Unknown execution profile "${override.trim()}". Expected one of: ${Object.keys(
|
|
68
|
+
PLAY_EXECUTION_PROFILES,
|
|
69
|
+
).join(', ')}.`,
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -27,6 +27,22 @@ export type PlayRuntimeProviderDescriptor = {
|
|
|
27
27
|
label: string;
|
|
28
28
|
};
|
|
29
29
|
|
|
30
|
+
export const PLAY_RUNTIME_PROVIDER_DISABLED_CODE =
|
|
31
|
+
'PLAY_RUNTIME_PROVIDER_DISABLED' as const;
|
|
32
|
+
|
|
33
|
+
export class PlayRuntimeProviderDisabledError extends Error {
|
|
34
|
+
readonly code = PLAY_RUNTIME_PROVIDER_DISABLED_CODE;
|
|
35
|
+
readonly profile: PlayRuntimeProviderId;
|
|
36
|
+
|
|
37
|
+
constructor(profile: PlayRuntimeProviderId) {
|
|
38
|
+
super(
|
|
39
|
+
`Play runtime profile "${profile}" is disabled. Deepline runs now execute on the "absurd" profile.`,
|
|
40
|
+
);
|
|
41
|
+
this.name = 'PlayRuntimeProviderDisabledError';
|
|
42
|
+
this.profile = profile;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
30
46
|
export const PLAY_RUNTIME_PROVIDERS: Record<
|
|
31
47
|
PlayRuntimeProviderId,
|
|
32
48
|
PlayRuntimeProviderDescriptor
|
|
@@ -57,6 +73,14 @@ export function defaultPlayRuntimeProvider(): PlayRuntimeProviderDescriptor {
|
|
|
57
73
|
return PLAY_RUNTIME_PROVIDERS.absurd;
|
|
58
74
|
}
|
|
59
75
|
|
|
76
|
+
export function assertPlayRuntimeProviderEnabled(
|
|
77
|
+
provider: PlayRuntimeProviderDescriptor,
|
|
78
|
+
): void {
|
|
79
|
+
if (provider.id === PLAY_RUNTIME_PROVIDER_IDS.workersEdge) {
|
|
80
|
+
throw new PlayRuntimeProviderDisabledError(provider.id);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
60
84
|
export function resolvePlayRuntimeProvider(
|
|
61
85
|
override?: string | null,
|
|
62
86
|
): PlayRuntimeProviderDescriptor {
|
|
@@ -73,3 +97,11 @@ export function resolvePlayRuntimeProvider(
|
|
|
73
97
|
}
|
|
74
98
|
return defaultPlayRuntimeProvider();
|
|
75
99
|
}
|
|
100
|
+
|
|
101
|
+
export function resolveEnabledPlayRuntimeProvider(
|
|
102
|
+
override?: string | null,
|
|
103
|
+
): PlayRuntimeProviderDescriptor {
|
|
104
|
+
const provider = resolvePlayRuntimeProvider(override);
|
|
105
|
+
assertPlayRuntimeProviderEnabled(provider);
|
|
106
|
+
return provider;
|
|
107
|
+
}
|
|
@@ -2,6 +2,8 @@ const CLOUDFLARE_DURABLE_OBJECT_RESET_RE =
|
|
|
2
2
|
/Durable Object.*(?:code (?:was|has been) updated|storage caused object)/;
|
|
3
3
|
const CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE =
|
|
4
4
|
/Too many subrequests by single Worker invocation/i;
|
|
5
|
+
const CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE =
|
|
6
|
+
/The requested endpoint could not be found, or you don't have access to it\. Please check the provided ID and try again\./i;
|
|
5
7
|
const VERCEL_RUNTIME_API_DEPLOYMENT_MISSING_RE =
|
|
6
8
|
/runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i;
|
|
7
9
|
|
|
@@ -9,6 +11,8 @@ export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
|
|
|
9
11
|
'Run interrupted by a platform deploy. Deepline retries this automatically when possible; if this error is still visible, re-run the same command.';
|
|
10
12
|
export const PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE =
|
|
11
13
|
'Worker subrequest limit hit. Deepline retries automatically; if still visible, re-run.';
|
|
14
|
+
export const PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE =
|
|
15
|
+
'Cloudflare workflow control was temporarily unavailable. Deepline retries automatically; if still visible, re-run.';
|
|
12
16
|
|
|
13
17
|
export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
|
|
14
18
|
'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
|
|
@@ -171,6 +175,15 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
|
|
|
171
175
|
cause,
|
|
172
176
|
};
|
|
173
177
|
}
|
|
178
|
+
if (CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE.test(cause)) {
|
|
179
|
+
return {
|
|
180
|
+
code: 'PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED',
|
|
181
|
+
phase: 'runtime',
|
|
182
|
+
message: PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE,
|
|
183
|
+
retryable: true,
|
|
184
|
+
cause,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
174
187
|
const playDepthBudgetMatch = cause.match(
|
|
175
188
|
/Play execution playDepth budget exceeded \((\d+)\/(\d+)\)\./,
|
|
176
189
|
);
|
|
@@ -27,6 +27,7 @@ type DaytonaPayloadEnvelope = {
|
|
|
27
27
|
export type StagedDaytonaPayload = {
|
|
28
28
|
workDir: string;
|
|
29
29
|
command: string;
|
|
30
|
+
readyPath: string;
|
|
30
31
|
outputPath: string;
|
|
31
32
|
exitCodePath: string;
|
|
32
33
|
progressEventPath: string;
|
|
@@ -343,6 +344,7 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
343
344
|
|
|
344
345
|
const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
|
|
345
346
|
const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
|
|
347
|
+
const readyPath = `${workDir}/deepline-play-ready-${randomUUID()}.json`;
|
|
346
348
|
const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
|
|
347
349
|
const runnerTraceEnv =
|
|
348
350
|
process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
|
|
@@ -354,17 +356,18 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
354
356
|
configPath,
|
|
355
357
|
artifactCodePath,
|
|
356
358
|
crashPusherPath,
|
|
357
|
-
})} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
359
|
+
})} && DEEPLINE_PLAY_RUNNER_READY_PATH=${shellQuote(readyPath)} DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
358
360
|
// Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
|
|
359
361
|
// pushes the parsed (or synthesized) terminal to the gateway so the parked
|
|
360
362
|
// worker wakes within seconds of ANY runner death — process.exit abuse, OOM
|
|
361
363
|
// SIGKILL, or a crash before the in-process push. Idempotent against a
|
|
362
364
|
// successful in-runner push (terminals are first-write-wins).
|
|
363
|
-
const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
|
|
365
|
+
const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(readyPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
|
|
364
366
|
|
|
365
367
|
return {
|
|
366
368
|
workDir: input.workDir,
|
|
367
369
|
command,
|
|
370
|
+
readyPath,
|
|
368
371
|
outputPath,
|
|
369
372
|
exitCodePath,
|
|
370
373
|
progressEventPath,
|
|
@@ -112,6 +112,16 @@ export type DetachedDaytonaCommandStart = {
|
|
|
112
112
|
cmdId: string;
|
|
113
113
|
};
|
|
114
114
|
|
|
115
|
+
const DAYTONA_RUNNER_READY_TIMEOUT_MS = 3_000;
|
|
116
|
+
const DAYTONA_RUNNER_READY_POLL_MS = 250;
|
|
117
|
+
|
|
118
|
+
export class DaytonaRunnerInitializationError extends Error {
|
|
119
|
+
constructor(message: string) {
|
|
120
|
+
super(message);
|
|
121
|
+
this.name = 'DaytonaRunnerInitializationError';
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
115
125
|
/**
|
|
116
126
|
* Start `command` detached in a fresh session. Returns the session + command id
|
|
117
127
|
* immediately (no await on completion).
|
|
@@ -137,3 +147,59 @@ export async function startDetachedDaytonaCommand(input: {
|
|
|
137
147
|
}
|
|
138
148
|
return { sessionId: input.sessionId, cmdId };
|
|
139
149
|
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* A detached command id only proves that Daytona accepted the request. Before
|
|
153
|
+
* the worker parks, require a marker written by the initialized runner process.
|
|
154
|
+
* This closes the unobservable handoff where a sandbox or command disappears
|
|
155
|
+
* after executeSessionCommand but before the first gateway heartbeat.
|
|
156
|
+
*/
|
|
157
|
+
export async function confirmDetachedDaytonaRunnerReady(input: {
|
|
158
|
+
sandbox: DaytonaSandbox;
|
|
159
|
+
sessionId: string;
|
|
160
|
+
cmdId: string;
|
|
161
|
+
readyPath: string;
|
|
162
|
+
exitCodePath: string;
|
|
163
|
+
timeoutMs?: number;
|
|
164
|
+
pollMs?: number;
|
|
165
|
+
now?: () => number;
|
|
166
|
+
sleep?: (ms: number) => Promise<void>;
|
|
167
|
+
}): Promise<void> {
|
|
168
|
+
const now = input.now ?? Date.now;
|
|
169
|
+
const sleep =
|
|
170
|
+
input.sleep ??
|
|
171
|
+
((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
|
172
|
+
const timeoutMs = Math.max(
|
|
173
|
+
1,
|
|
174
|
+
input.timeoutMs ?? DAYTONA_RUNNER_READY_TIMEOUT_MS,
|
|
175
|
+
);
|
|
176
|
+
const pollMs = Math.max(1, input.pollMs ?? DAYTONA_RUNNER_READY_POLL_MS);
|
|
177
|
+
const deadline = now() + timeoutMs;
|
|
178
|
+
let markerDiagnosis = 'ready_marker_unavailable';
|
|
179
|
+
|
|
180
|
+
while (now() < deadline) {
|
|
181
|
+
try {
|
|
182
|
+
const marker = await input.sandbox.fs.downloadFile(input.readyPath, 2);
|
|
183
|
+
const parsed = JSON.parse(marker.toString('utf-8')) as {
|
|
184
|
+
ready?: unknown;
|
|
185
|
+
};
|
|
186
|
+
if (parsed.ready === true) return;
|
|
187
|
+
markerDiagnosis = 'ready_marker_invalid';
|
|
188
|
+
} catch {
|
|
189
|
+
markerDiagnosis = 'ready_marker_unavailable';
|
|
190
|
+
}
|
|
191
|
+
await sleep(Math.min(pollMs, Math.max(1, deadline - now())));
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const diagnostic = await inspectDetachedDaytonaStartup({
|
|
195
|
+
sandbox: input.sandbox,
|
|
196
|
+
sessionId: input.sessionId,
|
|
197
|
+
cmdId: input.cmdId,
|
|
198
|
+
exitCodePath: input.exitCodePath,
|
|
199
|
+
});
|
|
200
|
+
const oomIndicated =
|
|
201
|
+
diagnostic.exitCode === 137 || diagnostic.exitCodeFile === 137;
|
|
202
|
+
throw new DaytonaRunnerInitializationError(
|
|
203
|
+
`RUNTIME_SANDBOX_START_FAILED: Daytona accepted detached command ${input.cmdId} in sandbox ${input.sandbox.id}, but the play runner did not initialize within ${timeoutMs}ms. Marker diagnosis: ${markerDiagnosis}. Daytona diagnosis: ${JSON.stringify(diagnostic)}. OOM is ${oomIndicated ? 'indicated by exit code 137' : 'not confirmed by the available Daytona evidence'}. The play was stopped and was not retried.`,
|
|
204
|
+
);
|
|
205
|
+
}
|
|
@@ -33,7 +33,8 @@ import {
|
|
|
33
33
|
} from './daytona-lifecycle';
|
|
34
34
|
import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
|
|
35
35
|
import {
|
|
36
|
-
|
|
36
|
+
confirmDetachedDaytonaRunnerReady,
|
|
37
|
+
DaytonaRunnerInitializationError,
|
|
37
38
|
startDetachedDaytonaCommand,
|
|
38
39
|
} from './daytona-session-execution';
|
|
39
40
|
|
|
@@ -45,8 +46,6 @@ const DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS = 2;
|
|
|
45
46
|
const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
|
|
46
47
|
const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
|
|
47
48
|
const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
|
|
48
|
-
const DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS = 15_000;
|
|
49
|
-
const DAYTONA_STARTUP_TRACE_ENV = 'DEEPLINE_DAYTONA_STARTUP_TRACE';
|
|
50
49
|
|
|
51
50
|
const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
|
|
52
51
|
/\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
|
|
@@ -397,6 +396,7 @@ export function retryableRuntimeInitializationFailureReason(
|
|
|
397
396
|
}
|
|
398
397
|
|
|
399
398
|
function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
|
|
399
|
+
if (error instanceof DaytonaRunnerInitializationError) return false;
|
|
400
400
|
const message = error instanceof Error ? error.message : String(error);
|
|
401
401
|
if (!message || /cancelled|aborted/i.test(message)) {
|
|
402
402
|
return false;
|
|
@@ -715,12 +715,11 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
715
715
|
});
|
|
716
716
|
// Push execution (B2-final, park-and-wake): start the runner command
|
|
717
717
|
// DETACHED via a Daytona session and return a `detached_runner`
|
|
718
|
-
// suspension
|
|
719
|
-
//
|
|
720
|
-
//
|
|
721
|
-
//
|
|
722
|
-
//
|
|
723
|
-
// wake leg owns finalize + sandbox GC.
|
|
718
|
+
// suspension after the runner writes its one-time ready marker. The
|
|
719
|
+
// marker is the handoff proof: a Daytona cmdId alone does not prove
|
|
720
|
+
// the sandbox process initialized. After that bounded startup check,
|
|
721
|
+
// the worker parks (releasing its claim and slot) and wakes only on
|
|
722
|
+
// the runner's pushed terminal or the ceiling timeout.
|
|
724
723
|
const sessionId = `deepline-play-${randomUUID()}`;
|
|
725
724
|
let start: Awaited<ReturnType<typeof startDetachedDaytonaCommand>>;
|
|
726
725
|
try {
|
|
@@ -744,6 +743,16 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
744
743
|
// classification below decide fresh-sandbox retry vs loud failure.
|
|
745
744
|
throw error;
|
|
746
745
|
}
|
|
746
|
+
await Promise.race([
|
|
747
|
+
confirmDetachedDaytonaRunnerReady({
|
|
748
|
+
sandbox,
|
|
749
|
+
sessionId: start.sessionId,
|
|
750
|
+
cmdId: start.cmdId,
|
|
751
|
+
readyPath: stagedPayload.readyPath,
|
|
752
|
+
exitCodePath: stagedPayload.exitCodePath,
|
|
753
|
+
}),
|
|
754
|
+
cancellationPromise,
|
|
755
|
+
]);
|
|
747
756
|
const runnerAttempt = Math.max(
|
|
748
757
|
0,
|
|
749
758
|
Math.floor(config.context.runAttempt ?? 0),
|
|
@@ -758,52 +767,6 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
758
767
|
exitCodePath: stagedPayload.exitCodePath,
|
|
759
768
|
elapsedMs: Date.now() - startedAt,
|
|
760
769
|
});
|
|
761
|
-
// Daytona acknowledges an async command with only a cmdId. It does
|
|
762
|
-
// not expose a queued/running state, and the backend parks before the
|
|
763
|
-
// runner contacts the runtime sheet. During startup investigations,
|
|
764
|
-
// retain a bounded snapshot while the sandbox is still alive; the
|
|
765
|
-
// normal cancellation path otherwise deletes the only evidence.
|
|
766
|
-
// This is opt-in because it adds Daytona control-plane calls to every
|
|
767
|
-
// run. The diagnostic excludes command lines and environment values.
|
|
768
|
-
if (process.env[DAYTONA_STARTUP_TRACE_ENV] === '1') {
|
|
769
|
-
const diagnosticTimer = setTimeout(() => {
|
|
770
|
-
void inspectDetachedDaytonaStartup({
|
|
771
|
-
sandbox,
|
|
772
|
-
sessionId: start.sessionId,
|
|
773
|
-
cmdId: start.cmdId,
|
|
774
|
-
exitCodePath: stagedPayload.exitCodePath,
|
|
775
|
-
}).then(
|
|
776
|
-
(diagnostic) =>
|
|
777
|
-
emitDaytonaStage(
|
|
778
|
-
callbacks,
|
|
779
|
-
config.context,
|
|
780
|
-
'execute:startup_diagnostic',
|
|
781
|
-
{
|
|
782
|
-
sandboxId: sandbox.id,
|
|
783
|
-
sessionId: start.sessionId,
|
|
784
|
-
cmdId: start.cmdId,
|
|
785
|
-
diagnostic,
|
|
786
|
-
elapsedMs: Date.now() - startedAt,
|
|
787
|
-
},
|
|
788
|
-
),
|
|
789
|
-
(error: unknown) =>
|
|
790
|
-
emitDaytonaStage(
|
|
791
|
-
callbacks,
|
|
792
|
-
config.context,
|
|
793
|
-
'execute:startup_diagnostic_failed',
|
|
794
|
-
{
|
|
795
|
-
sandboxId: sandbox.id,
|
|
796
|
-
sessionId: start.sessionId,
|
|
797
|
-
cmdId: start.cmdId,
|
|
798
|
-
errorType:
|
|
799
|
-
error instanceof Error ? error.name : 'unknown',
|
|
800
|
-
elapsedMs: Date.now() - startedAt,
|
|
801
|
-
},
|
|
802
|
-
),
|
|
803
|
-
);
|
|
804
|
-
}, DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS);
|
|
805
|
-
diagnosticTimer.unref?.();
|
|
806
|
-
}
|
|
807
770
|
return {
|
|
808
771
|
status: 'suspended',
|
|
809
772
|
suspension: {
|