deepline 0.1.214 → 0.1.216
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-manifest-resolver.ts +108 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +85 -10
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-runtime-proxy.ts +34 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +50 -36
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +12 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +4 -0
- package/dist/bundling-sources/sdk/src/client.ts +1 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +21 -3
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +95 -144
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +14 -3
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +37 -2
- package/dist/bundling-sources/shared_libs/play-runtime/profiles.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +32 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +5 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +66 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +18 -55
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +59 -27
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +17 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +11 -1
- package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +10 -0
- package/dist/cli/index.js +72 -21
- package/dist/cli/index.mjs +72 -21
- package/dist/index.d.mts +2 -3
- package/dist/index.d.ts +2 -3
- package/dist/index.js +7 -6
- package/dist/index.mjs +7 -6
- package/package.json +1 -1
|
@@ -27,6 +27,22 @@ export type PlayRuntimeProviderDescriptor = {
|
|
|
27
27
|
label: string;
|
|
28
28
|
};
|
|
29
29
|
|
|
30
|
+
export const PLAY_RUNTIME_PROVIDER_DISABLED_CODE =
|
|
31
|
+
'PLAY_RUNTIME_PROVIDER_DISABLED' as const;
|
|
32
|
+
|
|
33
|
+
export class PlayRuntimeProviderDisabledError extends Error {
|
|
34
|
+
readonly code = PLAY_RUNTIME_PROVIDER_DISABLED_CODE;
|
|
35
|
+
readonly profile: PlayRuntimeProviderId;
|
|
36
|
+
|
|
37
|
+
constructor(profile: PlayRuntimeProviderId) {
|
|
38
|
+
super(
|
|
39
|
+
`Play runtime profile "${profile}" is disabled. Deepline runs now execute on the "absurd" profile.`,
|
|
40
|
+
);
|
|
41
|
+
this.name = 'PlayRuntimeProviderDisabledError';
|
|
42
|
+
this.profile = profile;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
30
46
|
export const PLAY_RUNTIME_PROVIDERS: Record<
|
|
31
47
|
PlayRuntimeProviderId,
|
|
32
48
|
PlayRuntimeProviderDescriptor
|
|
@@ -57,6 +73,14 @@ export function defaultPlayRuntimeProvider(): PlayRuntimeProviderDescriptor {
|
|
|
57
73
|
return PLAY_RUNTIME_PROVIDERS.absurd;
|
|
58
74
|
}
|
|
59
75
|
|
|
76
|
+
export function assertPlayRuntimeProviderEnabled(
|
|
77
|
+
provider: PlayRuntimeProviderDescriptor,
|
|
78
|
+
): void {
|
|
79
|
+
if (provider.id === PLAY_RUNTIME_PROVIDER_IDS.workersEdge) {
|
|
80
|
+
throw new PlayRuntimeProviderDisabledError(provider.id);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
60
84
|
export function resolvePlayRuntimeProvider(
|
|
61
85
|
override?: string | null,
|
|
62
86
|
): PlayRuntimeProviderDescriptor {
|
|
@@ -73,3 +97,11 @@ export function resolvePlayRuntimeProvider(
|
|
|
73
97
|
}
|
|
74
98
|
return defaultPlayRuntimeProvider();
|
|
75
99
|
}
|
|
100
|
+
|
|
101
|
+
export function resolveEnabledPlayRuntimeProvider(
|
|
102
|
+
override?: string | null,
|
|
103
|
+
): PlayRuntimeProviderDescriptor {
|
|
104
|
+
const provider = resolvePlayRuntimeProvider(override);
|
|
105
|
+
assertPlayRuntimeProviderEnabled(provider);
|
|
106
|
+
return provider;
|
|
107
|
+
}
|
|
@@ -2,6 +2,8 @@ const CLOUDFLARE_DURABLE_OBJECT_RESET_RE =
|
|
|
2
2
|
/Durable Object.*(?:code (?:was|has been) updated|storage caused object)/;
|
|
3
3
|
const CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE =
|
|
4
4
|
/Too many subrequests by single Worker invocation/i;
|
|
5
|
+
const CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE =
|
|
6
|
+
/The requested endpoint could not be found, or you don't have access to it\. Please check the provided ID and try again\./i;
|
|
5
7
|
const VERCEL_RUNTIME_API_DEPLOYMENT_MISSING_RE =
|
|
6
8
|
/runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i;
|
|
7
9
|
|
|
@@ -9,6 +11,8 @@ export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
|
|
|
9
11
|
'Run interrupted by a platform deploy. Deepline retries this automatically when possible; if this error is still visible, re-run the same command.';
|
|
10
12
|
export const PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE =
|
|
11
13
|
'Worker subrequest limit hit. Deepline retries automatically; if still visible, re-run.';
|
|
14
|
+
export const PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE =
|
|
15
|
+
'Cloudflare workflow control was temporarily unavailable. Deepline retries automatically; if still visible, re-run.';
|
|
12
16
|
|
|
13
17
|
export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
|
|
14
18
|
'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
|
|
@@ -171,6 +175,15 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
|
|
|
171
175
|
cause,
|
|
172
176
|
};
|
|
173
177
|
}
|
|
178
|
+
if (CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE.test(cause)) {
|
|
179
|
+
return {
|
|
180
|
+
code: 'PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED',
|
|
181
|
+
phase: 'runtime',
|
|
182
|
+
message: PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE,
|
|
183
|
+
retryable: true,
|
|
184
|
+
cause,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
174
187
|
const playDepthBudgetMatch = cause.match(
|
|
175
188
|
/Play execution playDepth budget exceeded \((\d+)\/(\d+)\)\./,
|
|
176
189
|
);
|
|
@@ -27,6 +27,7 @@ type DaytonaPayloadEnvelope = {
|
|
|
27
27
|
export type StagedDaytonaPayload = {
|
|
28
28
|
workDir: string;
|
|
29
29
|
command: string;
|
|
30
|
+
readyPath: string;
|
|
30
31
|
outputPath: string;
|
|
31
32
|
exitCodePath: string;
|
|
32
33
|
progressEventPath: string;
|
|
@@ -343,6 +344,7 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
343
344
|
|
|
344
345
|
const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
|
|
345
346
|
const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
|
|
347
|
+
const readyPath = `${workDir}/deepline-play-ready-${randomUUID()}.json`;
|
|
346
348
|
const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
|
|
347
349
|
const runnerTraceEnv =
|
|
348
350
|
process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
|
|
@@ -354,17 +356,18 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
354
356
|
configPath,
|
|
355
357
|
artifactCodePath,
|
|
356
358
|
crashPusherPath,
|
|
357
|
-
})} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
359
|
+
})} && DEEPLINE_PLAY_RUNNER_READY_PATH=${shellQuote(readyPath)} DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
358
360
|
// Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
|
|
359
361
|
// pushes the parsed (or synthesized) terminal to the gateway so the parked
|
|
360
362
|
// worker wakes within seconds of ANY runner death — process.exit abuse, OOM
|
|
361
363
|
// SIGKILL, or a crash before the in-process push. Idempotent against a
|
|
362
364
|
// successful in-runner push (terminals are first-write-wins).
|
|
363
|
-
const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
|
|
365
|
+
const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(readyPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
|
|
364
366
|
|
|
365
367
|
return {
|
|
366
368
|
workDir: input.workDir,
|
|
367
369
|
command,
|
|
370
|
+
readyPath,
|
|
368
371
|
outputPath,
|
|
369
372
|
exitCodePath,
|
|
370
373
|
progressEventPath,
|
|
@@ -112,6 +112,16 @@ export type DetachedDaytonaCommandStart = {
|
|
|
112
112
|
cmdId: string;
|
|
113
113
|
};
|
|
114
114
|
|
|
115
|
+
const DAYTONA_RUNNER_READY_TIMEOUT_MS = 3_000;
|
|
116
|
+
const DAYTONA_RUNNER_READY_POLL_MS = 250;
|
|
117
|
+
|
|
118
|
+
export class DaytonaRunnerInitializationError extends Error {
|
|
119
|
+
constructor(message: string) {
|
|
120
|
+
super(message);
|
|
121
|
+
this.name = 'DaytonaRunnerInitializationError';
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
115
125
|
/**
|
|
116
126
|
* Start `command` detached in a fresh session. Returns the session + command id
|
|
117
127
|
* immediately (no await on completion).
|
|
@@ -137,3 +147,59 @@ export async function startDetachedDaytonaCommand(input: {
|
|
|
137
147
|
}
|
|
138
148
|
return { sessionId: input.sessionId, cmdId };
|
|
139
149
|
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* A detached command id only proves that Daytona accepted the request. Before
|
|
153
|
+
* the worker parks, require a marker written by the initialized runner process.
|
|
154
|
+
* This closes the unobservable handoff where a sandbox or command disappears
|
|
155
|
+
* after executeSessionCommand but before the first gateway heartbeat.
|
|
156
|
+
*/
|
|
157
|
+
export async function confirmDetachedDaytonaRunnerReady(input: {
|
|
158
|
+
sandbox: DaytonaSandbox;
|
|
159
|
+
sessionId: string;
|
|
160
|
+
cmdId: string;
|
|
161
|
+
readyPath: string;
|
|
162
|
+
exitCodePath: string;
|
|
163
|
+
timeoutMs?: number;
|
|
164
|
+
pollMs?: number;
|
|
165
|
+
now?: () => number;
|
|
166
|
+
sleep?: (ms: number) => Promise<void>;
|
|
167
|
+
}): Promise<void> {
|
|
168
|
+
const now = input.now ?? Date.now;
|
|
169
|
+
const sleep =
|
|
170
|
+
input.sleep ??
|
|
171
|
+
((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
|
172
|
+
const timeoutMs = Math.max(
|
|
173
|
+
1,
|
|
174
|
+
input.timeoutMs ?? DAYTONA_RUNNER_READY_TIMEOUT_MS,
|
|
175
|
+
);
|
|
176
|
+
const pollMs = Math.max(1, input.pollMs ?? DAYTONA_RUNNER_READY_POLL_MS);
|
|
177
|
+
const deadline = now() + timeoutMs;
|
|
178
|
+
let markerDiagnosis = 'ready_marker_unavailable';
|
|
179
|
+
|
|
180
|
+
while (now() < deadline) {
|
|
181
|
+
try {
|
|
182
|
+
const marker = await input.sandbox.fs.downloadFile(input.readyPath, 2);
|
|
183
|
+
const parsed = JSON.parse(marker.toString('utf-8')) as {
|
|
184
|
+
ready?: unknown;
|
|
185
|
+
};
|
|
186
|
+
if (parsed.ready === true) return;
|
|
187
|
+
markerDiagnosis = 'ready_marker_invalid';
|
|
188
|
+
} catch {
|
|
189
|
+
markerDiagnosis = 'ready_marker_unavailable';
|
|
190
|
+
}
|
|
191
|
+
await sleep(Math.min(pollMs, Math.max(1, deadline - now())));
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const diagnostic = await inspectDetachedDaytonaStartup({
|
|
195
|
+
sandbox: input.sandbox,
|
|
196
|
+
sessionId: input.sessionId,
|
|
197
|
+
cmdId: input.cmdId,
|
|
198
|
+
exitCodePath: input.exitCodePath,
|
|
199
|
+
});
|
|
200
|
+
const oomIndicated =
|
|
201
|
+
diagnostic.exitCode === 137 || diagnostic.exitCodeFile === 137;
|
|
202
|
+
throw new DaytonaRunnerInitializationError(
|
|
203
|
+
`RUNTIME_SANDBOX_START_FAILED: Daytona accepted detached command ${input.cmdId} in sandbox ${input.sandbox.id}, but the play runner did not initialize within ${timeoutMs}ms. Marker diagnosis: ${markerDiagnosis}. Daytona diagnosis: ${JSON.stringify(diagnostic)}. OOM is ${oomIndicated ? 'indicated by exit code 137' : 'not confirmed by the available Daytona evidence'}. The play was stopped and was not retried.`,
|
|
204
|
+
);
|
|
205
|
+
}
|
|
@@ -33,7 +33,8 @@ import {
|
|
|
33
33
|
} from './daytona-lifecycle';
|
|
34
34
|
import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
|
|
35
35
|
import {
|
|
36
|
-
|
|
36
|
+
confirmDetachedDaytonaRunnerReady,
|
|
37
|
+
DaytonaRunnerInitializationError,
|
|
37
38
|
startDetachedDaytonaCommand,
|
|
38
39
|
} from './daytona-session-execution';
|
|
39
40
|
|
|
@@ -45,8 +46,6 @@ const DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS = 2;
|
|
|
45
46
|
const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
|
|
46
47
|
const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
|
|
47
48
|
const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
|
|
48
|
-
const DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS = 15_000;
|
|
49
|
-
const DAYTONA_STARTUP_TRACE_ENV = 'DEEPLINE_DAYTONA_STARTUP_TRACE';
|
|
50
49
|
|
|
51
50
|
const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
|
|
52
51
|
/\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
|
|
@@ -397,6 +396,7 @@ export function retryableRuntimeInitializationFailureReason(
|
|
|
397
396
|
}
|
|
398
397
|
|
|
399
398
|
function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
|
|
399
|
+
if (error instanceof DaytonaRunnerInitializationError) return false;
|
|
400
400
|
const message = error instanceof Error ? error.message : String(error);
|
|
401
401
|
if (!message || /cancelled|aborted/i.test(message)) {
|
|
402
402
|
return false;
|
|
@@ -715,12 +715,11 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
715
715
|
});
|
|
716
716
|
// Push execution (B2-final, park-and-wake): start the runner command
|
|
717
717
|
// DETACHED via a Daytona session and return a `detached_runner`
|
|
718
|
-
// suspension
|
|
719
|
-
//
|
|
720
|
-
//
|
|
721
|
-
//
|
|
722
|
-
//
|
|
723
|
-
// wake leg owns finalize + sandbox GC.
|
|
718
|
+
// suspension after the runner writes its one-time ready marker. The
|
|
719
|
+
// marker is the handoff proof: a Daytona cmdId alone does not prove
|
|
720
|
+
// the sandbox process initialized. After that bounded startup check,
|
|
721
|
+
// the worker parks (releasing its claim and slot) and wakes only on
|
|
722
|
+
// the runner's pushed terminal or the ceiling timeout.
|
|
724
723
|
const sessionId = `deepline-play-${randomUUID()}`;
|
|
725
724
|
let start: Awaited<ReturnType<typeof startDetachedDaytonaCommand>>;
|
|
726
725
|
try {
|
|
@@ -744,6 +743,16 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
744
743
|
// classification below decide fresh-sandbox retry vs loud failure.
|
|
745
744
|
throw error;
|
|
746
745
|
}
|
|
746
|
+
await Promise.race([
|
|
747
|
+
confirmDetachedDaytonaRunnerReady({
|
|
748
|
+
sandbox,
|
|
749
|
+
sessionId: start.sessionId,
|
|
750
|
+
cmdId: start.cmdId,
|
|
751
|
+
readyPath: stagedPayload.readyPath,
|
|
752
|
+
exitCodePath: stagedPayload.exitCodePath,
|
|
753
|
+
}),
|
|
754
|
+
cancellationPromise,
|
|
755
|
+
]);
|
|
747
756
|
const runnerAttempt = Math.max(
|
|
748
757
|
0,
|
|
749
758
|
Math.floor(config.context.runAttempt ?? 0),
|
|
@@ -758,52 +767,6 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
758
767
|
exitCodePath: stagedPayload.exitCodePath,
|
|
759
768
|
elapsedMs: Date.now() - startedAt,
|
|
760
769
|
});
|
|
761
|
-
// Daytona acknowledges an async command with only a cmdId. It does
|
|
762
|
-
// not expose a queued/running state, and the backend parks before the
|
|
763
|
-
// runner contacts the runtime sheet. During startup investigations,
|
|
764
|
-
// retain a bounded snapshot while the sandbox is still alive; the
|
|
765
|
-
// normal cancellation path otherwise deletes the only evidence.
|
|
766
|
-
// This is opt-in because it adds Daytona control-plane calls to every
|
|
767
|
-
// run. The diagnostic excludes command lines and environment values.
|
|
768
|
-
if (process.env[DAYTONA_STARTUP_TRACE_ENV] === '1') {
|
|
769
|
-
const diagnosticTimer = setTimeout(() => {
|
|
770
|
-
void inspectDetachedDaytonaStartup({
|
|
771
|
-
sandbox,
|
|
772
|
-
sessionId: start.sessionId,
|
|
773
|
-
cmdId: start.cmdId,
|
|
774
|
-
exitCodePath: stagedPayload.exitCodePath,
|
|
775
|
-
}).then(
|
|
776
|
-
(diagnostic) =>
|
|
777
|
-
emitDaytonaStage(
|
|
778
|
-
callbacks,
|
|
779
|
-
config.context,
|
|
780
|
-
'execute:startup_diagnostic',
|
|
781
|
-
{
|
|
782
|
-
sandboxId: sandbox.id,
|
|
783
|
-
sessionId: start.sessionId,
|
|
784
|
-
cmdId: start.cmdId,
|
|
785
|
-
diagnostic,
|
|
786
|
-
elapsedMs: Date.now() - startedAt,
|
|
787
|
-
},
|
|
788
|
-
),
|
|
789
|
-
(error: unknown) =>
|
|
790
|
-
emitDaytonaStage(
|
|
791
|
-
callbacks,
|
|
792
|
-
config.context,
|
|
793
|
-
'execute:startup_diagnostic_failed',
|
|
794
|
-
{
|
|
795
|
-
sandboxId: sandbox.id,
|
|
796
|
-
sessionId: start.sessionId,
|
|
797
|
-
cmdId: start.cmdId,
|
|
798
|
-
errorType:
|
|
799
|
-
error instanceof Error ? error.name : 'unknown',
|
|
800
|
-
elapsedMs: Date.now() - startedAt,
|
|
801
|
-
},
|
|
802
|
-
),
|
|
803
|
-
);
|
|
804
|
-
}, DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS);
|
|
805
|
-
diagnosticTimer.unref?.();
|
|
806
|
-
}
|
|
807
770
|
return {
|
|
808
771
|
status: 'suspended',
|
|
809
772
|
suspension: {
|
|
@@ -95,7 +95,7 @@ import {
|
|
|
95
95
|
PLAY_RUNTIME_CONTRACT,
|
|
96
96
|
PLAY_RUNTIME_CONTRACT_HEADER,
|
|
97
97
|
} from './runtime-contract';
|
|
98
|
-
import {
|
|
98
|
+
import { PLAY_RUNTIME_API_CURRENT_PATH } from './runtime-api-paths';
|
|
99
99
|
import {
|
|
100
100
|
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
101
101
|
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
@@ -298,6 +298,7 @@ const RUNTIME_WORK_RECEIPT_LEASE_COLUMNS = [
|
|
|
298
298
|
'lease_expires_at',
|
|
299
299
|
] as const;
|
|
300
300
|
const RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN = 'failure_kind';
|
|
301
|
+
const RUNTIME_WORK_RECEIPT_KEY_INDEX = 'idx_deepline_step_receipts_k_unique';
|
|
301
302
|
const RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS = [
|
|
302
303
|
...RUNTIME_WORK_RECEIPT_LEASE_COLUMNS,
|
|
303
304
|
RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN,
|
|
@@ -504,7 +505,7 @@ function resolveRuntimeApiUrl(context: RuntimeApiContext): string {
|
|
|
504
505
|
if (!baseUrl) {
|
|
505
506
|
throw new Error('Runner runtime API requires a baseUrl.');
|
|
506
507
|
}
|
|
507
|
-
return `${baseUrl.replace(/\/$/, '')}${
|
|
508
|
+
return `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_CURRENT_PATH}`;
|
|
508
509
|
}
|
|
509
510
|
|
|
510
511
|
function resolveRuntimeApiHeaders(
|
|
@@ -586,11 +587,13 @@ async function postRuntimeApi<TResponse>(
|
|
|
586
587
|
? parsed.retry_after_ms
|
|
587
588
|
: RUNTIME_API_DEFAULT_RETRY_AFTER_MS;
|
|
588
589
|
const details =
|
|
589
|
-
typeof parsed?.
|
|
590
|
-
? parsed.
|
|
591
|
-
: typeof parsed?.
|
|
592
|
-
? parsed.
|
|
593
|
-
:
|
|
590
|
+
typeof parsed?.detail === 'string'
|
|
591
|
+
? parsed.detail
|
|
592
|
+
: typeof parsed?.details === 'string'
|
|
593
|
+
? parsed.details
|
|
594
|
+
: typeof parsed?.debug_error === 'string'
|
|
595
|
+
? parsed.debug_error
|
|
596
|
+
: null;
|
|
594
597
|
const shouldRetryRuntimeResponse =
|
|
595
598
|
(response.status === 503 &&
|
|
596
599
|
parsed?.code === 'ingestion_plane_not_ready') ||
|
|
@@ -2040,6 +2043,22 @@ function isMissingRuntimeWorkReceiptSelfHealColumnError(
|
|
|
2040
2043
|
);
|
|
2041
2044
|
}
|
|
2042
2045
|
|
|
2046
|
+
function isMissingRuntimeWorkReceiptKeyConstraintError(
|
|
2047
|
+
error: unknown,
|
|
2048
|
+
): boolean {
|
|
2049
|
+
if (!error || typeof error !== 'object') {
|
|
2050
|
+
return false;
|
|
2051
|
+
}
|
|
2052
|
+
const code = 'code' in error ? String(error.code) : '';
|
|
2053
|
+
const message = 'message' in error ? String(error.message) : '';
|
|
2054
|
+
return (
|
|
2055
|
+
code === '42P10' ||
|
|
2056
|
+
/no unique or exclusion constraint matching the ON CONFLICT specification/i.test(
|
|
2057
|
+
message,
|
|
2058
|
+
)
|
|
2059
|
+
);
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2043
2062
|
function runtimeWorkReceiptEnsureCacheKey(
|
|
2044
2063
|
session: RuntimePostgresSession,
|
|
2045
2064
|
): string {
|
|
@@ -2119,22 +2138,29 @@ async function ensureRuntimeWorkReceiptTable(
|
|
|
2119
2138
|
session,
|
|
2120
2139
|
client,
|
|
2121
2140
|
);
|
|
2122
|
-
if (missingColumns.length
|
|
2141
|
+
if (missingColumns.length > 0) {
|
|
2142
|
+
await client.query(`
|
|
2143
|
+
ALTER TABLE ${workReceiptTable(session)}
|
|
2144
|
+
${missingColumns
|
|
2145
|
+
.map((column) => {
|
|
2146
|
+
const type =
|
|
2147
|
+
column === 'lease_expires_at'
|
|
2148
|
+
? 'timestamptz'
|
|
2149
|
+
: column === 'lease_owner_attempt'
|
|
2150
|
+
? 'integer'
|
|
2151
|
+
: column === RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN
|
|
2152
|
+
? 'smallint NOT NULL DEFAULT 0'
|
|
2153
|
+
: 'text';
|
|
2154
|
+
return `ADD COLUMN IF NOT EXISTS ${column} ${type}`;
|
|
2155
|
+
})
|
|
2156
|
+
.join(',\n ')}
|
|
2157
|
+
`);
|
|
2158
|
+
}
|
|
2123
2159
|
await client.query(`
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
column === 'lease_expires_at'
|
|
2129
|
-
? 'timestamptz'
|
|
2130
|
-
: column === 'lease_owner_attempt'
|
|
2131
|
-
? 'integer'
|
|
2132
|
-
: column === RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN
|
|
2133
|
-
? 'smallint NOT NULL DEFAULT 0'
|
|
2134
|
-
: 'text';
|
|
2135
|
-
return `ADD COLUMN IF NOT EXISTS ${column} ${type}`;
|
|
2136
|
-
})
|
|
2137
|
-
.join(',\n ')}
|
|
2160
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${quoteIdentifier(
|
|
2161
|
+
RUNTIME_WORK_RECEIPT_KEY_INDEX,
|
|
2162
|
+
)}
|
|
2163
|
+
ON ${workReceiptTable(session)} (k)
|
|
2138
2164
|
`);
|
|
2139
2165
|
})
|
|
2140
2166
|
.then(() => undefined);
|
|
@@ -2238,7 +2264,8 @@ async function withRuntimeWorkReceiptClient<T>(
|
|
|
2238
2264
|
} catch (error) {
|
|
2239
2265
|
if (
|
|
2240
2266
|
isMissingRelationError(error) ||
|
|
2241
|
-
isMissingRuntimeWorkReceiptSelfHealColumnError(error)
|
|
2267
|
+
isMissingRuntimeWorkReceiptSelfHealColumnError(error) ||
|
|
2268
|
+
isMissingRuntimeWorkReceiptKeyConstraintError(error)
|
|
2242
2269
|
) {
|
|
2243
2270
|
runtimeWorkReceiptEnsureCache.delete(
|
|
2244
2271
|
runtimeWorkReceiptEnsureCacheKey(session),
|
|
@@ -4636,6 +4663,7 @@ export async function completeRuntimeWorkReceipt(
|
|
|
4636
4663
|
runAttempt?: number | null;
|
|
4637
4664
|
leaseId?: string | null;
|
|
4638
4665
|
output: unknown;
|
|
4666
|
+
returnOutput?: boolean;
|
|
4639
4667
|
},
|
|
4640
4668
|
): Promise<WorkReceipt | null> {
|
|
4641
4669
|
if (
|
|
@@ -4741,7 +4769,7 @@ export async function completeRuntimeWorkReceipt(
|
|
|
4741
4769
|
})}
|
|
4742
4770
|
RETURNING convert_from(k, 'UTF8') AS k,
|
|
4743
4771
|
status,
|
|
4744
|
-
output,
|
|
4772
|
+
CASE WHEN $7::boolean THEN output ELSE NULL::jsonb END AS output,
|
|
4745
4773
|
error,
|
|
4746
4774
|
failure_kind,
|
|
4747
4775
|
run_id,
|
|
@@ -4754,7 +4782,7 @@ export async function completeRuntimeWorkReceipt(
|
|
|
4754
4782
|
replay AS (
|
|
4755
4783
|
SELECT convert_from(k, 'UTF8') AS k,
|
|
4756
4784
|
status,
|
|
4757
|
-
output,
|
|
4785
|
+
CASE WHEN $7::boolean THEN output ELSE NULL::jsonb END AS output,
|
|
4758
4786
|
error,
|
|
4759
4787
|
failure_kind,
|
|
4760
4788
|
run_id,
|
|
@@ -4783,6 +4811,7 @@ export async function completeRuntimeWorkReceipt(
|
|
|
4783
4811
|
input.runId,
|
|
4784
4812
|
leaseId,
|
|
4785
4813
|
runAttempt,
|
|
4814
|
+
input.returnOutput !== false,
|
|
4786
4815
|
],
|
|
4787
4816
|
);
|
|
4788
4817
|
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
@@ -4801,6 +4830,7 @@ export async function completeRuntimeWorkReceipts(
|
|
|
4801
4830
|
leaseId?: string | null;
|
|
4802
4831
|
runAttempt?: number | null;
|
|
4803
4832
|
}>;
|
|
4833
|
+
returnOutput?: boolean;
|
|
4804
4834
|
},
|
|
4805
4835
|
): Promise<Array<WorkReceipt | null>> {
|
|
4806
4836
|
const receipts = input.receipts.filter((receipt) => receipt.key.trim());
|
|
@@ -4819,6 +4849,7 @@ export async function completeRuntimeWorkReceipts(
|
|
|
4819
4849
|
await completeRuntimeWorkReceipts(context, {
|
|
4820
4850
|
playName: input.playName,
|
|
4821
4851
|
receipts: siblingReceipts,
|
|
4852
|
+
returnOutput: input.returnOutput,
|
|
4822
4853
|
});
|
|
4823
4854
|
} catch (error) {
|
|
4824
4855
|
siblingError = error;
|
|
@@ -4938,7 +4969,7 @@ export async function completeRuntimeWorkReceipts(
|
|
|
4938
4969
|
})}
|
|
4939
4970
|
RETURNING target.k,
|
|
4940
4971
|
target.status,
|
|
4941
|
-
target.output,
|
|
4972
|
+
CASE WHEN $7::boolean THEN target.output ELSE NULL::jsonb END AS output,
|
|
4942
4973
|
target.error,
|
|
4943
4974
|
target.failure_kind,
|
|
4944
4975
|
target.run_id,
|
|
@@ -4952,7 +4983,7 @@ export async function completeRuntimeWorkReceipts(
|
|
|
4952
4983
|
replay AS (
|
|
4953
4984
|
SELECT target.k,
|
|
4954
4985
|
target.status,
|
|
4955
|
-
target.output,
|
|
4986
|
+
CASE WHEN $7::boolean THEN target.output ELSE NULL::jsonb END AS output,
|
|
4956
4987
|
target.error,
|
|
4957
4988
|
target.failure_kind,
|
|
4958
4989
|
target.run_id,
|
|
@@ -5020,6 +5051,7 @@ export async function completeRuntimeWorkReceipts(
|
|
|
5020
5051
|
),
|
|
5021
5052
|
),
|
|
5022
5053
|
RECEIPT_STATUS_COMPLETED,
|
|
5054
|
+
input.returnOutput !== false,
|
|
5023
5055
|
],
|
|
5024
5056
|
);
|
|
5025
5057
|
return rows.map((row) =>
|
|
@@ -1,8 +1,15 @@
|
|
|
1
|
-
export const PLAY_RUNTIME_CONTRACT =
|
|
1
|
+
export const PLAY_RUNTIME_CONTRACT = 3;
|
|
2
2
|
export const PLAY_RUNTIME_CONTRACT_MIN_SUPPORTED = 1;
|
|
3
3
|
export const PLAY_RUNTIME_CONTRACT_HEADER = 'x-deepline-runtime-contract';
|
|
4
4
|
|
|
5
5
|
export const RUNTIME_CONTRACT_CHANGELOG = [
|
|
6
|
+
{
|
|
7
|
+
contract: 3,
|
|
8
|
+
changed:
|
|
9
|
+
'Runtime receipt completion responses are compact acknowledgements; receipt reads remain the payload replay path.',
|
|
10
|
+
compatOwner: 'runtime',
|
|
11
|
+
compatRemoval: 'contract 2 support window closure',
|
|
12
|
+
},
|
|
6
13
|
{
|
|
7
14
|
contract: 2,
|
|
8
15
|
changed:
|
|
@@ -12,6 +19,15 @@ export const RUNTIME_CONTRACT_CHANGELOG = [
|
|
|
12
19
|
},
|
|
13
20
|
] as const;
|
|
14
21
|
|
|
22
|
+
export function runtimeReceiptCompletionResponse<
|
|
23
|
+
T extends { output?: unknown },
|
|
24
|
+
>(runtimeContract: number | null | undefined, receipt: T | null): T | null {
|
|
25
|
+
if (!receipt || (runtimeContract ?? 1) < 3) return receipt;
|
|
26
|
+
const { output, ...ack } = receipt;
|
|
27
|
+
void output;
|
|
28
|
+
return ack as T;
|
|
29
|
+
}
|
|
30
|
+
|
|
15
31
|
export type RuntimeContractResolution =
|
|
16
32
|
| { ok: true; contract: number }
|
|
17
33
|
| {
|
|
@@ -4,8 +4,9 @@
|
|
|
4
4
|
* One of three pluggable axes (alongside runner-backends and dedup-backends).
|
|
5
5
|
* Selected per-run via PlayExecutionProfile.
|
|
6
6
|
*
|
|
7
|
-
* Absurd is the
|
|
8
|
-
*
|
|
7
|
+
* Absurd is the enabled production scheduler. The Cloudflare Workflows
|
|
8
|
+
* adapter is retained for historical run interpretation while workers_edge
|
|
9
|
+
* launch admission is disabled.
|
|
9
10
|
*
|
|
10
11
|
* Customer plays are unaffected — this is purely the orchestration layer.
|
|
11
12
|
*/
|
|
@@ -5,10 +5,18 @@ const JWT_RE =
|
|
|
5
5
|
/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g;
|
|
6
6
|
const PRIVATE_KEY_RE =
|
|
7
7
|
/-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g;
|
|
8
|
+
// Only redact self-identifying credential prefixes here. Generic words such as
|
|
9
|
+
// `key`, `token`, and `secret` are common in harmless play/run slugs (for
|
|
10
|
+
// example `absurd-key-rotation-check`) and need an assignment context before
|
|
11
|
+
// they are sensitive; HIGH_ENTROPY_ASSIGNMENT_RE handles that case below.
|
|
8
12
|
const COMMON_SECRET_RE =
|
|
9
|
-
/\b(?:sk|pk|rk|pat|ghp|github_pat|xox[baprs]
|
|
13
|
+
/\b(?:sk|pk|rk|pat|ghp|github_pat|xox[baprs])[A-Za-z0-9_./+=:-]{12,}\b/gi;
|
|
14
|
+
const GENERIC_PREFIXED_SECRET_RE =
|
|
15
|
+
/\b(?:key|token|secret|api[_-]?key)_[A-Za-z0-9_./+=:-]{12,}\b/gi;
|
|
10
16
|
const URL_SECRET_VALUE_RE =
|
|
11
17
|
/\b(?:sk|pk|rk|pat|ghp|github_pat|xox)[A-Za-z0-9_./+=:-]{12,}\b/i;
|
|
18
|
+
const GENERIC_SECRET_ASSIGNMENT_RE =
|
|
19
|
+
/\b(?:api[_-]?key|token|secret|password)\b\s*[:=]\s*["']?[^"',\s]{12,}["']?/gi;
|
|
12
20
|
const HIGH_ENTROPY_ASSIGNMENT_RE =
|
|
13
21
|
/\b(?:api[_-]?key|token|secret|password|authorization|access[_-]?token|refresh[_-]?token)\b\s*[:=]\s*["']?[^"',\s]{16,}["']?/gi;
|
|
14
22
|
const SENSITIVE_URL_PARAM_RE =
|
|
@@ -39,6 +47,8 @@ export function redactSecretLikeString(value: string): string {
|
|
|
39
47
|
.replace(BEARER_TOKEN_RE, `Bearer ${SECRET_REDACTION_PLACEHOLDER}`)
|
|
40
48
|
.replace(JWT_RE, SECRET_REDACTION_PLACEHOLDER)
|
|
41
49
|
.replace(COMMON_SECRET_RE, SECRET_REDACTION_PLACEHOLDER)
|
|
50
|
+
.replace(GENERIC_PREFIXED_SECRET_RE, SECRET_REDACTION_PLACEHOLDER)
|
|
51
|
+
.replace(GENERIC_SECRET_ASSIGNMENT_RE, SECRET_REDACTION_PLACEHOLDER)
|
|
42
52
|
.replace(HIGH_ENTROPY_ASSIGNMENT_RE, (match) => {
|
|
43
53
|
const separator = match.includes('=') ? '=' : ':';
|
|
44
54
|
const [key] = match.split(separator, 1);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export function transientServiceUnavailableDetail(
|
|
2
|
+
error: unknown,
|
|
3
|
+
): string | null {
|
|
4
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
5
|
+
return /ServiceUnavailable|Service temporarily unavailable|temporarily unavailable \(Code:\s*\d+\)/i.test(
|
|
6
|
+
detail,
|
|
7
|
+
)
|
|
8
|
+
? detail
|
|
9
|
+
: null;
|
|
10
|
+
}
|