deepline 0.1.293 → 0.1.294
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +2 -151
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +24 -5
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +91 -36
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +10 -48
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +101 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +11 -1
- package/dist/cli/index.js +2 -1
- package/dist/cli/index.mjs +2 -1
- package/dist/index.js +2 -1
- package/dist/index.mjs +2 -1
- package/package.json +1 -1
|
@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
|
|
|
155
155
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
156
156
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
157
157
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
158
|
-
version: '0.1.
|
|
158
|
+
version: '0.1.294',
|
|
159
159
|
contracts: {
|
|
160
160
|
api: {
|
|
161
161
|
name: 'sdk-http-api',
|
|
@@ -24,7 +24,6 @@ import type {
|
|
|
24
24
|
PlayVisualNodeProgressMap,
|
|
25
25
|
} from '@shared_libs/play-runtime/worker-api-types';
|
|
26
26
|
import type { RuntimeStepReceipt } from '@shared_libs/play-runtime/ctx-types';
|
|
27
|
-
import type { WorkReceiptFailureKind } from '@shared_libs/play-runtime/work-receipts';
|
|
28
27
|
import type {
|
|
29
28
|
CreateDbSessionRequest,
|
|
30
29
|
CreateDbSessionResponse,
|
|
@@ -45,6 +44,7 @@ import {
|
|
|
45
44
|
import { PLAY_RUNTIME_API_COMPAT_PATH } from '@shared_libs/play-runtime/runtime-api-paths';
|
|
46
45
|
import { PLAY_RUNTIME_TEST_FAULT_HEADER } from '@shared_libs/play-runtime/test-runtime-seams';
|
|
47
46
|
import { vercelProtectionBypassHeaders } from '@shared_libs/play-runtime/vercel-protection';
|
|
47
|
+
import type { RuntimeReceiptAction } from '@shared_libs/play-runtime/runtime-actions';
|
|
48
48
|
|
|
49
49
|
export type StoredPlayArtifactPayload = {
|
|
50
50
|
sourceCode: string;
|
|
@@ -204,156 +204,7 @@ type RuntimeApiRequest =
|
|
|
204
204
|
| ({
|
|
205
205
|
action: 'governor_budget_charge';
|
|
206
206
|
} & PlayRunnerBudgetChargeInput)
|
|
207
|
-
|
|
|
208
|
-
action: 'get_runtime_step_receipt';
|
|
209
|
-
playName: string;
|
|
210
|
-
runId: string;
|
|
211
|
-
key: string;
|
|
212
|
-
}
|
|
213
|
-
| {
|
|
214
|
-
action: 'acquire_runtime_receipt_execution_lock';
|
|
215
|
-
playName: string;
|
|
216
|
-
runId: string;
|
|
217
|
-
key: string;
|
|
218
|
-
ownerExecutionId: string;
|
|
219
|
-
ttlMs: number;
|
|
220
|
-
}
|
|
221
|
-
| {
|
|
222
|
-
action: 'release_runtime_receipt_execution_lock';
|
|
223
|
-
playName: string;
|
|
224
|
-
runId: string;
|
|
225
|
-
key: string;
|
|
226
|
-
ownerExecutionId: string;
|
|
227
|
-
}
|
|
228
|
-
| {
|
|
229
|
-
action: 'claim_runtime_step_receipt';
|
|
230
|
-
playName: string;
|
|
231
|
-
runId: string;
|
|
232
|
-
key: string;
|
|
233
|
-
leaseId?: string;
|
|
234
|
-
runAttempt?: number | null;
|
|
235
|
-
leaseAware?: boolean;
|
|
236
|
-
reclaimRunning?: boolean;
|
|
237
|
-
forceRefresh?: boolean;
|
|
238
|
-
forceFailedRefresh?: boolean;
|
|
239
|
-
}
|
|
240
|
-
| {
|
|
241
|
-
action: 'complete_runtime_step_receipt';
|
|
242
|
-
playName: string;
|
|
243
|
-
runId: string;
|
|
244
|
-
key: string;
|
|
245
|
-
runAttempt?: number | null;
|
|
246
|
-
leaseId?: string | null;
|
|
247
|
-
output: unknown;
|
|
248
|
-
}
|
|
249
|
-
| {
|
|
250
|
-
action: 'release_runtime_step_receipt';
|
|
251
|
-
playName: string;
|
|
252
|
-
runId: string;
|
|
253
|
-
key: string;
|
|
254
|
-
runAttempt?: number | null;
|
|
255
|
-
leaseId?: string | null;
|
|
256
|
-
}
|
|
257
|
-
| {
|
|
258
|
-
action: 'fail_runtime_step_receipt';
|
|
259
|
-
playName: string;
|
|
260
|
-
runId: string;
|
|
261
|
-
key: string;
|
|
262
|
-
runAttempt?: number | null;
|
|
263
|
-
leaseId?: string | null;
|
|
264
|
-
error: string;
|
|
265
|
-
failureKind?: WorkReceiptFailureKind | null;
|
|
266
|
-
}
|
|
267
|
-
| {
|
|
268
|
-
action: 'skip_runtime_step_receipt';
|
|
269
|
-
playName: string;
|
|
270
|
-
runId: string;
|
|
271
|
-
key: string;
|
|
272
|
-
runAttempt?: number | null;
|
|
273
|
-
leaseId?: string | null;
|
|
274
|
-
output?: unknown;
|
|
275
|
-
}
|
|
276
|
-
| {
|
|
277
|
-
action: 'get_runtime_step_receipts';
|
|
278
|
-
playName: string;
|
|
279
|
-
runId: string;
|
|
280
|
-
keys: string[];
|
|
281
|
-
}
|
|
282
|
-
| {
|
|
283
|
-
action: 'claim_runtime_step_receipts';
|
|
284
|
-
playName: string;
|
|
285
|
-
runId: string;
|
|
286
|
-
keys: string[];
|
|
287
|
-
leaseIds?: string[];
|
|
288
|
-
runAttempt?: number | null;
|
|
289
|
-
leaseAware?: boolean;
|
|
290
|
-
reclaimRunning?: boolean;
|
|
291
|
-
forceRefresh?: boolean;
|
|
292
|
-
forceFailedRefresh?: boolean;
|
|
293
|
-
}
|
|
294
|
-
| {
|
|
295
|
-
action: 'mark_runtime_step_receipt_running';
|
|
296
|
-
playName: string;
|
|
297
|
-
runId: string;
|
|
298
|
-
key: string;
|
|
299
|
-
runAttempt?: number | null;
|
|
300
|
-
leaseId?: string | null;
|
|
301
|
-
}
|
|
302
|
-
| {
|
|
303
|
-
action: 'mark_runtime_step_receipts_running';
|
|
304
|
-
playName: string;
|
|
305
|
-
runId: string;
|
|
306
|
-
receipts: Array<{
|
|
307
|
-
key: string;
|
|
308
|
-
runId: string;
|
|
309
|
-
runAttempt?: number | null;
|
|
310
|
-
leaseId?: string | null;
|
|
311
|
-
}>;
|
|
312
|
-
}
|
|
313
|
-
| {
|
|
314
|
-
action: 'mark_runtime_step_receipts_queued';
|
|
315
|
-
playName: string;
|
|
316
|
-
runId: string;
|
|
317
|
-
receipts: Array<{
|
|
318
|
-
key: string;
|
|
319
|
-
runId: string;
|
|
320
|
-
runAttempt?: number | null;
|
|
321
|
-
leaseId?: string | null;
|
|
322
|
-
}>;
|
|
323
|
-
}
|
|
324
|
-
| {
|
|
325
|
-
action: 'complete_runtime_step_receipts';
|
|
326
|
-
playName: string;
|
|
327
|
-
runId: string;
|
|
328
|
-
receipts: Array<{
|
|
329
|
-
key: string;
|
|
330
|
-
runId: string;
|
|
331
|
-
runAttempt?: number | null;
|
|
332
|
-
leaseId?: string | null;
|
|
333
|
-
output: unknown;
|
|
334
|
-
}>;
|
|
335
|
-
}
|
|
336
|
-
| {
|
|
337
|
-
action: 'fail_runtime_step_receipts';
|
|
338
|
-
playName: string;
|
|
339
|
-
runId: string;
|
|
340
|
-
receipts: Array<{
|
|
341
|
-
key: string;
|
|
342
|
-
runId: string;
|
|
343
|
-
runAttempt?: number | null;
|
|
344
|
-
leaseId?: string | null;
|
|
345
|
-
error: string;
|
|
346
|
-
failureKind?: WorkReceiptFailureKind | null;
|
|
347
|
-
}>;
|
|
348
|
-
}
|
|
349
|
-
| {
|
|
350
|
-
action: 'heartbeat_runtime_step_receipts';
|
|
351
|
-
playName: string;
|
|
352
|
-
runId: string;
|
|
353
|
-
runAttempt?: number | null;
|
|
354
|
-
leaseId: string;
|
|
355
|
-
keys: string[];
|
|
356
|
-
};
|
|
207
|
+
| RuntimeReceiptAction;
|
|
357
208
|
|
|
358
209
|
export type WorkerRuntimeApiContext = {
|
|
359
210
|
baseUrl: string;
|
|
@@ -2,6 +2,7 @@ export type GatewayPostgresLane = 'ordinary' | 'commit';
|
|
|
2
2
|
|
|
3
3
|
export const GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE = 8;
|
|
4
4
|
export const GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE = 10;
|
|
5
|
+
export const GATEWAY_POSTGRES_MAX_CONSECUTIVE_COMMIT_PRIORITY_GRANTS = 8;
|
|
5
6
|
|
|
6
7
|
export type GatewayPostgresAdmissionSnapshot = {
|
|
7
8
|
active: Record<GatewayPostgresLane, number> & { total: number };
|
|
@@ -35,6 +36,7 @@ export class GatewayPostgresAdmission {
|
|
|
35
36
|
#commitActive = 0;
|
|
36
37
|
#cancelled = 0;
|
|
37
38
|
#commitPriorityGrants = 0;
|
|
39
|
+
#consecutiveCommitPriorityGrants = 0;
|
|
38
40
|
|
|
39
41
|
async acquire(
|
|
40
42
|
lane: GatewayPostgresLane,
|
|
@@ -115,21 +117,38 @@ export class GatewayPostgresAdmission {
|
|
|
115
117
|
|
|
116
118
|
#drain(): void {
|
|
117
119
|
while (this.#activeTotal() < GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE) {
|
|
120
|
+
const ordinaryCanRun =
|
|
121
|
+
this.#waiting.ordinary.length > 0 &&
|
|
122
|
+
this.#ordinaryActive < GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE;
|
|
123
|
+
if (
|
|
124
|
+
ordinaryCanRun &&
|
|
125
|
+
this.#waiting.commit.length > 0 &&
|
|
126
|
+
this.#consecutiveCommitPriorityGrants >=
|
|
127
|
+
GATEWAY_POSTGRES_MAX_CONSECUTIVE_COMMIT_PRIORITY_GRANTS
|
|
128
|
+
) {
|
|
129
|
+
this.#consecutiveCommitPriorityGrants = 0;
|
|
130
|
+
this.#resolve(this.#waiting.ordinary.shift()!);
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
118
133
|
const commit = this.#waiting.commit.shift();
|
|
119
134
|
if (commit) {
|
|
120
|
-
if (
|
|
135
|
+
if (ordinaryCanRun) {
|
|
121
136
|
this.#commitPriorityGrants += 1;
|
|
137
|
+
this.#consecutiveCommitPriorityGrants += 1;
|
|
138
|
+
} else {
|
|
139
|
+
this.#consecutiveCommitPriorityGrants = 0;
|
|
122
140
|
}
|
|
123
141
|
this.#resolve(commit);
|
|
124
142
|
continue;
|
|
125
143
|
}
|
|
126
|
-
if (
|
|
127
|
-
this.#
|
|
128
|
-
this.#waiting.ordinary.length > 0
|
|
129
|
-
) {
|
|
144
|
+
if (ordinaryCanRun) {
|
|
145
|
+
this.#consecutiveCommitPriorityGrants = 0;
|
|
130
146
|
this.#resolve(this.#waiting.ordinary.shift()!);
|
|
131
147
|
continue;
|
|
132
148
|
}
|
|
149
|
+
if (this.#waiting.ordinary.length === 0) {
|
|
150
|
+
this.#consecutiveCommitPriorityGrants = 0;
|
|
151
|
+
}
|
|
133
152
|
return;
|
|
134
153
|
}
|
|
135
154
|
}
|
|
@@ -41,6 +41,12 @@ export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
|
|
|
41
41
|
* is required to resume execution.
|
|
42
42
|
*/
|
|
43
43
|
export const RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
44
|
+
/**
|
|
45
|
+
* Receipt commands are measured before the runtime client adds its request
|
|
46
|
+
* envelope. Keep a full 4 MiB below the gateway body ceiling so one legal
|
|
47
|
+
* 10 MiB receipt fits while compatible fat receipts split before transport.
|
|
48
|
+
*/
|
|
49
|
+
export const RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
|
|
44
50
|
/**
|
|
45
51
|
* The terminal `result` line is the crash-recovery record in Daytona stdout.
|
|
46
52
|
* Once that line is written, retries may still emit runner-owned diagnostics.
|
|
@@ -111,9 +111,26 @@ export async function inspectDetachedDaytonaStartup(input: {
|
|
|
111
111
|
* verification.
|
|
112
112
|
*/
|
|
113
113
|
|
|
114
|
-
export type
|
|
115
|
-
sessionId: string
|
|
116
|
-
|
|
114
|
+
export type DetachedDaytonaRunnerState =
|
|
115
|
+
| { phase: 'preparing'; sessionId: string }
|
|
116
|
+
| {
|
|
117
|
+
phase: 'command_accepted';
|
|
118
|
+
sessionId: string;
|
|
119
|
+
cmdId: string;
|
|
120
|
+
baselineHeartbeatAt: string | null;
|
|
121
|
+
}
|
|
122
|
+
| {
|
|
123
|
+
phase: 'ready';
|
|
124
|
+
sessionId: string;
|
|
125
|
+
cmdId: string;
|
|
126
|
+
baselineHeartbeatAt: string | null;
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
export type DetachedDaytonaRunnerSupervisor = {
|
|
130
|
+
start(input: {
|
|
131
|
+
command: string;
|
|
132
|
+
exitCodePath: string;
|
|
133
|
+
}): Promise<Extract<DetachedDaytonaRunnerState, { phase: 'ready' }>>;
|
|
117
134
|
};
|
|
118
135
|
|
|
119
136
|
// The durable heartbeat proves runner-process liveness plus gateway reachability.
|
|
@@ -235,44 +252,82 @@ export class DaytonaRunnerInitializationError extends Error {
|
|
|
235
252
|
}
|
|
236
253
|
|
|
237
254
|
/**
|
|
238
|
-
*
|
|
239
|
-
*
|
|
255
|
+
* Begin the independent control-plane work needed by a detached runner while
|
|
256
|
+
* its payload uploads. `start` crosses the customer-code side-effect boundary:
|
|
257
|
+
* it accepts one command, then returns only after the scheduler has observed
|
|
258
|
+
* durable runner liveness.
|
|
240
259
|
*/
|
|
241
|
-
export
|
|
260
|
+
export function prepareDetachedDaytonaRunner(input: {
|
|
242
261
|
sandbox: DaytonaSandbox;
|
|
243
262
|
sessionId: string;
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
263
|
+
readiness: DetachedRunnerReadinessPort;
|
|
264
|
+
cancellation?: Promise<never>;
|
|
265
|
+
}): DetachedDaytonaRunnerSupervisor {
|
|
266
|
+
const cancellation = input.cancellation ?? new Promise<never>(() => {});
|
|
267
|
+
const sessionCreation = input.sandbox.process
|
|
268
|
+
.createSession(input.sessionId)
|
|
269
|
+
.then(
|
|
270
|
+
() => ({ ok: true as const }),
|
|
271
|
+
(error: unknown) => ({ ok: false as const, error }),
|
|
272
|
+
);
|
|
273
|
+
const readinessBaseline = captureDetachedDaytonaRunnerReadinessBaseline({
|
|
274
|
+
readiness: input.readiness,
|
|
275
|
+
}).then(
|
|
276
|
+
(heartbeatAt) => ({ ok: true as const, heartbeatAt }),
|
|
277
|
+
(error: unknown) => ({ ok: false as const, error }),
|
|
278
|
+
);
|
|
279
|
+
let state: DetachedDaytonaRunnerState = {
|
|
280
|
+
phase: 'preparing',
|
|
281
|
+
sessionId: input.sessionId,
|
|
282
|
+
};
|
|
283
|
+
let started = false;
|
|
256
284
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
285
|
+
return {
|
|
286
|
+
async start({ command, exitCodePath }) {
|
|
287
|
+
if (started) {
|
|
288
|
+
throw new Error(
|
|
289
|
+
`Detached Daytona runner session ${input.sessionId} was already started.`,
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
started = true;
|
|
293
|
+
const session = await Promise.race([sessionCreation, cancellation]);
|
|
294
|
+
if (session.ok === false) throw session.error;
|
|
295
|
+
const baseline = await Promise.race([readinessBaseline, cancellation]);
|
|
296
|
+
if (baseline.ok === false) throw baseline.error;
|
|
297
|
+
const response = await Promise.race([
|
|
298
|
+
input.sandbox.process.executeSessionCommand(input.sessionId, {
|
|
299
|
+
command,
|
|
300
|
+
runAsync: true,
|
|
301
|
+
}),
|
|
302
|
+
cancellation,
|
|
303
|
+
]);
|
|
304
|
+
const cmdId = response?.cmdId;
|
|
305
|
+
if (!cmdId) {
|
|
306
|
+
throw new Error(
|
|
307
|
+
`Daytona executeSessionCommand did not return a cmdId for session ${input.sessionId}.`,
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
state = {
|
|
311
|
+
phase: 'command_accepted',
|
|
312
|
+
sessionId: input.sessionId,
|
|
313
|
+
cmdId,
|
|
314
|
+
baselineHeartbeatAt: baseline.heartbeatAt,
|
|
315
|
+
};
|
|
316
|
+
await Promise.race([
|
|
317
|
+
confirmDetachedDaytonaRunnerReady({
|
|
318
|
+
sandbox: input.sandbox,
|
|
319
|
+
sessionId: state.sessionId,
|
|
320
|
+
cmdId: state.cmdId,
|
|
321
|
+
exitCodePath,
|
|
322
|
+
readiness: input.readiness,
|
|
323
|
+
baselineHeartbeatAt: state.baselineHeartbeatAt,
|
|
324
|
+
}),
|
|
325
|
+
cancellation,
|
|
326
|
+
]);
|
|
327
|
+
state = { ...state, phase: 'ready' };
|
|
328
|
+
return state;
|
|
267
329
|
},
|
|
268
|
-
|
|
269
|
-
const cmdId = response?.cmdId;
|
|
270
|
-
if (!cmdId) {
|
|
271
|
-
throw new Error(
|
|
272
|
-
`Daytona executeSessionCommand did not return a cmdId for session ${input.sessionId}.`,
|
|
273
|
-
);
|
|
274
|
-
}
|
|
275
|
-
return { sessionId: input.sessionId, cmdId };
|
|
330
|
+
};
|
|
276
331
|
}
|
|
277
332
|
|
|
278
333
|
/**
|
|
@@ -34,11 +34,8 @@ import {
|
|
|
34
34
|
} from './daytona-lifecycle';
|
|
35
35
|
import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
|
|
36
36
|
import {
|
|
37
|
-
captureDetachedDaytonaRunnerReadinessBaseline,
|
|
38
|
-
confirmDetachedDaytonaRunnerReady,
|
|
39
|
-
createDetachedDaytonaSession,
|
|
40
37
|
DaytonaRunnerInitializationError,
|
|
41
|
-
|
|
38
|
+
prepareDetachedDaytonaRunner,
|
|
42
39
|
} from './daytona-session-execution';
|
|
43
40
|
|
|
44
41
|
const DAYTONA_EXECUTE_TIMEOUT_SECONDS = STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
|
|
@@ -894,20 +891,12 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
894
891
|
// Session allocation is independent of payload staging. Start it
|
|
895
892
|
// now so Daytona control-plane latency overlaps compression/upload.
|
|
896
893
|
const sessionId = `deepline-play-${randomUUID()}`;
|
|
897
|
-
const
|
|
894
|
+
const detachedRunner = prepareDetachedDaytonaRunner({
|
|
898
895
|
sandbox,
|
|
899
896
|
sessionId,
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
);
|
|
904
|
-
const readinessBaseline =
|
|
905
|
-
captureDetachedDaytonaRunnerReadinessBaseline({
|
|
906
|
-
readiness: readRunnerReadiness,
|
|
907
|
-
}).then(
|
|
908
|
-
(heartbeatAt) => ({ ok: true as const, heartbeatAt }),
|
|
909
|
-
(error: unknown) => ({ ok: false as const, error }),
|
|
910
|
-
);
|
|
897
|
+
readiness: readRunnerReadiness,
|
|
898
|
+
cancellation: cancellationPromise,
|
|
899
|
+
});
|
|
911
900
|
|
|
912
901
|
const uploadStartedAt = Date.now();
|
|
913
902
|
let uploadDeadlineTimer: ReturnType<typeof setTimeout> | null = null;
|
|
@@ -996,28 +985,12 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
996
985
|
// gateway heartbeat. A Daytona cmdId alone does not prove liveness.
|
|
997
986
|
// After that bounded startup check, the worker parks (releasing its
|
|
998
987
|
// claim and slot) and wakes on terminal push or the ceiling timeout.
|
|
999
|
-
|
|
1000
|
-
sessionCreation,
|
|
1001
|
-
cancellationPromise,
|
|
1002
|
-
]);
|
|
1003
|
-
if (session.ok === false) throw session.error;
|
|
1004
|
-
const baseline = await Promise.race([
|
|
1005
|
-
readinessBaseline,
|
|
1006
|
-
cancellationPromise,
|
|
1007
|
-
]);
|
|
1008
|
-
if (baseline.ok === false) throw baseline.error;
|
|
1009
|
-
let start: Awaited<
|
|
1010
|
-
ReturnType<typeof executeDetachedDaytonaSessionCommand>
|
|
1011
|
-
>;
|
|
988
|
+
let start: Awaited<ReturnType<typeof detachedRunner.start>>;
|
|
1012
989
|
try {
|
|
1013
|
-
start = await
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
command: stagedPayload.command,
|
|
1018
|
-
}),
|
|
1019
|
-
cancellationPromise,
|
|
1020
|
-
]);
|
|
990
|
+
start = await detachedRunner.start({
|
|
991
|
+
command: stagedPayload.command,
|
|
992
|
+
exitCodePath: stagedPayload.exitCodePath,
|
|
993
|
+
});
|
|
1021
994
|
} catch (error) {
|
|
1022
995
|
if (
|
|
1023
996
|
cancellationRequested ||
|
|
@@ -1030,17 +1003,6 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1030
1003
|
// classification below decide fresh-sandbox retry vs loud failure.
|
|
1031
1004
|
throw error;
|
|
1032
1005
|
}
|
|
1033
|
-
await Promise.race([
|
|
1034
|
-
confirmDetachedDaytonaRunnerReady({
|
|
1035
|
-
sandbox,
|
|
1036
|
-
sessionId: start.sessionId,
|
|
1037
|
-
cmdId: start.cmdId,
|
|
1038
|
-
exitCodePath: stagedPayload.exitCodePath,
|
|
1039
|
-
readiness: readRunnerReadiness,
|
|
1040
|
-
baselineHeartbeatAt: baseline.heartbeatAt,
|
|
1041
|
-
}),
|
|
1042
|
-
cancellationPromise,
|
|
1043
|
-
]);
|
|
1044
1006
|
const runnerAttempt = Math.max(
|
|
1045
1007
|
0,
|
|
1046
1008
|
Math.floor(config.context.runAttempt ?? 0),
|
|
@@ -3,6 +3,7 @@ import type { DbLogicalTable, DbSessionOperation } from './db-session';
|
|
|
3
3
|
import type { PlayRowUpdate } from './ctx-types';
|
|
4
4
|
import type { PlaySheetContract } from '../plays/static-pipeline';
|
|
5
5
|
import type { PlayRunLedgerEvent } from './run-ledger';
|
|
6
|
+
import type { WorkReceiptFailureKind } from './work-receipts';
|
|
6
7
|
|
|
7
8
|
export type RuntimeAppendRunEventsAction = {
|
|
8
9
|
action: 'append_run_events';
|
|
@@ -94,11 +95,47 @@ export type RuntimeReceiptAction =
|
|
|
94
95
|
runId: string;
|
|
95
96
|
key: string;
|
|
96
97
|
}
|
|
98
|
+
| {
|
|
99
|
+
action: 'get_runtime_step_receipts';
|
|
100
|
+
playName: string;
|
|
101
|
+
runId: string;
|
|
102
|
+
keys: string[];
|
|
103
|
+
}
|
|
104
|
+
| {
|
|
105
|
+
action: 'acquire_runtime_receipt_execution_lock';
|
|
106
|
+
playName: string;
|
|
107
|
+
runId: string;
|
|
108
|
+
key: string;
|
|
109
|
+
ownerExecutionId: string;
|
|
110
|
+
ttlMs: number;
|
|
111
|
+
}
|
|
112
|
+
| {
|
|
113
|
+
action: 'release_runtime_receipt_execution_lock';
|
|
114
|
+
playName: string;
|
|
115
|
+
runId: string;
|
|
116
|
+
key: string;
|
|
117
|
+
ownerExecutionId: string;
|
|
118
|
+
}
|
|
97
119
|
| {
|
|
98
120
|
action: 'claim_runtime_step_receipt';
|
|
99
121
|
playName: string;
|
|
100
122
|
runId: string;
|
|
101
123
|
key: string;
|
|
124
|
+
leaseId?: string;
|
|
125
|
+
runAttempt?: number | null;
|
|
126
|
+
leaseAware?: boolean;
|
|
127
|
+
reclaimRunning?: boolean;
|
|
128
|
+
forceRefresh?: boolean;
|
|
129
|
+
forceFailedRefresh?: boolean;
|
|
130
|
+
}
|
|
131
|
+
| {
|
|
132
|
+
action: 'claim_runtime_step_receipts';
|
|
133
|
+
playName: string;
|
|
134
|
+
runId: string;
|
|
135
|
+
keys: string[];
|
|
136
|
+
leaseIds?: string[];
|
|
137
|
+
runAttempt?: number | null;
|
|
138
|
+
leaseAware?: boolean;
|
|
102
139
|
reclaimRunning?: boolean;
|
|
103
140
|
forceRefresh?: boolean;
|
|
104
141
|
forceFailedRefresh?: boolean;
|
|
@@ -108,6 +145,8 @@ export type RuntimeReceiptAction =
|
|
|
108
145
|
playName: string;
|
|
109
146
|
runId: string;
|
|
110
147
|
key: string;
|
|
148
|
+
runAttempt?: number | null;
|
|
149
|
+
leaseId?: string | null;
|
|
111
150
|
output: unknown;
|
|
112
151
|
}
|
|
113
152
|
| {
|
|
@@ -115,7 +154,7 @@ export type RuntimeReceiptAction =
|
|
|
115
154
|
playName: string;
|
|
116
155
|
runId: string;
|
|
117
156
|
key: string;
|
|
118
|
-
runAttempt?: number;
|
|
157
|
+
runAttempt?: number | null;
|
|
119
158
|
leaseId?: string | null;
|
|
120
159
|
}
|
|
121
160
|
| {
|
|
@@ -123,16 +162,77 @@ export type RuntimeReceiptAction =
|
|
|
123
162
|
playName: string;
|
|
124
163
|
runId: string;
|
|
125
164
|
key: string;
|
|
165
|
+
runAttempt?: number | null;
|
|
166
|
+
leaseId?: string | null;
|
|
126
167
|
error: string;
|
|
168
|
+
failureKind?: WorkReceiptFailureKind | null;
|
|
127
169
|
}
|
|
128
170
|
| {
|
|
129
171
|
action: 'skip_runtime_step_receipt';
|
|
130
172
|
playName: string;
|
|
131
173
|
runId: string;
|
|
132
174
|
key: string;
|
|
175
|
+
runAttempt?: number | null;
|
|
176
|
+
leaseId?: string | null;
|
|
133
177
|
output?: unknown;
|
|
178
|
+
}
|
|
179
|
+
| {
|
|
180
|
+
action: 'mark_runtime_step_receipt_running';
|
|
181
|
+
playName: string;
|
|
182
|
+
runId: string;
|
|
183
|
+
key: string;
|
|
184
|
+
runAttempt?: number | null;
|
|
185
|
+
leaseId?: string | null;
|
|
186
|
+
}
|
|
187
|
+
| {
|
|
188
|
+
action: 'mark_runtime_step_receipts_running';
|
|
189
|
+
playName: string;
|
|
190
|
+
runId: string;
|
|
191
|
+
receipts: RuntimeReceiptLease[];
|
|
192
|
+
}
|
|
193
|
+
| {
|
|
194
|
+
action: 'mark_runtime_step_receipts_queued';
|
|
195
|
+
playName: string;
|
|
196
|
+
runId: string;
|
|
197
|
+
receipts: RuntimeReceiptLease[];
|
|
198
|
+
}
|
|
199
|
+
| {
|
|
200
|
+
action: 'complete_runtime_step_receipts';
|
|
201
|
+
playName: string;
|
|
202
|
+
runId: string;
|
|
203
|
+
receipts: RuntimeReceiptCompletion[];
|
|
204
|
+
}
|
|
205
|
+
| {
|
|
206
|
+
action: 'fail_runtime_step_receipts';
|
|
207
|
+
playName: string;
|
|
208
|
+
runId: string;
|
|
209
|
+
receipts: RuntimeReceiptFailure[];
|
|
210
|
+
}
|
|
211
|
+
| {
|
|
212
|
+
action: 'heartbeat_runtime_step_receipts';
|
|
213
|
+
playName: string;
|
|
214
|
+
runId: string;
|
|
215
|
+
runAttempt?: number | null;
|
|
216
|
+
leaseId: string;
|
|
217
|
+
keys: string[];
|
|
134
218
|
};
|
|
135
219
|
|
|
220
|
+
export type RuntimeReceiptLease = {
|
|
221
|
+
key: string;
|
|
222
|
+
runId: string;
|
|
223
|
+
runAttempt?: number | null;
|
|
224
|
+
leaseId?: string | null;
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
export type RuntimeReceiptCompletion = RuntimeReceiptLease & {
|
|
228
|
+
output: unknown;
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
export type RuntimeReceiptFailure = RuntimeReceiptLease & {
|
|
232
|
+
error: string;
|
|
233
|
+
failureKind?: WorkReceiptFailureKind | null;
|
|
234
|
+
};
|
|
235
|
+
|
|
136
236
|
export type RuntimePlayResolutionAction = {
|
|
137
237
|
action: 'resolve_play';
|
|
138
238
|
playRef: string;
|
|
@@ -9,6 +9,7 @@ import type {
|
|
|
9
9
|
RuntimeStepReceipt,
|
|
10
10
|
SkipRuntimeStepReceiptInput,
|
|
11
11
|
} from './ctx-types';
|
|
12
|
+
import { RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES } from './output-size-limits';
|
|
12
13
|
import {
|
|
13
14
|
acquireRuntimeReceiptExecutionLockViaAppRuntime,
|
|
14
15
|
AppRuntimeApiResponseError,
|
|
@@ -120,6 +121,7 @@ export class RuntimeReceiptStoreAdapter {
|
|
|
120
121
|
playName: string;
|
|
121
122
|
runId: string;
|
|
122
123
|
maxBatchSize?: number;
|
|
124
|
+
maxBatchBytes?: number;
|
|
123
125
|
maxFlushMs?: number;
|
|
124
126
|
maxBufferedBytes?: number;
|
|
125
127
|
}) {
|
|
@@ -130,6 +132,8 @@ export class RuntimeReceiptStoreAdapter {
|
|
|
130
132
|
this.#writer = new RuntimeReceiptWriter({
|
|
131
133
|
batchKey: receiptCommandBatchKey,
|
|
132
134
|
maxBatchSize: options.maxBatchSize,
|
|
135
|
+
maxBatchBytes:
|
|
136
|
+
options.maxBatchBytes ?? RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES,
|
|
133
137
|
maxFlushMs: options.maxFlushMs,
|
|
134
138
|
maxBufferedBytes: options.maxBufferedBytes,
|
|
135
139
|
classifyRetryableError: classifyReceiptRetry,
|
|
@@ -2,6 +2,8 @@ type RetryableReceiptError = {
|
|
|
2
2
|
retryAfterMs: number;
|
|
3
3
|
};
|
|
4
4
|
|
|
5
|
+
const MAX_RETRY_JITTER_RATIO = 0.25;
|
|
6
|
+
|
|
5
7
|
type PendingWrite<Input, Output> = {
|
|
6
8
|
input: Input;
|
|
7
9
|
bytes: number;
|
|
@@ -320,7 +322,7 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
320
322
|
const retry = this.#classifyRetryableError(error);
|
|
321
323
|
if (!retry) throw error;
|
|
322
324
|
await this.#waitForRetry(
|
|
323
|
-
|
|
325
|
+
jitteredRetryDelayMs(retry.retryAfterMs),
|
|
324
326
|
batch,
|
|
325
327
|
);
|
|
326
328
|
}
|
|
@@ -489,6 +491,14 @@ function normalizePositiveInteger(
|
|
|
489
491
|
return Math.max(1, Math.floor(value ?? fallback));
|
|
490
492
|
}
|
|
491
493
|
|
|
494
|
+
function jitteredRetryDelayMs(delayMs: number): number {
|
|
495
|
+
const normalized = Math.max(0, Math.floor(delayMs));
|
|
496
|
+
if (normalized === 0) return 0;
|
|
497
|
+
return (
|
|
498
|
+
normalized + Math.floor(normalized * MAX_RETRY_JITTER_RATIO * Math.random())
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
|
|
492
502
|
function abortReason(signal: AbortSignal | null): unknown {
|
|
493
503
|
return signal?.reason ?? new Error('Runtime receipt write was aborted.');
|
|
494
504
|
}
|
package/dist/cli/index.js
CHANGED
|
@@ -718,7 +718,7 @@ var SDK_RELEASE = {
|
|
|
718
718
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
719
719
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
720
720
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
721
|
-
version: "0.1.
|
|
721
|
+
version: "0.1.294",
|
|
722
722
|
contracts: {
|
|
723
723
|
api: {
|
|
724
724
|
name: "sdk-http-api",
|
|
@@ -1600,6 +1600,7 @@ var CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 5 * 1024 * 1024;
|
|
|
1600
1600
|
var RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
|
|
1601
1601
|
var RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
|
|
1602
1602
|
var RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
1603
|
+
var RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
|
|
1603
1604
|
var RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES = 64 * 1024;
|
|
1604
1605
|
|
|
1605
1606
|
// ../shared_libs/play-runtime/ledger-safe-payload.ts
|
package/dist/cli/index.mjs
CHANGED
|
@@ -703,7 +703,7 @@ var SDK_RELEASE = {
|
|
|
703
703
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
704
704
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
705
705
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
706
|
-
version: "0.1.
|
|
706
|
+
version: "0.1.294",
|
|
707
707
|
contracts: {
|
|
708
708
|
api: {
|
|
709
709
|
name: "sdk-http-api",
|
|
@@ -1585,6 +1585,7 @@ var CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 5 * 1024 * 1024;
|
|
|
1585
1585
|
var RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
|
|
1586
1586
|
var RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
|
|
1587
1587
|
var RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
1588
|
+
var RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
|
|
1588
1589
|
var RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES = 64 * 1024;
|
|
1589
1590
|
|
|
1590
1591
|
// ../shared_libs/play-runtime/ledger-safe-payload.ts
|
package/dist/index.js
CHANGED
|
@@ -438,7 +438,7 @@ var SDK_RELEASE = {
|
|
|
438
438
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
439
439
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
440
440
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
441
|
-
version: "0.1.
|
|
441
|
+
version: "0.1.294",
|
|
442
442
|
contracts: {
|
|
443
443
|
api: {
|
|
444
444
|
name: "sdk-http-api",
|
|
@@ -1320,6 +1320,7 @@ var CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 5 * 1024 * 1024;
|
|
|
1320
1320
|
var RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
|
|
1321
1321
|
var RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
|
|
1322
1322
|
var RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
1323
|
+
var RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
|
|
1323
1324
|
var RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES = 64 * 1024;
|
|
1324
1325
|
|
|
1325
1326
|
// ../shared_libs/play-runtime/ledger-safe-payload.ts
|
package/dist/index.mjs
CHANGED
|
@@ -367,7 +367,7 @@ var SDK_RELEASE = {
|
|
|
367
367
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
368
368
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
369
369
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
370
|
-
version: "0.1.
|
|
370
|
+
version: "0.1.294",
|
|
371
371
|
contracts: {
|
|
372
372
|
api: {
|
|
373
373
|
name: "sdk-http-api",
|
|
@@ -1249,6 +1249,7 @@ var CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 5 * 1024 * 1024;
|
|
|
1249
1249
|
var RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
|
|
1250
1250
|
var RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
|
|
1251
1251
|
var RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
1252
|
+
var RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
|
|
1252
1253
|
var RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES = 64 * 1024;
|
|
1253
1254
|
|
|
1254
1255
|
// ../shared_libs/play-runtime/ledger-safe-payload.ts
|