deepline 0.1.201 → 0.1.202
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +30 -9
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +25 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +139 -124
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +17 -2
- package/dist/bundling-sources/shared_libs/play-runtime/bounded-dispatch.ts +52 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +152 -23
- package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +121 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-budget-state-backend.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +86 -19
- package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts +141 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/budget-state-backend.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +129 -41
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +40 -2
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +92 -33
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +13 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +122 -120
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +10 -8
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +140 -32
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +225 -29
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts +26 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +21 -0
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +32 -10
- package/dist/cli/index.js +198 -76
- package/dist/cli/index.mjs +204 -82
- package/dist/index.js +3 -2
- package/dist/index.mjs +3 -2
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/adaptive-tool-dispatcher.ts +0 -355
|
@@ -46,6 +46,71 @@ const DAYTONA_RUNTIME_DB_CONNECT_MAX_ATTEMPTS = 2;
|
|
|
46
46
|
const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
|
|
47
47
|
const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
|
|
48
48
|
const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
|
|
49
|
+
/**
|
|
50
|
+
* Client-side wall-clock grace for `sandbox.process.executeCommand`. The
|
|
51
|
+
* `timeout` argument we pass is a SERVER-side directive; the Daytona SDK's
|
|
52
|
+
* underlying HTTP call has no client deadline, so a wedged socket (observed:
|
|
53
|
+
* `AggregateError [ETIMEDOUT]`) leaves the await hanging forever and holds the
|
|
54
|
+
* worker slot. We race the call against this deadline = server timeout + grace,
|
|
55
|
+
* so a genuine server-side runtime-limit hit still returns first, but a wedged
|
|
56
|
+
* client transport fails the attempt loudly and lets the engine retry ladder
|
|
57
|
+
* take over instead of livelocking the slot.
|
|
58
|
+
*/
|
|
59
|
+
const DAYTONA_EXECUTE_CLIENT_GRACE_MS = 60_000;
|
|
60
|
+
const DAYTONA_EXECUTE_CLIENT_DEADLINE_MS =
|
|
61
|
+
DAYTONA_EXECUTE_TIMEOUT_SECONDS * 1_000 + DAYTONA_EXECUTE_CLIENT_GRACE_MS;
|
|
62
|
+
/**
|
|
63
|
+
* Bound for individual sandbox/runtime API calls that have no natural timeout
|
|
64
|
+
* (e.g. recording the acquired runtime resource). A wedged one of these must
|
|
65
|
+
* fail the attempt, not hold the slot.
|
|
66
|
+
*/
|
|
67
|
+
const DAYTONA_API_CALL_DEADLINE_MS = 60_000;
|
|
68
|
+
|
|
69
|
+
export class DaytonaOperationDeadlineError extends Error {
|
|
70
|
+
readonly operation: string;
|
|
71
|
+
readonly deadlineMs: number;
|
|
72
|
+
|
|
73
|
+
constructor(input: { operation: string; deadlineMs: number }) {
|
|
74
|
+
super(
|
|
75
|
+
`Daytona operation "${input.operation}" exceeded client deadline of ${input.deadlineMs}ms and was abandoned to free the worker slot.`,
|
|
76
|
+
);
|
|
77
|
+
this.name = 'DaytonaOperationDeadlineError';
|
|
78
|
+
this.operation = input.operation;
|
|
79
|
+
this.deadlineMs = input.deadlineMs;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Race `promise` against a wall-clock deadline. On breach, reject loudly with a
|
|
85
|
+
* `DaytonaOperationDeadlineError` (the caller's retry/failure classification
|
|
86
|
+
* decides what happens next). The timer is always cleared so a fast-resolving
|
|
87
|
+
* promise never leaves a dangling handle.
|
|
88
|
+
*/
|
|
89
|
+
export async function withDaytonaOperationDeadline<T>(input: {
|
|
90
|
+
operation: string;
|
|
91
|
+
deadlineMs: number;
|
|
92
|
+
promise: Promise<T>;
|
|
93
|
+
}): Promise<T> {
|
|
94
|
+
let deadlineTimer: ReturnType<typeof setTimeout> | null = null;
|
|
95
|
+
const deadlinePromise = new Promise<never>((_resolve, reject) => {
|
|
96
|
+
deadlineTimer = setTimeout(() => {
|
|
97
|
+
reject(
|
|
98
|
+
new DaytonaOperationDeadlineError({
|
|
99
|
+
operation: input.operation,
|
|
100
|
+
deadlineMs: input.deadlineMs,
|
|
101
|
+
}),
|
|
102
|
+
);
|
|
103
|
+
}, input.deadlineMs);
|
|
104
|
+
deadlineTimer.unref?.();
|
|
105
|
+
});
|
|
106
|
+
try {
|
|
107
|
+
return await Promise.race([input.promise, deadlinePromise]);
|
|
108
|
+
} finally {
|
|
109
|
+
if (deadlineTimer) {
|
|
110
|
+
clearTimeout(deadlineTimer);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
49
114
|
const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
|
|
50
115
|
/\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
|
|
51
116
|
const DAYTONA_INFRASTRUCTURE_RETRY_PATTERN =
|
|
@@ -223,10 +288,11 @@ function createDaytonaProgressSidecarPoller(input: {
|
|
|
223
288
|
context: DaytonaExecutionContext;
|
|
224
289
|
startedAt: number;
|
|
225
290
|
processedEventKeys: Set<string>;
|
|
226
|
-
}): { start(): void; stop(): Promise<void> } {
|
|
291
|
+
}): { start(): void; stop(flush?: boolean): Promise<void> } {
|
|
227
292
|
let timer: ReturnType<typeof setTimeout> | null = null;
|
|
228
293
|
let stopped = false;
|
|
229
294
|
let inFlight: Promise<void> = Promise.resolve();
|
|
295
|
+
let segment = 0;
|
|
230
296
|
let offsetBytes = 0;
|
|
231
297
|
let pendingLine = '';
|
|
232
298
|
let warned = false;
|
|
@@ -259,20 +325,39 @@ function createDaytonaProgressSidecarPoller(input: {
|
|
|
259
325
|
|
|
260
326
|
const poll = async (flushPartial = false): Promise<void> => {
|
|
261
327
|
try {
|
|
262
|
-
const
|
|
328
|
+
const manifest = await input.sandbox.fs.downloadFile(
|
|
263
329
|
input.progressEventPath,
|
|
264
330
|
5,
|
|
265
331
|
);
|
|
266
|
-
const
|
|
267
|
-
|
|
332
|
+
const parsedLatestSegment = Number.parseInt(
|
|
333
|
+
Buffer.isBuffer(manifest)
|
|
334
|
+
? manifest.toString('utf-8').trim()
|
|
335
|
+
: String(manifest).trim(),
|
|
336
|
+
10,
|
|
337
|
+
);
|
|
338
|
+
const latestSegment = Number.isFinite(parsedLatestSegment)
|
|
339
|
+
? Math.max(segment, parsedLatestSegment)
|
|
340
|
+
: segment;
|
|
341
|
+
while (segment <= latestSegment) {
|
|
342
|
+
const segmentPath = `${input.progressEventPath}.${String(
|
|
343
|
+
segment,
|
|
344
|
+
).padStart(6, '0')}`;
|
|
345
|
+
const file = await input.sandbox.fs.downloadFile(segmentPath, 5);
|
|
346
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(file);
|
|
347
|
+
if (buffer.length < offsetBytes) {
|
|
348
|
+
offsetBytes = 0;
|
|
349
|
+
pendingLine = '';
|
|
350
|
+
}
|
|
351
|
+
if (buffer.length > offsetBytes) {
|
|
352
|
+
const chunk = buffer.subarray(offsetBytes).toString('utf-8');
|
|
353
|
+
offsetBytes = buffer.length;
|
|
354
|
+
processText(chunk, flushPartial && segment === latestSegment);
|
|
355
|
+
}
|
|
356
|
+
if (segment >= latestSegment) break;
|
|
357
|
+
segment += 1;
|
|
268
358
|
offsetBytes = 0;
|
|
269
|
-
pendingLine = '';
|
|
270
359
|
}
|
|
271
|
-
if (
|
|
272
|
-
const chunk = buffer.subarray(offsetBytes).toString('utf-8');
|
|
273
|
-
offsetBytes = buffer.length;
|
|
274
|
-
processText(chunk, flushPartial);
|
|
275
|
-
} else if (flushPartial && pendingLine.trim()) {
|
|
360
|
+
if (flushPartial && pendingLine.trim()) {
|
|
276
361
|
processText('\n', true);
|
|
277
362
|
}
|
|
278
363
|
} catch (error) {
|
|
@@ -304,14 +389,14 @@ function createDaytonaProgressSidecarPoller(input: {
|
|
|
304
389
|
start() {
|
|
305
390
|
inFlight = inFlight.then(() => poll()).finally(schedule);
|
|
306
391
|
},
|
|
307
|
-
async stop() {
|
|
392
|
+
async stop(flush = true) {
|
|
308
393
|
stopped = true;
|
|
309
394
|
if (timer) {
|
|
310
395
|
clearTimeout(timer);
|
|
311
396
|
timer = null;
|
|
312
397
|
}
|
|
313
398
|
await inFlight.catch(() => {});
|
|
314
|
-
await poll(true);
|
|
399
|
+
if (flush) await poll(true);
|
|
315
400
|
},
|
|
316
401
|
};
|
|
317
402
|
}
|
|
@@ -422,6 +507,12 @@ function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
|
|
|
422
507
|
if (error instanceof DaytonaCommandTransportError) {
|
|
423
508
|
return false;
|
|
424
509
|
}
|
|
510
|
+
// A client-side deadline breach is a wedged transport (hung socket / SDK call
|
|
511
|
+
// with no client timeout). Retry once on a fresh sandbox before spending an
|
|
512
|
+
// engine attempt, same as the other transport-level infrastructure failures.
|
|
513
|
+
if (error instanceof DaytonaOperationDeadlineError) {
|
|
514
|
+
return true;
|
|
515
|
+
}
|
|
425
516
|
const message = error instanceof Error ? error.message : String(error);
|
|
426
517
|
if (!message || /cancelled|aborted/i.test(message)) {
|
|
427
518
|
return false;
|
|
@@ -566,15 +657,24 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
566
657
|
acquiredSandbox.billingStartedAt - startedAt;
|
|
567
658
|
const sandbox = acquiredSandbox.sandbox;
|
|
568
659
|
sandboxForAttempt = sandbox;
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
660
|
+
const resourceAcquiredNotice = callbacks?.onRuntimeResourceAcquired?.(
|
|
661
|
+
{
|
|
662
|
+
kind: 'daytona_sandbox',
|
|
663
|
+
sandboxId: sandbox.id,
|
|
664
|
+
billingStartedAt: acquiredSandbox.billingStartedAt,
|
|
665
|
+
cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
|
|
666
|
+
memoryGiB:
|
|
667
|
+
typeof sandbox.memory === 'number' ? sandbox.memory : null,
|
|
668
|
+
diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : null,
|
|
669
|
+
},
|
|
670
|
+
);
|
|
671
|
+
if (resourceAcquiredNotice) {
|
|
672
|
+
await withDaytonaOperationDeadline({
|
|
673
|
+
operation: 'onRuntimeResourceAcquired',
|
|
674
|
+
deadlineMs: DAYTONA_API_CALL_DEADLINE_MS,
|
|
675
|
+
promise: resourceAcquiredNotice,
|
|
676
|
+
});
|
|
677
|
+
}
|
|
578
678
|
throwIfCancellationRequested();
|
|
579
679
|
|
|
580
680
|
const { workdir: workDir } = loadDaytonaRunnerPathsConfig();
|
|
@@ -676,16 +776,19 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
676
776
|
try {
|
|
677
777
|
progressSidecar.start();
|
|
678
778
|
execution = await Promise.race([
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
779
|
+
withDaytonaOperationDeadline({
|
|
780
|
+
operation: 'sandbox.process.executeCommand',
|
|
781
|
+
deadlineMs: DAYTONA_EXECUTE_CLIENT_DEADLINE_MS,
|
|
782
|
+
promise: sandbox.process.executeCommand(
|
|
783
|
+
stagedPayload.command,
|
|
784
|
+
stagedPayload.workDir,
|
|
785
|
+
undefined,
|
|
786
|
+
DAYTONA_EXECUTE_TIMEOUT_SECONDS,
|
|
787
|
+
),
|
|
788
|
+
}),
|
|
685
789
|
cancellationPromise,
|
|
686
790
|
]);
|
|
687
791
|
} catch (error) {
|
|
688
|
-
await progressSidecar.stop();
|
|
689
792
|
if (
|
|
690
793
|
cancellationRequested ||
|
|
691
794
|
callbacks?.cancellationSignal?.aborted
|
|
@@ -729,7 +832,10 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
729
832
|
});
|
|
730
833
|
execution = recovered.execution;
|
|
731
834
|
} finally {
|
|
732
|
-
await progressSidecar.stop(
|
|
835
|
+
await progressSidecar.stop(
|
|
836
|
+
!cancellationRequested &&
|
|
837
|
+
callbacks?.cancellationSignal?.aborted !== true,
|
|
838
|
+
);
|
|
733
839
|
}
|
|
734
840
|
const runnerOutput = await resolveDaytonaCommandOutput({
|
|
735
841
|
sandbox,
|
|
@@ -818,9 +924,11 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
818
924
|
error:
|
|
819
925
|
typeof execution.exitCode === 'number' &&
|
|
820
926
|
execution.exitCode !== 0
|
|
821
|
-
? `Daytona play runner exited with code ${execution.exitCode}
|
|
822
|
-
|
|
823
|
-
|
|
927
|
+
? `Daytona play runner exited with code ${execution.exitCode}${
|
|
928
|
+
execution.exitCode === 137
|
|
929
|
+
? ' (sandbox OOM or forced SIGKILL)'
|
|
930
|
+
: ''
|
|
931
|
+
}.${outputTail ? ` Output tail: ${outputTail}` : ''}`
|
|
824
932
|
: 'Daytona play runner produced no result.',
|
|
825
933
|
runtimeTiming,
|
|
826
934
|
}),
|
|
@@ -115,7 +115,11 @@ type RuntimeApiContext = {
|
|
|
115
115
|
runId?: string | null;
|
|
116
116
|
userEmail?: string | null;
|
|
117
117
|
integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
|
|
118
|
-
dbSessionStrategy?:
|
|
118
|
+
dbSessionStrategy?:
|
|
119
|
+
| 'preloaded'
|
|
120
|
+
| 'sandbox_public_key'
|
|
121
|
+
| 'gateway_only'
|
|
122
|
+
| null;
|
|
119
123
|
preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
|
|
120
124
|
vercelProtectionBypassToken?: string | null;
|
|
121
125
|
runtimeTestFaultHeader?: string | null;
|
|
@@ -443,9 +447,33 @@ type RuntimeApiActionRequest =
|
|
|
443
447
|
action: 'repair_runtime_storage_grants';
|
|
444
448
|
playName: string;
|
|
445
449
|
runId?: string | null;
|
|
450
|
+
}
|
|
451
|
+
| {
|
|
452
|
+
action: 'runtime_sheet_start';
|
|
453
|
+
input: Parameters<typeof startRuntimeSheetDataset>[1];
|
|
454
|
+
}
|
|
455
|
+
| {
|
|
456
|
+
action: 'runtime_sheet_complete_map_rows';
|
|
457
|
+
input: Parameters<typeof completeRuntimeMapRows>[1];
|
|
458
|
+
}
|
|
459
|
+
| {
|
|
460
|
+
action: 'runtime_sheet_read_rows';
|
|
461
|
+
input: Parameters<typeof readRuntimeSheetDatasetRows>[1];
|
|
462
|
+
}
|
|
463
|
+
| {
|
|
464
|
+
action: 'runtime_sheet_read_row_keys';
|
|
465
|
+
input: Parameters<typeof readRuntimeSheetDatasetRowKeys>[1];
|
|
466
|
+
}
|
|
467
|
+
| {
|
|
468
|
+
action: 'runtime_sheet_release_attempt';
|
|
469
|
+
input: Parameters<typeof releaseRuntimeSheetAttempt>[1];
|
|
470
|
+
}
|
|
471
|
+
| {
|
|
472
|
+
action: 'runtime_receipts_release_attempt';
|
|
473
|
+
input: Parameters<typeof releaseRuntimeWorkReceipts>[1];
|
|
446
474
|
};
|
|
447
475
|
|
|
448
|
-
type RuntimeApiRowRecord = MapRowOutcome & {
|
|
476
|
+
export type RuntimeApiRowRecord = MapRowOutcome & {
|
|
449
477
|
inputIndex?: number | null;
|
|
450
478
|
};
|
|
451
479
|
|
|
@@ -614,8 +642,8 @@ function runtimeApiMaxAttempts(
|
|
|
614
642
|
return runtimeApiRetryDelaysForAction(action).length + 1;
|
|
615
643
|
}
|
|
616
644
|
|
|
617
|
-
function isTransientRuntimeApiErrorMessage(message: string): boolean {
|
|
618
|
-
return /timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|requested endpoint could not be found, or you don't have access/i.test(
|
|
645
|
+
export function isTransientRuntimeApiErrorMessage(message: string): boolean {
|
|
646
|
+
return /timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|requested endpoint could not be found, or you don't have access|RUNTIME_SCHEDULER_SATURATED|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open/i.test(
|
|
619
647
|
message,
|
|
620
648
|
);
|
|
621
649
|
}
|
|
@@ -4258,6 +4286,36 @@ export async function markRuntimeWorkReceiptsQueued(
|
|
|
4258
4286
|
);
|
|
4259
4287
|
}
|
|
4260
4288
|
|
|
4289
|
+
/**
|
|
4290
|
+
* Replay-idempotency fence for a receipt whose active complete/fail UPDATE
|
|
4291
|
+
* matched 0 rows. This fires when the durable write committed but the HTTP
|
|
4292
|
+
* response was lost and the SAME caller retries: the row is already terminal,
|
|
4293
|
+
* so the active-owner predicate (which requires status IN (queued, running))
|
|
4294
|
+
* can never match again. We surface the already-terminal row as success ONLY
|
|
4295
|
+
* when its run identity matches the retrying caller, so the retry does not
|
|
4296
|
+
* spuriously fail a receipt whose provider spend and durable write both
|
|
4297
|
+
* succeeded.
|
|
4298
|
+
*
|
|
4299
|
+
* The predicate mirrors the run-identity half of {@link
|
|
4300
|
+
* workReceiptActiveOwnerPredicateSql}: `COALESCE(lease_owner_run_id, run_id)`
|
|
4301
|
+
* must equal the caller's run id. complete/fail null the lease-owner and
|
|
4302
|
+
* attempt columns as part of the terminal write, so run id is the only durable
|
|
4303
|
+
* ownership marker left on a terminal row — a different run (the stale-owner
|
|
4304
|
+
* rejection case) never matches and still resolves to null exactly as before.
|
|
4305
|
+
* This is replay-idempotency only; it never lets a foreign run read or claim a
|
|
4306
|
+
* receipt it did not complete.
|
|
4307
|
+
*/
|
|
4308
|
+
function workReceiptTerminalReplayPredicateSql(input: {
|
|
4309
|
+
receiptTable: string;
|
|
4310
|
+
terminalStatusSql: string;
|
|
4311
|
+
ownerRunIdSql: string;
|
|
4312
|
+
}): string {
|
|
4313
|
+
const table = input.receiptTable;
|
|
4314
|
+
return `${table}.status = ${input.terminalStatusSql}::smallint
|
|
4315
|
+
/* replay-idempotency: same-run retry after a lost response; run identity is the only durable owner marker on a terminal row */
|
|
4316
|
+
AND COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.ownerRunIdSql}`;
|
|
4317
|
+
}
|
|
4318
|
+
|
|
4261
4319
|
export async function completeRuntimeWorkReceipt(
|
|
4262
4320
|
context: RuntimeApiContext,
|
|
4263
4321
|
input: {
|
|
@@ -4340,8 +4398,31 @@ export async function completeRuntimeWorkReceipt(
|
|
|
4340
4398
|
lease_owner_attempt,
|
|
4341
4399
|
lease_expires_at,
|
|
4342
4400
|
updated_at
|
|
4401
|
+
),
|
|
4402
|
+
replay AS (
|
|
4403
|
+
SELECT convert_from(k, 'UTF8') AS k,
|
|
4404
|
+
status,
|
|
4405
|
+
output,
|
|
4406
|
+
error,
|
|
4407
|
+
failure_kind,
|
|
4408
|
+
run_id,
|
|
4409
|
+
lease_id,
|
|
4410
|
+
lease_owner_run_id,
|
|
4411
|
+
lease_owner_attempt,
|
|
4412
|
+
lease_expires_at,
|
|
4413
|
+
updated_at
|
|
4414
|
+
FROM ${workReceiptTable(session)}
|
|
4415
|
+
WHERE k = decode($1, 'hex')
|
|
4416
|
+
AND NOT EXISTS (SELECT 1 FROM completed)
|
|
4417
|
+
AND ${workReceiptTerminalReplayPredicateSql({
|
|
4418
|
+
receiptTable: workReceiptTable(session),
|
|
4419
|
+
terminalStatusSql: '$2',
|
|
4420
|
+
ownerRunIdSql: '$4',
|
|
4421
|
+
})}
|
|
4343
4422
|
)
|
|
4344
4423
|
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM completed
|
|
4424
|
+
UNION ALL
|
|
4425
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM replay
|
|
4345
4426
|
`,
|
|
4346
4427
|
[
|
|
4347
4428
|
workReceiptKeyHex(input.key),
|
|
@@ -4478,21 +4559,49 @@ export async function completeRuntimeWorkReceipts(
|
|
|
4478
4559
|
target.updated_at,
|
|
4479
4560
|
inputs.ord
|
|
4480
4561
|
),
|
|
4562
|
+
replay AS (
|
|
4563
|
+
SELECT target.k,
|
|
4564
|
+
target.status,
|
|
4565
|
+
target.output,
|
|
4566
|
+
target.error,
|
|
4567
|
+
target.failure_kind,
|
|
4568
|
+
target.run_id,
|
|
4569
|
+
target.lease_id,
|
|
4570
|
+
target.lease_owner_run_id,
|
|
4571
|
+
target.lease_owner_attempt,
|
|
4572
|
+
target.lease_expires_at,
|
|
4573
|
+
target.updated_at,
|
|
4574
|
+
inputs.ord
|
|
4575
|
+
FROM inputs
|
|
4576
|
+
JOIN ${workReceiptTable(session)} AS target
|
|
4577
|
+
ON target.k = decode(inputs.key_hex, 'hex')
|
|
4578
|
+
WHERE NOT EXISTS (SELECT 1 FROM completed WHERE completed.ord = inputs.ord)
|
|
4579
|
+
AND ${workReceiptTerminalReplayPredicateSql({
|
|
4580
|
+
receiptTable: 'target',
|
|
4581
|
+
terminalStatusSql: '$6',
|
|
4582
|
+
ownerRunIdSql: 'inputs.run_id',
|
|
4583
|
+
})}
|
|
4584
|
+
),
|
|
4585
|
+
resolved AS (
|
|
4586
|
+
SELECT ord, k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM completed
|
|
4587
|
+
UNION ALL
|
|
4588
|
+
SELECT ord, k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM replay
|
|
4589
|
+
),
|
|
4481
4590
|
returned AS (
|
|
4482
|
-
SELECT
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
4591
|
+
SELECT resolved.k,
|
|
4592
|
+
resolved.status,
|
|
4593
|
+
resolved.output,
|
|
4594
|
+
resolved.error,
|
|
4595
|
+
resolved.failure_kind,
|
|
4596
|
+
resolved.run_id,
|
|
4597
|
+
resolved.lease_id,
|
|
4598
|
+
resolved.lease_owner_run_id,
|
|
4599
|
+
resolved.lease_owner_attempt,
|
|
4600
|
+
resolved.lease_expires_at,
|
|
4601
|
+
resolved.updated_at,
|
|
4493
4602
|
inputs.ord
|
|
4494
4603
|
FROM inputs
|
|
4495
|
-
LEFT JOIN
|
|
4604
|
+
LEFT JOIN resolved ON resolved.ord = inputs.ord
|
|
4496
4605
|
)
|
|
4497
4606
|
SELECT convert_from(returned.k, 'UTF8') AS k,
|
|
4498
4607
|
returned.status,
|
|
@@ -4584,8 +4693,31 @@ export async function failRuntimeWorkReceipt(
|
|
|
4584
4693
|
lease_owner_attempt,
|
|
4585
4694
|
lease_expires_at,
|
|
4586
4695
|
updated_at
|
|
4696
|
+
),
|
|
4697
|
+
replay AS (
|
|
4698
|
+
SELECT convert_from(k, 'UTF8') AS k,
|
|
4699
|
+
status,
|
|
4700
|
+
output,
|
|
4701
|
+
error,
|
|
4702
|
+
failure_kind,
|
|
4703
|
+
run_id,
|
|
4704
|
+
lease_id,
|
|
4705
|
+
lease_owner_run_id,
|
|
4706
|
+
lease_owner_attempt,
|
|
4707
|
+
lease_expires_at,
|
|
4708
|
+
updated_at
|
|
4709
|
+
FROM ${workReceiptTable(session)}
|
|
4710
|
+
WHERE k = decode($1, 'hex')
|
|
4711
|
+
AND NOT EXISTS (SELECT 1 FROM failed)
|
|
4712
|
+
AND ${workReceiptTerminalReplayPredicateSql({
|
|
4713
|
+
receiptTable: workReceiptTable(session),
|
|
4714
|
+
terminalStatusSql: '$2',
|
|
4715
|
+
ownerRunIdSql: '$4',
|
|
4716
|
+
})}
|
|
4587
4717
|
)
|
|
4588
4718
|
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM failed
|
|
4719
|
+
UNION ALL
|
|
4720
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM replay
|
|
4589
4721
|
`,
|
|
4590
4722
|
[
|
|
4591
4723
|
workReceiptKeyHex(input.key),
|
|
@@ -4749,21 +4881,49 @@ export async function failRuntimeWorkReceipts(
|
|
|
4749
4881
|
target.updated_at,
|
|
4750
4882
|
inputs.ord
|
|
4751
4883
|
),
|
|
4884
|
+
replay AS (
|
|
4885
|
+
SELECT target.k,
|
|
4886
|
+
target.status,
|
|
4887
|
+
target.output,
|
|
4888
|
+
target.error,
|
|
4889
|
+
target.failure_kind,
|
|
4890
|
+
target.run_id,
|
|
4891
|
+
target.lease_id,
|
|
4892
|
+
target.lease_owner_run_id,
|
|
4893
|
+
target.lease_owner_attempt,
|
|
4894
|
+
target.lease_expires_at,
|
|
4895
|
+
target.updated_at,
|
|
4896
|
+
inputs.ord
|
|
4897
|
+
FROM inputs
|
|
4898
|
+
JOIN ${workReceiptTable(session)} AS target
|
|
4899
|
+
ON target.k = decode(inputs.key_hex, 'hex')
|
|
4900
|
+
WHERE NOT EXISTS (SELECT 1 FROM failed WHERE failed.ord = inputs.ord)
|
|
4901
|
+
AND ${workReceiptTerminalReplayPredicateSql({
|
|
4902
|
+
receiptTable: 'target',
|
|
4903
|
+
terminalStatusSql: '$7',
|
|
4904
|
+
ownerRunIdSql: 'inputs.run_id',
|
|
4905
|
+
})}
|
|
4906
|
+
),
|
|
4907
|
+
resolved AS (
|
|
4908
|
+
SELECT ord, k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM failed
|
|
4909
|
+
UNION ALL
|
|
4910
|
+
SELECT ord, k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM replay
|
|
4911
|
+
),
|
|
4752
4912
|
returned AS (
|
|
4753
|
-
SELECT
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4913
|
+
SELECT resolved.k,
|
|
4914
|
+
resolved.status,
|
|
4915
|
+
resolved.output,
|
|
4916
|
+
resolved.error,
|
|
4917
|
+
resolved.failure_kind,
|
|
4918
|
+
resolved.run_id,
|
|
4919
|
+
resolved.lease_id,
|
|
4920
|
+
resolved.lease_owner_run_id,
|
|
4921
|
+
resolved.lease_owner_attempt,
|
|
4922
|
+
resolved.lease_expires_at,
|
|
4923
|
+
resolved.updated_at,
|
|
4764
4924
|
inputs.ord
|
|
4765
4925
|
FROM inputs
|
|
4766
|
-
LEFT JOIN
|
|
4926
|
+
LEFT JOIN resolved ON resolved.ord = inputs.ord
|
|
4767
4927
|
)
|
|
4768
4928
|
SELECT convert_from(returned.k, 'UTF8') AS k,
|
|
4769
4929
|
returned.status,
|
|
@@ -4992,6 +5152,12 @@ export async function startRuntimeSheetDataset(
|
|
|
4992
5152
|
force?: boolean;
|
|
4993
5153
|
},
|
|
4994
5154
|
): Promise<PrepareRuntimeSheetResult> {
|
|
5155
|
+
if (context.dbSessionStrategy === 'gateway_only') {
|
|
5156
|
+
return await postRuntimeApi<PrepareRuntimeSheetResult>(context, {
|
|
5157
|
+
action: 'runtime_sheet_start',
|
|
5158
|
+
input,
|
|
5159
|
+
});
|
|
5160
|
+
}
|
|
4995
5161
|
const totalStartedAt = Date.now();
|
|
4996
5162
|
const timings: RuntimeSheetTiming[] = [];
|
|
4997
5163
|
const playName = context.playName?.trim() || input.playName;
|
|
@@ -5383,6 +5549,12 @@ export async function releaseRuntimeSheetAttempt(
|
|
|
5383
5549
|
attemptSeq?: number | null;
|
|
5384
5550
|
},
|
|
5385
5551
|
): Promise<RuntimeSheetAttemptReleaseResult> {
|
|
5552
|
+
if (context.dbSessionStrategy === 'gateway_only') {
|
|
5553
|
+
return await postRuntimeApi<RuntimeSheetAttemptReleaseResult>(context, {
|
|
5554
|
+
action: 'runtime_sheet_release_attempt',
|
|
5555
|
+
input,
|
|
5556
|
+
});
|
|
5557
|
+
}
|
|
5386
5558
|
const playName = context.playName?.trim() || input.playName;
|
|
5387
5559
|
if (!playName) {
|
|
5388
5560
|
throw new Error('Runtime sheet attempt release requires a playName.');
|
|
@@ -5471,6 +5643,12 @@ export async function releaseRuntimeWorkReceipts(
|
|
|
5471
5643
|
runAttempt?: number | null;
|
|
5472
5644
|
},
|
|
5473
5645
|
): Promise<RuntimeWorkReceiptReleaseResult> {
|
|
5646
|
+
if (context.dbSessionStrategy === 'gateway_only') {
|
|
5647
|
+
return await postRuntimeApi<RuntimeWorkReceiptReleaseResult>(context, {
|
|
5648
|
+
action: 'runtime_receipts_release_attempt',
|
|
5649
|
+
input,
|
|
5650
|
+
});
|
|
5651
|
+
}
|
|
5474
5652
|
const runId = input.runId.trim();
|
|
5475
5653
|
if (!runId) {
|
|
5476
5654
|
return { released: 0, releasedKeys: [] };
|
|
@@ -5523,7 +5701,7 @@ type CompleteRuntimeMapRowChunksInput = {
|
|
|
5523
5701
|
outputFields: string[];
|
|
5524
5702
|
};
|
|
5525
5703
|
|
|
5526
|
-
type RuntimeMapRowsWriteResult = {
|
|
5704
|
+
export type RuntimeMapRowsWriteResult = {
|
|
5527
5705
|
updated: number;
|
|
5528
5706
|
fencedKeys: string[];
|
|
5529
5707
|
};
|
|
@@ -6444,6 +6622,12 @@ export async function completeRuntimeMapRows(
|
|
|
6444
6622
|
forceFailedRows?: boolean;
|
|
6445
6623
|
},
|
|
6446
6624
|
): Promise<RuntimeMapRowsWriteResult> {
|
|
6625
|
+
if (context.dbSessionStrategy === 'gateway_only') {
|
|
6626
|
+
return await postRuntimeApi<RuntimeMapRowsWriteResult>(context, {
|
|
6627
|
+
action: 'runtime_sheet_complete_map_rows',
|
|
6628
|
+
input,
|
|
6629
|
+
});
|
|
6630
|
+
}
|
|
6447
6631
|
if (input.rows.length === 0) {
|
|
6448
6632
|
return { updated: 0, fencedKeys: [] };
|
|
6449
6633
|
}
|
|
@@ -6598,6 +6782,12 @@ export async function readRuntimeSheetDatasetRows(
|
|
|
6598
6782
|
offset: number;
|
|
6599
6783
|
},
|
|
6600
6784
|
): Promise<{ rows: Record<string, unknown>[]; limit: number; offset: number }> {
|
|
6785
|
+
if (context.dbSessionStrategy === 'gateway_only') {
|
|
6786
|
+
return await postRuntimeApi(context, {
|
|
6787
|
+
action: 'runtime_sheet_read_rows',
|
|
6788
|
+
input,
|
|
6789
|
+
});
|
|
6790
|
+
}
|
|
6601
6791
|
const limit = Math.max(
|
|
6602
6792
|
1,
|
|
6603
6793
|
Math.min(DIRECT_POSTGRES_BATCH_SIZE, Math.floor(input.limit)),
|
|
@@ -6640,6 +6830,12 @@ export async function readRuntimeSheetDatasetRowKeys(
|
|
|
6640
6830
|
rowMode?: 'output' | 'all' | 'terminalAnyRun';
|
|
6641
6831
|
},
|
|
6642
6832
|
): Promise<{ keys: string[] }> {
|
|
6833
|
+
if (context.dbSessionStrategy === 'gateway_only') {
|
|
6834
|
+
return await postRuntimeApi(context, {
|
|
6835
|
+
action: 'runtime_sheet_read_row_keys',
|
|
6836
|
+
input,
|
|
6837
|
+
});
|
|
6838
|
+
}
|
|
6643
6839
|
const keys = [
|
|
6644
6840
|
...new Set(
|
|
6645
6841
|
input.keys.map((key) => key.trim()).filter((key) => key.length > 0),
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runtime-scheduler topology predicate shared by the app-side lane helpers
|
|
3
|
+
* (`src/lib/plays/scheduler-backends/absurd-shared.ts`) and the runner
|
|
4
|
+
* backends (`runner-backends/backends/daytona-lifecycle.ts`).
|
|
5
|
+
*
|
|
6
|
+
* ONE distinction drives several guards: a run whose scheduler schema is the
|
|
7
|
+
* prod default rides the production topology (base queue names, customer code,
|
|
8
|
+
* production egress policy), while any other schema is an ISOLATED topology —
|
|
9
|
+
* a per-PR preview fleet or a per-worktree dev fleet running synthetic
|
|
10
|
+
* internal test plays. Keeping the predicate here (dependency-free) lets
|
|
11
|
+
* shared_libs code use it without importing the app-side scheduler stack.
|
|
12
|
+
*/
|
|
13
|
+
export const PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA = 'runtime_scheduler';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* True when the scheduler schema names an isolated (preview CI / worktree)
|
|
17
|
+
* topology. Null/empty/prod-default => false (production topology). Callers
|
|
18
|
+
* that gate SECURITY policy on this must treat `false` as the fail-closed
|
|
19
|
+
* production posture — an absent schema is production, never "unknown".
|
|
20
|
+
*/
|
|
21
|
+
export function isIsolatedRuntimeSchedulerSchema(
|
|
22
|
+
schedulerSchema: string | null | undefined,
|
|
23
|
+
): boolean {
|
|
24
|
+
const schema = schedulerSchema?.trim();
|
|
25
|
+
return Boolean(schema) && schema !== PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA;
|
|
26
|
+
}
|