deepline 0.1.201 → 0.1.202

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +30 -9
  2. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +25 -0
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +139 -124
  4. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  5. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +17 -2
  6. package/dist/bundling-sources/shared_libs/play-runtime/bounded-dispatch.ts +52 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +152 -23
  8. package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +10 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +11 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +121 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts +6 -1
  12. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-budget-state-backend.ts +22 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +86 -19
  14. package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts +141 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/governor/budget-state-backend.ts +43 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +129 -41
  17. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +40 -2
  18. package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +92 -33
  19. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +13 -5
  20. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +122 -120
  21. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +10 -8
  22. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +140 -32
  23. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +225 -29
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts +26 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +21 -0
  26. package/dist/bundling-sources/shared_libs/plays/dataset.ts +32 -10
  27. package/dist/cli/index.js +198 -76
  28. package/dist/cli/index.mjs +204 -82
  29. package/dist/index.js +3 -2
  30. package/dist/index.mjs +3 -2
  31. package/package.json +1 -1
  32. package/dist/bundling-sources/shared_libs/play-runtime/adaptive-tool-dispatcher.ts +0 -355
@@ -46,6 +46,71 @@ const DAYTONA_RUNTIME_DB_CONNECT_MAX_ATTEMPTS = 2;
46
46
  const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
47
47
  const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
48
48
  const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
49
+ /**
50
+ * Client-side wall-clock grace for `sandbox.process.executeCommand`. The
51
+ * `timeout` argument we pass is a SERVER-side directive; the Daytona SDK's
52
+ * underlying HTTP call has no client deadline, so a wedged socket (observed:
53
+ * `AggregateError [ETIMEDOUT]`) leaves the await hanging forever and holds the
54
+ * worker slot. We race the call against this deadline = server timeout + grace,
55
+ * so a genuine server-side runtime-limit hit still returns first, but a wedged
56
+ * client transport fails the attempt loudly and lets the engine retry ladder
57
+ * take over instead of livelocking the slot.
58
+ */
59
+ const DAYTONA_EXECUTE_CLIENT_GRACE_MS = 60_000;
60
+ const DAYTONA_EXECUTE_CLIENT_DEADLINE_MS =
61
+ DAYTONA_EXECUTE_TIMEOUT_SECONDS * 1_000 + DAYTONA_EXECUTE_CLIENT_GRACE_MS;
62
+ /**
63
+ * Bound for individual sandbox/runtime API calls that have no natural timeout
64
+ * (e.g. recording the acquired runtime resource). A wedged one of these must
65
+ * fail the attempt, not hold the slot.
66
+ */
67
+ const DAYTONA_API_CALL_DEADLINE_MS = 60_000;
68
+
69
+ export class DaytonaOperationDeadlineError extends Error {
70
+ readonly operation: string;
71
+ readonly deadlineMs: number;
72
+
73
+ constructor(input: { operation: string; deadlineMs: number }) {
74
+ super(
75
+ `Daytona operation "${input.operation}" exceeded client deadline of ${input.deadlineMs}ms and was abandoned to free the worker slot.`,
76
+ );
77
+ this.name = 'DaytonaOperationDeadlineError';
78
+ this.operation = input.operation;
79
+ this.deadlineMs = input.deadlineMs;
80
+ }
81
+ }
82
+
83
+ /**
84
+ * Race `promise` against a wall-clock deadline. On breach, reject loudly with a
85
+ * `DaytonaOperationDeadlineError` (the caller's retry/failure classification
86
+ * decides what happens next). The timer is always cleared so a fast-resolving
87
+ * promise never leaves a dangling handle.
88
+ */
89
+ export async function withDaytonaOperationDeadline<T>(input: {
90
+ operation: string;
91
+ deadlineMs: number;
92
+ promise: Promise<T>;
93
+ }): Promise<T> {
94
+ let deadlineTimer: ReturnType<typeof setTimeout> | null = null;
95
+ const deadlinePromise = new Promise<never>((_resolve, reject) => {
96
+ deadlineTimer = setTimeout(() => {
97
+ reject(
98
+ new DaytonaOperationDeadlineError({
99
+ operation: input.operation,
100
+ deadlineMs: input.deadlineMs,
101
+ }),
102
+ );
103
+ }, input.deadlineMs);
104
+ deadlineTimer.unref?.();
105
+ });
106
+ try {
107
+ return await Promise.race([input.promise, deadlinePromise]);
108
+ } finally {
109
+ if (deadlineTimer) {
110
+ clearTimeout(deadlineTimer);
111
+ }
112
+ }
113
+ }
49
114
  const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
50
115
  /\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
51
116
  const DAYTONA_INFRASTRUCTURE_RETRY_PATTERN =
@@ -223,10 +288,11 @@ function createDaytonaProgressSidecarPoller(input: {
223
288
  context: DaytonaExecutionContext;
224
289
  startedAt: number;
225
290
  processedEventKeys: Set<string>;
226
- }): { start(): void; stop(): Promise<void> } {
291
+ }): { start(): void; stop(flush?: boolean): Promise<void> } {
227
292
  let timer: ReturnType<typeof setTimeout> | null = null;
228
293
  let stopped = false;
229
294
  let inFlight: Promise<void> = Promise.resolve();
295
+ let segment = 0;
230
296
  let offsetBytes = 0;
231
297
  let pendingLine = '';
232
298
  let warned = false;
@@ -259,20 +325,39 @@ function createDaytonaProgressSidecarPoller(input: {
259
325
 
260
326
  const poll = async (flushPartial = false): Promise<void> => {
261
327
  try {
262
- const file = await input.sandbox.fs.downloadFile(
328
+ const manifest = await input.sandbox.fs.downloadFile(
263
329
  input.progressEventPath,
264
330
  5,
265
331
  );
266
- const buffer = Buffer.isBuffer(file) ? file : Buffer.from(file);
267
- if (buffer.length < offsetBytes) {
332
+ const parsedLatestSegment = Number.parseInt(
333
+ Buffer.isBuffer(manifest)
334
+ ? manifest.toString('utf-8').trim()
335
+ : String(manifest).trim(),
336
+ 10,
337
+ );
338
+ const latestSegment = Number.isFinite(parsedLatestSegment)
339
+ ? Math.max(segment, parsedLatestSegment)
340
+ : segment;
341
+ while (segment <= latestSegment) {
342
+ const segmentPath = `${input.progressEventPath}.${String(
343
+ segment,
344
+ ).padStart(6, '0')}`;
345
+ const file = await input.sandbox.fs.downloadFile(segmentPath, 5);
346
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(file);
347
+ if (buffer.length < offsetBytes) {
348
+ offsetBytes = 0;
349
+ pendingLine = '';
350
+ }
351
+ if (buffer.length > offsetBytes) {
352
+ const chunk = buffer.subarray(offsetBytes).toString('utf-8');
353
+ offsetBytes = buffer.length;
354
+ processText(chunk, flushPartial && segment === latestSegment);
355
+ }
356
+ if (segment >= latestSegment) break;
357
+ segment += 1;
268
358
  offsetBytes = 0;
269
- pendingLine = '';
270
359
  }
271
- if (buffer.length > offsetBytes) {
272
- const chunk = buffer.subarray(offsetBytes).toString('utf-8');
273
- offsetBytes = buffer.length;
274
- processText(chunk, flushPartial);
275
- } else if (flushPartial && pendingLine.trim()) {
360
+ if (flushPartial && pendingLine.trim()) {
276
361
  processText('\n', true);
277
362
  }
278
363
  } catch (error) {
@@ -304,14 +389,14 @@ function createDaytonaProgressSidecarPoller(input: {
304
389
  start() {
305
390
  inFlight = inFlight.then(() => poll()).finally(schedule);
306
391
  },
307
- async stop() {
392
+ async stop(flush = true) {
308
393
  stopped = true;
309
394
  if (timer) {
310
395
  clearTimeout(timer);
311
396
  timer = null;
312
397
  }
313
398
  await inFlight.catch(() => {});
314
- await poll(true);
399
+ if (flush) await poll(true);
315
400
  },
316
401
  };
317
402
  }
@@ -422,6 +507,12 @@ function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
422
507
  if (error instanceof DaytonaCommandTransportError) {
423
508
  return false;
424
509
  }
510
+ // A client-side deadline breach is a wedged transport (hung socket / SDK call
511
+ // with no client timeout). Retry once on a fresh sandbox before spending an
512
+ // engine attempt, same as the other transport-level infrastructure failures.
513
+ if (error instanceof DaytonaOperationDeadlineError) {
514
+ return true;
515
+ }
425
516
  const message = error instanceof Error ? error.message : String(error);
426
517
  if (!message || /cancelled|aborted/i.test(message)) {
427
518
  return false;
@@ -566,15 +657,24 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
566
657
  acquiredSandbox.billingStartedAt - startedAt;
567
658
  const sandbox = acquiredSandbox.sandbox;
568
659
  sandboxForAttempt = sandbox;
569
- await callbacks?.onRuntimeResourceAcquired?.({
570
- kind: 'daytona_sandbox',
571
- sandboxId: sandbox.id,
572
- billingStartedAt: acquiredSandbox.billingStartedAt,
573
- cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
574
- memoryGiB:
575
- typeof sandbox.memory === 'number' ? sandbox.memory : null,
576
- diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : null,
577
- });
660
+ const resourceAcquiredNotice = callbacks?.onRuntimeResourceAcquired?.(
661
+ {
662
+ kind: 'daytona_sandbox',
663
+ sandboxId: sandbox.id,
664
+ billingStartedAt: acquiredSandbox.billingStartedAt,
665
+ cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
666
+ memoryGiB:
667
+ typeof sandbox.memory === 'number' ? sandbox.memory : null,
668
+ diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : null,
669
+ },
670
+ );
671
+ if (resourceAcquiredNotice) {
672
+ await withDaytonaOperationDeadline({
673
+ operation: 'onRuntimeResourceAcquired',
674
+ deadlineMs: DAYTONA_API_CALL_DEADLINE_MS,
675
+ promise: resourceAcquiredNotice,
676
+ });
677
+ }
578
678
  throwIfCancellationRequested();
579
679
 
580
680
  const { workdir: workDir } = loadDaytonaRunnerPathsConfig();
@@ -676,16 +776,19 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
676
776
  try {
677
777
  progressSidecar.start();
678
778
  execution = await Promise.race([
679
- sandbox.process.executeCommand(
680
- stagedPayload.command,
681
- stagedPayload.workDir,
682
- undefined,
683
- DAYTONA_EXECUTE_TIMEOUT_SECONDS,
684
- ),
779
+ withDaytonaOperationDeadline({
780
+ operation: 'sandbox.process.executeCommand',
781
+ deadlineMs: DAYTONA_EXECUTE_CLIENT_DEADLINE_MS,
782
+ promise: sandbox.process.executeCommand(
783
+ stagedPayload.command,
784
+ stagedPayload.workDir,
785
+ undefined,
786
+ DAYTONA_EXECUTE_TIMEOUT_SECONDS,
787
+ ),
788
+ }),
685
789
  cancellationPromise,
686
790
  ]);
687
791
  } catch (error) {
688
- await progressSidecar.stop();
689
792
  if (
690
793
  cancellationRequested ||
691
794
  callbacks?.cancellationSignal?.aborted
@@ -729,7 +832,10 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
729
832
  });
730
833
  execution = recovered.execution;
731
834
  } finally {
732
- await progressSidecar.stop();
835
+ await progressSidecar.stop(
836
+ !cancellationRequested &&
837
+ callbacks?.cancellationSignal?.aborted !== true,
838
+ );
733
839
  }
734
840
  const runnerOutput = await resolveDaytonaCommandOutput({
735
841
  sandbox,
@@ -818,9 +924,11 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
818
924
  error:
819
925
  typeof execution.exitCode === 'number' &&
820
926
  execution.exitCode !== 0
821
- ? `Daytona play runner exited with code ${execution.exitCode}.${
822
- outputTail ? ` Output tail: ${outputTail}` : ''
823
- }`
927
+ ? `Daytona play runner exited with code ${execution.exitCode}${
928
+ execution.exitCode === 137
929
+ ? ' (sandbox OOM or forced SIGKILL)'
930
+ : ''
931
+ }.${outputTail ? ` Output tail: ${outputTail}` : ''}`
824
932
  : 'Daytona play runner produced no result.',
825
933
  runtimeTiming,
826
934
  }),
@@ -115,7 +115,11 @@ type RuntimeApiContext = {
115
115
  runId?: string | null;
116
116
  userEmail?: string | null;
117
117
  integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
118
- dbSessionStrategy?: 'preloaded' | 'sandbox_public_key' | null;
118
+ dbSessionStrategy?:
119
+ | 'preloaded'
120
+ | 'sandbox_public_key'
121
+ | 'gateway_only'
122
+ | null;
119
123
  preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
120
124
  vercelProtectionBypassToken?: string | null;
121
125
  runtimeTestFaultHeader?: string | null;
@@ -443,9 +447,33 @@ type RuntimeApiActionRequest =
443
447
  action: 'repair_runtime_storage_grants';
444
448
  playName: string;
445
449
  runId?: string | null;
450
+ }
451
+ | {
452
+ action: 'runtime_sheet_start';
453
+ input: Parameters<typeof startRuntimeSheetDataset>[1];
454
+ }
455
+ | {
456
+ action: 'runtime_sheet_complete_map_rows';
457
+ input: Parameters<typeof completeRuntimeMapRows>[1];
458
+ }
459
+ | {
460
+ action: 'runtime_sheet_read_rows';
461
+ input: Parameters<typeof readRuntimeSheetDatasetRows>[1];
462
+ }
463
+ | {
464
+ action: 'runtime_sheet_read_row_keys';
465
+ input: Parameters<typeof readRuntimeSheetDatasetRowKeys>[1];
466
+ }
467
+ | {
468
+ action: 'runtime_sheet_release_attempt';
469
+ input: Parameters<typeof releaseRuntimeSheetAttempt>[1];
470
+ }
471
+ | {
472
+ action: 'runtime_receipts_release_attempt';
473
+ input: Parameters<typeof releaseRuntimeWorkReceipts>[1];
446
474
  };
447
475
 
448
- type RuntimeApiRowRecord = MapRowOutcome & {
476
+ export type RuntimeApiRowRecord = MapRowOutcome & {
449
477
  inputIndex?: number | null;
450
478
  };
451
479
 
@@ -614,8 +642,8 @@ function runtimeApiMaxAttempts(
614
642
  return runtimeApiRetryDelaysForAction(action).length + 1;
615
643
  }
616
644
 
617
- function isTransientRuntimeApiErrorMessage(message: string): boolean {
618
- return /timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|requested endpoint could not be found, or you don't have access/i.test(
645
+ export function isTransientRuntimeApiErrorMessage(message: string): boolean {
646
+ return /timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|requested endpoint could not be found, or you don't have access|RUNTIME_SCHEDULER_SATURATED|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open/i.test(
619
647
  message,
620
648
  );
621
649
  }
@@ -4258,6 +4286,36 @@ export async function markRuntimeWorkReceiptsQueued(
4258
4286
  );
4259
4287
  }
4260
4288
 
4289
+ /**
4290
+ * Replay-idempotency fence for a receipt whose active complete/fail UPDATE
4291
+ * matched 0 rows. This fires when the durable write committed but the HTTP
4292
+ * response was lost and the SAME caller retries: the row is already terminal,
4293
+ * so the active-owner predicate (which requires status IN (queued, running))
4294
+ * can never match again. We surface the already-terminal row as success ONLY
4295
+ * when its run identity matches the retrying caller, so the retry does not
4296
+ * spuriously fail a receipt whose provider spend and durable write both
4297
+ * succeeded.
4298
+ *
4299
+ * The predicate mirrors the run-identity half of {@link
4300
+ * workReceiptActiveOwnerPredicateSql}: `COALESCE(lease_owner_run_id, run_id)`
4301
+ * must equal the caller's run id. complete/fail null the lease-owner and
4302
+ * attempt columns as part of the terminal write, so run id is the only durable
4303
+ * ownership marker left on a terminal row — a different run (the stale-owner
4304
+ * rejection case) never matches and still resolves to null exactly as before.
4305
+ * This is replay-idempotency only; it never lets a foreign run read or claim a
4306
+ * receipt it did not complete.
4307
+ */
4308
+ function workReceiptTerminalReplayPredicateSql(input: {
4309
+ receiptTable: string;
4310
+ terminalStatusSql: string;
4311
+ ownerRunIdSql: string;
4312
+ }): string {
4313
+ const table = input.receiptTable;
4314
+ return `${table}.status = ${input.terminalStatusSql}::smallint
4315
+ /* replay-idempotency: same-run retry after a lost response; run identity is the only durable owner marker on a terminal row */
4316
+ AND COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.ownerRunIdSql}`;
4317
+ }
4318
+
4261
4319
  export async function completeRuntimeWorkReceipt(
4262
4320
  context: RuntimeApiContext,
4263
4321
  input: {
@@ -4340,8 +4398,31 @@ export async function completeRuntimeWorkReceipt(
4340
4398
  lease_owner_attempt,
4341
4399
  lease_expires_at,
4342
4400
  updated_at
4401
+ ),
4402
+ replay AS (
4403
+ SELECT convert_from(k, 'UTF8') AS k,
4404
+ status,
4405
+ output,
4406
+ error,
4407
+ failure_kind,
4408
+ run_id,
4409
+ lease_id,
4410
+ lease_owner_run_id,
4411
+ lease_owner_attempt,
4412
+ lease_expires_at,
4413
+ updated_at
4414
+ FROM ${workReceiptTable(session)}
4415
+ WHERE k = decode($1, 'hex')
4416
+ AND NOT EXISTS (SELECT 1 FROM completed)
4417
+ AND ${workReceiptTerminalReplayPredicateSql({
4418
+ receiptTable: workReceiptTable(session),
4419
+ terminalStatusSql: '$2',
4420
+ ownerRunIdSql: '$4',
4421
+ })}
4343
4422
  )
4344
4423
  SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM completed
4424
+ UNION ALL
4425
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM replay
4345
4426
  `,
4346
4427
  [
4347
4428
  workReceiptKeyHex(input.key),
@@ -4478,21 +4559,49 @@ export async function completeRuntimeWorkReceipts(
4478
4559
  target.updated_at,
4479
4560
  inputs.ord
4480
4561
  ),
4562
+ replay AS (
4563
+ SELECT target.k,
4564
+ target.status,
4565
+ target.output,
4566
+ target.error,
4567
+ target.failure_kind,
4568
+ target.run_id,
4569
+ target.lease_id,
4570
+ target.lease_owner_run_id,
4571
+ target.lease_owner_attempt,
4572
+ target.lease_expires_at,
4573
+ target.updated_at,
4574
+ inputs.ord
4575
+ FROM inputs
4576
+ JOIN ${workReceiptTable(session)} AS target
4577
+ ON target.k = decode(inputs.key_hex, 'hex')
4578
+ WHERE NOT EXISTS (SELECT 1 FROM completed WHERE completed.ord = inputs.ord)
4579
+ AND ${workReceiptTerminalReplayPredicateSql({
4580
+ receiptTable: 'target',
4581
+ terminalStatusSql: '$6',
4582
+ ownerRunIdSql: 'inputs.run_id',
4583
+ })}
4584
+ ),
4585
+ resolved AS (
4586
+ SELECT ord, k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM completed
4587
+ UNION ALL
4588
+ SELECT ord, k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM replay
4589
+ ),
4481
4590
  returned AS (
4482
- SELECT completed.k,
4483
- completed.status,
4484
- completed.output,
4485
- completed.error,
4486
- completed.failure_kind,
4487
- completed.run_id,
4488
- completed.lease_id,
4489
- completed.lease_owner_run_id,
4490
- completed.lease_owner_attempt,
4491
- completed.lease_expires_at,
4492
- completed.updated_at,
4591
+ SELECT resolved.k,
4592
+ resolved.status,
4593
+ resolved.output,
4594
+ resolved.error,
4595
+ resolved.failure_kind,
4596
+ resolved.run_id,
4597
+ resolved.lease_id,
4598
+ resolved.lease_owner_run_id,
4599
+ resolved.lease_owner_attempt,
4600
+ resolved.lease_expires_at,
4601
+ resolved.updated_at,
4493
4602
  inputs.ord
4494
4603
  FROM inputs
4495
- LEFT JOIN completed ON completed.ord = inputs.ord
4604
+ LEFT JOIN resolved ON resolved.ord = inputs.ord
4496
4605
  )
4497
4606
  SELECT convert_from(returned.k, 'UTF8') AS k,
4498
4607
  returned.status,
@@ -4584,8 +4693,31 @@ export async function failRuntimeWorkReceipt(
4584
4693
  lease_owner_attempt,
4585
4694
  lease_expires_at,
4586
4695
  updated_at
4696
+ ),
4697
+ replay AS (
4698
+ SELECT convert_from(k, 'UTF8') AS k,
4699
+ status,
4700
+ output,
4701
+ error,
4702
+ failure_kind,
4703
+ run_id,
4704
+ lease_id,
4705
+ lease_owner_run_id,
4706
+ lease_owner_attempt,
4707
+ lease_expires_at,
4708
+ updated_at
4709
+ FROM ${workReceiptTable(session)}
4710
+ WHERE k = decode($1, 'hex')
4711
+ AND NOT EXISTS (SELECT 1 FROM failed)
4712
+ AND ${workReceiptTerminalReplayPredicateSql({
4713
+ receiptTable: workReceiptTable(session),
4714
+ terminalStatusSql: '$2',
4715
+ ownerRunIdSql: '$4',
4716
+ })}
4587
4717
  )
4588
4718
  SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM failed
4719
+ UNION ALL
4720
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM replay
4589
4721
  `,
4590
4722
  [
4591
4723
  workReceiptKeyHex(input.key),
@@ -4749,21 +4881,49 @@ export async function failRuntimeWorkReceipts(
4749
4881
  target.updated_at,
4750
4882
  inputs.ord
4751
4883
  ),
4884
+ replay AS (
4885
+ SELECT target.k,
4886
+ target.status,
4887
+ target.output,
4888
+ target.error,
4889
+ target.failure_kind,
4890
+ target.run_id,
4891
+ target.lease_id,
4892
+ target.lease_owner_run_id,
4893
+ target.lease_owner_attempt,
4894
+ target.lease_expires_at,
4895
+ target.updated_at,
4896
+ inputs.ord
4897
+ FROM inputs
4898
+ JOIN ${workReceiptTable(session)} AS target
4899
+ ON target.k = decode(inputs.key_hex, 'hex')
4900
+ WHERE NOT EXISTS (SELECT 1 FROM failed WHERE failed.ord = inputs.ord)
4901
+ AND ${workReceiptTerminalReplayPredicateSql({
4902
+ receiptTable: 'target',
4903
+ terminalStatusSql: '$7',
4904
+ ownerRunIdSql: 'inputs.run_id',
4905
+ })}
4906
+ ),
4907
+ resolved AS (
4908
+ SELECT ord, k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM failed
4909
+ UNION ALL
4910
+ SELECT ord, k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM replay
4911
+ ),
4752
4912
  returned AS (
4753
- SELECT failed.k,
4754
- failed.status,
4755
- failed.output,
4756
- failed.error,
4757
- failed.failure_kind,
4758
- failed.run_id,
4759
- failed.lease_id,
4760
- failed.lease_owner_run_id,
4761
- failed.lease_owner_attempt,
4762
- failed.lease_expires_at,
4763
- failed.updated_at,
4913
+ SELECT resolved.k,
4914
+ resolved.status,
4915
+ resolved.output,
4916
+ resolved.error,
4917
+ resolved.failure_kind,
4918
+ resolved.run_id,
4919
+ resolved.lease_id,
4920
+ resolved.lease_owner_run_id,
4921
+ resolved.lease_owner_attempt,
4922
+ resolved.lease_expires_at,
4923
+ resolved.updated_at,
4764
4924
  inputs.ord
4765
4925
  FROM inputs
4766
- LEFT JOIN failed ON failed.ord = inputs.ord
4926
+ LEFT JOIN resolved ON resolved.ord = inputs.ord
4767
4927
  )
4768
4928
  SELECT convert_from(returned.k, 'UTF8') AS k,
4769
4929
  returned.status,
@@ -4992,6 +5152,12 @@ export async function startRuntimeSheetDataset(
4992
5152
  force?: boolean;
4993
5153
  },
4994
5154
  ): Promise<PrepareRuntimeSheetResult> {
5155
+ if (context.dbSessionStrategy === 'gateway_only') {
5156
+ return await postRuntimeApi<PrepareRuntimeSheetResult>(context, {
5157
+ action: 'runtime_sheet_start',
5158
+ input,
5159
+ });
5160
+ }
4995
5161
  const totalStartedAt = Date.now();
4996
5162
  const timings: RuntimeSheetTiming[] = [];
4997
5163
  const playName = context.playName?.trim() || input.playName;
@@ -5383,6 +5549,12 @@ export async function releaseRuntimeSheetAttempt(
5383
5549
  attemptSeq?: number | null;
5384
5550
  },
5385
5551
  ): Promise<RuntimeSheetAttemptReleaseResult> {
5552
+ if (context.dbSessionStrategy === 'gateway_only') {
5553
+ return await postRuntimeApi<RuntimeSheetAttemptReleaseResult>(context, {
5554
+ action: 'runtime_sheet_release_attempt',
5555
+ input,
5556
+ });
5557
+ }
5386
5558
  const playName = context.playName?.trim() || input.playName;
5387
5559
  if (!playName) {
5388
5560
  throw new Error('Runtime sheet attempt release requires a playName.');
@@ -5471,6 +5643,12 @@ export async function releaseRuntimeWorkReceipts(
5471
5643
  runAttempt?: number | null;
5472
5644
  },
5473
5645
  ): Promise<RuntimeWorkReceiptReleaseResult> {
5646
+ if (context.dbSessionStrategy === 'gateway_only') {
5647
+ return await postRuntimeApi<RuntimeWorkReceiptReleaseResult>(context, {
5648
+ action: 'runtime_receipts_release_attempt',
5649
+ input,
5650
+ });
5651
+ }
5474
5652
  const runId = input.runId.trim();
5475
5653
  if (!runId) {
5476
5654
  return { released: 0, releasedKeys: [] };
@@ -5523,7 +5701,7 @@ type CompleteRuntimeMapRowChunksInput = {
5523
5701
  outputFields: string[];
5524
5702
  };
5525
5703
 
5526
- type RuntimeMapRowsWriteResult = {
5704
+ export type RuntimeMapRowsWriteResult = {
5527
5705
  updated: number;
5528
5706
  fencedKeys: string[];
5529
5707
  };
@@ -6444,6 +6622,12 @@ export async function completeRuntimeMapRows(
6444
6622
  forceFailedRows?: boolean;
6445
6623
  },
6446
6624
  ): Promise<RuntimeMapRowsWriteResult> {
6625
+ if (context.dbSessionStrategy === 'gateway_only') {
6626
+ return await postRuntimeApi<RuntimeMapRowsWriteResult>(context, {
6627
+ action: 'runtime_sheet_complete_map_rows',
6628
+ input,
6629
+ });
6630
+ }
6447
6631
  if (input.rows.length === 0) {
6448
6632
  return { updated: 0, fencedKeys: [] };
6449
6633
  }
@@ -6598,6 +6782,12 @@ export async function readRuntimeSheetDatasetRows(
6598
6782
  offset: number;
6599
6783
  },
6600
6784
  ): Promise<{ rows: Record<string, unknown>[]; limit: number; offset: number }> {
6785
+ if (context.dbSessionStrategy === 'gateway_only') {
6786
+ return await postRuntimeApi(context, {
6787
+ action: 'runtime_sheet_read_rows',
6788
+ input,
6789
+ });
6790
+ }
6601
6791
  const limit = Math.max(
6602
6792
  1,
6603
6793
  Math.min(DIRECT_POSTGRES_BATCH_SIZE, Math.floor(input.limit)),
@@ -6640,6 +6830,12 @@ export async function readRuntimeSheetDatasetRowKeys(
6640
6830
  rowMode?: 'output' | 'all' | 'terminalAnyRun';
6641
6831
  },
6642
6832
  ): Promise<{ keys: string[] }> {
6833
+ if (context.dbSessionStrategy === 'gateway_only') {
6834
+ return await postRuntimeApi(context, {
6835
+ action: 'runtime_sheet_read_row_keys',
6836
+ input,
6837
+ });
6838
+ }
6643
6839
  const keys = [
6644
6840
  ...new Set(
6645
6841
  input.keys.map((key) => key.trim()).filter((key) => key.length > 0),
@@ -0,0 +1,26 @@
1
+ /**
2
+ * Runtime-scheduler topology predicate shared by the app-side lane helpers
3
+ * (`src/lib/plays/scheduler-backends/absurd-shared.ts`) and the runner
4
+ * backends (`runner-backends/backends/daytona-lifecycle.ts`).
5
+ *
6
+ * ONE distinction drives several guards: a run whose scheduler schema is the
7
+ * prod default rides the production topology (base queue names, customer code,
8
+ * production egress policy), while any other schema is an ISOLATED topology —
9
+ * a per-PR preview fleet or a per-worktree dev fleet running synthetic
10
+ * internal test plays. Keeping the predicate here (dependency-free) lets
11
+ * shared_libs code use it without importing the app-side scheduler stack.
12
+ */
13
+ export const PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA = 'runtime_scheduler';
14
+
15
+ /**
16
+ * True when the scheduler schema names an isolated (preview CI / worktree)
17
+ * topology. Null/empty/prod-default => false (production topology). Callers
18
+ * that gate SECURITY policy on this must treat `false` as the fail-closed
19
+ * production posture — an absent schema is production, never "unknown".
20
+ */
21
+ export function isIsolatedRuntimeSchedulerSchema(
22
+ schedulerSchema: string | null | undefined,
23
+ ): boolean {
24
+ const schema = schedulerSchema?.trim();
25
+ return Boolean(schema) && schema !== PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA;
26
+ }