deepline 0.3.289 → 0.3.290

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +30 -10
  3. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +108 -12
  4. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +1 -1
  5. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
  8. package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +17 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +6 -5
  10. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +7 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/run-event-catalog.ts +2 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +31 -3
  14. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +23 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +30 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +38 -3
  17. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +2 -3
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +22 -1
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +1 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool-options.ts +145 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +1 -5
  23. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -139
  24. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +19 -5
  25. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +37 -15
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +7 -5
  27. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +961 -281
  28. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +55 -25
  29. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +97 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +33 -14
  31. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +452 -43
  32. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +10 -1
  33. package/dist/bundling-sources/shared_libs/play-runtime/test-admission-batching.ts +103 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +75 -28
  35. package/dist/cli/index.js +3 -1
  36. package/dist/cli/index.mjs +3 -1
  37. package/dist/index.js +3 -1
  38. package/dist/index.mjs +3 -1
  39. package/dist/install-integrity.json +3 -0
  40. package/dist/release.d.mts +1 -1
  41. package/dist/release.d.ts +1 -1
  42. package/dist/release.js +1 -1
  43. package/dist/release.mjs +1 -1
  44. package/package.json +1 -1
@@ -320,6 +320,7 @@ export type PlayRunLedgerEvent =
320
320
  lines: string[];
321
321
  /** Search-only context aligned to `lines`; stripped before Convex. */
322
322
  lineContexts?: Array<PlayLogContext | null>;
323
+ lineIds?: Array<string | null>;
323
324
  /**
324
325
  * Absolute per-run sequence (1-based) of `lines[0]`. Assigned by Run
325
326
  * Log Stream ingestion in Convex only; producers never set it.
@@ -430,6 +431,7 @@ export type PlayRunLedgerStatusPatch = {
430
431
  liveLogs?: readonly string[] | null;
431
432
  /** Search-only context aligned to `liveLogs`; stripped before Convex. */
432
433
  liveLogContexts?: readonly (PlayLogContext | null)[] | null;
434
+ liveLogIds?: readonly (string | null)[] | null;
433
435
  /**
434
436
  * Positional cursor for `liveLogs[0]` on this producer's log channel (the
435
437
  * count of lines emitted before it). Computed by the producer-side cursor
@@ -2369,12 +2371,14 @@ function terminalEventTypeForStatus(
2369
2371
  export function slicePositionalLogLines(input: {
2370
2372
  bufferLines: readonly string[];
2371
2373
  bufferContexts?: readonly (PlayLogContext | null)[];
2374
+ bufferIds?: readonly (string | null)[];
2372
2375
  bufferTotalCount: number;
2373
2376
  sentCount: number;
2374
2377
  }): {
2375
2378
  lines: string[];
2376
2379
  channelOffset: number;
2377
2380
  lineContexts?: Array<PlayLogContext | null>;
2381
+ lineIds?: Array<string | null>;
2378
2382
  } | null {
2379
2383
  if (
2380
2384
  input.bufferContexts !== undefined &&
@@ -2382,6 +2386,9 @@ export function slicePositionalLogLines(input: {
2382
2386
  ) {
2383
2387
  throw new TypeError('Run log contexts must align with the retained lines.');
2384
2388
  }
2389
+ if (input.bufferIds && input.bufferIds.length !== input.bufferLines.length) {
2390
+ throw new TypeError('Run log identities must align with retained lines.');
2391
+ }
2385
2392
  const total = Math.max(input.bufferTotalCount, input.bufferLines.length);
2386
2393
  const sent = Math.max(0, input.sentCount);
2387
2394
  if (total <= sent) {
@@ -2398,6 +2405,7 @@ export function slicePositionalLogLines(input: {
2398
2405
  return {
2399
2406
  lines: [...lines],
2400
2407
  channelOffset: sendFromOffset,
2408
+ ...(input.bufferIds ? { lineIds: input.bufferIds.slice(start) } : {}),
2401
2409
  ...(lineContexts?.some(Boolean) ? { lineContexts: [...lineContexts] } : {}),
2402
2410
  };
2403
2411
  }
@@ -2427,6 +2435,7 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
2427
2435
  runId: string;
2428
2436
  lines: readonly string[];
2429
2437
  lineContexts?: readonly (PlayLogContext | null)[];
2438
+ lineIds?: readonly (string | null)[];
2430
2439
  source?: PlayRunLedgerEventSource;
2431
2440
  occurredAt?: number;
2432
2441
  channelOffset?: number | null;
@@ -2448,10 +2457,14 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
2448
2457
  ) {
2449
2458
  throw new TypeError('Run log contexts must align with their lines.');
2450
2459
  }
2460
+ if (input.lineIds && input.lineIds.length !== input.lines.length) {
2461
+ throw new TypeError('Run log identities must align with their lines.');
2462
+ }
2451
2463
  const entries = input.lines
2452
2464
  .map((line, index) => ({
2453
2465
  line: positional ? line.trim() || '(blank log line)' : line.trim(),
2454
2466
  context: input.lineContexts?.[index] ?? null,
2467
+ entryId: input.lineIds?.[index] ?? null,
2455
2468
  }))
2456
2469
  .filter((entry) => positional || entry.line.length > 0);
2457
2470
  const newLines = entries.map((entry) => entry.line);
@@ -2470,6 +2483,13 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
2470
2483
  source,
2471
2484
  occurredAt,
2472
2485
  lines: newLines.slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT),
2486
+ ...(input.lineIds
2487
+ ? {
2488
+ lineIds: entries
2489
+ .slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT)
2490
+ .map((entry) => entry.entryId),
2491
+ }
2492
+ : {}),
2473
2493
  ...(entries
2474
2494
  .slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT)
2475
2495
  .some((entry) => entry.context)
@@ -2507,6 +2527,7 @@ export function buildTerminalLogReplayEvents(input: {
2507
2527
  runId: string;
2508
2528
  lines: readonly string[];
2509
2529
  lineContexts?: readonly (PlayLogContext | null)[];
2530
+ lineIds?: readonly (string | null)[];
2510
2531
  source?: PlayRunLedgerEventSource;
2511
2532
  occurredAt?: number;
2512
2533
  liveLogTotalCount?: number;
@@ -2521,6 +2542,7 @@ export function buildTerminalLogReplayEvents(input: {
2521
2542
  runId: input.runId,
2522
2543
  lines: input.lines,
2523
2544
  lineContexts: input.lineContexts,
2545
+ lineIds: input.lineIds,
2524
2546
  source: 'coordinator',
2525
2547
  occurredAt: input.occurredAt,
2526
2548
  channelOffset,
@@ -2562,6 +2584,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
2562
2584
  runId: patch.runId,
2563
2585
  lines: patch.liveLogs,
2564
2586
  lineContexts: patch.liveLogContexts ?? undefined,
2587
+ lineIds: patch.liveLogIds ?? undefined,
2565
2588
  source,
2566
2589
  occurredAt: checkpointAt,
2567
2590
  channelOffset: patch.liveLogsChannelOffset ?? null,
@@ -25,6 +25,8 @@ export function boundRunLogLine(line: string): string {
25
25
  */
26
26
  export class RunLogBuffer {
27
27
  readonly lines: string[] = [];
28
+ readonly ids: string[] = [];
29
+ private readonly origin = crypto.randomUUID();
28
30
  readonly contexts: Array<PlayLogContext | null> = [];
29
31
  private bytes = 0;
30
32
  private contextBytes = 0;
@@ -56,6 +58,7 @@ export class RunLogBuffer {
56
58
  true,
57
59
  );
58
60
  }
61
+ this.ids.push(`${this.origin}:${this.lines.length}`);
59
62
  this.lines.push(line);
60
63
  const retainContext =
61
64
  normalizedContext !== null &&
@@ -80,3 +83,30 @@ export class RunLogBuffer {
80
83
  };
81
84
  }
82
85
  }
86
+
87
+ /** Optional origin identities: old producers omit them; never infer them from text. */
88
+ export function normalizeRunLogIds(
89
+ value: unknown,
90
+ lines: readonly unknown[],
91
+ ): Array<string | null> | undefined {
92
+ if (value === undefined || value === null) return undefined;
93
+ if (
94
+ !Array.isArray(value) ||
95
+ value.length !== lines.length ||
96
+ value.some(
97
+ (id) =>
98
+ id !== null &&
99
+ (typeof id !== 'string' || !/^[A-Za-z0-9:_-]{1,128}$/.test(id)),
100
+ )
101
+ ) {
102
+ throw new TypeError(
103
+ 'Run log identities must be bounded strings aligned with their lines.',
104
+ );
105
+ }
106
+ const ids = value.filter((id): id is string => id !== null);
107
+ if (new Set(ids).size !== ids.length)
108
+ throw new TypeError(
109
+ 'Run log identities must identify distinct occurrences.',
110
+ );
111
+ return value;
112
+ }
@@ -1903,6 +1903,14 @@ async function releaseRuntimeLeasesOnAppRunnerSettlement(input: {
1903
1903
  }
1904
1904
  }
1905
1905
 
1906
+ function terminalLogIdsWithPhaseSummary(ctx: {
1907
+ getLogIds(): string[];
1908
+ }): string[] {
1909
+ // Summaries are new occurrences at result construction. Terminal retries
1910
+ // reuse that result and its IDs, including the frozen summary bodies.
1911
+ return [...ctx.getLogIds(), crypto.randomUUID(), crypto.randomUUID()];
1912
+ }
1913
+
1906
1914
  function terminalLogsWithPhaseSummary(ctx: {
1907
1915
  getLogs(): string[];
1908
1916
  getRuntimeResourceSnapshot(): unknown;
@@ -2488,7 +2496,7 @@ async function run(
2488
2496
  });
2489
2497
  }
2490
2498
  : undefined,
2491
- onLog: (line, context) => {
2499
+ onLog: (line, context, entryId) => {
2492
2500
  const timestampMatch = line.match(/^\[([^\]]+)\]/);
2493
2501
  const timestamp = timestampMatch?.[1];
2494
2502
  const at =
@@ -2501,6 +2509,9 @@ async function run(
2501
2509
  at,
2502
2510
  source: 'play',
2503
2511
  line,
2512
+ ...(config.context.runtimeLogIdentity === 'origin' && entryId
2513
+ ? { entryId }
2514
+ : {}),
2504
2515
  ...(context ? { context } : {}),
2505
2516
  },
2506
2517
  options,
@@ -3128,6 +3139,9 @@ async function run(
3128
3139
  error: runtimeLimitExceededCause(runtimeLimitSeconds),
3129
3140
  errors: [runtimeLimitExceededFailure(runtimeLimitSeconds)],
3130
3141
  logs: terminalLogsWithPhaseSummary(ctx),
3142
+ ...(config.context.runtimeLogIdentity === 'origin'
3143
+ ? { logIds: terminalLogIdsWithPhaseSummary(ctx) }
3144
+ : {}),
3131
3145
  logContexts: terminalLogContextsWithPhaseSummary(ctx),
3132
3146
  stats: runtimeStats(),
3133
3147
  steps: ctx.getSteps(),
@@ -3372,6 +3386,9 @@ async function run(
3372
3386
  : {}),
3373
3387
  outputRowCount,
3374
3388
  logs,
3389
+ ...(config.context.runtimeLogIdentity === 'origin'
3390
+ ? { logIds: terminalLogIdsWithPhaseSummary(ctx) }
3391
+ : {}),
3375
3392
  logContexts: terminalLogContextsWithPhaseSummary(ctx),
3376
3393
  ...(runtimeDiagnostics.length > 0 ? { runtimeDiagnostics } : {}),
3377
3394
  ...(rowOutcomes ? { rowOutcomes } : {}),
@@ -3427,6 +3444,9 @@ async function run(
3427
3444
  status: 'suspended',
3428
3445
  suspension: error.suspension,
3429
3446
  logs: terminalLogsWithPhaseSummary(ctx),
3447
+ ...(config.context.runtimeLogIdentity === 'origin'
3448
+ ? { logIds: terminalLogIdsWithPhaseSummary(ctx) }
3449
+ : {}),
3430
3450
  logContexts: terminalLogContextsWithPhaseSummary(ctx),
3431
3451
  stats: runtimeStats(),
3432
3452
  steps: ctx.getSteps(),
@@ -3463,6 +3483,9 @@ async function run(
3463
3483
  error: formatRunnerFailure(error),
3464
3484
  errors: [failure],
3465
3485
  logs: terminalLogsWithPhaseSummary(ctx),
3486
+ ...(config.context.runtimeLogIdentity === 'origin'
3487
+ ? { logIds: terminalLogIdsWithPhaseSummary(ctx) }
3488
+ : {}),
3466
3489
  logContexts: terminalLogContextsWithPhaseSummary(ctx),
3467
3490
  ...(rowOutcomes ? { rowOutcomes } : {}),
3468
3491
  stats: runtimeStats(),
@@ -3483,11 +3506,23 @@ export async function executePlayRunner(
3483
3506
  config: PlayRunnerExecutionConfig,
3484
3507
  options?: ExecutePlayRunnerOptions,
3485
3508
  ): Promise<PlayRunnerResult> {
3486
- const result = await run(config, options);
3509
+ const emissionOptions: ExecutePlayRunnerOptions = {
3510
+ ...options,
3511
+ emitEvent: (event) =>
3512
+ emitRunnerEvent(
3513
+ config.context.runtimeLogIdentity === 'origin' &&
3514
+ event.type === 'log' &&
3515
+ !event.entryId
3516
+ ? { ...event, entryId: crypto.randomUUID() }
3517
+ : event,
3518
+ options,
3519
+ ),
3520
+ };
3521
+ const result = await run(config, emissionOptions);
3487
3522
  // Inline-executor mirror of the child terminal drain: the caller's finalize
3488
3523
  // reads the observations table right after this returns, so land in-flight
3489
3524
  // observation posts first (bounded, best-effort, docflow-gated by emptiness).
3490
- await drainPendingDocflowObservationPosts(options);
3525
+ await drainPendingDocflowObservationPosts(emissionOptions);
3491
3526
  return result;
3492
3527
  }
3493
3528
 
@@ -1,4 +1,5 @@
1
1
  import type { PlayRunnerEvent } from '../protocol';
2
+ import { isDurableDirectRunnerLog } from '../log-provenance';
2
3
  import { jsonByteLengthUpTo } from '../output-size-limits';
3
4
  import { RUNTIME_RUNNER_TERMINAL_PROGRESS_MAX_EVENTS } from '../runtime-contract';
4
5
 
@@ -116,9 +117,7 @@ type PendingEvent = {
116
117
  };
117
118
 
118
119
  function isDurableLog(event: PlayRunnerEvent): boolean {
119
- return (
120
- event.type === 'log' && event.source !== 'worker' && event.line.length > 0
121
- );
120
+ return event.type === 'log' && isDurableDirectRunnerLog(event);
122
121
  }
123
122
 
124
123
  export function createRunnerProgressPusher(
@@ -269,12 +269,18 @@ function appendTerminalDiagnosticsMarker(
269
269
  logs: string[],
270
270
  truncatedFields: string[],
271
271
  logContexts?: Array<PlayLogContext | null>,
272
- ): { logs: string[]; logContexts?: Array<PlayLogContext | null> } {
272
+ logIds?: string[],
273
+ ): {
274
+ logs: string[];
275
+ logContexts?: Array<PlayLogContext | null>;
276
+ logIds?: string[];
277
+ } {
273
278
  const marker =
274
279
  `[deepline] terminal diagnostic metadata was bounded (${truncatedFields.join(', ')}); ` +
275
280
  'the authored play output is complete.';
276
281
  const marked = [marker, ...logs];
277
282
  const markedContexts = logContexts ? [null, ...logContexts] : undefined;
283
+ const markedIds = logIds ? [crypto.randomUUID(), ...logIds] : undefined;
278
284
  while (
279
285
  marked.length > 1 &&
280
286
  !fitsJsonByteLimit(marked, TERMINAL_TRANSPORT_LOG_TAIL_MAX_BYTES)
@@ -289,6 +295,7 @@ function appendTerminalDiagnosticsMarker(
289
295
  const removeAt = removableIndex === -1 ? 1 : removableIndex;
290
296
  marked.splice(removeAt, 1);
291
297
  markedContexts?.splice(removeAt, 1);
298
+ markedIds?.splice(removeAt, 1);
292
299
  }
293
300
  const logsFit = fitsJsonByteLimit(
294
301
  marked,
@@ -298,6 +305,9 @@ function appendTerminalDiagnosticsMarker(
298
305
  const retainedContexts = logsFit ? markedContexts : [null];
299
306
  return {
300
307
  logs: retainedLogs,
308
+ ...(markedIds
309
+ ? { logIds: logsFit ? markedIds : markedIds.slice(0, 1) }
310
+ : {}),
301
311
  ...(retainedContexts?.some(Boolean)
302
312
  ? { logContexts: retainedContexts }
303
313
  : {}),
@@ -325,6 +335,11 @@ function boundSettledTerminalTransportDiagnostics(
325
335
  const logContexts = result.logContexts?.length
326
336
  ? logProjection.indexes.map((index) => result.logContexts![index] ?? null)
327
337
  : undefined;
338
+ if (result.logIds && result.logIds.length !== result.logs.length)
339
+ throw new TypeError('Terminal log identities must align with result logs.');
340
+ const logIds = result.logIds
341
+ ? logProjection.indexes.map((index) => result.logIds![index]!)
342
+ : undefined;
328
343
  const statsProjection = retainRecordWithinJsonByteLimit(
329
344
  result.stats,
330
345
  TERMINAL_TRANSPORT_STATS_MAX_BYTES,
@@ -368,15 +383,18 @@ function boundSettledTerminalTransportDiagnostics(
368
383
  logProjection.values,
369
384
  truncatedFields,
370
385
  logContexts,
386
+ logIds,
371
387
  )
372
388
  : {
373
389
  logs: logProjection.values,
390
+ ...(logIds ? { logIds } : {}),
374
391
  ...(logContexts?.some(Boolean) ? { logContexts } : {}),
375
392
  };
376
393
  return {
377
394
  ...result,
378
395
  logs: boundedLogs.logs,
379
396
  logContexts: boundedLogs.logContexts,
397
+ logIds: boundedLogs.logIds,
380
398
  stats: statsProjection.value,
381
399
  steps: stepsProjection.values,
382
400
  ...(result.outputWarnings
@@ -408,9 +426,11 @@ function boundSettledTerminalTransportDiagnostics(
408
426
  logProjection.values,
409
427
  truncatedFields,
410
428
  logContexts,
429
+ logIds,
411
430
  )
412
431
  : {
413
432
  logs: logProjection.values,
433
+ ...(logIds ? { logIds } : {}),
414
434
  ...(logContexts?.some(Boolean) ? { logContexts } : {}),
415
435
  };
416
436
  return {
@@ -418,6 +438,7 @@ function boundSettledTerminalTransportDiagnostics(
418
438
  error: error.value,
419
439
  logs: boundedLogs.logs,
420
440
  logContexts: boundedLogs.logContexts,
441
+ logIds: boundedLogs.logIds,
421
442
  stats: statsProjection.value,
422
443
  steps: stepsProjection.values,
423
444
  ...(result.errors ? { errors: errorsProjection.values } : {}),
@@ -90,7 +90,7 @@ const DAYTONA_PUBLIC_STATUS_TRANSIENT_RETRY_PATTERN =
90
90
  const DAYTONA_RUNNER_STARTUP_TRANSPORT_RETRY_PATTERN =
91
91
  /\bDaytonaRunnerInitializationError:\s*RUNTIME_SANDBOX_START_FAILED:.*\bScheduler diagnosis:\s*runner_heartbeat_not_observed\..*"schedulerReadinessObserved":false.*"commandFound":true.*"exitCode":null.*"sessionFound":true.*"exitCodeFile":null.*"runnerStartup":\{"phase":"heartbeat_transport","outcome":"network_error","errorCode":"(?:ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT)"/i;
92
92
  const DAYTONA_INFRASTRUCTURE_RETRY_PATTERN =
93
- /\b(?:Request failed with status code (?:408|429|500|502|503|504)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|fetch failed|socket hang up|Daytona sandbox create failed across|Daytona sandbox create did not complete|Daytona readiness baseline)\b/i;
93
+ /\b(?:Request failed with status code (?:408|429|500|502|503|504)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|fetch failed|socket hang up|The socket connection was closed unexpectedly|Daytona sandbox create failed across|Daytona sandbox create did not complete|Daytona readiness baseline)\b/i;
94
94
 
95
95
  type DaytonaExecution = { exitCode: number | null; result: string };
96
96
  export type DaytonaLookupFailure = {
@@ -86,6 +86,8 @@ export type PlaySchedulerSubmitInput = RetainedPipelineCarrier & {
86
86
  * Omitted for legacy launches whose Play Run row was persisted synchronously.
87
87
  */
88
88
  requiresRunCreatedProjection?: boolean;
89
+ /** Admission-owned producer pin. Absent keeps launch-pinned legacy logging. */
90
+ runtimeLogIdentity?: 'origin';
89
91
  /** Optional inline artifact for the Node scheduler. */
90
92
  runtimeArtifact?: unknown;
91
93
  artifactHash: string;
@@ -0,0 +1,145 @@
1
+ import { AsyncLocalStorage } from 'node:async_hooks';
2
+ import {
3
+ PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS,
4
+ PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES,
5
+ PLAY_ADMISSION_MAX_PARALLELISM,
6
+ } from '../admission-transport-policy';
7
+ import {
8
+ RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS,
9
+ RUNTIME_SCHEDULER_SERVERLESS_IDLE_TIMEOUT_MS,
10
+ resolveWorkflowRuntimePostgresUrl,
11
+ type RuntimeSchedulerPoolOptions,
12
+ } from './postgres-pool';
13
+
14
+ // All queues in one worker share this single process pool. Scheduler queries
15
+ // are short and batched; execution concurrency belongs to Daytona/Fly slots,
16
+ // not to a growing set of database clients. Keeping this at two makes a
17
+ // horizontally scaled fleet predictable against the shared PgBouncer budget.
18
+ export const ABSURD_RUNTIME_POOL_MAX_CLIENTS =
19
+ RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS;
20
+
21
+ /**
22
+ * Vercel Fluid invocations share a process and its scheduler pool. A single
23
+ * client serializes unrelated launches, reads, and critical lease lookups;
24
+ * one held connection can make every other request hit its acquire timeout.
25
+ * Use the shared two-client control-plane budget so another request can make
26
+ * progress while keeping each instance's PgBouncer footprint bounded.
27
+ */
28
+ export const ABSURD_SERVERLESS_POOL_MAX_CLIENTS =
29
+ RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS;
30
+
31
+ /**
32
+ * A Vercel isolate may survive long after a webhook has returned. Keep its
33
+ * scheduler clients only long enough to reuse them for an immediate follow-up,
34
+ * then return idle PgBouncer clients before the source's delayed Workpool
35
+ * recovery is due. The worker fleet keeps the ordinary 30-second idle lifetime.
36
+ */
37
+ export const ABSURD_SERVERLESS_POOL_IDLE_TIMEOUT_MS =
38
+ RUNTIME_SCHEDULER_SERVERLESS_IDLE_TIMEOUT_MS;
39
+
40
+ // Ordinary Vercel requests share the bounded control-plane pool above. The
41
+ // private batch adapter is different: Workpool elects one global up-to-1,024-
42
+ // source collection before entering this context, so one batch request may use
43
+ // its larger bounded scheduler capacity without multiplying it per source.
44
+ // Other Workpool adapters can still overlap during rollout.
45
+ type ServerlessAdmissionPoolContext = {
46
+ maxClients: number;
47
+ maxAcquireQueueDepth: number;
48
+ maxAcquireQueueWaitMs: number;
49
+ };
50
+
51
+ const serverlessAdmissionPoolContext =
52
+ new AsyncLocalStorage<ServerlessAdmissionPoolContext>();
53
+
54
+ export async function withServerlessAdmissionPoolMaxClients<T>(input: {
55
+ maxClients: number;
56
+ maxAcquireQueueDepth?: number;
57
+ maxAcquireQueueWaitMs?: number;
58
+ fn: () => Promise<T>;
59
+ }): Promise<T> {
60
+ if (
61
+ !Number.isInteger(input.maxClients) ||
62
+ input.maxClients < 1 ||
63
+ input.maxClients > PLAY_ADMISSION_MAX_PARALLELISM
64
+ ) {
65
+ throw new Error(
66
+ 'Serverless admission pool capacity is outside the supported range.',
67
+ );
68
+ }
69
+ const maxAcquireQueueDepth =
70
+ input.maxAcquireQueueDepth ?? PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES;
71
+ const maxAcquireQueueWaitMs =
72
+ input.maxAcquireQueueWaitMs ?? PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS;
73
+ if (
74
+ !Number.isSafeInteger(maxAcquireQueueDepth) ||
75
+ maxAcquireQueueDepth < 1 ||
76
+ maxAcquireQueueDepth > PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES
77
+ ) {
78
+ throw new Error(
79
+ 'Serverless admission acquire queue capacity is outside the supported range.',
80
+ );
81
+ }
82
+ if (
83
+ !Number.isSafeInteger(maxAcquireQueueWaitMs) ||
84
+ maxAcquireQueueWaitMs < 5_000 ||
85
+ maxAcquireQueueWaitMs > PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS
86
+ ) {
87
+ throw new Error(
88
+ 'Serverless admission acquire queue wait is outside the supported range.',
89
+ );
90
+ }
91
+ return await serverlessAdmissionPoolContext.run(
92
+ {
93
+ maxClients: input.maxClients,
94
+ maxAcquireQueueDepth,
95
+ maxAcquireQueueWaitMs,
96
+ },
97
+ input.fn,
98
+ );
99
+ }
100
+
101
+ /**
102
+ * The private batch adapter owns this request-local scheduler pool. Reserve
103
+ * one client for executor-token and other critical control-plane traffic; the
104
+ * rest may submit the independently idempotent batch sources in parallel.
105
+ */
106
+ export function serverlessAdmissionSubmitMaxActive(): number | null {
107
+ const context = serverlessAdmissionPoolContext.getStore();
108
+ return context === undefined ? null : Math.max(1, context.maxClients - 1);
109
+ }
110
+
111
+ /**
112
+ * Resolve the runtime scheduler Postgres pool options for the absurd control
113
+ * plane. Loud failure over silent fallback: the absurd worker and scheduler
114
+ * only run under Node where `WORKFLOW_RUNTIME_POSTGRES` is configured.
115
+ */
116
+ export function resolveAbsurdPoolOptions(
117
+ explicitUrl?: string | null,
118
+ ): RuntimeSchedulerPoolOptions {
119
+ const url = explicitUrl?.trim() || resolveWorkflowRuntimePostgresUrl();
120
+ if (!url) {
121
+ throw new Error(
122
+ 'absurd scheduler backend requires WORKFLOW_RUNTIME_POSTGRES (runtime ' +
123
+ 'scheduler Postgres URL). Refusing to fall back to local execution.',
124
+ );
125
+ }
126
+ const serverless = process.env.VERCEL?.trim();
127
+ const context = serverlessAdmissionPoolContext.getStore();
128
+ return {
129
+ url,
130
+ applicationName: 'deepline-absurd-runtime',
131
+ maxClients:
132
+ serverless || context !== undefined
133
+ ? (context?.maxClients ?? ABSURD_SERVERLESS_POOL_MAX_CLIENTS)
134
+ : ABSURD_RUNTIME_POOL_MAX_CLIENTS,
135
+ ...(serverless || context !== undefined
136
+ ? { idleTimeoutMs: ABSURD_SERVERLESS_POOL_IDLE_TIMEOUT_MS }
137
+ : {}),
138
+ ...(context
139
+ ? {
140
+ maxAcquireQueueDepth: context.maxAcquireQueueDepth,
141
+ maxAcquireQueueWaitMs: context.maxAcquireQueueWaitMs,
142
+ }
143
+ : {}),
144
+ };
145
+ }
@@ -4463,7 +4463,6 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
4463
4463
  );
4464
4464
  });
4465
4465
  };
4466
- let projectionStartedAtMs: number | undefined;
4467
4466
  return createPlayRunAttemptWorkerTerminalProjection({
4468
4467
  async observeFenceLost() {
4469
4468
  // A lost fence means this attempt is dead, which a cancelled run is the
@@ -4479,7 +4478,6 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
4479
4478
  });
4480
4479
  },
4481
4480
  async reconcileTerminalTruth() {
4482
- projectionStartedAtMs ??= Date.now();
4483
4481
  // ADR 0016 rule 3: detached Daytona runs may provide no live events, so
4484
4482
  // fold stored docflow/runtime-sheet facts into progress after truth wins.
4485
4483
  await reconcileTerminalTruthIntoReporter({
@@ -4564,9 +4562,7 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
4564
4562
  runId: input.runId,
4565
4563
  options: input.options,
4566
4564
  trigger: 'committed_terminal',
4567
- ...(projectionStartedAtMs === undefined
4568
- ? {}
4569
- : { projectionStartedAtMs }),
4565
+ projectionStartedAtMs: Date.now(),
4570
4566
  });
4571
4567
  },
4572
4568
  });