deepline 0.3.289 → 0.3.290

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts

@@ -11,7 +11,6 @@ import { RUN_EVENT_KIND } from '../run-event-catalog';
  * absurd-specific wiring (queue, event names, checkpoint boundary shaping).
  */
 import { createHash } from 'node:crypto';
-import { AsyncLocalStorage } from 'node:async_hooks';
 import type {
   PlayCheckpoint,
   PlayExecutionEvent,
@@ -22,11 +21,6 @@ import type { PlayRunLedgerEvent } from '../run-ledger';
 import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
 import type { WorkerRuntimeApiContext } from '../app-runtime-api';
 import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
-import {
-  PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS,
-  PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES,
-  PLAY_ADMISSION_MAX_PARALLELISM,
-} from '../admission-transport-policy';
 import { assertKnownResolvedBoundary } from '../checkpoint-boundaries';
 import type { Absurd, Log, RetryStrategy } from '@deepline/absurd/sdk';
 import { isIsolatedRuntimeSchedulerSchema } from '../runtime-scheduler-topology';
@@ -37,8 +31,6 @@ import {
 } from './postgres';
 import { createAbsurdForRuntimeScheduler } from './absurd-pool';
 import {
-  RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS,
-  resolveWorkflowRuntimePostgresUrl,
   withRuntimeSchedulerClient,
   type RuntimeSchedulerPoolOptions,
 } from './postgres-pool';
@@ -548,137 +540,14 @@ export type AbsurdRuntimeResult = {
   error?: string;
 };
 
-// All queues in one worker share this single process pool. Scheduler queries
-// are short and batched; execution concurrency belongs to Daytona/Fly slots,
-// not to a growing set of database clients. Keeping this at two makes a
-// horizontally scaled fleet predictable against the shared PgBouncer budget.
-export const ABSURD_RUNTIME_POOL_MAX_CLIENTS =
-  RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS;
-
-/**
- * Vercel Fluid invocations share a process and its scheduler pool. A single
- * client serializes unrelated launches, reads, and critical lease lookups;
- * one held connection can make every other request hit its acquire timeout.
- * Use the shared two-client control-plane budget so another request can make
- * progress while keeping each instance's PgBouncer footprint bounded.
- */
-export const ABSURD_SERVERLESS_POOL_MAX_CLIENTS =
-  RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS;
-
-/**
- * A Vercel isolate may survive long after a webhook has returned. Keep its
- * scheduler clients only long enough to reuse them for an immediate follow-up,
- * then return idle PgBouncer clients before the source's delayed Workpool
- * recovery is due. The worker fleet keeps the ordinary 30-second idle lifetime.
- */
-export const ABSURD_SERVERLESS_POOL_IDLE_TIMEOUT_MS = 1_000;
-
-// Ordinary Vercel requests share the bounded control-plane pool above. The
-// private batch adapter is different: Workpool elects one global up-to-1,024-
-// source collection before entering this context, so one batch request may use
-// its larger bounded scheduler capacity without multiplying it per source.
-// Other Workpool adapters can still overlap during rollout.
-type ServerlessAdmissionPoolContext = {
-  maxClients: number;
-  maxAcquireQueueDepth: number;
-  maxAcquireQueueWaitMs: number;
-};
-
-const serverlessAdmissionPoolContext =
-  new AsyncLocalStorage<ServerlessAdmissionPoolContext>();
-
-export async function withServerlessAdmissionPoolMaxClients<T>(input: {
-  maxClients: number;
-  maxAcquireQueueDepth?: number;
-  maxAcquireQueueWaitMs?: number;
-  fn: () => Promise<T>;
-}): Promise<T> {
-  if (
-    !Number.isInteger(input.maxClients) ||
-    input.maxClients < 1 ||
-    input.maxClients > PLAY_ADMISSION_MAX_PARALLELISM
-  ) {
-    throw new Error(
-      'Serverless admission pool capacity is outside the supported range.',
-    );
-  }
-  const maxAcquireQueueDepth =
-    input.maxAcquireQueueDepth ?? PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES;
-  const maxAcquireQueueWaitMs =
-    input.maxAcquireQueueWaitMs ?? PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS;
-  if (
-    !Number.isSafeInteger(maxAcquireQueueDepth) ||
-    maxAcquireQueueDepth < 1 ||
-    maxAcquireQueueDepth > PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES
-  ) {
-    throw new Error(
-      'Serverless admission acquire queue capacity is outside the supported range.',
-    );
-  }
-  if (
-    !Number.isSafeInteger(maxAcquireQueueWaitMs) ||
-    maxAcquireQueueWaitMs < 5_000 ||
-    maxAcquireQueueWaitMs > PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS
-  ) {
-    throw new Error(
-      'Serverless admission acquire queue wait is outside the supported range.',
-    );
-  }
-  return await serverlessAdmissionPoolContext.run(
-    {
-      maxClients: input.maxClients,
-      maxAcquireQueueDepth,
-      maxAcquireQueueWaitMs,
-    },
-    input.fn,
-  );
-}
-
-/**
- * The private batch adapter owns this request-local scheduler pool. Reserve
- * one client for executor-token and other critical control-plane traffic; the
- * rest may submit the independently idempotent batch sources in parallel.
- */
-export function serverlessAdmissionSubmitMaxActive(): number | null {
-  const context = serverlessAdmissionPoolContext.getStore();
-  return context === undefined ? null : Math.max(1, context.maxClients - 1);
-}
-
-/**
- * Resolve the runtime scheduler Postgres pool options for the absurd control
- * plane. Loud failure over silent fallback: the absurd worker and scheduler
- * only run under Node where `WORKFLOW_RUNTIME_POSTGRES` is configured.
- */
-export function resolveAbsurdPoolOptions(
-  explicitUrl?: string | null,
-): RuntimeSchedulerPoolOptions {
-  const url = explicitUrl?.trim() || resolveWorkflowRuntimePostgresUrl();
-  if (!url) {
-    throw new Error(
-      'absurd scheduler backend requires WORKFLOW_RUNTIME_POSTGRES (runtime ' +
-        'scheduler Postgres URL). Refusing to fall back to local execution.',
-    );
-  }
-  const serverless = process.env.VERCEL?.trim();
-  const context = serverlessAdmissionPoolContext.getStore();
-  return {
-    url,
-    applicationName: 'deepline-absurd-runtime',
-    maxClients:
-      serverless || context !== undefined
-        ? (context?.maxClients ?? ABSURD_SERVERLESS_POOL_MAX_CLIENTS)
-        : ABSURD_RUNTIME_POOL_MAX_CLIENTS,
-    ...(serverless || context !== undefined
-      ? { idleTimeoutMs: ABSURD_SERVERLESS_POOL_IDLE_TIMEOUT_MS }
-      : {}),
-    ...(context
-      ? {
-          maxAcquireQueueDepth: context.maxAcquireQueueDepth,
-          maxAcquireQueueWaitMs: context.maxAcquireQueueWaitMs,
-        }
-      : {}),
-  };
-}
+export {
+  ABSURD_RUNTIME_POOL_MAX_CLIENTS,
+  ABSURD_SERVERLESS_POOL_MAX_CLIENTS,
+  ABSURD_SERVERLESS_POOL_IDLE_TIMEOUT_MS,
+  withServerlessAdmissionPoolMaxClients,
+  serverlessAdmissionSubmitMaxActive,
+  resolveAbsurdPoolOptions,
+} from './absurd-pool-options';
 
 export function normalizeAbsurdSchedulerOptions(
   options: { url?: string | null; schema?: string | null } = {},

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts

@@ -22,6 +22,7 @@
  * connection. Notifications are strictly a wakeup hint — the tables remain
  * the source of truth and every waiter still reads the DB.
  */
+import { attachDatabasePool } from '@vercel/functions/db-connections';
 import { Client, Pool } from 'pg';
 import {
   RuntimeSchedulerPgBouncerCapacityError,
@@ -175,6 +176,8 @@ export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
   keepAliveIntervalMs: 60_000,
 } as const;
 
+export const RUNTIME_SCHEDULER_SERVERLESS_IDLE_TIMEOUT_MS = 1_000;
+
 /**
  * A control-plane process only needs a few short scheduler transactions at
  * once. This shared cap keeps horizontally scaled workers and the receipt
@@ -197,8 +200,8 @@ export type RuntimeSchedulerPoolOptions = {
   maxClients?: number;
   /**
    * Serverless launchers must not pin a scarce PgBouncer client between
-   * requests. Long-lived workers retain the default; Vercel callers opt into
-   * a short idle lifetime through the shared Absurd pool resolver.
+   * requests. Vercel pools share one short idle lifetime; this override only
+   * applies to long-lived processes.
    */
   idleTimeoutMs?: number;
   /**
@@ -447,6 +450,7 @@ export class RuntimeSchedulerPoolAdmission {
 
 type PoolEntry = {
   pool: Pool;
+  databasePoolAttached?: boolean;
   maxClients: number;
   maxAcquireQueueDepth: number;
   admission: RuntimeSchedulerPoolAdmission;
@@ -723,7 +727,7 @@ function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
   assertRuntimeSchedulerHotPathUrl(options.url);
   const maxClients =
     options.maxClients ?? RUNTIME_SCHEDULER_POOL_DEFAULTS.maxClients;
-  const idleTimeoutMs =
+  const requestedIdleTimeoutMs =
     options.idleTimeoutMs ?? RUNTIME_SCHEDULER_POOL_DEFAULTS.idleTimeoutMs;
   const maxAcquireQueueDepth =
     options.maxAcquireQueueDepth ??
@@ -738,11 +742,17 @@ function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
         'instead of widening a single process pool.',
     );
   }
-  if (!Number.isSafeInteger(idleTimeoutMs) || idleTimeoutMs < 0) {
+  if (
+    !Number.isSafeInteger(requestedIdleTimeoutMs) ||
+    requestedIdleTimeoutMs < 0
+  ) {
     throw new Error(
-      `Runtime scheduler pool idleTimeoutMs=${idleTimeoutMs} must be a non-negative integer.`,
+      `Runtime scheduler pool idleTimeoutMs=${requestedIdleTimeoutMs} must be a non-negative integer.`,
     );
   }
+  const idleTimeoutMs = process.env.VERCEL?.trim()
+    ? RUNTIME_SCHEDULER_SERVERLESS_IDLE_TIMEOUT_MS
+    : requestedIdleTimeoutMs;
   if (
     !Number.isSafeInteger(maxAcquireQueueDepth) ||
     maxAcquireQueueDepth < 1 ||
@@ -1049,6 +1059,10 @@ export async function withRuntimeSchedulerClient<T>(
         let queryCount = 0;
         let queryMs = 0;
         try {
+          if (!entry.databasePoolAttached) {
+            attachDatabasePool(entry.pool);
+            entry.databasePoolAttached = true;
+          }
           throwIfSchedulerAborted(signal);
           const result = await fn({
             query: async (sql, params) => {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -14,6 +14,7 @@ import { RUN_EVENT_KIND } from '../run-event-catalog';
  * Runtime Sheet row durability belongs to the context-level row writer.
  */
 import { PlayStepLifecycleTracker } from '../step-lifecycle-tracker';
+import { isDurableDirectRunnerLog } from '../log-provenance';
 import {
   updateRunStatusViaAppRuntime,
   type RuntimeStatusUpdate,
@@ -268,18 +269,7 @@ function shouldPublishDurableLogEvent(
     | Pick<PlayRunnerLogEvent, 'source' | 'line'>
     | { source?: unknown; line?: unknown },
 ): boolean {
-  // Daytona host lifecycle logs use source "worker" (sandbox
-  // create/upload/execute). Keep those in worker logs while still storing
-  // play-authored ctx.log events in the durable Run Log Stream. The one
-  // exception is the structured performance marker: it is bounded (one
-  // terminal phase observation per runner phase), run-scoped, and is the
-  // canonical black-box regression signal for dispatcher throughput. Do not
-  // broaden this to arbitrary worker output.
-  return (
-    mode !== 'direct_worker' ||
-    event.source !== 'worker' ||
-    (typeof event.line === 'string' && event.line.startsWith('[perf-trace] '))
-  );
+  return mode !== 'direct_worker' || isDurableDirectRunnerLog(event);
 }
 
 function shouldPublishTerminalResultLog(
@@ -305,14 +295,25 @@ type TerminalLogReplayPayload = NonNullable<
 function terminalReplayEntriesForMode(input: {
   resultLogs: readonly string[];
   resultLogContexts?: readonly (PlayLogContext | null)[];
+  resultLogIds?: readonly string[];
   durableLogMode: PostgresSchedulerDurableLogMode;
-}): Array<{ line: string; context: PlayLogContext | null }> {
+}): Array<{
+  line: string;
+  context: PlayLogContext | null;
+  entryId: string | null;
+}> {
   if (
     input.resultLogContexts !== undefined &&
     input.resultLogContexts.length !== input.resultLogs.length
   ) {
     throw new TypeError('Terminal log contexts must align with result logs.');
   }
+  if (
+    input.resultLogIds &&
+    input.resultLogIds.length !== input.resultLogs.length
+  ) {
+    throw new TypeError('Terminal log identities must align with result logs.');
+  }
   return input.resultLogs.flatMap((rawLine, index) => {
     const line = rawLine.trim();
     if (
@@ -325,6 +326,7 @@ function terminalReplayEntriesForMode(input: {
       {
         line,
         context: normalizePlayLogContext(input.resultLogContexts?.[index]),
+        entryId: input.resultLogIds?.[index] ?? null,
       },
     ];
   });
@@ -333,6 +335,7 @@ function terminalReplayEntriesForMode(input: {
 function terminalReplayPayloadForMode(input: {
   resultLogs: readonly string[];
   resultLogContexts?: readonly (PlayLogContext | null)[];
+  resultLogIds?: readonly string[];
   durableLogMode: PostgresSchedulerDurableLogMode;
   liveLogs: readonly string[];
   liveLogContexts: readonly (PlayLogContext | null)[];
@@ -351,6 +354,7 @@ function terminalReplayPayloadForMode(input: {
   let totalCount = entries.length;
   if (
     input.durableLogMode === 'direct_worker' &&
+    !input.resultLogIds &&
     !input.replayAllTerminalLogs
   ) {
     const observedCounts = new Map<string, number>();
@@ -380,6 +384,9 @@ function terminalReplayPayloadForMode(input: {
   return {
     lines,
     ...(lineContexts.some(Boolean) ? { lineContexts } : {}),
+    ...(input.resultLogIds
+      ? { lineIds: entries.map((entry) => entry.entryId) }
+      : {}),
     ...(input.replayAllTerminalLogs ? { dedupeMode: 'exact' as const } : {}),
     ...(typeof input.producerAttempt === 'number'
       ? { producerAttempt: input.producerAttempt }
@@ -439,6 +446,7 @@ export function createPostgresSchedulerProgressReporter(input: {
   let nodeProgressSealed = false;
   let liveLogs: string[] = [];
   let liveLogContexts: Array<PlayLogContext | null> = [];
+  let liveLogIds: Array<string | null> = [];
   let liveLogsTotalCount = input.initialLiveLogTotalCount ?? 0;
   let liveLogsDirty = false;
   let workCounters: RuntimeStatusUpdate['workCounters'];
@@ -541,6 +549,7 @@ export function createPostgresSchedulerProgressReporter(input: {
             : undefined;
           const nextLogs = liveLogs.slice();
           const nextLogContexts = liveLogContexts.slice();
+          const nextLogIds = liveLogIds.slice();
           const nextLogTotalCount = liveLogsTotalCount;
           const nextDatasetLifecycleEvents = includeProgress
             ? pendingDatasetLifecycleEvents
@@ -566,7 +575,11 @@ export function createPostgresSchedulerProgressReporter(input: {
                 liveNodeProgress: nodeProgressSealed ? {} : nextProgress,
                 liveLogs: nextLogs,
                 liveLogContexts: nextLogContexts,
+                liveLogIds: nextLogIds,
                 liveLogTotalCount: nextLogTotalCount,
+                ...(input.initialLiveLogTotalCount !== undefined
+                  ? { liveLogBatchOffset: nextLogTotalCount - nextLogs.length }
+                  : {}),
                 liveLogProducerAttempt: producerAttempt,
                 datasetLifecycleEvents: nextDatasetLifecycleEvents,
                 activityObservations: nextActivityObservations,
@@ -667,7 +680,11 @@ export function createPostgresSchedulerProgressReporter(input: {
     void flush({ force }).catch(() => undefined);
   };
 
-  const appendLiveLog = (line: string, context?: unknown) => {
+  const appendLiveLog = (
+    line: string,
+    context?: unknown,
+    entryId?: unknown,
+  ) => {
     const trimmed = line.trim();
     if (!trimmed) {
       return;
@@ -675,8 +692,10 @@ export function createPostgresSchedulerProgressReporter(input: {
     liveLogsTotalCount += 1;
     liveLogs = [...liveLogs, trimmed];
     liveLogContexts = [...liveLogContexts, normalizePlayLogContext(context)];
+    liveLogIds = [...liveLogIds, typeof entryId === 'string' ? entryId : null];
     if (liveLogs.length > LIVE_LOG_TAIL_LIMIT) {
       liveLogs = liveLogs.slice(liveLogs.length - LIVE_LOG_TAIL_LIMIT);
+      liveLogIds = liveLogIds.slice(liveLogIds.length - LIVE_LOG_TAIL_LIMIT);
       liveLogContexts = liveLogContexts.slice(
         liveLogContexts.length - LIVE_LOG_TAIL_LIMIT,
       );
@@ -1321,13 +1340,14 @@ export function createPostgresSchedulerProgressReporter(input: {
         line?: unknown;
         source?: unknown;
         context?: unknown;
+        entryId?: unknown;
       };
       const rawLine = logEvent.line;
       if (typeof rawLine !== 'string' || !rawLine) {
         return;
       }
       if (shouldPublishDurableLogEvent(durableLogMode, logEvent)) {
-        appendLiveLog(rawLine, logEvent.context);
+        appendLiveLog(rawLine, logEvent.context, logEvent.entryId);
       }
       markMapStartedFromLog(rawLine);
       markMapStartedFromPerfLog(rawLine);
@@ -1629,6 +1649,7 @@ export function createPostgresSchedulerProgressReporter(input: {
           : terminalReplayPayloadForMode({
               resultLogs: result.logs,
               resultLogContexts: result.logContexts,
+              resultLogIds: result.logIds,
               durableLogMode,
               liveLogs,
               liveLogContexts,
@@ -1832,6 +1853,7 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
       liveNodeProgress: finalUpdate.liveNodeProgress,
       liveLogs,
       liveLogContexts: finalUpdate.liveLogContexts,
+      liveLogIds: finalUpdate.liveLogIds,
       liveLogsChannelOffset: Math.max(0, liveLogTotalCount - liveLogs.length),
       liveLogsProducerAttempt:
         finalUpdate.liveLogProducerAttempt ?? input.attempt,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -10,6 +10,7 @@ import type {
 import {
   DEFAULT_ADVISORY_PROVIDER_MAX_RPS,
   DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
+  DEFAULT_UNKNOWN_PROVIDER_MIN_RPS,
   PROVIDER_AIMD_COOLDOWN_CAP_MS,
   type PacingRule,
 } from '../governor/rate-state-backend';
@@ -431,7 +432,8 @@ function normalizeRules(
  * per second; `capacity` is one window of burst. The adaptive `[min,max]` range
  * is derived value-first so the same wire shape drives both declared and
  * default-pacing buckets:
- *   - min_rps = min(5/60, max)
+ *   - min_rps = 2 for the unknown shared-provider scope
+ *   - min_rps = min(5/60, max) for declared rules
  *     (a lower explicit adaptive maximum remains authoritative)
  *   - max_rps = max(base, 100)     (catalogued-rate recommendation)
  *   - max_rps = explicit adaptiveMaxRps when supplied
@@ -454,11 +456,11 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
     rule.adaptiveMaxRps != null && rule.adaptiveMaxRps > 0
       ? Math.max(rule.adaptiveMaxRps, base)
       : Math.max(base, DEFAULT_ADVISORY_PROVIDER_MAX_RPS);
-  // Undeclared providers keep their existing 10-RPS fallback. Other provider
-  // scopes use the five-RPM floor, capped by an explicit adaptive maximum.
+  // The unknown-provider seed is a guess, not a quota floor. Declared provider
+  // scopes keep the five-RPM floor, capped by an explicit adaptive maximum.
   const minRps =
     rule.ruleId === 'default:provider-scope'
-      ? base
+      ? DEFAULT_UNKNOWN_PROVIDER_MIN_RPS
       : Math.min(MIN_ADAPTIVE_RPS_FLOOR, maxRps);
   return {
     rule_id: rule.ruleId,
@@ -472,7 +474,7 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
 
 function minimumRateRpsForRules(rules: readonly AcquirePacingRule[]): number {
   if (rules.some((rule) => rule.ruleId === 'default:provider-scope')) {
-    return DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS;
+    return DEFAULT_UNKNOWN_PROVIDER_MIN_RPS;
   }
   const explicitAdaptiveMaxima = rules.flatMap((rule) => {
     if (rule.adaptiveMaxRps == null || rule.adaptiveMaxRps <= 0) return [];