deepline 0.3.128 → 0.3.130

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts

@@ -33,6 +33,7 @@ import {
   PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS,
 } from '../admission-transport-policy';
 import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
+import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
 export {
   isRuntimeSchedulerCapacityError,
   parseRuntimeSchedulerCapacityError,
@@ -126,8 +127,8 @@ export function assertRuntimeSchedulerHotPathUrl(url: string): void {
 }
 
 export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
-  /** Per-process client cap. Workers override via options; keep small on serverless. */
-  maxClients: 8,
+  /** One shared per-process client cap for scheduler control-plane lanes. */
+  maxClients: RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients,
   /** Refuse to construct a pool above this — protects the shared PgBouncer/Postgres. */
   maxClientsHardCap: 32,
   /** Max time a caller waits for a pooled connection before erroring. */
@@ -163,7 +164,11 @@ export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
  * once. This shared cap keeps horizontally scaled workers and the receipt
  * gateway from consuming PgBouncer's global client budget independently.
  */
-export const RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS = 2;
+export const RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS =
+  RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients;
+
+export const RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS = 2_000;
+export const RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS = 15_000;
 
 export type RuntimeSchedulerPoolOptions = {
   url: string;
@@ -188,6 +193,11 @@ export type RuntimeSchedulerPoolOptions = {
   maxAcquireQueueWaitMs?: number;
 };
 
+export type RuntimeSchedulerOperationOptions = {
+  signal?: AbortSignal;
+  maxAttempts?: number;
+};
+
 export type RuntimeSchedulerQueryClient = {
   query<Row extends Record<string, unknown> = Record<string, unknown>>(
     sql: string,
@@ -195,6 +205,22 @@ export type RuntimeSchedulerQueryClient = {
   ): Promise<{ rows: Row[] }>;
 };
 
+export function isRuntimeSchedulerQueryCancellationError(
+  error: unknown,
+): boolean {
+  if (!error || typeof error !== 'object') return false;
+  if ((error as { code?: unknown }).code === '57014') return true;
+  const name = (error as { name?: unknown }).name;
+  if (name === 'TimeoutError' || name === 'AbortError') return true;
+  const message = (error as { message?: unknown }).message;
+  return (
+    typeof message === 'string' &&
+    /(?:statement timeout|canceling statement due to user request|query read timeout)/i.test(
+      message,
+    )
+  );
+}
+
 export class RuntimeSchedulerCircuitOpenError extends Error {
   readonly code = 'RUNTIME_SCHEDULER_UNAVAILABLE';
 
@@ -206,6 +232,47 @@ export class RuntimeSchedulerCircuitOpenError extends Error {
   }
 }
 
+/**
+ * Errors can cross the app/worker package boundary, where `instanceof` is not
+ * reliable. Keep this guard tied to the one canonical error name and message
+ * instead of making every status endpoint duplicate a string match.
+ */
+export function isRuntimeSchedulerCircuitOpenError(
+  error: unknown,
+): error is RuntimeSchedulerCircuitOpenError {
+  if (error instanceof RuntimeSchedulerCircuitOpenError) return true;
+  return (
+    error instanceof Error &&
+    error.name === 'RuntimeSchedulerCircuitOpenError' &&
+    error.message.startsWith(
+      'Runtime scheduler DB circuit breaker is open after repeated connection failures:',
+    )
+  );
+}
+
+type SchedulerCircuitWait = (delayMs: number) => Promise<void>;
+
+/**
+ * Give the existing scheduler-pool breaker one half-open trial before an
+ * idempotent control-plane read is surfaced as unavailable. This is not a
+ * second retry policy or congestion controller: it waits exactly the breaker
+ * cooldown already enforced by `withRuntimeSchedulerClient`, retries once,
+ * and only accepts read operations supplied by the caller.
+ */
+export async function retryOnceAfterRuntimeSchedulerCircuitOpen<T>(
+  read: () => Promise<T>,
+  options: { wait?: SchedulerCircuitWait } = {},
+): Promise<T> {
+  try {
+    return await read();
+  } catch (error) {
+    if (!isRuntimeSchedulerCircuitOpenError(error)) throw error;
+    const wait = options.wait ?? sleep;
+    await wait(RUNTIME_SCHEDULER_POOL_DEFAULTS.breaker.openMs);
+    return read();
+  }
+}
+
 type LatencyRing = {
   samples: number[];
   next: number;
@@ -697,11 +764,21 @@ export async function withRuntimeSchedulerClient<T>(
     phase: string;
     priority?: RuntimeSchedulerPoolPriority;
   },
-  operationOptions: { signal?: AbortSignal } = {},
+  operationOptions: RuntimeSchedulerOperationOptions = {},
 ): Promise<T> {
   const signal = operationOptions.signal;
   throwIfSchedulerAborted(signal);
   const entry = getPoolEntry(options);
+  const maxAttempts = Math.max(
+    1,
+    Math.min(
+      RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts,
+      Math.floor(
+        operationOptions.maxAttempts ??
+          RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts,
+      ),
+    ),
+  );
   const operationStartedAt = performance.now();
   const admissionStartedAt = performance.now();
   const releaseAdmission = await entry.admission.acquire(
@@ -712,11 +789,7 @@ export async function withRuntimeSchedulerClient<T>(
   let lastError: unknown = null;
   try {
     throwIfSchedulerAborted(signal);
-    for (
-      let attempt = 0;
-      attempt < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts;
-      attempt += 1
-    ) {
+    for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
       throwIfSchedulerAborted(signal);
       breakerGate(entry);
       if (entry.pool.waitingCount >= entry.maxAcquireQueueDepth) {
@@ -755,7 +828,7 @@ export async function withRuntimeSchedulerClient<T>(
         recordBreakerFailure(entry, error);
         entry.stats.connectionErrors += 1;
         lastError = error;
-        if (attempt + 1 < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts) {
+        if (attempt + 1 < maxAttempts) {
           entry.stats.retries += 1;
           await sleep(jitteredBackoffMs(attempt));
           throwIfSchedulerAborted(signal);
@@ -805,9 +878,18 @@ export async function withRuntimeSchedulerClient<T>(
             const queryStartedAt = performance.now();
             queryCount += 1;
             try {
-              const queryResult = await poolClient.query(sql, params);
-              throwIfSchedulerAborted(signal);
-              return { rows: queryResult.rows };
+              try {
+                const queryResult = await poolClient.query(sql, params);
+                throwIfSchedulerAborted(signal);
+                return { rows: queryResult.rows };
+              } catch (error) {
+                if (isRuntimeSchedulerQueryCancellationError(error)) {
+                  releasePoolClient(
+                    error instanceof Error ? error : new Error(String(error)),
+                  );
+                }
+                throw error;
+              }
             } finally {
               queryMs += performance.now() - queryStartedAt;
             }
@@ -873,6 +955,14 @@ export async function withRuntimeSchedulerClient<T>(
             error instanceof Error ? error.message : String(error),
           );
         }
+        if (isRuntimeSchedulerQueryCancellationError(error)) {
+          entry.stats.queryErrors += 1;
+          recordBreakerSuccess(entry);
+          releasePoolClient(
+            error instanceof Error ? error : new Error(String(error)),
+          );
+          throw error;
+        }
         if (isConnectionCategoryError(error)) {
           recordBreakerFailure(entry, error);
           entry.stats.connectionErrors += 1;
@@ -881,7 +971,7 @@ export async function withRuntimeSchedulerClient<T>(
           releasePoolClient(
             error instanceof Error ? error : new Error(String(error)),
           );
-          if (attempt + 1 < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts) {
+          if (attempt + 1 < maxAttempts) {
             entry.stats.retries += 1;
             await sleep(jitteredBackoffMs(attempt));
             throwIfSchedulerAborted(signal);
@@ -899,7 +989,7 @@ export async function withRuntimeSchedulerClient<T>(
           releasePoolClient(
             error instanceof Error ? error : new Error(String(error)),
           );
-          if (attempt + 1 < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts) {
+          if (attempt + 1 < maxAttempts) {
             entry.stats.retries += 1;
             await sleep(jitteredBackoffMs(attempt));
             throwIfSchedulerAborted(signal);
@@ -923,6 +1013,47 @@ export async function withRuntimeSchedulerClient<T>(
   }
 }
 
+export async function withRuntimeSchedulerTransaction<T>(
+  options: RuntimeSchedulerPoolOptions,
+  timeouts: { lockTimeoutMs: number; statementTimeoutMs: number },
+  fn: (client: RuntimeSchedulerQueryClient) => Promise<T>,
+  trace?: {
+    runId: string;
+    phase: string;
+    priority?: RuntimeSchedulerPoolPriority;
+  },
+  operationOptions: RuntimeSchedulerOperationOptions = {},
+): Promise<T> {
+  const lockTimeoutMs = Math.max(1, Math.floor(timeouts.lockTimeoutMs));
+  const statementTimeoutMs = Math.max(
+    lockTimeoutMs,
+    Math.floor(timeouts.statementTimeoutMs),
+  );
+  return withRuntimeSchedulerClient(
+    options,
+    async (client) => {
+      await client.query('BEGIN');
+      try {
+        await client.query(
+          `SELECT set_config('lock_timeout', $1, true),
+                  set_config('statement_timeout', $2, true)`,
+          [`${lockTimeoutMs}ms`, `${statementTimeoutMs}ms`],
+        );
+        const result = await fn(client);
+        await client.query('COMMIT');
+        return result;
+      } catch (error) {
+        if (!isRuntimeSchedulerQueryCancellationError(error)) {
+          await client.query('ROLLBACK').catch(() => undefined);
+        }
+        throw error;
+      }
+    },
+    trace,
+    { ...operationOptions, maxAttempts: operationOptions.maxAttempts ?? 1 },
+  );
+}
+
 export type RuntimeSchedulerPoolStats = {
   url: string;
   maxClients: number;

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -19,12 +19,18 @@ import {
 } from '../app-runtime-api';
 import type { PlayVisualNodeProgressMap } from '../worker-api-types';
 import type { PlayExecutionEvent } from '../ctx-types';
+import type { PlayActivityObservation } from '../activity-observation';
 import type {
   PlayRunnerLogEvent,
   PlayRunnerResult,
 } from '../protocol';
 import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
 import type { DocflowObservationRecord } from '../docflow-observation';
+import {
+  buildPlayRunLedgerEventsFromStatusPatch,
+  createEmptyPlayRunLedgerSnapshot,
+  type PlayRunLedgerEvent,
+} from '../run-ledger';
 import {
   getTopLevelPipelineSubsteps,
   type PlayStaticPipeline,
@@ -248,7 +254,16 @@ function shouldPublishTerminalResultLog(
   mode: PostgresSchedulerDurableLogMode,
   line: string,
 ): boolean {
-  return mode !== 'direct_worker' || CTX_LOG_TIMESTAMP_PREFIX_RE.test(line);
+  // Direct workers keep host lifecycle noise out of the customer Run Log
+  // Stream. The resource summary is one bounded, machine-readable terminal
+  // diagnostic rather than a host log, though: the fixture harness needs it
+  // after the runner has gone away and the normal terminal result is no longer
+  // available through the status projection.
+  return (
+    mode !== 'direct_worker' ||
+    CTX_LOG_TIMESTAMP_PREFIX_RE.test(line) ||
+    line.startsWith('[runtime.phase_summary] ')
+  );
 }
 
 type TerminalLogReplayPayload = NonNullable<
@@ -323,14 +338,34 @@ export function createPostgresSchedulerProgressReporter(input: {
   durableLogMode?: PostgresSchedulerDurableLogMode;
   producerAttempt?: number;
   isCurrentAttempt?: () => boolean | Promise<boolean>;
+  /**
+   * The caller already read the durable attempt fence for THIS request and it
+   * was current. The first projection flush then reuses that read instead of
+   * paying a second identical round trip. A reporter lives for exactly one
+   * request, and the credit is consumed by the first flush, so a later flush —
+   * and every other request — re-reads the fence.
+   */
+  attemptAlreadyVerified?: boolean;
   /** Durable log position before this reporter's first event. Push runners own
    * this cursor so successive batches may land on different gateway Machines. */
   initialLiveLogTotalCount?: number;
+  /** Override the app-runtime transport when a caller needs to capture the
+   * folded terminal patch for a durable scheduler outbox instead of posting
+   * it synchronously. */
+  publishStatusUpdate?: (update: RuntimeStatusUpdate) => Promise<void>;
 }): PostgresSchedulerProgressReporter {
   const launch = input.launch;
   const durableLogMode = input.durableLogMode ?? 'progress_reporter';
   const producerAttempt = input.producerAttempt;
-  const isCurrentAttempt = input.isCurrentAttempt ?? (() => true);
+  const readAttemptFence = input.isCurrentAttempt ?? (() => true);
+  let verifiedAttemptCredit = input.attemptAlreadyVerified === true;
+  const isCurrentAttempt = async (): Promise<boolean> => {
+    if (verifiedAttemptCredit) {
+      verifiedAttemptCredit = false;
+      return true;
+    }
+    return await readAttemptFence();
+  };
   const orderedNodes = buildOrderedNodeList(launch.contractSnapshot);
   const mapNodeLookup = buildMapNodeLookup(launch.contractSnapshot);
   const runtimeApi = {
@@ -338,6 +373,10 @@ export function createPostgresSchedulerProgressReporter(input: {
     executorToken: launch.executorToken,
     vercelProtectionBypassToken: launch.vercelProtectionBypassToken ?? null,
   };
+  const publishStatusUpdate =
+    input.publishStatusUpdate ??
+    ((update: RuntimeStatusUpdate) =>
+      updateRunStatusViaAppRuntime(runtimeApi, update));
 
   let liveNodeProgress: PlayVisualNodeProgressMap = {};
   let liveNodeProgressDirty = false;
@@ -427,7 +466,7 @@ export function createPostgresSchedulerProgressReporter(input: {
 
           try {
             if (await isCurrentAttempt()) {
-              await updateRunStatusViaAppRuntime(runtimeApi, {
+              await publishStatusUpdate({
                 playId: launch.runId,
                 status: request.status,
                 liveNodeProgress: nextProgress,
@@ -1246,10 +1285,102 @@ export const createDirectWorkerProgressReporter = (input: {
   launch: PlaySchedulerSubmitInput;
   producerAttempt?: number;
   isCurrentAttempt?: () => boolean | Promise<boolean>;
+  /** See `createPostgresSchedulerProgressReporter`: one request, one credit. */
+  attemptAlreadyVerified?: boolean;
   initialLiveLogTotalCount?: number;
+  publishStatusUpdate?: (update: RuntimeStatusUpdate) => Promise<void>;
 }) =>
   createPostgresSchedulerProgressReporter({
     ...input,
     durableLogMode: 'direct_worker',
   });
 export type DirectWorkerProgressReporter = PostgresSchedulerProgressReporter;
+
+/**
+ * Fold a detached runner's final progress without crossing the app-runtime
+ * network boundary. The returned events are stored in the same Postgres
+ * terminal outbox row as `run.completed`, so the projector can append the
+ * final step/log/activity facts and the terminal fact in one ordered Convex
+ * mutation. This is the completed-run fast path; failed/suspended runs retain
+ * the compatibility projection until their parked worker resumes.
+ */
+export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
+  launch: PlaySchedulerSubmitInput;
+  attempt: number;
+  result: PlayRunnerResult;
+  events: unknown[];
+  liveLogOffset: number;
+}): Promise<PlayRunLedgerEvent[]> {
+  const captured: { update?: RuntimeStatusUpdate } = {};
+  const reporter = createDirectWorkerProgressReporter({
+    launch: input.launch,
+    producerAttempt: input.attempt,
+    initialLiveLogTotalCount: input.liveLogOffset,
+    isCurrentAttempt: () => true,
+    attemptAlreadyVerified: true,
+    publishStatusUpdate: async (update) => {
+      captured.update = update;
+    },
+  });
+  for (const event of input.events) reporter.onRunnerEvent(event);
+  await reporter.finalize(input.result, {
+    status: 'running',
+    required: true,
+    skipTerminalLogReplay: true,
+    terminalLogProducerAttempt: input.attempt,
+  });
+  const finalUpdate = captured.update;
+  if (!finalUpdate) return [];
+
+  const liveLogs = finalUpdate.liveLogs ?? [];
+  const liveLogTotalCount =
+    finalUpdate.liveLogTotalCount ?? liveLogs.length + input.liveLogOffset;
+  const progressEvents = buildPlayRunLedgerEventsFromStatusPatch({
+    patch: {
+      runId: input.launch.runId,
+      status: 'running',
+      lastCheckpointAt: finalUpdate.lastCheckpointAt,
+      liveNodeProgress: finalUpdate.liveNodeProgress,
+      liveLogs,
+      liveLogsChannelOffset: Math.max(0, liveLogTotalCount - liveLogs.length),
+      liveLogsProducerAttempt:
+        finalUpdate.liveLogProducerAttempt ?? input.attempt,
+    },
+    previousSnapshot: createEmptyPlayRunLedgerSnapshot({
+      runId: input.launch.runId,
+      status: 'queued',
+    }),
+    source: 'worker',
+  });
+  const datasetEvents: PlayRunLedgerEvent[] = (
+    finalUpdate.datasetLifecycleEvents ?? []
+  ).map((event) => ({
+    type: 'dataset.lifecycle',
+    runId: input.launch.runId,
+    source: 'worker',
+    occurredAt: event.at,
+    datasetId: event.datasetId,
+    path: event.path,
+    tableNamespace: event.tableNamespace,
+    phase: event.phase,
+    ...(event.persistedRows === undefined
+      ? {}
+      : { persistedRows: event.persistedRows }),
+    ...(event.succeededRows === undefined
+      ? {}
+      : { succeededRows: event.succeededRows }),
+    ...(event.failedRows === undefined ? {} : { failedRows: event.failedRows }),
+    ...(event.complete === undefined ? {} : { complete: event.complete }),
+    ...(event.bornFrom === undefined ? {} : { bornFrom: event.bornFrom }),
+  }));
+  const activityEvents: PlayRunLedgerEvent[] = (
+    finalUpdate.activityObservations ?? []
+  ).map((observation: PlayActivityObservation) => ({
+    type: 'activity.observed',
+    runId: input.launch.runId,
+    source: 'worker',
+    occurredAt: observation.observedAt,
+    observation,
+  }));
+  return [...progressEvents, ...datasetEvents, ...activityEvents];
+}