deepline 0.3.129 → 0.3.131

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.129',
+  version: '0.3.131',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/sdk/src/types.ts

@@ -947,6 +947,8 @@ export interface StopPlayRunResult {
   runId: string;
   /** Whether the server confirmed the run was stopped. */
   stopped: boolean;
+  /** True when the durable cancellation lane accepted the request and is draining. */
+  cancelling?: boolean;
   /** Number of open HITL interactions marked cancelled. */
   hitlCancelledCount: number;
   /**
@@ -965,6 +967,7 @@ export interface StopPlayRunResult {
  */
 export interface StopAllPlayRunsStopResult {
   stopped: number;
+  cancelling?: number;
   failed: number;
   skipped: number;
   /** True when the bounded candidate query may have left active runs behind. */

package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts

@@ -1,4 +1,5 @@
 import { withRuntimeSchedulerClient } from './/scheduler-backends/postgres-pool';
+import { RUNTIME_CAPACITY_POLICY } from './/runtime-capacity-policy';
 import { isIsolatedRuntimeSchedulerSchema } from './/runtime-scheduler-topology';
 
 /** A single controller owns this fleet-wide sweep. Workers clean resources for
@@ -60,7 +61,9 @@ export function startDaytonaSandboxJanitor(
         {
           url: options.url,
           applicationName: 'deepline-daytona-janitor',
-          maxClients: options.maxClients ?? 2,
+          maxClients:
+            options.maxClients ??
+            RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients,
         },
         (client) =>
           reapOrphanDaytonaSandboxes({

package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts

@@ -7,11 +7,24 @@
  * and tests cannot independently invent concurrency or timeout budgets.
  */
 export const RUNTIME_CAPACITY_POLICY = {
+  runtimePostgres: {
+    /**
+     * One process-local scheduler pool budget for every runtime control-plane
+     * caller. More worker slots must queue behind this number; they must not
+     * create one pool per feature or widen the database connection fan-out.
+     */
+    schedulerPoolMaxClients: 2,
+  },
   absurd: {
     /** Warm floor for availability and deploy rollovers. */
     activeLaneMachines: 2,
-    /** 32 Machines x 8 claim slots permits 256 concurrent launch/resume legs. */
-    maxActiveLaneMachines: 32,
+    /**
+     * Keep the fleet below the PlanetScale direct-session budget. Each worker
+     * has one elected direct wake listener and a two-client pooled scheduler
+     * budget; scaling past this point amplifies connection pressure faster
+     * than it adds useful control-plane throughput.
+     */
+    maxActiveLaneMachines: 8,
     workerSlotsPerMachine: 8,
     /** React within one control interval when a due backlog is not draining. */
     scaleUpQueueAgeMs: 1_000,
@@ -174,6 +187,16 @@ function nonNegativeNumber(value: number): number {
 }
 
 export function assertRuntimeCapacityPolicy(): void {
+  if (
+    !Number.isInteger(
+      RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients,
+    ) ||
+    RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients < 1
+  ) {
+    throw new Error(
+      'Runtime scheduler pool client cap must be a positive integer.',
+    );
+  }
   if (
     !Number.isInteger(
       RUNTIME_CAPACITY_POLICY.sandboxAcquisition

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -257,6 +257,14 @@ export type PlaySchedulerRunHandle = {
   result(): Promise<PlaySchedulerResultEnvelope>;
 };
 
+/**
+ * The durable engine cancellation lane accepted the stop request, but the
+ * scheduler run has not reached its authoritative terminal state yet.
+ * Callers must surface this as an in-progress cancellation, never as success.
+ */
+export const RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE =
+  'RUNTIME_SCHEDULER_CANCELLATION_PENDING';
+
 export type PlaySchedulerSignalPayload = {
   kind: 'integration_event' | 'cancel' | 'custom';
   eventKey?: string;

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts

@@ -3,6 +3,12 @@ import type {
   PostgresSchedulerOptions,
   PostgresSchedulerQueryClient,
 } from './postgres';
+import {
+  isRuntimeSchedulerQueryCancellationError,
+  isRuntimeSchedulerRetryableTransactionError,
+  RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
+  RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
+} from './postgres-pool';
 import { ABSURD_PLAY_RUN_TASK } from './absurd-shared';
 import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
 
@@ -14,18 +20,20 @@ function quoteAbsurdTable(table: string): string {
 }
 
 /**
- * Project a play task cancelled by Absurd's indexed `max_delay` sweep into the
- * scheduler's authoritative run state.
+ * Project a play task cancelled by Absurd into the scheduler's authoritative
+ * run state. This covers both indexed `max_delay` cancellation and the manual
+ * cancellation lane. Manual cancellation stamps `cancel_at` in the adapter so
+ * existing Absurd v4 queues are discoverable without a schema cutover.
  *
  * The engine can cancel an overdue task before or after a worker claims its
  * handler. A cancelled task has no handler path left to write the normal
  * terminal state. This reconciler
  * closes that gap: `cancelPostgresSchedulerRun` atomically writes the runtime
  * terminal and its run-ledger outbox event, and is idempotent if a user cancel
- * or a racing worker already terminalized the run. The next empty claim invokes
- * it immediately after Absurd performs its bounded cancellation sweep.
+ * or a racing worker already terminalized the run. The next claim/maintenance
+ * pass invokes it after Absurd performs its bounded cancellation sweep.
  */
-export async function reconcileAbsurdRuntimeMaxDelayCancellations(input: {
+export async function reconcileAbsurdRuntimeCancellations(input: {
   client: PostgresSchedulerQueryClient;
   queue: string;
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
@@ -46,18 +54,30 @@ export async function reconcileAbsurdRuntimeMaxDelayCancellations(input: {
       resolveRuntimeSchedulerSchemaEnv() ||
       'runtime_scheduler',
   );
-  const cancelled = await input.client.query<{ run_id: string | null }>(
+  const cancelled = await input.client.query<{
+    run_id: string | null;
+    cancellation_kind: 'manual' | 'deadline';
+  }>(
     `
-      SELECT task.params ->> 'runId' AS run_id
-      FROM absurd.${taskTable} AS task
-      JOIN ${schedulerSchema}.runs AS run
-        ON run.run_id = task.params ->> 'runId'
+      SELECT run.run_id,
+             CASE
+               WHEN task.cancellation ? 'manual' THEN 'manual'
+               ELSE 'deadline'
+             END AS cancellation_kind
+      FROM ${schedulerSchema}.runs AS run
+      JOIN absurd.${taskTable} AS task
+        ON task.task_id::text = run.absurd_task_id
        AND run.status IN ('queued', 'running', 'waiting')
       WHERE task.state = 'cancelled'
         AND task.task_name = $1
-        AND task.cancellation ? 'max_delay'
         AND task.cancel_at IS NOT NULL
-        AND task.cancelled_at >= task.cancel_at
+        AND (
+          task.cancellation ? 'manual'
+          OR (
+            task.cancellation ? 'max_delay'
+            AND task.cancelled_at >= task.cancel_at
+          )
+        )
       ORDER BY task.cancelled_at ASC NULLS LAST, task.task_id ASC
       LIMIT $2
     `,
@@ -72,17 +92,51 @@ export async function reconcileAbsurdRuntimeMaxDelayCancellations(input: {
       skipped += 1;
       continue;
     }
-    const didCancel = await cancelPostgresSchedulerRun(
-      input.client,
-      {
-        runId,
-        reason:
-          'Run did not start within its configured scheduler queue deadline.',
-      },
-      input.schedulerOptions,
-    );
-    if (didCancel) reconciled += 1;
-    else skipped += 1;
+    await input.client.query('BEGIN');
+    try {
+      await input.client.query(
+        `SELECT set_config('lock_timeout', $1, true),
+                set_config('statement_timeout', $2, true)`,
+        [
+          `${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
+          `${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
+        ],
+      );
+      const didCancel = await cancelPostgresSchedulerRun(
+        input.client,
+        {
+          runId,
+          reason:
+            row.cancellation_kind === 'manual'
+              ? 'Run cancellation was requested by the caller.'
+              : 'Run did not start within its configured scheduler queue deadline.',
+        },
+        input.schedulerOptions,
+      );
+      await input.client.query('COMMIT');
+      if (didCancel) reconciled += 1;
+      else skipped += 1;
+    } catch (error) {
+      await input.client.query('ROLLBACK').catch(() => undefined);
+      // A worker-held lock is expected to be retried by the next lane sweep;
+      // never let one blocked cancellation stall the claim loop or create a
+      // second unbounded cancellation pile.
+      if (
+        isRuntimeSchedulerQueryCancellationError(error) ||
+        isRuntimeSchedulerRetryableTransactionError(error)
+      ) {
+        skipped += 1;
+        // The hardened pool destroys a client after a statement timeout or a
+        // retryable transaction error. Do not issue the next candidate on that
+        // same socket; the next worker sweep acquires a fresh client.
+        break;
+      }
+      throw error;
+    }
   }
   return { reconciled, skipped };
 }
+
+/** Compatibility export for callers/tests that still name the old deadline-only lane. */
+export const reconcileAbsurdRuntimeMaxDelayCancellations =
+  reconcileAbsurdRuntimeCancellations;

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -262,7 +262,7 @@ import { runtimeApiBaseUrlFromLaunch } from './runtime-api-origin';
 import { shouldTerminalizeSandboxCapacityRejection } from '../launch-source-policy';
 import {
   ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
-  reconcileAbsurdRuntimeMaxDelayCancellations,
+  reconcileAbsurdRuntimeCancellations,
 } from './absurd-deadline-reconciliation';
 import {
   abandonRuntimeSandboxOperationReservation,
@@ -3254,8 +3254,7 @@ async function executeAssignedAbsurdAttempt(input: {
                     };
               await enforceRuntimeTrafficPolicyBeforeSandboxStart({
                 policy: trafficPolicy,
-                defaultSandboxStartsPerMinute:
-                  defaultDaytonaStartRatePerMinute,
+                defaultSandboxStartsPerMinute: defaultDaytonaStartRatePerMinute,
                 nowMs: Date.now(),
                 acquireStartPermit: async (sandboxStartsPerMinute) =>
                   await deps.runWithClient((client) =>
@@ -3273,8 +3272,7 @@ async function executeAssignedAbsurdAttempt(input: {
                           maxConcurrency: null,
                           adaptiveMaxRps:
                             (daytonaStartRateConfig?.maximumRatePerMinute ??
-                              sandboxStartsPerMinute) /
-                            60,
+                              sandboxStartsPerMinute) / 60,
                         },
                       ],
                       // The permit itself is the successful admission
@@ -3632,10 +3630,7 @@ async function executeAssignedAbsurdAttempt(input: {
       await cleanupAfterExecutionError(error, runnerConfig);
       const recoveryError =
         recoveryForDaytonaSandboxAcquisitionUnavailable(error) ?? error;
-      if (
-        recoveryError !== error &&
-        daytonaStartRateConfig !== null
-      ) {
+      if (recoveryError !== error && daytonaStartRateConfig !== null) {
         await deps.runWithClient((client) =>
           penalizePostgresRateState(client, {
             bucketId: 'runtime:traffic:sandbox-start',
@@ -4003,9 +3998,7 @@ export async function enforceRuntimeTrafficPolicyBeforeSandboxStart(input: {
         : Math.max(
             0,
             permit.waitMs,
-            scheduledAtMs > input.nowMs
-              ? scheduledAtMs - schedulerNowMs
-              : 0,
+            scheduledAtMs > input.nowMs ? scheduledAtMs - schedulerNowMs : 0,
           );
     // A sub-second schedule retains smooth pacing without spending a worker
     // attempt. Longer waits become a durable, bounded defer, allowing the
@@ -6954,28 +6947,27 @@ export async function startRuntimeWorkerHost(
       schedulerOptions.schema,
     ),
   });
-  const reconcileDeadlineCancellations = async () => {
+  const reconcileRuntimeCancellations = async () => {
     await withRuntimeSchedulerClient(poolOptions, async (client) => {
-      const deadlineCancellations =
-        await reconcileAbsurdRuntimeMaxDelayCancellations({
-          client,
-          queue,
-          schedulerOptions,
-        });
-      if (deadlineCancellations.reconciled > 0) {
-        console.info('[absurd-runtime.worker] max_delay_reconciled', {
+      const cancellations = await reconcileAbsurdRuntimeCancellations({
+        client,
+        queue,
+        schedulerOptions,
+      });
+      if (cancellations.reconciled > 0) {
+        console.info('[absurd-runtime.worker] cancellations_reconciled', {
           queue,
           releaseId,
-          ...deadlineCancellations,
+          ...cancellations,
         });
       }
     });
   };
-  const sweepAndReconcileDeadlineCancellations = async () => {
+  const sweepAndReconcileCancellations = async () => {
     await absurdClient.sweepCancellations(
       ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
     );
-    await reconcileDeadlineCancellations();
+    await reconcileRuntimeCancellations();
   };
   const reportQueueError = createRateLimitedRuntimeLogger({
     log: (...args) => console.error(...args),
@@ -6987,12 +6979,13 @@ export async function startRuntimeWorkerHost(
     pollInterval: ABSURD_IDLE_POLL_INTERVAL_SECONDS,
     // `claim_task` performs the indexed max_delay sweep. Reconcile after every
     // sweep, including claims that return backlog work, so a saturated queue
-    // cannot postpone the scheduler/outbox terminal indefinitely.
-    afterClaim: reconcileDeadlineCancellations,
+    // cannot postpone a cancellation's scheduler/outbox terminal indefinitely.
+    afterClaim: reconcileRuntimeCancellations,
     // claim_task cannot run while every handler slot is occupied. Keep the
-    // enqueue deadline independent from handler duration by sweeping without
-    // claiming, then project cancellation into the scheduler/outbox.
-    saturatedMaintenance: sweepAndReconcileDeadlineCancellations,
+    // enqueue deadline and cancellation lane independent from handler duration
+    // by sweeping without claiming, then project cancellation into the
+    // scheduler/outbox.
+    saturatedMaintenance: sweepAndReconcileCancellations,
     nextPollDelayMs: () =>
       withRuntimeSchedulerClient(poolOptions, async (client) => {
         const nextDue = await readAbsurdQueueNextDue(client, queue);

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts

@@ -14,7 +14,8 @@
  *             Persist the absurd task id
  *             on the run row so a later cancel can target it.
  *   observe/result → reuse the postgres LISTEN/NOTIFY run handle.
- *   cancel  → postgres terminal cancel intent + absurd cancelTask.
+ *   cancel  → durable Absurd cancellation lane + bounded postgres terminal
+ *             projection (with worker reconciliation when locks are busy).
  *   signal  → durable signal row (authoritative payload) + absurd emitEvent
  *             (the wakeup only; payload is re-read from the signal row).
  */
@@ -30,6 +31,7 @@ import {
 } from '../runner-backends/runtime-sandbox-reconciliation';
 import {
   PLAY_SCHEDULER_BACKENDS,
+  RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE,
   type PlaySchedulerBackend,
   type PlaySchedulerRunHandle,
   type PlaySchedulerSubmitInput,
@@ -45,7 +47,12 @@ import {
   createPlayRunAdmissionJob,
   createPostgresPlayRunAdmissionPorts,
 } from '../play-runs/admission';
-import { withRuntimeSchedulerClient } from './postgres-pool';
+import {
+  RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
+  RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
+  withRuntimeSchedulerClient,
+  withRuntimeSchedulerTransaction,
+} from './postgres-pool';
 import { withRuntimeSchedulerSubmitAdmission } from './submit-admission';
 import {
   ABSURD_PLAY_RUN_MAX_ATTEMPTS,
@@ -75,6 +82,26 @@ export type AbsurdSchedulerBackendOptions = AbsurdSchedulerOptions & {
   queueSuffix?: string | null;
 };
 
+const cancellationFlights = new Map<string, Promise<void>>();
+
+function quotePostgresIdentifier(identifier: string): string {
+  return `"${identifier.replaceAll('"', '""')}"`;
+}
+function runCancellationSingleFlight(
+  key: string,
+  operation: () => Promise<void>,
+): Promise<void> {
+  const existing = cancellationFlights.get(key);
+  if (existing) return existing;
+  const flight = operation().finally(() => {
+    if (cancellationFlights.get(key) === flight) {
+      cancellationFlights.delete(key);
+    }
+  });
+  cancellationFlights.set(key, flight);
+  return flight;
+}
+
 function recordAbsurdSubmitPhase(
   input: Pick<PlaySchedulerSubmitInput, 'runId' | 'playName' | 'graphHash'>,
   phase: string,
@@ -133,14 +160,86 @@ function absurdRunHandle(input: {
   const releaseId =
     input.releaseId?.trim() || absurdReleaseIdForSuffix(input.queueSuffix);
   const runsQueue = absurdQueueNameForRelease(releaseId);
+  const cancellationKey = `${poolOptions.url}::${effectiveOptions.schema ?? ''}::${input.runId}`;
 
   const cancelAbsurdTask = async () => {
     const taskId = await withRuntimeSchedulerClient(poolOptions, (client) =>
       readPostgresSchedulerAbsurdTaskId(client, input.runId, effectiveOptions),
     );
     if (!taskId) return;
-    const absurd = createAbsurdRuntimeClient(poolOptions, runsQueue);
-    await absurd.cancelTask(taskId, runsQueue);
+    // Keep cancellation on the same bounded control-plane contract as the
+    // scheduler terminal write. This is a durable lane transition, not an
+    // unbounded best-effort query which can pile up behind a worker lock.
+    await withRuntimeSchedulerTransaction(
+      poolOptions,
+      {
+        lockTimeoutMs: RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
+        statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
+      },
+      async (client) => {
+        await client.query(`SELECT absurd.cancel_task($1, $2::uuid)`, [
+          runsQueue,
+          taskId,
+        ]);
+        // Older v4 Absurd functions do not materialize a manual cancellation
+        // deadline or reason. Stamp both in the adapter so existing queues
+        // participate in the same durable reconciliation lane without a
+        // schema cutover. The explicit marker keeps a user cancel distinct
+        // from a task cancelled by its configured max_delay sweep.
+        await client.query(
+          `UPDATE absurd.${quotePostgresIdentifier(`t_${runsQueue}`)}
+              SET cancel_at = COALESCE(cancel_at, clock_timestamp()),
+                  cancellation = COALESCE(cancellation, '{}'::jsonb)
+                    || jsonb_build_object('manual', true)
+            WHERE task_id = $1::uuid
+              AND state = 'cancelled'`,
+          [taskId],
+        );
+      },
+      {
+        runId: input.runId,
+        phase: 'cancel_engine_task',
+        priority: 'critical',
+      },
+      { maxAttempts: 1 },
+    );
+  };
+
+  const cancellationPendingError = (cause: unknown): Error => {
+    const error = new Error(
+      `Run cancellation was durably requested, but scheduler terminalization is pending for ${input.runId}.`,
+    );
+    Object.assign(error, {
+      code: RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE,
+      cause,
+    });
+    return error;
+  };
+
+  const requestCancellation = async () => {
+    await runCancellationSingleFlight(cancellationKey, async () => {
+      // First move the durable engine task into its cancellation lane. If the
+      // scheduler tables are briefly locked, the worker reconciler can finish
+      // the terminal write later without another user request piling up.
+      await cancelAbsurdTask();
+      try {
+        await delegate.cancel();
+      } catch (error) {
+        throw cancellationPendingError(error);
+      }
+      try {
+        await cleanupPersistedRuntimeSandboxes();
+      } catch (error) {
+        // The scheduler terminal transaction already won. Its cleanup jobs are
+        // durable, so a best-effort eager cleanup failure must not turn a real
+        // cancellation into a false 502 or cause the caller to retry the
+        // terminal write.
+        console.warn('[absurd.runtime_sandbox_cleanup_deferred]', {
+          runId: input.runId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    });
   };
 
   const cleanupPersistedRuntimeSandboxes = async () => {
@@ -229,18 +328,10 @@ function absurdRunHandle(input: {
     ...delegate,
     runId: input.runId,
     initialState: input.initialState ?? undefined,
-    cancel: async () => {
-      const cleanup = cleanupPersistedRuntimeSandboxes();
-      await Promise.all([delegate.cancel(), cancelAbsurdTask(), cleanup]);
-    },
+    cancel: requestCancellation,
     signal: async (payload) => {
       if (payload.kind === 'cancel') {
-        const cleanup = cleanupPersistedRuntimeSandboxes();
-        await Promise.all([
-          delegate.signal(payload),
-          cancelAbsurdTask(),
-          cleanup,
-        ]);
+        await requestCancellation();
         return;
       }
       if (payload.kind !== 'integration_event' || !payload.eventKey) {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts

@@ -33,6 +33,7 @@ import {
   PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS,
 } from '../admission-transport-policy';
 import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
+import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
 export {
   isRuntimeSchedulerCapacityError,
   parseRuntimeSchedulerCapacityError,
@@ -126,8 +127,8 @@ export function assertRuntimeSchedulerHotPathUrl(url: string): void {
 }
 
 export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
-  /** Per-process client cap. Workers override via options; keep small on serverless. */
-  maxClients: 8,
+  /** One shared per-process client cap for every scheduler control-plane lane. */
+  maxClients: RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients,
   /** Refuse to construct a pool above this — protects the shared PgBouncer/Postgres. */
   maxClientsHardCap: 32,
   /** Max time a caller waits for a pooled connection before erroring. */
@@ -163,7 +164,16 @@ export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
  * once. This shared cap keeps horizontally scaled workers and the receipt
  * gateway from consuming PgBouncer's global client budget independently.
  */
-export const RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS = 2;
+export const RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS =
+  RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients;
+
+/**
+ * Cancellation is control-plane work, not customer execution. A blocked
+ * cancellation must give up quickly instead of sitting behind a worker-held
+ * row lock and spawning more identical cancellation statements on retry.
+ */
+export const RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS = 2_000;
+export const RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS = 15_000;
 
 export type RuntimeSchedulerPoolOptions = {
   url: string;
@@ -188,6 +198,12 @@ export type RuntimeSchedulerPoolOptions = {
   maxAcquireQueueWaitMs?: number;
 };
 
+export type RuntimeSchedulerOperationOptions = {
+  signal?: AbortSignal;
+  /** Override only for an operation with its own retry/lock contract. */
+  maxAttempts?: number;
+};
+
 export type RuntimeSchedulerQueryClient = {
   query<Row extends Record<string, unknown> = Record<string, unknown>>(
     sql: string,
@@ -195,6 +211,23 @@ export type RuntimeSchedulerQueryClient = {
   ): Promise<{ rows: Row[] }>;
 };
 
+export function isRuntimeSchedulerQueryCancellationError(
+  error: unknown,
+): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const code = (error as { code?: unknown }).code;
+  if (code === '57014') return true;
+  const name = (error as { name?: unknown }).name;
+  if (name === 'TimeoutError' || name === 'AbortError') return true;
+  const message = (error as { message?: unknown }).message;
+  return (
+    typeof message === 'string' &&
+    /(?:statement timeout|canceling statement due to user request|query read timeout)/i.test(
+      message,
+    )
+  );
+}
+
 export class RuntimeSchedulerCircuitOpenError extends Error {
   readonly code = 'RUNTIME_SCHEDULER_UNAVAILABLE';
 
@@ -738,11 +771,21 @@ export async function withRuntimeSchedulerClient<T>(
     phase: string;
     priority?: RuntimeSchedulerPoolPriority;
   },
-  operationOptions: { signal?: AbortSignal } = {},
+  operationOptions: RuntimeSchedulerOperationOptions = {},
 ): Promise<T> {
   const signal = operationOptions.signal;
   throwIfSchedulerAborted(signal);
   const entry = getPoolEntry(options);
+  const maxAttempts = Math.max(
+    1,
+    Math.min(
+      RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts,
+      Math.floor(
+        operationOptions.maxAttempts ??
+          RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts,
+      ),
+    ),
+  );
   const operationStartedAt = performance.now();
   const admissionStartedAt = performance.now();
   const releaseAdmission = await entry.admission.acquire(
@@ -753,11 +796,7 @@ export async function withRuntimeSchedulerClient<T>(
   let lastError: unknown = null;
   try {
     throwIfSchedulerAborted(signal);
-    for (
-      let attempt = 0;
-      attempt < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts;
-      attempt += 1
-    ) {
+    for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
       throwIfSchedulerAborted(signal);
       breakerGate(entry);
       if (entry.pool.waitingCount >= entry.maxAcquireQueueDepth) {
@@ -796,7 +835,7 @@ export async function withRuntimeSchedulerClient<T>(
         recordBreakerFailure(entry, error);
         entry.stats.connectionErrors += 1;
         lastError = error;
-        if (attempt + 1 < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts) {
+        if (attempt + 1 < maxAttempts) {
           entry.stats.retries += 1;
           await sleep(jitteredBackoffMs(attempt));
           throwIfSchedulerAborted(signal);
@@ -846,9 +885,22 @@ export async function withRuntimeSchedulerClient<T>(
             const queryStartedAt = performance.now();
             queryCount += 1;
             try {
-              const queryResult = await poolClient.query(sql, params);
-              throwIfSchedulerAborted(signal);
-              return { rows: queryResult.rows };
+              try {
+                const queryResult = await poolClient.query(sql, params);
+                throwIfSchedulerAborted(signal);
+                return { rows: queryResult.rows };
+              } catch (error) {
+                // node-postgres' query timeout and an AbortSignal reject the
+                // promise, but the server may still be processing the SQL.
+                // Never return that socket to the pool: doing so lets the
+                // next retry overlap the first statement on the same backend.
+                if (isRuntimeSchedulerQueryCancellationError(error)) {
+                  releasePoolClient(
+                    error instanceof Error ? error : new Error(String(error)),
+                  );
+                }
+                throw error;
+              }
             } finally {
               queryMs += performance.now() - queryStartedAt;
             }
@@ -914,6 +966,14 @@ export async function withRuntimeSchedulerClient<T>(
             error instanceof Error ? error.message : String(error),
           );
         }
+        if (isRuntimeSchedulerQueryCancellationError(error)) {
+          entry.stats.queryErrors += 1;
+          recordBreakerSuccess(entry);
+          releasePoolClient(
+            error instanceof Error ? error : new Error(String(error)),
+          );
+          throw error;
+        }
         if (isConnectionCategoryError(error)) {
           recordBreakerFailure(entry, error);
           entry.stats.connectionErrors += 1;
@@ -922,7 +982,7 @@ export async function withRuntimeSchedulerClient<T>(
           releasePoolClient(
             error instanceof Error ? error : new Error(String(error)),
           );
-          if (attempt + 1 < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts) {
+          if (attempt + 1 < maxAttempts) {
             entry.stats.retries += 1;
             await sleep(jitteredBackoffMs(attempt));
             throwIfSchedulerAborted(signal);
@@ -940,7 +1000,7 @@ export async function withRuntimeSchedulerClient<T>(
           releasePoolClient(
             error instanceof Error ? error : new Error(String(error)),
           );
-          if (attempt + 1 < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts) {
+          if (attempt + 1 < maxAttempts) {
             entry.stats.retries += 1;
             await sleep(jitteredBackoffMs(attempt));
             throwIfSchedulerAborted(signal);
@@ -964,6 +1024,63 @@ export async function withRuntimeSchedulerClient<T>(
   }
 }
 
+/**
+ * Run one bounded scheduler transaction on a checked-out pooled client.
+ * `SET LOCAL` is deliberately issued inside the transaction because the
+ * runtime hot path uses PgBouncer transaction pooling and must not leak
+ * session settings between callers.
+ */
+export async function withRuntimeSchedulerTransaction<T>(
+  options: RuntimeSchedulerPoolOptions,
+  timeouts: {
+    lockTimeoutMs: number;
+    statementTimeoutMs: number;
+  },
+  fn: (client: RuntimeSchedulerQueryClient) => Promise<T>,
+  trace?: {
+    runId: string;
+    phase: string;
+    priority?: RuntimeSchedulerPoolPriority;
+  },
+  operationOptions: RuntimeSchedulerOperationOptions = {},
+): Promise<T> {
+  const lockTimeoutMs = Math.max(1, Math.floor(timeouts.lockTimeoutMs));
+  const statementTimeoutMs = Math.max(
+    lockTimeoutMs,
+    Math.floor(timeouts.statementTimeoutMs),
+  );
+  return withRuntimeSchedulerClient(
+    options,
+    async (client) => {
+      await client.query('BEGIN');
+      try {
+        await client.query(
+          `SELECT set_config('lock_timeout', $1, true),
+                  set_config('statement_timeout', $2, true)`,
+          [`${lockTimeoutMs}ms`, `${statementTimeoutMs}ms`],
+        );
+        const result = await fn(client);
+        await client.query('COMMIT');
+        return result;
+      } catch (error) {
+        if (!isRuntimeSchedulerQueryCancellationError(error)) {
+          await client.query('ROLLBACK').catch(() => undefined);
+        }
+        throw error;
+      }
+    },
+    trace,
+    {
+      ...operationOptions,
+      // A cancellation is a user/control-plane decision. If its lock cannot
+      // be acquired inside the bounded window, the caller must report the
+      // active run honestly; retrying here would recreate the incident's
+      // cancellation pile.
+      maxAttempts: operationOptions.maxAttempts ?? 1,
+    },
+  );
+}
+
 export type RuntimeSchedulerPoolStats = {
   url: string;
   maxClients: number;

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -6,6 +6,9 @@ import {
   acquirePgNotifyHub,
   resolveWorkflowRuntimePostgresUrl,
   withRuntimeSchedulerClient,
+  withRuntimeSchedulerTransaction,
+  RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
+  RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
   type PgNotifyHub,
   type PgNotifyHubState,
   type RuntimeSchedulerPoolOptions,
@@ -16974,15 +16977,35 @@ export function postgresSchedulerRunHandle(
     },
     cancel: async () => {
       const poolOptions = resolve();
-      await withRuntimeSchedulerClient(poolOptions, (client) =>
-        cancelPostgresSchedulerRun(client, { runId }, options),
+      await withRuntimeSchedulerTransaction(
+        poolOptions,
+        {
+          lockTimeoutMs: RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
+          statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
+        },
+        (client) => cancelPostgresSchedulerRun(client, { runId }, options),
+        {
+          runId,
+          phase: 'cancel_run',
+          priority: 'critical',
+        },
       );
     },
     signal: async (payload) => {
       if (payload.kind === 'cancel') {
         const poolOptions = resolve();
-        await withRuntimeSchedulerClient(poolOptions, (client) =>
-          cancelPostgresSchedulerRun(client, { runId }, options),
+        await withRuntimeSchedulerTransaction(
+          poolOptions,
+          {
+            lockTimeoutMs: RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
+            statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
+          },
+          (client) => cancelPostgresSchedulerRun(client, { runId }, options),
+          {
+            runId,
+            phase: 'cancel_run',
+            priority: 'critical',
+          },
         );
         return;
       }

package/dist/cli/index.js

@@ -3068,7 +3068,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.129",
+  version: "0.3.131",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/cli/index.mjs

@@ -3063,7 +3063,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.129",
+  version: "0.3.131",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/index.d.mts

@@ -1891,6 +1891,8 @@ interface StopPlayRunResult {
     runId: string;
     /** Whether the server confirmed the run was stopped. */
     stopped: boolean;
+    /** True when the durable cancellation lane accepted the request and is draining. */
+    cancelling?: boolean;
     /** Number of open HITL interactions marked cancelled. */
     hitlCancelledCount: number;
     /**
@@ -1908,6 +1910,7 @@ interface StopPlayRunResult {
  */
 interface StopAllPlayRunsStopResult {
     stopped: number;
+    cancelling?: number;
     failed: number;
     skipped: number;
     /** True when the bounded candidate query may have left active runs behind. */

package/dist/index.d.ts

@@ -1891,6 +1891,8 @@ interface StopPlayRunResult {
     runId: string;
     /** Whether the server confirmed the run was stopped. */
     stopped: boolean;
+    /** True when the durable cancellation lane accepted the request and is draining. */
+    cancelling?: boolean;
     /** Number of open HITL interactions marked cancelled. */
     hitlCancelledCount: number;
     /**
@@ -1908,6 +1910,7 @@ interface StopPlayRunResult {
  */
 interface StopAllPlayRunsStopResult {
     stopped: number;
+    cancelling?: number;
     failed: number;
     skipped: number;
     /** True when the bounded candidate query may have left active runs behind. */

package/dist/index.js

@@ -864,7 +864,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.129",
+  version: "0.3.131",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/index.mjs

@@ -768,7 +768,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.129",
+  version: "0.3.131",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/release.d.mts

@@ -149,7 +149,7 @@ type SdkRelease = {
     supportPolicy: SdkSupportPolicy;
 };
 declare const SDK_RELEASE: {
-    readonly version: "0.3.129";
+    readonly version: "0.3.131";
     readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
     readonly packageCapabilities: {
         readonly updatePreferences: 1;

package/dist/release.d.ts

@@ -149,7 +149,7 @@ type SdkRelease = {
     supportPolicy: SdkSupportPolicy;
 };
 declare const SDK_RELEASE: {
-    readonly version: "0.3.129";
+    readonly version: "0.3.131";
     readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
     readonly packageCapabilities: {
         readonly updatePreferences: 1;

package/dist/release.js

@@ -74,7 +74,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.129",
+  version: "0.3.131",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/release.mjs

@@ -48,7 +48,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.129",
+  version: "0.3.131",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.3.129",
+  "version": "0.3.131",
   "description": "GTM data CLI and TypeScript SDK for coding agents",
   "license": "MIT",
   "homepage": "https://code.deepline.com",