deepline 0.3.130 → 0.3.131

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
200
200
  // getters keep their established compatibility behavior.
201
201
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
202
202
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
203
- version: '0.3.130',
203
+ version: '0.3.131',
204
204
  updateSummary:
205
205
  'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
206
206
  packageCapabilities: {
@@ -947,6 +947,8 @@ export interface StopPlayRunResult {
947
947
  runId: string;
948
948
  /** Whether the server confirmed the run was stopped. */
949
949
  stopped: boolean;
950
+ /** True when the durable cancellation lane accepted the request and is draining. */
951
+ cancelling?: boolean;
950
952
  /** Number of open HITL interactions marked cancelled. */
951
953
  hitlCancelledCount: number;
952
954
  /**
@@ -965,6 +967,7 @@ export interface StopPlayRunResult {
965
967
  */
966
968
  export interface StopAllPlayRunsStopResult {
967
969
  stopped: number;
970
+ cancelling?: number;
968
971
  failed: number;
969
972
  skipped: number;
970
973
  /** True when the bounded candidate query may have left active runs behind. */
@@ -8,13 +8,22 @@
8
8
  */
9
9
  export const RUNTIME_CAPACITY_POLICY = {
10
10
  runtimePostgres: {
11
- /** One process-local scheduler pool budget for every control-plane lane. */
11
+ /**
12
+ * One process-local scheduler pool budget for every runtime control-plane
13
+ * caller. More worker slots must queue behind this number; they must not
14
+ * create one pool per feature or widen the database connection fan-out.
15
+ */
12
16
  schedulerPoolMaxClients: 2,
13
17
  },
14
18
  absurd: {
15
19
  /** Warm floor for availability and deploy rollovers. */
16
20
  activeLaneMachines: 2,
17
- /** Keep worker pools and direct wake listeners below the shared DB budget. */
21
+ /**
22
+ * Keep the fleet below the PlanetScale direct-session budget. Each worker
23
+ * has one elected direct wake listener and a two-client pooled scheduler
24
+ * budget; scaling past this point amplifies connection pressure faster
25
+ * than it adds useful control-plane throughput.
26
+ */
18
27
  maxActiveLaneMachines: 8,
19
28
  workerSlotsPerMachine: 8,
20
29
  /** React within one control interval when a due backlog is not draining. */
@@ -257,6 +257,14 @@ export type PlaySchedulerRunHandle = {
257
257
  result(): Promise<PlaySchedulerResultEnvelope>;
258
258
  };
259
259
 
260
+ /**
261
+ * The durable engine cancellation lane accepted the stop request, but the
262
+ * scheduler run has not reached its authoritative terminal state yet.
263
+ * Callers must surface this as an in-progress cancellation, never as success.
264
+ */
265
+ export const RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE =
266
+ 'RUNTIME_SCHEDULER_CANCELLATION_PENDING';
267
+
260
268
  export type PlaySchedulerSignalPayload = {
261
269
  kind: 'integration_event' | 'cancel' | 'custom';
262
270
  eventKey?: string;
@@ -3,6 +3,12 @@ import type {
3
3
  PostgresSchedulerOptions,
4
4
  PostgresSchedulerQueryClient,
5
5
  } from './postgres';
6
+ import {
7
+ isRuntimeSchedulerQueryCancellationError,
8
+ isRuntimeSchedulerRetryableTransactionError,
9
+ RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
10
+ RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
11
+ } from './postgres-pool';
6
12
  import { ABSURD_PLAY_RUN_TASK } from './absurd-shared';
7
13
  import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
8
14
 
@@ -14,18 +20,20 @@ function quoteAbsurdTable(table: string): string {
14
20
  }
15
21
 
16
22
  /**
17
- * Project a play task cancelled by Absurd's indexed `max_delay` sweep into the
18
- * scheduler's authoritative run state.
23
+ * Project a play task cancelled by Absurd into the scheduler's authoritative
24
+ * run state. This covers both indexed `max_delay` cancellation and the manual
25
+ * cancellation lane. Manual cancellation stamps `cancel_at` in the adapter so
26
+ * existing Absurd v4 queues are discoverable without a schema cutover.
19
27
  *
20
28
  * The engine can cancel an overdue task before or after a worker claims its
21
29
  * handler. A cancelled task has no handler path left to write the normal
22
30
  * terminal state. This reconciler
23
31
  * closes that gap: `cancelPostgresSchedulerRun` atomically writes the runtime
24
32
  * terminal and its run-ledger outbox event, and is idempotent if a user cancel
25
- * or a racing worker already terminalized the run. The next empty claim invokes
26
- * it immediately after Absurd performs its bounded cancellation sweep.
33
+ * or a racing worker already terminalized the run. The next claim/maintenance
34
+ * pass invokes it after Absurd performs its bounded cancellation sweep.
27
35
  */
28
- export async function reconcileAbsurdRuntimeMaxDelayCancellations(input: {
36
+ export async function reconcileAbsurdRuntimeCancellations(input: {
29
37
  client: PostgresSchedulerQueryClient;
30
38
  queue: string;
31
39
  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
@@ -46,18 +54,30 @@ export async function reconcileAbsurdRuntimeMaxDelayCancellations(input: {
46
54
  resolveRuntimeSchedulerSchemaEnv() ||
47
55
  'runtime_scheduler',
48
56
  );
49
- const cancelled = await input.client.query<{ run_id: string | null }>(
57
+ const cancelled = await input.client.query<{
58
+ run_id: string | null;
59
+ cancellation_kind: 'manual' | 'deadline';
60
+ }>(
50
61
  `
51
- SELECT task.params ->> 'runId' AS run_id
52
- FROM absurd.${taskTable} AS task
53
- JOIN ${schedulerSchema}.runs AS run
54
- ON run.run_id = task.params ->> 'runId'
62
+ SELECT run.run_id,
63
+ CASE
64
+ WHEN task.cancellation ? 'manual' THEN 'manual'
65
+ ELSE 'deadline'
66
+ END AS cancellation_kind
67
+ FROM ${schedulerSchema}.runs AS run
68
+ JOIN absurd.${taskTable} AS task
69
+ ON task.task_id::text = run.absurd_task_id
55
70
  AND run.status IN ('queued', 'running', 'waiting')
56
71
  WHERE task.state = 'cancelled'
57
72
  AND task.task_name = $1
58
- AND task.cancellation ? 'max_delay'
59
73
  AND task.cancel_at IS NOT NULL
60
- AND task.cancelled_at >= task.cancel_at
74
+ AND (
75
+ task.cancellation ? 'manual'
76
+ OR (
77
+ task.cancellation ? 'max_delay'
78
+ AND task.cancelled_at >= task.cancel_at
79
+ )
80
+ )
61
81
  ORDER BY task.cancelled_at ASC NULLS LAST, task.task_id ASC
62
82
  LIMIT $2
63
83
  `,
@@ -72,17 +92,51 @@ export async function reconcileAbsurdRuntimeMaxDelayCancellations(input: {
72
92
  skipped += 1;
73
93
  continue;
74
94
  }
75
- const didCancel = await cancelPostgresSchedulerRun(
76
- input.client,
77
- {
78
- runId,
79
- reason:
80
- 'Run did not start within its configured scheduler queue deadline.',
81
- },
82
- input.schedulerOptions,
83
- );
84
- if (didCancel) reconciled += 1;
85
- else skipped += 1;
95
+ await input.client.query('BEGIN');
96
+ try {
97
+ await input.client.query(
98
+ `SELECT set_config('lock_timeout', $1, true),
99
+ set_config('statement_timeout', $2, true)`,
100
+ [
101
+ `${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
102
+ `${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
103
+ ],
104
+ );
105
+ const didCancel = await cancelPostgresSchedulerRun(
106
+ input.client,
107
+ {
108
+ runId,
109
+ reason:
110
+ row.cancellation_kind === 'manual'
111
+ ? 'Run cancellation was requested by the caller.'
112
+ : 'Run did not start within its configured scheduler queue deadline.',
113
+ },
114
+ input.schedulerOptions,
115
+ );
116
+ await input.client.query('COMMIT');
117
+ if (didCancel) reconciled += 1;
118
+ else skipped += 1;
119
+ } catch (error) {
120
+ await input.client.query('ROLLBACK').catch(() => undefined);
121
+ // A worker-held lock is expected to be retried by the next lane sweep;
122
+ // never let one blocked cancellation stall the claim loop or create a
123
+ // second unbounded cancellation pile.
124
+ if (
125
+ isRuntimeSchedulerQueryCancellationError(error) ||
126
+ isRuntimeSchedulerRetryableTransactionError(error)
127
+ ) {
128
+ skipped += 1;
129
+ // The hardened pool destroys a client after a statement timeout or a
130
+ // retryable transaction error. Do not issue the next candidate on that
131
+ // same socket; the next worker sweep acquires a fresh client.
132
+ break;
133
+ }
134
+ throw error;
135
+ }
86
136
  }
87
137
  return { reconciled, skipped };
88
138
  }
139
+
140
+ /** Compatibility export for callers/tests that still name the old deadline-only lane. */
141
+ export const reconcileAbsurdRuntimeMaxDelayCancellations =
142
+ reconcileAbsurdRuntimeCancellations;
@@ -262,7 +262,7 @@ import { runtimeApiBaseUrlFromLaunch } from './runtime-api-origin';
262
262
  import { shouldTerminalizeSandboxCapacityRejection } from '../launch-source-policy';
263
263
  import {
264
264
  ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
265
- reconcileAbsurdRuntimeMaxDelayCancellations,
265
+ reconcileAbsurdRuntimeCancellations,
266
266
  } from './absurd-deadline-reconciliation';
267
267
  import {
268
268
  abandonRuntimeSandboxOperationReservation,
@@ -3254,8 +3254,7 @@ async function executeAssignedAbsurdAttempt(input: {
3254
3254
  };
3255
3255
  await enforceRuntimeTrafficPolicyBeforeSandboxStart({
3256
3256
  policy: trafficPolicy,
3257
- defaultSandboxStartsPerMinute:
3258
- defaultDaytonaStartRatePerMinute,
3257
+ defaultSandboxStartsPerMinute: defaultDaytonaStartRatePerMinute,
3259
3258
  nowMs: Date.now(),
3260
3259
  acquireStartPermit: async (sandboxStartsPerMinute) =>
3261
3260
  await deps.runWithClient((client) =>
@@ -3273,8 +3272,7 @@ async function executeAssignedAbsurdAttempt(input: {
3273
3272
  maxConcurrency: null,
3274
3273
  adaptiveMaxRps:
3275
3274
  (daytonaStartRateConfig?.maximumRatePerMinute ??
3276
- sandboxStartsPerMinute) /
3277
- 60,
3275
+ sandboxStartsPerMinute) / 60,
3278
3276
  },
3279
3277
  ],
3280
3278
  // The permit itself is the successful admission
@@ -3632,10 +3630,7 @@ async function executeAssignedAbsurdAttempt(input: {
3632
3630
  await cleanupAfterExecutionError(error, runnerConfig);
3633
3631
  const recoveryError =
3634
3632
  recoveryForDaytonaSandboxAcquisitionUnavailable(error) ?? error;
3635
- if (
3636
- recoveryError !== error &&
3637
- daytonaStartRateConfig !== null
3638
- ) {
3633
+ if (recoveryError !== error && daytonaStartRateConfig !== null) {
3639
3634
  await deps.runWithClient((client) =>
3640
3635
  penalizePostgresRateState(client, {
3641
3636
  bucketId: 'runtime:traffic:sandbox-start',
@@ -4003,9 +3998,7 @@ export async function enforceRuntimeTrafficPolicyBeforeSandboxStart(input: {
4003
3998
  : Math.max(
4004
3999
  0,
4005
4000
  permit.waitMs,
4006
- scheduledAtMs > input.nowMs
4007
- ? scheduledAtMs - schedulerNowMs
4008
- : 0,
4001
+ scheduledAtMs > input.nowMs ? scheduledAtMs - schedulerNowMs : 0,
4009
4002
  );
4010
4003
  // A sub-second schedule retains smooth pacing without spending a worker
4011
4004
  // attempt. Longer waits become a durable, bounded defer, allowing the
@@ -6954,28 +6947,27 @@ export async function startRuntimeWorkerHost(
6954
6947
  schedulerOptions.schema,
6955
6948
  ),
6956
6949
  });
6957
- const reconcileDeadlineCancellations = async () => {
6950
+ const reconcileRuntimeCancellations = async () => {
6958
6951
  await withRuntimeSchedulerClient(poolOptions, async (client) => {
6959
- const deadlineCancellations =
6960
- await reconcileAbsurdRuntimeMaxDelayCancellations({
6961
- client,
6962
- queue,
6963
- schedulerOptions,
6964
- });
6965
- if (deadlineCancellations.reconciled > 0) {
6966
- console.info('[absurd-runtime.worker] max_delay_reconciled', {
6952
+ const cancellations = await reconcileAbsurdRuntimeCancellations({
6953
+ client,
6954
+ queue,
6955
+ schedulerOptions,
6956
+ });
6957
+ if (cancellations.reconciled > 0) {
6958
+ console.info('[absurd-runtime.worker] cancellations_reconciled', {
6967
6959
  queue,
6968
6960
  releaseId,
6969
- ...deadlineCancellations,
6961
+ ...cancellations,
6970
6962
  });
6971
6963
  }
6972
6964
  });
6973
6965
  };
6974
- const sweepAndReconcileDeadlineCancellations = async () => {
6966
+ const sweepAndReconcileCancellations = async () => {
6975
6967
  await absurdClient.sweepCancellations(
6976
6968
  ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
6977
6969
  );
6978
- await reconcileDeadlineCancellations();
6970
+ await reconcileRuntimeCancellations();
6979
6971
  };
6980
6972
  const reportQueueError = createRateLimitedRuntimeLogger({
6981
6973
  log: (...args) => console.error(...args),
@@ -6987,12 +6979,13 @@ export async function startRuntimeWorkerHost(
6987
6979
  pollInterval: ABSURD_IDLE_POLL_INTERVAL_SECONDS,
6988
6980
  // `claim_task` performs the indexed max_delay sweep. Reconcile after every
6989
6981
  // sweep, including claims that return backlog work, so a saturated queue
6990
- // cannot postpone the scheduler/outbox terminal indefinitely.
6991
- afterClaim: reconcileDeadlineCancellations,
6982
+ // cannot postpone a cancellation's scheduler/outbox terminal indefinitely.
6983
+ afterClaim: reconcileRuntimeCancellations,
6992
6984
  // claim_task cannot run while every handler slot is occupied. Keep the
6993
- // enqueue deadline independent from handler duration by sweeping without
6994
- // claiming, then project cancellation into the scheduler/outbox.
6995
- saturatedMaintenance: sweepAndReconcileDeadlineCancellations,
6985
+ // enqueue deadline and cancellation lane independent from handler duration
6986
+ // by sweeping without claiming, then project cancellation into the
6987
+ // scheduler/outbox.
6988
+ saturatedMaintenance: sweepAndReconcileCancellations,
6996
6989
  nextPollDelayMs: () =>
6997
6990
  withRuntimeSchedulerClient(poolOptions, async (client) => {
6998
6991
  const nextDue = await readAbsurdQueueNextDue(client, queue);
@@ -14,7 +14,8 @@
14
14
  * Persist the absurd task id
15
15
  * on the run row so a later cancel can target it.
16
16
  * observe/result → reuse the postgres LISTEN/NOTIFY run handle.
17
- * cancel → postgres terminal cancel intent + absurd cancelTask.
17
+ * cancel → durable Absurd cancellation lane + bounded postgres terminal
18
+ * projection (with worker reconciliation when locks are busy).
18
19
  * signal → durable signal row (authoritative payload) + absurd emitEvent
19
20
  * (the wakeup only; payload is re-read from the signal row).
20
21
  */
@@ -30,6 +31,7 @@ import {
30
31
  } from '../runner-backends/runtime-sandbox-reconciliation';
31
32
  import {
32
33
  PLAY_SCHEDULER_BACKENDS,
34
+ RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE,
33
35
  type PlaySchedulerBackend,
34
36
  type PlaySchedulerRunHandle,
35
37
  type PlaySchedulerSubmitInput,
@@ -45,7 +47,12 @@ import {
45
47
  createPlayRunAdmissionJob,
46
48
  createPostgresPlayRunAdmissionPorts,
47
49
  } from '../play-runs/admission';
48
- import { withRuntimeSchedulerClient } from './postgres-pool';
50
+ import {
51
+ RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
52
+ RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
53
+ withRuntimeSchedulerClient,
54
+ withRuntimeSchedulerTransaction,
55
+ } from './postgres-pool';
49
56
  import { withRuntimeSchedulerSubmitAdmission } from './submit-admission';
50
57
  import {
51
58
  ABSURD_PLAY_RUN_MAX_ATTEMPTS,
@@ -77,6 +84,9 @@ export type AbsurdSchedulerBackendOptions = AbsurdSchedulerOptions & {
77
84
 
78
85
  const cancellationFlights = new Map<string, Promise<void>>();
79
86
 
87
+ function quotePostgresIdentifier(identifier: string): string {
88
+ return `"${identifier.replaceAll('"', '""')}"`;
89
+ }
80
90
  function runCancellationSingleFlight(
81
91
  key: string,
82
92
  operation: () => Promise<void>,
@@ -157,8 +167,79 @@ function absurdRunHandle(input: {
157
167
  readPostgresSchedulerAbsurdTaskId(client, input.runId, effectiveOptions),
158
168
  );
159
169
  if (!taskId) return;
160
- const absurd = createAbsurdRuntimeClient(poolOptions, runsQueue);
161
- await absurd.cancelTask(taskId, runsQueue);
170
+ // Keep cancellation on the same bounded control-plane contract as the
171
+ // scheduler terminal write. This is a durable lane transition, not an
172
+ // unbounded best-effort query which can pile up behind a worker lock.
173
+ await withRuntimeSchedulerTransaction(
174
+ poolOptions,
175
+ {
176
+ lockTimeoutMs: RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
177
+ statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
178
+ },
179
+ async (client) => {
180
+ await client.query(`SELECT absurd.cancel_task($1, $2::uuid)`, [
181
+ runsQueue,
182
+ taskId,
183
+ ]);
184
+ // Older v4 Absurd functions do not materialize a manual cancellation
185
+ // deadline or reason. Stamp both in the adapter so existing queues
186
+ // participate in the same durable reconciliation lane without a
187
+ // schema cutover. The explicit marker keeps a user cancel distinct
188
+ // from a task cancelled by its configured max_delay sweep.
189
+ await client.query(
190
+ `UPDATE absurd.${quotePostgresIdentifier(`t_${runsQueue}`)}
191
+ SET cancel_at = COALESCE(cancel_at, clock_timestamp()),
192
+ cancellation = COALESCE(cancellation, '{}'::jsonb)
193
+ || jsonb_build_object('manual', true)
194
+ WHERE task_id = $1::uuid
195
+ AND state = 'cancelled'`,
196
+ [taskId],
197
+ );
198
+ },
199
+ {
200
+ runId: input.runId,
201
+ phase: 'cancel_engine_task',
202
+ priority: 'critical',
203
+ },
204
+ { maxAttempts: 1 },
205
+ );
206
+ };
207
+
208
+ const cancellationPendingError = (cause: unknown): Error => {
209
+ const error = new Error(
210
+ `Run cancellation was durably requested, but scheduler terminalization is pending for ${input.runId}.`,
211
+ );
212
+ Object.assign(error, {
213
+ code: RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE,
214
+ cause,
215
+ });
216
+ return error;
217
+ };
218
+
219
+ const requestCancellation = async () => {
220
+ await runCancellationSingleFlight(cancellationKey, async () => {
221
+ // First move the durable engine task into its cancellation lane. If the
222
+ // scheduler tables are briefly locked, the worker reconciler can finish
223
+ // the terminal write later without another user request piling up.
224
+ await cancelAbsurdTask();
225
+ try {
226
+ await delegate.cancel();
227
+ } catch (error) {
228
+ throw cancellationPendingError(error);
229
+ }
230
+ try {
231
+ await cleanupPersistedRuntimeSandboxes();
232
+ } catch (error) {
233
+ // The scheduler terminal transaction already won. Its cleanup jobs are
234
+ // durable, so a best-effort eager cleanup failure must not turn a real
235
+ // cancellation into a false 502 or cause the caller to retry the
236
+ // terminal write.
237
+ console.warn('[absurd.runtime_sandbox_cleanup_deferred]', {
238
+ runId: input.runId,
239
+ error: error instanceof Error ? error.message : String(error),
240
+ });
241
+ }
242
+ });
162
243
  };
163
244
 
164
245
  const cleanupPersistedRuntimeSandboxes = async () => {
@@ -247,20 +328,10 @@ function absurdRunHandle(input: {
247
328
  ...delegate,
248
329
  runId: input.runId,
249
330
  initialState: input.initialState ?? undefined,
250
- cancel: async () => {
251
- await runCancellationSingleFlight(cancellationKey, async () => {
252
- await delegate.cancel();
253
- await cancelAbsurdTask();
254
- await cleanupPersistedRuntimeSandboxes();
255
- });
256
- },
331
+ cancel: requestCancellation,
257
332
  signal: async (payload) => {
258
333
  if (payload.kind === 'cancel') {
259
- await runCancellationSingleFlight(cancellationKey, async () => {
260
- await delegate.signal(payload);
261
- await cancelAbsurdTask();
262
- await cleanupPersistedRuntimeSandboxes();
263
- });
334
+ await requestCancellation();
264
335
  return;
265
336
  }
266
337
  if (payload.kind !== 'integration_event' || !payload.eventKey) {
@@ -127,7 +127,7 @@ export function assertRuntimeSchedulerHotPathUrl(url: string): void {
127
127
  }
128
128
 
129
129
  export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
130
- /** One shared per-process client cap for scheduler control-plane lanes. */
130
+ /** One shared per-process client cap for every scheduler control-plane lane. */
131
131
  maxClients: RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients,
132
132
  /** Refuse to construct a pool above this — protects the shared PgBouncer/Postgres. */
133
133
  maxClientsHardCap: 32,
@@ -167,6 +167,11 @@ export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
167
167
  export const RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS =
168
168
  RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients;
169
169
 
170
+ /**
171
+ * Cancellation is control-plane work, not customer execution. A blocked
172
+ * cancellation must give up quickly instead of sitting behind a worker-held
173
+ * row lock and spawning more identical cancellation statements on retry.
174
+ */
170
175
  export const RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS = 2_000;
171
176
  export const RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS = 15_000;
172
177
 
@@ -195,6 +200,7 @@ export type RuntimeSchedulerPoolOptions = {
195
200
 
196
201
  export type RuntimeSchedulerOperationOptions = {
197
202
  signal?: AbortSignal;
203
+ /** Override only for an operation with its own retry/lock contract. */
198
204
  maxAttempts?: number;
199
205
  };
200
206
 
@@ -209,7 +215,8 @@ export function isRuntimeSchedulerQueryCancellationError(
209
215
  error: unknown,
210
216
  ): boolean {
211
217
  if (!error || typeof error !== 'object') return false;
212
- if ((error as { code?: unknown }).code === '57014') return true;
218
+ const code = (error as { code?: unknown }).code;
219
+ if (code === '57014') return true;
213
220
  const name = (error as { name?: unknown }).name;
214
221
  if (name === 'TimeoutError' || name === 'AbortError') return true;
215
222
  const message = (error as { message?: unknown }).message;
@@ -883,6 +890,10 @@ export async function withRuntimeSchedulerClient<T>(
883
890
  throwIfSchedulerAborted(signal);
884
891
  return { rows: queryResult.rows };
885
892
  } catch (error) {
893
+ // node-postgres' query timeout and an AbortSignal reject the
894
+ // promise, but the server may still be processing the SQL.
895
+ // Never return that socket to the pool: doing so lets the
896
+ // next retry overlap the first statement on the same backend.
886
897
  if (isRuntimeSchedulerQueryCancellationError(error)) {
887
898
  releasePoolClient(
888
899
  error instanceof Error ? error : new Error(String(error)),
@@ -1013,9 +1024,18 @@ export async function withRuntimeSchedulerClient<T>(
1013
1024
  }
1014
1025
  }
1015
1026
 
1027
+ /**
1028
+ * Run one bounded scheduler transaction on a checked-out pooled client.
1029
+ * `SET LOCAL` is deliberately issued inside the transaction because the
1030
+ * runtime hot path uses PgBouncer transaction pooling and must not leak
1031
+ * session settings between callers.
1032
+ */
1016
1033
  export async function withRuntimeSchedulerTransaction<T>(
1017
1034
  options: RuntimeSchedulerPoolOptions,
1018
- timeouts: { lockTimeoutMs: number; statementTimeoutMs: number },
1035
+ timeouts: {
1036
+ lockTimeoutMs: number;
1037
+ statementTimeoutMs: number;
1038
+ },
1019
1039
  fn: (client: RuntimeSchedulerQueryClient) => Promise<T>,
1020
1040
  trace?: {
1021
1041
  runId: string;
@@ -1050,7 +1070,14 @@ export async function withRuntimeSchedulerTransaction<T>(
1050
1070
  }
1051
1071
  },
1052
1072
  trace,
1053
- { ...operationOptions, maxAttempts: operationOptions.maxAttempts ?? 1 },
1073
+ {
1074
+ ...operationOptions,
1075
+ // A cancellation is a user/control-plane decision. If its lock cannot
1076
+ // be acquired inside the bounded window, the caller must report the
1077
+ // active run honestly; retrying here would recreate the incident's
1078
+ // cancellation pile.
1079
+ maxAttempts: operationOptions.maxAttempts ?? 1,
1080
+ },
1054
1081
  );
1055
1082
  }
1056
1083
 
@@ -16984,7 +16984,11 @@ export function postgresSchedulerRunHandle(
16984
16984
  statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
16985
16985
  },
16986
16986
  (client) => cancelPostgresSchedulerRun(client, { runId }, options),
16987
- { runId, phase: 'cancel_run', priority: 'critical' },
16987
+ {
16988
+ runId,
16989
+ phase: 'cancel_run',
16990
+ priority: 'critical',
16991
+ },
16988
16992
  );
16989
16993
  },
16990
16994
  signal: async (payload) => {
@@ -16997,7 +17001,11 @@ export function postgresSchedulerRunHandle(
16997
17001
  statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
16998
17002
  },
16999
17003
  (client) => cancelPostgresSchedulerRun(client, { runId }, options),
17000
- { runId, phase: 'cancel_run', priority: 'critical' },
17004
+ {
17005
+ runId,
17006
+ phase: 'cancel_run',
17007
+ priority: 'critical',
17008
+ },
17001
17009
  );
17002
17010
  return;
17003
17011
  }
package/dist/cli/index.js CHANGED
@@ -3068,7 +3068,7 @@ var SDK_RELEASE = {
3068
3068
  // getters keep their established compatibility behavior.
3069
3069
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
3070
3070
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
3071
- version: "0.3.130",
3071
+ version: "0.3.131",
3072
3072
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
3073
3073
  packageCapabilities: {
3074
3074
  updatePreferences: 1
@@ -3063,7 +3063,7 @@ var SDK_RELEASE = {
3063
3063
  // getters keep their established compatibility behavior.
3064
3064
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
3065
3065
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
3066
- version: "0.3.130",
3066
+ version: "0.3.131",
3067
3067
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
3068
3068
  packageCapabilities: {
3069
3069
  updatePreferences: 1
package/dist/index.d.mts CHANGED
@@ -1891,6 +1891,8 @@ interface StopPlayRunResult {
1891
1891
  runId: string;
1892
1892
  /** Whether the server confirmed the run was stopped. */
1893
1893
  stopped: boolean;
1894
+ /** True when the durable cancellation lane accepted the request and is draining. */
1895
+ cancelling?: boolean;
1894
1896
  /** Number of open HITL interactions marked cancelled. */
1895
1897
  hitlCancelledCount: number;
1896
1898
  /**
@@ -1908,6 +1910,7 @@ interface StopPlayRunResult {
1908
1910
  */
1909
1911
  interface StopAllPlayRunsStopResult {
1910
1912
  stopped: number;
1913
+ cancelling?: number;
1911
1914
  failed: number;
1912
1915
  skipped: number;
1913
1916
  /** True when the bounded candidate query may have left active runs behind. */
package/dist/index.d.ts CHANGED
@@ -1891,6 +1891,8 @@ interface StopPlayRunResult {
1891
1891
  runId: string;
1892
1892
  /** Whether the server confirmed the run was stopped. */
1893
1893
  stopped: boolean;
1894
+ /** True when the durable cancellation lane accepted the request and is draining. */
1895
+ cancelling?: boolean;
1894
1896
  /** Number of open HITL interactions marked cancelled. */
1895
1897
  hitlCancelledCount: number;
1896
1898
  /**
@@ -1908,6 +1910,7 @@ interface StopPlayRunResult {
1908
1910
  */
1909
1911
  interface StopAllPlayRunsStopResult {
1910
1912
  stopped: number;
1913
+ cancelling?: number;
1911
1914
  failed: number;
1912
1915
  skipped: number;
1913
1916
  /** True when the bounded candidate query may have left active runs behind. */
package/dist/index.js CHANGED
@@ -864,7 +864,7 @@ var SDK_RELEASE = {
864
864
  // getters keep their established compatibility behavior.
865
865
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
866
866
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
867
- version: "0.3.130",
867
+ version: "0.3.131",
868
868
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
869
869
  packageCapabilities: {
870
870
  updatePreferences: 1
package/dist/index.mjs CHANGED
@@ -768,7 +768,7 @@ var SDK_RELEASE = {
768
768
  // getters keep their established compatibility behavior.
769
769
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
770
770
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
771
- version: "0.3.130",
771
+ version: "0.3.131",
772
772
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
773
773
  packageCapabilities: {
774
774
  updatePreferences: 1
@@ -149,7 +149,7 @@ type SdkRelease = {
149
149
  supportPolicy: SdkSupportPolicy;
150
150
  };
151
151
  declare const SDK_RELEASE: {
152
- readonly version: "0.3.130";
152
+ readonly version: "0.3.131";
153
153
  readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
154
154
  readonly packageCapabilities: {
155
155
  readonly updatePreferences: 1;
package/dist/release.d.ts CHANGED
@@ -149,7 +149,7 @@ type SdkRelease = {
149
149
  supportPolicy: SdkSupportPolicy;
150
150
  };
151
151
  declare const SDK_RELEASE: {
152
- readonly version: "0.3.130";
152
+ readonly version: "0.3.131";
153
153
  readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
154
154
  readonly packageCapabilities: {
155
155
  readonly updatePreferences: 1;
package/dist/release.js CHANGED
@@ -74,7 +74,7 @@ var SDK_RELEASE = {
74
74
  // getters keep their established compatibility behavior.
75
75
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
76
76
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
77
- version: "0.3.130",
77
+ version: "0.3.131",
78
78
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
79
79
  packageCapabilities: {
80
80
  updatePreferences: 1
package/dist/release.mjs CHANGED
@@ -48,7 +48,7 @@ var SDK_RELEASE = {
48
48
  // getters keep their established compatibility behavior.
49
49
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
50
50
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
51
- version: "0.3.130",
51
+ version: "0.3.131",
52
52
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
53
53
  packageCapabilities: {
54
54
  updatePreferences: 1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "deepline",
3
- "version": "0.3.130",
3
+ "version": "0.3.131",
4
4
  "description": "GTM data CLI and TypeScript SDK for coding agents",
5
5
  "license": "MIT",
6
6
  "homepage": "https://code.deepline.com",