deepline 0.3.130 → 0.3.131
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/types.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +11 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts +77 -23
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +22 -29
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +87 -16
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +31 -4
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +10 -2
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.d.mts +3 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.131',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -947,6 +947,8 @@ export interface StopPlayRunResult {
|
|
|
947
947
|
runId: string;
|
|
948
948
|
/** Whether the server confirmed the run was stopped. */
|
|
949
949
|
stopped: boolean;
|
|
950
|
+
/** True when the durable cancellation lane accepted the request and is draining. */
|
|
951
|
+
cancelling?: boolean;
|
|
950
952
|
/** Number of open HITL interactions marked cancelled. */
|
|
951
953
|
hitlCancelledCount: number;
|
|
952
954
|
/**
|
|
@@ -965,6 +967,7 @@ export interface StopPlayRunResult {
|
|
|
965
967
|
*/
|
|
966
968
|
export interface StopAllPlayRunsStopResult {
|
|
967
969
|
stopped: number;
|
|
970
|
+
cancelling?: number;
|
|
968
971
|
failed: number;
|
|
969
972
|
skipped: number;
|
|
970
973
|
/** True when the bounded candidate query may have left active runs behind. */
|
|
@@ -8,13 +8,22 @@
|
|
|
8
8
|
*/
|
|
9
9
|
export const RUNTIME_CAPACITY_POLICY = {
|
|
10
10
|
runtimePostgres: {
|
|
11
|
-
/**
|
|
11
|
+
/**
|
|
12
|
+
* One process-local scheduler pool budget for every runtime control-plane
|
|
13
|
+
* caller. More worker slots must queue behind this number; they must not
|
|
14
|
+
* create one pool per feature or widen the database connection fan-out.
|
|
15
|
+
*/
|
|
12
16
|
schedulerPoolMaxClients: 2,
|
|
13
17
|
},
|
|
14
18
|
absurd: {
|
|
15
19
|
/** Warm floor for availability and deploy rollovers. */
|
|
16
20
|
activeLaneMachines: 2,
|
|
17
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* Keep the fleet below the PlanetScale direct-session budget. Each worker
|
|
23
|
+
* has one elected direct wake listener and a two-client pooled scheduler
|
|
24
|
+
* budget; scaling past this point amplifies connection pressure faster
|
|
25
|
+
* than it adds useful control-plane throughput.
|
|
26
|
+
*/
|
|
18
27
|
maxActiveLaneMachines: 8,
|
|
19
28
|
workerSlotsPerMachine: 8,
|
|
20
29
|
/** React within one control interval when a due backlog is not draining. */
|
|
@@ -257,6 +257,14 @@ export type PlaySchedulerRunHandle = {
|
|
|
257
257
|
result(): Promise<PlaySchedulerResultEnvelope>;
|
|
258
258
|
};
|
|
259
259
|
|
|
260
|
+
/**
|
|
261
|
+
* The durable engine cancellation lane accepted the stop request, but the
|
|
262
|
+
* scheduler run has not reached its authoritative terminal state yet.
|
|
263
|
+
* Callers must surface this as an in-progress cancellation, never as success.
|
|
264
|
+
*/
|
|
265
|
+
export const RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE =
|
|
266
|
+
'RUNTIME_SCHEDULER_CANCELLATION_PENDING';
|
|
267
|
+
|
|
260
268
|
export type PlaySchedulerSignalPayload = {
|
|
261
269
|
kind: 'integration_event' | 'cancel' | 'custom';
|
|
262
270
|
eventKey?: string;
|
|
@@ -3,6 +3,12 @@ import type {
|
|
|
3
3
|
PostgresSchedulerOptions,
|
|
4
4
|
PostgresSchedulerQueryClient,
|
|
5
5
|
} from './postgres';
|
|
6
|
+
import {
|
|
7
|
+
isRuntimeSchedulerQueryCancellationError,
|
|
8
|
+
isRuntimeSchedulerRetryableTransactionError,
|
|
9
|
+
RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
|
|
10
|
+
RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
|
|
11
|
+
} from './postgres-pool';
|
|
6
12
|
import { ABSURD_PLAY_RUN_TASK } from './absurd-shared';
|
|
7
13
|
import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
|
|
8
14
|
|
|
@@ -14,18 +20,20 @@ function quoteAbsurdTable(table: string): string {
|
|
|
14
20
|
}
|
|
15
21
|
|
|
16
22
|
/**
|
|
17
|
-
* Project a play task cancelled by Absurd
|
|
18
|
-
*
|
|
23
|
+
* Project a play task cancelled by Absurd into the scheduler's authoritative
|
|
24
|
+
* run state. This covers both indexed `max_delay` cancellation and the manual
|
|
25
|
+
* cancellation lane. Manual cancellation stamps `cancel_at` in the adapter so
|
|
26
|
+
* existing Absurd v4 queues are discoverable without a schema cutover.
|
|
19
27
|
*
|
|
20
28
|
* The engine can cancel an overdue task before or after a worker claims its
|
|
21
29
|
* handler. A cancelled task has no handler path left to write the normal
|
|
22
30
|
* terminal state. This reconciler
|
|
23
31
|
* closes that gap: `cancelPostgresSchedulerRun` atomically writes the runtime
|
|
24
32
|
* terminal and its run-ledger outbox event, and is idempotent if a user cancel
|
|
25
|
-
* or a racing worker already terminalized the run. The next
|
|
26
|
-
* it
|
|
33
|
+
* or a racing worker already terminalized the run. The next claim/maintenance
|
|
34
|
+
* pass invokes it after Absurd performs its bounded cancellation sweep.
|
|
27
35
|
*/
|
|
28
|
-
export async function
|
|
36
|
+
export async function reconcileAbsurdRuntimeCancellations(input: {
|
|
29
37
|
client: PostgresSchedulerQueryClient;
|
|
30
38
|
queue: string;
|
|
31
39
|
schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
|
|
@@ -46,18 +54,30 @@ export async function reconcileAbsurdRuntimeMaxDelayCancellations(input: {
|
|
|
46
54
|
resolveRuntimeSchedulerSchemaEnv() ||
|
|
47
55
|
'runtime_scheduler',
|
|
48
56
|
);
|
|
49
|
-
const cancelled = await input.client.query<{
|
|
57
|
+
const cancelled = await input.client.query<{
|
|
58
|
+
run_id: string | null;
|
|
59
|
+
cancellation_kind: 'manual' | 'deadline';
|
|
60
|
+
}>(
|
|
50
61
|
`
|
|
51
|
-
SELECT
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
62
|
+
SELECT run.run_id,
|
|
63
|
+
CASE
|
|
64
|
+
WHEN task.cancellation ? 'manual' THEN 'manual'
|
|
65
|
+
ELSE 'deadline'
|
|
66
|
+
END AS cancellation_kind
|
|
67
|
+
FROM ${schedulerSchema}.runs AS run
|
|
68
|
+
JOIN absurd.${taskTable} AS task
|
|
69
|
+
ON task.task_id::text = run.absurd_task_id
|
|
55
70
|
AND run.status IN ('queued', 'running', 'waiting')
|
|
56
71
|
WHERE task.state = 'cancelled'
|
|
57
72
|
AND task.task_name = $1
|
|
58
|
-
AND task.cancellation ? 'max_delay'
|
|
59
73
|
AND task.cancel_at IS NOT NULL
|
|
60
|
-
AND
|
|
74
|
+
AND (
|
|
75
|
+
task.cancellation ? 'manual'
|
|
76
|
+
OR (
|
|
77
|
+
task.cancellation ? 'max_delay'
|
|
78
|
+
AND task.cancelled_at >= task.cancel_at
|
|
79
|
+
)
|
|
80
|
+
)
|
|
61
81
|
ORDER BY task.cancelled_at ASC NULLS LAST, task.task_id ASC
|
|
62
82
|
LIMIT $2
|
|
63
83
|
`,
|
|
@@ -72,17 +92,51 @@ export async function reconcileAbsurdRuntimeMaxDelayCancellations(input: {
|
|
|
72
92
|
skipped += 1;
|
|
73
93
|
continue;
|
|
74
94
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
95
|
+
await input.client.query('BEGIN');
|
|
96
|
+
try {
|
|
97
|
+
await input.client.query(
|
|
98
|
+
`SELECT set_config('lock_timeout', $1, true),
|
|
99
|
+
set_config('statement_timeout', $2, true)`,
|
|
100
|
+
[
|
|
101
|
+
`${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
|
|
102
|
+
`${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
|
|
103
|
+
],
|
|
104
|
+
);
|
|
105
|
+
const didCancel = await cancelPostgresSchedulerRun(
|
|
106
|
+
input.client,
|
|
107
|
+
{
|
|
108
|
+
runId,
|
|
109
|
+
reason:
|
|
110
|
+
row.cancellation_kind === 'manual'
|
|
111
|
+
? 'Run cancellation was requested by the caller.'
|
|
112
|
+
: 'Run did not start within its configured scheduler queue deadline.',
|
|
113
|
+
},
|
|
114
|
+
input.schedulerOptions,
|
|
115
|
+
);
|
|
116
|
+
await input.client.query('COMMIT');
|
|
117
|
+
if (didCancel) reconciled += 1;
|
|
118
|
+
else skipped += 1;
|
|
119
|
+
} catch (error) {
|
|
120
|
+
await input.client.query('ROLLBACK').catch(() => undefined);
|
|
121
|
+
// A worker-held lock is expected to be retried by the next lane sweep;
|
|
122
|
+
// never let one blocked cancellation stall the claim loop or create a
|
|
123
|
+
// second unbounded cancellation pile.
|
|
124
|
+
if (
|
|
125
|
+
isRuntimeSchedulerQueryCancellationError(error) ||
|
|
126
|
+
isRuntimeSchedulerRetryableTransactionError(error)
|
|
127
|
+
) {
|
|
128
|
+
skipped += 1;
|
|
129
|
+
// The hardened pool destroys a client after a statement timeout or a
|
|
130
|
+
// retryable transaction error. Do not issue the next candidate on that
|
|
131
|
+
// same socket; the next worker sweep acquires a fresh client.
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
throw error;
|
|
135
|
+
}
|
|
86
136
|
}
|
|
87
137
|
return { reconciled, skipped };
|
|
88
138
|
}
|
|
139
|
+
|
|
140
|
+
/** Compatibility export for callers/tests that still name the old deadline-only lane. */
|
|
141
|
+
export const reconcileAbsurdRuntimeMaxDelayCancellations =
|
|
142
|
+
reconcileAbsurdRuntimeCancellations;
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts
CHANGED
|
@@ -262,7 +262,7 @@ import { runtimeApiBaseUrlFromLaunch } from './runtime-api-origin';
|
|
|
262
262
|
import { shouldTerminalizeSandboxCapacityRejection } from '../launch-source-policy';
|
|
263
263
|
import {
|
|
264
264
|
ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
|
|
265
|
-
|
|
265
|
+
reconcileAbsurdRuntimeCancellations,
|
|
266
266
|
} from './absurd-deadline-reconciliation';
|
|
267
267
|
import {
|
|
268
268
|
abandonRuntimeSandboxOperationReservation,
|
|
@@ -3254,8 +3254,7 @@ async function executeAssignedAbsurdAttempt(input: {
|
|
|
3254
3254
|
};
|
|
3255
3255
|
await enforceRuntimeTrafficPolicyBeforeSandboxStart({
|
|
3256
3256
|
policy: trafficPolicy,
|
|
3257
|
-
defaultSandboxStartsPerMinute:
|
|
3258
|
-
defaultDaytonaStartRatePerMinute,
|
|
3257
|
+
defaultSandboxStartsPerMinute: defaultDaytonaStartRatePerMinute,
|
|
3259
3258
|
nowMs: Date.now(),
|
|
3260
3259
|
acquireStartPermit: async (sandboxStartsPerMinute) =>
|
|
3261
3260
|
await deps.runWithClient((client) =>
|
|
@@ -3273,8 +3272,7 @@ async function executeAssignedAbsurdAttempt(input: {
|
|
|
3273
3272
|
maxConcurrency: null,
|
|
3274
3273
|
adaptiveMaxRps:
|
|
3275
3274
|
(daytonaStartRateConfig?.maximumRatePerMinute ??
|
|
3276
|
-
sandboxStartsPerMinute) /
|
|
3277
|
-
60,
|
|
3275
|
+
sandboxStartsPerMinute) / 60,
|
|
3278
3276
|
},
|
|
3279
3277
|
],
|
|
3280
3278
|
// The permit itself is the successful admission
|
|
@@ -3632,10 +3630,7 @@ async function executeAssignedAbsurdAttempt(input: {
|
|
|
3632
3630
|
await cleanupAfterExecutionError(error, runnerConfig);
|
|
3633
3631
|
const recoveryError =
|
|
3634
3632
|
recoveryForDaytonaSandboxAcquisitionUnavailable(error) ?? error;
|
|
3635
|
-
if (
|
|
3636
|
-
recoveryError !== error &&
|
|
3637
|
-
daytonaStartRateConfig !== null
|
|
3638
|
-
) {
|
|
3633
|
+
if (recoveryError !== error && daytonaStartRateConfig !== null) {
|
|
3639
3634
|
await deps.runWithClient((client) =>
|
|
3640
3635
|
penalizePostgresRateState(client, {
|
|
3641
3636
|
bucketId: 'runtime:traffic:sandbox-start',
|
|
@@ -4003,9 +3998,7 @@ export async function enforceRuntimeTrafficPolicyBeforeSandboxStart(input: {
|
|
|
4003
3998
|
: Math.max(
|
|
4004
3999
|
0,
|
|
4005
4000
|
permit.waitMs,
|
|
4006
|
-
scheduledAtMs > input.nowMs
|
|
4007
|
-
? scheduledAtMs - schedulerNowMs
|
|
4008
|
-
: 0,
|
|
4001
|
+
scheduledAtMs > input.nowMs ? scheduledAtMs - schedulerNowMs : 0,
|
|
4009
4002
|
);
|
|
4010
4003
|
// A sub-second schedule retains smooth pacing without spending a worker
|
|
4011
4004
|
// attempt. Longer waits become a durable, bounded defer, allowing the
|
|
@@ -6954,28 +6947,27 @@ export async function startRuntimeWorkerHost(
|
|
|
6954
6947
|
schedulerOptions.schema,
|
|
6955
6948
|
),
|
|
6956
6949
|
});
|
|
6957
|
-
const
|
|
6950
|
+
const reconcileRuntimeCancellations = async () => {
|
|
6958
6951
|
await withRuntimeSchedulerClient(poolOptions, async (client) => {
|
|
6959
|
-
const
|
|
6960
|
-
|
|
6961
|
-
|
|
6962
|
-
|
|
6963
|
-
|
|
6964
|
-
|
|
6965
|
-
|
|
6966
|
-
console.info('[absurd-runtime.worker] max_delay_reconciled', {
|
|
6952
|
+
const cancellations = await reconcileAbsurdRuntimeCancellations({
|
|
6953
|
+
client,
|
|
6954
|
+
queue,
|
|
6955
|
+
schedulerOptions,
|
|
6956
|
+
});
|
|
6957
|
+
if (cancellations.reconciled > 0) {
|
|
6958
|
+
console.info('[absurd-runtime.worker] cancellations_reconciled', {
|
|
6967
6959
|
queue,
|
|
6968
6960
|
releaseId,
|
|
6969
|
-
...
|
|
6961
|
+
...cancellations,
|
|
6970
6962
|
});
|
|
6971
6963
|
}
|
|
6972
6964
|
});
|
|
6973
6965
|
};
|
|
6974
|
-
const
|
|
6966
|
+
const sweepAndReconcileCancellations = async () => {
|
|
6975
6967
|
await absurdClient.sweepCancellations(
|
|
6976
6968
|
ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
|
|
6977
6969
|
);
|
|
6978
|
-
await
|
|
6970
|
+
await reconcileRuntimeCancellations();
|
|
6979
6971
|
};
|
|
6980
6972
|
const reportQueueError = createRateLimitedRuntimeLogger({
|
|
6981
6973
|
log: (...args) => console.error(...args),
|
|
@@ -6987,12 +6979,13 @@ export async function startRuntimeWorkerHost(
|
|
|
6987
6979
|
pollInterval: ABSURD_IDLE_POLL_INTERVAL_SECONDS,
|
|
6988
6980
|
// `claim_task` performs the indexed max_delay sweep. Reconcile after every
|
|
6989
6981
|
// sweep, including claims that return backlog work, so a saturated queue
|
|
6990
|
-
// cannot postpone
|
|
6991
|
-
afterClaim:
|
|
6982
|
+
// cannot postpone a cancellation's scheduler/outbox terminal indefinitely.
|
|
6983
|
+
afterClaim: reconcileRuntimeCancellations,
|
|
6992
6984
|
// claim_task cannot run while every handler slot is occupied. Keep the
|
|
6993
|
-
// enqueue deadline independent from handler duration
|
|
6994
|
-
// claiming, then project cancellation into the
|
|
6995
|
-
|
|
6985
|
+
// enqueue deadline and cancellation lane independent from handler duration
|
|
6986
|
+
// by sweeping without claiming, then project cancellation into the
|
|
6987
|
+
// scheduler/outbox.
|
|
6988
|
+
saturatedMaintenance: sweepAndReconcileCancellations,
|
|
6996
6989
|
nextPollDelayMs: () =>
|
|
6997
6990
|
withRuntimeSchedulerClient(poolOptions, async (client) => {
|
|
6998
6991
|
const nextDue = await readAbsurdQueueNextDue(client, queue);
|
|
@@ -14,7 +14,8 @@
|
|
|
14
14
|
* Persist the absurd task id
|
|
15
15
|
* on the run row so a later cancel can target it.
|
|
16
16
|
* observe/result → reuse the postgres LISTEN/NOTIFY run handle.
|
|
17
|
-
* cancel →
|
|
17
|
+
* cancel → durable Absurd cancellation lane + bounded postgres terminal
|
|
18
|
+
* projection (with worker reconciliation when locks are busy).
|
|
18
19
|
* signal → durable signal row (authoritative payload) + absurd emitEvent
|
|
19
20
|
* (the wakeup only; payload is re-read from the signal row).
|
|
20
21
|
*/
|
|
@@ -30,6 +31,7 @@ import {
|
|
|
30
31
|
} from '../runner-backends/runtime-sandbox-reconciliation';
|
|
31
32
|
import {
|
|
32
33
|
PLAY_SCHEDULER_BACKENDS,
|
|
34
|
+
RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE,
|
|
33
35
|
type PlaySchedulerBackend,
|
|
34
36
|
type PlaySchedulerRunHandle,
|
|
35
37
|
type PlaySchedulerSubmitInput,
|
|
@@ -45,7 +47,12 @@ import {
|
|
|
45
47
|
createPlayRunAdmissionJob,
|
|
46
48
|
createPostgresPlayRunAdmissionPorts,
|
|
47
49
|
} from '../play-runs/admission';
|
|
48
|
-
import {
|
|
50
|
+
import {
|
|
51
|
+
RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
|
|
52
|
+
RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
|
|
53
|
+
withRuntimeSchedulerClient,
|
|
54
|
+
withRuntimeSchedulerTransaction,
|
|
55
|
+
} from './postgres-pool';
|
|
49
56
|
import { withRuntimeSchedulerSubmitAdmission } from './submit-admission';
|
|
50
57
|
import {
|
|
51
58
|
ABSURD_PLAY_RUN_MAX_ATTEMPTS,
|
|
@@ -77,6 +84,9 @@ export type AbsurdSchedulerBackendOptions = AbsurdSchedulerOptions & {
|
|
|
77
84
|
|
|
78
85
|
const cancellationFlights = new Map<string, Promise<void>>();
|
|
79
86
|
|
|
87
|
+
function quotePostgresIdentifier(identifier: string): string {
|
|
88
|
+
return `"${identifier.replaceAll('"', '""')}"`;
|
|
89
|
+
}
|
|
80
90
|
function runCancellationSingleFlight(
|
|
81
91
|
key: string,
|
|
82
92
|
operation: () => Promise<void>,
|
|
@@ -157,8 +167,79 @@ function absurdRunHandle(input: {
|
|
|
157
167
|
readPostgresSchedulerAbsurdTaskId(client, input.runId, effectiveOptions),
|
|
158
168
|
);
|
|
159
169
|
if (!taskId) return;
|
|
160
|
-
|
|
161
|
-
|
|
170
|
+
// Keep cancellation on the same bounded control-plane contract as the
|
|
171
|
+
// scheduler terminal write. This is a durable lane transition, not an
|
|
172
|
+
// unbounded best-effort query which can pile up behind a worker lock.
|
|
173
|
+
await withRuntimeSchedulerTransaction(
|
|
174
|
+
poolOptions,
|
|
175
|
+
{
|
|
176
|
+
lockTimeoutMs: RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
|
|
177
|
+
statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
|
|
178
|
+
},
|
|
179
|
+
async (client) => {
|
|
180
|
+
await client.query(`SELECT absurd.cancel_task($1, $2::uuid)`, [
|
|
181
|
+
runsQueue,
|
|
182
|
+
taskId,
|
|
183
|
+
]);
|
|
184
|
+
// Older v4 Absurd functions do not materialize a manual cancellation
|
|
185
|
+
// deadline or reason. Stamp both in the adapter so existing queues
|
|
186
|
+
// participate in the same durable reconciliation lane without a
|
|
187
|
+
// schema cutover. The explicit marker keeps a user cancel distinct
|
|
188
|
+
// from a task cancelled by its configured max_delay sweep.
|
|
189
|
+
await client.query(
|
|
190
|
+
`UPDATE absurd.${quotePostgresIdentifier(`t_${runsQueue}`)}
|
|
191
|
+
SET cancel_at = COALESCE(cancel_at, clock_timestamp()),
|
|
192
|
+
cancellation = COALESCE(cancellation, '{}'::jsonb)
|
|
193
|
+
|| jsonb_build_object('manual', true)
|
|
194
|
+
WHERE task_id = $1::uuid
|
|
195
|
+
AND state = 'cancelled'`,
|
|
196
|
+
[taskId],
|
|
197
|
+
);
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
runId: input.runId,
|
|
201
|
+
phase: 'cancel_engine_task',
|
|
202
|
+
priority: 'critical',
|
|
203
|
+
},
|
|
204
|
+
{ maxAttempts: 1 },
|
|
205
|
+
);
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
const cancellationPendingError = (cause: unknown): Error => {
|
|
209
|
+
const error = new Error(
|
|
210
|
+
`Run cancellation was durably requested, but scheduler terminalization is pending for ${input.runId}.`,
|
|
211
|
+
);
|
|
212
|
+
Object.assign(error, {
|
|
213
|
+
code: RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE,
|
|
214
|
+
cause,
|
|
215
|
+
});
|
|
216
|
+
return error;
|
|
217
|
+
};
|
|
218
|
+
|
|
219
|
+
const requestCancellation = async () => {
|
|
220
|
+
await runCancellationSingleFlight(cancellationKey, async () => {
|
|
221
|
+
// First move the durable engine task into its cancellation lane. If the
|
|
222
|
+
// scheduler tables are briefly locked, the worker reconciler can finish
|
|
223
|
+
// the terminal write later without another user request piling up.
|
|
224
|
+
await cancelAbsurdTask();
|
|
225
|
+
try {
|
|
226
|
+
await delegate.cancel();
|
|
227
|
+
} catch (error) {
|
|
228
|
+
throw cancellationPendingError(error);
|
|
229
|
+
}
|
|
230
|
+
try {
|
|
231
|
+
await cleanupPersistedRuntimeSandboxes();
|
|
232
|
+
} catch (error) {
|
|
233
|
+
// The scheduler terminal transaction already won. Its cleanup jobs are
|
|
234
|
+
// durable, so a best-effort eager cleanup failure must not turn a real
|
|
235
|
+
// cancellation into a false 502 or cause the caller to retry the
|
|
236
|
+
// terminal write.
|
|
237
|
+
console.warn('[absurd.runtime_sandbox_cleanup_deferred]', {
|
|
238
|
+
runId: input.runId,
|
|
239
|
+
error: error instanceof Error ? error.message : String(error),
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
});
|
|
162
243
|
};
|
|
163
244
|
|
|
164
245
|
const cleanupPersistedRuntimeSandboxes = async () => {
|
|
@@ -247,20 +328,10 @@ function absurdRunHandle(input: {
|
|
|
247
328
|
...delegate,
|
|
248
329
|
runId: input.runId,
|
|
249
330
|
initialState: input.initialState ?? undefined,
|
|
250
|
-
cancel:
|
|
251
|
-
await runCancellationSingleFlight(cancellationKey, async () => {
|
|
252
|
-
await delegate.cancel();
|
|
253
|
-
await cancelAbsurdTask();
|
|
254
|
-
await cleanupPersistedRuntimeSandboxes();
|
|
255
|
-
});
|
|
256
|
-
},
|
|
331
|
+
cancel: requestCancellation,
|
|
257
332
|
signal: async (payload) => {
|
|
258
333
|
if (payload.kind === 'cancel') {
|
|
259
|
-
await
|
|
260
|
-
await delegate.signal(payload);
|
|
261
|
-
await cancelAbsurdTask();
|
|
262
|
-
await cleanupPersistedRuntimeSandboxes();
|
|
263
|
-
});
|
|
334
|
+
await requestCancellation();
|
|
264
335
|
return;
|
|
265
336
|
}
|
|
266
337
|
if (payload.kind !== 'integration_event' || !payload.eventKey) {
|
|
@@ -127,7 +127,7 @@ export function assertRuntimeSchedulerHotPathUrl(url: string): void {
|
|
|
127
127
|
}
|
|
128
128
|
|
|
129
129
|
export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
|
|
130
|
-
/** One shared per-process client cap for scheduler control-plane
|
|
130
|
+
/** One shared per-process client cap for every scheduler control-plane lane. */
|
|
131
131
|
maxClients: RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients,
|
|
132
132
|
/** Refuse to construct a pool above this — protects the shared PgBouncer/Postgres. */
|
|
133
133
|
maxClientsHardCap: 32,
|
|
@@ -167,6 +167,11 @@ export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
|
|
|
167
167
|
export const RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS =
|
|
168
168
|
RUNTIME_CAPACITY_POLICY.runtimePostgres.schedulerPoolMaxClients;
|
|
169
169
|
|
|
170
|
+
/**
|
|
171
|
+
* Cancellation is control-plane work, not customer execution. A blocked
|
|
172
|
+
* cancellation must give up quickly instead of sitting behind a worker-held
|
|
173
|
+
* row lock and spawning more identical cancellation statements on retry.
|
|
174
|
+
*/
|
|
170
175
|
export const RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS = 2_000;
|
|
171
176
|
export const RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS = 15_000;
|
|
172
177
|
|
|
@@ -195,6 +200,7 @@ export type RuntimeSchedulerPoolOptions = {
|
|
|
195
200
|
|
|
196
201
|
export type RuntimeSchedulerOperationOptions = {
|
|
197
202
|
signal?: AbortSignal;
|
|
203
|
+
/** Override only for an operation with its own retry/lock contract. */
|
|
198
204
|
maxAttempts?: number;
|
|
199
205
|
};
|
|
200
206
|
|
|
@@ -209,7 +215,8 @@ export function isRuntimeSchedulerQueryCancellationError(
|
|
|
209
215
|
error: unknown,
|
|
210
216
|
): boolean {
|
|
211
217
|
if (!error || typeof error !== 'object') return false;
|
|
212
|
-
|
|
218
|
+
const code = (error as { code?: unknown }).code;
|
|
219
|
+
if (code === '57014') return true;
|
|
213
220
|
const name = (error as { name?: unknown }).name;
|
|
214
221
|
if (name === 'TimeoutError' || name === 'AbortError') return true;
|
|
215
222
|
const message = (error as { message?: unknown }).message;
|
|
@@ -883,6 +890,10 @@ export async function withRuntimeSchedulerClient<T>(
|
|
|
883
890
|
throwIfSchedulerAborted(signal);
|
|
884
891
|
return { rows: queryResult.rows };
|
|
885
892
|
} catch (error) {
|
|
893
|
+
// node-postgres' query timeout and an AbortSignal reject the
|
|
894
|
+
// promise, but the server may still be processing the SQL.
|
|
895
|
+
// Never return that socket to the pool: doing so lets the
|
|
896
|
+
// next retry overlap the first statement on the same backend.
|
|
886
897
|
if (isRuntimeSchedulerQueryCancellationError(error)) {
|
|
887
898
|
releasePoolClient(
|
|
888
899
|
error instanceof Error ? error : new Error(String(error)),
|
|
@@ -1013,9 +1024,18 @@ export async function withRuntimeSchedulerClient<T>(
|
|
|
1013
1024
|
}
|
|
1014
1025
|
}
|
|
1015
1026
|
|
|
1027
|
+
/**
|
|
1028
|
+
* Run one bounded scheduler transaction on a checked-out pooled client.
|
|
1029
|
+
* `SET LOCAL` is deliberately issued inside the transaction because the
|
|
1030
|
+
* runtime hot path uses PgBouncer transaction pooling and must not leak
|
|
1031
|
+
* session settings between callers.
|
|
1032
|
+
*/
|
|
1016
1033
|
export async function withRuntimeSchedulerTransaction<T>(
|
|
1017
1034
|
options: RuntimeSchedulerPoolOptions,
|
|
1018
|
-
timeouts: {
|
|
1035
|
+
timeouts: {
|
|
1036
|
+
lockTimeoutMs: number;
|
|
1037
|
+
statementTimeoutMs: number;
|
|
1038
|
+
},
|
|
1019
1039
|
fn: (client: RuntimeSchedulerQueryClient) => Promise<T>,
|
|
1020
1040
|
trace?: {
|
|
1021
1041
|
runId: string;
|
|
@@ -1050,7 +1070,14 @@ export async function withRuntimeSchedulerTransaction<T>(
|
|
|
1050
1070
|
}
|
|
1051
1071
|
},
|
|
1052
1072
|
trace,
|
|
1053
|
-
{
|
|
1073
|
+
{
|
|
1074
|
+
...operationOptions,
|
|
1075
|
+
// A cancellation is a user/control-plane decision. If its lock cannot
|
|
1076
|
+
// be acquired inside the bounded window, the caller must report the
|
|
1077
|
+
// active run honestly; retrying here would recreate the incident's
|
|
1078
|
+
// cancellation pile.
|
|
1079
|
+
maxAttempts: operationOptions.maxAttempts ?? 1,
|
|
1080
|
+
},
|
|
1054
1081
|
);
|
|
1055
1082
|
}
|
|
1056
1083
|
|
|
@@ -16984,7 +16984,11 @@ export function postgresSchedulerRunHandle(
|
|
|
16984
16984
|
statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
|
|
16985
16985
|
},
|
|
16986
16986
|
(client) => cancelPostgresSchedulerRun(client, { runId }, options),
|
|
16987
|
-
{
|
|
16987
|
+
{
|
|
16988
|
+
runId,
|
|
16989
|
+
phase: 'cancel_run',
|
|
16990
|
+
priority: 'critical',
|
|
16991
|
+
},
|
|
16988
16992
|
);
|
|
16989
16993
|
},
|
|
16990
16994
|
signal: async (payload) => {
|
|
@@ -16997,7 +17001,11 @@ export function postgresSchedulerRunHandle(
|
|
|
16997
17001
|
statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
|
|
16998
17002
|
},
|
|
16999
17003
|
(client) => cancelPostgresSchedulerRun(client, { runId }, options),
|
|
17000
|
-
{
|
|
17004
|
+
{
|
|
17005
|
+
runId,
|
|
17006
|
+
phase: 'cancel_run',
|
|
17007
|
+
priority: 'critical',
|
|
17008
|
+
},
|
|
17001
17009
|
);
|
|
17002
17010
|
return;
|
|
17003
17011
|
}
|
package/dist/cli/index.js
CHANGED
|
@@ -3068,7 +3068,7 @@ var SDK_RELEASE = {
|
|
|
3068
3068
|
// getters keep their established compatibility behavior.
|
|
3069
3069
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3070
3070
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3071
|
-
version: "0.3.
|
|
3071
|
+
version: "0.3.131",
|
|
3072
3072
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3073
3073
|
packageCapabilities: {
|
|
3074
3074
|
updatePreferences: 1
|
package/dist/cli/index.mjs
CHANGED
|
@@ -3063,7 +3063,7 @@ var SDK_RELEASE = {
|
|
|
3063
3063
|
// getters keep their established compatibility behavior.
|
|
3064
3064
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3065
3065
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3066
|
-
version: "0.3.
|
|
3066
|
+
version: "0.3.131",
|
|
3067
3067
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3068
3068
|
packageCapabilities: {
|
|
3069
3069
|
updatePreferences: 1
|
package/dist/index.d.mts
CHANGED
|
@@ -1891,6 +1891,8 @@ interface StopPlayRunResult {
|
|
|
1891
1891
|
runId: string;
|
|
1892
1892
|
/** Whether the server confirmed the run was stopped. */
|
|
1893
1893
|
stopped: boolean;
|
|
1894
|
+
/** True when the durable cancellation lane accepted the request and is draining. */
|
|
1895
|
+
cancelling?: boolean;
|
|
1894
1896
|
/** Number of open HITL interactions marked cancelled. */
|
|
1895
1897
|
hitlCancelledCount: number;
|
|
1896
1898
|
/**
|
|
@@ -1908,6 +1910,7 @@ interface StopPlayRunResult {
|
|
|
1908
1910
|
*/
|
|
1909
1911
|
interface StopAllPlayRunsStopResult {
|
|
1910
1912
|
stopped: number;
|
|
1913
|
+
cancelling?: number;
|
|
1911
1914
|
failed: number;
|
|
1912
1915
|
skipped: number;
|
|
1913
1916
|
/** True when the bounded candidate query may have left active runs behind. */
|
package/dist/index.d.ts
CHANGED
|
@@ -1891,6 +1891,8 @@ interface StopPlayRunResult {
|
|
|
1891
1891
|
runId: string;
|
|
1892
1892
|
/** Whether the server confirmed the run was stopped. */
|
|
1893
1893
|
stopped: boolean;
|
|
1894
|
+
/** True when the durable cancellation lane accepted the request and is draining. */
|
|
1895
|
+
cancelling?: boolean;
|
|
1894
1896
|
/** Number of open HITL interactions marked cancelled. */
|
|
1895
1897
|
hitlCancelledCount: number;
|
|
1896
1898
|
/**
|
|
@@ -1908,6 +1910,7 @@ interface StopPlayRunResult {
|
|
|
1908
1910
|
*/
|
|
1909
1911
|
interface StopAllPlayRunsStopResult {
|
|
1910
1912
|
stopped: number;
|
|
1913
|
+
cancelling?: number;
|
|
1911
1914
|
failed: number;
|
|
1912
1915
|
skipped: number;
|
|
1913
1916
|
/** True when the bounded candidate query may have left active runs behind. */
|
package/dist/index.js
CHANGED
|
@@ -864,7 +864,7 @@ var SDK_RELEASE = {
|
|
|
864
864
|
// getters keep their established compatibility behavior.
|
|
865
865
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
866
866
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
867
|
-
version: "0.3.
|
|
867
|
+
version: "0.3.131",
|
|
868
868
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
869
869
|
packageCapabilities: {
|
|
870
870
|
updatePreferences: 1
|
package/dist/index.mjs
CHANGED
|
@@ -768,7 +768,7 @@ var SDK_RELEASE = {
|
|
|
768
768
|
// getters keep their established compatibility behavior.
|
|
769
769
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
770
770
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
771
|
-
version: "0.3.
|
|
771
|
+
version: "0.3.131",
|
|
772
772
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
773
773
|
packageCapabilities: {
|
|
774
774
|
updatePreferences: 1
|
package/dist/release.d.mts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.131";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.d.ts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.131";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.js
CHANGED
|
@@ -74,7 +74,7 @@ var SDK_RELEASE = {
|
|
|
74
74
|
// getters keep their established compatibility behavior.
|
|
75
75
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
76
76
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
77
|
-
version: "0.3.
|
|
77
|
+
version: "0.3.131",
|
|
78
78
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
79
79
|
packageCapabilities: {
|
|
80
80
|
updatePreferences: 1
|
package/dist/release.mjs
CHANGED
|
@@ -48,7 +48,7 @@ var SDK_RELEASE = {
|
|
|
48
48
|
// getters keep their established compatibility behavior.
|
|
49
49
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
50
50
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
51
|
-
version: "0.3.
|
|
51
|
+
version: "0.3.131",
|
|
52
52
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
53
53
|
packageCapabilities: {
|
|
54
54
|
updatePreferences: 1
|