deepline 0.3.200 → 0.3.201
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/types.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts +160 -58
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +111 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +51 -46
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +168 -8
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +133 -5
- package/dist/cli/index.js +22 -1
- package/dist/cli/index.mjs +22 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.201',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -1045,8 +1045,8 @@ export interface StopPlayRunResult {
|
|
|
1045
1045
|
|
|
1046
1046
|
/**
|
|
1047
1047
|
* Result of an actual workspace stop-all. `stopped` counts only runs whose
|
|
1048
|
-
* scheduler cancellation was confirmed;
|
|
1049
|
-
*
|
|
1048
|
+
* scheduler cancellation was confirmed; queued requests are counted in
|
|
1049
|
+
* `cancelling`, while requests that could not be queued count as `failed`.
|
|
1050
1050
|
*/
|
|
1051
1051
|
export interface StopAllPlayRunsStopResult {
|
|
1052
1052
|
stopped: number;
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
abandonUnstartedModalCancellationReservations,
|
|
3
|
+
cancelPostgresSchedulerRun,
|
|
4
|
+
enqueuePostgresSchedulerSandboxCleanupForRun,
|
|
5
|
+
hasPendingPostgresSchedulerSandboxCleanupForRun,
|
|
6
|
+
readPostgresSchedulerResult,
|
|
7
|
+
} from './postgres';
|
|
2
8
|
import type {
|
|
3
9
|
PostgresSchedulerOptions,
|
|
4
10
|
PostgresSchedulerQueryClient,
|
|
@@ -14,31 +20,100 @@ import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
|
|
|
14
20
|
|
|
15
21
|
/** Bound deadline reconciliation just like Absurd's own cancellation sweep. */
|
|
16
22
|
export const ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT = 100;
|
|
23
|
+
const QUEUED_CANCELLATION_RECONCILE_LIMIT = 20;
|
|
17
24
|
|
|
18
25
|
function quoteAbsurdTable(table: string): string {
|
|
19
26
|
return `"${table.replaceAll('"', '""')}"`;
|
|
20
27
|
}
|
|
21
28
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
29
|
+
async function finalizeCancelledRunAfterCleanup(input: {
|
|
30
|
+
client: PostgresSchedulerQueryClient;
|
|
31
|
+
runId: string;
|
|
32
|
+
reason: string;
|
|
33
|
+
schedulerSchema: string;
|
|
34
|
+
schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
|
|
35
|
+
controlKey: string;
|
|
36
|
+
}): Promise<'pending' | 'cancelled' | 'already_terminal'> {
|
|
37
|
+
await input.client.query('BEGIN');
|
|
38
|
+
try {
|
|
39
|
+
await input.client.query(
|
|
40
|
+
`SELECT set_config('lock_timeout', $1, true),
|
|
41
|
+
set_config('statement_timeout', $2, true)`,
|
|
42
|
+
[
|
|
43
|
+
`${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
|
|
44
|
+
`${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
|
|
45
|
+
],
|
|
46
|
+
);
|
|
47
|
+
// Sandbox admission locks the run before reserving an operation; resource
|
|
48
|
+
// writers lock attempts. Hold both through the final cleanup check.
|
|
49
|
+
await input.client.query(
|
|
50
|
+
`SELECT run_id FROM ${input.schedulerSchema}.runs
|
|
51
|
+
WHERE run_id = $1 FOR UPDATE`,
|
|
52
|
+
[input.runId],
|
|
53
|
+
);
|
|
54
|
+
await input.client.query(
|
|
55
|
+
`SELECT attempt FROM ${input.schedulerSchema}.attempts
|
|
56
|
+
WHERE run_id = $1 FOR UPDATE`,
|
|
57
|
+
[input.runId],
|
|
58
|
+
);
|
|
59
|
+
await abandonUnstartedModalCancellationReservations(
|
|
60
|
+
input.client,
|
|
61
|
+
{ runId: input.runId },
|
|
62
|
+
input.schedulerOptions,
|
|
63
|
+
);
|
|
64
|
+
await enqueuePostgresSchedulerSandboxCleanupForRun(
|
|
65
|
+
input.client,
|
|
66
|
+
{ runId: input.runId },
|
|
67
|
+
input.schedulerOptions,
|
|
68
|
+
);
|
|
69
|
+
if (
|
|
70
|
+
await hasPendingPostgresSchedulerSandboxCleanupForRun(
|
|
71
|
+
input.client,
|
|
72
|
+
{ runId: input.runId },
|
|
73
|
+
input.schedulerOptions,
|
|
74
|
+
)
|
|
75
|
+
) {
|
|
76
|
+
await input.client.query('COMMIT');
|
|
77
|
+
return 'pending';
|
|
78
|
+
}
|
|
79
|
+
const cancelled = await cancelPostgresSchedulerRun(
|
|
80
|
+
input.client,
|
|
81
|
+
{ runId: input.runId, reason: input.reason },
|
|
82
|
+
input.schedulerOptions,
|
|
83
|
+
);
|
|
84
|
+
if (
|
|
85
|
+
!cancelled &&
|
|
86
|
+
!(await readPostgresSchedulerResult(
|
|
87
|
+
input.client,
|
|
88
|
+
input.runId,
|
|
89
|
+
input.schedulerOptions,
|
|
90
|
+
))
|
|
91
|
+
) {
|
|
92
|
+
throw new Error(
|
|
93
|
+
`Cancellation did not terminalize scheduler run "${input.runId}".`,
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
await input.client.query(
|
|
97
|
+
`DELETE FROM ${input.schedulerSchema}.control WHERE key = $1`,
|
|
98
|
+
[input.controlKey],
|
|
99
|
+
);
|
|
100
|
+
await input.client.query('COMMIT');
|
|
101
|
+
return cancelled ? 'cancelled' : 'already_terminal';
|
|
102
|
+
} catch (error) {
|
|
103
|
+
await input.client.query('ROLLBACK').catch(() => undefined);
|
|
104
|
+
throw error;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Put legacy manual and deadline cancellations into the same durable queue
|
|
109
|
+
* as caller requests. A queued task is excluded from later scans, so one
|
|
110
|
+
* pending sandbox cannot hide other cancelled tasks behind the batch limit. */
|
|
36
111
|
export async function reconcileAbsurdRuntimeCancellations(input: {
|
|
37
112
|
client: PostgresSchedulerQueryClient;
|
|
38
113
|
queue: string;
|
|
39
114
|
schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
|
|
40
115
|
limit?: number;
|
|
41
|
-
}): Promise<{
|
|
116
|
+
}): Promise<{ queued: number }> {
|
|
42
117
|
const requestedLimit = Math.floor(
|
|
43
118
|
input.limit ?? ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
|
|
44
119
|
);
|
|
@@ -54,16 +129,21 @@ export async function reconcileAbsurdRuntimeCancellations(input: {
|
|
|
54
129
|
resolveRuntimeSchedulerSchemaEnv() ||
|
|
55
130
|
'runtime_scheduler',
|
|
56
131
|
);
|
|
57
|
-
const
|
|
58
|
-
run_id: string | null;
|
|
59
|
-
cancellation_kind: 'manual' | 'deadline';
|
|
60
|
-
}>(
|
|
132
|
+
const queued = await input.client.query<{ key: string }>(
|
|
61
133
|
`
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
134
|
+
INSERT INTO ${schedulerSchema}.control (key, value)
|
|
135
|
+
SELECT 'runtime_cancel:' || run.run_id,
|
|
136
|
+
jsonb_build_object(
|
|
137
|
+
'reason', CASE
|
|
138
|
+
WHEN task.cancellation ? 'manual'
|
|
139
|
+
THEN COALESCE(
|
|
140
|
+
NULLIF(task.cancellation->>'reason', ''),
|
|
141
|
+
'Run cancellation was requested by the caller.'
|
|
142
|
+
)
|
|
143
|
+
ELSE 'Run did not start within its configured scheduler queue deadline.'
|
|
144
|
+
END,
|
|
145
|
+
'requestedAt', clock_timestamp()
|
|
146
|
+
)
|
|
67
147
|
FROM ${schedulerSchema}.runs AS run
|
|
68
148
|
JOIN absurd.${taskTable} AS task
|
|
69
149
|
ON task.task_id::text = run.absurd_task_id
|
|
@@ -78,57 +158,79 @@ export async function reconcileAbsurdRuntimeCancellations(input: {
|
|
|
78
158
|
AND task.cancelled_at >= task.cancel_at
|
|
79
159
|
)
|
|
80
160
|
)
|
|
161
|
+
AND NOT EXISTS (
|
|
162
|
+
SELECT 1 FROM ${schedulerSchema}.control AS pending
|
|
163
|
+
WHERE pending.key = 'runtime_cancel:' || run.run_id
|
|
164
|
+
)
|
|
81
165
|
ORDER BY task.cancelled_at ASC NULLS LAST, task.task_id ASC
|
|
82
166
|
LIMIT $2
|
|
167
|
+
ON CONFLICT (key) DO NOTHING
|
|
168
|
+
RETURNING key
|
|
83
169
|
`,
|
|
84
170
|
[ABSURD_PLAY_RUN_TASK, limit],
|
|
85
171
|
);
|
|
172
|
+
return { queued: queued.rows.length };
|
|
173
|
+
}
|
|
86
174
|
|
|
175
|
+
/** Drain caller-requested cancellations independently of their release lane. */
|
|
176
|
+
export async function reconcileQueuedAbsurdRuntimeCancellations(input: {
|
|
177
|
+
client: PostgresSchedulerQueryClient;
|
|
178
|
+
schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
|
|
179
|
+
limit?: number;
|
|
180
|
+
}): Promise<{ reconciled: number; skipped: number }> {
|
|
181
|
+
const schedulerSchema = quoteAbsurdTable(
|
|
182
|
+
input.schedulerOptions?.schema?.trim() ||
|
|
183
|
+
resolveRuntimeSchedulerSchemaEnv() ||
|
|
184
|
+
'runtime_scheduler',
|
|
185
|
+
);
|
|
186
|
+
const requestedLimit = Math.floor(
|
|
187
|
+
input.limit ?? QUEUED_CANCELLATION_RECONCILE_LIMIT,
|
|
188
|
+
);
|
|
189
|
+
const limit = Number.isFinite(requestedLimit)
|
|
190
|
+
? Math.max(1, Math.min(QUEUED_CANCELLATION_RECONCILE_LIMIT, requestedLimit))
|
|
191
|
+
: QUEUED_CANCELLATION_RECONCILE_LIMIT;
|
|
192
|
+
const queued = await input.client.query<{
|
|
193
|
+
key: string;
|
|
194
|
+
reason: string | null;
|
|
195
|
+
}>(
|
|
196
|
+
`WITH selected AS (
|
|
197
|
+
SELECT key FROM ${schedulerSchema}.control
|
|
198
|
+
WHERE starts_with(key, 'runtime_cancel:')
|
|
199
|
+
ORDER BY updated_at ASC, key ASC
|
|
200
|
+
LIMIT $1
|
|
201
|
+
FOR UPDATE SKIP LOCKED
|
|
202
|
+
)
|
|
203
|
+
UPDATE ${schedulerSchema}.control AS request
|
|
204
|
+
SET updated_at = clock_timestamp()
|
|
205
|
+
FROM selected
|
|
206
|
+
WHERE request.key = selected.key
|
|
207
|
+
RETURNING request.key, request.value->>'reason' AS reason`,
|
|
208
|
+
[limit],
|
|
209
|
+
);
|
|
87
210
|
let reconciled = 0;
|
|
88
211
|
let skipped = 0;
|
|
89
|
-
for (const row of
|
|
90
|
-
const runId = row.
|
|
212
|
+
for (const row of queued.rows) {
|
|
213
|
+
const runId = row.key.slice('runtime_cancel:'.length);
|
|
91
214
|
if (!runId) {
|
|
92
|
-
|
|
93
|
-
continue;
|
|
215
|
+
throw new Error(`Malformed runtime cancellation request "${row.key}".`);
|
|
94
216
|
}
|
|
95
|
-
await input.client.query('BEGIN');
|
|
96
217
|
try {
|
|
97
|
-
await
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
);
|
|
105
|
-
|
|
106
|
-
input.client,
|
|
107
|
-
{
|
|
108
|
-
runId,
|
|
109
|
-
reason:
|
|
110
|
-
row.cancellation_kind === 'manual'
|
|
111
|
-
? 'Run cancellation was requested by the caller.'
|
|
112
|
-
: 'Run did not start within its configured scheduler queue deadline.',
|
|
113
|
-
},
|
|
114
|
-
input.schedulerOptions,
|
|
115
|
-
);
|
|
116
|
-
await input.client.query('COMMIT');
|
|
117
|
-
if (didCancel) reconciled += 1;
|
|
218
|
+
const outcome = await finalizeCancelledRunAfterCleanup({
|
|
219
|
+
client: input.client,
|
|
220
|
+
runId,
|
|
221
|
+
reason: row.reason || 'Cancelled play run.',
|
|
222
|
+
schedulerSchema,
|
|
223
|
+
schedulerOptions: input.schedulerOptions,
|
|
224
|
+
controlKey: row.key,
|
|
225
|
+
});
|
|
226
|
+
if (outcome === 'cancelled') reconciled += 1;
|
|
118
227
|
else skipped += 1;
|
|
119
228
|
} catch (error) {
|
|
120
|
-
await input.client.query('ROLLBACK').catch(() => undefined);
|
|
121
|
-
// A worker-held lock is expected to be retried by the next lane sweep;
|
|
122
|
-
// never let one blocked cancellation stall the claim loop or create a
|
|
123
|
-
// second unbounded cancellation pile.
|
|
124
229
|
if (
|
|
125
230
|
isRuntimeSchedulerQueryCancellationError(error) ||
|
|
126
231
|
isRuntimeSchedulerRetryableTransactionError(error)
|
|
127
232
|
) {
|
|
128
233
|
skipped += 1;
|
|
129
|
-
// The hardened pool destroys a client after a statement timeout or a
|
|
130
|
-
// retryable transaction error. Do not issue the next candidate on that
|
|
131
|
-
// same socket; the next worker sweep acquires a fresh client.
|
|
132
234
|
break;
|
|
133
235
|
}
|
|
134
236
|
throw error;
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts
CHANGED
|
@@ -262,10 +262,13 @@ import { shouldTerminalizeSandboxCapacityRejection } from '../launch-source-poli
|
|
|
262
262
|
import {
|
|
263
263
|
ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
|
|
264
264
|
reconcileAbsurdRuntimeCancellations,
|
|
265
|
+
reconcileQueuedAbsurdRuntimeCancellations,
|
|
265
266
|
} from './absurd-deadline-reconciliation';
|
|
266
267
|
import {
|
|
267
268
|
abandonRuntimeSandboxOperationReservation,
|
|
268
269
|
bindRuntimeSandboxCapacityLease,
|
|
270
|
+
readReservedDaytonaCancellationOperation,
|
|
271
|
+
recordRecoveredDaytonaCancellationSandbox,
|
|
269
272
|
reserveRuntimeSandboxOperation,
|
|
270
273
|
runtimeSandboxCapacityAdmissionMode,
|
|
271
274
|
RuntimeSandboxCapacityLimitError,
|
|
@@ -373,8 +376,8 @@ export const ABSURD_ADOPTED_LANE_POLL_INTERVAL_MS =
|
|
|
373
376
|
const HEARTBEAT_THROTTLE_MS = (HEARTBEAT_CLAIM_TIMEOUT_SECONDS * 1000) / 3;
|
|
374
377
|
const HEARTBEAT_TIMEOUT_RETRY_DELAY_MS = 1_000;
|
|
375
378
|
/**
|
|
376
|
-
* Sandbox deletion is
|
|
377
|
-
* Daytona SDK may hang before its own delete timeout (for example while
|
|
379
|
+
* Sandbox deletion is a durable obligation and may precede a cancellation's
|
|
380
|
+
* terminal write. The Daytona SDK may hang before its own delete timeout (for example while
|
|
378
381
|
* looking up an already-autostopped sandbox), so never let that external call
|
|
379
382
|
* occupy an Absurd worker slot indefinitely.
|
|
380
383
|
*/
|
|
@@ -6644,6 +6647,75 @@ export async function reconcilePostgresSchedulerAmbiguousModalCreatesOnce(input:
|
|
|
6644
6647
|
return { observed: intents.length, resourcesRecorded, resolvedAbsent };
|
|
6645
6648
|
}
|
|
6646
6649
|
|
|
6650
|
+
async function reconcileReservedDaytonaCancellationOnce(input: {
|
|
6651
|
+
runWithClient: AbsurdRuntimeWorkerDeps['runWithClient'];
|
|
6652
|
+
environment: 'preview' | 'production';
|
|
6653
|
+
schedulerOptions: Pick<PostgresSchedulerOptions, 'schema'>;
|
|
6654
|
+
}): Promise<void> {
|
|
6655
|
+
const operation = await input.runWithClient((client) =>
|
|
6656
|
+
readReservedDaytonaCancellationOperation(
|
|
6657
|
+
client,
|
|
6658
|
+
input.environment,
|
|
6659
|
+
input.schedulerOptions,
|
|
6660
|
+
),
|
|
6661
|
+
);
|
|
6662
|
+
if (!operation) return;
|
|
6663
|
+
const [{ Daytona }, { loadDaytonaRequiredConfig }] = await Promise.all([
|
|
6664
|
+
import('@daytonaio/sdk'),
|
|
6665
|
+
import('../daytona-runtime-config'),
|
|
6666
|
+
]);
|
|
6667
|
+
const daytona = new Daytona(
|
|
6668
|
+
loadDaytonaRequiredConfig({ environment: input.environment }).clientOptions,
|
|
6669
|
+
);
|
|
6670
|
+
const sandboxName = `dl-${operation.providerOperationToken}`;
|
|
6671
|
+
let sandbox: Awaited<ReturnType<typeof daytona.get>>;
|
|
6672
|
+
try {
|
|
6673
|
+
sandbox = await deleteRuntimeSandboxBounded({
|
|
6674
|
+
deleteRuntimeSandbox: (name) => daytona.get(name),
|
|
6675
|
+
sandboxId: sandboxName,
|
|
6676
|
+
timeoutMs: ABSURD_SANDBOX_GC_TIMEOUT_MS,
|
|
6677
|
+
});
|
|
6678
|
+
} catch (error) {
|
|
6679
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6680
|
+
if (!/\b(?:sandbox )?not found\b|\b404\b/i.test(message)) throw error;
|
|
6681
|
+
if (
|
|
6682
|
+
Date.now() - operation.createdAtMs <
|
|
6683
|
+
RUNTIME_RELIABILITY_POLICY.recovery.horizonMs
|
|
6684
|
+
) {
|
|
6685
|
+
return;
|
|
6686
|
+
}
|
|
6687
|
+
await input.runWithClient((client) =>
|
|
6688
|
+
abandonRuntimeSandboxOperationReservation(
|
|
6689
|
+
client,
|
|
6690
|
+
{ operationKey: operation.operationKey },
|
|
6691
|
+
input.schedulerOptions,
|
|
6692
|
+
),
|
|
6693
|
+
);
|
|
6694
|
+
return;
|
|
6695
|
+
}
|
|
6696
|
+
if (sandbox.labels?.operationKey !== operation.providerOperationToken) {
|
|
6697
|
+
throw new Error(
|
|
6698
|
+
`Daytona cancellation lookup ${sandboxName} lacks its exact operation label.`,
|
|
6699
|
+
);
|
|
6700
|
+
}
|
|
6701
|
+
if (!sandbox.organizationId) {
|
|
6702
|
+
throw new Error(
|
|
6703
|
+
`Daytona cancellation lookup ${sandboxName} lacks its routing organization.`,
|
|
6704
|
+
);
|
|
6705
|
+
}
|
|
6706
|
+
await input.runWithClient((client) =>
|
|
6707
|
+
recordRecoveredDaytonaCancellationSandbox(
|
|
6708
|
+
client,
|
|
6709
|
+
{
|
|
6710
|
+
operationKey: operation.operationKey,
|
|
6711
|
+
sandboxId: sandbox.id,
|
|
6712
|
+
routingDomain: sandbox.organizationId,
|
|
6713
|
+
},
|
|
6714
|
+
input.schedulerOptions,
|
|
6715
|
+
),
|
|
6716
|
+
);
|
|
6717
|
+
}
|
|
6718
|
+
|
|
6647
6719
|
export async function runPostgresSchedulerSandboxCleanupController(input: {
|
|
6648
6720
|
runWithClient: AbsurdRuntimeWorkerDeps['runWithClient'];
|
|
6649
6721
|
deleteRuntimeSandbox: AbsurdRuntimeWorkerDeps['deleteRuntimeSandbox'];
|
|
@@ -6681,6 +6753,8 @@ export async function runPostgresSchedulerSandboxCleanupController(input: {
|
|
|
6681
6753
|
const reportReconciliationPollFailure = createRateLimitedRuntimeLogger({
|
|
6682
6754
|
log: (...args) => console.warn(...args),
|
|
6683
6755
|
});
|
|
6756
|
+
let reservedRecovery: Promise<void> | null = null;
|
|
6757
|
+
let nextReservedRecoveryAt = 0;
|
|
6684
6758
|
|
|
6685
6759
|
const startJob = (job: PostgresSchedulerSandboxCleanupJob): void => {
|
|
6686
6760
|
const task = processPostgresSchedulerSandboxCleanupJob({
|
|
@@ -6726,6 +6800,37 @@ export async function runPostgresSchedulerSandboxCleanupController(input: {
|
|
|
6726
6800
|
};
|
|
6727
6801
|
|
|
6728
6802
|
while (!input.shouldStop()) {
|
|
6803
|
+
if (
|
|
6804
|
+
(!input.providers || input.providers.includes('daytona')) &&
|
|
6805
|
+
!reservedRecovery &&
|
|
6806
|
+
Date.now() >= nextReservedRecoveryAt
|
|
6807
|
+
) {
|
|
6808
|
+
nextReservedRecoveryAt = Date.now() + 60_000;
|
|
6809
|
+
const recovery = reconcileReservedDaytonaCancellationOnce(input)
|
|
6810
|
+
.catch((error) => {
|
|
6811
|
+
reportReconciliationPollFailure(
|
|
6812
|
+
'[absurd-runtime.worker] daytona_cancellation_reservation_recovery_failed',
|
|
6813
|
+
{ error: error instanceof Error ? error.message : String(error) },
|
|
6814
|
+
);
|
|
6815
|
+
})
|
|
6816
|
+
.finally(() => {
|
|
6817
|
+
if (reservedRecovery === recovery) reservedRecovery = null;
|
|
6818
|
+
});
|
|
6819
|
+
reservedRecovery = recovery;
|
|
6820
|
+
}
|
|
6821
|
+
try {
|
|
6822
|
+
await input.runWithClient((client) =>
|
|
6823
|
+
reconcileQueuedAbsurdRuntimeCancellations({
|
|
6824
|
+
client,
|
|
6825
|
+
schedulerOptions: input.schedulerOptions,
|
|
6826
|
+
}),
|
|
6827
|
+
);
|
|
6828
|
+
} catch (error) {
|
|
6829
|
+
reportReconciliationPollFailure(
|
|
6830
|
+
'[absurd-runtime.worker] cancellation_reconciliation_poll_failed',
|
|
6831
|
+
{ error: error instanceof Error ? error.message : String(error) },
|
|
6832
|
+
);
|
|
6833
|
+
}
|
|
6729
6834
|
if (input.providers?.includes('modal')) {
|
|
6730
6835
|
try {
|
|
6731
6836
|
const reconciliation =
|
|
@@ -7082,9 +7187,9 @@ export async function startRuntimeWorkerHost(
|
|
|
7082
7187
|
queue,
|
|
7083
7188
|
schedulerOptions,
|
|
7084
7189
|
});
|
|
7085
|
-
if (cancellations.
|
|
7190
|
+
if (cancellations.queued > 0) {
|
|
7086
7191
|
reportCancellationProgress(
|
|
7087
|
-
'[absurd-runtime.worker]
|
|
7192
|
+
'[absurd-runtime.worker] cancellations_queued',
|
|
7088
7193
|
{
|
|
7089
7194
|
queue,
|
|
7090
7195
|
releaseId,
|
|
@@ -7244,9 +7349,9 @@ export async function startRuntimeWorkerHost(
|
|
|
7244
7349
|
queue: adoptedQueue,
|
|
7245
7350
|
schedulerOptions,
|
|
7246
7351
|
});
|
|
7247
|
-
if (cancellations.
|
|
7352
|
+
if (cancellations.queued > 0) {
|
|
7248
7353
|
reportAdoptedCancellationProgress(
|
|
7249
|
-
'[absurd-runtime.worker]
|
|
7354
|
+
'[absurd-runtime.worker] cancellations_queued',
|
|
7250
7355
|
{
|
|
7251
7356
|
queue: adoptedQueue,
|
|
7252
7357
|
releaseId,
|
|
@@ -14,8 +14,8 @@
|
|
|
14
14
|
* Persist the absurd task id
|
|
15
15
|
* on the run row so a later cancel can target it.
|
|
16
16
|
* observe/result → reuse the postgres LISTEN/NOTIFY run handle.
|
|
17
|
-
* cancel →
|
|
18
|
-
* the
|
|
17
|
+
* cancel → commit the Absurd stop and a durable cancellation request;
|
|
18
|
+
* the worker deletes sandboxes before the final run write.
|
|
19
19
|
* signal → durable signal row (authoritative payload) + absurd emitEvent
|
|
20
20
|
* (the wakeup only; payload is re-read from the signal row).
|
|
21
21
|
*/
|
|
@@ -29,8 +29,6 @@ import {
|
|
|
29
29
|
type PlaySchedulerSubmitInput,
|
|
30
30
|
} from '../scheduler-backend';
|
|
31
31
|
import {
|
|
32
|
-
cancelPostgresSchedulerRun,
|
|
33
|
-
postgresSchedulerCancellationOutboxLease,
|
|
34
32
|
postgresSchedulerRunHandle,
|
|
35
33
|
readPostgresSchedulerLaunch,
|
|
36
34
|
readPostgresSchedulerAbsurdTaskId,
|
|
@@ -48,6 +46,8 @@ import {
|
|
|
48
46
|
withRuntimeSchedulerClient,
|
|
49
47
|
withRuntimeSchedulerTransaction,
|
|
50
48
|
} from './postgres-pool';
|
|
49
|
+
import { RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE } from '../scheduler-backend';
|
|
50
|
+
import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
|
|
51
51
|
import { withRuntimeSchedulerSubmitAdmission } from './submit-admission';
|
|
52
52
|
import {
|
|
53
53
|
ABSURD_PLAY_RUN_MAX_ATTEMPTS,
|
|
@@ -200,7 +200,7 @@ function absurdRunHandle(input: {
|
|
|
200
200
|
const runsQueue = absurdQueueNameForRelease(releaseId);
|
|
201
201
|
const cancellationKey = `${poolOptions.url}::${effectiveOptions.schema ?? ''}::${input.runId}`;
|
|
202
202
|
|
|
203
|
-
const
|
|
203
|
+
const queueCancellation = async (reason?: string) =>
|
|
204
204
|
withRuntimeSchedulerTransaction(
|
|
205
205
|
poolOptions,
|
|
206
206
|
{
|
|
@@ -208,9 +208,8 @@ function absurdRunHandle(input: {
|
|
|
208
208
|
statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
|
|
209
209
|
},
|
|
210
210
|
async (client) => {
|
|
211
|
-
// Resolve the immutable release lane
|
|
212
|
-
//
|
|
213
|
-
// has no preliminary lane lookup that can succeed or fail separately.
|
|
211
|
+
// Resolve the immutable release lane in the transaction that records
|
|
212
|
+
// the request. A failed final run write cannot roll this marker back.
|
|
214
213
|
const launch = await readPostgresSchedulerLaunch(
|
|
215
214
|
client,
|
|
216
215
|
input.runId,
|
|
@@ -231,67 +230,73 @@ function absurdRunHandle(input: {
|
|
|
231
230
|
input.runId,
|
|
232
231
|
effectiveOptions,
|
|
233
232
|
);
|
|
233
|
+
if (
|
|
234
|
+
await readPostgresSchedulerResult(
|
|
235
|
+
client,
|
|
236
|
+
input.runId,
|
|
237
|
+
effectiveOptions,
|
|
238
|
+
)
|
|
239
|
+
) {
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
234
242
|
if (taskId) {
|
|
235
|
-
const
|
|
243
|
+
const engineTaskPresent = await cancelAbsurdTaskIfPresent(client, {
|
|
236
244
|
queue: cancellationQueue,
|
|
237
245
|
taskId,
|
|
238
246
|
runId: input.runId,
|
|
239
247
|
});
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
// schema cutover. The explicit marker keeps a user cancel distinct
|
|
244
|
-
// from a task cancelled by its configured max_delay sweep.
|
|
245
|
-
if (engineTaskCancelled) {
|
|
248
|
+
if (engineTaskPresent) {
|
|
249
|
+
// Older workers only reconcile `manual` or `max_delay`. This
|
|
250
|
+
// marker is diagnostic; the control row is the cross-lane queue.
|
|
246
251
|
await client.query(
|
|
247
252
|
`UPDATE absurd.${quotePostgresIdentifier(`t_${cancellationQueue}`)}
|
|
248
253
|
SET cancel_at = COALESCE(cancel_at, clock_timestamp()),
|
|
249
254
|
cancellation = COALESCE(cancellation, '{}'::jsonb)
|
|
250
|
-
|| jsonb_build_object('
|
|
251
|
-
WHERE task_id = $1::uuid
|
|
252
|
-
AND state = 'cancelled'`,
|
|
255
|
+
|| jsonb_build_object('sandboxFirst', true)
|
|
256
|
+
WHERE task_id = $1::uuid`,
|
|
253
257
|
[taskId],
|
|
254
258
|
);
|
|
255
259
|
}
|
|
256
260
|
}
|
|
257
|
-
const
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
reason,
|
|
262
|
-
outboxLease: postgresSchedulerCancellationOutboxLease(input.runId),
|
|
263
|
-
},
|
|
264
|
-
effectiveOptions,
|
|
261
|
+
const schedulerSchema = quotePostgresIdentifier(
|
|
262
|
+
effectiveOptions.schema?.trim() ||
|
|
263
|
+
resolveRuntimeSchedulerSchemaEnv() ||
|
|
264
|
+
'runtime_scheduler',
|
|
265
265
|
);
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
}
|
|
280
|
-
}
|
|
266
|
+
await client.query(
|
|
267
|
+
`INSERT INTO ${schedulerSchema}.control (key, value)
|
|
268
|
+
VALUES ($1, jsonb_build_object(
|
|
269
|
+
'reason', left($2::text, 4000),
|
|
270
|
+
'requestedAt', clock_timestamp()
|
|
271
|
+
))
|
|
272
|
+
ON CONFLICT (key) DO NOTHING`,
|
|
273
|
+
[
|
|
274
|
+
`runtime_cancel:${input.runId}`,
|
|
275
|
+
reason?.trim() || 'Cancelled play run.',
|
|
276
|
+
],
|
|
277
|
+
);
|
|
278
|
+
return true;
|
|
281
279
|
},
|
|
282
280
|
{
|
|
283
281
|
runId: input.runId,
|
|
284
|
-
phase: '
|
|
282
|
+
phase: 'queue_run_cancellation',
|
|
285
283
|
priority: 'critical',
|
|
286
284
|
},
|
|
285
|
+
{ maxAttempts: 1 },
|
|
287
286
|
);
|
|
288
287
|
|
|
289
288
|
const requestCancellation = async (reason?: string) => {
|
|
290
289
|
await runCancellationSingleFlight(cancellationKey, async () => {
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
290
|
+
const queued = await queueCancellation(reason);
|
|
291
|
+
if (queued) {
|
|
292
|
+
const error = new Error(
|
|
293
|
+
`Cancellation queued for ${input.runId}; sandbox cleanup and final run update are pending.`,
|
|
294
|
+
);
|
|
295
|
+
Object.assign(error, {
|
|
296
|
+
code: RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE,
|
|
297
|
+
});
|
|
298
|
+
throw error;
|
|
299
|
+
}
|
|
295
300
|
});
|
|
296
301
|
};
|
|
297
302
|
|
|
@@ -8774,13 +8774,29 @@ export async function resolvePostgresSchedulerAmbiguousModalCreateIntentFromProv
|
|
|
8774
8774
|
): Promise<boolean> {
|
|
8775
8775
|
const resolved = await client.query<{ run_id: string }>(
|
|
8776
8776
|
`
|
|
8777
|
-
|
|
8778
|
-
|
|
8779
|
-
|
|
8780
|
-
|
|
8781
|
-
|
|
8782
|
-
|
|
8783
|
-
|
|
8777
|
+
WITH resolved AS (
|
|
8778
|
+
UPDATE ${tableName('ambiguous_modal_creates', options)}
|
|
8779
|
+
SET status = 'resolved', resolved_at = clock_timestamp()
|
|
8780
|
+
WHERE run_id = $1
|
|
8781
|
+
AND attempt = $2
|
|
8782
|
+
AND modal_app_id = $3
|
|
8783
|
+
AND status = 'pending'
|
|
8784
|
+
RETURNING run_id, sandbox_capacity_lease_id
|
|
8785
|
+
), abandoned AS (
|
|
8786
|
+
UPDATE ${tableName('runtime_sandbox_operations', options)} AS operation
|
|
8787
|
+
SET state = 'abandoned', updated_at = clock_timestamp()
|
|
8788
|
+
FROM resolved
|
|
8789
|
+
WHERE operation.operation_key = resolved.sandbox_capacity_lease_id
|
|
8790
|
+
AND operation.state = 'reserved'
|
|
8791
|
+
RETURNING operation.operation_key
|
|
8792
|
+
), released AS (
|
|
8793
|
+
UPDATE ${tableName('runtime_sandbox_capacity_leases', options)} AS lease
|
|
8794
|
+
SET released_at = clock_timestamp()
|
|
8795
|
+
FROM abandoned
|
|
8796
|
+
WHERE lease.lease_id = abandoned.operation_key
|
|
8797
|
+
AND lease.released_at IS NULL
|
|
8798
|
+
)
|
|
8799
|
+
SELECT run_id FROM resolved
|
|
8784
8800
|
`,
|
|
8785
8801
|
[input.runId, input.attempt, input.modalAppId],
|
|
8786
8802
|
);
|
|
@@ -10587,6 +10603,131 @@ export async function enqueuePostgresSchedulerSandboxCleanupForAttempt(
|
|
|
10587
10603
|
return enqueued.rows.length;
|
|
10588
10604
|
}
|
|
10589
10605
|
|
|
10606
|
+
/** Queue every known sandbox before a requested cancellation can terminalize. */
|
|
10607
|
+
export async function enqueuePostgresSchedulerSandboxCleanupForRun(
|
|
10608
|
+
client: PostgresSchedulerQueryClient,
|
|
10609
|
+
input: { runId: string },
|
|
10610
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
10611
|
+
): Promise<void> {
|
|
10612
|
+
await client.query(
|
|
10613
|
+
`
|
|
10614
|
+
INSERT INTO ${tableName('sandbox_cleanup_jobs_v2', options)} (
|
|
10615
|
+
provider, sandbox_id, run_id, attempt, environment, routing_domain
|
|
10616
|
+
)
|
|
10617
|
+
SELECT DISTINCT
|
|
10618
|
+
CASE resource->>'kind'
|
|
10619
|
+
WHEN 'modal_sandbox' THEN 'modal'
|
|
10620
|
+
ELSE 'daytona'
|
|
10621
|
+
END,
|
|
10622
|
+
resource->>'sandboxId',
|
|
10623
|
+
attempt.run_id,
|
|
10624
|
+
attempt.attempt,
|
|
10625
|
+
COALESCE(
|
|
10626
|
+
NULLIF(resource->>'runtimeEnvironment', ''),
|
|
10627
|
+
NULLIF(resource->>'daytonaEnvironment', ''),
|
|
10628
|
+
'production'
|
|
10629
|
+
),
|
|
10630
|
+
CASE resource->>'kind'
|
|
10631
|
+
WHEN 'modal_sandbox' THEN NULLIF(resource->>'modalAppId', '')
|
|
10632
|
+
ELSE NULLIF(resource->>'daytonaOrganizationId', '')
|
|
10633
|
+
END
|
|
10634
|
+
FROM ${tableName('attempts', options)} AS attempt
|
|
10635
|
+
CROSS JOIN LATERAL jsonb_array_elements(
|
|
10636
|
+
CASE
|
|
10637
|
+
WHEN jsonb_typeof(attempt.resource_json) = 'array'
|
|
10638
|
+
THEN attempt.resource_json
|
|
10639
|
+
ELSE jsonb_build_array(attempt.resource_json)
|
|
10640
|
+
END
|
|
10641
|
+
) AS resource
|
|
10642
|
+
WHERE attempt.run_id = $1
|
|
10643
|
+
AND attempt.resource_json IS NOT NULL
|
|
10644
|
+
AND resource->>'kind' IN ('daytona_sandbox', 'modal_sandbox')
|
|
10645
|
+
AND NULLIF(resource->>'sandboxId', '') IS NOT NULL
|
|
10646
|
+
UNION ALL
|
|
10647
|
+
SELECT operation.provider, operation.sandbox_id, operation.run_id,
|
|
10648
|
+
operation.attempt, operation.environment, operation.routing_domain
|
|
10649
|
+
FROM ${tableName('runtime_sandbox_operations', options)} AS operation
|
|
10650
|
+
WHERE operation.run_id = $1
|
|
10651
|
+
AND operation.state IN ('live', 'deleting')
|
|
10652
|
+
AND operation.sandbox_id IS NOT NULL
|
|
10653
|
+
ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
|
|
10654
|
+
`,
|
|
10655
|
+
[input.runId],
|
|
10656
|
+
);
|
|
10657
|
+
}
|
|
10658
|
+
|
|
10659
|
+
/** Modal cannot create without a durable intent; an old reservation with no
|
|
10660
|
+
* pending selector or recorded resource is safe to release after recovery time. */
|
|
10661
|
+
export async function abandonUnstartedModalCancellationReservations(
|
|
10662
|
+
client: PostgresSchedulerQueryClient,
|
|
10663
|
+
input: { runId: string },
|
|
10664
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
10665
|
+
): Promise<void> {
|
|
10666
|
+
await client.query(
|
|
10667
|
+
`WITH abandoned AS (
|
|
10668
|
+
UPDATE ${tableName('runtime_sandbox_operations', options)} AS operation
|
|
10669
|
+
SET state = 'abandoned', updated_at = clock_timestamp()
|
|
10670
|
+
WHERE operation.run_id = $1
|
|
10671
|
+
AND operation.provider = 'modal'
|
|
10672
|
+
AND operation.state = 'reserved'
|
|
10673
|
+
AND operation.created_at <= clock_timestamp()
|
|
10674
|
+
- ($2::double precision * interval '1 millisecond')
|
|
10675
|
+
AND NOT EXISTS (
|
|
10676
|
+
SELECT 1 FROM ${tableName('ambiguous_modal_creates', options)} AS intent
|
|
10677
|
+
WHERE intent.sandbox_capacity_lease_id = operation.operation_key
|
|
10678
|
+
AND intent.status = 'pending'
|
|
10679
|
+
)
|
|
10680
|
+
AND NOT EXISTS (
|
|
10681
|
+
SELECT 1 FROM ${tableName('attempts', options)} AS attempt
|
|
10682
|
+
CROSS JOIN LATERAL jsonb_array_elements(
|
|
10683
|
+
CASE
|
|
10684
|
+
WHEN jsonb_typeof(attempt.resource_json) = 'array'
|
|
10685
|
+
THEN attempt.resource_json
|
|
10686
|
+
WHEN attempt.resource_json IS NULL THEN '[]'::jsonb
|
|
10687
|
+
ELSE jsonb_build_array(attempt.resource_json)
|
|
10688
|
+
END
|
|
10689
|
+
) AS resource
|
|
10690
|
+
WHERE attempt.run_id = operation.run_id
|
|
10691
|
+
AND resource->>'sandboxCapacityLeaseId' = operation.operation_key
|
|
10692
|
+
)
|
|
10693
|
+
RETURNING operation.operation_key
|
|
10694
|
+
)
|
|
10695
|
+
UPDATE ${tableName('runtime_sandbox_capacity_leases', options)} AS lease
|
|
10696
|
+
SET released_at = clock_timestamp()
|
|
10697
|
+
FROM abandoned
|
|
10698
|
+
WHERE lease.lease_id = abandoned.operation_key
|
|
10699
|
+
AND lease.released_at IS NULL`,
|
|
10700
|
+
[input.runId, RUNTIME_RELIABILITY_POLICY.recovery.horizonMs],
|
|
10701
|
+
);
|
|
10702
|
+
}
|
|
10703
|
+
|
|
10704
|
+
/** A reserved create or unconfirmed deletion keeps cancellation pending. */
|
|
10705
|
+
export async function hasPendingPostgresSchedulerSandboxCleanupForRun(
|
|
10706
|
+
client: PostgresSchedulerQueryClient,
|
|
10707
|
+
input: { runId: string },
|
|
10708
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
10709
|
+
): Promise<boolean> {
|
|
10710
|
+
const pending = await client.query<{ pending: boolean }>(
|
|
10711
|
+
`
|
|
10712
|
+
SELECT EXISTS (
|
|
10713
|
+
SELECT 1
|
|
10714
|
+
FROM ${tableName('sandbox_cleanup_jobs_v2', options)}
|
|
10715
|
+
WHERE run_id = $1 AND status <> 'deleted'
|
|
10716
|
+
) OR EXISTS (
|
|
10717
|
+
SELECT 1
|
|
10718
|
+
FROM ${tableName('runtime_sandbox_operations', options)}
|
|
10719
|
+
WHERE run_id = $1 AND state = 'reserved'
|
|
10720
|
+
) OR EXISTS (
|
|
10721
|
+
SELECT 1
|
|
10722
|
+
FROM ${tableName('ambiguous_modal_creates', options)}
|
|
10723
|
+
WHERE run_id = $1 AND status = 'pending'
|
|
10724
|
+
) AS pending
|
|
10725
|
+
`,
|
|
10726
|
+
[input.runId],
|
|
10727
|
+
);
|
|
10728
|
+
return pending.rows[0]?.pending ?? true;
|
|
10729
|
+
}
|
|
10730
|
+
|
|
10590
10731
|
export async function claimPostgresSchedulerSandboxCleanupJobs(
|
|
10591
10732
|
client: PostgresSchedulerQueryClient,
|
|
10592
10733
|
input: {
|
|
@@ -12278,7 +12419,26 @@ export async function cancelPostgresSchedulerRun(
|
|
|
12278
12419
|
WHEN elem->>'kind' IN ('daytona_sandbox', 'modal_sandbox')
|
|
12279
12420
|
AND (elem->'billingEndedAt' IS NULL OR elem->'billingEndedAt' = 'null'::jsonb)
|
|
12280
12421
|
THEN elem || jsonb_build_object(
|
|
12281
|
-
'billingEndedAt',
|
|
12422
|
+
'billingEndedAt', COALESCE(
|
|
12423
|
+
(
|
|
12424
|
+
SELECT extract(epoch FROM cleanup.deleted_at) * 1000
|
|
12425
|
+
FROM ${tableName('sandbox_cleanup_jobs_v2', options)} AS cleanup
|
|
12426
|
+
WHERE cleanup.provider = CASE elem->>'kind'
|
|
12427
|
+
WHEN 'modal_sandbox' THEN 'modal'
|
|
12428
|
+
ELSE 'daytona'
|
|
12429
|
+
END
|
|
12430
|
+
AND cleanup.sandbox_id = elem->>'sandboxId'
|
|
12431
|
+
AND cleanup.routing_domain_key = COALESCE(
|
|
12432
|
+
CASE elem->>'kind'
|
|
12433
|
+
WHEN 'modal_sandbox' THEN NULLIF(elem->>'modalAppId', '')
|
|
12434
|
+
ELSE NULLIF(elem->>'daytonaOrganizationId', '')
|
|
12435
|
+
END,
|
|
12436
|
+
''
|
|
12437
|
+
)
|
|
12438
|
+
AND cleanup.status = 'deleted'
|
|
12439
|
+
),
|
|
12440
|
+
extract(epoch FROM clock_timestamp()) * 1000
|
|
12441
|
+
),
|
|
12282
12442
|
'terminalAt', extract(epoch FROM clock_timestamp()) * 1000,
|
|
12283
12443
|
'terminalReason', 'cancelled'
|
|
12284
12444
|
)
|
|
@@ -64,6 +64,7 @@ function schemaTables(options?: Pick<PostgresSchedulerOptions, 'schema'>) {
|
|
|
64
64
|
leases: `${schema}."runtime_sandbox_capacity_leases"`,
|
|
65
65
|
operations: `${schema}."runtime_sandbox_operations"`,
|
|
66
66
|
cleanup: `${schema}."sandbox_cleanup_jobs_v2"`,
|
|
67
|
+
runs: `${schema}."runs"`,
|
|
67
68
|
};
|
|
68
69
|
}
|
|
69
70
|
|
|
@@ -266,7 +267,7 @@ export async function reserveRuntimeSandboxCapacityLease(
|
|
|
266
267
|
> {
|
|
267
268
|
const orgId = assertId(input.orgId, 'orgId');
|
|
268
269
|
const runId = assertId(input.runId, 'runId');
|
|
269
|
-
const { cache, leases, operations } = schemaTables(options);
|
|
270
|
+
const { cache, leases, operations, runs } = schemaTables(options);
|
|
270
271
|
// This is deliberately a soft admission check: concurrent workers may both
|
|
271
272
|
// observe space and insert. Avoid locks and long transactions; the provider
|
|
272
273
|
// has its own global safety cap and the 50-minute lease bounds crashes.
|
|
@@ -469,13 +470,21 @@ export async function reserveRuntimeSandboxCapacityLease(
|
|
|
469
470
|
}
|
|
470
471
|
await client.query(
|
|
471
472
|
`
|
|
472
|
-
WITH
|
|
473
|
+
WITH active_run AS MATERIALIZED (
|
|
474
|
+
SELECT run_id FROM ${runs}
|
|
475
|
+
WHERE run_id = $3 AND status IN ('queued', 'running', 'waiting')
|
|
476
|
+
AND NOT EXISTS (
|
|
477
|
+
SELECT 1 FROM ${quoteIdent(options?.schema ?? 'runtime_scheduler')}."control"
|
|
478
|
+
WHERE key = 'runtime_cancel:' || $3::text
|
|
479
|
+
)
|
|
480
|
+
FOR UPDATE
|
|
481
|
+
), reserved_lease AS (
|
|
473
482
|
INSERT INTO ${leases} (
|
|
474
483
|
lease_id, org_id, run_id, provider, expires_at
|
|
475
|
-
) VALUES (
|
|
476
|
-
$1, $2, $3, $4,
|
|
477
|
-
clock_timestamp() + ($5::integer * interval '1 millisecond')
|
|
478
484
|
)
|
|
485
|
+
SELECT $1, $2, $3, $4,
|
|
486
|
+
clock_timestamp() + ($5::integer * interval '1 millisecond')
|
|
487
|
+
WHERE $6::integer IS NULL OR EXISTS (SELECT 1 FROM active_run)
|
|
479
488
|
ON CONFLICT (lease_id) DO NOTHING
|
|
480
489
|
RETURNING lease_id
|
|
481
490
|
)
|
|
@@ -726,6 +735,125 @@ export async function abandonRuntimeSandboxOperationReservation(
|
|
|
726
735
|
return result.rows.length === 1;
|
|
727
736
|
}
|
|
728
737
|
|
|
738
|
+
export async function readReservedDaytonaCancellationOperation(
|
|
739
|
+
client: PostgresSchedulerQueryClient,
|
|
740
|
+
environment: 'preview' | 'production',
|
|
741
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
742
|
+
): Promise<{
|
|
743
|
+
operationKey: string;
|
|
744
|
+
providerOperationToken: string;
|
|
745
|
+
createdAtMs: number;
|
|
746
|
+
} | null> {
|
|
747
|
+
const { operations } = schemaTables(options);
|
|
748
|
+
const schema = quoteIdent(options?.schema ?? 'runtime_scheduler');
|
|
749
|
+
const rows = await client.query<{
|
|
750
|
+
operation_key: string;
|
|
751
|
+
provider_operation_token: string | null;
|
|
752
|
+
created_at_ms: number;
|
|
753
|
+
}>(
|
|
754
|
+
`WITH candidate AS (
|
|
755
|
+
SELECT operation.operation_key
|
|
756
|
+
FROM ${operations} AS operation
|
|
757
|
+
JOIN ${schema}."control" AS request
|
|
758
|
+
ON request.key = 'runtime_cancel:' || operation.run_id
|
|
759
|
+
WHERE operation.provider = 'daytona'
|
|
760
|
+
AND operation.environment = $1
|
|
761
|
+
AND operation.state = 'reserved'
|
|
762
|
+
ORDER BY operation.updated_at ASC, operation.operation_key ASC
|
|
763
|
+
LIMIT 1
|
|
764
|
+
FOR UPDATE OF operation SKIP LOCKED
|
|
765
|
+
)
|
|
766
|
+
UPDATE ${operations} AS operation
|
|
767
|
+
SET updated_at = clock_timestamp()
|
|
768
|
+
FROM candidate
|
|
769
|
+
WHERE operation.operation_key = candidate.operation_key
|
|
770
|
+
RETURNING operation.operation_key, operation.provider_operation_token,
|
|
771
|
+
floor(extract(epoch FROM operation.created_at) * 1000)::bigint AS created_at_ms`,
|
|
772
|
+
[environment],
|
|
773
|
+
);
|
|
774
|
+
const row = rows.rows[0];
|
|
775
|
+
if (!row) return null;
|
|
776
|
+
if (!row.provider_operation_token) {
|
|
777
|
+
throw new Error(
|
|
778
|
+
`Reserved Daytona cancellation operation ${row.operation_key} lacks its provider token.`,
|
|
779
|
+
);
|
|
780
|
+
}
|
|
781
|
+
return {
|
|
782
|
+
operationKey: row.operation_key,
|
|
783
|
+
providerOperationToken: row.provider_operation_token,
|
|
784
|
+
createdAtMs: Number(row.created_at_ms),
|
|
785
|
+
};
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
export async function recordRecoveredDaytonaCancellationSandbox(
|
|
789
|
+
client: PostgresSchedulerQueryClient,
|
|
790
|
+
input: { operationKey: string; sandboxId: string; routingDomain: string },
|
|
791
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
792
|
+
): Promise<boolean> {
|
|
793
|
+
const { operations, leases, cleanup } = schemaTables(options);
|
|
794
|
+
const schema = quoteIdent(options?.schema ?? 'runtime_scheduler');
|
|
795
|
+
const recorded = await client.query<{ operation_key: string }>(
|
|
796
|
+
`WITH promoted AS (
|
|
797
|
+
UPDATE ${operations} AS operation
|
|
798
|
+
SET state = 'deleting',
|
|
799
|
+
sandbox_id = $2,
|
|
800
|
+
routing_domain = $3,
|
|
801
|
+
provider_created_at = clock_timestamp(),
|
|
802
|
+
updated_at = clock_timestamp()
|
|
803
|
+
WHERE operation.operation_key = $1
|
|
804
|
+
AND operation.provider = 'daytona'
|
|
805
|
+
AND operation.state = 'reserved'
|
|
806
|
+
RETURNING operation.operation_key, operation.run_id, operation.attempt,
|
|
807
|
+
operation.environment, operation.created_at
|
|
808
|
+
), resource AS (
|
|
809
|
+
SELECT promoted.*,
|
|
810
|
+
jsonb_build_object(
|
|
811
|
+
'kind', 'daytona_sandbox',
|
|
812
|
+
'sandboxId', $2::text,
|
|
813
|
+
'runtimeEnvironment', promoted.environment,
|
|
814
|
+
'daytonaOrganizationId', $3::text,
|
|
815
|
+
'billingStartedAt', extract(epoch FROM promoted.created_at) * 1000,
|
|
816
|
+
'sandboxCapacityLeaseId', promoted.operation_key,
|
|
817
|
+
'lateAcquired', true
|
|
818
|
+
) AS value
|
|
819
|
+
FROM promoted
|
|
820
|
+
), recorded_attempt AS (
|
|
821
|
+
UPDATE ${schema}."attempts" AS attempt
|
|
822
|
+
SET resource_json = CASE
|
|
823
|
+
WHEN attempt.resource_json IS NULL THEN jsonb_build_array(resource.value)
|
|
824
|
+
WHEN jsonb_typeof(attempt.resource_json) = 'array'
|
|
825
|
+
THEN attempt.resource_json || jsonb_build_array(resource.value)
|
|
826
|
+
ELSE jsonb_build_array(attempt.resource_json, resource.value)
|
|
827
|
+
END,
|
|
828
|
+
updated_at = clock_timestamp()
|
|
829
|
+
FROM resource
|
|
830
|
+
WHERE attempt.run_id = resource.run_id
|
|
831
|
+
AND attempt.attempt = resource.attempt
|
|
832
|
+
RETURNING attempt.run_id
|
|
833
|
+
), bound AS (
|
|
834
|
+
UPDATE ${leases} AS lease
|
|
835
|
+
SET sandbox_id = $2, routing_domain = $3
|
|
836
|
+
FROM promoted
|
|
837
|
+
WHERE lease.lease_id = promoted.operation_key
|
|
838
|
+
AND lease.released_at IS NULL
|
|
839
|
+
RETURNING lease.lease_id
|
|
840
|
+
), queued AS (
|
|
841
|
+
INSERT INTO ${cleanup} (
|
|
842
|
+
provider, sandbox_id, run_id, attempt, environment, routing_domain
|
|
843
|
+
)
|
|
844
|
+
SELECT 'daytona', $2, promoted.run_id, promoted.attempt,
|
|
845
|
+
promoted.environment, $3
|
|
846
|
+
FROM promoted
|
|
847
|
+
ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
|
|
848
|
+
)
|
|
849
|
+
SELECT promoted.operation_key FROM promoted
|
|
850
|
+
JOIN recorded_attempt ON recorded_attempt.run_id = promoted.run_id
|
|
851
|
+
JOIN bound ON bound.lease_id = promoted.operation_key`,
|
|
852
|
+
[input.operationKey, input.sandboxId, input.routingDomain],
|
|
853
|
+
);
|
|
854
|
+
return recorded.rows.length === 1;
|
|
855
|
+
}
|
|
856
|
+
|
|
729
857
|
/** Release only after the provider confirms deletion (or already absent). */
|
|
730
858
|
export async function releaseRuntimeSandboxCapacityLeaseForSandbox(
|
|
731
859
|
client: PostgresSchedulerQueryClient,
|
package/dist/cli/index.js
CHANGED
|
@@ -3051,7 +3051,7 @@ var SDK_RELEASE = {
|
|
|
3051
3051
|
// getters keep their established compatibility behavior.
|
|
3052
3052
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3053
3053
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3054
|
-
version: "0.3.
|
|
3054
|
+
version: "0.3.201",
|
|
3055
3055
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3056
3056
|
packageCapabilities: {
|
|
3057
3057
|
updatePreferences: 1
|
|
@@ -29004,6 +29004,26 @@ async function handleRunStop(args) {
|
|
|
29004
29004
|
}
|
|
29005
29005
|
const client2 = new DeeplineClient();
|
|
29006
29006
|
const result = await client2.runs.stop(runId, { reason });
|
|
29007
|
+
if (result.cancelling === true) {
|
|
29008
|
+
printCommandEnvelope(
|
|
29009
|
+
{
|
|
29010
|
+
...result,
|
|
29011
|
+
render: {
|
|
29012
|
+
sections: [
|
|
29013
|
+
{
|
|
29014
|
+
title: "run stop",
|
|
29015
|
+
lines: [
|
|
29016
|
+
`Cancellation queued for ${result.runId || runId}; sandbox cleanup is pending`,
|
|
29017
|
+
`inspect: deepline runs get ${runId} --full --json`
|
|
29018
|
+
]
|
|
29019
|
+
}
|
|
29020
|
+
]
|
|
29021
|
+
}
|
|
29022
|
+
},
|
|
29023
|
+
{ json: argsWantJson(args) }
|
|
29024
|
+
);
|
|
29025
|
+
return 0;
|
|
29026
|
+
}
|
|
29007
29027
|
const stopNotConfirmed = result.stopped === false || result.staleSchedulerState === true;
|
|
29008
29028
|
if (stopNotConfirmed) {
|
|
29009
29029
|
const detail = typeof result.error === "string" && result.error.trim() ? result.error.trim() : result.staleSchedulerState === true ? "scheduler state for the run is stale" : "the server did not confirm the stop";
|
|
@@ -29118,6 +29138,7 @@ async function handleRunStopAll(args) {
|
|
|
29118
29138
|
const result = await client2.runs.stopAll({ reason });
|
|
29119
29139
|
const lines = [
|
|
29120
29140
|
`Stopped ${result.stopped} active run${result.stopped === 1 ? "" : "s"}`,
|
|
29141
|
+
...result.cancelling ? [`cancellation queued: ${result.cancelling}`] : [],
|
|
29121
29142
|
...result.partial ? ["partial: active runs may remain; rerun stop-all"] : [],
|
|
29122
29143
|
...result.failed > 0 ? [`failed stops: ${result.failed}`] : [],
|
|
29123
29144
|
...result.skipped > 0 ? [`skipped: ${result.skipped}`] : []
|
package/dist/cli/index.mjs
CHANGED
|
@@ -3046,7 +3046,7 @@ var SDK_RELEASE = {
|
|
|
3046
3046
|
// getters keep their established compatibility behavior.
|
|
3047
3047
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3048
3048
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3049
|
-
version: "0.3.
|
|
3049
|
+
version: "0.3.201",
|
|
3050
3050
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3051
3051
|
packageCapabilities: {
|
|
3052
3052
|
updatePreferences: 1
|
|
@@ -29076,6 +29076,26 @@ async function handleRunStop(args) {
|
|
|
29076
29076
|
}
|
|
29077
29077
|
const client2 = new DeeplineClient();
|
|
29078
29078
|
const result = await client2.runs.stop(runId, { reason });
|
|
29079
|
+
if (result.cancelling === true) {
|
|
29080
|
+
printCommandEnvelope(
|
|
29081
|
+
{
|
|
29082
|
+
...result,
|
|
29083
|
+
render: {
|
|
29084
|
+
sections: [
|
|
29085
|
+
{
|
|
29086
|
+
title: "run stop",
|
|
29087
|
+
lines: [
|
|
29088
|
+
`Cancellation queued for ${result.runId || runId}; sandbox cleanup is pending`,
|
|
29089
|
+
`inspect: deepline runs get ${runId} --full --json`
|
|
29090
|
+
]
|
|
29091
|
+
}
|
|
29092
|
+
]
|
|
29093
|
+
}
|
|
29094
|
+
},
|
|
29095
|
+
{ json: argsWantJson(args) }
|
|
29096
|
+
);
|
|
29097
|
+
return 0;
|
|
29098
|
+
}
|
|
29079
29099
|
const stopNotConfirmed = result.stopped === false || result.staleSchedulerState === true;
|
|
29080
29100
|
if (stopNotConfirmed) {
|
|
29081
29101
|
const detail = typeof result.error === "string" && result.error.trim() ? result.error.trim() : result.staleSchedulerState === true ? "scheduler state for the run is stale" : "the server did not confirm the stop";
|
|
@@ -29190,6 +29210,7 @@ async function handleRunStopAll(args) {
|
|
|
29190
29210
|
const result = await client2.runs.stopAll({ reason });
|
|
29191
29211
|
const lines = [
|
|
29192
29212
|
`Stopped ${result.stopped} active run${result.stopped === 1 ? "" : "s"}`,
|
|
29213
|
+
...result.cancelling ? [`cancellation queued: ${result.cancelling}`] : [],
|
|
29193
29214
|
...result.partial ? ["partial: active runs may remain; rerun stop-all"] : [],
|
|
29194
29215
|
...result.failed > 0 ? [`failed stops: ${result.failed}`] : [],
|
|
29195
29216
|
...result.skipped > 0 ? [`skipped: ${result.skipped}`] : []
|
package/dist/index.d.mts
CHANGED
|
@@ -1936,8 +1936,8 @@ interface StopPlayRunResult {
|
|
|
1936
1936
|
}
|
|
1937
1937
|
/**
|
|
1938
1938
|
* Result of an actual workspace stop-all. `stopped` counts only runs whose
|
|
1939
|
-
* scheduler cancellation was confirmed;
|
|
1940
|
-
*
|
|
1939
|
+
* scheduler cancellation was confirmed; queued requests are counted in
|
|
1940
|
+
* `cancelling`, while requests that could not be queued count as `failed`.
|
|
1941
1941
|
*/
|
|
1942
1942
|
interface StopAllPlayRunsStopResult {
|
|
1943
1943
|
stopped: number;
|
package/dist/index.d.ts
CHANGED
|
@@ -1936,8 +1936,8 @@ interface StopPlayRunResult {
|
|
|
1936
1936
|
}
|
|
1937
1937
|
/**
|
|
1938
1938
|
* Result of an actual workspace stop-all. `stopped` counts only runs whose
|
|
1939
|
-
* scheduler cancellation was confirmed;
|
|
1940
|
-
*
|
|
1939
|
+
* scheduler cancellation was confirmed; queued requests are counted in
|
|
1940
|
+
* `cancelling`, while requests that could not be queued count as `failed`.
|
|
1941
1941
|
*/
|
|
1942
1942
|
interface StopAllPlayRunsStopResult {
|
|
1943
1943
|
stopped: number;
|
package/dist/index.js
CHANGED
|
@@ -864,7 +864,7 @@ var SDK_RELEASE = {
|
|
|
864
864
|
// getters keep their established compatibility behavior.
|
|
865
865
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
866
866
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
867
|
-
version: "0.3.
|
|
867
|
+
version: "0.3.201",
|
|
868
868
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
869
869
|
packageCapabilities: {
|
|
870
870
|
updatePreferences: 1
|
package/dist/index.mjs
CHANGED
|
@@ -768,7 +768,7 @@ var SDK_RELEASE = {
|
|
|
768
768
|
// getters keep their established compatibility behavior.
|
|
769
769
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
770
770
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
771
|
-
version: "0.3.
|
|
771
|
+
version: "0.3.201",
|
|
772
772
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
773
773
|
packageCapabilities: {
|
|
774
774
|
updatePreferences: 1
|
package/dist/release.d.mts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.201";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.d.ts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.201";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.js
CHANGED
|
@@ -74,7 +74,7 @@ var SDK_RELEASE = {
|
|
|
74
74
|
// getters keep their established compatibility behavior.
|
|
75
75
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
76
76
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
77
|
-
version: "0.3.
|
|
77
|
+
version: "0.3.201",
|
|
78
78
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
79
79
|
packageCapabilities: {
|
|
80
80
|
updatePreferences: 1
|
package/dist/release.mjs
CHANGED
|
@@ -48,7 +48,7 @@ var SDK_RELEASE = {
|
|
|
48
48
|
// getters keep their established compatibility behavior.
|
|
49
49
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
50
50
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
51
|
-
version: "0.3.
|
|
51
|
+
version: "0.3.201",
|
|
52
52
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
53
53
|
packageCapabilities: {
|
|
54
54
|
updatePreferences: 1
|