@shipfox/api-runners 10.2.0 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +43 -0
- package/dist/db/job-executions.d.ts +10 -5
- package/dist/db/job-executions.d.ts.map +1 -1
- package/dist/db/job-executions.js +59 -23
- package/dist/db/job-executions.js.map +1 -1
- package/dist/db/rate-limits.d.ts +6 -15
- package/dist/db/rate-limits.d.ts.map +1 -1
- package/dist/db/rate-limits.js +17 -18
- package/dist/db/rate-limits.js.map +1 -1
- package/dist/presentation/routes/rate-limit.d.ts.map +1 -1
- package/dist/presentation/routes/rate-limit.js +33 -54
- package/dist/presentation/routes/rate-limit.js.map +1 -1
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.js +3 -3
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +13 -13
- package/src/db/job-executions.test.ts +270 -22
- package/src/db/job-executions.ts +101 -32
- package/src/db/rate-limits.ts +30 -42
- package/src/presentation/routes/heartbeat.test.ts +3 -3
- package/src/presentation/routes/list-active-runners.test.ts +8 -2
- package/src/presentation/routes/manual-registration-tokens.test.ts +8 -2
- package/src/presentation/routes/provisioner-tokens.test.ts +8 -2
- package/src/presentation/routes/rate-limit.ts +33 -64
- package/src/presentation/subscribers/on-workflows-job-execution-timed-out.test.ts +16 -0
- package/src/presentation/subscribers/on-workflows-job-execution-timed-out.ts +3 -3
- package/tsconfig.build.tsbuildinfo +1 -1
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
$ shipfox-swc && shipfox-temporal-bundle dist/temporal/workflows/index.js
|
|
2
|
-
Successfully compiled: 101 files with swc (
|
|
3
|
-
2026-
|
|
2
|
+
Successfully compiled: 101 files with swc (538.03ms)
|
|
3
|
+
2026-08-04T08:17:16.817Z [INFO] asset workflow-bundle-0c3da2487e29e352d64e.js 3.17 MiB [emitted] [immutable] (name: main)
|
|
4
4
|
orphan modules 33.5 KiB [orphan] 21 modules
|
|
5
5
|
runtime modules 1.13 KiB 5 modules
|
|
6
6
|
modules by path ../../../node_modules/.pnpm/ 970 KiB 196 modules
|
|
@@ -14,6 +14,6 @@ optional modules 30 bytes [optional]
|
|
|
14
14
|
__temporal_custom_payload_converter (ignored) 15 bytes [optional] [built] [code generated]
|
|
15
15
|
__temporal_custom_failure_converter (ignored) 15 bytes [optional] [built] [code generated]
|
|
16
16
|
../../shared/node/temporal/dist/workflow-error-interceptor.js 1.21 KiB [built] [code generated]
|
|
17
|
-
webpack 5.107.2 compiled successfully in
|
|
18
|
-
2026-
|
|
17
|
+
webpack 5.107.2 compiled successfully in 3353 ms
|
|
18
|
+
2026-08-04T08:17:16.821Z [INFO] Workflow bundle created { size: '3.17MB' }
|
|
19
19
|
/home/runner/work/shipfox/shipfox/libs/api/runners/dist/temporal/workflows/index.js: 3326598 bytes
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,48 @@
|
|
|
1
1
|
# @shipfox/api-runners
|
|
2
2
|
|
|
3
|
+
## 12.0.0
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- f78740d: Remove Unicode dash punctuation from package prose and source comments.
|
|
8
|
+
- 9ebc5b4: Add an authenticated, rate-limited workspace slug availability endpoint.
|
|
9
|
+
- d3222a9: Atomically reconcile pending and running runner state when a job execution reaches a terminal state.
|
|
10
|
+
- Updated dependencies [ee2ce67]
|
|
11
|
+
- Updated dependencies [f7939c7]
|
|
12
|
+
- Updated dependencies [f78740d]
|
|
13
|
+
- Updated dependencies [dea1ffd]
|
|
14
|
+
- Updated dependencies [adf07e7]
|
|
15
|
+
- Updated dependencies [285fff2]
|
|
16
|
+
- Updated dependencies [e44a279]
|
|
17
|
+
- Updated dependencies [f13e8bb]
|
|
18
|
+
- Updated dependencies [9ebc5b4]
|
|
19
|
+
- Updated dependencies [54c820e]
|
|
20
|
+
- Updated dependencies [35a42bd]
|
|
21
|
+
- Updated dependencies [d77baaa]
|
|
22
|
+
- Updated dependencies [34a5639]
|
|
23
|
+
- Updated dependencies [032d316]
|
|
24
|
+
- Updated dependencies [c2a8e54]
|
|
25
|
+
- Updated dependencies [cb0abfa]
|
|
26
|
+
- @shipfox/api-workflows-dto@12.0.0
|
|
27
|
+
- @shipfox/api-auth-dto@12.0.0
|
|
28
|
+
- @shipfox/api-common-dto@12.0.0
|
|
29
|
+
- @shipfox/inter-module@0.2.3
|
|
30
|
+
- @shipfox/node-fastify@0.4.1
|
|
31
|
+
- @shipfox/node-module@1.0.5
|
|
32
|
+
- @shipfox/node-postgres@0.5.0
|
|
33
|
+
- @shipfox/node-rate-limit@0.4.0
|
|
34
|
+
- @shipfox/api-auth-context@12.0.0
|
|
35
|
+
- @shipfox/api-runners-dto@12.0.0
|
|
36
|
+
- @shipfox/node-drizzle@0.3.5
|
|
37
|
+
- @shipfox/node-outbox@0.2.6
|
|
38
|
+
|
|
39
|
+
## 11.0.0
|
|
40
|
+
|
|
41
|
+
### Patch Changes
|
|
42
|
+
|
|
43
|
+
- Updated dependencies [25158c8]
|
|
44
|
+
- @shipfox/api-auth-context@11.0.0
|
|
45
|
+
|
|
3
46
|
## 10.2.0
|
|
4
47
|
|
|
5
48
|
### Minor Changes
|
|
@@ -59,7 +59,7 @@ export declare function claimPendingJobExecution(params: {
|
|
|
59
59
|
* Releases a job execution's lease when the orchestration workflow finalizes it: deletes the
|
|
60
60
|
* running-job-execution row AND any lingering pending row for the same execution, in one tx.
|
|
61
61
|
* Idempotent (0-row no-op), no token scope (the workflow is authoritative), and
|
|
62
|
-
* emits no event
|
|
62
|
+
* emits no event: the workflow already owns the outcome. Sweeping the pending row
|
|
63
63
|
* too closes the at-least-once window where an enqueue retry left an orphan that a
|
|
64
64
|
* later claim would otherwise pick up for an already-finished job execution.
|
|
65
65
|
*/
|
|
@@ -75,9 +75,9 @@ export declare function releaseJobExecution(params: {
|
|
|
75
75
|
* pending row: a failed best-effort `releaseJobExecution` would otherwise leave an orphan
|
|
76
76
|
* that a later claim re-runs as an already-finished job execution.
|
|
77
77
|
*
|
|
78
|
-
* Locks
|
|
79
|
-
*
|
|
80
|
-
*
|
|
78
|
+
* Locks pending-then-running to match `claimPendingJobExecution`, `releaseJobExecution`, and
|
|
79
|
+
* `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock running
|
|
80
|
+
* rows first; the running-row DELETE re-checks the stale predicate after the pending-row sweep.
|
|
81
81
|
*/
|
|
82
82
|
export declare function expireStuckJobExecutions(params: {
|
|
83
83
|
thresholdSeconds: number;
|
|
@@ -126,7 +126,12 @@ export declare function recordHeartbeat(params: {
|
|
|
126
126
|
runnerSessionId: string;
|
|
127
127
|
};
|
|
128
128
|
}>;
|
|
129
|
-
|
|
129
|
+
/**
|
|
130
|
+
* Reconciles a terminal job execution with runner state in one transaction: removes its pending
|
|
131
|
+
* queue row and requests cancellation on its running lease, if either exists. The operation is
|
|
132
|
+
* idempotent and preserves the first cancellation-request timestamp.
|
|
133
|
+
*/
|
|
134
|
+
export declare function reconcileTerminalJobExecution(params: {
|
|
130
135
|
jobExecutionId: string;
|
|
131
136
|
}): Promise<void>;
|
|
132
137
|
export declare function cancelRunnerJobs(params: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-executions.d.ts","sourceRoot":"","sources":["../../src/db/job-executions.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,yBAAyB,EAC/B,MAAM,0BAA0B,CAAC;AA0BlC,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"job-executions.d.ts","sourceRoot":"","sources":["../../src/db/job-executions.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,yBAAyB,EAC/B,MAAM,0BAA0B,CAAC;AA0BlC,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAgBhC,MAAM,WAAW,yBAAyB;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,GAAG,OAAO,CAAC,KAAK,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,CAAC,CAAC,CAwBzE;AAOD,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAoD1F;AAED,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,eAAe,EAAE,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,+BAA+B;IAC9C,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,eAAe,EAAE,IAAI,CAAC;IACtB,uBAAuB,EAAE,IAAI,GAAG,IAAI,CAAC;CACtC;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,oCAAoC,EAAE,MAAM,CAAC;CAC9C,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAwJtC;AAmBD;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CAAC,MAAM,EAAE;IAAC,cAAc,EAAE,MAAM,CAAA;CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAezF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,4BAA4B,EAAE,MAAM,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CACT,KAAK,CAAC;IACJ,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CACH,CAwFA;AAED,wBAAsB,yBAAyB,IAAI,OAAO,CAAC;IACzD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC,CAOD;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAyBvC;AAED,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GACA,OAAO,CAAC,+BAA+B,EAAE,CAAC,CA2E5C;AAMD,wBAAsB,gBAAgB,CAAC,MAAM,EAAE;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAED,wBAAsB,eAAe,CAAC,MAAM,EAAE;IAC5C,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,yBAAyB,GAAG,IAAI,CAAC;CACrD,GAAG,OAAO,CAAC;IACV,qBAAqB,EAAE,OAAO,CAAC;IAC/B,wBAAwB,EAAE,yBAAyB,GAAG,IAAI,CAAC;IAC3D,uBAAuB,EAAE,yBAAyB,GAAG,IAAI,CAAC;IAC1D,mBAAmB,EAAE;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH,CAAC,CAoED;AAED;;;;GAIG;AACH,wBAAsB,6BAA6B,CAAC,MAAM,EAAE;IAC1D,cAAc,EAAE,MAAM,CAAC;CACxB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiBhB;AAED,wBAAsB,gBAAgB,CAAC,MAAM,EAAE;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAYhF"}
|
|
@@ -11,6 +11,10 @@ import { pendingJobExecutions } from './schema/pending-job-executions.js';
|
|
|
11
11
|
import { providerRunners } from './schema/runner-instances.js';
|
|
12
12
|
import { runnerSessions } from './schema/runner-sessions.js';
|
|
13
13
|
import { runningJobExecutions } from './schema/running-job-executions.js';
|
|
14
|
+
const runnerJobExecutionLockPrefix = 'runners_job_execution:';
|
|
15
|
+
async function lockJobExecution(tx, jobExecutionId) {
|
|
16
|
+
await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`);
|
|
17
|
+
}
|
|
14
18
|
export async function getWorkspaceJobCounts(params) {
|
|
15
19
|
const [pendingRows, runningRows] = await Promise.all([
|
|
16
20
|
db().select({
|
|
@@ -37,18 +41,21 @@ export async function getWorkspaceJobCounts(params) {
|
|
|
37
41
|
}));
|
|
38
42
|
}
|
|
39
43
|
// Idempotent while the job execution is still pending: a duplicate jobExecutionId already in
|
|
40
|
-
// `runners_pending_jobs` is a no-op. Temporal retries the enqueue activity
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
//
|
|
44
|
-
// can reinsert an orphan pending row, which a later `claimPendingJobExecution` drops
|
|
45
|
-
// via the running-execution unique constraint (onConflictDoNothing) instead of failing.
|
|
44
|
+
// `runners_pending_jobs` is a no-op. Temporal retries the enqueue activity at-least-once, so a
|
|
45
|
+
// unique-violation throw on a retry-after-lost-result would permanently fail a healthy execution.
|
|
46
|
+
// The per-execution advisory lock serializes retries with lease expiry/reconciliation, and the
|
|
47
|
+
// durable lease-expired event prevents a retry from re-queueing an execution already reaped.
|
|
46
48
|
export async function enqueueJobExecution(params) {
|
|
47
49
|
const requiredLabels = [
|
|
48
50
|
...canonicalizeLabels(params.requiredLabels)
|
|
49
51
|
];
|
|
50
52
|
if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();
|
|
51
53
|
const enqueued = await db().transaction(async (tx)=>{
|
|
54
|
+
await lockJobExecution(tx, params.jobExecutionId);
|
|
55
|
+
const [leaseExpired] = await tx.select({
|
|
56
|
+
id: runnersOutbox.id
|
|
57
|
+
}).from(runnersOutbox).where(and(eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED), sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`)).limit(1);
|
|
58
|
+
if (leaseExpired) return false;
|
|
52
59
|
const [inserted] = await tx.insert(pendingJobExecutions).values({
|
|
53
60
|
workspaceId: params.workspaceId,
|
|
54
61
|
workflowRunId: params.workflowRunId,
|
|
@@ -110,10 +117,17 @@ export async function claimPendingJobExecution(params) {
|
|
|
110
117
|
providerRunnerId = session.providerRunnerId;
|
|
111
118
|
}
|
|
112
119
|
// `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker
|
|
113
|
-
// for rows sharing a created_at within a batch.
|
|
114
|
-
|
|
120
|
+
// for rows sharing a created_at within a batch. Lock only the FIFO candidate before
|
|
121
|
+
// attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this
|
|
122
|
+
// predicate would evaluate it while scanning and temporarily lock many queue entries.
|
|
123
|
+
const candidate = tx.select().from(pendingJobExecutions).where(and(eq(pendingJobExecutions.workspaceId, params.workspaceId), arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels))).orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id)).limit(1).for('update', {
|
|
115
124
|
skipLocked: true
|
|
116
|
-
});
|
|
125
|
+
}).as('pending_candidate');
|
|
126
|
+
const [row] = await tx.select().from(candidate).where(sql`
|
|
127
|
+
pg_try_advisory_xact_lock(
|
|
128
|
+
hashtext(${runnerJobExecutionLockPrefix} || ${candidate.jobExecutionId}::text)
|
|
129
|
+
)
|
|
130
|
+
`);
|
|
117
131
|
if (!row) return null;
|
|
118
132
|
await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));
|
|
119
133
|
// An enqueue retry that lands after a prior claim can leave an orphan pending
|
|
@@ -185,11 +199,12 @@ async function touchRunnerSessionLiveness(params) {
|
|
|
185
199
|
* Releases a job execution's lease when the orchestration workflow finalizes it: deletes the
|
|
186
200
|
* running-job-execution row AND any lingering pending row for the same execution, in one tx.
|
|
187
201
|
* Idempotent (0-row no-op), no token scope (the workflow is authoritative), and
|
|
188
|
-
* emits no event
|
|
202
|
+
* emits no event: the workflow already owns the outcome. Sweeping the pending row
|
|
189
203
|
* too closes the at-least-once window where an enqueue retry left an orphan that a
|
|
190
204
|
* later claim would otherwise pick up for an already-finished job execution.
|
|
191
205
|
*/ export async function releaseJobExecution(params) {
|
|
192
206
|
await db().transaction(async (tx)=>{
|
|
207
|
+
await lockJobExecution(tx, params.jobExecutionId);
|
|
193
208
|
// Delete pending before running to match `claimPendingJobExecution`'s lock-acquisition
|
|
194
209
|
// order (it locks the pending row first, then the running row). A concurrent
|
|
195
210
|
// claim picking up an orphan pending row for this same job execution would otherwise
|
|
@@ -207,19 +222,30 @@ async function touchRunnerSessionLiveness(params) {
|
|
|
207
222
|
* pending row: a failed best-effort `releaseJobExecution` would otherwise leave an orphan
|
|
208
223
|
* that a later claim re-runs as an already-finished job execution.
|
|
209
224
|
*
|
|
210
|
-
* Locks
|
|
211
|
-
*
|
|
212
|
-
*
|
|
225
|
+
* Locks pending-then-running to match `claimPendingJobExecution`, `releaseJobExecution`, and
|
|
226
|
+
* `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock running
|
|
227
|
+
* rows first; the running-row DELETE re-checks the stale predicate after the pending-row sweep.
|
|
213
228
|
*/ export async function expireStuckJobExecutions(params) {
|
|
214
229
|
const reaped = await db().transaction(async (tx)=>{
|
|
215
230
|
const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} || ' seconds')::interval`;
|
|
216
231
|
const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} || ' seconds')::interval`;
|
|
217
232
|
const stalePredicate = or(and(isNull(runningJobExecutions.firstHeartbeatAt), lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt), lt(runningJobExecutions.startedAt, firstHeartbeatCutoff)), and(or(isNotNull(runningJobExecutions.firstHeartbeatAt), gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt)), lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff)));
|
|
218
|
-
const
|
|
219
|
-
id: runningJobExecutions.id
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
233
|
+
const staleRows = await tx.select({
|
|
234
|
+
id: runningJobExecutions.id,
|
|
235
|
+
jobExecutionId: runningJobExecutions.jobExecutionId
|
|
236
|
+
}).from(runningJobExecutions).where(and(stalePredicate, sql`
|
|
237
|
+
pg_try_advisory_xact_lock(
|
|
238
|
+
hashtext(${runnerJobExecutionLockPrefix} || ${runningJobExecutions.jobExecutionId}::text)
|
|
239
|
+
)
|
|
240
|
+
`)).orderBy(asc(sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`), asc(runningJobExecutions.id)).limit(params.limit ?? 100);
|
|
241
|
+
if (staleRows.length === 0) return [];
|
|
242
|
+
const staleIds = staleRows.map((row)=>row.id);
|
|
243
|
+
const staleJobExecutionIds = staleRows.map((row)=>row.jobExecutionId);
|
|
244
|
+
// Sweep pending rows first so this transaction cannot hold a running-row lock while waiting
|
|
245
|
+
// for a pending-row lock held by reconciliation or a claim of an orphan pending row. The
|
|
246
|
+
// advisory lock acquired in the candidate scan also makes enqueue retries wait until this
|
|
247
|
+
// transaction has either reaped the execution or released its lock without selecting it.
|
|
248
|
+
await tx.delete(pendingJobExecutions).where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));
|
|
223
249
|
const deleted = await tx.delete(runningJobExecutions).where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate)).returning({
|
|
224
250
|
workflowRunId: runningJobExecutions.workflowRunId,
|
|
225
251
|
workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,
|
|
@@ -227,7 +253,6 @@ async function touchRunnerSessionLiveness(params) {
|
|
|
227
253
|
jobExecutionId: runningJobExecutions.jobExecutionId
|
|
228
254
|
});
|
|
229
255
|
if (deleted.length === 0) return [];
|
|
230
|
-
await tx.delete(pendingJobExecutions).where(inArray(pendingJobExecutions.jobExecutionId, deleted.map((row)=>row.jobExecutionId)));
|
|
231
256
|
await writeOutboxEvents(tx, runnersOutbox, deleted.map((row)=>({
|
|
232
257
|
type: RUNNER_JOB_LEASE_EXPIRED,
|
|
233
258
|
payload: {
|
|
@@ -373,10 +398,21 @@ export async function recordHeartbeat(params) {
|
|
|
373
398
|
}
|
|
374
399
|
};
|
|
375
400
|
}
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
401
|
+
/**
|
|
402
|
+
* Reconciles a terminal job execution with runner state in one transaction: removes its pending
|
|
403
|
+
* queue row and requests cancellation on its running lease, if either exists. The operation is
|
|
404
|
+
* idempotent and preserves the first cancellation-request timestamp.
|
|
405
|
+
*/ export async function reconcileTerminalJobExecution(params) {
|
|
406
|
+
await db().transaction(async (tx)=>{
|
|
407
|
+
await lockJobExecution(tx, params.jobExecutionId);
|
|
408
|
+
// Delete pending before updating running to match claim/release lock order. Claim locks
|
|
409
|
+
// pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes
|
|
410
|
+
// a concurrent terminal reconciliation either win before claim or cancel the new lease.
|
|
411
|
+
await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));
|
|
412
|
+
await tx.update(runningJobExecutions).set({
|
|
413
|
+
cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`
|
|
414
|
+
}).where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));
|
|
415
|
+
});
|
|
380
416
|
}
|
|
381
417
|
export async function cancelRunnerJobs(params) {
|
|
382
418
|
if (params.jobIds.length === 0) return;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/db/job-executions.ts"],"sourcesContent":["import {\n RUNNER_JOB_CLAIMED,\n RUNNER_JOB_LEASE_EXPIRED,\n RUNNER_JOB_QUEUED,\n type RunnersEventMap,\n type RunnerToolCapabilitiesDto,\n} from '@shipfox/api-runners-dto';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n and,\n arrayContained,\n asc,\n count,\n desc,\n eq,\n gt,\n inArray,\n isNotNull,\n isNull,\n lt,\n lte,\n or,\n sql,\n} from 'drizzle-orm';\nimport {\n EmptyRequiredLabelsError,\n RunnerSessionExhaustedError,\n RunningJobExecutionNotFoundError,\n} from '#core/errors.js';\nimport {jobExecutionEnqueuedCount, jobExecutionLeaseExpiredCount} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {runnersOutbox} from './schema/outbox.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {providerRunners} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nexport interface EnqueueJobExecutionParams {\n workspaceId: string;\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n requiredLabels: string[];\n}\n\nexport async function getWorkspaceJobCounts(params: {\n workspaceIds: string[];\n}): Promise<Array<{workspaceId: string; queued: number; running: number}>> {\n const [pendingRows, runningRows] = await Promise.all([\n db()\n .select({workspaceId: pendingJobExecutions.workspaceId, count: count()})\n .from(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.workspaceId, params.workspaceIds))\n .groupBy(pendingJobExecutions.workspaceId),\n db()\n .select({workspaceId: runningJobExecutions.workspaceId, count: count()})\n .from(runningJobExecutions)\n .where(inArray(runningJobExecutions.workspaceId, params.workspaceIds))\n .groupBy(runningJobExecutions.workspaceId),\n ]);\n\n const queuedByWorkspace = new Map(pendingRows.map((row) => [row.workspaceId, Number(row.count)]));\n const runningByWorkspace = new Map(\n runningRows.map((row) => [row.workspaceId, Number(row.count)]),\n );\n\n return params.workspaceIds.map((workspaceId) => ({\n workspaceId,\n queued: queuedByWorkspace.get(workspaceId) ?? 0,\n running: runningByWorkspace.get(workspaceId) ?? 0,\n }));\n}\n\n// Idempotent while the job execution is still pending: a duplicate jobExecutionId already in\n// `runners_pending_jobs` is a no-op. Temporal retries the enqueue activity\n// at-least-once, so a unique-violation throw on a retry-after-lost-result\n// would permanently fail a healthy job execution's workflow. The guard does not extend\n// past the claim: once the job execution has moved to `runners_running_jobs`, a retry\n// can reinsert an orphan pending row, which a later `claimPendingJobExecution` drops\n// via the running-execution unique constraint (onConflictDoNothing) instead of failing.\nexport async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {\n const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];\n if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();\n\n const enqueued = await db().transaction(async (tx) => {\n const [inserted] = await tx\n .insert(pendingJobExecutions)\n .values({\n workspaceId: params.workspaceId,\n workflowRunId: params.workflowRunId,\n workflowRunAttemptId: params.workflowRunAttemptId,\n jobId: params.jobId,\n jobExecutionId: params.jobExecutionId,\n projectId: params.projectId,\n requiredLabels,\n })\n .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})\n .returning({createdAt: pendingJobExecutions.createdAt});\n\n // A retry that hits the conflict inserts nothing: the first enqueue already\n // emitted the queued event (durably, in the outbox), so re-emitting would\n // only add a redundant row the subscriber coalesces away. Skip it.\n if (!inserted) return false;\n\n await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n type: RUNNER_JOB_QUEUED,\n payload: {\n workflowRunId: params.workflowRunId,\n workflowRunAttemptId: params.workflowRunAttemptId,\n jobId: params.jobId,\n jobExecutionId: params.jobExecutionId,\n queuedAt: inserted.createdAt.toISOString(),\n },\n });\n return true;\n });\n\n if (enqueued) jobExecutionEnqueuedCount.add(1);\n}\n\nexport interface ClaimedJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n}\n\nexport interface ActiveRunningJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n runnerSessionId: string;\n provisionerId: string | null;\n providerRunnerId: string | null;\n requiredLabels: string[];\n runnerLabels: string[];\n startedAt: Date;\n lastHeartbeatAt: Date;\n}\n\nexport interface RunnerInstanceBoundJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n providerRunnerId: string;\n startedAt: Date;\n lastHeartbeatAt: Date;\n cancellationRequestedAt: Date | null;\n}\n\nexport async function claimPendingJobExecution(params: {\n workspaceId: string;\n runnerSessionId: string;\n sessionLabels: string[];\n maxClaims: number | null;\n runnerSessionLivenessThrottleSeconds: number;\n}): Promise<ClaimedJobExecution | null> {\n await touchRunnerSessionLiveness({\n workspaceId: params.workspaceId,\n runnerSessionId: params.runnerSessionId,\n throttleSeconds: params.runnerSessionLivenessThrottleSeconds,\n });\n\n if (params.sessionLabels.length === 0) return null;\n\n return await db().transaction(async (tx) => {\n let provisionerId: string | null = null;\n let providerRunnerId: string | null = null;\n let runnerInstanceId: string | null = null;\n\n if (params.maxClaims !== null) {\n const [session] = await tx\n .select({\n maxClaims: runnerSessions.maxClaims,\n claimsUsed: runnerSessions.claimsUsed,\n revokedAt: runnerSessions.revokedAt,\n runnerInstanceId: runnerSessions.runnerInstanceId,\n provisionerId: runnerSessions.provisionerId,\n providerRunnerId: runnerSessions.providerRunnerId,\n })\n .from(runnerSessions)\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .limit(1)\n .for('update');\n\n if (\n !session ||\n session.revokedAt ||\n session.maxClaims === null ||\n session.claimsUsed >= session.maxClaims\n ) {\n throw new RunnerSessionExhaustedError(params.runnerSessionId);\n }\n\n // Ephemeral sessions are the only capped sessions, and the DB check keeps\n // their provisioned-runner link present as a pair.\n runnerInstanceId = session.runnerInstanceId;\n provisionerId = session.provisionerId;\n providerRunnerId = session.providerRunnerId;\n }\n\n // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker\n // for rows sharing a created_at within a batch.\n const [row] = await tx\n .select()\n .from(pendingJobExecutions)\n .where(\n and(\n eq(pendingJobExecutions.workspaceId, params.workspaceId),\n arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels),\n ),\n )\n .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))\n .limit(1)\n .for('update', {skipLocked: true});\n\n if (!row) return null;\n\n await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));\n\n // An enqueue retry that lands after a prior claim can leave an orphan pending\n // row whose jobExecutionId is already in `runners_running_jobs`. Insert-or-skip\n // on the jobExecutionId unique constraint: when the job execution is already running\n // the insert touches no row, so we commit the orphan's deletion and return\n // null rather than let the unique violation roll the claim back into a poison\n // loop. The runner just re-polls for a real job execution.\n const inserted = await tx\n .insert(runningJobExecutions)\n .values({\n workspaceId: row.workspaceId,\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n runnerSessionId: params.runnerSessionId,\n provisionerId,\n providerRunnerId,\n requiredLabels: row.requiredLabels,\n runnerLabels: params.sessionLabels,\n })\n .onConflictDoNothing({target: runningJobExecutions.jobExecutionId})\n .returning({claimedAt: runningJobExecutions.startedAt});\n\n const claimed = inserted[0];\n if (!claimed) return null;\n\n const runnerInstanceCondition = runnerInstanceId\n ? eq(providerRunners.id, runnerInstanceId)\n : provisionerId && providerRunnerId\n ? and(\n eq(providerRunners.provisionerId, provisionerId),\n eq(providerRunners.providerRunnerId, providerRunnerId),\n )\n : null;\n if (runnerInstanceCondition) {\n await tx\n .update(providerRunners)\n .set({\n firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,\n updatedAt: sql`now()`,\n })\n .where(runnerInstanceCondition);\n }\n\n if (params.maxClaims !== null) {\n await tx\n .update(runnerSessions)\n .set({claimsUsed: sql`${runnerSessions.claimsUsed} + 1`, updatedAt: sql`now()`})\n .where(eq(runnerSessions.id, params.runnerSessionId));\n }\n\n // The running-row insert is the runner claiming the job execution. Emit in the same tx; the\n // payload carries the row's own claim instant so a consumer records the true time,\n // not the outbox drain time.\n await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n type: RUNNER_JOB_CLAIMED,\n payload: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n claimedAt: claimed.claimedAt.toISOString(),\n },\n });\n\n return {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n };\n });\n}\n\nasync function touchRunnerSessionLiveness(params: {\n workspaceId: string;\n runnerSessionId: string;\n throttleSeconds: number;\n}): Promise<void> {\n await db()\n .update(runnerSessions)\n .set({updatedAt: sql`now()`})\n .where(\n and(\n eq(runnerSessions.id, params.runnerSessionId),\n eq(runnerSessions.workspaceId, params.workspaceId),\n sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} || ' seconds')::interval`,\n ),\n );\n}\n\n/**\n * Releases a job execution's lease when the orchestration workflow finalizes it: deletes the\n * running-job-execution row AND any lingering pending row for the same execution, in one tx.\n * Idempotent (0-row no-op), no token scope (the workflow is authoritative), and\n * emits no event — the workflow already owns the outcome. Sweeping the pending row\n * too closes the at-least-once window where an enqueue retry left an orphan that a\n * later claim would otherwise pick up for an already-finished job execution.\n */\nexport async function releaseJobExecution(params: {jobExecutionId: string}): Promise<void> {\n await db().transaction(async (tx) => {\n // Delete pending before running to match `claimPendingJobExecution`'s lock-acquisition\n // order (it locks the pending row first, then the running row). A concurrent\n // claim picking up an orphan pending row for this same job execution would otherwise\n // deadlock against the reverse order here.\n await tx\n .delete(pendingJobExecutions)\n .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n await tx\n .delete(runningJobExecutions)\n .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));\n });\n}\n\n/**\n * Reaps stale leases (bounded by `limit`), emitting one\n * `runners.job.lease_expired` event per reaped job execution.\n *\n * The cutoff is re-checked in the DELETE, not just the locking subquery, so a\n * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its\n * pending row: a failed best-effort `releaseJobExecution` would otherwise leave an orphan\n * that a later claim re-runs as an already-finished job execution.\n *\n * Locks running-then-pending, the inverse of `claimPendingJobExecution` / `releaseJobExecution`.\n * That pre-existing asymmetry opens a narrow deadlock window against a concurrent\n * claim of the same orphan-pending job execution; Postgres breaks it and the cron retries.\n */\nexport async function expireStuckJobExecutions(params: {\n thresholdSeconds: number;\n noFirstHeartbeatGraceSeconds: number;\n limit?: number;\n}): Promise<\n Array<{\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n }>\n> {\n const reaped = await db().transaction(async (tx) => {\n const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} || ' seconds')::interval`;\n const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} || ' seconds')::interval`;\n const stalePredicate = or(\n and(\n isNull(runningJobExecutions.firstHeartbeatAt),\n lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n lt(runningJobExecutions.startedAt, firstHeartbeatCutoff),\n ),\n and(\n or(\n isNotNull(runningJobExecutions.firstHeartbeatAt),\n gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n ),\n lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff),\n ),\n );\n\n const staleIds = tx\n .select({id: runningJobExecutions.id})\n .from(runningJobExecutions)\n .where(stalePredicate)\n .orderBy(\n asc(\n sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,\n ),\n )\n .limit(params.limit ?? 100)\n .for('update', {skipLocked: true});\n\n const deleted = await tx\n .delete(runningJobExecutions)\n .where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate))\n .returning({\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n });\n\n if (deleted.length === 0) return [];\n\n await tx.delete(pendingJobExecutions).where(\n inArray(\n pendingJobExecutions.jobExecutionId,\n deleted.map((row) => row.jobExecutionId),\n ),\n );\n\n await writeOutboxEvents<RunnersEventMap>(\n tx,\n runnersOutbox,\n deleted.map((row) => ({\n type: RUNNER_JOB_LEASE_EXPIRED,\n payload: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n },\n })),\n );\n\n return deleted;\n });\n\n if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);\n\n return reaped;\n}\n\nexport async function getJobExecutionQueueDepth(): Promise<{\n pendingJobExecutions: number;\n runningJobExecutions: number;\n}> {\n const [pending] = await db().select({value: count()}).from(pendingJobExecutions);\n const [running] = await db().select({value: count()}).from(runningJobExecutions);\n return {\n pendingJobExecutions: pending?.value ?? 0,\n runningJobExecutions: running?.value ?? 0,\n };\n}\n\nexport async function listActiveRunningJobExecutions(params: {\n workspaceId: string;\n windowSeconds: number;\n limit?: number;\n}): Promise<ActiveRunningJobExecution[]> {\n return await db()\n .select({\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n projectId: runningJobExecutions.projectId,\n runnerSessionId: runningJobExecutions.runnerSessionId,\n provisionerId: runningJobExecutions.provisionerId,\n providerRunnerId: runningJobExecutions.providerRunnerId,\n requiredLabels: runningJobExecutions.requiredLabels,\n runnerLabels: runningJobExecutions.runnerLabels,\n startedAt: runningJobExecutions.startedAt,\n lastHeartbeatAt: runningJobExecutions.lastHeartbeatAt,\n })\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.workspaceId, params.workspaceId),\n sql`${runningJobExecutions.lastHeartbeatAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n ),\n )\n .orderBy(desc(runningJobExecutions.lastHeartbeatAt), desc(runningJobExecutions.id))\n .limit(params.limit ?? 1000);\n}\n\nexport async function listRunningJobExecutionsByRunnerInstanceTx(\n tx: Tx,\n params: {\n workspaceId: string;\n provisionerId: string;\n providerRunnerIds: string[];\n },\n): Promise<RunnerInstanceBoundJobExecution[]> {\n if (params.providerRunnerIds.length === 0) return [];\n\n const duplicateRows = await tx\n .select({\n providerRunnerId: runningJobExecutions.providerRunnerId,\n count: count(),\n })\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.workspaceId, params.workspaceId),\n eq(runningJobExecutions.provisionerId, params.provisionerId),\n inArray(runningJobExecutions.providerRunnerId, params.providerRunnerIds),\n ),\n )\n .groupBy(runningJobExecutions.providerRunnerId)\n .having(sql`count(*) > 1`);\n\n const duplicateRunnerInstanceIds = duplicateRows.flatMap((row) =>\n row.providerRunnerId ? [row.providerRunnerId] : [],\n );\n if (duplicateRunnerInstanceIds.length > 0) {\n logger().warn(\n {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n providerRunnerIds: duplicateRunnerInstanceIds,\n },\n 'multiple running job executions are bound to the same provisioned runner',\n );\n }\n\n const result = await tx.execute<{\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n providerRunnerId: string;\n startedAt: Date | string;\n lastHeartbeatAt: Date | string;\n cancellationRequestedAt: Date | string | null;\n }>(sql`\n SELECT DISTINCT ON (${runningJobExecutions.providerRunnerId})\n ${runningJobExecutions.workflowRunId} AS \"workflowRunId\",\n ${runningJobExecutions.workflowRunAttemptId} AS \"workflowRunAttemptId\",\n ${runningJobExecutions.jobId} AS \"jobId\",\n ${runningJobExecutions.jobExecutionId} AS \"jobExecutionId\",\n ${runningJobExecutions.providerRunnerId} AS \"providerRunnerId\",\n ${runningJobExecutions.startedAt} AS \"startedAt\",\n ${runningJobExecutions.lastHeartbeatAt} AS \"lastHeartbeatAt\",\n ${runningJobExecutions.cancellationRequestedAt} AS \"cancellationRequestedAt\"\n FROM ${runningJobExecutions}\n WHERE\n ${runningJobExecutions.workspaceId} = ${params.workspaceId}\n AND ${runningJobExecutions.provisionerId} = ${params.provisionerId}\n AND ${runningJobExecutions.providerRunnerId} IN (${sql.join(\n params.providerRunnerIds.map((providerRunnerId) => sql`${providerRunnerId}`),\n sql`, `,\n )})\n ORDER BY ${runningJobExecutions.providerRunnerId}, ${runningJobExecutions.startedAt} DESC, ${runningJobExecutions.jobExecutionId} DESC\n `);\n\n return result.rows.map((row) => ({\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n providerRunnerId: row.providerRunnerId,\n startedAt: toDate(row.startedAt),\n lastHeartbeatAt: toDate(row.lastHeartbeatAt),\n cancellationRequestedAt: row.cancellationRequestedAt\n ? toDate(row.cancellationRequestedAt)\n : null,\n }));\n}\n\nfunction toDate(value: Date | string): Date {\n return value instanceof Date ? value : new Date(value);\n}\n\nexport async function isJobLeaseActive(params: {\n jobId?: string;\n jobExecutionId: string;\n runnerSessionId: string;\n}): Promise<boolean> {\n const [row] = await db()\n .select({id: runningJobExecutions.id})\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n params.jobId === undefined ? undefined : eq(runningJobExecutions.jobId, params.jobId),\n ),\n )\n .limit(1);\n\n return row !== undefined;\n}\n\nexport async function recordHeartbeat(params: {\n jobExecutionId: string;\n runnerSessionId: string;\n toolCapabilities?: RunnerToolCapabilitiesDto | null;\n}): Promise<{\n cancellationRequested: boolean;\n previousToolCapabilities: RunnerToolCapabilitiesDto | null;\n currentToolCapabilities: RunnerToolCapabilitiesDto | null;\n runningJobExecution: {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n workspaceId: string;\n runnerSessionId: string;\n };\n}> {\n const result = await db().transaction(async (tx) => {\n const updated = await tx\n .update(runningJobExecutions)\n .set({\n firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,\n lastHeartbeatAt: sql`now()`,\n })\n .where(\n and(\n eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n ),\n )\n .returning({\n cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n projectId: runningJobExecutions.projectId,\n workspaceId: runningJobExecutions.workspaceId,\n runnerSessionId: runningJobExecutions.runnerSessionId,\n });\n\n const row = updated[0];\n if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n\n const [previous] = await tx\n .select({toolCapabilities: runnerSessions.toolCapabilities})\n .from(runnerSessions)\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .limit(1)\n .for('update');\n\n const [session] = await tx\n .update(runnerSessions)\n .set({\n toolCapabilities: params.toolCapabilities ?? null,\n toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,\n updatedAt: sql`now()`,\n })\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .returning({toolCapabilities: runnerSessions.toolCapabilities});\n\n return {\n row,\n previousToolCapabilities: previous?.toolCapabilities ?? null,\n currentToolCapabilities: session?.toolCapabilities ?? null,\n };\n });\n\n const row = result.row;\n if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n return {\n cancellationRequested: row.cancellationRequestedAt !== null,\n previousToolCapabilities: result.previousToolCapabilities,\n currentToolCapabilities: result.currentToolCapabilities,\n runningJobExecution: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n workspaceId: row.workspaceId,\n runnerSessionId: row.runnerSessionId,\n },\n };\n}\n\nexport async function requestJobExecutionCancellation(params: {\n jobExecutionId: string;\n}): Promise<void> {\n await db()\n .update(runningJobExecutions)\n .set({\n cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n })\n .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));\n}\n\nexport async function cancelRunnerJobs(params: {jobIds: string[]}): Promise<void> {\n if (params.jobIds.length === 0) return;\n\n await db().transaction(async (tx) => {\n await tx.delete(pendingJobExecutions).where(inArray(pendingJobExecutions.jobId, params.jobIds));\n await tx\n .update(runningJobExecutions)\n .set({\n cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n })\n .where(inArray(runningJobExecutions.jobId, params.jobIds));\n });\n}\n"],"names":["RUNNER_JOB_CLAIMED","RUNNER_JOB_LEASE_EXPIRED","RUNNER_JOB_QUEUED","logger","writeOutboxEvent","writeOutboxEvents","canonicalizeLabels","and","arrayContained","asc","count","desc","eq","gt","inArray","isNotNull","isNull","lt","lte","or","sql","EmptyRequiredLabelsError","RunnerSessionExhaustedError","RunningJobExecutionNotFoundError","jobExecutionEnqueuedCount","jobExecutionLeaseExpiredCount","db","runnersOutbox","pendingJobExecutions","providerRunners","runnerSessions","runningJobExecutions","getWorkspaceJobCounts","params","pendingRows","runningRows","Promise","all","select","workspaceId","from","where","workspaceIds","groupBy","queuedByWorkspace","Map","map","row","Number","runningByWorkspace","queued","get","running","enqueueJobExecution","requiredLabels","length","enqueued","transaction","tx","inserted","insert","values","workflowRunId","workflowRunAttemptId","jobId","jobExecutionId","projectId","onConflictDoNothing","target","returning","createdAt","type","payload","queuedAt","toISOString","add","claimPendingJobExecution","touchRunnerSessionLiveness","runnerSessionId","throttleSeconds","runnerSessionLivenessThrottleSeconds","sessionLabels","provisionerId","providerRunnerId","runnerInstanceId","maxClaims","session","claimsUsed","revokedAt","id","limit","for","orderBy","skipLocked","delete","runnerLabels","claimedAt","startedAt","claimed","runnerInstanceCondition","update","set","firstClaimedAt","updatedAt","releaseJobExecution","expireStuckJobExecutions","reaped","heartbeatCutoff","thresholdSeconds","firstHeartbeatCutoff","noFirstHeartbeatGraceSeconds","stalePredicate","firstHeartbeatAt","lastHeartbeatAt","staleIds","deleted","getJobExecutionQueueDepth","pending","value","listActiveRunningJobExecutions","windowSeconds","listRunningJobExecutionsByRunnerInstanceTx","providerRunnerIds","duplicateRows","having","duplicateRunnerInstanceIds","flatMap","warn","result","execute","cancellationRequestedAt","join","rows","toDate","Date","isJobLeaseActive","undefined","recordHeartbeat","updated","previous","toolCapabilities","toolCapabilitiesReportedAt","previousToolCapabilities","currentToolCapabilities","cancellationRequested","runningJobExecution","requestJobExecutionCancellation","cancelRunnerJobs","jobIds"],"mappings":"AAAA,SACEA,kBAAkB,EAClBC,wBAAwB,EACxBC,iBAAiB,QAGZ,2BAA2B;AAClC,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,gBAAgB,EAAEC,iBAAiB,QAAO,uBAAuB;AACzE,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,cAAc,EACdC,GAAG,EACHC,KAAK,EACLC,IAAI,EACJC,EAAE,EACFC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,GAAG,EACHC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SACEC,wBAAwB,EACxBC,2BAA2B,EAC3BC,gCAAgC,QAC3B,kBAAkB;AACzB,SAAQC,yBAAyB,EAAEC,6BAA6B,QAAO,uBAAuB;AAE9F,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,eAAe,QAAO,+BAA+B;AAC7D,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAYxE,OAAO,eAAeC,sBAAsBC,MAE3C;IACC,MAAM,CAACC,aAAaC,YAAY,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACnDX,KACGY,MAAM,CAAC;YAACC,aAAaX,qBAAqBW,WAAW;YAAE7B,OAAOA;QAAO,GACrE8B,IAAI,CAACZ,sBACLa,KAAK,CAAC3B,QAAQc,qBAAqBW,WAAW,EAAEN,OAAOS,YAAY,GACnEC,OAAO,CAACf,qBAAqBW,WAAW;QAC3Cb,KACGY,MAAM,CAAC;YAACC,aAAaR,qBAAqBQ,WAAW;YAAE7B,OAAOA;QAAO,GACrE8B,IAAI,CAACT,sBACLU,KAAK,CAAC3B,QAAQiB,qBAAqBQ,WAAW,EAAEN,OAAOS,YAAY,GACnEC,OAAO,CAACZ,qBAAqBQ,WAAW;KAC5C;IAED,MAAMK,oBAAoB,IAAIC,IAAIX,YAAYY,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIR,WAAW;YAAES,OAAOD,IAAIrC,KAAK;SAAE;IAC/F,MAAMuC,qBAAqB,IAAIJ,IAC7BV,YAAYW,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIR,WAAW;YAAES,OAAOD,IAAIrC,KAAK;SAAE;IAG/D,OAAOuB,OAAOS,YAAY,CAACI,GAAG,CAAC,CAACP,cAAiB,CAAA;YAC/CA;YACAW,QAAQN,kBAAkBO,GAAG,CAACZ,gBAAgB;YAC9Ca,SAASH,mBAAmBE,GAAG,CAACZ,gBAAgB;QAClD,CAAA;AACF;AAEA,6FAA6F;AAC7F,2EAA2E;AAC3E,0EAA0E;AAC1E,uFAAuF;AACvF,sFAAsF;AACtF,qFAAqF;AACrF,wFAAwF;AACxF,OAAO,eAAec,oBAAoBpB,MAAiC;IACzE,MAAMqB,iBAAiB;WAAIhD,mBAAmB2B,OAAOqB,cAAc;KAAE;IACrE,IAAIA,eAAeC,MAAM,KAAK,GAAG,MAAM,IAAIlC;IAE3C,MAAMmC,WAAW,MAAM9B,KAAK+B,WAAW,CAAC,OAAOC;QAC7C,MAAM,CAACC,SAAS,GAAG,MAAMD,GACtBE,MAAM,CAAChC,sBACPiC,MAAM,CAAC;YACNtB,aAAaN,OAAOM,WAAW;YAC/BuB,eAAe7B,OAAO6B,aAAa;YACnCC,sBAAsB9B,OAAO8B,oBAAoB;YACjDC,OAAO/B,OAAO+B,KAAK;YACnBC,gBAAgBhC,OAAOgC,cAAc;YACrCC,WAAWjC,OAAOiC,SAAS;YAC3BZ;QACF,GACCa,mBAAmB,CAAC;YAACC,QAAQxC,qBAAqBqC,cAAc;QAAA,GAChEI,SAAS,CAAC;YAACC,WAAW1C,qBAAqB0C,SAAS;QAAA;QAEvD,4EAA4E;QAC5E,0EAA0E;QAC1E,mEAAmE;QACnE,IAAI,CAACX,UAAU,OAAO;QAEtB,MAAMvD,iBAAkCsD,IAAI/B,eAAe;YACzD4C,MAAMrE;YACNsE,SAAS;gBACPV,eAAe7B,OAAO6B,aAAa;gBACnCC,sBAAsB9B,OAAO8B,oBAAoB;gBACjDC,OAAO/B,OAAO+B,KAAK;gBACnBC,gBAAgBhC,OAAOgC,cAAc;gBACrCQ,UAAUd,SAASW,SAAS,CAACI,WAAW;YAC1C;QACF;QACA,OAAO;IACT;IAEA,IAAIlB,UAAUhC,0BAA0BmD,GAAG,CAAC;AAC9C;AAoCA,OAAO,eAAeC,yBAAyB3C,MAM9C;IACC,MAAM4C,2BAA2B;QAC/BtC,aAAaN,OAAOM,WAAW;QAC/BuC,iBAAiB7C,OAAO6C,eAAe;QACvCC,iBAAiB9C,OAAO+C,oCAAoC;IAC9D;IAEA,IAAI/C,OAAOgD,aAAa,CAAC1B,MAAM,KAAK,GAAG,OAAO;IAE9C,OAAO,MAAM7B,KAAK+B,WAAW,CAAC,OAAOC;QACnC,IAAIwB,gBAA+B;QACnC,IAAIC,mBAAkC;QACtC,IAAIC,mBAAkC;QAEtC,IAAInD,OAAOoD,SAAS,KAAK,MAAM;YAC7B,MAAM,CAACC,QAAQ,GAAG,MAAM5B,GACrBpB,MAAM,CAAC;gBACN+C,WAAWvD,eAAeuD,SAAS;gBACnCE,YAAYzD,eAAeyD,UAAU;gBACrCC,WAAW1D,eAAe0D,SAAS;gBACnCJ,kBAAkBtD,eAAesD,gBAAgB;gBACjDF,eAAepD,eAAeoD,aAAa;gBAC3CC,kBAAkBrD,eAAeqD,gBAAgB;YACnD,GACC3C,IAAI,CAACV,gBACLW,KAAK,CAAC7B,GAAGkB,eAAe2D,EAAE,EAAExD,OAAO6C,eAAe,GAClDY,KAAK,CAAC,GACNC,GAAG,CAAC;YAEP,IACE,CAACL,WACDA,QAAQE,SAAS,IACjBF,QAAQD,SAAS,KAAK,QACtBC,QAAQC,UAAU,IAAID,QAAQD,SAAS,EACvC;gBACA,MAAM,IAAI/D,4BAA4BW,OAAO6C,eAAe;YAC9D;YAEA,0EAA0E;YAC1E,mDAAmD;YACnDM,mBAAmBE,QAAQF,gBAAgB;YAC3CF,gBAAgBI,QAAQJ,aAAa;YACrCC,mBAAmBG,QAAQH,gBAAgB;QAC7C;QAEA,4EAA4E;QAC5E,gDAAgD;QAChD,MAAM,CAACpC,IAAI,GAAG,MAAMW,GACjBpB,MAAM,GACNE,IAAI,CAACZ,sBACLa,KAAK,CACJlC,IACEK,GAAGgB,qBAAqBW,WAAW,EAAEN,OAAOM,WAAW,GACvD/B,eAAeoB,qBAAqB0B,cAAc,EAAErB,OAAOgD,aAAa,IAG3EW,OAAO,CAACnF,IAAImB,qBAAqB0C,SAAS,GAAG7D,IAAImB,qBAAqB6D,EAAE,GACxEC,KAAK,CAAC,GACNC,GAAG,CAAC,UAAU;YAACE,YAAY;QAAI;QAElC,IAAI,CAAC9C,KAAK,OAAO;QAEjB,MAAMW,GAAGoC,MAAM,CAAClE,sBAAsBa,KAAK,CAAC7B,GAAGgB,qBAAqB6D,EAAE,EAAE1C,IAAI0C,EAAE;QAE9E,8EAA8E;QAC9E,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,8EAA8E;QAC9E,2DAA2D;QAC3D,MAAM9B,WAAW,MAAMD,GACpBE,MAAM,CAAC7B,sBACP8B,MAAM,CAAC;YACNtB,aAAaQ,IAAIR,WAAW;YAC5BuB,eAAef,IAAIe,aAAa;YAChCC,sBAAsBhB,IAAIgB,oBAAoB;YAC9CC,OAAOjB,IAAIiB,KAAK;YAChBC,gBAAgBlB,IAAIkB,cAAc;YAClCC,WAAWnB,IAAImB,SAAS;YACxBY,iBAAiB7C,OAAO6C,eAAe;YACvCI;YACAC;YACA7B,gBAAgBP,IAAIO,cAAc;YAClCyC,cAAc9D,OAAOgD,aAAa;QACpC,GACCd,mBAAmB,CAAC;YAACC,QAAQrC,qBAAqBkC,cAAc;QAAA,GAChEI,SAAS,CAAC;YAAC2B,WAAWjE,qBAAqBkE,SAAS;QAAA;QAEvD,MAAMC,UAAUvC,QAAQ,CAAC,EAAE;QAC3B,IAAI,CAACuC,SAAS,OAAO;QAErB,MAAMC,0BAA0Bf,mBAC5BxE,GAAGiB,gBAAgB4D,EAAE,EAAEL,oBACvBF,iBAAiBC,mBACf5E,IACEK,GAAGiB,gBAAgBqD,aAAa,EAAEA,gBAClCtE,GAAGiB,gBAAgBsD,gBAAgB,EAAEA,qBAEvC;QACN,IAAIgB,yBAAyB;YAC3B,MAAMzC,GACH0C,MAAM,CAACvE,iBACPwE,GAAG,CAAC;gBACHC,gBAAgBlF,GAAG,CAAC,SAAS,EAAES,gBAAgByE,cAAc,CAAC,EAAE,EAAEJ,QAAQF,SAAS,CAAC,CAAC,CAAC;gBACtFO,WAAWnF,GAAG,CAAC,KAAK,CAAC;YACvB,GACCqB,KAAK,CAAC0D;QACX;QAEA,IAAIlE,OAAOoD,SAAS,KAAK,MAAM;YAC7B,MAAM3B,GACH0C,MAAM,CAACtE,gBACPuE,GAAG,CAAC;gBAACd,YAAYnE,GAAG,CAAC,EAAEU,eAAeyD,UAAU,CAAC,IAAI,CAAC;gBAAEgB,WAAWnF,GAAG,CAAC,KAAK,CAAC;YAAA,GAC7EqB,KAAK,CAAC7B,GAAGkB,eAAe2D,EAAE,EAAExD,OAAO6C,eAAe;QACvD;QAEA,4FAA4F;QAC5F,mFAAmF;QACnF,6BAA6B;QAC7B,MAAM1E,iBAAkCsD,IAAI/B,eAAe;YACzD4C,MAAMvE;YACNwE,SAAS;gBACPV,eAAef,IAAIe,aAAa;gBAChCC,sBAAsBhB,IAAIgB,oBAAoB;gBAC9CC,OAAOjB,IAAIiB,KAAK;gBAChBC,gBAAgBlB,IAAIkB,cAAc;gBAClC+B,WAAWE,QAAQF,SAAS,CAACtB,WAAW;YAC1C;QACF;QAEA,OAAO;YACLZ,eAAef,IAAIe,aAAa;YAChCC,sBAAsBhB,IAAIgB,oBAAoB;YAC9CC,OAAOjB,IAAIiB,KAAK;YAChBC,gBAAgBlB,IAAIkB,cAAc;YAClCC,WAAWnB,IAAImB,SAAS;QAC1B;IACF;AACF;AAEA,eAAeW,2BAA2B5C,MAIzC;IACC,MAAMP,KACH0E,MAAM,CAACtE,gBACPuE,GAAG,CAAC;QAACE,WAAWnF,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1BqB,KAAK,CACJlC,IACEK,GAAGkB,eAAe2D,EAAE,EAAExD,OAAO6C,eAAe,GAC5ClE,GAAGkB,eAAeS,WAAW,EAAEN,OAAOM,WAAW,GACjDnB,GAAG,CAAC,EAAEU,eAAeyE,SAAS,CAAC,YAAY,EAAEtE,OAAO8C,eAAe,CAAC,yBAAyB,CAAC;AAGtG;AAEA;;;;;;;CAOC,GACD,OAAO,eAAeyB,oBAAoBvE,MAAgC;IACxE,MAAMP,KAAK+B,WAAW,CAAC,OAAOC;QAC5B,uFAAuF;QACvF,6EAA6E;QAC7E,qFAAqF;QACrF,2CAA2C;QAC3C,MAAMA,GACHoC,MAAM,CAAClE,sBACPa,KAAK,CAAC7B,GAAGgB,qBAAqBqC,cAAc,EAAEhC,OAAOgC,cAAc;QACtE,MAAMP,GACHoC,MAAM,CAAC/D,sBACPU,KAAK,CAAC7B,GAAGmB,qBAAqBkC,cAAc,EAAEhC,OAAOgC,cAAc;IACxE;AACF;AAEA;;;;;;;;;;;;CAYC,GACD,OAAO,eAAewC,yBAAyBxE,MAI9C;IAQC,MAAMyE,SAAS,MAAMhF,KAAK+B,WAAW,CAAC,OAAOC;QAC3C,MAAMiD,kBAAkBvF,GAAG,CAAC,SAAS,EAAEa,OAAO2E,gBAAgB,CAAC,yBAAyB,CAAC;QACzF,MAAMC,uBAAuBzF,GAAG,CAAC,SAAS,EAAEa,OAAO6E,4BAA4B,CAAC,yBAAyB,CAAC;QAC1G,MAAMC,iBAAiB5F,GACrBZ,IACES,OAAOe,qBAAqBiF,gBAAgB,GAC5C9F,IAAIa,qBAAqBkF,eAAe,EAAElF,qBAAqBkE,SAAS,GACxEhF,GAAGc,qBAAqBkE,SAAS,EAAEY,wBAErCtG,IACEY,GACEJ,UAAUgB,qBAAqBiF,gBAAgB,GAC/CnG,GAAGkB,qBAAqBkF,eAAe,EAAElF,qBAAqBkE,SAAS,IAEzEhF,GAAGc,qBAAqBkF,eAAe,EAAEN;QAI7C,MAAMO,WAAWxD,GACdpB,MAAM,CAAC;YAACmD,IAAI1D,qBAAqB0D,EAAE;QAAA,GACnCjD,IAAI,CAACT,sBACLU,KAAK,CAACsE,gBACNnB,OAAO,CACNnF,IACEW,GAAG,CAAC,UAAU,EAAEW,qBAAqBiF,gBAAgB,CAAC,aAAa,EAAEjF,qBAAqBkF,eAAe,CAAC,IAAI,EAAElF,qBAAqBkE,SAAS,CAAC,MAAM,EAAElE,qBAAqBkE,SAAS,CAAC,MAAM,EAAElE,qBAAqBkF,eAAe,CAAC,IAAI,CAAC,GAG3OvB,KAAK,CAACzD,OAAOyD,KAAK,IAAI,KACtBC,GAAG,CAAC,UAAU;YAACE,YAAY;QAAI;QAElC,MAAMsB,UAAU,MAAMzD,GACnBoC,MAAM,CAAC/D,sBACPU,KAAK,CAAClC,IAAIO,QAAQiB,qBAAqB0D,EAAE,EAAEyB,WAAWH,iBACtD1C,SAAS,CAAC;YACTP,eAAe/B,qBAAqB+B,aAAa;YACjDC,sBAAsBhC,qBAAqBgC,oBAAoB;YAC/DC,OAAOjC,qBAAqBiC,KAAK;YACjCC,gBAAgBlC,qBAAqBkC,cAAc;QACrD;QAEF,IAAIkD,QAAQ5D,MAAM,KAAK,GAAG,OAAO,EAAE;QAEnC,MAAMG,GAAGoC,MAAM,CAAClE,sBAAsBa,KAAK,CACzC3B,QACEc,qBAAqBqC,cAAc,EACnCkD,QAAQrE,GAAG,CAAC,CAACC,MAAQA,IAAIkB,cAAc;QAI3C,MAAM5D,kBACJqD,IACA/B,eACAwF,QAAQrE,GAAG,CAAC,CAACC,MAAS,CAAA;gBACpBwB,MAAMtE;gBACNuE,SAAS;oBACPV,eAAef,IAAIe,aAAa;oBAChCC,sBAAsBhB,IAAIgB,oBAAoB;oBAC9CC,OAAOjB,IAAIiB,KAAK;oBAChBC,gBAAgBlB,IAAIkB,cAAc;gBACpC;YACF,CAAA;QAGF,OAAOkD;IACT;IAEA,IAAIT,OAAOnD,MAAM,GAAG,GAAG9B,8BAA8BkD,GAAG,CAAC+B,OAAOnD,MAAM;IAEtE,OAAOmD;AACT;AAEA,OAAO,eAAeU;IAIpB,MAAM,CAACC,QAAQ,GAAG,MAAM3F,KAAKY,MAAM,CAAC;QAACgF,OAAO5G;IAAO,GAAG8B,IAAI,CAACZ;IAC3D,MAAM,CAACwB,QAAQ,GAAG,MAAM1B,KAAKY,MAAM,CAAC;QAACgF,OAAO5G;IAAO,GAAG8B,IAAI,CAACT;IAC3D,OAAO;QACLH,sBAAsByF,SAASC,SAAS;QACxCvF,sBAAsBqB,SAASkE,SAAS;IAC1C;AACF;AAEA,OAAO,eAAeC,+BAA+BtF,MAIpD;IACC,OAAO,MAAMP,KACVY,MAAM,CAAC;QACN0B,OAAOjC,qBAAqBiC,KAAK;QACjCC,gBAAgBlC,qBAAqBkC,cAAc;QACnDH,eAAe/B,qBAAqB+B,aAAa;QACjDC,sBAAsBhC,qBAAqBgC,oBAAoB;QAC/DG,WAAWnC,qBAAqBmC,SAAS;QACzCY,iBAAiB/C,qBAAqB+C,eAAe;QACrDI,eAAenD,qBAAqBmD,aAAa;QACjDC,kBAAkBpD,qBAAqBoD,gBAAgB;QACvD7B,gBAAgBvB,qBAAqBuB,cAAc;QACnDyC,cAAchE,qBAAqBgE,YAAY;QAC/CE,WAAWlE,qBAAqBkE,SAAS;QACzCgB,iBAAiBlF,qBAAqBkF,eAAe;IACvD,GACCzE,IAAI,CAACT,sBACLU,KAAK,CACJlC,IACEK,GAAGmB,qBAAqBQ,WAAW,EAAEN,OAAOM,WAAW,GACvDnB,GAAG,CAAC,EAAEW,qBAAqBkF,eAAe,CAAC,YAAY,EAAEhF,OAAOuF,aAAa,CAAC,yBAAyB,CAAC,GAG3G5B,OAAO,CAACjF,KAAKoB,qBAAqBkF,eAAe,GAAGtG,KAAKoB,qBAAqB0D,EAAE,GAChFC,KAAK,CAACzD,OAAOyD,KAAK,IAAI;AAC3B;AAEA,OAAO,eAAe+B,2CACpB/D,EAAM,EACNzB,MAIC;IAED,IAAIA,OAAOyF,iBAAiB,CAACnE,MAAM,KAAK,GAAG,OAAO,EAAE;IAEpD,MAAMoE,gBAAgB,MAAMjE,GACzBpB,MAAM,CAAC;QACN6C,kBAAkBpD,qBAAqBoD,gBAAgB;QACvDzE,OAAOA;IACT,GACC8B,IAAI,CAACT,sBACLU,KAAK,CACJlC,IACEK,GAAGmB,qBAAqBQ,WAAW,EAAEN,OAAOM,WAAW,GACvD3B,GAAGmB,qBAAqBmD,aAAa,EAAEjD,OAAOiD,aAAa,GAC3DpE,QAAQiB,qBAAqBoD,gBAAgB,EAAElD,OAAOyF,iBAAiB,IAG1E/E,OAAO,CAACZ,qBAAqBoD,gBAAgB,EAC7CyC,MAAM,CAACxG,GAAG,CAAC,YAAY,CAAC;IAE3B,MAAMyG,6BAA6BF,cAAcG,OAAO,CAAC,CAAC/E,MACxDA,IAAIoC,gBAAgB,GAAG;YAACpC,IAAIoC,gBAAgB;SAAC,GAAG,EAAE;IAEpD,IAAI0C,2BAA2BtE,MAAM,GAAG,GAAG;QACzCpD,SAAS4H,IAAI,CACX;YACExF,aAAaN,OAAOM,WAAW;YAC/B2C,eAAejD,OAAOiD,aAAa;YACnCwC,mBAAmBG;QACrB,GACA;IAEJ;IAEA,MAAMG,SAAS,MAAMtE,GAAGuE,OAAO,CAS5B7G,GAAG,CAAC;wBACe,EAAEW,qBAAqBoD,gBAAgB,CAAC;MAC1D,EAAEpD,qBAAqB+B,aAAa,CAAC;MACrC,EAAE/B,qBAAqBgC,oBAAoB,CAAC;MAC5C,EAAEhC,qBAAqBiC,KAAK,CAAC;MAC7B,EAAEjC,qBAAqBkC,cAAc,CAAC;MACtC,EAAElC,qBAAqBoD,gBAAgB,CAAC;MACxC,EAAEpD,qBAAqBkE,SAAS,CAAC;MACjC,EAAElE,qBAAqBkF,eAAe,CAAC;MACvC,EAAElF,qBAAqBmG,uBAAuB,CAAC;SAC5C,EAAEnG,qBAAqB;;MAE1B,EAAEA,qBAAqBQ,WAAW,CAAC,GAAG,EAAEN,OAAOM,WAAW,CAAC;UACvD,EAAER,qBAAqBmD,aAAa,CAAC,GAAG,EAAEjD,OAAOiD,aAAa,CAAC;UAC/D,EAAEnD,qBAAqBoD,gBAAgB,CAAC,KAAK,EAAE/D,IAAI+G,IAAI,CACzDlG,OAAOyF,iBAAiB,CAAC5E,GAAG,CAAC,CAACqC,mBAAqB/D,GAAG,CAAC,EAAE+D,iBAAiB,CAAC,GAC3E/D,GAAG,CAAC,EAAE,CAAC,EACP;aACK,EAAEW,qBAAqBoD,gBAAgB,CAAC,EAAE,EAAEpD,qBAAqBkE,SAAS,CAAC,OAAO,EAAElE,qBAAqBkC,cAAc,CAAC;EACnI,CAAC;IAED,OAAO+D,OAAOI,IAAI,CAACtF,GAAG,CAAC,CAACC,MAAS,CAAA;YAC/Be,eAAef,IAAIe,aAAa;YAChCC,sBAAsBhB,IAAIgB,oBAAoB;YAC9CC,OAAOjB,IAAIiB,KAAK;YAChBC,gBAAgBlB,IAAIkB,cAAc;YAClCkB,kBAAkBpC,IAAIoC,gBAAgB;YACtCc,WAAWoC,OAAOtF,IAAIkD,SAAS;YAC/BgB,iBAAiBoB,OAAOtF,IAAIkE,eAAe;YAC3CiB,yBAAyBnF,IAAImF,uBAAuB,GAChDG,OAAOtF,IAAImF,uBAAuB,IAClC;QACN,CAAA;AACF;AAEA,SAASG,OAAOf,KAAoB;IAClC,OAAOA,iBAAiBgB,OAAOhB,QAAQ,IAAIgB,KAAKhB;AAClD;AAEA,OAAO,eAAeiB,iBAAiBtG,MAItC;IACC,MAAM,CAACc,IAAI,GAAG,MAAMrB,KACjBY,MAAM,CAAC;QAACmD,IAAI1D,qBAAqB0D,EAAE;IAAA,GACnCjD,IAAI,CAACT,sBACLU,KAAK,CACJlC,IACEK,GAAGmB,qBAAqBkC,cAAc,EAAEhC,OAAOgC,cAAc,GAC7DrD,GAAGmB,qBAAqB+C,eAAe,EAAE7C,OAAO6C,eAAe,GAC/D7C,OAAO+B,KAAK,KAAKwE,YAAYA,YAAY5H,GAAGmB,qBAAqBiC,KAAK,EAAE/B,OAAO+B,KAAK,IAGvF0B,KAAK,CAAC;IAET,OAAO3C,QAAQyF;AACjB;AAEA,OAAO,eAAeC,gBAAgBxG,MAIrC;IAcC,MAAM+F,SAAS,MAAMtG,KAAK+B,WAAW,CAAC,OAAOC;QAC3C,MAAMgF,UAAU,MAAMhF,GACnB0C,MAAM,CAACrE,sBACPsE,GAAG,CAAC;YACHW,kBAAkB5F,GAAG,CAAC,SAAS,EAAEW,qBAAqBiF,gBAAgB,CAAC,QAAQ,CAAC;YAChFC,iBAAiB7F,GAAG,CAAC,KAAK,CAAC;QAC7B,GACCqB,KAAK,CACJlC,IACEK,GAAGmB,qBAAqBkC,cAAc,EAAEhC,OAAOgC,cAAc,GAC7DrD,GAAGmB,qBAAqB+C,eAAe,EAAE7C,OAAO6C,eAAe,IAGlET,SAAS,CAAC;YACT6D,yBAAyBnG,qBAAqBmG,uBAAuB;YACrEpE,eAAe/B,qBAAqB+B,aAAa;YACjDC,sBAAsBhC,qBAAqBgC,oBAAoB;YAC/DC,OAAOjC,qBAAqBiC,KAAK;YACjCC,gBAAgBlC,qBAAqBkC,cAAc;YACnDC,WAAWnC,qBAAqBmC,SAAS;YACzC3B,aAAaR,qBAAqBQ,WAAW;YAC7CuC,iBAAiB/C,qBAAqB+C,eAAe;QACvD;QAEF,MAAM/B,MAAM2F,OAAO,CAAC,EAAE;QACtB,IAAI,CAAC3F,KAAK,MAAM,IAAIxB,iCAAiCU,OAAOgC,cAAc;QAE1E,MAAM,CAAC0E,SAAS,GAAG,MAAMjF,GACtBpB,MAAM,CAAC;YAACsG,kBAAkB9G,eAAe8G,gBAAgB;QAAA,GACzDpG,IAAI,CAACV,gBACLW,KAAK,CAAC7B,GAAGkB,eAAe2D,EAAE,EAAExD,OAAO6C,eAAe,GAClDY,KAAK,CAAC,GACNC,GAAG,CAAC;QAEP,MAAM,CAACL,QAAQ,GAAG,MAAM5B,GACrB0C,MAAM,CAACtE,gBACPuE,GAAG,CAAC;YACHuC,kBAAkB3G,OAAO2G,gBAAgB,IAAI;YAC7CC,4BAA4B5G,OAAO2G,gBAAgB,GAAGxH,GAAG,CAAC,KAAK,CAAC,GAAG;YACnEmF,WAAWnF,GAAG,CAAC,KAAK,CAAC;QACvB,GACCqB,KAAK,CAAC7B,GAAGkB,eAAe2D,EAAE,EAAExD,OAAO6C,eAAe,GAClDT,SAAS,CAAC;YAACuE,kBAAkB9G,eAAe8G,gBAAgB;QAAA;QAE/D,OAAO;YACL7F;YACA+F,0BAA0BH,UAAUC,oBAAoB;YACxDG,yBAAyBzD,SAASsD,oBAAoB;QACxD;IACF;IAEA,MAAM7F,MAAMiF,OAAOjF,GAAG;IACtB,IAAI,CAACA,KAAK,MAAM,IAAIxB,iCAAiCU,OAAOgC,cAAc;IAC1E,OAAO;QACL+E,uBAAuBjG,IAAImF,uBAAuB,KAAK;QACvDY,0BAA0Bd,OAAOc,wBAAwB;QACzDC,yBAAyBf,OAAOe,uBAAuB;QACvDE,qBAAqB;YACnBnF,eAAef,IAAIe,aAAa;YAChCC,sBAAsBhB,IAAIgB,oBAAoB;YAC9CC,OAAOjB,IAAIiB,KAAK;YAChBC,gBAAgBlB,IAAIkB,cAAc;YAClCC,WAAWnB,IAAImB,SAAS;YACxB3B,aAAaQ,IAAIR,WAAW;YAC5BuC,iBAAiB/B,IAAI+B,eAAe;QACtC;IACF;AACF;AAEA,OAAO,eAAeoE,gCAAgCjH,MAErD;IACC,MAAMP,KACH0E,MAAM,CAACrE,sBACPsE,GAAG,CAAC;QACH6B,yBAAyB9G,GAAG,CAAC,SAAS,EAAEW,qBAAqBmG,uBAAuB,CAAC,QAAQ,CAAC;IAChG,GACCzF,KAAK,CAAC7B,GAAGmB,qBAAqBkC,cAAc,EAAEhC,OAAOgC,cAAc;AACxE;AAEA,OAAO,eAAekF,iBAAiBlH,MAA0B;IAC/D,IAAIA,OAAOmH,MAAM,CAAC7F,MAAM,KAAK,GAAG;IAEhC,MAAM7B,KAAK+B,WAAW,CAAC,OAAOC;QAC5B,MAAMA,GAAGoC,MAAM,CAAClE,sBAAsBa,KAAK,CAAC3B,QAAQc,qBAAqBoC,KAAK,EAAE/B,OAAOmH,MAAM;QAC7F,MAAM1F,GACH0C,MAAM,CAACrE,sBACPsE,GAAG,CAAC;YACH6B,yBAAyB9G,GAAG,CAAC,SAAS,EAAEW,qBAAqBmG,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACCzF,KAAK,CAAC3B,QAAQiB,qBAAqBiC,KAAK,EAAE/B,OAAOmH,MAAM;IAC5D;AACF"}
|
|
1
|
+
{"version":3,"sources":["../../src/db/job-executions.ts"],"sourcesContent":["import {\n RUNNER_JOB_CLAIMED,\n RUNNER_JOB_LEASE_EXPIRED,\n RUNNER_JOB_QUEUED,\n type RunnersEventMap,\n type RunnerToolCapabilitiesDto,\n} from '@shipfox/api-runners-dto';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n and,\n arrayContained,\n asc,\n count,\n desc,\n eq,\n gt,\n inArray,\n isNotNull,\n isNull,\n lt,\n lte,\n or,\n sql,\n} from 'drizzle-orm';\nimport {\n EmptyRequiredLabelsError,\n RunnerSessionExhaustedError,\n RunningJobExecutionNotFoundError,\n} from '#core/errors.js';\nimport {jobExecutionEnqueuedCount, jobExecutionLeaseExpiredCount} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {runnersOutbox} from './schema/outbox.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {providerRunners} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nconst runnerJobExecutionLockPrefix = 'runners_job_execution:';\n\nasync function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {\n await tx.execute(\n sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`,\n );\n}\n\nexport interface EnqueueJobExecutionParams {\n workspaceId: string;\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n requiredLabels: string[];\n}\n\nexport async function getWorkspaceJobCounts(params: {\n workspaceIds: string[];\n}): Promise<Array<{workspaceId: string; queued: number; running: number}>> {\n const [pendingRows, runningRows] = await Promise.all([\n db()\n .select({workspaceId: pendingJobExecutions.workspaceId, count: count()})\n .from(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.workspaceId, params.workspaceIds))\n .groupBy(pendingJobExecutions.workspaceId),\n db()\n .select({workspaceId: runningJobExecutions.workspaceId, count: count()})\n .from(runningJobExecutions)\n .where(inArray(runningJobExecutions.workspaceId, params.workspaceIds))\n .groupBy(runningJobExecutions.workspaceId),\n ]);\n\n const queuedByWorkspace = new Map(pendingRows.map((row) => [row.workspaceId, Number(row.count)]));\n const runningByWorkspace = new Map(\n runningRows.map((row) => [row.workspaceId, Number(row.count)]),\n );\n\n return params.workspaceIds.map((workspaceId) => ({\n workspaceId,\n queued: queuedByWorkspace.get(workspaceId) ?? 0,\n running: runningByWorkspace.get(workspaceId) ?? 0,\n }));\n}\n\n// Idempotent while the job execution is still pending: a duplicate jobExecutionId already in\n// `runners_pending_jobs` is a no-op. Temporal retries the enqueue activity at-least-once, so a\n// unique-violation throw on a retry-after-lost-result would permanently fail a healthy execution.\n// The per-execution advisory lock serializes retries with lease expiry/reconciliation, and the\n// durable lease-expired event prevents a retry from re-queueing an execution already reaped.\nexport async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {\n const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];\n if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();\n\n const enqueued = await db().transaction(async (tx) => {\n await lockJobExecution(tx, params.jobExecutionId);\n\n const [leaseExpired] = await tx\n .select({id: runnersOutbox.id})\n .from(runnersOutbox)\n .where(\n and(\n eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED),\n sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`,\n ),\n )\n .limit(1);\n if (leaseExpired) return false;\n\n const [inserted] = await tx\n .insert(pendingJobExecutions)\n .values({\n workspaceId: params.workspaceId,\n workflowRunId: params.workflowRunId,\n workflowRunAttemptId: params.workflowRunAttemptId,\n jobId: params.jobId,\n jobExecutionId: params.jobExecutionId,\n projectId: params.projectId,\n requiredLabels,\n })\n .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})\n .returning({createdAt: pendingJobExecutions.createdAt});\n\n // A retry that hits the conflict inserts nothing: the first enqueue already\n // emitted the queued event (durably, in the outbox), so re-emitting would\n // only add a redundant row the subscriber coalesces away. Skip it.\n if (!inserted) return false;\n\n await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n type: RUNNER_JOB_QUEUED,\n payload: {\n workflowRunId: params.workflowRunId,\n workflowRunAttemptId: params.workflowRunAttemptId,\n jobId: params.jobId,\n jobExecutionId: params.jobExecutionId,\n queuedAt: inserted.createdAt.toISOString(),\n },\n });\n return true;\n });\n\n if (enqueued) jobExecutionEnqueuedCount.add(1);\n}\n\nexport interface ClaimedJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n}\n\nexport interface ActiveRunningJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n runnerSessionId: string;\n provisionerId: string | null;\n providerRunnerId: string | null;\n requiredLabels: string[];\n runnerLabels: string[];\n startedAt: Date;\n lastHeartbeatAt: Date;\n}\n\nexport interface RunnerInstanceBoundJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n providerRunnerId: string;\n startedAt: Date;\n lastHeartbeatAt: Date;\n cancellationRequestedAt: Date | null;\n}\n\nexport async function claimPendingJobExecution(params: {\n workspaceId: string;\n runnerSessionId: string;\n sessionLabels: string[];\n maxClaims: number | null;\n runnerSessionLivenessThrottleSeconds: number;\n}): Promise<ClaimedJobExecution | null> {\n await touchRunnerSessionLiveness({\n workspaceId: params.workspaceId,\n runnerSessionId: params.runnerSessionId,\n throttleSeconds: params.runnerSessionLivenessThrottleSeconds,\n });\n\n if (params.sessionLabels.length === 0) return null;\n\n return await db().transaction(async (tx) => {\n let provisionerId: string | null = null;\n let providerRunnerId: string | null = null;\n let runnerInstanceId: string | null = null;\n\n if (params.maxClaims !== null) {\n const [session] = await tx\n .select({\n maxClaims: runnerSessions.maxClaims,\n claimsUsed: runnerSessions.claimsUsed,\n revokedAt: runnerSessions.revokedAt,\n runnerInstanceId: runnerSessions.runnerInstanceId,\n provisionerId: runnerSessions.provisionerId,\n providerRunnerId: runnerSessions.providerRunnerId,\n })\n .from(runnerSessions)\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .limit(1)\n .for('update');\n\n if (\n !session ||\n session.revokedAt ||\n session.maxClaims === null ||\n session.claimsUsed >= session.maxClaims\n ) {\n throw new RunnerSessionExhaustedError(params.runnerSessionId);\n }\n\n // Ephemeral sessions are the only capped sessions, and the DB check keeps\n // their provisioned-runner link present as a pair.\n runnerInstanceId = session.runnerInstanceId;\n provisionerId = session.provisionerId;\n providerRunnerId = session.providerRunnerId;\n }\n\n // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker\n // for rows sharing a created_at within a batch. Lock only the FIFO candidate before\n // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this\n // predicate would evaluate it while scanning and temporarily lock many queue entries.\n const candidate = tx\n .select()\n .from(pendingJobExecutions)\n .where(\n and(\n eq(pendingJobExecutions.workspaceId, params.workspaceId),\n arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels),\n ),\n )\n .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))\n .limit(1)\n .for('update', {skipLocked: true})\n .as('pending_candidate');\n\n const [row] = await tx\n .select()\n .from(candidate)\n .where(\n sql`\n pg_try_advisory_xact_lock(\n hashtext(${runnerJobExecutionLockPrefix} || ${candidate.jobExecutionId}::text)\n )\n `,\n );\n\n if (!row) return null;\n\n await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));\n\n // An enqueue retry that lands after a prior claim can leave an orphan pending\n // row whose jobExecutionId is already in `runners_running_jobs`. Insert-or-skip\n // on the jobExecutionId unique constraint: when the job execution is already running\n // the insert touches no row, so we commit the orphan's deletion and return\n // null rather than let the unique violation roll the claim back into a poison\n // loop. The runner just re-polls for a real job execution.\n const inserted = await tx\n .insert(runningJobExecutions)\n .values({\n workspaceId: row.workspaceId,\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n runnerSessionId: params.runnerSessionId,\n provisionerId,\n providerRunnerId,\n requiredLabels: row.requiredLabels,\n runnerLabels: params.sessionLabels,\n })\n .onConflictDoNothing({target: runningJobExecutions.jobExecutionId})\n .returning({claimedAt: runningJobExecutions.startedAt});\n\n const claimed = inserted[0];\n if (!claimed) return null;\n\n const runnerInstanceCondition = runnerInstanceId\n ? eq(providerRunners.id, runnerInstanceId)\n : provisionerId && providerRunnerId\n ? and(\n eq(providerRunners.provisionerId, provisionerId),\n eq(providerRunners.providerRunnerId, providerRunnerId),\n )\n : null;\n if (runnerInstanceCondition) {\n await tx\n .update(providerRunners)\n .set({\n firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,\n updatedAt: sql`now()`,\n })\n .where(runnerInstanceCondition);\n }\n\n if (params.maxClaims !== null) {\n await tx\n .update(runnerSessions)\n .set({claimsUsed: sql`${runnerSessions.claimsUsed} + 1`, updatedAt: sql`now()`})\n .where(eq(runnerSessions.id, params.runnerSessionId));\n }\n\n // The running-row insert is the runner claiming the job execution. Emit in the same tx; the\n // payload carries the row's own claim instant so a consumer records the true time,\n // not the outbox drain time.\n await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n type: RUNNER_JOB_CLAIMED,\n payload: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n claimedAt: claimed.claimedAt.toISOString(),\n },\n });\n\n return {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n };\n });\n}\n\nasync function touchRunnerSessionLiveness(params: {\n workspaceId: string;\n runnerSessionId: string;\n throttleSeconds: number;\n}): Promise<void> {\n await db()\n .update(runnerSessions)\n .set({updatedAt: sql`now()`})\n .where(\n and(\n eq(runnerSessions.id, params.runnerSessionId),\n eq(runnerSessions.workspaceId, params.workspaceId),\n sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} || ' seconds')::interval`,\n ),\n );\n}\n\n/**\n * Releases a job execution's lease when the orchestration workflow finalizes it: deletes the\n * running-job-execution row AND any lingering pending row for the same execution, in one tx.\n * Idempotent (0-row no-op), no token scope (the workflow is authoritative), and\n * emits no event: the workflow already owns the outcome. Sweeping the pending row\n * too closes the at-least-once window where an enqueue retry left an orphan that a\n * later claim would otherwise pick up for an already-finished job execution.\n */\nexport async function releaseJobExecution(params: {jobExecutionId: string}): Promise<void> {\n await db().transaction(async (tx) => {\n await lockJobExecution(tx, params.jobExecutionId);\n\n // Delete pending before running to match `claimPendingJobExecution`'s lock-acquisition\n // order (it locks the pending row first, then the running row). A concurrent\n // claim picking up an orphan pending row for this same job execution would otherwise\n // deadlock against the reverse order here.\n await tx\n .delete(pendingJobExecutions)\n .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n await tx\n .delete(runningJobExecutions)\n .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));\n });\n}\n\n/**\n * Reaps stale leases (bounded by `limit`), emitting one\n * `runners.job.lease_expired` event per reaped job execution.\n *\n * The cutoff is re-checked in the DELETE, not just the locking subquery, so a\n * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its\n * pending row: a failed best-effort `releaseJobExecution` would otherwise leave an orphan\n * that a later claim re-runs as an already-finished job execution.\n *\n * Locks pending-then-running to match `claimPendingJobExecution`, `releaseJobExecution`, and\n * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock running\n * rows first; the running-row DELETE re-checks the stale predicate after the pending-row sweep.\n */\nexport async function expireStuckJobExecutions(params: {\n thresholdSeconds: number;\n noFirstHeartbeatGraceSeconds: number;\n limit?: number;\n}): Promise<\n Array<{\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n }>\n> {\n const reaped = await db().transaction(async (tx) => {\n const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} || ' seconds')::interval`;\n const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} || ' seconds')::interval`;\n const stalePredicate = or(\n and(\n isNull(runningJobExecutions.firstHeartbeatAt),\n lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n lt(runningJobExecutions.startedAt, firstHeartbeatCutoff),\n ),\n and(\n or(\n isNotNull(runningJobExecutions.firstHeartbeatAt),\n gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n ),\n lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff),\n ),\n );\n\n const staleRows = await tx\n .select({\n id: runningJobExecutions.id,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n })\n .from(runningJobExecutions)\n .where(\n and(\n stalePredicate,\n sql`\n pg_try_advisory_xact_lock(\n hashtext(${runnerJobExecutionLockPrefix} || ${runningJobExecutions.jobExecutionId}::text)\n )\n `,\n ),\n )\n .orderBy(\n asc(\n sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,\n ),\n asc(runningJobExecutions.id),\n )\n .limit(params.limit ?? 100);\n\n if (staleRows.length === 0) return [];\n\n const staleIds = staleRows.map((row) => row.id);\n const staleJobExecutionIds = staleRows.map((row) => row.jobExecutionId);\n\n // Sweep pending rows first so this transaction cannot hold a running-row lock while waiting\n // for a pending-row lock held by reconciliation or a claim of an orphan pending row. The\n // advisory lock acquired in the candidate scan also makes enqueue retries wait until this\n // transaction has either reaped the execution or released its lock without selecting it.\n await tx\n .delete(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));\n\n const deleted = await tx\n .delete(runningJobExecutions)\n .where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate))\n .returning({\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n });\n\n if (deleted.length === 0) return [];\n\n await writeOutboxEvents<RunnersEventMap>(\n tx,\n runnersOutbox,\n deleted.map((row) => ({\n type: RUNNER_JOB_LEASE_EXPIRED,\n payload: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n },\n })),\n );\n\n return deleted;\n });\n\n if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);\n\n return reaped;\n}\n\nexport async function getJobExecutionQueueDepth(): Promise<{\n pendingJobExecutions: number;\n runningJobExecutions: number;\n}> {\n const [pending] = await db().select({value: count()}).from(pendingJobExecutions);\n const [running] = await db().select({value: count()}).from(runningJobExecutions);\n return {\n pendingJobExecutions: pending?.value ?? 0,\n runningJobExecutions: running?.value ?? 0,\n };\n}\n\nexport async function listActiveRunningJobExecutions(params: {\n workspaceId: string;\n windowSeconds: number;\n limit?: number;\n}): Promise<ActiveRunningJobExecution[]> {\n return await db()\n .select({\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n projectId: runningJobExecutions.projectId,\n runnerSessionId: runningJobExecutions.runnerSessionId,\n provisionerId: runningJobExecutions.provisionerId,\n providerRunnerId: runningJobExecutions.providerRunnerId,\n requiredLabels: runningJobExecutions.requiredLabels,\n runnerLabels: runningJobExecutions.runnerLabels,\n startedAt: runningJobExecutions.startedAt,\n lastHeartbeatAt: runningJobExecutions.lastHeartbeatAt,\n })\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.workspaceId, params.workspaceId),\n sql`${runningJobExecutions.lastHeartbeatAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n ),\n )\n .orderBy(desc(runningJobExecutions.lastHeartbeatAt), desc(runningJobExecutions.id))\n .limit(params.limit ?? 1000);\n}\n\nexport async function listRunningJobExecutionsByRunnerInstanceTx(\n tx: Tx,\n params: {\n workspaceId: string;\n provisionerId: string;\n providerRunnerIds: string[];\n },\n): Promise<RunnerInstanceBoundJobExecution[]> {\n if (params.providerRunnerIds.length === 0) return [];\n\n const duplicateRows = await tx\n .select({\n providerRunnerId: runningJobExecutions.providerRunnerId,\n count: count(),\n })\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.workspaceId, params.workspaceId),\n eq(runningJobExecutions.provisionerId, params.provisionerId),\n inArray(runningJobExecutions.providerRunnerId, params.providerRunnerIds),\n ),\n )\n .groupBy(runningJobExecutions.providerRunnerId)\n .having(sql`count(*) > 1`);\n\n const duplicateRunnerInstanceIds = duplicateRows.flatMap((row) =>\n row.providerRunnerId ? [row.providerRunnerId] : [],\n );\n if (duplicateRunnerInstanceIds.length > 0) {\n logger().warn(\n {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n providerRunnerIds: duplicateRunnerInstanceIds,\n },\n 'multiple running job executions are bound to the same provisioned runner',\n );\n }\n\n const result = await tx.execute<{\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n providerRunnerId: string;\n startedAt: Date | string;\n lastHeartbeatAt: Date | string;\n cancellationRequestedAt: Date | string | null;\n }>(sql`\n SELECT DISTINCT ON (${runningJobExecutions.providerRunnerId})\n ${runningJobExecutions.workflowRunId} AS \"workflowRunId\",\n ${runningJobExecutions.workflowRunAttemptId} AS \"workflowRunAttemptId\",\n ${runningJobExecutions.jobId} AS \"jobId\",\n ${runningJobExecutions.jobExecutionId} AS \"jobExecutionId\",\n ${runningJobExecutions.providerRunnerId} AS \"providerRunnerId\",\n ${runningJobExecutions.startedAt} AS \"startedAt\",\n ${runningJobExecutions.lastHeartbeatAt} AS \"lastHeartbeatAt\",\n ${runningJobExecutions.cancellationRequestedAt} AS \"cancellationRequestedAt\"\n FROM ${runningJobExecutions}\n WHERE\n ${runningJobExecutions.workspaceId} = ${params.workspaceId}\n AND ${runningJobExecutions.provisionerId} = ${params.provisionerId}\n AND ${runningJobExecutions.providerRunnerId} IN (${sql.join(\n params.providerRunnerIds.map((providerRunnerId) => sql`${providerRunnerId}`),\n sql`, `,\n )})\n ORDER BY ${runningJobExecutions.providerRunnerId}, ${runningJobExecutions.startedAt} DESC, ${runningJobExecutions.jobExecutionId} DESC\n `);\n\n return result.rows.map((row) => ({\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n providerRunnerId: row.providerRunnerId,\n startedAt: toDate(row.startedAt),\n lastHeartbeatAt: toDate(row.lastHeartbeatAt),\n cancellationRequestedAt: row.cancellationRequestedAt\n ? toDate(row.cancellationRequestedAt)\n : null,\n }));\n}\n\nfunction toDate(value: Date | string): Date {\n return value instanceof Date ? value : new Date(value);\n}\n\nexport async function isJobLeaseActive(params: {\n jobId?: string;\n jobExecutionId: string;\n runnerSessionId: string;\n}): Promise<boolean> {\n const [row] = await db()\n .select({id: runningJobExecutions.id})\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n params.jobId === undefined ? undefined : eq(runningJobExecutions.jobId, params.jobId),\n ),\n )\n .limit(1);\n\n return row !== undefined;\n}\n\nexport async function recordHeartbeat(params: {\n jobExecutionId: string;\n runnerSessionId: string;\n toolCapabilities?: RunnerToolCapabilitiesDto | null;\n}): Promise<{\n cancellationRequested: boolean;\n previousToolCapabilities: RunnerToolCapabilitiesDto | null;\n currentToolCapabilities: RunnerToolCapabilitiesDto | null;\n runningJobExecution: {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n workspaceId: string;\n runnerSessionId: string;\n };\n}> {\n const result = await db().transaction(async (tx) => {\n const updated = await tx\n .update(runningJobExecutions)\n .set({\n firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,\n lastHeartbeatAt: sql`now()`,\n })\n .where(\n and(\n eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n ),\n )\n .returning({\n cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n projectId: runningJobExecutions.projectId,\n workspaceId: runningJobExecutions.workspaceId,\n runnerSessionId: runningJobExecutions.runnerSessionId,\n });\n\n const row = updated[0];\n if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n\n const [previous] = await tx\n .select({toolCapabilities: runnerSessions.toolCapabilities})\n .from(runnerSessions)\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .limit(1)\n .for('update');\n\n const [session] = await tx\n .update(runnerSessions)\n .set({\n toolCapabilities: params.toolCapabilities ?? null,\n toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,\n updatedAt: sql`now()`,\n })\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .returning({toolCapabilities: runnerSessions.toolCapabilities});\n\n return {\n row,\n previousToolCapabilities: previous?.toolCapabilities ?? null,\n currentToolCapabilities: session?.toolCapabilities ?? null,\n };\n });\n\n const row = result.row;\n if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n return {\n cancellationRequested: row.cancellationRequestedAt !== null,\n previousToolCapabilities: result.previousToolCapabilities,\n currentToolCapabilities: result.currentToolCapabilities,\n runningJobExecution: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n workspaceId: row.workspaceId,\n runnerSessionId: row.runnerSessionId,\n },\n };\n}\n\n/**\n * Reconciles a terminal job execution with runner state in one transaction: removes its pending\n * queue row and requests cancellation on its running lease, if either exists. The operation is\n * idempotent and preserves the first cancellation-request timestamp.\n */\nexport async function reconcileTerminalJobExecution(params: {\n jobExecutionId: string;\n}): Promise<void> {\n await db().transaction(async (tx) => {\n await lockJobExecution(tx, params.jobExecutionId);\n\n // Delete pending before updating running to match claim/release lock order. Claim locks\n // pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes\n // a concurrent terminal reconciliation either win before claim or cancel the new lease.\n await tx\n .delete(pendingJobExecutions)\n .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n await tx\n .update(runningJobExecutions)\n .set({\n cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n })\n .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));\n });\n}\n\nexport async function cancelRunnerJobs(params: {jobIds: string[]}): Promise<void> {\n if (params.jobIds.length === 0) return;\n\n await db().transaction(async (tx) => {\n await tx.delete(pendingJobExecutions).where(inArray(pendingJobExecutions.jobId, params.jobIds));\n await tx\n .update(runningJobExecutions)\n .set({\n cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n })\n .where(inArray(runningJobExecutions.jobId, params.jobIds));\n });\n}\n"],"names":["RUNNER_JOB_CLAIMED","RUNNER_JOB_LEASE_EXPIRED","RUNNER_JOB_QUEUED","logger","writeOutboxEvent","writeOutboxEvents","canonicalizeLabels","and","arrayContained","asc","count","desc","eq","gt","inArray","isNotNull","isNull","lt","lte","or","sql","EmptyRequiredLabelsError","RunnerSessionExhaustedError","RunningJobExecutionNotFoundError","jobExecutionEnqueuedCount","jobExecutionLeaseExpiredCount","db","runnersOutbox","pendingJobExecutions","providerRunners","runnerSessions","runningJobExecutions","runnerJobExecutionLockPrefix","lockJobExecution","tx","jobExecutionId","execute","getWorkspaceJobCounts","params","pendingRows","runningRows","Promise","all","select","workspaceId","from","where","workspaceIds","groupBy","queuedByWorkspace","Map","map","row","Number","runningByWorkspace","queued","get","running","enqueueJobExecution","requiredLabels","length","enqueued","transaction","leaseExpired","id","eventType","payload","limit","inserted","insert","values","workflowRunId","workflowRunAttemptId","jobId","projectId","onConflictDoNothing","target","returning","createdAt","type","queuedAt","toISOString","add","claimPendingJobExecution","touchRunnerSessionLiveness","runnerSessionId","throttleSeconds","runnerSessionLivenessThrottleSeconds","sessionLabels","provisionerId","providerRunnerId","runnerInstanceId","maxClaims","session","claimsUsed","revokedAt","for","candidate","orderBy","skipLocked","as","delete","runnerLabels","claimedAt","startedAt","claimed","runnerInstanceCondition","update","set","firstClaimedAt","updatedAt","releaseJobExecution","expireStuckJobExecutions","reaped","heartbeatCutoff","thresholdSeconds","firstHeartbeatCutoff","noFirstHeartbeatGraceSeconds","stalePredicate","firstHeartbeatAt","lastHeartbeatAt","staleRows","staleIds","staleJobExecutionIds","deleted","getJobExecutionQueueDepth","pending","value","listActiveRunningJobExecutions","windowSeconds","listRunningJobExecutionsByRunnerInstanceTx","providerRunnerIds","duplicateRows","having","duplicateRunnerInstanceIds","flatMap","warn","result","cancellationRequestedAt","join","rows","toDate","Date","isJobLeaseActive","undefined","recordHeartbeat","updated","previous","toolCapabilities","toolCapabilitiesReportedAt","previousToolCapabilities","currentToolCapabilities","cancellationRequested","runningJobExecution","reconcileTerminalJobExecution","cancelRunnerJobs","jobIds"],"mappings":"AAAA,SACEA,kBAAkB,EAClBC,wBAAwB,EACxBC,iBAAiB,QAGZ,2BAA2B;AAClC,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,gBAAgB,EAAEC,iBAAiB,QAAO,uBAAuB;AACzE,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,cAAc,EACdC,GAAG,EACHC,KAAK,EACLC,IAAI,EACJC,EAAE,EACFC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,GAAG,EACHC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SACEC,wBAAwB,EACxBC,2BAA2B,EAC3BC,gCAAgC,QAC3B,kBAAkB;AACzB,SAAQC,yBAAyB,EAAEC,6BAA6B,QAAO,uBAAuB;AAE9F,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,eAAe,QAAO,+BAA+B;AAC7D,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAExE,MAAMC,+BAA+B;AAErC,eAAeC,iBAAiBC,EAAM,EAAEC,cAAsB;IAC5D,MAAMD,GAAGE,OAAO,CACdhB,GAAG,CAAC,sCAAsC,EAAE,GAAGY,+BAA+BG,gBAAgB,CAAC,EAAE,CAAC;AAEtG;AAYA,OAAO,eAAeE,sBAAsBC,MAE3C;IACC,MAAM,CAACC,aAAaC,YAAY,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACnDhB,KACGiB,MAAM,CAAC;YAACC,aAAahB,qBAAqBgB,WAAW;YAAElC,OAAOA;QAAO,GACrEmC,IAAI,CAACjB,sBACLkB,KAAK,CAAChC,QAAQc,qBAAqBgB,WAAW,EAAEN,OAAOS,YAAY,GACnEC,OAAO,CAACpB,qBAAqBgB,WAAW;QAC3ClB,KACGiB,MAAM,CAAC;YAACC,aAAab,qBAAqBa,WAAW;YAAElC,OAAOA;QAAO,GACrEmC,IAAI,CAACd,sBACLe,KAAK,CAAChC,QAAQiB,qBAAqBa,WAAW,EAAEN,OAAOS,YAAY,GACnEC,OAAO,CAACjB,qBAAqBa,WAAW;KAC5C;IAED,MAAMK,oBAAoB,IAAIC,IAAIX,YAAYY,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIR,WAAW;YAAES,OAAOD,IAAI1C,KAAK;SAAE;IAC/F,MAAM4C,qBAAqB,IAAIJ,IAC7BV,YAAYW,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIR,WAAW;YAAES,OAAOD,IAAI1C,KAAK;SAAE;IAG/D,OAAO4B,OAAOS,YAAY,CAACI,GAAG,CAAC,CAACP,cAAiB,CAAA;YAC/CA;YACAW,QAAQN,kBAAkBO,GAAG,CAACZ,gBAAgB;YAC9Ca,SAASH,mBAAmBE,GAAG,CAACZ,gBAAgB;QAClD,CAAA;AACF;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,+FAA+F;AAC/F,6FAA6F;AAC7F,OAAO,eAAec,oBAAoBpB,MAAiC;IACzE,MAAMqB,iBAAiB;WAAIrD,mBAAmBgC,OAAOqB,cAAc;KAAE;IACrE,IAAIA,eAAeC,MAAM,KAAK,GAAG,MAAM,IAAIvC;IAE3C,MAAMwC,WAAW,MAAMnC,KAAKoC,WAAW,CAAC,OAAO5B;QAC7C,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,MAAM,CAAC4B,aAAa,GAAG,MAAM7B,GAC1BS,MAAM,CAAC;YAACqB,IAAIrC,cAAcqC,EAAE;QAAA,GAC5BnB,IAAI,CAAClB,eACLmB,KAAK,CACJvC,IACEK,GAAGe,cAAcsC,SAAS,EAAEhE,2BAC5BmB,GAAG,CAAC,EAAEO,cAAcuC,OAAO,CAAC,sBAAsB,EAAE5B,OAAOH,cAAc,CAAC,CAAC,GAG9EgC,KAAK,CAAC;QACT,IAAIJ,cAAc,OAAO;QAEzB,MAAM,CAACK,SAAS,GAAG,MAAMlC,GACtBmC,MAAM,CAACzC,sBACP0C,MAAM,CAAC;YACN1B,aAAaN,OAAOM,WAAW;YAC/B2B,eAAejC,OAAOiC,aAAa;YACnCC,sBAAsBlC,OAAOkC,oBAAoB;YACjDC,OAAOnC,OAAOmC,KAAK;YACnBtC,gBAAgBG,OAAOH,cAAc;YACrCuC,WAAWpC,OAAOoC,SAAS;YAC3Bf;QACF,GACCgB,mBAAmB,CAAC;YAACC,QAAQhD,qBAAqBO,cAAc;QAAA,GAChE0C,SAAS,CAAC;YAACC,WAAWlD,qBAAqBkD,SAAS;QAAA;QAEvD,4EAA4E;QAC5E,0EAA0E;QAC1E,mEAAmE;QACnE,IAAI,CAACV,UAAU,OAAO;QAEtB,MAAMhE,iBAAkC8B,IAAIP,eAAe;YACzDoD,MAAM7E;YACNgE,SAAS;gBACPK,eAAejC,OAAOiC,aAAa;gBACnCC,sBAAsBlC,OAAOkC,oBAAoB;gBACjDC,OAAOnC,OAAOmC,KAAK;gBACnBtC,gBAAgBG,OAAOH,cAAc;gBACrC6C,UAAUZ,SAASU,SAAS,CAACG,WAAW;YAC1C;QACF;QACA,OAAO;IACT;IAEA,IAAIpB,UAAUrC,0BAA0B0D,GAAG,CAAC;AAC9C;AAoCA,OAAO,eAAeC,yBAAyB7C,MAM9C;IACC,MAAM8C,2BAA2B;QAC/BxC,aAAaN,OAAOM,WAAW;QAC/ByC,iBAAiB/C,OAAO+C,eAAe;QACvCC,iBAAiBhD,OAAOiD,oCAAoC;IAC9D;IAEA,IAAIjD,OAAOkD,aAAa,CAAC5B,MAAM,KAAK,GAAG,OAAO;IAE9C,OAAO,MAAMlC,KAAKoC,WAAW,CAAC,OAAO5B;QACnC,IAAIuD,gBAA+B;QACnC,IAAIC,mBAAkC;QACtC,IAAIC,mBAAkC;QAEtC,IAAIrD,OAAOsD,SAAS,KAAK,MAAM;YAC7B,MAAM,CAACC,QAAQ,GAAG,MAAM3D,GACrBS,MAAM,CAAC;gBACNiD,WAAW9D,eAAe8D,SAAS;gBACnCE,YAAYhE,eAAegE,UAAU;gBACrCC,WAAWjE,eAAeiE,SAAS;gBACnCJ,kBAAkB7D,eAAe6D,gBAAgB;gBACjDF,eAAe3D,eAAe2D,aAAa;gBAC3CC,kBAAkB5D,eAAe4D,gBAAgB;YACnD,GACC7C,IAAI,CAACf,gBACLgB,KAAK,CAAClC,GAAGkB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDlB,KAAK,CAAC,GACN6B,GAAG,CAAC;YAEP,IACE,CAACH,WACDA,QAAQE,SAAS,IACjBF,QAAQD,SAAS,KAAK,QACtBC,QAAQC,UAAU,IAAID,QAAQD,SAAS,EACvC;gBACA,MAAM,IAAItE,4BAA4BgB,OAAO+C,eAAe;YAC9D;YAEA,0EAA0E;YAC1E,mDAAmD;YACnDM,mBAAmBE,QAAQF,gBAAgB;YAC3CF,gBAAgBI,QAAQJ,aAAa;YACrCC,mBAAmBG,QAAQH,gBAAgB;QAC7C;QAEA,4EAA4E;QAC5E,oFAAoF;QACpF,oFAAoF;QACpF,sFAAsF;QACtF,MAAMO,YAAY/D,GACfS,MAAM,GACNE,IAAI,CAACjB,sBACLkB,KAAK,CACJvC,IACEK,GAAGgB,qBAAqBgB,WAAW,EAAEN,OAAOM,WAAW,GACvDpC,eAAeoB,qBAAqB+B,cAAc,EAAErB,OAAOkD,aAAa,IAG3EU,OAAO,CAACzF,IAAImB,qBAAqBkD,SAAS,GAAGrE,IAAImB,qBAAqBoC,EAAE,GACxEG,KAAK,CAAC,GACN6B,GAAG,CAAC,UAAU;YAACG,YAAY;QAAI,GAC/BC,EAAE,CAAC;QAEN,MAAM,CAAChD,IAAI,GAAG,MAAMlB,GACjBS,MAAM,GACNE,IAAI,CAACoD,WACLnD,KAAK,CACJ1B,GAAG,CAAC;;qBAES,EAAEY,6BAA6B,IAAI,EAAEiE,UAAU9D,cAAc,CAAC;;QAE3E,CAAC;QAGL,IAAI,CAACiB,KAAK,OAAO;QAEjB,MAAMlB,GAAGmE,MAAM,CAACzE,sBAAsBkB,KAAK,CAAClC,GAAGgB,qBAAqBoC,EAAE,EAAEZ,IAAIY,EAAE;QAE9E,8EAA8E;QAC9E,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,8EAA8E;QAC9E,2DAA2D;QAC3D,MAAMI,WAAW,MAAMlC,GACpBmC,MAAM,CAACtC,sBACPuC,MAAM,CAAC;YACN1B,aAAaQ,IAAIR,WAAW;YAC5B2B,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;YACxBW,iBAAiB/C,OAAO+C,eAAe;YACvCI;YACAC;YACA/B,gBAAgBP,IAAIO,cAAc;YAClC2C,cAAchE,OAAOkD,aAAa;QACpC,GACCb,mBAAmB,CAAC;YAACC,QAAQ7C,qBAAqBI,cAAc;QAAA,GAChE0C,SAAS,CAAC;YAAC0B,WAAWxE,qBAAqByE,SAAS;QAAA;QAEvD,MAAMC,UAAUrC,QAAQ,CAAC,EAAE;QAC3B,IAAI,CAACqC,SAAS,OAAO;QAErB,MAAMC,0BAA0Bf,mBAC5B/E,GAAGiB,gBAAgBmC,EAAE,EAAE2B,oBACvBF,iBAAiBC,mBACfnF,IACEK,GAAGiB,gBAAgB4D,aAAa,EAAEA,gBAClC7E,GAAGiB,gBAAgB6D,gBAAgB,EAAEA,qBAEvC;QACN,IAAIgB,yBAAyB;YAC3B,MAAMxE,GACHyE,MAAM,CAAC9E,iBACP+E,GAAG,CAAC;gBACHC,gBAAgBzF,GAAG,CAAC,SAAS,EAAES,gBAAgBgF,cAAc,CAAC,EAAE,EAAEJ,QAAQF,SAAS,CAAC,CAAC,CAAC;gBACtFO,WAAW1F,GAAG,CAAC,KAAK,CAAC;YACvB,GACC0B,KAAK,CAAC4D;QACX;QAEA,IAAIpE,OAAOsD,SAAS,KAAK,MAAM;YAC7B,MAAM1D,GACHyE,MAAM,CAAC7E,gBACP8E,GAAG,CAAC;gBAACd,YAAY1E,GAAG,CAAC,EAAEU,eAAegE,UAAU,CAAC,IAAI,CAAC;gBAAEgB,WAAW1F,GAAG,CAAC,KAAK,CAAC;YAAA,GAC7E0B,KAAK,CAAClC,GAAGkB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe;QACvD;QAEA,4FAA4F;QAC5F,mFAAmF;QACnF,6BAA6B;QAC7B,MAAMjF,iBAAkC8B,IAAIP,eAAe;YACzDoD,MAAM/E;YACNkE,SAAS;gBACPK,eAAenB,IAAImB,aAAa;gBAChCC,sBAAsBpB,IAAIoB,oBAAoB;gBAC9CC,OAAOrB,IAAIqB,KAAK;gBAChBtC,gBAAgBiB,IAAIjB,cAAc;gBAClCoE,WAAWE,QAAQF,SAAS,CAACtB,WAAW;YAC1C;QACF;QAEA,OAAO;YACLV,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;QAC1B;IACF;AACF;AAEA,eAAeU,2BAA2B9C,MAIzC;IACC,MAAMZ,KACHiF,MAAM,CAAC7E,gBACP8E,GAAG,CAAC;QAACE,WAAW1F,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1B0B,KAAK,CACJvC,IACEK,GAAGkB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAC5CzE,GAAGkB,eAAec,WAAW,EAAEN,OAAOM,WAAW,GACjDxB,GAAG,CAAC,EAAEU,eAAegF,SAAS,CAAC,YAAY,EAAExE,OAAOgD,eAAe,CAAC,yBAAyB,CAAC;AAGtG;AAEA;;;;;;;CAOC,GACD,OAAO,eAAeyB,oBAAoBzE,MAAgC;IACxE,MAAMZ,KAAKoC,WAAW,CAAC,OAAO5B;QAC5B,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,uFAAuF;QACvF,6EAA6E;QAC7E,qFAAqF;QACrF,2CAA2C;QAC3C,MAAMD,GACHmE,MAAM,CAACzE,sBACPkB,KAAK,CAAClC,GAAGgB,qBAAqBO,cAAc,EAAEG,OAAOH,cAAc;QACtE,MAAMD,GACHmE,MAAM,CAACtE,sBACPe,KAAK,CAAClC,GAAGmB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc;IACxE;AACF;AAEA;;;;;;;;;;;;CAYC,GACD,OAAO,eAAe6E,yBAAyB1E,MAI9C;IAQC,MAAM2E,SAAS,MAAMvF,KAAKoC,WAAW,CAAC,OAAO5B;QAC3C,MAAMgF,kBAAkB9F,GAAG,CAAC,SAAS,EAAEkB,OAAO6E,gBAAgB,CAAC,yBAAyB,CAAC;QACzF,MAAMC,uBAAuBhG,GAAG,CAAC,SAAS,EAAEkB,OAAO+E,4BAA4B,CAAC,yBAAyB,CAAC;QAC1G,MAAMC,iBAAiBnG,GACrBZ,IACES,OAAOe,qBAAqBwF,gBAAgB,GAC5CrG,IAAIa,qBAAqByF,eAAe,EAAEzF,qBAAqByE,SAAS,GACxEvF,GAAGc,qBAAqByE,SAAS,EAAEY,wBAErC7G,IACEY,GACEJ,UAAUgB,qBAAqBwF,gBAAgB,GAC/C1G,GAAGkB,qBAAqByF,eAAe,EAAEzF,qBAAqByE,SAAS,IAEzEvF,GAAGc,qBAAqByF,eAAe,EAAEN;QAI7C,MAAMO,YAAY,MAAMvF,GACrBS,MAAM,CAAC;YACNqB,IAAIjC,qBAAqBiC,EAAE;YAC3B7B,gBAAgBJ,qBAAqBI,cAAc;QACrD,GACCU,IAAI,CAACd,sBACLe,KAAK,CACJvC,IACE+G,gBACAlG,GAAG,CAAC;;uBAES,EAAEY,6BAA6B,IAAI,EAAED,qBAAqBI,cAAc,CAAC;;UAEtF,CAAC,GAGJ+D,OAAO,CACNzF,IACEW,GAAG,CAAC,UAAU,EAAEW,qBAAqBwF,gBAAgB,CAAC,aAAa,EAAExF,qBAAqByF,eAAe,CAAC,IAAI,EAAEzF,qBAAqByE,SAAS,CAAC,MAAM,EAAEzE,qBAAqByE,SAAS,CAAC,MAAM,EAAEzE,qBAAqByF,eAAe,CAAC,IAAI,CAAC,GAE1O/G,IAAIsB,qBAAqBiC,EAAE,GAE5BG,KAAK,CAAC7B,OAAO6B,KAAK,IAAI;QAEzB,IAAIsD,UAAU7D,MAAM,KAAK,GAAG,OAAO,EAAE;QAErC,MAAM8D,WAAWD,UAAUtE,GAAG,CAAC,CAACC,MAAQA,IAAIY,EAAE;QAC9C,MAAM2D,uBAAuBF,UAAUtE,GAAG,CAAC,CAACC,MAAQA,IAAIjB,cAAc;QAEtE,4FAA4F;QAC5F,yFAAyF;QACzF,0FAA0F;QAC1F,yFAAyF;QACzF,MAAMD,GACHmE,MAAM,CAACzE,sBACPkB,KAAK,CAAChC,QAAQc,qBAAqBO,cAAc,EAAEwF;QAEtD,MAAMC,UAAU,MAAM1F,GACnBmE,MAAM,CAACtE,sBACPe,KAAK,CAACvC,IAAIO,QAAQiB,qBAAqBiC,EAAE,EAAE0D,WAAWJ,iBACtDzC,SAAS,CAAC;YACTN,eAAexC,qBAAqBwC,aAAa;YACjDC,sBAAsBzC,qBAAqByC,oBAAoB;YAC/DC,OAAO1C,qBAAqB0C,KAAK;YACjCtC,gBAAgBJ,qBAAqBI,cAAc;QACrD;QAEF,IAAIyF,QAAQhE,MAAM,KAAK,GAAG,OAAO,EAAE;QAEnC,MAAMvD,kBACJ6B,IACAP,eACAiG,QAAQzE,GAAG,CAAC,CAACC,MAAS,CAAA;gBACpB2B,MAAM9E;gBACNiE,SAAS;oBACPK,eAAenB,IAAImB,aAAa;oBAChCC,sBAAsBpB,IAAIoB,oBAAoB;oBAC9CC,OAAOrB,IAAIqB,KAAK;oBAChBtC,gBAAgBiB,IAAIjB,cAAc;gBACpC;YACF,CAAA;QAGF,OAAOyF;IACT;IAEA,IAAIX,OAAOrD,MAAM,GAAG,GAAGnC,8BAA8ByD,GAAG,CAAC+B,OAAOrD,MAAM;IAEtE,OAAOqD;AACT;AAEA,OAAO,eAAeY;IAIpB,MAAM,CAACC,QAAQ,GAAG,MAAMpG,KAAKiB,MAAM,CAAC;QAACoF,OAAOrH;IAAO,GAAGmC,IAAI,CAACjB;IAC3D,MAAM,CAAC6B,QAAQ,GAAG,MAAM/B,KAAKiB,MAAM,CAAC;QAACoF,OAAOrH;IAAO,GAAGmC,IAAI,CAACd;IAC3D,OAAO;QACLH,sBAAsBkG,SAASC,SAAS;QACxChG,sBAAsB0B,SAASsE,SAAS;IAC1C;AACF;AAEA,OAAO,eAAeC,+BAA+B1F,MAIpD;IACC,OAAO,MAAMZ,KACViB,MAAM,CAAC;QACN8B,OAAO1C,qBAAqB0C,KAAK;QACjCtC,gBAAgBJ,qBAAqBI,cAAc;QACnDoC,eAAexC,qBAAqBwC,aAAa;QACjDC,sBAAsBzC,qBAAqByC,oBAAoB;QAC/DE,WAAW3C,qBAAqB2C,SAAS;QACzCW,iBAAiBtD,qBAAqBsD,eAAe;QACrDI,eAAe1D,qBAAqB0D,aAAa;QACjDC,kBAAkB3D,qBAAqB2D,gBAAgB;QACvD/B,gBAAgB5B,qBAAqB4B,cAAc;QACnD2C,cAAcvE,qBAAqBuE,YAAY;QAC/CE,WAAWzE,qBAAqByE,SAAS;QACzCgB,iBAAiBzF,qBAAqByF,eAAe;IACvD,GACC3E,IAAI,CAACd,sBACLe,KAAK,CACJvC,IACEK,GAAGmB,qBAAqBa,WAAW,EAAEN,OAAOM,WAAW,GACvDxB,GAAG,CAAC,EAAEW,qBAAqByF,eAAe,CAAC,YAAY,EAAElF,OAAO2F,aAAa,CAAC,yBAAyB,CAAC,GAG3G/B,OAAO,CAACvF,KAAKoB,qBAAqByF,eAAe,GAAG7G,KAAKoB,qBAAqBiC,EAAE,GAChFG,KAAK,CAAC7B,OAAO6B,KAAK,IAAI;AAC3B;AAEA,OAAO,eAAe+D,2CACpBhG,EAAM,EACNI,MAIC;IAED,IAAIA,OAAO6F,iBAAiB,CAACvE,MAAM,KAAK,GAAG,OAAO,EAAE;IAEpD,MAAMwE,gBAAgB,MAAMlG,GACzBS,MAAM,CAAC;QACN+C,kBAAkB3D,qBAAqB2D,gBAAgB;QACvDhF,OAAOA;IACT,GACCmC,IAAI,CAACd,sBACLe,KAAK,CACJvC,IACEK,GAAGmB,qBAAqBa,WAAW,EAAEN,OAAOM,WAAW,GACvDhC,GAAGmB,qBAAqB0D,aAAa,EAAEnD,OAAOmD,aAAa,GAC3D3E,QAAQiB,qBAAqB2D,gBAAgB,EAAEpD,OAAO6F,iBAAiB,IAG1EnF,OAAO,CAACjB,qBAAqB2D,gBAAgB,EAC7C2C,MAAM,CAACjH,GAAG,CAAC,YAAY,CAAC;IAE3B,MAAMkH,6BAA6BF,cAAcG,OAAO,CAAC,CAACnF,MACxDA,IAAIsC,gBAAgB,GAAG;YAACtC,IAAIsC,gBAAgB;SAAC,GAAG,EAAE;IAEpD,IAAI4C,2BAA2B1E,MAAM,GAAG,GAAG;QACzCzD,SAASqI,IAAI,CACX;YACE5F,aAAaN,OAAOM,WAAW;YAC/B6C,eAAenD,OAAOmD,aAAa;YACnC0C,mBAAmBG;QACrB,GACA;IAEJ;IAEA,MAAMG,SAAS,MAAMvG,GAAGE,OAAO,CAS5BhB,GAAG,CAAC;wBACe,EAAEW,qBAAqB2D,gBAAgB,CAAC;MAC1D,EAAE3D,qBAAqBwC,aAAa,CAAC;MACrC,EAAExC,qBAAqByC,oBAAoB,CAAC;MAC5C,EAAEzC,qBAAqB0C,KAAK,CAAC;MAC7B,EAAE1C,qBAAqBI,cAAc,CAAC;MACtC,EAAEJ,qBAAqB2D,gBAAgB,CAAC;MACxC,EAAE3D,qBAAqByE,SAAS,CAAC;MACjC,EAAEzE,qBAAqByF,eAAe,CAAC;MACvC,EAAEzF,qBAAqB2G,uBAAuB,CAAC;SAC5C,EAAE3G,qBAAqB;;MAE1B,EAAEA,qBAAqBa,WAAW,CAAC,GAAG,EAAEN,OAAOM,WAAW,CAAC;UACvD,EAAEb,qBAAqB0D,aAAa,CAAC,GAAG,EAAEnD,OAAOmD,aAAa,CAAC;UAC/D,EAAE1D,qBAAqB2D,gBAAgB,CAAC,KAAK,EAAEtE,IAAIuH,IAAI,CACzDrG,OAAO6F,iBAAiB,CAAChF,GAAG,CAAC,CAACuC,mBAAqBtE,GAAG,CAAC,EAAEsE,iBAAiB,CAAC,GAC3EtE,GAAG,CAAC,EAAE,CAAC,EACP;aACK,EAAEW,qBAAqB2D,gBAAgB,CAAC,EAAE,EAAE3D,qBAAqByE,SAAS,CAAC,OAAO,EAAEzE,qBAAqBI,cAAc,CAAC;EACnI,CAAC;IAED,OAAOsG,OAAOG,IAAI,CAACzF,GAAG,CAAC,CAACC,MAAS,CAAA;YAC/BmB,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuD,kBAAkBtC,IAAIsC,gBAAgB;YACtCc,WAAWqC,OAAOzF,IAAIoD,SAAS;YAC/BgB,iBAAiBqB,OAAOzF,IAAIoE,eAAe;YAC3CkB,yBAAyBtF,IAAIsF,uBAAuB,GAChDG,OAAOzF,IAAIsF,uBAAuB,IAClC;QACN,CAAA;AACF;AAEA,SAASG,OAAOd,KAAoB;IAClC,OAAOA,iBAAiBe,OAAOf,QAAQ,IAAIe,KAAKf;AAClD;AAEA,OAAO,eAAegB,iBAAiBzG,MAItC;IACC,MAAM,CAACc,IAAI,GAAG,MAAM1B,KACjBiB,MAAM,CAAC;QAACqB,IAAIjC,qBAAqBiC,EAAE;IAAA,GACnCnB,IAAI,CAACd,sBACLe,KAAK,CACJvC,IACEK,GAAGmB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc,GAC7DvB,GAAGmB,qBAAqBsD,eAAe,EAAE/C,OAAO+C,eAAe,GAC/D/C,OAAOmC,KAAK,KAAKuE,YAAYA,YAAYpI,GAAGmB,qBAAqB0C,KAAK,EAAEnC,OAAOmC,KAAK,IAGvFN,KAAK,CAAC;IAET,OAAOf,QAAQ4F;AACjB;AAEA,OAAO,eAAeC,gBAAgB3G,MAIrC;IAcC,MAAMmG,SAAS,MAAM/G,KAAKoC,WAAW,CAAC,OAAO5B;QAC3C,MAAMgH,UAAU,MAAMhH,GACnByE,MAAM,CAAC5E,sBACP6E,GAAG,CAAC;YACHW,kBAAkBnG,GAAG,CAAC,SAAS,EAAEW,qBAAqBwF,gBAAgB,CAAC,QAAQ,CAAC;YAChFC,iBAAiBpG,GAAG,CAAC,KAAK,CAAC;QAC7B,GACC0B,KAAK,CACJvC,IACEK,GAAGmB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc,GAC7DvB,GAAGmB,qBAAqBsD,eAAe,EAAE/C,OAAO+C,eAAe,IAGlER,SAAS,CAAC;YACT6D,yBAAyB3G,qBAAqB2G,uBAAuB;YACrEnE,eAAexC,qBAAqBwC,aAAa;YACjDC,sBAAsBzC,qBAAqByC,oBAAoB;YAC/DC,OAAO1C,qBAAqB0C,KAAK;YACjCtC,gBAAgBJ,qBAAqBI,cAAc;YACnDuC,WAAW3C,qBAAqB2C,SAAS;YACzC9B,aAAab,qBAAqBa,WAAW;YAC7CyC,iBAAiBtD,qBAAqBsD,eAAe;QACvD;QAEF,MAAMjC,MAAM8F,OAAO,CAAC,EAAE;QACtB,IAAI,CAAC9F,KAAK,MAAM,IAAI7B,iCAAiCe,OAAOH,cAAc;QAE1E,MAAM,CAACgH,SAAS,GAAG,MAAMjH,GACtBS,MAAM,CAAC;YAACyG,kBAAkBtH,eAAesH,gBAAgB;QAAA,GACzDvG,IAAI,CAACf,gBACLgB,KAAK,CAAClC,GAAGkB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDlB,KAAK,CAAC,GACN6B,GAAG,CAAC;QAEP,MAAM,CAACH,QAAQ,GAAG,MAAM3D,GACrByE,MAAM,CAAC7E,gBACP8E,GAAG,CAAC;YACHwC,kBAAkB9G,OAAO8G,gBAAgB,IAAI;YAC7CC,4BAA4B/G,OAAO8G,gBAAgB,GAAGhI,GAAG,CAAC,KAAK,CAAC,GAAG;YACnE0F,WAAW1F,GAAG,CAAC,KAAK,CAAC;QACvB,GACC0B,KAAK,CAAClC,GAAGkB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDR,SAAS,CAAC;YAACuE,kBAAkBtH,eAAesH,gBAAgB;QAAA;QAE/D,OAAO;YACLhG;YACAkG,0BAA0BH,UAAUC,oBAAoB;YACxDG,yBAAyB1D,SAASuD,oBAAoB;QACxD;IACF;IAEA,MAAMhG,MAAMqF,OAAOrF,GAAG;IACtB,IAAI,CAACA,KAAK,MAAM,IAAI7B,iCAAiCe,OAAOH,cAAc;IAC1E,OAAO;QACLqH,uBAAuBpG,IAAIsF,uBAAuB,KAAK;QACvDY,0BAA0Bb,OAAOa,wBAAwB;QACzDC,yBAAyBd,OAAOc,uBAAuB;QACvDE,qBAAqB;YACnBlF,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;YACxB9B,aAAaQ,IAAIR,WAAW;YAC5ByC,iBAAiBjC,IAAIiC,eAAe;QACtC;IACF;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAeqE,8BAA8BpH,MAEnD;IACC,MAAMZ,KAAKoC,WAAW,CAAC,OAAO5B;QAC5B,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,wFAAwF;QACxF,2FAA2F;QAC3F,wFAAwF;QACxF,MAAMD,GACHmE,MAAM,CAACzE,sBACPkB,KAAK,CAAClC,GAAGgB,qBAAqBO,cAAc,EAAEG,OAAOH,cAAc;QACtE,MAAMD,GACHyE,MAAM,CAAC5E,sBACP6E,GAAG,CAAC;YACH8B,yBAAyBtH,GAAG,CAAC,SAAS,EAAEW,qBAAqB2G,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACC5F,KAAK,CAAClC,GAAGmB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc;IACxE;AACF;AAEA,OAAO,eAAewH,iBAAiBrH,MAA0B;IAC/D,IAAIA,OAAOsH,MAAM,CAAChG,MAAM,KAAK,GAAG;IAEhC,MAAMlC,KAAKoC,WAAW,CAAC,OAAO5B;QAC5B,MAAMA,GAAGmE,MAAM,CAACzE,sBAAsBkB,KAAK,CAAChC,QAAQc,qBAAqB6C,KAAK,EAAEnC,OAAOsH,MAAM;QAC7F,MAAM1H,GACHyE,MAAM,CAAC5E,sBACP6E,GAAG,CAAC;YACH8B,yBAAyBtH,GAAG,CAAC,SAAS,EAAEW,qBAAqB2G,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACC5F,KAAK,CAAChC,QAAQiB,qBAAqB0C,KAAK,EAAEnC,OAAOsH,MAAM;IAC5D;AACF"}
|
package/dist/db/rate-limits.d.ts
CHANGED
|
@@ -1,18 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
expiresAt: Date;
|
|
7
|
-
timeoutMs: number;
|
|
8
|
-
}
|
|
9
|
-
export interface ConsumeRunnersRateLimitResult {
|
|
10
|
-
count: number;
|
|
11
|
-
expiresAt: Date;
|
|
12
|
-
}
|
|
13
|
-
export declare function consumeRunnersRateLimit(params: ConsumeRunnersRateLimitParams): Promise<ConsumeRunnersRateLimitResult>;
|
|
14
|
-
export declare function pruneExpiredRunnersRateLimits(params?: {
|
|
1
|
+
import { type ConsumeRateLimitParams, type ConsumeRateLimitResult } from '@shipfox/node-rate-limit';
|
|
2
|
+
export type ConsumeRunnersRateLimitParams = ConsumeRateLimitParams<string, string>;
|
|
3
|
+
export type ConsumeRunnersRateLimitResult = ConsumeRateLimitResult;
|
|
4
|
+
export declare const consumeRunnersRateLimit: (params: ConsumeRateLimitParams<string, string>) => Promise<ConsumeRateLimitResult>;
|
|
5
|
+
export declare const pruneExpiredRunnersRateLimits: (params?: {
|
|
15
6
|
now?: Date | undefined;
|
|
16
7
|
minIntervalMs?: number | undefined;
|
|
17
|
-
})
|
|
8
|
+
}) => Promise<number | undefined>;
|
|
18
9
|
//# sourceMappingURL=rate-limits.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rate-limits.d.ts","sourceRoot":"","sources":["../../src/db/rate-limits.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rate-limits.d.ts","sourceRoot":"","sources":["../../src/db/rate-limits.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAE5B,MAAM,0BAA0B,CAAC;AAiDlC,MAAM,MAAM,6BAA6B,GAAG,sBAAsB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACnF,MAAM,MAAM,6BAA6B,GAAG,sBAAsB,CAAC;AACnE,eAAO,MAAM,uBAAuB,qFAAsB,CAAC;AAC3D,eAAO,MAAM,6BAA6B;;;iCAAoB,CAAC"}
|
package/dist/db/rate-limits.js
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
|
+
import { createRateLimitPersistence } from '@shipfox/node-rate-limit';
|
|
1
2
|
import { lt, sql } from 'drizzle-orm';
|
|
2
3
|
import { db } from './db.js';
|
|
3
4
|
import { runnersRateLimits } from './schema/rate-limits.js';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
const persistence = createRateLimitPersistence({
|
|
6
|
+
transaction: (callback)=>db().transaction(callback),
|
|
7
|
+
setStatementTimeout: async (transaction, timeoutMs)=>{
|
|
8
|
+
await transaction.execute(sql`select set_config('statement_timeout', ${`${timeoutMs}ms`}, true)`);
|
|
9
|
+
},
|
|
10
|
+
consume: async (transaction, params)=>{
|
|
11
|
+
const rows = await transaction.insert(runnersRateLimits).values({
|
|
8
12
|
action: params.action,
|
|
9
13
|
scope: params.scope,
|
|
10
14
|
identifierHmac: params.identifierHmac,
|
|
@@ -26,19 +30,14 @@ export async function consumeRunnersRateLimit(params) {
|
|
|
26
30
|
count: runnersRateLimits.count,
|
|
27
31
|
expiresAt: runnersRateLimits.expiresAt
|
|
28
32
|
});
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if (minIntervalMs > 0 && now.getTime() < nextPruneAt) return undefined;
|
|
39
|
-
nextPruneAt = now.getTime() + minIntervalMs;
|
|
40
|
-
const result = await db().delete(runnersRateLimits).where(lt(runnersRateLimits.expiresAt, now));
|
|
41
|
-
return result.rowCount ?? 0;
|
|
42
|
-
}
|
|
33
|
+
return rows[0];
|
|
34
|
+
},
|
|
35
|
+
prune: async (now)=>{
|
|
36
|
+
const result = await db().delete(runnersRateLimits).where(lt(runnersRateLimits.expiresAt, now));
|
|
37
|
+
return result.rowCount ?? 0;
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
export const consumeRunnersRateLimit = persistence.consume;
|
|
41
|
+
export const pruneExpiredRunnersRateLimits = persistence.prune;
|
|
43
42
|
|
|
44
43
|
//# sourceMappingURL=rate-limits.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/db/rate-limits.ts"],"sourcesContent":["import {lt, sql} from 'drizzle-orm';\nimport {db} from './db.js';\nimport {runnersRateLimits} from './schema/rate-limits.js';\n\
|
|
1
|
+
{"version":3,"sources":["../../src/db/rate-limits.ts"],"sourcesContent":["import {\n type ConsumeRateLimitParams,\n type ConsumeRateLimitResult,\n createRateLimitPersistence,\n} from '@shipfox/node-rate-limit';\nimport {lt, sql} from 'drizzle-orm';\nimport {db} from './db.js';\nimport {runnersRateLimits} from './schema/rate-limits.js';\n\ntype RunnersRateLimitTransaction = Parameters<\n Parameters<ReturnType<typeof db>['transaction']>[0]\n>[0];\n\nconst persistence = createRateLimitPersistence<RunnersRateLimitTransaction>({\n transaction: (callback) => db().transaction(callback),\n setStatementTimeout: async (transaction, timeoutMs) => {\n await transaction.execute(\n sql`select set_config('statement_timeout', ${`${timeoutMs}ms`}, true)`,\n );\n },\n consume: async (transaction, params) => {\n const rows = await transaction\n .insert(runnersRateLimits)\n .values({\n action: params.action,\n scope: params.scope,\n identifierHmac: params.identifierHmac,\n windowStart: params.windowStart,\n count: 1,\n expiresAt: params.expiresAt,\n })\n .onConflictDoUpdate({\n target: [\n runnersRateLimits.action,\n runnersRateLimits.scope,\n runnersRateLimits.identifierHmac,\n runnersRateLimits.windowStart,\n ],\n set: {\n count: sql`${runnersRateLimits.count} + 1`,\n updatedAt: sql`now()`,\n },\n })\n .returning({count: runnersRateLimits.count, expiresAt: runnersRateLimits.expiresAt});\n\n return rows[0];\n },\n prune: async (now) => {\n const result = await db().delete(runnersRateLimits).where(lt(runnersRateLimits.expiresAt, now));\n return result.rowCount ?? 0;\n },\n});\n\nexport type ConsumeRunnersRateLimitParams = ConsumeRateLimitParams<string, string>;\nexport type ConsumeRunnersRateLimitResult = ConsumeRateLimitResult;\nexport const consumeRunnersRateLimit = persistence.consume;\nexport const pruneExpiredRunnersRateLimits = persistence.prune;\n"],"names":["createRateLimitPersistence","lt","sql","db","runnersRateLimits","persistence","transaction","callback","setStatementTimeout","timeoutMs","execute","consume","params","rows","insert","values","action","scope","identifierHmac","windowStart","count","expiresAt","onConflictDoUpdate","target","set","updatedAt","returning","prune","now","result","delete","where","rowCount","consumeRunnersRateLimit","pruneExpiredRunnersRateLimits"],"mappings":"AAAA,SAGEA,0BAA0B,QACrB,2BAA2B;AAClC,SAAQC,EAAE,EAAEC,GAAG,QAAO,cAAc;AACpC,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,iBAAiB,QAAO,0BAA0B;AAM1D,MAAMC,cAAcL,2BAAwD;IAC1EM,aAAa,CAACC,WAAaJ,KAAKG,WAAW,CAACC;IAC5CC,qBAAqB,OAAOF,aAAaG;QACvC,MAAMH,YAAYI,OAAO,CACvBR,GAAG,CAAC,uCAAuC,EAAE,GAAGO,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC;IAE1E;IACAE,SAAS,OAAOL,aAAaM;QAC3B,MAAMC,OAAO,MAAMP,YAChBQ,MAAM,CAACV,mBACPW,MAAM,CAAC;YACNC,QAAQJ,OAAOI,MAAM;YACrBC,OAAOL,OAAOK,KAAK;YACnBC,gBAAgBN,OAAOM,cAAc;YACrCC,aAAaP,OAAOO,WAAW;YAC/BC,OAAO;YACPC,WAAWT,OAAOS,SAAS;QAC7B,GACCC,kBAAkB,CAAC;YAClBC,QAAQ;gBACNnB,kBAAkBY,MAAM;gBACxBZ,kBAAkBa,KAAK;gBACvBb,kBAAkBc,cAAc;gBAChCd,kBAAkBe,WAAW;aAC9B;YACDK,KAAK;gBACHJ,OAAOlB,GAAG,CAAC,EAAEE,kBAAkBgB,KAAK,CAAC,IAAI,CAAC;gBAC1CK,WAAWvB,GAAG,CAAC,KAAK,CAAC;YACvB;QACF,GACCwB,SAAS,CAAC;YAACN,OAAOhB,kBAAkBgB,KAAK;YAAEC,WAAWjB,kBAAkBiB,SAAS;QAAA;QAEpF,OAAOR,IAAI,CAAC,EAAE;IAChB;IACAc,OAAO,OAAOC;QACZ,MAAMC,SAAS,MAAM1B,KAAK2B,MAAM,CAAC1B,mBAAmB2B,KAAK,CAAC9B,GAAGG,kBAAkBiB,SAAS,EAAEO;QAC1F,OAAOC,OAAOG,QAAQ,IAAI;IAC5B;AACF;AAIA,OAAO,MAAMC,0BAA0B5B,YAAYM,OAAO,CAAC;AAC3D,OAAO,MAAMuB,gCAAgC7B,YAAYsB,KAAK,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rate-limit.d.ts","sourceRoot":"","sources":["../../../src/presentation/routes/rate-limit.ts"],"names":[],"mappings":"AACA,OAAO,EAAc,KAAK,YAAY,EAAE,KAAK,cAAc,EAAC,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"rate-limit.d.ts","sourceRoot":"","sources":["../../../src/presentation/routes/rate-limit.ts"],"names":[],"mappings":"AACA,OAAO,EAAc,KAAK,YAAY,EAAE,KAAK,cAAc,EAAC,MAAM,uBAAuB,CAAC;AAoD1F,wBAAgB,wCAAwC,KACxC,SAAS,cAAc,EAAE,OAAO,YAAY,KAAG,OAAO,CAAC,IAAI,CAAC,CAY3E;AAED,wBAAgB,0CAA0C,KAC1C,SAAS,cAAc,EAAE,OAAO,YAAY,KAAG,OAAO,CAAC,IAAI,CAAC,CAc3E"}
|