@shipfox/api-runners 12.2.0 → 12.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +34 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +10 -3
- package/dist/config.js.map +1 -1
- package/dist/core/demand.d.ts +4 -2
- package/dist/core/demand.d.ts.map +1 -1
- package/dist/core/demand.js +22 -6
- package/dist/core/demand.js.map +1 -1
- package/dist/core/entities/reservation.d.ts +1 -0
- package/dist/core/entities/reservation.d.ts.map +1 -1
- package/dist/core/entities/reservation.js.map +1 -1
- package/dist/core/entities/runner-instance.d.ts +3 -0
- package/dist/core/entities/runner-instance.d.ts.map +1 -1
- package/dist/core/entities/runner-instance.js.map +1 -1
- package/dist/core/runner-activation.d.ts +4 -1
- package/dist/core/runner-activation.d.ts.map +1 -1
- package/dist/core/runner-activation.js +32 -3
- package/dist/core/runner-activation.js.map +1 -1
- package/dist/core/runner-control-sessions.d.ts +8 -4
- package/dist/core/runner-control-sessions.d.ts.map +1 -1
- package/dist/core/runner-control-sessions.js +69 -11
- package/dist/core/runner-control-sessions.js.map +1 -1
- package/dist/core/runner-instances.d.ts +1 -0
- package/dist/core/runner-instances.d.ts.map +1 -1
- package/dist/core/runner-instances.js +1 -0
- package/dist/core/runner-instances.js.map +1 -1
- package/dist/db/db.d.ts +68 -0
- package/dist/db/db.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.js +1 -1
- package/dist/db/ephemeral-registration-tokens.js.map +1 -1
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/db/reservation-locks.d.ts +6 -0
- package/dist/db/reservation-locks.d.ts.map +1 -0
- package/dist/db/reservation-locks.js +15 -0
- package/dist/db/reservation-locks.js.map +1 -0
- package/dist/db/reservations.d.ts +17 -3
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +256 -45
- package/dist/db/reservations.js.map +1 -1
- package/dist/db/runner-assignments.d.ts +17 -0
- package/dist/db/runner-assignments.d.ts.map +1 -1
- package/dist/db/runner-assignments.js +103 -8
- package/dist/db/runner-assignments.js.map +1 -1
- package/dist/db/runner-instances.d.ts +5 -2
- package/dist/db/runner-instances.d.ts.map +1 -1
- package/dist/db/runner-instances.js +134 -25
- package/dist/db/runner-instances.js.map +1 -1
- package/dist/db/runner-sessions.d.ts.map +1 -1
- package/dist/db/runner-sessions.js +22 -15
- package/dist/db/runner-sessions.js.map +1 -1
- package/dist/db/runner-states.d.ts +2 -0
- package/dist/db/runner-states.d.ts.map +1 -0
- package/dist/db/runner-states.js +7 -0
- package/dist/db/runner-states.js.map +1 -0
- package/dist/db/schema/reservations.d.ts +18 -0
- package/dist/db/schema/reservations.d.ts.map +1 -1
- package/dist/db/schema/reservations.js +7 -1
- package/dist/db/schema/reservations.js.map +1 -1
- package/dist/db/schema/runner-instances.d.ts +18 -0
- package/dist/db/schema/runner-instances.d.ts.map +1 -1
- package/dist/db/schema/runner-instances.js +8 -0
- package/dist/db/schema/runner-instances.js.map +1 -1
- package/dist/metrics/index.d.ts +2 -2
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js +1 -1
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/instance.d.ts +24 -2
- package/dist/metrics/instance.d.ts.map +1 -1
- package/dist/metrics/instance.js +23 -0
- package/dist/metrics/instance.js.map +1 -1
- package/dist/metrics/service.d.ts.map +1 -1
- package/dist/metrics/service.js +20 -4
- package/dist/metrics/service.js.map +1 -1
- package/dist/presentation/dto/poll-demand.d.ts.map +1 -1
- package/dist/presentation/dto/poll-demand.js +3 -0
- package/dist/presentation/dto/poll-demand.js.map +1 -1
- package/dist/presentation/dto/runner-instances.d.ts +4 -1
- package/dist/presentation/dto/runner-instances.d.ts.map +1 -1
- package/dist/presentation/dto/runner-instances.js +4 -1
- package/dist/presentation/dto/runner-instances.js.map +1 -1
- package/dist/presentation/routes/assign-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/assign-runner-instances.js +3 -3
- package/dist/presentation/routes/assign-runner-instances.js.map +1 -1
- package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
- package/dist/presentation/routes/poll-demand.js +1 -0
- package/dist/presentation/routes/poll-demand.js.map +1 -1
- package/dist/presentation/routes/reconcile-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/reconcile-runner-instances.js +4 -2
- package/dist/presentation/routes/reconcile-runner-instances.js.map +1 -1
- package/dist/presentation/routes/runner-enrollment.d.ts +4 -1
- package/dist/presentation/routes/runner-enrollment.d.ts.map +1 -1
- package/dist/presentation/routes/runner-enrollment.js +11 -6
- package/dist/presentation/routes/runner-enrollment.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0005_previous_lenny_balinger.sql +5 -0
- package/drizzle/meta/0005_snapshot.json +2220 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +4 -4
- package/src/config.test.ts +24 -2
- package/src/config.ts +16 -3
- package/src/core/demand.test.ts +192 -3
- package/src/core/demand.ts +44 -4
- package/src/core/entities/reservation.ts +1 -0
- package/src/core/entities/runner-instance.ts +4 -0
- package/src/core/runner-activation.ts +48 -2
- package/src/core/runner-control-sessions.test.ts +230 -4
- package/src/core/runner-control-sessions.ts +84 -10
- package/src/core/runner-instances.test.ts +19 -0
- package/src/core/runner-instances.ts +2 -0
- package/src/core/runner-sessions.test.ts +121 -1
- package/src/db/ephemeral-registration-tokens.test.ts +20 -1
- package/src/db/ephemeral-registration-tokens.ts +1 -0
- package/src/db/index.ts +1 -0
- package/src/db/reservation-locks.ts +22 -0
- package/src/db/reservations.test.ts +1110 -31
- package/src/db/reservations.ts +434 -105
- package/src/db/runner-assignments.test.ts +105 -1
- package/src/db/runner-assignments.ts +178 -9
- package/src/db/runner-instances.test.ts +496 -2
- package/src/db/runner-instances.ts +247 -56
- package/src/db/runner-sessions.ts +26 -15
- package/src/db/runner-states.ts +7 -0
- package/src/db/schema/reservations.ts +5 -1
- package/src/db/schema/runner-instances.ts +9 -0
- package/src/metrics/index.ts +10 -1
- package/src/metrics/instance.ts +59 -2
- package/src/metrics/service.test.ts +106 -0
- package/src/metrics/service.ts +24 -4
- package/src/presentation/dto/poll-demand.ts +3 -0
- package/src/presentation/dto/runner-instances.ts +8 -1
- package/src/presentation/routes/assign-runner-instances.ts +6 -2
- package/src/presentation/routes/late-runner-enrollment.test.ts +447 -0
- package/src/presentation/routes/poll-demand.test.ts +79 -6
- package/src/presentation/routes/poll-demand.ts +1 -0
- package/src/presentation/routes/reconcile-runner-instances.test.ts +34 -2
- package/src/presentation/routes/reconcile-runner-instances.ts +7 -1
- package/src/presentation/routes/runner-enrollment.test.ts +316 -3
- package/src/presentation/routes/runner-enrollment.ts +8 -5
- package/test/factories/reservation.ts +2 -0
- package/test/factories/runner-instance.ts +6 -0
- package/test/fixtures/late-runner-enrollment.ts +103 -0
- package/test/index.ts +5 -0
- package/tsconfig.build.tsbuildinfo +1 -1
package/dist/db/reservations.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { canonicalizeLabels } from '@shipfox/runner-labels';
|
|
2
|
-
import { and, arrayContains, asc, eq, exists, gt, inArray, isNotNull, isNull, lt, or, sql } from 'drizzle-orm';
|
|
2
|
+
import { and, arrayContains, asc, eq, exists, gt, inArray, isNotNull, isNull, lt, notExists, notInArray, or, sql } from 'drizzle-orm';
|
|
3
|
+
import { recordProviderRunnerActivationOutcome } from '#metrics/instance.js';
|
|
3
4
|
import { db } from './db.js';
|
|
5
|
+
import { lockRunnerReservationAdvisoryKeysTx } from './reservation-locks.js';
|
|
6
|
+
import { terminalStates } from './runner-states.js';
|
|
4
7
|
import { pendingJobExecutions } from './schema/pending-job-executions.js';
|
|
5
8
|
import { provisionerCapabilitySnapshots } from './schema/provisioner-capability-snapshots.js';
|
|
6
9
|
import { provisionerTokens } from './schema/provisioner-tokens.js';
|
|
@@ -9,13 +12,20 @@ import { runnerActivationTokens } from './schema/runner-activation-tokens.js';
|
|
|
9
12
|
import { runnerControlSessions } from './schema/runner-control-sessions.js';
|
|
10
13
|
import { providerRunners } from './schema/runner-instances.js';
|
|
11
14
|
export async function pollDemandAndReserve(params) {
|
|
12
|
-
|
|
15
|
+
const result = await db().transaction(async (tx)=>{
|
|
13
16
|
return await pollDemandAndReserveTx(tx, params);
|
|
14
17
|
});
|
|
18
|
+
return {
|
|
19
|
+
stats: result.stats,
|
|
20
|
+
reservations: result.reservations
|
|
21
|
+
};
|
|
15
22
|
}
|
|
16
23
|
export async function pollDemandAndReserveTx(tx, params) {
|
|
17
24
|
await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId}))`);
|
|
18
|
-
return await pollDemandAndReserveLockedTx(tx,
|
|
25
|
+
return await pollDemandAndReserveLockedTx(tx, {
|
|
26
|
+
...params,
|
|
27
|
+
scope: 'workspace'
|
|
28
|
+
});
|
|
19
29
|
}
|
|
20
30
|
export async function pollInstallationDemandAndReserve(params) {
|
|
21
31
|
const candidateWorkspaceIds = await listInstallationDemandWorkspaceIds(params.eligibleWorkspaceIds);
|
|
@@ -35,32 +45,43 @@ export async function pollInstallationDemandAndReserve(params) {
|
|
|
35
45
|
const locked = lockResult.rows[0];
|
|
36
46
|
if (!locked?.locked) return {
|
|
37
47
|
stats: [],
|
|
38
|
-
reservations: []
|
|
48
|
+
reservations: [],
|
|
49
|
+
newlyReservedUnits: []
|
|
39
50
|
};
|
|
40
51
|
return await pollDemandAndReserveLockedTx(tx, {
|
|
41
52
|
workspaceId,
|
|
42
53
|
provisionerId: params.provisionerId,
|
|
43
54
|
maxReservations: remainingMaxReservations,
|
|
44
55
|
ttlSeconds: params.ttlSeconds,
|
|
56
|
+
...params.activationGraceSeconds !== undefined ? {
|
|
57
|
+
activationGraceSeconds: params.activationGraceSeconds
|
|
58
|
+
} : {},
|
|
45
59
|
templates: remainingTemplates.map((template)=>({
|
|
46
60
|
...template,
|
|
47
61
|
availableSlots: template.remainingSlots
|
|
48
62
|
})),
|
|
49
|
-
capabilityWindowSeconds: params.capabilityWindowSeconds
|
|
63
|
+
capabilityWindowSeconds: params.capabilityWindowSeconds,
|
|
64
|
+
scope: 'installation'
|
|
50
65
|
});
|
|
51
66
|
});
|
|
52
67
|
results.push(result);
|
|
53
68
|
consumeInstallationTemplateSlots(remainingTemplates, result.reservations);
|
|
54
69
|
params.onReservations?.(result.reservations);
|
|
55
|
-
remainingMaxReservations -= result.
|
|
70
|
+
remainingMaxReservations -= result.newlyReservedUnits.reduce((total, reservation)=>total + reservation.count, 0);
|
|
56
71
|
}
|
|
72
|
+
const newlyReservedCount = results.reduce((total, result)=>total + result.newlyReservedUnits.reduce((subtotal, reservation)=>subtotal + reservation.count, 0), 0);
|
|
57
73
|
return {
|
|
58
74
|
stats: results.flatMap((result)=>result.stats),
|
|
59
|
-
reservations: results.flatMap((result)=>result.reservations)
|
|
75
|
+
reservations: results.flatMap((result)=>result.reservations),
|
|
76
|
+
...newlyReservedCount > 0 ? {
|
|
77
|
+
newlyReservedCount
|
|
78
|
+
} : {}
|
|
60
79
|
};
|
|
61
80
|
}
|
|
62
|
-
function consumeInstallationTemplateSlots(templates,
|
|
63
|
-
|
|
81
|
+
function consumeInstallationTemplateSlots(templates, launchGrants) {
|
|
82
|
+
// Adopted runners are already included in the running count behind availableSlots.
|
|
83
|
+
// Only units that still need a launch consume advertised template capacity.
|
|
84
|
+
for (const reservation of launchGrants){
|
|
64
85
|
const satisfyingTemplates = templates.filter((template)=>isSubset(reservation.labels, template.labels)).sort((a, b)=>a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey));
|
|
65
86
|
drawSlots(satisfyingTemplates, reservation.count);
|
|
66
87
|
}
|
|
@@ -89,10 +110,10 @@ async function pollDemandAndReserveLockedTx(tx, params) {
|
|
|
89
110
|
labelKey(row.requiredLabels),
|
|
90
111
|
row.reserved
|
|
91
112
|
]));
|
|
92
|
-
const activeProvisionerReservationRows = await tx
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
})
|
|
113
|
+
const activeProvisionerReservationRows = await listActiveProvisionerReservationRowsTx(tx, {
|
|
114
|
+
workspaceId: params.workspaceId,
|
|
115
|
+
provisionerId: params.provisionerId
|
|
116
|
+
});
|
|
96
117
|
const templates = params.templates.map((template)=>({
|
|
97
118
|
templateKey: template.templateKey,
|
|
98
119
|
labels: [
|
|
@@ -103,6 +124,7 @@ async function pollDemandAndReserveLockedTx(tx, params) {
|
|
|
103
124
|
deductProvisionerReservations(templates, activeProvisionerReservationRows);
|
|
104
125
|
const stats = [];
|
|
105
126
|
const grants = [];
|
|
127
|
+
const newlyReservedUnits = [];
|
|
106
128
|
let remainingMaxReservations = params.maxReservations;
|
|
107
129
|
const workspaceIdField = params.capabilityWindowSeconds === undefined ? {} : {
|
|
108
130
|
workspaceId: params.workspaceId
|
|
@@ -116,34 +138,68 @@ async function pollDemandAndReserveLockedTx(tx, params) {
|
|
|
116
138
|
const grant = Math.min(unreserved, capacity, remainingMaxReservations);
|
|
117
139
|
let reservedAfterGrant = reserved;
|
|
118
140
|
if (grant > 0 && params.maxReservations > 0) {
|
|
119
|
-
const
|
|
120
|
-
workspaceId: params.workspaceId,
|
|
141
|
+
const idleRunners = await listIdleRunnerInstancesTx(tx, {
|
|
121
142
|
provisionerId: params.provisionerId,
|
|
122
143
|
requiredLabels: demand.requiredLabels,
|
|
123
144
|
count: grant,
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
id: reservations.id,
|
|
127
|
-
expiresAt: reservations.expiresAt
|
|
145
|
+
workspaceId: params.workspaceId,
|
|
146
|
+
scope: params.scope
|
|
128
147
|
});
|
|
129
|
-
if (!inserted) throw new Error('Insert returned no rows');
|
|
130
148
|
remainingMaxReservations -= grant;
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
149
|
+
const boundCount = idleRunners.length;
|
|
150
|
+
if (boundCount > 0) {
|
|
151
|
+
const [boundReservation] = await tx.insert(reservations).values({
|
|
152
|
+
workspaceId: params.workspaceId,
|
|
153
|
+
provisionerId: params.provisionerId,
|
|
154
|
+
requiredLabels: demand.requiredLabels,
|
|
155
|
+
count: boundCount,
|
|
156
|
+
kind: 'bound',
|
|
157
|
+
expiresAt: sql`now() + (${params.activationGraceSeconds ?? params.ttlSeconds} || ' seconds')::interval`
|
|
158
|
+
}).returning({
|
|
159
|
+
id: reservations.id
|
|
160
|
+
});
|
|
161
|
+
if (!boundReservation) throw new Error('Insert returned no rows');
|
|
162
|
+
await bindIdleRunnerInstancesTx(tx, {
|
|
163
|
+
provisionerId: params.provisionerId,
|
|
164
|
+
reservationId: boundReservation.id,
|
|
165
|
+
workspaceId: params.workspaceId,
|
|
166
|
+
requiredLabels: demand.requiredLabels,
|
|
167
|
+
scope: params.scope,
|
|
168
|
+
idleRunners
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
const launchCount = grant - boundCount;
|
|
172
|
+
drawSlots(satisfyingTemplates, launchCount);
|
|
173
|
+
let launchReservation;
|
|
174
|
+
if (launchCount > 0) {
|
|
175
|
+
const [inserted] = await tx.insert(reservations).values({
|
|
176
|
+
workspaceId: params.workspaceId,
|
|
177
|
+
provisionerId: params.provisionerId,
|
|
178
|
+
requiredLabels: demand.requiredLabels,
|
|
179
|
+
count: launchCount,
|
|
180
|
+
kind: 'launch',
|
|
181
|
+
expiresAt: sql`now() + (${params.ttlSeconds} || ' seconds')::interval`
|
|
182
|
+
}).returning({
|
|
183
|
+
id: reservations.id,
|
|
184
|
+
expiresAt: reservations.expiresAt
|
|
185
|
+
});
|
|
186
|
+
if (!inserted) throw new Error('Insert returned no rows');
|
|
187
|
+
launchReservation = inserted;
|
|
188
|
+
}
|
|
189
|
+
newlyReservedUnits.push({
|
|
143
190
|
labels: demand.requiredLabels,
|
|
144
|
-
count: grant
|
|
145
|
-
expiresAt: inserted.expiresAt
|
|
191
|
+
count: grant
|
|
146
192
|
});
|
|
193
|
+
reservedAfterGrant += grant;
|
|
194
|
+
if (launchReservation) {
|
|
195
|
+
grants.push({
|
|
196
|
+
reservationId: launchReservation.id,
|
|
197
|
+
...workspaceIdField,
|
|
198
|
+
labels: demand.requiredLabels,
|
|
199
|
+
count: launchCount,
|
|
200
|
+
expiresAt: launchReservation.expiresAt
|
|
201
|
+
});
|
|
202
|
+
}
|
|
147
203
|
}
|
|
148
204
|
stats.push({
|
|
149
205
|
...workspaceIdField,
|
|
@@ -155,22 +211,160 @@ async function pollDemandAndReserveLockedTx(tx, params) {
|
|
|
155
211
|
}
|
|
156
212
|
return {
|
|
157
213
|
stats,
|
|
158
|
-
reservations: grants
|
|
214
|
+
reservations: grants,
|
|
215
|
+
newlyReservedUnits
|
|
159
216
|
};
|
|
160
217
|
}
|
|
161
|
-
async function
|
|
162
|
-
const
|
|
163
|
-
id:
|
|
164
|
-
}).from(
|
|
218
|
+
async function listActiveProvisionerReservationRowsTx(tx, params) {
|
|
219
|
+
const candidateRows = await tx.select({
|
|
220
|
+
id: reservations.id
|
|
221
|
+
}).from(reservations).where(and(eq(reservations.workspaceId, params.workspaceId), eq(reservations.provisionerId, params.provisionerId), gt(reservations.expiresAt, sql`now()`)));
|
|
222
|
+
const candidateIds = candidateRows.map((row)=>row.id);
|
|
223
|
+
if (candidateIds.length === 0) return [];
|
|
224
|
+
// Assignment and provider-report transactions use this key before changing runner
|
|
225
|
+
// links. Lock it before counting so an in-flight assignment cannot expose its unit twice.
|
|
226
|
+
await lockRunnerReservationAdvisoryKeysTx(tx, {
|
|
227
|
+
provisionerId: params.provisionerId,
|
|
228
|
+
reservationIds: candidateIds
|
|
229
|
+
});
|
|
230
|
+
const activeRows = await tx.select({
|
|
231
|
+
id: reservations.id,
|
|
232
|
+
requiredLabels: reservations.requiredLabels,
|
|
233
|
+
count: reservations.count
|
|
234
|
+
}).from(reservations).where(and(eq(reservations.workspaceId, params.workspaceId), eq(reservations.provisionerId, params.provisionerId), gt(reservations.expiresAt, sql`now()`))).orderBy(asc(reservations.id)).for('update');
|
|
235
|
+
if (activeRows.length === 0) return [];
|
|
236
|
+
const activeIds = activeRows.map((row)=>row.id);
|
|
237
|
+
const activeIdSet = new Set(activeIds);
|
|
238
|
+
const usedRunnerRows = await tx.select({
|
|
239
|
+
id: providerRunners.id,
|
|
240
|
+
reservationId: providerRunners.reservationId,
|
|
241
|
+
intendedReservationId: providerRunners.intendedReservationId,
|
|
242
|
+
state: providerRunners.state
|
|
243
|
+
}).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), isNull(providerRunners.reservationReleasedAt), or(inArray(providerRunners.reservationId, activeIds), and(inArray(providerRunners.intendedReservationId, activeIds), notInArray(providerRunners.state, [
|
|
244
|
+
...terminalStates
|
|
245
|
+
])))));
|
|
246
|
+
const usedByReservation = new Map();
|
|
247
|
+
for (const runner of usedRunnerRows){
|
|
248
|
+
if (runner.reservationId && activeIdSet.has(runner.reservationId)) {
|
|
249
|
+
addUsedRunner(usedByReservation, runner.reservationId, runner.id);
|
|
250
|
+
}
|
|
251
|
+
if (runner.intendedReservationId && activeIdSet.has(runner.intendedReservationId) && !terminalStates.includes(runner.state)) {
|
|
252
|
+
addUsedRunner(usedByReservation, runner.intendedReservationId, runner.id);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return activeRows.flatMap((reservation)=>{
|
|
256
|
+
const used = usedByReservation.get(reservation.id)?.size ?? 0;
|
|
257
|
+
const pending = Math.max(0, reservation.count - used);
|
|
258
|
+
return pending > 0 ? [
|
|
259
|
+
{
|
|
260
|
+
requiredLabels: reservation.requiredLabels,
|
|
261
|
+
reserved: pending
|
|
262
|
+
}
|
|
263
|
+
] : [];
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
function addUsedRunner(usedByReservation, reservationId, runnerId) {
|
|
267
|
+
const runnerIds = usedByReservation.get(reservationId);
|
|
268
|
+
if (runnerIds) runnerIds.add(runnerId);
|
|
269
|
+
else usedByReservation.set(reservationId, new Set([
|
|
270
|
+
runnerId
|
|
271
|
+
]));
|
|
272
|
+
}
|
|
273
|
+
// An expired or missing reservation is stale. A live reservation protects a runner that is
|
|
274
|
+
// still booting or waiting for its activation grace period.
|
|
275
|
+
function canBindRunner(tx, params) {
|
|
276
|
+
return and(or(and(isNull(providerRunners.workspaceId), isNull(providerRunners.reservationId)), and(isNotNull(providerRunners.reservationId), params.scope === 'installation' ? undefined : or(isNull(providerRunners.workspaceId), eq(providerRunners.workspaceId, params.workspaceId)), notExists(tx.select({
|
|
277
|
+
id: reservations.id
|
|
278
|
+
}).from(reservations).where(and(eq(reservations.id, providerRunners.reservationId), gt(reservations.expiresAt, sql`now()`)))))), or(isNull(providerRunners.intendedReservationId), notExists(tx.select({
|
|
279
|
+
id: reservations.id
|
|
280
|
+
}).from(reservations).where(and(eq(reservations.id, providerRunners.intendedReservationId), gt(reservations.expiresAt, sql`now()`))))), isNull(providerRunners.reservationReleasedAt), isNull(providerRunners.runnerSessionId));
|
|
281
|
+
}
|
|
282
|
+
function isBindableRunner(tx, params) {
|
|
283
|
+
return and(eq(providerRunners.provisionerId, params.provisionerId), canBindRunner(tx, params), isNotNull(providerRunners.providerRunnerId), eq(providerRunners.state, 'running'), arrayContains(providerRunners.labels, params.requiredLabels), exists(tx.select({
|
|
165
284
|
id: runnerControlSessions.id
|
|
166
|
-
}).from(runnerControlSessions).where(and(eq(runnerControlSessions.runnerInstanceId, providerRunners.id), eq(runnerControlSessions.provisionerId, params.provisionerId), isNull(runnerControlSessions.closedAt), gt(runnerControlSessions.expiresAt, sql`now()`)))))
|
|
167
|
-
|
|
168
|
-
|
|
285
|
+
}).from(runnerControlSessions).where(and(eq(runnerControlSessions.runnerInstanceId, providerRunners.id), eq(runnerControlSessions.provisionerId, params.provisionerId), isNull(runnerControlSessions.closedAt), gt(runnerControlSessions.expiresAt, sql`now()`)))));
|
|
286
|
+
}
|
|
287
|
+
async function listIdleRunnerInstancesTx(tx, params) {
|
|
288
|
+
const lockedRunners = await selectIdleRunnerInstancesTx(tx, params);
|
|
289
|
+
if (lockedRunners.length === 0) return lockedRunners;
|
|
290
|
+
// A SELECT ... FOR UPDATE can wake with a fresh target row but the original
|
|
291
|
+
// statement snapshot for reservation subqueries. Re-check the full predicate
|
|
292
|
+
// in a new statement while retaining the row locks from the first query.
|
|
293
|
+
return await selectIdleRunnerInstancesTx(tx, {
|
|
294
|
+
...params,
|
|
295
|
+
runnerIds: lockedRunners.map((runner)=>runner.id),
|
|
296
|
+
count: lockedRunners.length
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
async function selectIdleRunnerInstancesTx(tx, params) {
|
|
300
|
+
if (params.runnerIds) {
|
|
301
|
+
// The first pass retains the row locks. Recheck the full predicate in a fresh statement so
|
|
302
|
+
// reservation subqueries see the state that was current when the lock was acquired.
|
|
303
|
+
return await tx.select({
|
|
304
|
+
id: providerRunners.id,
|
|
305
|
+
launchKind: providerRunners.launchKind
|
|
306
|
+
}).from(providerRunners).where(and(inArray(providerRunners.id, params.runnerIds), isBindableRunner(tx, params))).orderBy(asc(providerRunners.createdAt), asc(providerRunners.id)).limit(params.count);
|
|
307
|
+
}
|
|
308
|
+
// Select the oldest candidates first, but leave row locking to a second pass. The cleanup
|
|
309
|
+
// path locks runner rows by id, so the locking pass must use that same order. A skipped row
|
|
310
|
+
// rolls back only this nested savepoint and retries the bounded candidate scan, allowing the
|
|
311
|
+
// next-oldest eligible runner to refill the grant without retaining a partial lock set.
|
|
312
|
+
const retrySelection = Symbol('retry bindable runner selection');
|
|
313
|
+
const idleRunners = await (async ()=>{
|
|
314
|
+
while(true){
|
|
315
|
+
try {
|
|
316
|
+
return await tx.transaction(async (lockTx)=>{
|
|
317
|
+
const candidateRunners = await lockTx.select({
|
|
318
|
+
id: providerRunners.id,
|
|
319
|
+
launchKind: providerRunners.launchKind
|
|
320
|
+
}).from(providerRunners).where(isBindableRunner(lockTx, params)).orderBy(asc(providerRunners.createdAt), asc(providerRunners.id)).limit(params.count);
|
|
321
|
+
if (candidateRunners.length === 0) return candidateRunners;
|
|
322
|
+
// PostgreSQL can acquire row locks before sorting a multi-row FOR UPDATE result. Lock
|
|
323
|
+
// each selected candidate individually so both polling and cleanup use id order.
|
|
324
|
+
const lockedRunners = [];
|
|
325
|
+
for (const candidate of [
|
|
326
|
+
...candidateRunners
|
|
327
|
+
].sort(compareRunnerIds)){
|
|
328
|
+
const [runner] = await lockTx.select({
|
|
329
|
+
id: providerRunners.id,
|
|
330
|
+
launchKind: providerRunners.launchKind
|
|
331
|
+
}).from(providerRunners).where(and(eq(providerRunners.id, candidate.id), isBindableRunner(lockTx, params))).for('update');
|
|
332
|
+
if (runner) lockedRunners.push(runner);
|
|
333
|
+
}
|
|
334
|
+
if (lockedRunners.length !== candidateRunners.length) throw retrySelection;
|
|
335
|
+
return lockedRunners;
|
|
336
|
+
});
|
|
337
|
+
} catch (error) {
|
|
338
|
+
if (error !== retrySelection) throw error;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
})();
|
|
342
|
+
return idleRunners;
|
|
343
|
+
}
|
|
344
|
+
async function bindIdleRunnerInstancesTx(tx, params) {
|
|
345
|
+
const idleRunnerIds = params.idleRunners.map((runner)=>runner.id);
|
|
346
|
+
await tx.update(runnerActivationTokens).set({
|
|
347
|
+
revokedAt: sql`now()`
|
|
348
|
+
}).where(and(inArray(runnerActivationTokens.runnerInstanceId, idleRunnerIds), isNull(runnerActivationTokens.consumedAt), isNull(runnerActivationTokens.revokedAt)));
|
|
349
|
+
// listIdleRunnerInstancesTx re-checks the full eligibility predicate in a fresh statement
|
|
350
|
+
// after acquiring row locks. Those locks prevent the validated set from changing before this
|
|
351
|
+
// update runs.
|
|
352
|
+
const boundRunners = await tx.update(providerRunners).set({
|
|
169
353
|
workspaceId: params.workspaceId,
|
|
170
354
|
reservationId: params.reservationId,
|
|
355
|
+
intendedReservationId: null,
|
|
171
356
|
assignedAt: sql`now()`,
|
|
172
357
|
updatedAt: sql`now()`
|
|
173
|
-
}).where(and(inArray(providerRunners.id,
|
|
358
|
+
}).where(and(inArray(providerRunners.id, idleRunnerIds), isBindableRunner(tx, params))).returning({
|
|
359
|
+
id: providerRunners.id,
|
|
360
|
+
launchKind: providerRunners.launchKind
|
|
361
|
+
});
|
|
362
|
+
if (boundRunners.length !== params.idleRunners.length) throw new Error('Locked idle runner set changed before binding');
|
|
363
|
+
const reboundCount = boundRunners.filter((runner)=>runner.launchKind === 'demand').length;
|
|
364
|
+
if (reboundCount > 0) recordProviderRunnerActivationOutcome({
|
|
365
|
+
outcome: 'rebound',
|
|
366
|
+
count: reboundCount
|
|
367
|
+
});
|
|
174
368
|
}
|
|
175
369
|
async function listInstallationDemandWorkspaceIds(eligibleWorkspaceIds) {
|
|
176
370
|
if (eligibleWorkspaceIds.size === 0) return [];
|
|
@@ -212,13 +406,27 @@ export async function deleteReservationsByIds(ids) {
|
|
|
212
406
|
}
|
|
213
407
|
async function deleteReservationsWithCleanupTx(tx, ids, params = {}) {
|
|
214
408
|
if (ids.length === 0) return 0;
|
|
215
|
-
const
|
|
409
|
+
const isAffectedRunner = ()=>or(and(inArray(providerRunners.reservationId, ids), isNull(providerRunners.runnerSessionId), isNull(providerRunners.reservationReleasedAt)), inArray(providerRunners.intendedReservationId, ids));
|
|
410
|
+
const candidateRunners = await tx.select({
|
|
216
411
|
id: providerRunners.id,
|
|
217
412
|
reservationId: providerRunners.reservationId,
|
|
218
413
|
intendedReservationId: providerRunners.intendedReservationId,
|
|
219
414
|
runnerSessionId: providerRunners.runnerSessionId,
|
|
220
415
|
reservationReleasedAt: providerRunners.reservationReleasedAt
|
|
221
|
-
}).from(providerRunners).where(
|
|
416
|
+
}).from(providerRunners).where(isAffectedRunner()).orderBy(asc(providerRunners.id));
|
|
417
|
+
const affectedRunners = [];
|
|
418
|
+
for (const candidate of [
|
|
419
|
+
...candidateRunners
|
|
420
|
+
].sort(compareRunnerIds)){
|
|
421
|
+
const [runner] = await tx.select({
|
|
422
|
+
id: providerRunners.id,
|
|
423
|
+
reservationId: providerRunners.reservationId,
|
|
424
|
+
intendedReservationId: providerRunners.intendedReservationId,
|
|
425
|
+
runnerSessionId: providerRunners.runnerSessionId,
|
|
426
|
+
reservationReleasedAt: providerRunners.reservationReleasedAt
|
|
427
|
+
}).from(providerRunners).where(and(eq(providerRunners.id, candidate.id), isAffectedRunner())).for('update');
|
|
428
|
+
if (runner) affectedRunners.push(runner);
|
|
429
|
+
}
|
|
222
430
|
const reservationRows = await tx.select({
|
|
223
431
|
id: reservations.id
|
|
224
432
|
}).from(reservations).where(and(inArray(reservations.id, ids), params.expiredOnly ? lt(reservations.expiresAt, sql`now()`) : undefined)).for('update');
|
|
@@ -274,6 +482,9 @@ function deductProvisionerReservations(templates, activeReservations) {
|
|
|
274
482
|
drawSlots(satisfyingTemplates, reservation.reserved);
|
|
275
483
|
}
|
|
276
484
|
}
|
|
485
|
+
function compareRunnerIds(left, right) {
|
|
486
|
+
return left.id < right.id ? -1 : left.id > right.id ? 1 : 0;
|
|
487
|
+
}
|
|
277
488
|
function sortDemandRows(rows) {
|
|
278
489
|
return [
|
|
279
490
|
...rows
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/db/reservations.ts"],"sourcesContent":["import {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n and,\n arrayContains,\n asc,\n eq,\n exists,\n gt,\n inArray,\n isNotNull,\n isNull,\n lt,\n or,\n sql,\n} from 'drizzle-orm';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {provisionerCapabilitySnapshots} from './schema/provisioner-capability-snapshots.js';\nimport {provisionerTokens} from './schema/provisioner-tokens.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerActivationTokens} from './schema/runner-activation-tokens.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {providerRunners} from './schema/runner-instances.js';\n\nexport interface ReservationTemplate {\n templateKey: string;\n labels: string[];\n availableSlots: number;\n starting: number;\n running: number;\n}\n\nexport interface DemandStat {\n workspaceId?: string;\n labels: string[];\n queued: number;\n reserved: number;\n oldestQueuedAt: Date;\n}\n\nexport interface ReservationGrant {\n reservationId: string;\n workspaceId?: string;\n labels: string[];\n count: number;\n expiresAt: Date;\n}\n\nexport interface PollDemandAndReserveParams {\n workspaceId: string;\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds?: number;\n}\n\nexport interface InstallationPollDemandAndReserveParams {\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds: number;\n eligibleWorkspaceIds: ReadonlySet<string>;\n signal?: AbortSignal;\n onReservations?: (reservations: ReservationGrant[]) => void;\n}\n\ninterface NormalizedTemplate {\n templateKey: string;\n labels: string[];\n remainingSlots: number;\n}\n\ninterface DemandRow {\n requiredLabels: string[];\n queued: number;\n oldestQueuedAt: Date;\n}\n\nexport async function pollDemandAndReserve(\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n return await db().transaction(async (tx) => {\n return await pollDemandAndReserveTx(tx, params);\n });\n}\n\nexport async function pollDemandAndReserveTx(\n tx: Tx,\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId}))`);\n return await pollDemandAndReserveLockedTx(tx, params);\n}\n\nexport async function pollInstallationDemandAndReserve(\n params: InstallationPollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n const candidateWorkspaceIds = await listInstallationDemandWorkspaceIds(\n params.eligibleWorkspaceIds,\n );\n const results: Array<{stats: DemandStat[]; reservations: ReservationGrant[]}> = [];\n let remainingMaxReservations = params.maxReservations;\n const remainingTemplates = params.templates.map((template) => ({\n ...template,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n for (const workspaceId of candidateWorkspaceIds) {\n if (params.signal?.aborted || remainingMaxReservations === 0) break;\n const result = await db().transaction(async (tx) => {\n const lockResult = await tx.execute<{locked: boolean}>(\n sql`select pg_try_advisory_xact_lock(hashtext(${workspaceId})) as locked`,\n );\n const locked = lockResult.rows[0];\n if (!locked?.locked) return {stats: [], reservations: []};\n return await pollDemandAndReserveLockedTx(tx, {\n workspaceId,\n provisionerId: params.provisionerId,\n maxReservations: remainingMaxReservations,\n ttlSeconds: params.ttlSeconds,\n templates: remainingTemplates.map((template) => ({\n ...template,\n availableSlots: template.remainingSlots,\n })),\n capabilityWindowSeconds: params.capabilityWindowSeconds,\n });\n });\n results.push(result);\n consumeInstallationTemplateSlots(remainingTemplates, result.reservations);\n params.onReservations?.(result.reservations);\n remainingMaxReservations -= result.reservations.reduce(\n (total, reservation) => total + reservation.count,\n 0,\n );\n }\n return {\n stats: results.flatMap((result) => result.stats),\n reservations: results.flatMap((result) => result.reservations),\n };\n}\n\nfunction consumeInstallationTemplateSlots(\n templates: NormalizedTemplate[],\n reservations: ReservationGrant[],\n): void {\n for (const reservation of reservations) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.labels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.count);\n }\n}\n\nasync function pollDemandAndReserveLockedTx(\n tx: Tx,\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n let demandRows = (\n await tx\n .select({\n requiredLabels: pendingJobExecutions.requiredLabels,\n queued: sql<number>`count(*)::int`,\n oldestQueuedAt: sql<Date | string>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(eq(pendingJobExecutions.workspaceId, params.workspaceId))\n .groupBy(pendingJobExecutions.requiredLabels)\n ).map((row) => ({\n ...row,\n oldestQueuedAt: new Date(row.oldestQueuedAt),\n }));\n if (params.capabilityWindowSeconds !== undefined) {\n const capabilityLabels = await listActiveWorkspaceCapabilityLabelsTx(tx, {\n workspaceId: params.workspaceId,\n windowSeconds: params.capabilityWindowSeconds,\n });\n demandRows = demandRows.filter(\n (demand) => !capabilityLabels.some((labels) => isSubset(demand.requiredLabels, labels)),\n );\n }\n\n const activeReservationRows = await tx\n .select({\n requiredLabels: reservations.requiredLabels,\n reserved: sql<number>`coalesce(sum(${reservations.count}), 0)::int`,\n })\n .from(reservations)\n .where(\n and(eq(reservations.workspaceId, params.workspaceId), gt(reservations.expiresAt, sql`now()`)),\n )\n .groupBy(reservations.requiredLabels);\n\n const reservedByLabels = new Map(\n activeReservationRows.map((row) => [labelKey(row.requiredLabels), row.reserved]),\n );\n const activeProvisionerReservationRows = await tx\n .select({\n requiredLabels: reservations.requiredLabels,\n reserved: sql<number>`coalesce(sum(${reservations.count}), 0)::int`,\n })\n .from(reservations)\n .where(\n and(\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .groupBy(reservations.requiredLabels);\n\n const templates = params.templates.map((template) => ({\n templateKey: template.templateKey,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n deductProvisionerReservations(templates, activeProvisionerReservationRows);\n const stats: DemandStat[] = [];\n const grants: ReservationGrant[] = [];\n let remainingMaxReservations = params.maxReservations;\n const workspaceIdField =\n params.capabilityWindowSeconds === undefined ? {} : {workspaceId: params.workspaceId};\n\n for (const demand of sortDemandRows(demandRows)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(demand.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n\n if (satisfyingTemplates.length === 0) continue;\n\n const reserved = reservedByLabels.get(labelKey(demand.requiredLabels)) ?? 0;\n const unreserved = Math.max(0, demand.queued - reserved);\n const capacity = satisfyingTemplates.reduce(\n (total, template) => total + template.remainingSlots,\n 0,\n );\n const grant = Math.min(unreserved, capacity, remainingMaxReservations);\n let reservedAfterGrant = reserved;\n\n if (grant > 0 && params.maxReservations > 0) {\n const [inserted] = await tx\n .insert(reservations)\n .values({\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: grant,\n expiresAt: sql`now() + (${params.ttlSeconds} || ' seconds')::interval`,\n })\n .returning({id: reservations.id, expiresAt: reservations.expiresAt});\n\n if (!inserted) throw new Error('Insert returned no rows');\n\n remainingMaxReservations -= grant;\n drawSlots(satisfyingTemplates, grant);\n await bindIdleRunnerInstancesTx(tx, {\n provisionerId: params.provisionerId,\n reservationId: inserted.id,\n requiredLabels: demand.requiredLabels,\n count: grant,\n workspaceId: params.workspaceId,\n });\n reservedAfterGrant += grant;\n grants.push({\n reservationId: inserted.id,\n ...workspaceIdField,\n labels: demand.requiredLabels,\n count: grant,\n expiresAt: inserted.expiresAt,\n });\n }\n\n stats.push({\n ...workspaceIdField,\n labels: demand.requiredLabels,\n queued: demand.queued,\n reserved: reservedAfterGrant,\n oldestQueuedAt: demand.oldestQueuedAt,\n });\n }\n\n return {stats, reservations: grants};\n}\n\nasync function bindIdleRunnerInstancesTx(\n tx: Tx,\n params: {\n provisionerId: string;\n reservationId: string;\n workspaceId: string;\n requiredLabels: string[];\n count: number;\n },\n): Promise<void> {\n const idleRunners = await tx\n .select({id: providerRunners.id})\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n isNull(providerRunners.workspaceId),\n isNull(providerRunners.reservationId),\n isNull(providerRunners.intendedReservationId),\n isNull(providerRunners.runnerSessionId),\n isNotNull(providerRunners.providerRunnerId),\n eq(providerRunners.state, 'running'),\n arrayContains(providerRunners.labels, params.requiredLabels),\n exists(\n tx\n .select({id: runnerControlSessions.id})\n .from(runnerControlSessions)\n .where(\n and(\n eq(runnerControlSessions.runnerInstanceId, providerRunners.id),\n eq(runnerControlSessions.provisionerId, params.provisionerId),\n isNull(runnerControlSessions.closedAt),\n gt(runnerControlSessions.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n )\n .orderBy(asc(providerRunners.createdAt), asc(providerRunners.id))\n .limit(params.count)\n .for('update');\n\n if (idleRunners.length === 0) return;\n\n await tx\n .update(providerRunners)\n .set({\n workspaceId: params.workspaceId,\n reservationId: params.reservationId,\n assignedAt: sql`now()`,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n inArray(\n providerRunners.id,\n idleRunners.map((runner) => runner.id),\n ),\n isNull(providerRunners.workspaceId),\n isNull(providerRunners.reservationId),\n isNull(providerRunners.runnerSessionId),\n ),\n );\n}\n\nasync function listInstallationDemandWorkspaceIds(eligibleWorkspaceIds: ReadonlySet<string>) {\n if (eligibleWorkspaceIds.size === 0) return [];\n const rows = await db()\n .select({\n workspaceId: pendingJobExecutions.workspaceId,\n oldestQueuedAt: sql<Date>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.workspaceId, [...eligibleWorkspaceIds]))\n .groupBy(pendingJobExecutions.workspaceId)\n .orderBy(\n asc(sql`min(${pendingJobExecutions.createdAt})`),\n asc(pendingJobExecutions.workspaceId),\n );\n return rows.map((row) => row.workspaceId);\n}\n\nexport async function listQueuedDemandWorkspaceIds(): Promise<string[]> {\n const rows = await db()\n .select({workspaceId: pendingJobExecutions.workspaceId})\n .from(pendingJobExecutions)\n .groupBy(pendingJobExecutions.workspaceId);\n return rows.map((row) => row.workspaceId);\n}\n\nasync function listActiveWorkspaceCapabilityLabelsTx(\n tx: Tx,\n params: {workspaceId: string; windowSeconds: number},\n): Promise<string[][]> {\n const rows = await tx\n .select({labels: provisionerCapabilitySnapshots.labels})\n .from(provisionerCapabilitySnapshots)\n .innerJoin(\n provisionerTokens,\n eq(provisionerTokens.id, provisionerCapabilitySnapshots.provisionerId),\n )\n .where(\n and(\n eq(provisionerCapabilitySnapshots.workspaceId, params.workspaceId),\n eq(provisionerTokens.workspaceId, params.workspaceId),\n eq(provisionerTokens.scope, 'workspace'),\n sql`${provisionerCapabilitySnapshots.advertisedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n sql`${provisionerTokens.revokedAt} is null`,\n sql`(${provisionerTokens.expiresAt} is null or ${provisionerTokens.expiresAt} > now())`,\n ),\n );\n return rows.map((row) => row.labels);\n}\n\nexport async function deleteExpiredReservations(params?: {limit?: number}): Promise<number> {\n return await db().transaction(async (tx) => {\n const expiredRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(lt(reservations.expiresAt, sql`now()`))\n .orderBy(asc(reservations.expiresAt))\n .limit(params?.limit ?? 1000);\n return await deleteReservationsWithCleanupTx(\n tx,\n expiredRows.map((reservation) => reservation.id),\n {expiredOnly: true},\n );\n });\n}\n\nexport async function deleteReservationsByIds(ids: string[]): Promise<number> {\n if (ids.length === 0) return 0;\n\n return await db().transaction(async (tx) => {\n return await deleteReservationsWithCleanupTx(tx, ids);\n });\n}\n\nasync function deleteReservationsWithCleanupTx(\n tx: Tx,\n ids: string[],\n params: {expiredOnly?: boolean} = {},\n): Promise<number> {\n if (ids.length === 0) return 0;\n\n const affectedRunners = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n runnerSessionId: providerRunners.runnerSessionId,\n reservationReleasedAt: providerRunners.reservationReleasedAt,\n })\n .from(providerRunners)\n .where(\n or(\n and(\n inArray(providerRunners.reservationId, ids),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n inArray(providerRunners.intendedReservationId, ids),\n ),\n )\n .orderBy(asc(providerRunners.id))\n .for('update');\n const reservationRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n inArray(reservations.id, ids),\n params.expiredOnly ? lt(reservations.expiresAt, sql`now()`) : undefined,\n ),\n )\n .for('update');\n const reservationIds = reservationRows.map((reservation) => reservation.id);\n\n if (reservationIds.length === 0) return 0;\n\n const assignedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.reservationId &&\n reservationIds.includes(runner.reservationId) &&\n !runner.runnerSessionId &&\n !runner.reservationReleasedAt,\n )\n .map((runner) => runner.id);\n const intendedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.intendedReservationId && reservationIds.includes(runner.intendedReservationId),\n )\n .map((runner) => runner.id);\n if (assignedRunnerIds.length > 0) {\n await tx\n .update(runnerActivationTokens)\n .set({revokedAt: sql`now()`})\n .where(\n and(\n inArray(runnerActivationTokens.runnerInstanceId, assignedRunnerIds),\n isNull(runnerActivationTokens.consumedAt),\n isNull(runnerActivationTokens.revokedAt),\n ),\n );\n await tx\n .update(providerRunners)\n .set({\n workspaceId: null,\n reservationId: null,\n assignedAt: null,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n inArray(providerRunners.id, assignedRunnerIds),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n );\n }\n if (intendedRunnerIds.length > 0) {\n await tx\n .update(providerRunners)\n .set({intendedReservationId: null, updatedAt: sql`now()`})\n .where(\n and(\n inArray(providerRunners.id, intendedRunnerIds),\n inArray(providerRunners.intendedReservationId, reservationIds),\n ),\n );\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(inArray(reservations.id, reservationIds))\n .returning({id: reservations.id});\n\n return deleted.length;\n}\n\nexport async function releaseReservationUnits(\n tx: Tx,\n params: {\n workspaceId: string;\n provisionerId: string;\n releases: Array<{reservationId: string; count: number}>;\n },\n): Promise<number> {\n let released = 0;\n\n for (const release of params.releases) {\n if (release.count <= 0) continue;\n\n const decremented = await tx\n .update(reservations)\n .set({count: sql`${reservations.count} - ${release.count}`})\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.count, release.count),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({id: reservations.id});\n\n if (decremented.length > 0) {\n released += release.count;\n continue;\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n sql`${reservations.count} <= ${release.count}`,\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({count: reservations.count});\n\n released += deleted.reduce((total, row) => total + row.count, 0);\n }\n\n return released;\n}\n\nfunction deductProvisionerReservations(\n templates: NormalizedTemplate[],\n activeReservations: {requiredLabels: string[]; reserved: number}[],\n): void {\n for (const reservation of sortReservationRows(activeReservations)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.reserved);\n }\n}\n\nfunction sortDemandRows(rows: DemandRow[]): DemandRow[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return a.oldestQueuedAt.getTime() - b.oldestQueuedAt.getTime();\n });\n}\n\nfunction sortReservationRows<T extends {requiredLabels: string[]}>(rows: T[]): T[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return labelKey(a.requiredLabels).localeCompare(labelKey(b.requiredLabels));\n });\n}\n\nfunction isSubset(requiredLabels: string[], availableLabels: string[]): boolean {\n return requiredLabels.every((label) => availableLabels.includes(label));\n}\n\nfunction drawSlots(templates: NormalizedTemplate[], count: number): void {\n let remaining = count;\n for (const template of templates) {\n if (remaining === 0) return;\n const used = Math.min(template.remainingSlots, remaining);\n template.remainingSlots -= used;\n remaining -= used;\n }\n}\n\nfunction labelKey(labels: string[]): string {\n return JSON.stringify(labels);\n}\n"],"names":["canonicalizeLabels","and","arrayContains","asc","eq","exists","gt","inArray","isNotNull","isNull","lt","or","sql","db","pendingJobExecutions","provisionerCapabilitySnapshots","provisionerTokens","reservations","runnerActivationTokens","runnerControlSessions","providerRunners","pollDemandAndReserve","params","transaction","tx","pollDemandAndReserveTx","execute","workspaceId","pollDemandAndReserveLockedTx","pollInstallationDemandAndReserve","candidateWorkspaceIds","listInstallationDemandWorkspaceIds","eligibleWorkspaceIds","results","remainingMaxReservations","maxReservations","remainingTemplates","templates","map","template","labels","remainingSlots","availableSlots","signal","aborted","result","lockResult","locked","rows","stats","provisionerId","ttlSeconds","capabilityWindowSeconds","push","consumeInstallationTemplateSlots","onReservations","reduce","total","reservation","count","flatMap","satisfyingTemplates","filter","isSubset","sort","a","b","length","templateKey","localeCompare","drawSlots","demandRows","select","requiredLabels","queued","oldestQueuedAt","createdAt","from","where","groupBy","row","Date","undefined","capabilityLabels","listActiveWorkspaceCapabilityLabelsTx","windowSeconds","demand","some","activeReservationRows","reserved","expiresAt","reservedByLabels","Map","labelKey","activeProvisionerReservationRows","deductProvisionerReservations","grants","workspaceIdField","sortDemandRows","get","unreserved","Math","max","capacity","grant","min","reservedAfterGrant","inserted","insert","values","returning","id","Error","bindIdleRunnerInstancesTx","reservationId","idleRunners","intendedReservationId","runnerSessionId","providerRunnerId","state","runnerInstanceId","closedAt","orderBy","limit","for","update","set","assignedAt","updatedAt","runner","size","listQueuedDemandWorkspaceIds","innerJoin","scope","advertisedAt","revokedAt","deleteExpiredReservations","expiredRows","deleteReservationsWithCleanupTx","expiredOnly","deleteReservationsByIds","ids","affectedRunners","reservationReleasedAt","reservationRows","reservationIds","assignedRunnerIds","includes","intendedRunnerIds","consumedAt","deleted","delete","releaseReservationUnits","released","release","releases","decremented","activeReservations","sortReservationRows","specificity","getTime","availableLabels","every","label","remaining","used","JSON","stringify"],"mappings":"AAAA,SAAQA,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,aAAa,EACbC,GAAG,EACHC,EAAE,EACFC,MAAM,EACNC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,EAAE,EACFC,GAAG,QACE,cAAc;AAErB,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,8BAA8B,QAAO,+CAA+C;AAC5F,SAAQC,iBAAiB,QAAO,iCAAiC;AACjE,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,sBAAsB,QAAO,uCAAuC;AAC5E,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAAQC,eAAe,QAAO,+BAA+B;AA0D7D,OAAO,eAAeC,qBACpBC,MAAkC;IAElC,OAAO,MAAMT,KAAKU,WAAW,CAAC,OAAOC;QACnC,OAAO,MAAMC,uBAAuBD,IAAIF;IAC1C;AACF;AAEA,OAAO,eAAeG,uBACpBD,EAAM,EACNF,MAAkC;IAElC,MAAME,GAAGE,OAAO,CAACd,GAAG,CAAC,sCAAsC,EAAEU,OAAOK,WAAW,CAAC,EAAE,CAAC;IACnF,OAAO,MAAMC,6BAA6BJ,IAAIF;AAChD;AAEA,OAAO,eAAeO,iCACpBP,MAA8C;IAE9C,MAAMQ,wBAAwB,MAAMC,mCAClCT,OAAOU,oBAAoB;IAE7B,MAAMC,UAA0E,EAAE;IAClF,IAAIC,2BAA2BZ,OAAOa,eAAe;IACrD,MAAMC,qBAAqBd,OAAOe,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YAC7D,GAAGA,QAAQ;YACXC,QAAQ;mBAAIxC,mBAAmBuC,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACA,KAAK,MAAMf,eAAeG,sBAAuB;QAC/C,IAAIR,OAAOqB,MAAM,EAAEC,WAAWV,6BAA6B,GAAG;QAC9D,MAAMW,SAAS,MAAMhC,KAAKU,WAAW,CAAC,OAAOC;YAC3C,MAAMsB,aAAa,MAAMtB,GAAGE,OAAO,CACjCd,GAAG,CAAC,0CAA0C,EAAEe,YAAY,YAAY,CAAC;YAE3E,MAAMoB,SAASD,WAAWE,IAAI,CAAC,EAAE;YACjC,IAAI,CAACD,QAAQA,QAAQ,OAAO;gBAACE,OAAO,EAAE;gBAAEhC,cAAc,EAAE;YAAA;YACxD,OAAO,MAAMW,6BAA6BJ,IAAI;gBAC5CG;gBACAuB,eAAe5B,OAAO4B,aAAa;gBACnCf,iBAAiBD;gBACjBiB,YAAY7B,OAAO6B,UAAU;gBAC7Bd,WAAWD,mBAAmBE,GAAG,CAAC,CAACC,WAAc,CAAA;wBAC/C,GAAGA,QAAQ;wBACXG,gBAAgBH,SAASE,cAAc;oBACzC,CAAA;gBACAW,yBAAyB9B,OAAO8B,uBAAuB;YACzD;QACF;QACAnB,QAAQoB,IAAI,CAACR;QACbS,iCAAiClB,oBAAoBS,OAAO5B,YAAY;QACxEK,OAAOiC,cAAc,GAAGV,OAAO5B,YAAY;QAC3CiB,4BAA4BW,OAAO5B,YAAY,CAACuC,MAAM,CACpD,CAACC,OAAOC,cAAgBD,QAAQC,YAAYC,KAAK,EACjD;IAEJ;IACA,OAAO;QACLV,OAAOhB,QAAQ2B,OAAO,CAAC,CAACf,SAAWA,OAAOI,KAAK;QAC/ChC,cAAcgB,QAAQ2B,OAAO,CAAC,CAACf,SAAWA,OAAO5B,YAAY;IAC/D;AACF;AAEA,SAASqC,iCACPjB,SAA+B,EAC/BpB,YAAgC;IAEhC,KAAK,MAAMyC,eAAezC,aAAc;QACtC,MAAM4C,sBAAsBxB,UACzByB,MAAM,CAAC,CAACvB,WAAawB,SAASL,YAAYlB,MAAM,EAAED,SAASC,MAAM,GACjEwB,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAEzB,MAAM,CAAC2B,MAAM,GAAGD,EAAE1B,MAAM,CAAC2B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBH,YAAYC,KAAK;IAClD;AACF;AAEA,eAAe/B,6BACbJ,EAAM,EACNF,MAAkC;IAElC,IAAIiD,aAAa,AACf,CAAA,MAAM/C,GACHgD,MAAM,CAAC;QACNC,gBAAgB3D,qBAAqB2D,cAAc;QACnDC,QAAQ9D,GAAW,CAAC,aAAa,CAAC;QAClC+D,gBAAgB/D,GAAkB,CAAC,IAAI,EAAEE,qBAAqB8D,SAAS,CAAC,CAAC,CAAC;IAC5E,GACCC,IAAI,CAAC/D,sBACLgE,KAAK,CAAC1E,GAAGU,qBAAqBa,WAAW,EAAEL,OAAOK,WAAW,GAC7DoD,OAAO,CAACjE,qBAAqB2D,cAAc,CAAA,EAC9CnC,GAAG,CAAC,CAAC0C,MAAS,CAAA;YACd,GAAGA,GAAG;YACNL,gBAAgB,IAAIM,KAAKD,IAAIL,cAAc;QAC7C,CAAA;IACA,IAAIrD,OAAO8B,uBAAuB,KAAK8B,WAAW;QAChD,MAAMC,mBAAmB,MAAMC,sCAAsC5D,IAAI;YACvEG,aAAaL,OAAOK,WAAW;YAC/B0D,eAAe/D,OAAO8B,uBAAuB;QAC/C;QACAmB,aAAaA,WAAWT,MAAM,CAC5B,CAACwB,SAAW,CAACH,iBAAiBI,IAAI,CAAC,CAAC/C,SAAWuB,SAASuB,OAAOb,cAAc,EAAEjC;IAEnF;IAEA,MAAMgD,wBAAwB,MAAMhE,GACjCgD,MAAM,CAAC;QACNC,gBAAgBxD,aAAawD,cAAc;QAC3CgB,UAAU7E,GAAW,CAAC,aAAa,EAAEK,aAAa0C,KAAK,CAAC,UAAU,CAAC;IACrE,GACCkB,IAAI,CAAC5D,cACL6D,KAAK,CACJ7E,IAAIG,GAAGa,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAAGrB,GAAGW,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,IAE5FmE,OAAO,CAAC9D,aAAawD,cAAc;IAEtC,MAAMkB,mBAAmB,IAAIC,IAC3BJ,sBAAsBlD,GAAG,CAAC,CAAC0C,MAAQ;YAACa,SAASb,IAAIP,cAAc;YAAGO,IAAIS,QAAQ;SAAC;IAEjF,MAAMK,mCAAmC,MAAMtE,GAC5CgD,MAAM,CAAC;QACNC,gBAAgBxD,aAAawD,cAAc;QAC3CgB,UAAU7E,GAAW,CAAC,aAAa,EAAEK,aAAa0C,KAAK,CAAC,UAAU,CAAC;IACrE,GACCkB,IAAI,CAAC5D,cACL6D,KAAK,CACJ7E,IACEG,GAAGa,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAC/CvB,GAAGa,aAAaiC,aAAa,EAAE5B,OAAO4B,aAAa,GACnD5C,GAAGW,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,IAGxCmE,OAAO,CAAC9D,aAAawD,cAAc;IAEtC,MAAMpC,YAAYf,OAAOe,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YACpD6B,aAAa7B,SAAS6B,WAAW;YACjC5B,QAAQ;mBAAIxC,mBAAmBuC,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACAqD,8BAA8B1D,WAAWyD;IACzC,MAAM7C,QAAsB,EAAE;IAC9B,MAAM+C,SAA6B,EAAE;IACrC,IAAI9D,2BAA2BZ,OAAOa,eAAe;IACrD,MAAM8D,mBACJ3E,OAAO8B,uBAAuB,KAAK8B,YAAY,CAAC,IAAI;QAACvD,aAAaL,OAAOK,WAAW;IAAA;IAEtF,KAAK,MAAM2D,UAAUY,eAAe3B,YAAa;QAC/C,MAAMV,sBAAsBxB,UACzByB,MAAM,CAAC,CAACvB,WAAawB,SAASuB,OAAOb,cAAc,EAAElC,SAASC,MAAM,GACpEwB,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAEzB,MAAM,CAAC2B,MAAM,GAAGD,EAAE1B,MAAM,CAAC2B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAG5F,IAAIP,oBAAoBM,MAAM,KAAK,GAAG;QAEtC,MAAMsB,WAAWE,iBAAiBQ,GAAG,CAACN,SAASP,OAAOb,cAAc,MAAM;QAC1E,MAAM2B,aAAaC,KAAKC,GAAG,CAAC,GAAGhB,OAAOZ,MAAM,GAAGe;QAC/C,MAAMc,WAAW1C,oBAAoBL,MAAM,CACzC,CAACC,OAAOlB,WAAakB,QAAQlB,SAASE,cAAc,EACpD;QAEF,MAAM+D,QAAQH,KAAKI,GAAG,CAACL,YAAYG,UAAUrE;QAC7C,IAAIwE,qBAAqBjB;QAEzB,IAAIe,QAAQ,KAAKlF,OAAOa,eAAe,GAAG,GAAG;YAC3C,MAAM,CAACwE,SAAS,GAAG,MAAMnF,GACtBoF,MAAM,CAAC3F,cACP4F,MAAM,CAAC;gBACNlF,aAAaL,OAAOK,WAAW;gBAC/BuB,eAAe5B,OAAO4B,aAAa;gBACnCuB,gBAAgBa,OAAOb,cAAc;gBACrCd,OAAO6C;gBACPd,WAAW9E,GAAG,CAAC,SAAS,EAAEU,OAAO6B,UAAU,CAAC,yBAAyB,CAAC;YACxE,GACC2D,SAAS,CAAC;gBAACC,IAAI9F,aAAa8F,EAAE;gBAAErB,WAAWzE,aAAayE,SAAS;YAAA;YAEpE,IAAI,CAACiB,UAAU,MAAM,IAAIK,MAAM;YAE/B9E,4BAA4BsE;YAC5BlC,UAAUT,qBAAqB2C;YAC/B,MAAMS,0BAA0BzF,IAAI;gBAClC0B,eAAe5B,OAAO4B,aAAa;gBACnCgE,eAAeP,SAASI,EAAE;gBAC1BtC,gBAAgBa,OAAOb,cAAc;gBACrCd,OAAO6C;gBACP7E,aAAaL,OAAOK,WAAW;YACjC;YACA+E,sBAAsBF;YACtBR,OAAO3C,IAAI,CAAC;gBACV6D,eAAeP,SAASI,EAAE;gBAC1B,GAAGd,gBAAgB;gBACnBzD,QAAQ8C,OAAOb,cAAc;gBAC7Bd,OAAO6C;gBACPd,WAAWiB,SAASjB,SAAS;YAC/B;QACF;QAEAzC,MAAMI,IAAI,CAAC;YACT,GAAG4C,gBAAgB;YACnBzD,QAAQ8C,OAAOb,cAAc;YAC7BC,QAAQY,OAAOZ,MAAM;YACrBe,UAAUiB;YACV/B,gBAAgBW,OAAOX,cAAc;QACvC;IACF;IAEA,OAAO;QAAC1B;QAAOhC,cAAc+E;IAAM;AACrC;AAEA,eAAeiB,0BACbzF,EAAM,EACNF,MAMC;IAED,MAAM6F,cAAc,MAAM3F,GACvBgD,MAAM,CAAC;QAACuC,IAAI3F,gBAAgB2F,EAAE;IAAA,GAC9BlC,IAAI,CAACzD,iBACL0D,KAAK,CACJ7E,IACEG,GAAGgB,gBAAgB8B,aAAa,EAAE5B,OAAO4B,aAAa,GACtDzC,OAAOW,gBAAgBO,WAAW,GAClClB,OAAOW,gBAAgB8F,aAAa,GACpCzG,OAAOW,gBAAgBgG,qBAAqB,GAC5C3G,OAAOW,gBAAgBiG,eAAe,GACtC7G,UAAUY,gBAAgBkG,gBAAgB,GAC1ClH,GAAGgB,gBAAgBmG,KAAK,EAAE,YAC1BrH,cAAckB,gBAAgBoB,MAAM,EAAElB,OAAOmD,cAAc,GAC3DpE,OACEmB,GACGgD,MAAM,CAAC;QAACuC,IAAI5F,sBAAsB4F,EAAE;IAAA,GACpClC,IAAI,CAAC1D,uBACL2D,KAAK,CACJ7E,IACEG,GAAGe,sBAAsBqG,gBAAgB,EAAEpG,gBAAgB2F,EAAE,GAC7D3G,GAAGe,sBAAsB+B,aAAa,EAAE5B,OAAO4B,aAAa,GAC5DzC,OAAOU,sBAAsBsG,QAAQ,GACrCnH,GAAGa,sBAAsBuE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,OAMzD8G,OAAO,CAACvH,IAAIiB,gBAAgBwD,SAAS,GAAGzE,IAAIiB,gBAAgB2F,EAAE,GAC9DY,KAAK,CAACrG,OAAOqC,KAAK,EAClBiE,GAAG,CAAC;IAEP,IAAIT,YAAYhD,MAAM,KAAK,GAAG;IAE9B,MAAM3C,GACHqG,MAAM,CAACzG,iBACP0G,GAAG,CAAC;QACHnG,aAAaL,OAAOK,WAAW;QAC/BuF,eAAe5F,OAAO4F,aAAa;QACnCa,YAAYnH,GAAG,CAAC,KAAK,CAAC;QACtBoH,WAAWpH,GAAG,CAAC,KAAK,CAAC;IACvB,GACCkE,KAAK,CACJ7E,IACEM,QACEa,gBAAgB2F,EAAE,EAClBI,YAAY7E,GAAG,CAAC,CAAC2F,SAAWA,OAAOlB,EAAE,IAEvCtG,OAAOW,gBAAgBO,WAAW,GAClClB,OAAOW,gBAAgB8F,aAAa,GACpCzG,OAAOW,gBAAgBiG,eAAe;AAG9C;AAEA,eAAetF,mCAAmCC,oBAAyC;IACzF,IAAIA,qBAAqBkG,IAAI,KAAK,GAAG,OAAO,EAAE;IAC9C,MAAMlF,OAAO,MAAMnC,KAChB2D,MAAM,CAAC;QACN7C,aAAab,qBAAqBa,WAAW;QAC7CgD,gBAAgB/D,GAAS,CAAC,IAAI,EAAEE,qBAAqB8D,SAAS,CAAC,CAAC,CAAC;IACnE,GACCC,IAAI,CAAC/D,sBACLgE,KAAK,CAACvE,QAAQO,qBAAqBa,WAAW,EAAE;WAAIK;KAAqB,GACzE+C,OAAO,CAACjE,qBAAqBa,WAAW,EACxC+F,OAAO,CACNvH,IAAIS,GAAG,CAAC,IAAI,EAAEE,qBAAqB8D,SAAS,CAAC,CAAC,CAAC,GAC/CzE,IAAIW,qBAAqBa,WAAW;IAExC,OAAOqB,KAAKV,GAAG,CAAC,CAAC0C,MAAQA,IAAIrD,WAAW;AAC1C;AAEA,OAAO,eAAewG;IACpB,MAAMnF,OAAO,MAAMnC,KAChB2D,MAAM,CAAC;QAAC7C,aAAab,qBAAqBa,WAAW;IAAA,GACrDkD,IAAI,CAAC/D,sBACLiE,OAAO,CAACjE,qBAAqBa,WAAW;IAC3C,OAAOqB,KAAKV,GAAG,CAAC,CAAC0C,MAAQA,IAAIrD,WAAW;AAC1C;AAEA,eAAeyD,sCACb5D,EAAM,EACNF,MAAoD;IAEpD,MAAM0B,OAAO,MAAMxB,GAChBgD,MAAM,CAAC;QAAChC,QAAQzB,+BAA+ByB,MAAM;IAAA,GACrDqC,IAAI,CAAC9D,gCACLqH,SAAS,CACRpH,mBACAZ,GAAGY,kBAAkB+F,EAAE,EAAEhG,+BAA+BmC,aAAa,GAEtE4B,KAAK,CACJ7E,IACEG,GAAGW,+BAA+BY,WAAW,EAAEL,OAAOK,WAAW,GACjEvB,GAAGY,kBAAkBW,WAAW,EAAEL,OAAOK,WAAW,GACpDvB,GAAGY,kBAAkBqH,KAAK,EAAE,cAC5BzH,GAAG,CAAC,EAAEG,+BAA+BuH,YAAY,CAAC,YAAY,EAAEhH,OAAO+D,aAAa,CAAC,yBAAyB,CAAC,EAC/GzE,GAAG,CAAC,EAAEI,kBAAkBuH,SAAS,CAAC,QAAQ,CAAC,EAC3C3H,GAAG,CAAC,CAAC,EAAEI,kBAAkB0E,SAAS,CAAC,YAAY,EAAE1E,kBAAkB0E,SAAS,CAAC,SAAS,CAAC;IAG7F,OAAO1C,KAAKV,GAAG,CAAC,CAAC0C,MAAQA,IAAIxC,MAAM;AACrC;AAEA,OAAO,eAAegG,0BAA0BlH,MAAyB;IACvE,OAAO,MAAMT,KAAKU,WAAW,CAAC,OAAOC;QACnC,MAAMiH,cAAc,MAAMjH,GACvBgD,MAAM,CAAC;YAACuC,IAAI9F,aAAa8F,EAAE;QAAA,GAC3BlC,IAAI,CAAC5D,cACL6D,KAAK,CAACpE,GAAGO,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,GAC3C8G,OAAO,CAACvH,IAAIc,aAAayE,SAAS,GAClCiC,KAAK,CAACrG,QAAQqG,SAAS;QAC1B,OAAO,MAAMe,gCACXlH,IACAiH,YAAYnG,GAAG,CAAC,CAACoB,cAAgBA,YAAYqD,EAAE,GAC/C;YAAC4B,aAAa;QAAI;IAEtB;AACF;AAEA,OAAO,eAAeC,wBAAwBC,GAAa;IACzD,IAAIA,IAAI1E,MAAM,KAAK,GAAG,OAAO;IAE7B,OAAO,MAAMtD,KAAKU,WAAW,CAAC,OAAOC;QACnC,OAAO,MAAMkH,gCAAgClH,IAAIqH;IACnD;AACF;AAEA,eAAeH,gCACblH,EAAM,EACNqH,GAAa,EACbvH,SAAkC,CAAC,CAAC;IAEpC,IAAIuH,IAAI1E,MAAM,KAAK,GAAG,OAAO;IAE7B,MAAM2E,kBAAkB,MAAMtH,GAC3BgD,MAAM,CAAC;QACNuC,IAAI3F,gBAAgB2F,EAAE;QACtBG,eAAe9F,gBAAgB8F,aAAa;QAC5CE,uBAAuBhG,gBAAgBgG,qBAAqB;QAC5DC,iBAAiBjG,gBAAgBiG,eAAe;QAChD0B,uBAAuB3H,gBAAgB2H,qBAAqB;IAC9D,GACClE,IAAI,CAACzD,iBACL0D,KAAK,CACJnE,GACEV,IACEM,QAAQa,gBAAgB8F,aAAa,EAAE2B,MACvCpI,OAAOW,gBAAgBiG,eAAe,GACtC5G,OAAOW,gBAAgB2H,qBAAqB,IAE9CxI,QAAQa,gBAAgBgG,qBAAqB,EAAEyB,OAGlDnB,OAAO,CAACvH,IAAIiB,gBAAgB2F,EAAE,GAC9Ba,GAAG,CAAC;IACP,MAAMoB,kBAAkB,MAAMxH,GAC3BgD,MAAM,CAAC;QAACuC,IAAI9F,aAAa8F,EAAE;IAAA,GAC3BlC,IAAI,CAAC5D,cACL6D,KAAK,CACJ7E,IACEM,QAAQU,aAAa8F,EAAE,EAAE8B,MACzBvH,OAAOqH,WAAW,GAAGjI,GAAGO,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,IAAIsE,YAGjE0C,GAAG,CAAC;IACP,MAAMqB,iBAAiBD,gBAAgB1G,GAAG,CAAC,CAACoB,cAAgBA,YAAYqD,EAAE;IAE1E,IAAIkC,eAAe9E,MAAM,KAAK,GAAG,OAAO;IAExC,MAAM+E,oBAAoBJ,gBACvBhF,MAAM,CACL,CAACmE,SACCA,OAAOf,aAAa,IACpB+B,eAAeE,QAAQ,CAAClB,OAAOf,aAAa,KAC5C,CAACe,OAAOZ,eAAe,IACvB,CAACY,OAAOc,qBAAqB,EAEhCzG,GAAG,CAAC,CAAC2F,SAAWA,OAAOlB,EAAE;IAC5B,MAAMqC,oBAAoBN,gBACvBhF,MAAM,CACL,CAACmE,SACCA,OAAOb,qBAAqB,IAAI6B,eAAeE,QAAQ,CAAClB,OAAOb,qBAAqB,GAEvF9E,GAAG,CAAC,CAAC2F,SAAWA,OAAOlB,EAAE;IAC5B,IAAImC,kBAAkB/E,MAAM,GAAG,GAAG;QAChC,MAAM3C,GACHqG,MAAM,CAAC3G,wBACP4G,GAAG,CAAC;YAACS,WAAW3H,GAAG,CAAC,KAAK,CAAC;QAAA,GAC1BkE,KAAK,CACJ7E,IACEM,QAAQW,uBAAuBsG,gBAAgB,EAAE0B,oBACjDzI,OAAOS,uBAAuBmI,UAAU,GACxC5I,OAAOS,uBAAuBqH,SAAS;QAG7C,MAAM/G,GACHqG,MAAM,CAACzG,iBACP0G,GAAG,CAAC;YACHnG,aAAa;YACbuF,eAAe;YACfa,YAAY;YACZC,WAAWpH,GAAG,CAAC,KAAK,CAAC;QACvB,GACCkE,KAAK,CACJ7E,IACEM,QAAQa,gBAAgB2F,EAAE,EAAEmC,oBAC5BzI,OAAOW,gBAAgBiG,eAAe,GACtC5G,OAAOW,gBAAgB2H,qBAAqB;IAGpD;IACA,IAAIK,kBAAkBjF,MAAM,GAAG,GAAG;QAChC,MAAM3C,GACHqG,MAAM,CAACzG,iBACP0G,GAAG,CAAC;YAACV,uBAAuB;YAAMY,WAAWpH,GAAG,CAAC,KAAK,CAAC;QAAA,GACvDkE,KAAK,CACJ7E,IACEM,QAAQa,gBAAgB2F,EAAE,EAAEqC,oBAC5B7I,QAAQa,gBAAgBgG,qBAAqB,EAAE6B;IAGvD;IAEA,MAAMK,UAAU,MAAM9H,GACnB+H,MAAM,CAACtI,cACP6D,KAAK,CAACvE,QAAQU,aAAa8F,EAAE,EAAEkC,iBAC/BnC,SAAS,CAAC;QAACC,IAAI9F,aAAa8F,EAAE;IAAA;IAEjC,OAAOuC,QAAQnF,MAAM;AACvB;AAEA,OAAO,eAAeqF,wBACpBhI,EAAM,EACNF,MAIC;IAED,IAAImI,WAAW;IAEf,KAAK,MAAMC,WAAWpI,OAAOqI,QAAQ,CAAE;QACrC,IAAID,QAAQ/F,KAAK,IAAI,GAAG;QAExB,MAAMiG,cAAc,MAAMpI,GACvBqG,MAAM,CAAC5G,cACP6G,GAAG,CAAC;YAACnE,OAAO/C,GAAG,CAAC,EAAEK,aAAa0C,KAAK,CAAC,GAAG,EAAE+F,QAAQ/F,KAAK,CAAC,CAAC;QAAA,GACzDmB,KAAK,CACJ7E,IACEG,GAAGa,aAAa8F,EAAE,EAAE2C,QAAQxC,aAAa,GACzC9G,GAAGa,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAC/CvB,GAAGa,aAAaiC,aAAa,EAAE5B,OAAO4B,aAAa,GACnD5C,GAAGW,aAAa0C,KAAK,EAAE+F,QAAQ/F,KAAK,GACpCrD,GAAGW,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,IAGxCkG,SAAS,CAAC;YAACC,IAAI9F,aAAa8F,EAAE;QAAA;QAEjC,IAAI6C,YAAYzF,MAAM,GAAG,GAAG;YAC1BsF,YAAYC,QAAQ/F,KAAK;YACzB;QACF;QAEA,MAAM2F,UAAU,MAAM9H,GACnB+H,MAAM,CAACtI,cACP6D,KAAK,CACJ7E,IACEG,GAAGa,aAAa8F,EAAE,EAAE2C,QAAQxC,aAAa,GACzC9G,GAAGa,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAC/CvB,GAAGa,aAAaiC,aAAa,EAAE5B,OAAO4B,aAAa,GACnDtC,GAAG,CAAC,EAAEK,aAAa0C,KAAK,CAAC,IAAI,EAAE+F,QAAQ/F,KAAK,CAAC,CAAC,EAC9CrD,GAAGW,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,IAGxCkG,SAAS,CAAC;YAACnD,OAAO1C,aAAa0C,KAAK;QAAA;QAEvC8F,YAAYH,QAAQ9F,MAAM,CAAC,CAACC,OAAOuB,MAAQvB,QAAQuB,IAAIrB,KAAK,EAAE;IAChE;IAEA,OAAO8F;AACT;AAEA,SAAS1D,8BACP1D,SAA+B,EAC/BwH,kBAAkE;IAElE,KAAK,MAAMnG,eAAeoG,oBAAoBD,oBAAqB;QACjE,MAAMhG,sBAAsBxB,UACzByB,MAAM,CAAC,CAACvB,WAAawB,SAASL,YAAYe,cAAc,EAAElC,SAASC,MAAM,GACzEwB,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAEzB,MAAM,CAAC2B,MAAM,GAAGD,EAAE1B,MAAM,CAAC2B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBH,YAAY+B,QAAQ;IACrD;AACF;AAEA,SAASS,eAAelD,IAAiB;IACvC,OAAO;WAAIA;KAAK,CAACgB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAM6F,cAAc7F,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAI4F,gBAAgB,GAAG,OAAOA;QAC9B,OAAO9F,EAAEU,cAAc,CAACqF,OAAO,KAAK9F,EAAES,cAAc,CAACqF,OAAO;IAC9D;AACF;AAEA,SAASF,oBAA0D9G,IAAS;IAC1E,OAAO;WAAIA;KAAK,CAACgB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAM6F,cAAc7F,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAI4F,gBAAgB,GAAG,OAAOA;QAC9B,OAAOlE,SAAS5B,EAAEQ,cAAc,EAAEJ,aAAa,CAACwB,SAAS3B,EAAEO,cAAc;IAC3E;AACF;AAEA,SAASV,SAASU,cAAwB,EAAEwF,eAAyB;IACnE,OAAOxF,eAAeyF,KAAK,CAAC,CAACC,QAAUF,gBAAgBd,QAAQ,CAACgB;AAClE;AAEA,SAAS7F,UAAUjC,SAA+B,EAAEsB,KAAa;IAC/D,IAAIyG,YAAYzG;IAChB,KAAK,MAAMpB,YAAYF,UAAW;QAChC,IAAI+H,cAAc,GAAG;QACrB,MAAMC,OAAOhE,KAAKI,GAAG,CAAClE,SAASE,cAAc,EAAE2H;QAC/C7H,SAASE,cAAc,IAAI4H;QAC3BD,aAAaC;IACf;AACF;AAEA,SAASxE,SAASrD,MAAgB;IAChC,OAAO8H,KAAKC,SAAS,CAAC/H;AACxB"}
|
|
1
|
+
{"version":3,"sources":["../../src/db/reservations.ts"],"sourcesContent":["import {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n and,\n arrayContains,\n asc,\n eq,\n exists,\n gt,\n inArray,\n isNotNull,\n isNull,\n lt,\n notExists,\n notInArray,\n or,\n sql,\n} from 'drizzle-orm';\nimport {recordProviderRunnerActivationOutcome} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';\nimport {terminalStates} from './runner-states.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {provisionerCapabilitySnapshots} from './schema/provisioner-capability-snapshots.js';\nimport {provisionerTokens} from './schema/provisioner-tokens.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerActivationTokens} from './schema/runner-activation-tokens.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {type providerRunnerLaunchKindEnum, providerRunners} from './schema/runner-instances.js';\n\nexport interface ReservationTemplate {\n templateKey: string;\n labels: string[];\n availableSlots: number;\n starting: number;\n running: number;\n}\n\nexport interface DemandStat {\n workspaceId?: string;\n labels: string[];\n queued: number;\n reserved: number;\n oldestQueuedAt: Date;\n}\n\nexport interface ReservationGrant {\n reservationId: string;\n workspaceId?: string;\n labels: string[];\n count: number;\n expiresAt: Date;\n}\n\nexport interface PollDemandAndReserveParams {\n workspaceId: string;\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n /** Lifetime of the short reservation that gives rebound runners time to activate. */\n activationGraceSeconds?: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds?: number;\n}\n\nexport interface InstallationPollDemandAndReserveParams {\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n /** Lifetime of the short reservation that gives rebound runners time to activate. */\n activationGraceSeconds?: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds: number;\n eligibleWorkspaceIds: ReadonlySet<string>;\n signal?: AbortSignal;\n onReservations?: (reservations: ReservationGrant[]) => void;\n}\n\ntype DemandScope = 'installation' | 'workspace';\n\ninterface IdleRunnerCandidate {\n id: string;\n launchKind: (typeof providerRunnerLaunchKindEnum.enumValues)[number];\n}\n\ninterface BindableRunnerParams {\n provisionerId: string;\n workspaceId: string;\n requiredLabels: string[];\n scope: DemandScope;\n}\n\ninterface IdleRunnerSelectionParams extends BindableRunnerParams {\n count: number;\n}\n\ntype PollDemandAndReserveLockedParams = PollDemandAndReserveParams & {\n scope: DemandScope;\n};\n\ninterface NormalizedTemplate {\n templateKey: string;\n labels: string[];\n remainingSlots: number;\n}\n\ninterface DemandRow {\n requiredLabels: string[];\n queued: number;\n oldestQueuedAt: Date;\n}\n\ninterface NewReservationUnits {\n labels: string[];\n count: number;\n}\n\ninterface PollDemandAndReserveResult {\n stats: DemandStat[];\n /** Only launch reservations are exposed to the provisioner. */\n reservations: ReservationGrant[];\n /** Internal allocation accounting for bound and launch rows together. */\n newlyReservedUnits: NewReservationUnits[];\n}\n\nexport async function pollDemandAndReserve(\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n const result = await db().transaction(async (tx) => {\n return await pollDemandAndReserveTx(tx, params);\n });\n return {stats: result.stats, reservations: result.reservations};\n}\n\nexport async function pollDemandAndReserveTx(\n tx: Tx,\n params: PollDemandAndReserveParams,\n): Promise<PollDemandAndReserveResult> {\n await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId}))`);\n return await pollDemandAndReserveLockedTx(tx, {...params, scope: 'workspace'});\n}\n\nexport async function pollInstallationDemandAndReserve(\n params: InstallationPollDemandAndReserveParams,\n): Promise<{\n stats: DemandStat[];\n reservations: ReservationGrant[];\n newlyReservedCount?: number;\n}> {\n const candidateWorkspaceIds = await listInstallationDemandWorkspaceIds(\n params.eligibleWorkspaceIds,\n );\n const results: PollDemandAndReserveResult[] = [];\n let remainingMaxReservations = params.maxReservations;\n const remainingTemplates = params.templates.map((template) => ({\n ...template,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n for (const workspaceId of candidateWorkspaceIds) {\n if (params.signal?.aborted || remainingMaxReservations === 0) break;\n const result = await db().transaction(async (tx) => {\n const lockResult = await tx.execute<{locked: boolean}>(\n sql`select pg_try_advisory_xact_lock(hashtext(${workspaceId})) as locked`,\n );\n const locked = lockResult.rows[0];\n if (!locked?.locked) return {stats: [], reservations: [], newlyReservedUnits: []};\n return await pollDemandAndReserveLockedTx(tx, {\n workspaceId,\n provisionerId: params.provisionerId,\n maxReservations: remainingMaxReservations,\n ttlSeconds: params.ttlSeconds,\n ...(params.activationGraceSeconds !== undefined\n ? {activationGraceSeconds: params.activationGraceSeconds}\n : {}),\n templates: remainingTemplates.map((template) => ({\n ...template,\n availableSlots: template.remainingSlots,\n })),\n capabilityWindowSeconds: params.capabilityWindowSeconds,\n scope: 'installation',\n });\n });\n results.push(result);\n consumeInstallationTemplateSlots(remainingTemplates, result.reservations);\n params.onReservations?.(result.reservations);\n remainingMaxReservations -= result.newlyReservedUnits.reduce(\n (total, reservation) => total + reservation.count,\n 0,\n );\n }\n const newlyReservedCount = results.reduce(\n (total, result) =>\n total +\n result.newlyReservedUnits.reduce((subtotal, reservation) => subtotal + reservation.count, 0),\n 0,\n );\n return {\n stats: results.flatMap((result) => result.stats),\n reservations: results.flatMap((result) => result.reservations),\n ...(newlyReservedCount > 0 ? {newlyReservedCount} : {}),\n };\n}\n\nfunction consumeInstallationTemplateSlots(\n templates: NormalizedTemplate[],\n launchGrants: ReservationGrant[],\n): void {\n // Adopted runners are already included in the running count behind availableSlots.\n // Only units that still need a launch consume advertised template capacity.\n for (const reservation of launchGrants) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.labels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.count);\n }\n}\n\nasync function pollDemandAndReserveLockedTx(\n tx: Tx,\n params: PollDemandAndReserveLockedParams,\n): Promise<PollDemandAndReserveResult> {\n let demandRows = (\n await tx\n .select({\n requiredLabels: pendingJobExecutions.requiredLabels,\n queued: sql<number>`count(*)::int`,\n oldestQueuedAt: sql<Date | string>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(eq(pendingJobExecutions.workspaceId, params.workspaceId))\n .groupBy(pendingJobExecutions.requiredLabels)\n ).map((row) => ({\n ...row,\n oldestQueuedAt: new Date(row.oldestQueuedAt),\n }));\n if (params.capabilityWindowSeconds !== undefined) {\n const capabilityLabels = await listActiveWorkspaceCapabilityLabelsTx(tx, {\n workspaceId: params.workspaceId,\n windowSeconds: params.capabilityWindowSeconds,\n });\n demandRows = demandRows.filter(\n (demand) => !capabilityLabels.some((labels) => isSubset(demand.requiredLabels, labels)),\n );\n }\n\n const activeReservationRows = await tx\n .select({\n requiredLabels: reservations.requiredLabels,\n reserved: sql<number>`coalesce(sum(${reservations.count}), 0)::int`,\n })\n .from(reservations)\n .where(\n and(eq(reservations.workspaceId, params.workspaceId), gt(reservations.expiresAt, sql`now()`)),\n )\n .groupBy(reservations.requiredLabels);\n\n const reservedByLabels = new Map(\n activeReservationRows.map((row) => [labelKey(row.requiredLabels), row.reserved]),\n );\n const activeProvisionerReservationRows = await listActiveProvisionerReservationRowsTx(tx, {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n });\n\n const templates = params.templates.map((template) => ({\n templateKey: template.templateKey,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n deductProvisionerReservations(templates, activeProvisionerReservationRows);\n const stats: DemandStat[] = [];\n const grants: ReservationGrant[] = [];\n const newlyReservedUnits: NewReservationUnits[] = [];\n let remainingMaxReservations = params.maxReservations;\n const workspaceIdField =\n params.capabilityWindowSeconds === undefined ? {} : {workspaceId: params.workspaceId};\n\n for (const demand of sortDemandRows(demandRows)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(demand.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n\n if (satisfyingTemplates.length === 0) continue;\n\n const reserved = reservedByLabels.get(labelKey(demand.requiredLabels)) ?? 0;\n const unreserved = Math.max(0, demand.queued - reserved);\n const capacity = satisfyingTemplates.reduce(\n (total, template) => total + template.remainingSlots,\n 0,\n );\n const grant = Math.min(unreserved, capacity, remainingMaxReservations);\n let reservedAfterGrant = reserved;\n\n if (grant > 0 && params.maxReservations > 0) {\n const idleRunners = await listIdleRunnerInstancesTx(tx, {\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: grant,\n workspaceId: params.workspaceId,\n scope: params.scope,\n });\n\n remainingMaxReservations -= grant;\n\n const boundCount = idleRunners.length;\n if (boundCount > 0) {\n const [boundReservation] = await tx\n .insert(reservations)\n .values({\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: boundCount,\n kind: 'bound',\n expiresAt: sql`now() + (${params.activationGraceSeconds ?? params.ttlSeconds} || ' seconds')::interval`,\n })\n .returning({id: reservations.id});\n\n if (!boundReservation) throw new Error('Insert returned no rows');\n\n await bindIdleRunnerInstancesTx(tx, {\n provisionerId: params.provisionerId,\n reservationId: boundReservation.id,\n workspaceId: params.workspaceId,\n requiredLabels: demand.requiredLabels,\n scope: params.scope,\n idleRunners,\n });\n }\n\n const launchCount = grant - boundCount;\n drawSlots(satisfyingTemplates, launchCount);\n let launchReservation: {id: string; expiresAt: Date} | undefined;\n if (launchCount > 0) {\n const [inserted] = await tx\n .insert(reservations)\n .values({\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: launchCount,\n kind: 'launch',\n expiresAt: sql`now() + (${params.ttlSeconds} || ' seconds')::interval`,\n })\n .returning({id: reservations.id, expiresAt: reservations.expiresAt});\n\n if (!inserted) throw new Error('Insert returned no rows');\n launchReservation = inserted;\n }\n\n newlyReservedUnits.push({labels: demand.requiredLabels, count: grant});\n reservedAfterGrant += grant;\n if (launchReservation) {\n grants.push({\n reservationId: launchReservation.id,\n ...workspaceIdField,\n labels: demand.requiredLabels,\n count: launchCount,\n expiresAt: launchReservation.expiresAt,\n });\n }\n }\n\n stats.push({\n ...workspaceIdField,\n labels: demand.requiredLabels,\n queued: demand.queued,\n reserved: reservedAfterGrant,\n oldestQueuedAt: demand.oldestQueuedAt,\n });\n }\n\n return {stats, reservations: grants, newlyReservedUnits};\n}\n\nasync function listActiveProvisionerReservationRowsTx(\n tx: Tx,\n params: {workspaceId: string; provisionerId: string},\n): Promise<Array<{requiredLabels: string[]; reserved: number}>> {\n const candidateRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n );\n const candidateIds = candidateRows.map((row) => row.id);\n if (candidateIds.length === 0) return [];\n\n // Assignment and provider-report transactions use this key before changing runner\n // links. Lock it before counting so an in-flight assignment cannot expose its unit twice.\n await lockRunnerReservationAdvisoryKeysTx(tx, {\n provisionerId: params.provisionerId,\n reservationIds: candidateIds,\n });\n\n const activeRows = await tx\n .select({\n id: reservations.id,\n requiredLabels: reservations.requiredLabels,\n count: reservations.count,\n })\n .from(reservations)\n .where(\n and(\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .orderBy(asc(reservations.id))\n .for('update');\n if (activeRows.length === 0) return [];\n\n const activeIds = activeRows.map((row) => row.id);\n const activeIdSet = new Set(activeIds);\n const usedRunnerRows = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n state: providerRunners.state,\n })\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n isNull(providerRunners.reservationReleasedAt),\n or(\n inArray(providerRunners.reservationId, activeIds),\n and(\n inArray(providerRunners.intendedReservationId, activeIds),\n notInArray(providerRunners.state, [...terminalStates]),\n ),\n ),\n ),\n );\n const usedByReservation = new Map<string, Set<string>>();\n for (const runner of usedRunnerRows) {\n if (runner.reservationId && activeIdSet.has(runner.reservationId)) {\n addUsedRunner(usedByReservation, runner.reservationId, runner.id);\n }\n if (\n runner.intendedReservationId &&\n activeIdSet.has(runner.intendedReservationId) &&\n !terminalStates.includes(runner.state as (typeof terminalStates)[number])\n ) {\n addUsedRunner(usedByReservation, runner.intendedReservationId, runner.id);\n }\n }\n\n return activeRows.flatMap((reservation) => {\n const used = usedByReservation.get(reservation.id)?.size ?? 0;\n const pending = Math.max(0, reservation.count - used);\n return pending > 0 ? [{requiredLabels: reservation.requiredLabels, reserved: pending}] : [];\n });\n}\n\nfunction addUsedRunner(\n usedByReservation: Map<string, Set<string>>,\n reservationId: string,\n runnerId: string,\n): void {\n const runnerIds = usedByReservation.get(reservationId);\n if (runnerIds) runnerIds.add(runnerId);\n else usedByReservation.set(reservationId, new Set([runnerId]));\n}\n\n// An expired or missing reservation is stale. A live reservation protects a runner that is\n// still booting or waiting for its activation grace period.\nfunction canBindRunner(tx: Tx, params: BindableRunnerParams) {\n return and(\n or(\n and(isNull(providerRunners.workspaceId), isNull(providerRunners.reservationId)),\n and(\n isNotNull(providerRunners.reservationId),\n params.scope === 'installation'\n ? undefined\n : or(\n isNull(providerRunners.workspaceId),\n eq(providerRunners.workspaceId, params.workspaceId),\n ),\n notExists(\n tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.id, providerRunners.reservationId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n ),\n or(\n isNull(providerRunners.intendedReservationId),\n notExists(\n tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.id, providerRunners.intendedReservationId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n isNull(providerRunners.reservationReleasedAt),\n isNull(providerRunners.runnerSessionId),\n );\n}\n\nfunction isBindableRunner(tx: Tx, params: BindableRunnerParams) {\n return and(\n eq(providerRunners.provisionerId, params.provisionerId),\n canBindRunner(tx, params),\n isNotNull(providerRunners.providerRunnerId),\n eq(providerRunners.state, 'running'),\n arrayContains(providerRunners.labels, params.requiredLabels),\n exists(\n tx\n .select({id: runnerControlSessions.id})\n .from(runnerControlSessions)\n .where(\n and(\n eq(runnerControlSessions.runnerInstanceId, providerRunners.id),\n eq(runnerControlSessions.provisionerId, params.provisionerId),\n isNull(runnerControlSessions.closedAt),\n gt(runnerControlSessions.expiresAt, sql`now()`),\n ),\n ),\n ),\n );\n}\n\nasync function listIdleRunnerInstancesTx(\n tx: Tx,\n params: IdleRunnerSelectionParams & {count: number},\n): Promise<IdleRunnerCandidate[]> {\n const lockedRunners = await selectIdleRunnerInstancesTx(tx, params);\n if (lockedRunners.length === 0) return lockedRunners;\n\n // A SELECT ... FOR UPDATE can wake with a fresh target row but the original\n // statement snapshot for reservation subqueries. Re-check the full predicate\n // in a new statement while retaining the row locks from the first query.\n return await selectIdleRunnerInstancesTx(tx, {\n ...params,\n runnerIds: lockedRunners.map((runner) => runner.id),\n count: lockedRunners.length,\n });\n}\n\nasync function selectIdleRunnerInstancesTx(\n tx: Tx,\n params: IdleRunnerSelectionParams & {count: number; runnerIds?: string[]},\n): Promise<IdleRunnerCandidate[]> {\n if (params.runnerIds) {\n // The first pass retains the row locks. Recheck the full predicate in a fresh statement so\n // reservation subqueries see the state that was current when the lock was acquired.\n return await tx\n .select({id: providerRunners.id, launchKind: providerRunners.launchKind})\n .from(providerRunners)\n .where(and(inArray(providerRunners.id, params.runnerIds), isBindableRunner(tx, params)))\n .orderBy(asc(providerRunners.createdAt), asc(providerRunners.id))\n .limit(params.count);\n }\n\n // Select the oldest candidates first, but leave row locking to a second pass. The cleanup\n // path locks runner rows by id, so the locking pass must use that same order. A skipped row\n // rolls back only this nested savepoint and retries the bounded candidate scan, allowing the\n // next-oldest eligible runner to refill the grant without retaining a partial lock set.\n const retrySelection = Symbol('retry bindable runner selection');\n const idleRunners = await (async () => {\n while (true) {\n try {\n return await tx.transaction(async (lockTx) => {\n const candidateRunners = await lockTx\n .select({id: providerRunners.id, launchKind: providerRunners.launchKind})\n .from(providerRunners)\n .where(isBindableRunner(lockTx, params))\n .orderBy(asc(providerRunners.createdAt), asc(providerRunners.id))\n .limit(params.count);\n\n if (candidateRunners.length === 0) return candidateRunners;\n\n // PostgreSQL can acquire row locks before sorting a multi-row FOR UPDATE result. Lock\n // each selected candidate individually so both polling and cleanup use id order.\n const lockedRunners: typeof candidateRunners = [];\n for (const candidate of [...candidateRunners].sort(compareRunnerIds)) {\n const [runner] = await lockTx\n .select({id: providerRunners.id, launchKind: providerRunners.launchKind})\n .from(providerRunners)\n .where(and(eq(providerRunners.id, candidate.id), isBindableRunner(lockTx, params)))\n .for('update');\n if (runner) lockedRunners.push(runner);\n }\n\n if (lockedRunners.length !== candidateRunners.length) throw retrySelection;\n return lockedRunners;\n });\n } catch (error) {\n if (error !== retrySelection) throw error;\n }\n }\n })();\n\n return idleRunners;\n}\n\nasync function bindIdleRunnerInstancesTx(\n tx: Tx,\n params: {\n provisionerId: string;\n reservationId: string;\n workspaceId: string;\n requiredLabels: string[];\n scope: DemandScope;\n idleRunners: IdleRunnerCandidate[];\n },\n): Promise<void> {\n const idleRunnerIds = params.idleRunners.map((runner) => runner.id);\n\n await tx\n .update(runnerActivationTokens)\n .set({revokedAt: sql`now()`})\n .where(\n and(\n inArray(runnerActivationTokens.runnerInstanceId, idleRunnerIds),\n isNull(runnerActivationTokens.consumedAt),\n isNull(runnerActivationTokens.revokedAt),\n ),\n );\n\n // listIdleRunnerInstancesTx re-checks the full eligibility predicate in a fresh statement\n // after acquiring row locks. Those locks prevent the validated set from changing before this\n // update runs.\n const boundRunners = await tx\n .update(providerRunners)\n .set({\n workspaceId: params.workspaceId,\n reservationId: params.reservationId,\n intendedReservationId: null,\n assignedAt: sql`now()`,\n updatedAt: sql`now()`,\n })\n .where(and(inArray(providerRunners.id, idleRunnerIds), isBindableRunner(tx, params)))\n .returning({id: providerRunners.id, launchKind: providerRunners.launchKind});\n\n if (boundRunners.length !== params.idleRunners.length)\n throw new Error('Locked idle runner set changed before binding');\n\n const reboundCount = boundRunners.filter((runner) => runner.launchKind === 'demand').length;\n if (reboundCount > 0)\n recordProviderRunnerActivationOutcome({outcome: 'rebound', count: reboundCount});\n}\n\nasync function listInstallationDemandWorkspaceIds(eligibleWorkspaceIds: ReadonlySet<string>) {\n if (eligibleWorkspaceIds.size === 0) return [];\n const rows = await db()\n .select({\n workspaceId: pendingJobExecutions.workspaceId,\n oldestQueuedAt: sql<Date>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.workspaceId, [...eligibleWorkspaceIds]))\n .groupBy(pendingJobExecutions.workspaceId)\n .orderBy(\n asc(sql`min(${pendingJobExecutions.createdAt})`),\n asc(pendingJobExecutions.workspaceId),\n );\n return rows.map((row) => row.workspaceId);\n}\n\nexport async function listQueuedDemandWorkspaceIds(): Promise<string[]> {\n const rows = await db()\n .select({workspaceId: pendingJobExecutions.workspaceId})\n .from(pendingJobExecutions)\n .groupBy(pendingJobExecutions.workspaceId);\n return rows.map((row) => row.workspaceId);\n}\n\nasync function listActiveWorkspaceCapabilityLabelsTx(\n tx: Tx,\n params: {workspaceId: string; windowSeconds: number},\n): Promise<string[][]> {\n const rows = await tx\n .select({labels: provisionerCapabilitySnapshots.labels})\n .from(provisionerCapabilitySnapshots)\n .innerJoin(\n provisionerTokens,\n eq(provisionerTokens.id, provisionerCapabilitySnapshots.provisionerId),\n )\n .where(\n and(\n eq(provisionerCapabilitySnapshots.workspaceId, params.workspaceId),\n eq(provisionerTokens.workspaceId, params.workspaceId),\n eq(provisionerTokens.scope, 'workspace'),\n sql`${provisionerCapabilitySnapshots.advertisedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n sql`${provisionerTokens.revokedAt} is null`,\n sql`(${provisionerTokens.expiresAt} is null or ${provisionerTokens.expiresAt} > now())`,\n ),\n );\n return rows.map((row) => row.labels);\n}\n\nexport async function deleteExpiredReservations(params?: {limit?: number}): Promise<number> {\n return await db().transaction(async (tx) => {\n const expiredRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(lt(reservations.expiresAt, sql`now()`))\n .orderBy(asc(reservations.expiresAt))\n .limit(params?.limit ?? 1000);\n return await deleteReservationsWithCleanupTx(\n tx,\n expiredRows.map((reservation) => reservation.id),\n {expiredOnly: true},\n );\n });\n}\n\nexport async function deleteReservationsByIds(ids: string[]): Promise<number> {\n if (ids.length === 0) return 0;\n\n return await db().transaction(async (tx) => {\n return await deleteReservationsWithCleanupTx(tx, ids);\n });\n}\n\nasync function deleteReservationsWithCleanupTx(\n tx: Tx,\n ids: string[],\n params: {expiredOnly?: boolean} = {},\n): Promise<number> {\n if (ids.length === 0) return 0;\n\n const isAffectedRunner = () =>\n or(\n and(\n inArray(providerRunners.reservationId, ids),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n inArray(providerRunners.intendedReservationId, ids),\n );\n const candidateRunners = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n runnerSessionId: providerRunners.runnerSessionId,\n reservationReleasedAt: providerRunners.reservationReleasedAt,\n })\n .from(providerRunners)\n .where(isAffectedRunner())\n .orderBy(asc(providerRunners.id));\n const affectedRunners: typeof candidateRunners = [];\n for (const candidate of [...candidateRunners].sort(compareRunnerIds)) {\n const [runner] = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n runnerSessionId: providerRunners.runnerSessionId,\n reservationReleasedAt: providerRunners.reservationReleasedAt,\n })\n .from(providerRunners)\n .where(and(eq(providerRunners.id, candidate.id), isAffectedRunner()))\n .for('update');\n if (runner) affectedRunners.push(runner);\n }\n const reservationRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n inArray(reservations.id, ids),\n params.expiredOnly ? lt(reservations.expiresAt, sql`now()`) : undefined,\n ),\n )\n .for('update');\n const reservationIds = reservationRows.map((reservation) => reservation.id);\n\n if (reservationIds.length === 0) return 0;\n\n const assignedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.reservationId &&\n reservationIds.includes(runner.reservationId) &&\n !runner.runnerSessionId &&\n !runner.reservationReleasedAt,\n )\n .map((runner) => runner.id);\n const intendedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.intendedReservationId && reservationIds.includes(runner.intendedReservationId),\n )\n .map((runner) => runner.id);\n if (assignedRunnerIds.length > 0) {\n await tx\n .update(runnerActivationTokens)\n .set({revokedAt: sql`now()`})\n .where(\n and(\n inArray(runnerActivationTokens.runnerInstanceId, assignedRunnerIds),\n isNull(runnerActivationTokens.consumedAt),\n isNull(runnerActivationTokens.revokedAt),\n ),\n );\n await tx\n .update(providerRunners)\n .set({\n workspaceId: null,\n reservationId: null,\n assignedAt: null,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n inArray(providerRunners.id, assignedRunnerIds),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n );\n }\n if (intendedRunnerIds.length > 0) {\n await tx\n .update(providerRunners)\n .set({intendedReservationId: null, updatedAt: sql`now()`})\n .where(\n and(\n inArray(providerRunners.id, intendedRunnerIds),\n inArray(providerRunners.intendedReservationId, reservationIds),\n ),\n );\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(inArray(reservations.id, reservationIds))\n .returning({id: reservations.id});\n\n return deleted.length;\n}\n\nexport async function releaseReservationUnits(\n tx: Tx,\n params: {\n workspaceId: string;\n provisionerId: string;\n releases: Array<{reservationId: string; count: number}>;\n },\n): Promise<number> {\n let released = 0;\n\n for (const release of params.releases) {\n if (release.count <= 0) continue;\n\n const decremented = await tx\n .update(reservations)\n .set({count: sql`${reservations.count} - ${release.count}`})\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.count, release.count),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({id: reservations.id});\n\n if (decremented.length > 0) {\n released += release.count;\n continue;\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n sql`${reservations.count} <= ${release.count}`,\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({count: reservations.count});\n\n released += deleted.reduce((total, row) => total + row.count, 0);\n }\n\n return released;\n}\n\nfunction deductProvisionerReservations(\n templates: NormalizedTemplate[],\n activeReservations: {requiredLabels: string[]; reserved: number}[],\n): void {\n for (const reservation of sortReservationRows(activeReservations)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.reserved);\n }\n}\n\nfunction compareRunnerIds(left: {id: string}, right: {id: string}): number {\n return left.id < right.id ? -1 : left.id > right.id ? 1 : 0;\n}\n\nfunction sortDemandRows(rows: DemandRow[]): DemandRow[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return a.oldestQueuedAt.getTime() - b.oldestQueuedAt.getTime();\n });\n}\n\nfunction sortReservationRows<T extends {requiredLabels: string[]}>(rows: T[]): T[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return labelKey(a.requiredLabels).localeCompare(labelKey(b.requiredLabels));\n });\n}\n\nfunction isSubset(requiredLabels: string[], availableLabels: string[]): boolean {\n return requiredLabels.every((label) => availableLabels.includes(label));\n}\n\nfunction drawSlots(templates: NormalizedTemplate[], count: number): void {\n let remaining = count;\n for (const template of templates) {\n if (remaining === 0) return;\n const used = Math.min(template.remainingSlots, remaining);\n template.remainingSlots -= used;\n remaining -= used;\n }\n}\n\nfunction labelKey(labels: string[]): string {\n return JSON.stringify(labels);\n}\n"],"names":["canonicalizeLabels","and","arrayContains","asc","eq","exists","gt","inArray","isNotNull","isNull","lt","notExists","notInArray","or","sql","recordProviderRunnerActivationOutcome","db","lockRunnerReservationAdvisoryKeysTx","terminalStates","pendingJobExecutions","provisionerCapabilitySnapshots","provisionerTokens","reservations","runnerActivationTokens","runnerControlSessions","providerRunners","pollDemandAndReserve","params","result","transaction","tx","pollDemandAndReserveTx","stats","execute","workspaceId","pollDemandAndReserveLockedTx","scope","pollInstallationDemandAndReserve","candidateWorkspaceIds","listInstallationDemandWorkspaceIds","eligibleWorkspaceIds","results","remainingMaxReservations","maxReservations","remainingTemplates","templates","map","template","labels","remainingSlots","availableSlots","signal","aborted","lockResult","locked","rows","newlyReservedUnits","provisionerId","ttlSeconds","activationGraceSeconds","undefined","capabilityWindowSeconds","push","consumeInstallationTemplateSlots","onReservations","reduce","total","reservation","count","newlyReservedCount","subtotal","flatMap","launchGrants","satisfyingTemplates","filter","isSubset","sort","a","b","length","templateKey","localeCompare","drawSlots","demandRows","select","requiredLabels","queued","oldestQueuedAt","createdAt","from","where","groupBy","row","Date","capabilityLabels","listActiveWorkspaceCapabilityLabelsTx","windowSeconds","demand","some","activeReservationRows","reserved","expiresAt","reservedByLabels","Map","labelKey","activeProvisionerReservationRows","listActiveProvisionerReservationRowsTx","deductProvisionerReservations","grants","workspaceIdField","sortDemandRows","get","unreserved","Math","max","capacity","grant","min","reservedAfterGrant","idleRunners","listIdleRunnerInstancesTx","boundCount","boundReservation","insert","values","kind","returning","id","Error","bindIdleRunnerInstancesTx","reservationId","launchCount","launchReservation","inserted","candidateRows","candidateIds","reservationIds","activeRows","orderBy","for","activeIds","activeIdSet","Set","usedRunnerRows","intendedReservationId","state","reservationReleasedAt","usedByReservation","runner","has","addUsedRunner","includes","used","size","pending","runnerId","runnerIds","add","set","canBindRunner","runnerSessionId","isBindableRunner","providerRunnerId","runnerInstanceId","closedAt","lockedRunners","selectIdleRunnerInstancesTx","launchKind","limit","retrySelection","Symbol","lockTx","candidateRunners","candidate","compareRunnerIds","error","idleRunnerIds","update","revokedAt","consumedAt","boundRunners","assignedAt","updatedAt","reboundCount","outcome","listQueuedDemandWorkspaceIds","innerJoin","advertisedAt","deleteExpiredReservations","expiredRows","deleteReservationsWithCleanupTx","expiredOnly","deleteReservationsByIds","ids","isAffectedRunner","affectedRunners","reservationRows","assignedRunnerIds","intendedRunnerIds","deleted","delete","releaseReservationUnits","released","release","releases","decremented","activeReservations","sortReservationRows","left","right","specificity","getTime","availableLabels","every","label","remaining","JSON","stringify"],"mappings":"AAAA,SAAQA,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,aAAa,EACbC,GAAG,EACHC,EAAE,EACFC,MAAM,EACNC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,SAAS,EACTC,UAAU,EACVC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SAAQC,qCAAqC,QAAO,uBAAuB;AAE3E,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,mCAAmC,QAAO,yBAAyB;AAC3E,SAAQC,cAAc,QAAO,qBAAqB;AAClD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,8BAA8B,QAAO,+CAA+C;AAC5F,SAAQC,iBAAiB,QAAO,iCAAiC;AACjE,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,sBAAsB,QAAO,uCAAuC;AAC5E,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAA2CC,eAAe,QAAO,+BAA+B;AAiGhG,OAAO,eAAeC,qBACpBC,MAAkC;IAElC,MAAMC,SAAS,MAAMZ,KAAKa,WAAW,CAAC,OAAOC;QAC3C,OAAO,MAAMC,uBAAuBD,IAAIH;IAC1C;IACA,OAAO;QAACK,OAAOJ,OAAOI,KAAK;QAAEV,cAAcM,OAAON,YAAY;IAAA;AAChE;AAEA,OAAO,eAAeS,uBACpBD,EAAM,EACNH,MAAkC;IAElC,MAAMG,GAAGG,OAAO,CAACnB,GAAG,CAAC,sCAAsC,EAAEa,OAAOO,WAAW,CAAC,EAAE,CAAC;IACnF,OAAO,MAAMC,6BAA6BL,IAAI;QAAC,GAAGH,MAAM;QAAES,OAAO;IAAW;AAC9E;AAEA,OAAO,eAAeC,iCACpBV,MAA8C;IAM9C,MAAMW,wBAAwB,MAAMC,mCAClCZ,OAAOa,oBAAoB;IAE7B,MAAMC,UAAwC,EAAE;IAChD,IAAIC,2BAA2Bf,OAAOgB,eAAe;IACrD,MAAMC,qBAAqBjB,OAAOkB,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YAC7D,GAAGA,QAAQ;YACXC,QAAQ;mBAAIhD,mBAAmB+C,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACA,KAAK,MAAMhB,eAAeI,sBAAuB;QAC/C,IAAIX,OAAOwB,MAAM,EAAEC,WAAWV,6BAA6B,GAAG;QAC9D,MAAMd,SAAS,MAAMZ,KAAKa,WAAW,CAAC,OAAOC;YAC3C,MAAMuB,aAAa,MAAMvB,GAAGG,OAAO,CACjCnB,GAAG,CAAC,0CAA0C,EAAEoB,YAAY,YAAY,CAAC;YAE3E,MAAMoB,SAASD,WAAWE,IAAI,CAAC,EAAE;YACjC,IAAI,CAACD,QAAQA,QAAQ,OAAO;gBAACtB,OAAO,EAAE;gBAAEV,cAAc,EAAE;gBAAEkC,oBAAoB,EAAE;YAAA;YAChF,OAAO,MAAMrB,6BAA6BL,IAAI;gBAC5CI;gBACAuB,eAAe9B,OAAO8B,aAAa;gBACnCd,iBAAiBD;gBACjBgB,YAAY/B,OAAO+B,UAAU;gBAC7B,GAAI/B,OAAOgC,sBAAsB,KAAKC,YAClC;oBAACD,wBAAwBhC,OAAOgC,sBAAsB;gBAAA,IACtD,CAAC,CAAC;gBACNd,WAAWD,mBAAmBE,GAAG,CAAC,CAACC,WAAc,CAAA;wBAC/C,GAAGA,QAAQ;wBACXG,gBAAgBH,SAASE,cAAc;oBACzC,CAAA;gBACAY,yBAAyBlC,OAAOkC,uBAAuB;gBACvDzB,OAAO;YACT;QACF;QACAK,QAAQqB,IAAI,CAAClC;QACbmC,iCAAiCnB,oBAAoBhB,OAAON,YAAY;QACxEK,OAAOqC,cAAc,GAAGpC,OAAON,YAAY;QAC3CoB,4BAA4Bd,OAAO4B,kBAAkB,CAACS,MAAM,CAC1D,CAACC,OAAOC,cAAgBD,QAAQC,YAAYC,KAAK,EACjD;IAEJ;IACA,MAAMC,qBAAqB5B,QAAQwB,MAAM,CACvC,CAACC,OAAOtC,SACNsC,QACAtC,OAAO4B,kBAAkB,CAACS,MAAM,CAAC,CAACK,UAAUH,cAAgBG,WAAWH,YAAYC,KAAK,EAAE,IAC5F;IAEF,OAAO;QACLpC,OAAOS,QAAQ8B,OAAO,CAAC,CAAC3C,SAAWA,OAAOI,KAAK;QAC/CV,cAAcmB,QAAQ8B,OAAO,CAAC,CAAC3C,SAAWA,OAAON,YAAY;QAC7D,GAAI+C,qBAAqB,IAAI;YAACA;QAAkB,IAAI,CAAC,CAAC;IACxD;AACF;AAEA,SAASN,iCACPlB,SAA+B,EAC/B2B,YAAgC;IAEhC,mFAAmF;IACnF,4EAA4E;IAC5E,KAAK,MAAML,eAAeK,aAAc;QACtC,MAAMC,sBAAsB5B,UACzB6B,MAAM,CAAC,CAAC3B,WAAa4B,SAASR,YAAYnB,MAAM,EAAED,SAASC,MAAM,GACjE4B,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAE7B,MAAM,CAAC+B,MAAM,GAAGD,EAAE9B,MAAM,CAAC+B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBN,YAAYC,KAAK;IAClD;AACF;AAEA,eAAejC,6BACbL,EAAM,EACNH,MAAwC;IAExC,IAAIwD,aAAa,AACf,CAAA,MAAMrD,GACHsD,MAAM,CAAC;QACNC,gBAAgBlE,qBAAqBkE,cAAc;QACnDC,QAAQxE,GAAW,CAAC,aAAa,CAAC;QAClCyE,gBAAgBzE,GAAkB,CAAC,IAAI,EAAEK,qBAAqBqE,SAAS,CAAC,CAAC,CAAC;IAC5E,GACCC,IAAI,CAACtE,sBACLuE,KAAK,CAACtF,GAAGe,qBAAqBe,WAAW,EAAEP,OAAOO,WAAW,GAC7DyD,OAAO,CAACxE,qBAAqBkE,cAAc,CAAA,EAC9CvC,GAAG,CAAC,CAAC8C,MAAS,CAAA;YACd,GAAGA,GAAG;YACNL,gBAAgB,IAAIM,KAAKD,IAAIL,cAAc;QAC7C,CAAA;IACA,IAAI5D,OAAOkC,uBAAuB,KAAKD,WAAW;QAChD,MAAMkC,mBAAmB,MAAMC,sCAAsCjE,IAAI;YACvEI,aAAaP,OAAOO,WAAW;YAC/B8D,eAAerE,OAAOkC,uBAAuB;QAC/C;QACAsB,aAAaA,WAAWT,MAAM,CAC5B,CAACuB,SAAW,CAACH,iBAAiBI,IAAI,CAAC,CAAClD,SAAW2B,SAASsB,OAAOZ,cAAc,EAAErC;IAEnF;IAEA,MAAMmD,wBAAwB,MAAMrE,GACjCsD,MAAM,CAAC;QACNC,gBAAgB/D,aAAa+D,cAAc;QAC3Ce,UAAUtF,GAAW,CAAC,aAAa,EAAEQ,aAAa8C,KAAK,CAAC,UAAU,CAAC;IACrE,GACCqB,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IAAIG,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAAG5B,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAE5F6E,OAAO,CAACrE,aAAa+D,cAAc;IAEtC,MAAMiB,mBAAmB,IAAIC,IAC3BJ,sBAAsBrD,GAAG,CAAC,CAAC8C,MAAQ;YAACY,SAASZ,IAAIP,cAAc;YAAGO,IAAIQ,QAAQ;SAAC;IAEjF,MAAMK,mCAAmC,MAAMC,uCAAuC5E,IAAI;QACxFI,aAAaP,OAAOO,WAAW;QAC/BuB,eAAe9B,OAAO8B,aAAa;IACrC;IAEA,MAAMZ,YAAYlB,OAAOkB,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YACpDiC,aAAajC,SAASiC,WAAW;YACjChC,QAAQ;mBAAIhD,mBAAmB+C,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACAyD,8BAA8B9D,WAAW4D;IACzC,MAAMzE,QAAsB,EAAE;IAC9B,MAAM4E,SAA6B,EAAE;IACrC,MAAMpD,qBAA4C,EAAE;IACpD,IAAId,2BAA2Bf,OAAOgB,eAAe;IACrD,MAAMkE,mBACJlF,OAAOkC,uBAAuB,KAAKD,YAAY,CAAC,IAAI;QAAC1B,aAAaP,OAAOO,WAAW;IAAA;IAEtF,KAAK,MAAM+D,UAAUa,eAAe3B,YAAa;QAC/C,MAAMV,sBAAsB5B,UACzB6B,MAAM,CAAC,CAAC3B,WAAa4B,SAASsB,OAAOZ,cAAc,EAAEtC,SAASC,MAAM,GACpE4B,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAE7B,MAAM,CAAC+B,MAAM,GAAGD,EAAE9B,MAAM,CAAC+B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAG5F,IAAIP,oBAAoBM,MAAM,KAAK,GAAG;QAEtC,MAAMqB,WAAWE,iBAAiBS,GAAG,CAACP,SAASP,OAAOZ,cAAc,MAAM;QAC1E,MAAM2B,aAAaC,KAAKC,GAAG,CAAC,GAAGjB,OAAOX,MAAM,GAAGc;QAC/C,MAAMe,WAAW1C,oBAAoBR,MAAM,CACzC,CAACC,OAAOnB,WAAamB,QAAQnB,SAASE,cAAc,EACpD;QAEF,MAAMmE,QAAQH,KAAKI,GAAG,CAACL,YAAYG,UAAUzE;QAC7C,IAAI4E,qBAAqBlB;QAEzB,IAAIgB,QAAQ,KAAKzF,OAAOgB,eAAe,GAAG,GAAG;YAC3C,MAAM4E,cAAc,MAAMC,0BAA0B1F,IAAI;gBACtD2B,eAAe9B,OAAO8B,aAAa;gBACnC4B,gBAAgBY,OAAOZ,cAAc;gBACrCjB,OAAOgD;gBACPlF,aAAaP,OAAOO,WAAW;gBAC/BE,OAAOT,OAAOS,KAAK;YACrB;YAEAM,4BAA4B0E;YAE5B,MAAMK,aAAaF,YAAYxC,MAAM;YACrC,IAAI0C,aAAa,GAAG;gBAClB,MAAM,CAACC,iBAAiB,GAAG,MAAM5F,GAC9B6F,MAAM,CAACrG,cACPsG,MAAM,CAAC;oBACN1F,aAAaP,OAAOO,WAAW;oBAC/BuB,eAAe9B,OAAO8B,aAAa;oBACnC4B,gBAAgBY,OAAOZ,cAAc;oBACrCjB,OAAOqD;oBACPI,MAAM;oBACNxB,WAAWvF,GAAG,CAAC,SAAS,EAAEa,OAAOgC,sBAAsB,IAAIhC,OAAO+B,UAAU,CAAC,yBAAyB,CAAC;gBACzG,GACCoE,SAAS,CAAC;oBAACC,IAAIzG,aAAayG,EAAE;gBAAA;gBAEjC,IAAI,CAACL,kBAAkB,MAAM,IAAIM,MAAM;gBAEvC,MAAMC,0BAA0BnG,IAAI;oBAClC2B,eAAe9B,OAAO8B,aAAa;oBACnCyE,eAAeR,iBAAiBK,EAAE;oBAClC7F,aAAaP,OAAOO,WAAW;oBAC/BmD,gBAAgBY,OAAOZ,cAAc;oBACrCjD,OAAOT,OAAOS,KAAK;oBACnBmF;gBACF;YACF;YAEA,MAAMY,cAAcf,QAAQK;YAC5BvC,UAAUT,qBAAqB0D;YAC/B,IAAIC;YACJ,IAAID,cAAc,GAAG;gBACnB,MAAM,CAACE,SAAS,GAAG,MAAMvG,GACtB6F,MAAM,CAACrG,cACPsG,MAAM,CAAC;oBACN1F,aAAaP,OAAOO,WAAW;oBAC/BuB,eAAe9B,OAAO8B,aAAa;oBACnC4B,gBAAgBY,OAAOZ,cAAc;oBACrCjB,OAAO+D;oBACPN,MAAM;oBACNxB,WAAWvF,GAAG,CAAC,SAAS,EAAEa,OAAO+B,UAAU,CAAC,yBAAyB,CAAC;gBACxE,GACCoE,SAAS,CAAC;oBAACC,IAAIzG,aAAayG,EAAE;oBAAE1B,WAAW/E,aAAa+E,SAAS;gBAAA;gBAEpE,IAAI,CAACgC,UAAU,MAAM,IAAIL,MAAM;gBAC/BI,oBAAoBC;YACtB;YAEA7E,mBAAmBM,IAAI,CAAC;gBAACd,QAAQiD,OAAOZ,cAAc;gBAAEjB,OAAOgD;YAAK;YACpEE,sBAAsBF;YACtB,IAAIgB,mBAAmB;gBACrBxB,OAAO9C,IAAI,CAAC;oBACVoE,eAAeE,kBAAkBL,EAAE;oBACnC,GAAGlB,gBAAgB;oBACnB7D,QAAQiD,OAAOZ,cAAc;oBAC7BjB,OAAO+D;oBACP9B,WAAW+B,kBAAkB/B,SAAS;gBACxC;YACF;QACF;QAEArE,MAAM8B,IAAI,CAAC;YACT,GAAG+C,gBAAgB;YACnB7D,QAAQiD,OAAOZ,cAAc;YAC7BC,QAAQW,OAAOX,MAAM;YACrBc,UAAUkB;YACV/B,gBAAgBU,OAAOV,cAAc;QACvC;IACF;IAEA,OAAO;QAACvD;QAAOV,cAAcsF;QAAQpD;IAAkB;AACzD;AAEA,eAAekD,uCACb5E,EAAM,EACNH,MAAoD;IAEpD,MAAM2G,gBAAgB,MAAMxG,GACzBsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnDnD,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC;IAG3C,MAAMyH,eAAeD,cAAcxF,GAAG,CAAC,CAAC8C,MAAQA,IAAImC,EAAE;IACtD,IAAIQ,aAAaxD,MAAM,KAAK,GAAG,OAAO,EAAE;IAExC,kFAAkF;IAClF,0FAA0F;IAC1F,MAAM9D,oCAAoCa,IAAI;QAC5C2B,eAAe9B,OAAO8B,aAAa;QACnC+E,gBAAgBD;IAClB;IAEA,MAAME,aAAa,MAAM3G,GACtBsD,MAAM,CAAC;QACN2C,IAAIzG,aAAayG,EAAE;QACnB1C,gBAAgB/D,aAAa+D,cAAc;QAC3CjB,OAAO9C,aAAa8C,KAAK;IAC3B,GACCqB,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnDnD,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAGxC4H,OAAO,CAACvI,IAAImB,aAAayG,EAAE,GAC3BY,GAAG,CAAC;IACP,IAAIF,WAAW1D,MAAM,KAAK,GAAG,OAAO,EAAE;IAEtC,MAAM6D,YAAYH,WAAW3F,GAAG,CAAC,CAAC8C,MAAQA,IAAImC,EAAE;IAChD,MAAMc,cAAc,IAAIC,IAAIF;IAC5B,MAAMG,iBAAiB,MAAMjH,GAC1BsD,MAAM,CAAC;QACN2C,IAAItG,gBAAgBsG,EAAE;QACtBG,eAAezG,gBAAgByG,aAAa;QAC5Cc,uBAAuBvH,gBAAgBuH,qBAAqB;QAC5DC,OAAOxH,gBAAgBwH,KAAK;IAC9B,GACCxD,IAAI,CAAChE,iBACLiE,KAAK,CACJzF,IACEG,GAAGqB,gBAAgBgC,aAAa,EAAE9B,OAAO8B,aAAa,GACtDhD,OAAOgB,gBAAgByH,qBAAqB,GAC5CrI,GACEN,QAAQkB,gBAAgByG,aAAa,EAAEU,YACvC3I,IACEM,QAAQkB,gBAAgBuH,qBAAqB,EAAEJ,YAC/ChI,WAAWa,gBAAgBwH,KAAK,EAAE;WAAI/H;KAAe;IAK/D,MAAMiI,oBAAoB,IAAI5C;IAC9B,KAAK,MAAM6C,UAAUL,eAAgB;QACnC,IAAIK,OAAOlB,aAAa,IAAIW,YAAYQ,GAAG,CAACD,OAAOlB,aAAa,GAAG;YACjEoB,cAAcH,mBAAmBC,OAAOlB,aAAa,EAAEkB,OAAOrB,EAAE;QAClE;QACA,IACEqB,OAAOJ,qBAAqB,IAC5BH,YAAYQ,GAAG,CAACD,OAAOJ,qBAAqB,KAC5C,CAAC9H,eAAeqI,QAAQ,CAACH,OAAOH,KAAK,GACrC;YACAK,cAAcH,mBAAmBC,OAAOJ,qBAAqB,EAAEI,OAAOrB,EAAE;QAC1E;IACF;IAEA,OAAOU,WAAWlE,OAAO,CAAC,CAACJ;QACzB,MAAMqF,OAAOL,kBAAkBpC,GAAG,CAAC5C,YAAY4D,EAAE,GAAG0B,QAAQ;QAC5D,MAAMC,UAAUzC,KAAKC,GAAG,CAAC,GAAG/C,YAAYC,KAAK,GAAGoF;QAChD,OAAOE,UAAU,IAAI;YAAC;gBAACrE,gBAAgBlB,YAAYkB,cAAc;gBAAEe,UAAUsD;YAAO;SAAE,GAAG,EAAE;IAC7F;AACF;AAEA,SAASJ,cACPH,iBAA2C,EAC3CjB,aAAqB,EACrByB,QAAgB;IAEhB,MAAMC,YAAYT,kBAAkBpC,GAAG,CAACmB;IACxC,IAAI0B,WAAWA,UAAUC,GAAG,CAACF;SACxBR,kBAAkBW,GAAG,CAAC5B,eAAe,IAAIY,IAAI;QAACa;KAAS;AAC9D;AAEA,2FAA2F;AAC3F,4DAA4D;AAC5D,SAASI,cAAcjI,EAAM,EAAEH,MAA4B;IACzD,OAAO1B,IACLY,GACEZ,IAAIQ,OAAOgB,gBAAgBS,WAAW,GAAGzB,OAAOgB,gBAAgByG,aAAa,IAC7EjI,IACEO,UAAUiB,gBAAgByG,aAAa,GACvCvG,OAAOS,KAAK,KAAK,iBACbwB,YACA/C,GACEJ,OAAOgB,gBAAgBS,WAAW,GAClC9B,GAAGqB,gBAAgBS,WAAW,EAAEP,OAAOO,WAAW,IAExDvB,UACEmB,GACGsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAEtG,gBAAgByG,aAAa,GACjD5H,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,QAMjDD,GACEJ,OAAOgB,gBAAgBuH,qBAAqB,GAC5CrI,UACEmB,GACGsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAEtG,gBAAgBuH,qBAAqB,GACzD1I,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,OAK/CL,OAAOgB,gBAAgByH,qBAAqB,GAC5CzI,OAAOgB,gBAAgBuI,eAAe;AAE1C;AAEA,SAASC,iBAAiBnI,EAAM,EAAEH,MAA4B;IAC5D,OAAO1B,IACLG,GAAGqB,gBAAgBgC,aAAa,EAAE9B,OAAO8B,aAAa,GACtDsG,cAAcjI,IAAIH,SAClBnB,UAAUiB,gBAAgByI,gBAAgB,GAC1C9J,GAAGqB,gBAAgBwH,KAAK,EAAE,YAC1B/I,cAAcuB,gBAAgBuB,MAAM,EAAErB,OAAO0D,cAAc,GAC3DhF,OACEyB,GACGsD,MAAM,CAAC;QAAC2C,IAAIvG,sBAAsBuG,EAAE;IAAA,GACpCtC,IAAI,CAACjE,uBACLkE,KAAK,CACJzF,IACEG,GAAGoB,sBAAsB2I,gBAAgB,EAAE1I,gBAAgBsG,EAAE,GAC7D3H,GAAGoB,sBAAsBiC,aAAa,EAAE9B,OAAO8B,aAAa,GAC5DhD,OAAOe,sBAAsB4I,QAAQ,GACrC9J,GAAGkB,sBAAsB6E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC;AAK1D;AAEA,eAAe0G,0BACb1F,EAAM,EACNH,MAAmD;IAEnD,MAAM0I,gBAAgB,MAAMC,4BAA4BxI,IAAIH;IAC5D,IAAI0I,cAActF,MAAM,KAAK,GAAG,OAAOsF;IAEvC,4EAA4E;IAC5E,6EAA6E;IAC7E,yEAAyE;IACzE,OAAO,MAAMC,4BAA4BxI,IAAI;QAC3C,GAAGH,MAAM;QACTiI,WAAWS,cAAcvH,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;QAClD3D,OAAOiG,cAActF,MAAM;IAC7B;AACF;AAEA,eAAeuF,4BACbxI,EAAM,EACNH,MAAyE;IAEzE,IAAIA,OAAOiI,SAAS,EAAE;QACpB,2FAA2F;QAC3F,oFAAoF;QACpF,OAAO,MAAM9H,GACVsD,MAAM,CAAC;YAAC2C,IAAItG,gBAAgBsG,EAAE;YAAEwC,YAAY9I,gBAAgB8I,UAAU;QAAA,GACtE9E,IAAI,CAAChE,iBACLiE,KAAK,CAACzF,IAAIM,QAAQkB,gBAAgBsG,EAAE,EAAEpG,OAAOiI,SAAS,GAAGK,iBAAiBnI,IAAIH,UAC9E+G,OAAO,CAACvI,IAAIsB,gBAAgB+D,SAAS,GAAGrF,IAAIsB,gBAAgBsG,EAAE,GAC9DyC,KAAK,CAAC7I,OAAOyC,KAAK;IACvB;IAEA,0FAA0F;IAC1F,4FAA4F;IAC5F,6FAA6F;IAC7F,wFAAwF;IACxF,MAAMqG,iBAAiBC,OAAO;IAC9B,MAAMnD,cAAc,MAAM,AAAC,CAAA;QACzB,MAAO,KAAM;YACX,IAAI;gBACF,OAAO,MAAMzF,GAAGD,WAAW,CAAC,OAAO8I;oBACjC,MAAMC,mBAAmB,MAAMD,OAC5BvF,MAAM,CAAC;wBAAC2C,IAAItG,gBAAgBsG,EAAE;wBAAEwC,YAAY9I,gBAAgB8I,UAAU;oBAAA,GACtE9E,IAAI,CAAChE,iBACLiE,KAAK,CAACuE,iBAAiBU,QAAQhJ,SAC/B+G,OAAO,CAACvI,IAAIsB,gBAAgB+D,SAAS,GAAGrF,IAAIsB,gBAAgBsG,EAAE,GAC9DyC,KAAK,CAAC7I,OAAOyC,KAAK;oBAErB,IAAIwG,iBAAiB7F,MAAM,KAAK,GAAG,OAAO6F;oBAE1C,sFAAsF;oBACtF,iFAAiF;oBACjF,MAAMP,gBAAyC,EAAE;oBACjD,KAAK,MAAMQ,aAAa;2BAAID;qBAAiB,CAAChG,IAAI,CAACkG,kBAAmB;wBACpE,MAAM,CAAC1B,OAAO,GAAG,MAAMuB,OACpBvF,MAAM,CAAC;4BAAC2C,IAAItG,gBAAgBsG,EAAE;4BAAEwC,YAAY9I,gBAAgB8I,UAAU;wBAAA,GACtE9E,IAAI,CAAChE,iBACLiE,KAAK,CAACzF,IAAIG,GAAGqB,gBAAgBsG,EAAE,EAAE8C,UAAU9C,EAAE,GAAGkC,iBAAiBU,QAAQhJ,UACzEgH,GAAG,CAAC;wBACP,IAAIS,QAAQiB,cAAcvG,IAAI,CAACsF;oBACjC;oBAEA,IAAIiB,cAActF,MAAM,KAAK6F,iBAAiB7F,MAAM,EAAE,MAAM0F;oBAC5D,OAAOJ;gBACT;YACF,EAAE,OAAOU,OAAO;gBACd,IAAIA,UAAUN,gBAAgB,MAAMM;YACtC;QACF;IACF,CAAA;IAEA,OAAOxD;AACT;AAEA,eAAeU,0BACbnG,EAAM,EACNH,MAOC;IAED,MAAMqJ,gBAAgBrJ,OAAO4F,WAAW,CAACzE,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;IAElE,MAAMjG,GACHmJ,MAAM,CAAC1J,wBACPuI,GAAG,CAAC;QAACoB,WAAWpK,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1B4E,KAAK,CACJzF,IACEM,QAAQgB,uBAAuB4I,gBAAgB,EAAEa,gBACjDvK,OAAOc,uBAAuB4J,UAAU,GACxC1K,OAAOc,uBAAuB2J,SAAS;IAI7C,0FAA0F;IAC1F,6FAA6F;IAC7F,eAAe;IACf,MAAME,eAAe,MAAMtJ,GACxBmJ,MAAM,CAACxJ,iBACPqI,GAAG,CAAC;QACH5H,aAAaP,OAAOO,WAAW;QAC/BgG,eAAevG,OAAOuG,aAAa;QACnCc,uBAAuB;QACvBqC,YAAYvK,GAAG,CAAC,KAAK,CAAC;QACtBwK,WAAWxK,GAAG,CAAC,KAAK,CAAC;IACvB,GACC4E,KAAK,CAACzF,IAAIM,QAAQkB,gBAAgBsG,EAAE,EAAEiD,gBAAgBf,iBAAiBnI,IAAIH,UAC3EmG,SAAS,CAAC;QAACC,IAAItG,gBAAgBsG,EAAE;QAAEwC,YAAY9I,gBAAgB8I,UAAU;IAAA;IAE5E,IAAIa,aAAarG,MAAM,KAAKpD,OAAO4F,WAAW,CAACxC,MAAM,EACnD,MAAM,IAAIiD,MAAM;IAElB,MAAMuD,eAAeH,aAAa1G,MAAM,CAAC,CAAC0E,SAAWA,OAAOmB,UAAU,KAAK,UAAUxF,MAAM;IAC3F,IAAIwG,eAAe,GACjBxK,sCAAsC;QAACyK,SAAS;QAAWpH,OAAOmH;IAAY;AAClF;AAEA,eAAehJ,mCAAmCC,oBAAyC;IACzF,IAAIA,qBAAqBiH,IAAI,KAAK,GAAG,OAAO,EAAE;IAC9C,MAAMlG,OAAO,MAAMvC,KAChBoE,MAAM,CAAC;QACNlD,aAAaf,qBAAqBe,WAAW;QAC7CqD,gBAAgBzE,GAAS,CAAC,IAAI,EAAEK,qBAAqBqE,SAAS,CAAC,CAAC,CAAC;IACnE,GACCC,IAAI,CAACtE,sBACLuE,KAAK,CAACnF,QAAQY,qBAAqBe,WAAW,EAAE;WAAIM;KAAqB,GACzEmD,OAAO,CAACxE,qBAAqBe,WAAW,EACxCwG,OAAO,CACNvI,IAAIW,GAAG,CAAC,IAAI,EAAEK,qBAAqBqE,SAAS,CAAC,CAAC,CAAC,GAC/CrF,IAAIgB,qBAAqBe,WAAW;IAExC,OAAOqB,KAAKT,GAAG,CAAC,CAAC8C,MAAQA,IAAI1D,WAAW;AAC1C;AAEA,OAAO,eAAeuJ;IACpB,MAAMlI,OAAO,MAAMvC,KAChBoE,MAAM,CAAC;QAAClD,aAAaf,qBAAqBe,WAAW;IAAA,GACrDuD,IAAI,CAACtE,sBACLwE,OAAO,CAACxE,qBAAqBe,WAAW;IAC3C,OAAOqB,KAAKT,GAAG,CAAC,CAAC8C,MAAQA,IAAI1D,WAAW;AAC1C;AAEA,eAAe6D,sCACbjE,EAAM,EACNH,MAAoD;IAEpD,MAAM4B,OAAO,MAAMzB,GAChBsD,MAAM,CAAC;QAACpC,QAAQ5B,+BAA+B4B,MAAM;IAAA,GACrDyC,IAAI,CAACrE,gCACLsK,SAAS,CACRrK,mBACAjB,GAAGiB,kBAAkB0G,EAAE,EAAE3G,+BAA+BqC,aAAa,GAEtEiC,KAAK,CACJzF,IACEG,GAAGgB,+BAA+Bc,WAAW,EAAEP,OAAOO,WAAW,GACjE9B,GAAGiB,kBAAkBa,WAAW,EAAEP,OAAOO,WAAW,GACpD9B,GAAGiB,kBAAkBe,KAAK,EAAE,cAC5BtB,GAAG,CAAC,EAAEM,+BAA+BuK,YAAY,CAAC,YAAY,EAAEhK,OAAOqE,aAAa,CAAC,yBAAyB,CAAC,EAC/GlF,GAAG,CAAC,EAAEO,kBAAkB6J,SAAS,CAAC,QAAQ,CAAC,EAC3CpK,GAAG,CAAC,CAAC,EAAEO,kBAAkBgF,SAAS,CAAC,YAAY,EAAEhF,kBAAkBgF,SAAS,CAAC,SAAS,CAAC;IAG7F,OAAO9C,KAAKT,GAAG,CAAC,CAAC8C,MAAQA,IAAI5C,MAAM;AACrC;AAEA,OAAO,eAAe4I,0BAA0BjK,MAAyB;IACvE,OAAO,MAAMX,KAAKa,WAAW,CAAC,OAAOC;QACnC,MAAM+J,cAAc,MAAM/J,GACvBsD,MAAM,CAAC;YAAC2C,IAAIzG,aAAayG,EAAE;QAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CAAChF,GAAGY,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,GAC3C4H,OAAO,CAACvI,IAAImB,aAAa+E,SAAS,GAClCmE,KAAK,CAAC7I,QAAQ6I,SAAS;QAC1B,OAAO,MAAMsB,gCACXhK,IACA+J,YAAY/I,GAAG,CAAC,CAACqB,cAAgBA,YAAY4D,EAAE,GAC/C;YAACgE,aAAa;QAAI;IAEtB;AACF;AAEA,OAAO,eAAeC,wBAAwBC,GAAa;IACzD,IAAIA,IAAIlH,MAAM,KAAK,GAAG,OAAO;IAE7B,OAAO,MAAM/D,KAAKa,WAAW,CAAC,OAAOC;QACnC,OAAO,MAAMgK,gCAAgChK,IAAImK;IACnD;AACF;AAEA,eAAeH,gCACbhK,EAAM,EACNmK,GAAa,EACbtK,SAAkC,CAAC,CAAC;IAEpC,IAAIsK,IAAIlH,MAAM,KAAK,GAAG,OAAO;IAE7B,MAAMmH,mBAAmB,IACvBrL,GACEZ,IACEM,QAAQkB,gBAAgByG,aAAa,EAAE+D,MACvCxL,OAAOgB,gBAAgBuI,eAAe,GACtCvJ,OAAOgB,gBAAgByH,qBAAqB,IAE9C3I,QAAQkB,gBAAgBuH,qBAAqB,EAAEiD;IAEnD,MAAMrB,mBAAmB,MAAM9I,GAC5BsD,MAAM,CAAC;QACN2C,IAAItG,gBAAgBsG,EAAE;QACtBG,eAAezG,gBAAgByG,aAAa;QAC5Cc,uBAAuBvH,gBAAgBuH,qBAAqB;QAC5DgB,iBAAiBvI,gBAAgBuI,eAAe;QAChDd,uBAAuBzH,gBAAgByH,qBAAqB;IAC9D,GACCzD,IAAI,CAAChE,iBACLiE,KAAK,CAACwG,oBACNxD,OAAO,CAACvI,IAAIsB,gBAAgBsG,EAAE;IACjC,MAAMoE,kBAA2C,EAAE;IACnD,KAAK,MAAMtB,aAAa;WAAID;KAAiB,CAAChG,IAAI,CAACkG,kBAAmB;QACpE,MAAM,CAAC1B,OAAO,GAAG,MAAMtH,GACpBsD,MAAM,CAAC;YACN2C,IAAItG,gBAAgBsG,EAAE;YACtBG,eAAezG,gBAAgByG,aAAa;YAC5Cc,uBAAuBvH,gBAAgBuH,qBAAqB;YAC5DgB,iBAAiBvI,gBAAgBuI,eAAe;YAChDd,uBAAuBzH,gBAAgByH,qBAAqB;QAC9D,GACCzD,IAAI,CAAChE,iBACLiE,KAAK,CAACzF,IAAIG,GAAGqB,gBAAgBsG,EAAE,EAAE8C,UAAU9C,EAAE,GAAGmE,qBAChDvD,GAAG,CAAC;QACP,IAAIS,QAAQ+C,gBAAgBrI,IAAI,CAACsF;IACnC;IACA,MAAMgD,kBAAkB,MAAMtK,GAC3BsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEM,QAAQe,aAAayG,EAAE,EAAEkE,MACzBtK,OAAOoK,WAAW,GAAGrL,GAAGY,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAAI8C,YAGjE+E,GAAG,CAAC;IACP,MAAMH,iBAAiB4D,gBAAgBtJ,GAAG,CAAC,CAACqB,cAAgBA,YAAY4D,EAAE;IAE1E,IAAIS,eAAezD,MAAM,KAAK,GAAG,OAAO;IAExC,MAAMsH,oBAAoBF,gBACvBzH,MAAM,CACL,CAAC0E,SACCA,OAAOlB,aAAa,IACpBM,eAAee,QAAQ,CAACH,OAAOlB,aAAa,KAC5C,CAACkB,OAAOY,eAAe,IACvB,CAACZ,OAAOF,qBAAqB,EAEhCpG,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;IAC5B,MAAMuE,oBAAoBH,gBACvBzH,MAAM,CACL,CAAC0E,SACCA,OAAOJ,qBAAqB,IAAIR,eAAee,QAAQ,CAACH,OAAOJ,qBAAqB,GAEvFlG,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;IAC5B,IAAIsE,kBAAkBtH,MAAM,GAAG,GAAG;QAChC,MAAMjD,GACHmJ,MAAM,CAAC1J,wBACPuI,GAAG,CAAC;YAACoB,WAAWpK,GAAG,CAAC,KAAK,CAAC;QAAA,GAC1B4E,KAAK,CACJzF,IACEM,QAAQgB,uBAAuB4I,gBAAgB,EAAEkC,oBACjD5L,OAAOc,uBAAuB4J,UAAU,GACxC1K,OAAOc,uBAAuB2J,SAAS;QAG7C,MAAMpJ,GACHmJ,MAAM,CAACxJ,iBACPqI,GAAG,CAAC;YACH5H,aAAa;YACbgG,eAAe;YACfmD,YAAY;YACZC,WAAWxK,GAAG,CAAC,KAAK,CAAC;QACvB,GACC4E,KAAK,CACJzF,IACEM,QAAQkB,gBAAgBsG,EAAE,EAAEsE,oBAC5B5L,OAAOgB,gBAAgBuI,eAAe,GACtCvJ,OAAOgB,gBAAgByH,qBAAqB;IAGpD;IACA,IAAIoD,kBAAkBvH,MAAM,GAAG,GAAG;QAChC,MAAMjD,GACHmJ,MAAM,CAACxJ,iBACPqI,GAAG,CAAC;YAACd,uBAAuB;YAAMsC,WAAWxK,GAAG,CAAC,KAAK,CAAC;QAAA,GACvD4E,KAAK,CACJzF,IACEM,QAAQkB,gBAAgBsG,EAAE,EAAEuE,oBAC5B/L,QAAQkB,gBAAgBuH,qBAAqB,EAAER;IAGvD;IAEA,MAAM+D,UAAU,MAAMzK,GACnB0K,MAAM,CAAClL,cACPoE,KAAK,CAACnF,QAAQe,aAAayG,EAAE,EAAES,iBAC/BV,SAAS,CAAC;QAACC,IAAIzG,aAAayG,EAAE;IAAA;IAEjC,OAAOwE,QAAQxH,MAAM;AACvB;AAEA,OAAO,eAAe0H,wBACpB3K,EAAM,EACNH,MAIC;IAED,IAAI+K,WAAW;IAEf,KAAK,MAAMC,WAAWhL,OAAOiL,QAAQ,CAAE;QACrC,IAAID,QAAQvI,KAAK,IAAI,GAAG;QAExB,MAAMyI,cAAc,MAAM/K,GACvBmJ,MAAM,CAAC3J,cACPwI,GAAG,CAAC;YAAC1F,OAAOtD,GAAG,CAAC,EAAEQ,aAAa8C,KAAK,CAAC,GAAG,EAAEuI,QAAQvI,KAAK,CAAC,CAAC;QAAA,GACzDsB,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAE4E,QAAQzE,aAAa,GACzC9H,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnDnD,GAAGgB,aAAa8C,KAAK,EAAEuI,QAAQvI,KAAK,GACpC9D,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAGxCgH,SAAS,CAAC;YAACC,IAAIzG,aAAayG,EAAE;QAAA;QAEjC,IAAI8E,YAAY9H,MAAM,GAAG,GAAG;YAC1B2H,YAAYC,QAAQvI,KAAK;YACzB;QACF;QAEA,MAAMmI,UAAU,MAAMzK,GACnB0K,MAAM,CAAClL,cACPoE,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAE4E,QAAQzE,aAAa,GACzC9H,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnD3C,GAAG,CAAC,EAAEQ,aAAa8C,KAAK,CAAC,IAAI,EAAEuI,QAAQvI,KAAK,CAAC,CAAC,EAC9C9D,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAGxCgH,SAAS,CAAC;YAAC1D,OAAO9C,aAAa8C,KAAK;QAAA;QAEvCsI,YAAYH,QAAQtI,MAAM,CAAC,CAACC,OAAO0B,MAAQ1B,QAAQ0B,IAAIxB,KAAK,EAAE;IAChE;IAEA,OAAOsI;AACT;AAEA,SAAS/F,8BACP9D,SAA+B,EAC/BiK,kBAAkE;IAElE,KAAK,MAAM3I,eAAe4I,oBAAoBD,oBAAqB;QACjE,MAAMrI,sBAAsB5B,UACzB6B,MAAM,CAAC,CAAC3B,WAAa4B,SAASR,YAAYkB,cAAc,EAAEtC,SAASC,MAAM,GACzE4B,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAE7B,MAAM,CAAC+B,MAAM,GAAGD,EAAE9B,MAAM,CAAC+B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBN,YAAYiC,QAAQ;IACrD;AACF;AAEA,SAAS0E,iBAAiBkC,IAAkB,EAAEC,KAAmB;IAC/D,OAAOD,KAAKjF,EAAE,GAAGkF,MAAMlF,EAAE,GAAG,CAAC,IAAIiF,KAAKjF,EAAE,GAAGkF,MAAMlF,EAAE,GAAG,IAAI;AAC5D;AAEA,SAASjB,eAAevD,IAAiB;IACvC,OAAO;WAAIA;KAAK,CAACqB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAMoI,cAAcpI,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAImI,gBAAgB,GAAG,OAAOA;QAC9B,OAAOrI,EAAEU,cAAc,CAAC4H,OAAO,KAAKrI,EAAES,cAAc,CAAC4H,OAAO;IAC9D;AACF;AAEA,SAASJ,oBAA0DxJ,IAAS;IAC1E,OAAO;WAAIA;KAAK,CAACqB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAMoI,cAAcpI,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAImI,gBAAgB,GAAG,OAAOA;QAC9B,OAAO1G,SAAS3B,EAAEQ,cAAc,EAAEJ,aAAa,CAACuB,SAAS1B,EAAEO,cAAc;IAC3E;AACF;AAEA,SAASV,SAASU,cAAwB,EAAE+H,eAAyB;IACnE,OAAO/H,eAAegI,KAAK,CAAC,CAACC,QAAUF,gBAAgB7D,QAAQ,CAAC+D;AAClE;AAEA,SAASpI,UAAUrC,SAA+B,EAAEuB,KAAa;IAC/D,IAAImJ,YAAYnJ;IAChB,KAAK,MAAMrB,YAAYF,UAAW;QAChC,IAAI0K,cAAc,GAAG;QACrB,MAAM/D,OAAOvC,KAAKI,GAAG,CAACtE,SAASE,cAAc,EAAEsK;QAC/CxK,SAASE,cAAc,IAAIuG;QAC3B+D,aAAa/D;IACf;AACF;AAEA,SAAShD,SAASxD,MAAgB;IAChC,OAAOwK,KAAKC,SAAS,CAACzK;AACxB"}
|