@shipfox/api-runners 12.2.0 → 12.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/.turbo/turbo-build.log +4 -4
  2. package/CHANGELOG.md +34 -0
  3. package/dist/config.d.ts +1 -0
  4. package/dist/config.d.ts.map +1 -1
  5. package/dist/config.js +10 -3
  6. package/dist/config.js.map +1 -1
  7. package/dist/core/demand.d.ts +4 -2
  8. package/dist/core/demand.d.ts.map +1 -1
  9. package/dist/core/demand.js +22 -6
  10. package/dist/core/demand.js.map +1 -1
  11. package/dist/core/entities/reservation.d.ts +1 -0
  12. package/dist/core/entities/reservation.d.ts.map +1 -1
  13. package/dist/core/entities/reservation.js.map +1 -1
  14. package/dist/core/entities/runner-instance.d.ts +3 -0
  15. package/dist/core/entities/runner-instance.d.ts.map +1 -1
  16. package/dist/core/entities/runner-instance.js.map +1 -1
  17. package/dist/core/runner-activation.d.ts +4 -1
  18. package/dist/core/runner-activation.d.ts.map +1 -1
  19. package/dist/core/runner-activation.js +32 -3
  20. package/dist/core/runner-activation.js.map +1 -1
  21. package/dist/core/runner-control-sessions.d.ts +8 -4
  22. package/dist/core/runner-control-sessions.d.ts.map +1 -1
  23. package/dist/core/runner-control-sessions.js +69 -11
  24. package/dist/core/runner-control-sessions.js.map +1 -1
  25. package/dist/core/runner-instances.d.ts +1 -0
  26. package/dist/core/runner-instances.d.ts.map +1 -1
  27. package/dist/core/runner-instances.js +1 -0
  28. package/dist/core/runner-instances.js.map +1 -1
  29. package/dist/db/db.d.ts +68 -0
  30. package/dist/db/db.d.ts.map +1 -1
  31. package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
  32. package/dist/db/ephemeral-registration-tokens.js +1 -1
  33. package/dist/db/ephemeral-registration-tokens.js.map +1 -1
  34. package/dist/db/index.d.ts +1 -1
  35. package/dist/db/index.d.ts.map +1 -1
  36. package/dist/db/index.js +1 -1
  37. package/dist/db/index.js.map +1 -1
  38. package/dist/db/reservation-locks.d.ts +6 -0
  39. package/dist/db/reservation-locks.d.ts.map +1 -0
  40. package/dist/db/reservation-locks.js +15 -0
  41. package/dist/db/reservation-locks.js.map +1 -0
  42. package/dist/db/reservations.d.ts +17 -3
  43. package/dist/db/reservations.d.ts.map +1 -1
  44. package/dist/db/reservations.js +256 -45
  45. package/dist/db/reservations.js.map +1 -1
  46. package/dist/db/runner-assignments.d.ts +17 -0
  47. package/dist/db/runner-assignments.d.ts.map +1 -1
  48. package/dist/db/runner-assignments.js +103 -8
  49. package/dist/db/runner-assignments.js.map +1 -1
  50. package/dist/db/runner-instances.d.ts +5 -2
  51. package/dist/db/runner-instances.d.ts.map +1 -1
  52. package/dist/db/runner-instances.js +134 -25
  53. package/dist/db/runner-instances.js.map +1 -1
  54. package/dist/db/runner-sessions.d.ts.map +1 -1
  55. package/dist/db/runner-sessions.js +22 -15
  56. package/dist/db/runner-sessions.js.map +1 -1
  57. package/dist/db/runner-states.d.ts +2 -0
  58. package/dist/db/runner-states.d.ts.map +1 -0
  59. package/dist/db/runner-states.js +7 -0
  60. package/dist/db/runner-states.js.map +1 -0
  61. package/dist/db/schema/reservations.d.ts +18 -0
  62. package/dist/db/schema/reservations.d.ts.map +1 -1
  63. package/dist/db/schema/reservations.js +7 -1
  64. package/dist/db/schema/reservations.js.map +1 -1
  65. package/dist/db/schema/runner-instances.d.ts +18 -0
  66. package/dist/db/schema/runner-instances.d.ts.map +1 -1
  67. package/dist/db/schema/runner-instances.js +8 -0
  68. package/dist/db/schema/runner-instances.js.map +1 -1
  69. package/dist/metrics/index.d.ts +2 -2
  70. package/dist/metrics/index.d.ts.map +1 -1
  71. package/dist/metrics/index.js +1 -1
  72. package/dist/metrics/index.js.map +1 -1
  73. package/dist/metrics/instance.d.ts +24 -2
  74. package/dist/metrics/instance.d.ts.map +1 -1
  75. package/dist/metrics/instance.js +23 -0
  76. package/dist/metrics/instance.js.map +1 -1
  77. package/dist/metrics/service.d.ts.map +1 -1
  78. package/dist/metrics/service.js +20 -4
  79. package/dist/metrics/service.js.map +1 -1
  80. package/dist/presentation/dto/poll-demand.d.ts.map +1 -1
  81. package/dist/presentation/dto/poll-demand.js +3 -0
  82. package/dist/presentation/dto/poll-demand.js.map +1 -1
  83. package/dist/presentation/dto/runner-instances.d.ts +4 -1
  84. package/dist/presentation/dto/runner-instances.d.ts.map +1 -1
  85. package/dist/presentation/dto/runner-instances.js +4 -1
  86. package/dist/presentation/dto/runner-instances.js.map +1 -1
  87. package/dist/presentation/routes/assign-runner-instances.d.ts.map +1 -1
  88. package/dist/presentation/routes/assign-runner-instances.js +3 -3
  89. package/dist/presentation/routes/assign-runner-instances.js.map +1 -1
  90. package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
  91. package/dist/presentation/routes/poll-demand.js +1 -0
  92. package/dist/presentation/routes/poll-demand.js.map +1 -1
  93. package/dist/presentation/routes/reconcile-runner-instances.d.ts.map +1 -1
  94. package/dist/presentation/routes/reconcile-runner-instances.js +4 -2
  95. package/dist/presentation/routes/reconcile-runner-instances.js.map +1 -1
  96. package/dist/presentation/routes/runner-enrollment.d.ts +4 -1
  97. package/dist/presentation/routes/runner-enrollment.d.ts.map +1 -1
  98. package/dist/presentation/routes/runner-enrollment.js +11 -6
  99. package/dist/presentation/routes/runner-enrollment.js.map +1 -1
  100. package/dist/tsconfig.test.tsbuildinfo +1 -1
  101. package/drizzle/0005_previous_lenny_balinger.sql +5 -0
  102. package/drizzle/meta/0005_snapshot.json +2220 -0
  103. package/drizzle/meta/_journal.json +7 -0
  104. package/package.json +4 -4
  105. package/src/config.test.ts +24 -2
  106. package/src/config.ts +16 -3
  107. package/src/core/demand.test.ts +192 -3
  108. package/src/core/demand.ts +44 -4
  109. package/src/core/entities/reservation.ts +1 -0
  110. package/src/core/entities/runner-instance.ts +4 -0
  111. package/src/core/runner-activation.ts +48 -2
  112. package/src/core/runner-control-sessions.test.ts +230 -4
  113. package/src/core/runner-control-sessions.ts +84 -10
  114. package/src/core/runner-instances.test.ts +19 -0
  115. package/src/core/runner-instances.ts +2 -0
  116. package/src/core/runner-sessions.test.ts +121 -1
  117. package/src/db/ephemeral-registration-tokens.test.ts +20 -1
  118. package/src/db/ephemeral-registration-tokens.ts +1 -0
  119. package/src/db/index.ts +1 -0
  120. package/src/db/reservation-locks.ts +22 -0
  121. package/src/db/reservations.test.ts +1110 -31
  122. package/src/db/reservations.ts +434 -105
  123. package/src/db/runner-assignments.test.ts +105 -1
  124. package/src/db/runner-assignments.ts +178 -9
  125. package/src/db/runner-instances.test.ts +496 -2
  126. package/src/db/runner-instances.ts +247 -56
  127. package/src/db/runner-sessions.ts +26 -15
  128. package/src/db/runner-states.ts +7 -0
  129. package/src/db/schema/reservations.ts +5 -1
  130. package/src/db/schema/runner-instances.ts +9 -0
  131. package/src/metrics/index.ts +10 -1
  132. package/src/metrics/instance.ts +59 -2
  133. package/src/metrics/service.test.ts +106 -0
  134. package/src/metrics/service.ts +24 -4
  135. package/src/presentation/dto/poll-demand.ts +3 -0
  136. package/src/presentation/dto/runner-instances.ts +8 -1
  137. package/src/presentation/routes/assign-runner-instances.ts +6 -2
  138. package/src/presentation/routes/late-runner-enrollment.test.ts +447 -0
  139. package/src/presentation/routes/poll-demand.test.ts +79 -6
  140. package/src/presentation/routes/poll-demand.ts +1 -0
  141. package/src/presentation/routes/reconcile-runner-instances.test.ts +34 -2
  142. package/src/presentation/routes/reconcile-runner-instances.ts +7 -1
  143. package/src/presentation/routes/runner-enrollment.test.ts +316 -3
  144. package/src/presentation/routes/runner-enrollment.ts +8 -5
  145. package/test/factories/reservation.ts +2 -0
  146. package/test/factories/runner-instance.ts +6 -0
  147. package/test/fixtures/late-runner-enrollment.ts +103 -0
  148. package/test/index.ts +5 -0
  149. package/tsconfig.build.tsbuildinfo +1 -1
@@ -1,6 +1,9 @@
1
1
  import { canonicalizeLabels } from '@shipfox/runner-labels';
2
- import { and, arrayContains, asc, eq, exists, gt, inArray, isNotNull, isNull, lt, or, sql } from 'drizzle-orm';
2
+ import { and, arrayContains, asc, eq, exists, gt, inArray, isNotNull, isNull, lt, notExists, notInArray, or, sql } from 'drizzle-orm';
3
+ import { recordProviderRunnerActivationOutcome } from '#metrics/instance.js';
3
4
  import { db } from './db.js';
5
+ import { lockRunnerReservationAdvisoryKeysTx } from './reservation-locks.js';
6
+ import { terminalStates } from './runner-states.js';
4
7
  import { pendingJobExecutions } from './schema/pending-job-executions.js';
5
8
  import { provisionerCapabilitySnapshots } from './schema/provisioner-capability-snapshots.js';
6
9
  import { provisionerTokens } from './schema/provisioner-tokens.js';
@@ -9,13 +12,20 @@ import { runnerActivationTokens } from './schema/runner-activation-tokens.js';
9
12
  import { runnerControlSessions } from './schema/runner-control-sessions.js';
10
13
  import { providerRunners } from './schema/runner-instances.js';
11
14
  export async function pollDemandAndReserve(params) {
12
- return await db().transaction(async (tx)=>{
15
+ const result = await db().transaction(async (tx)=>{
13
16
  return await pollDemandAndReserveTx(tx, params);
14
17
  });
18
+ return {
19
+ stats: result.stats,
20
+ reservations: result.reservations
21
+ };
15
22
  }
16
23
  export async function pollDemandAndReserveTx(tx, params) {
17
24
  await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId}))`);
18
- return await pollDemandAndReserveLockedTx(tx, params);
25
+ return await pollDemandAndReserveLockedTx(tx, {
26
+ ...params,
27
+ scope: 'workspace'
28
+ });
19
29
  }
20
30
  export async function pollInstallationDemandAndReserve(params) {
21
31
  const candidateWorkspaceIds = await listInstallationDemandWorkspaceIds(params.eligibleWorkspaceIds);
@@ -35,32 +45,43 @@ export async function pollInstallationDemandAndReserve(params) {
35
45
  const locked = lockResult.rows[0];
36
46
  if (!locked?.locked) return {
37
47
  stats: [],
38
- reservations: []
48
+ reservations: [],
49
+ newlyReservedUnits: []
39
50
  };
40
51
  return await pollDemandAndReserveLockedTx(tx, {
41
52
  workspaceId,
42
53
  provisionerId: params.provisionerId,
43
54
  maxReservations: remainingMaxReservations,
44
55
  ttlSeconds: params.ttlSeconds,
56
+ ...params.activationGraceSeconds !== undefined ? {
57
+ activationGraceSeconds: params.activationGraceSeconds
58
+ } : {},
45
59
  templates: remainingTemplates.map((template)=>({
46
60
  ...template,
47
61
  availableSlots: template.remainingSlots
48
62
  })),
49
- capabilityWindowSeconds: params.capabilityWindowSeconds
63
+ capabilityWindowSeconds: params.capabilityWindowSeconds,
64
+ scope: 'installation'
50
65
  });
51
66
  });
52
67
  results.push(result);
53
68
  consumeInstallationTemplateSlots(remainingTemplates, result.reservations);
54
69
  params.onReservations?.(result.reservations);
55
- remainingMaxReservations -= result.reservations.reduce((total, reservation)=>total + reservation.count, 0);
70
+ remainingMaxReservations -= result.newlyReservedUnits.reduce((total, reservation)=>total + reservation.count, 0);
56
71
  }
72
+ const newlyReservedCount = results.reduce((total, result)=>total + result.newlyReservedUnits.reduce((subtotal, reservation)=>subtotal + reservation.count, 0), 0);
57
73
  return {
58
74
  stats: results.flatMap((result)=>result.stats),
59
- reservations: results.flatMap((result)=>result.reservations)
75
+ reservations: results.flatMap((result)=>result.reservations),
76
+ ...newlyReservedCount > 0 ? {
77
+ newlyReservedCount
78
+ } : {}
60
79
  };
61
80
  }
62
- function consumeInstallationTemplateSlots(templates, reservations) {
63
- for (const reservation of reservations){
81
+ function consumeInstallationTemplateSlots(templates, launchGrants) {
82
+ // Adopted runners are already included in the running count behind availableSlots.
83
+ // Only units that still need a launch consume advertised template capacity.
84
+ for (const reservation of launchGrants){
64
85
  const satisfyingTemplates = templates.filter((template)=>isSubset(reservation.labels, template.labels)).sort((a, b)=>a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey));
65
86
  drawSlots(satisfyingTemplates, reservation.count);
66
87
  }
@@ -89,10 +110,10 @@ async function pollDemandAndReserveLockedTx(tx, params) {
89
110
  labelKey(row.requiredLabels),
90
111
  row.reserved
91
112
  ]));
92
- const activeProvisionerReservationRows = await tx.select({
93
- requiredLabels: reservations.requiredLabels,
94
- reserved: sql`coalesce(sum(${reservations.count}), 0)::int`
95
- }).from(reservations).where(and(eq(reservations.workspaceId, params.workspaceId), eq(reservations.provisionerId, params.provisionerId), gt(reservations.expiresAt, sql`now()`))).groupBy(reservations.requiredLabels);
113
+ const activeProvisionerReservationRows = await listActiveProvisionerReservationRowsTx(tx, {
114
+ workspaceId: params.workspaceId,
115
+ provisionerId: params.provisionerId
116
+ });
96
117
  const templates = params.templates.map((template)=>({
97
118
  templateKey: template.templateKey,
98
119
  labels: [
@@ -103,6 +124,7 @@ async function pollDemandAndReserveLockedTx(tx, params) {
103
124
  deductProvisionerReservations(templates, activeProvisionerReservationRows);
104
125
  const stats = [];
105
126
  const grants = [];
127
+ const newlyReservedUnits = [];
106
128
  let remainingMaxReservations = params.maxReservations;
107
129
  const workspaceIdField = params.capabilityWindowSeconds === undefined ? {} : {
108
130
  workspaceId: params.workspaceId
@@ -116,34 +138,68 @@ async function pollDemandAndReserveLockedTx(tx, params) {
116
138
  const grant = Math.min(unreserved, capacity, remainingMaxReservations);
117
139
  let reservedAfterGrant = reserved;
118
140
  if (grant > 0 && params.maxReservations > 0) {
119
- const [inserted] = await tx.insert(reservations).values({
120
- workspaceId: params.workspaceId,
141
+ const idleRunners = await listIdleRunnerInstancesTx(tx, {
121
142
  provisionerId: params.provisionerId,
122
143
  requiredLabels: demand.requiredLabels,
123
144
  count: grant,
124
- expiresAt: sql`now() + (${params.ttlSeconds} || ' seconds')::interval`
125
- }).returning({
126
- id: reservations.id,
127
- expiresAt: reservations.expiresAt
145
+ workspaceId: params.workspaceId,
146
+ scope: params.scope
128
147
  });
129
- if (!inserted) throw new Error('Insert returned no rows');
130
148
  remainingMaxReservations -= grant;
131
- drawSlots(satisfyingTemplates, grant);
132
- await bindIdleRunnerInstancesTx(tx, {
133
- provisionerId: params.provisionerId,
134
- reservationId: inserted.id,
135
- requiredLabels: demand.requiredLabels,
136
- count: grant,
137
- workspaceId: params.workspaceId
138
- });
139
- reservedAfterGrant += grant;
140
- grants.push({
141
- reservationId: inserted.id,
142
- ...workspaceIdField,
149
+ const boundCount = idleRunners.length;
150
+ if (boundCount > 0) {
151
+ const [boundReservation] = await tx.insert(reservations).values({
152
+ workspaceId: params.workspaceId,
153
+ provisionerId: params.provisionerId,
154
+ requiredLabels: demand.requiredLabels,
155
+ count: boundCount,
156
+ kind: 'bound',
157
+ expiresAt: sql`now() + (${params.activationGraceSeconds ?? params.ttlSeconds} || ' seconds')::interval`
158
+ }).returning({
159
+ id: reservations.id
160
+ });
161
+ if (!boundReservation) throw new Error('Insert returned no rows');
162
+ await bindIdleRunnerInstancesTx(tx, {
163
+ provisionerId: params.provisionerId,
164
+ reservationId: boundReservation.id,
165
+ workspaceId: params.workspaceId,
166
+ requiredLabels: demand.requiredLabels,
167
+ scope: params.scope,
168
+ idleRunners
169
+ });
170
+ }
171
+ const launchCount = grant - boundCount;
172
+ drawSlots(satisfyingTemplates, launchCount);
173
+ let launchReservation;
174
+ if (launchCount > 0) {
175
+ const [inserted] = await tx.insert(reservations).values({
176
+ workspaceId: params.workspaceId,
177
+ provisionerId: params.provisionerId,
178
+ requiredLabels: demand.requiredLabels,
179
+ count: launchCount,
180
+ kind: 'launch',
181
+ expiresAt: sql`now() + (${params.ttlSeconds} || ' seconds')::interval`
182
+ }).returning({
183
+ id: reservations.id,
184
+ expiresAt: reservations.expiresAt
185
+ });
186
+ if (!inserted) throw new Error('Insert returned no rows');
187
+ launchReservation = inserted;
188
+ }
189
+ newlyReservedUnits.push({
143
190
  labels: demand.requiredLabels,
144
- count: grant,
145
- expiresAt: inserted.expiresAt
191
+ count: grant
146
192
  });
193
+ reservedAfterGrant += grant;
194
+ if (launchReservation) {
195
+ grants.push({
196
+ reservationId: launchReservation.id,
197
+ ...workspaceIdField,
198
+ labels: demand.requiredLabels,
199
+ count: launchCount,
200
+ expiresAt: launchReservation.expiresAt
201
+ });
202
+ }
147
203
  }
148
204
  stats.push({
149
205
  ...workspaceIdField,
@@ -155,22 +211,160 @@ async function pollDemandAndReserveLockedTx(tx, params) {
155
211
  }
156
212
  return {
157
213
  stats,
158
- reservations: grants
214
+ reservations: grants,
215
+ newlyReservedUnits
159
216
  };
160
217
  }
161
- async function bindIdleRunnerInstancesTx(tx, params) {
162
- const idleRunners = await tx.select({
163
- id: providerRunners.id
164
- }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), isNull(providerRunners.workspaceId), isNull(providerRunners.reservationId), isNull(providerRunners.intendedReservationId), isNull(providerRunners.runnerSessionId), isNotNull(providerRunners.providerRunnerId), eq(providerRunners.state, 'running'), arrayContains(providerRunners.labels, params.requiredLabels), exists(tx.select({
218
+ async function listActiveProvisionerReservationRowsTx(tx, params) {
219
+ const candidateRows = await tx.select({
220
+ id: reservations.id
221
+ }).from(reservations).where(and(eq(reservations.workspaceId, params.workspaceId), eq(reservations.provisionerId, params.provisionerId), gt(reservations.expiresAt, sql`now()`)));
222
+ const candidateIds = candidateRows.map((row)=>row.id);
223
+ if (candidateIds.length === 0) return [];
224
+ // Assignment and provider-report transactions use this key before changing runner
225
+ // links. Lock it before counting so an in-flight assignment cannot expose its unit twice.
226
+ await lockRunnerReservationAdvisoryKeysTx(tx, {
227
+ provisionerId: params.provisionerId,
228
+ reservationIds: candidateIds
229
+ });
230
+ const activeRows = await tx.select({
231
+ id: reservations.id,
232
+ requiredLabels: reservations.requiredLabels,
233
+ count: reservations.count
234
+ }).from(reservations).where(and(eq(reservations.workspaceId, params.workspaceId), eq(reservations.provisionerId, params.provisionerId), gt(reservations.expiresAt, sql`now()`))).orderBy(asc(reservations.id)).for('update');
235
+ if (activeRows.length === 0) return [];
236
+ const activeIds = activeRows.map((row)=>row.id);
237
+ const activeIdSet = new Set(activeIds);
238
+ const usedRunnerRows = await tx.select({
239
+ id: providerRunners.id,
240
+ reservationId: providerRunners.reservationId,
241
+ intendedReservationId: providerRunners.intendedReservationId,
242
+ state: providerRunners.state
243
+ }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), isNull(providerRunners.reservationReleasedAt), or(inArray(providerRunners.reservationId, activeIds), and(inArray(providerRunners.intendedReservationId, activeIds), notInArray(providerRunners.state, [
244
+ ...terminalStates
245
+ ])))));
246
+ const usedByReservation = new Map();
247
+ for (const runner of usedRunnerRows){
248
+ if (runner.reservationId && activeIdSet.has(runner.reservationId)) {
249
+ addUsedRunner(usedByReservation, runner.reservationId, runner.id);
250
+ }
251
+ if (runner.intendedReservationId && activeIdSet.has(runner.intendedReservationId) && !terminalStates.includes(runner.state)) {
252
+ addUsedRunner(usedByReservation, runner.intendedReservationId, runner.id);
253
+ }
254
+ }
255
+ return activeRows.flatMap((reservation)=>{
256
+ const used = usedByReservation.get(reservation.id)?.size ?? 0;
257
+ const pending = Math.max(0, reservation.count - used);
258
+ return pending > 0 ? [
259
+ {
260
+ requiredLabels: reservation.requiredLabels,
261
+ reserved: pending
262
+ }
263
+ ] : [];
264
+ });
265
+ }
266
+ function addUsedRunner(usedByReservation, reservationId, runnerId) {
267
+ const runnerIds = usedByReservation.get(reservationId);
268
+ if (runnerIds) runnerIds.add(runnerId);
269
+ else usedByReservation.set(reservationId, new Set([
270
+ runnerId
271
+ ]));
272
+ }
273
+ // An expired or missing reservation is stale. A live reservation protects a runner that is
274
+ // still booting or waiting for its activation grace period.
275
+ function canBindRunner(tx, params) {
276
+ return and(or(and(isNull(providerRunners.workspaceId), isNull(providerRunners.reservationId)), and(isNotNull(providerRunners.reservationId), params.scope === 'installation' ? undefined : or(isNull(providerRunners.workspaceId), eq(providerRunners.workspaceId, params.workspaceId)), notExists(tx.select({
277
+ id: reservations.id
278
+ }).from(reservations).where(and(eq(reservations.id, providerRunners.reservationId), gt(reservations.expiresAt, sql`now()`)))))), or(isNull(providerRunners.intendedReservationId), notExists(tx.select({
279
+ id: reservations.id
280
+ }).from(reservations).where(and(eq(reservations.id, providerRunners.intendedReservationId), gt(reservations.expiresAt, sql`now()`))))), isNull(providerRunners.reservationReleasedAt), isNull(providerRunners.runnerSessionId));
281
+ }
282
+ function isBindableRunner(tx, params) {
283
+ return and(eq(providerRunners.provisionerId, params.provisionerId), canBindRunner(tx, params), isNotNull(providerRunners.providerRunnerId), eq(providerRunners.state, 'running'), arrayContains(providerRunners.labels, params.requiredLabels), exists(tx.select({
165
284
  id: runnerControlSessions.id
166
- }).from(runnerControlSessions).where(and(eq(runnerControlSessions.runnerInstanceId, providerRunners.id), eq(runnerControlSessions.provisionerId, params.provisionerId), isNull(runnerControlSessions.closedAt), gt(runnerControlSessions.expiresAt, sql`now()`)))))).orderBy(asc(providerRunners.createdAt), asc(providerRunners.id)).limit(params.count).for('update');
167
- if (idleRunners.length === 0) return;
168
- await tx.update(providerRunners).set({
285
+ }).from(runnerControlSessions).where(and(eq(runnerControlSessions.runnerInstanceId, providerRunners.id), eq(runnerControlSessions.provisionerId, params.provisionerId), isNull(runnerControlSessions.closedAt), gt(runnerControlSessions.expiresAt, sql`now()`)))));
286
+ }
287
+ async function listIdleRunnerInstancesTx(tx, params) {
288
+ const lockedRunners = await selectIdleRunnerInstancesTx(tx, params);
289
+ if (lockedRunners.length === 0) return lockedRunners;
290
+ // A SELECT ... FOR UPDATE can wake with a fresh target row but the original
291
+ // statement snapshot for reservation subqueries. Re-check the full predicate
292
+ // in a new statement while retaining the row locks from the first query.
293
+ return await selectIdleRunnerInstancesTx(tx, {
294
+ ...params,
295
+ runnerIds: lockedRunners.map((runner)=>runner.id),
296
+ count: lockedRunners.length
297
+ });
298
+ }
299
+ async function selectIdleRunnerInstancesTx(tx, params) {
300
+ if (params.runnerIds) {
301
+ // The first pass retains the row locks. Recheck the full predicate in a fresh statement so
302
+ // reservation subqueries see the state that was current when the lock was acquired.
303
+ return await tx.select({
304
+ id: providerRunners.id,
305
+ launchKind: providerRunners.launchKind
306
+ }).from(providerRunners).where(and(inArray(providerRunners.id, params.runnerIds), isBindableRunner(tx, params))).orderBy(asc(providerRunners.createdAt), asc(providerRunners.id)).limit(params.count);
307
+ }
308
+ // Select the oldest candidates first, but leave row locking to a second pass. The cleanup
309
+ // path locks runner rows by id, so the locking pass must use that same order. A skipped row
310
+ // rolls back only this nested savepoint and retries the bounded candidate scan, allowing the
311
+ // next-oldest eligible runner to refill the grant without retaining a partial lock set.
312
+ const retrySelection = Symbol('retry bindable runner selection');
313
+ const idleRunners = await (async ()=>{
314
+ while(true){
315
+ try {
316
+ return await tx.transaction(async (lockTx)=>{
317
+ const candidateRunners = await lockTx.select({
318
+ id: providerRunners.id,
319
+ launchKind: providerRunners.launchKind
320
+ }).from(providerRunners).where(isBindableRunner(lockTx, params)).orderBy(asc(providerRunners.createdAt), asc(providerRunners.id)).limit(params.count);
321
+ if (candidateRunners.length === 0) return candidateRunners;
322
+ // PostgreSQL can acquire row locks before sorting a multi-row FOR UPDATE result. Lock
323
+ // each selected candidate individually so both polling and cleanup use id order.
324
+ const lockedRunners = [];
325
+ for (const candidate of [
326
+ ...candidateRunners
327
+ ].sort(compareRunnerIds)){
328
+ const [runner] = await lockTx.select({
329
+ id: providerRunners.id,
330
+ launchKind: providerRunners.launchKind
331
+ }).from(providerRunners).where(and(eq(providerRunners.id, candidate.id), isBindableRunner(lockTx, params))).for('update');
332
+ if (runner) lockedRunners.push(runner);
333
+ }
334
+ if (lockedRunners.length !== candidateRunners.length) throw retrySelection;
335
+ return lockedRunners;
336
+ });
337
+ } catch (error) {
338
+ if (error !== retrySelection) throw error;
339
+ }
340
+ }
341
+ })();
342
+ return idleRunners;
343
+ }
344
+ async function bindIdleRunnerInstancesTx(tx, params) {
345
+ const idleRunnerIds = params.idleRunners.map((runner)=>runner.id);
346
+ await tx.update(runnerActivationTokens).set({
347
+ revokedAt: sql`now()`
348
+ }).where(and(inArray(runnerActivationTokens.runnerInstanceId, idleRunnerIds), isNull(runnerActivationTokens.consumedAt), isNull(runnerActivationTokens.revokedAt)));
349
+ // listIdleRunnerInstancesTx re-checks the full eligibility predicate in a fresh statement
350
+ // after acquiring row locks. Those locks prevent the validated set from changing before this
351
+ // update runs.
352
+ const boundRunners = await tx.update(providerRunners).set({
169
353
  workspaceId: params.workspaceId,
170
354
  reservationId: params.reservationId,
355
+ intendedReservationId: null,
171
356
  assignedAt: sql`now()`,
172
357
  updatedAt: sql`now()`
173
- }).where(and(inArray(providerRunners.id, idleRunners.map((runner)=>runner.id)), isNull(providerRunners.workspaceId), isNull(providerRunners.reservationId), isNull(providerRunners.runnerSessionId)));
358
+ }).where(and(inArray(providerRunners.id, idleRunnerIds), isBindableRunner(tx, params))).returning({
359
+ id: providerRunners.id,
360
+ launchKind: providerRunners.launchKind
361
+ });
362
+ if (boundRunners.length !== params.idleRunners.length) throw new Error('Locked idle runner set changed before binding');
363
+ const reboundCount = boundRunners.filter((runner)=>runner.launchKind === 'demand').length;
364
+ if (reboundCount > 0) recordProviderRunnerActivationOutcome({
365
+ outcome: 'rebound',
366
+ count: reboundCount
367
+ });
174
368
  }
175
369
  async function listInstallationDemandWorkspaceIds(eligibleWorkspaceIds) {
176
370
  if (eligibleWorkspaceIds.size === 0) return [];
@@ -212,13 +406,27 @@ export async function deleteReservationsByIds(ids) {
212
406
  }
213
407
  async function deleteReservationsWithCleanupTx(tx, ids, params = {}) {
214
408
  if (ids.length === 0) return 0;
215
- const affectedRunners = await tx.select({
409
+ const isAffectedRunner = ()=>or(and(inArray(providerRunners.reservationId, ids), isNull(providerRunners.runnerSessionId), isNull(providerRunners.reservationReleasedAt)), inArray(providerRunners.intendedReservationId, ids));
410
+ const candidateRunners = await tx.select({
216
411
  id: providerRunners.id,
217
412
  reservationId: providerRunners.reservationId,
218
413
  intendedReservationId: providerRunners.intendedReservationId,
219
414
  runnerSessionId: providerRunners.runnerSessionId,
220
415
  reservationReleasedAt: providerRunners.reservationReleasedAt
221
- }).from(providerRunners).where(or(and(inArray(providerRunners.reservationId, ids), isNull(providerRunners.runnerSessionId), isNull(providerRunners.reservationReleasedAt)), inArray(providerRunners.intendedReservationId, ids))).orderBy(asc(providerRunners.id)).for('update');
416
+ }).from(providerRunners).where(isAffectedRunner()).orderBy(asc(providerRunners.id));
417
+ const affectedRunners = [];
418
+ for (const candidate of [
419
+ ...candidateRunners
420
+ ].sort(compareRunnerIds)){
421
+ const [runner] = await tx.select({
422
+ id: providerRunners.id,
423
+ reservationId: providerRunners.reservationId,
424
+ intendedReservationId: providerRunners.intendedReservationId,
425
+ runnerSessionId: providerRunners.runnerSessionId,
426
+ reservationReleasedAt: providerRunners.reservationReleasedAt
427
+ }).from(providerRunners).where(and(eq(providerRunners.id, candidate.id), isAffectedRunner())).for('update');
428
+ if (runner) affectedRunners.push(runner);
429
+ }
222
430
  const reservationRows = await tx.select({
223
431
  id: reservations.id
224
432
  }).from(reservations).where(and(inArray(reservations.id, ids), params.expiredOnly ? lt(reservations.expiresAt, sql`now()`) : undefined)).for('update');
@@ -274,6 +482,9 @@ function deductProvisionerReservations(templates, activeReservations) {
274
482
  drawSlots(satisfyingTemplates, reservation.reserved);
275
483
  }
276
484
  }
485
+ function compareRunnerIds(left, right) {
486
+ return left.id < right.id ? -1 : left.id > right.id ? 1 : 0;
487
+ }
277
488
  function sortDemandRows(rows) {
278
489
  return [
279
490
  ...rows
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/db/reservations.ts"],"sourcesContent":["import {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n and,\n arrayContains,\n asc,\n eq,\n exists,\n gt,\n inArray,\n isNotNull,\n isNull,\n lt,\n or,\n sql,\n} from 'drizzle-orm';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {provisionerCapabilitySnapshots} from './schema/provisioner-capability-snapshots.js';\nimport {provisionerTokens} from './schema/provisioner-tokens.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerActivationTokens} from './schema/runner-activation-tokens.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {providerRunners} from './schema/runner-instances.js';\n\nexport interface ReservationTemplate {\n templateKey: string;\n labels: string[];\n availableSlots: number;\n starting: number;\n running: number;\n}\n\nexport interface DemandStat {\n workspaceId?: string;\n labels: string[];\n queued: number;\n reserved: number;\n oldestQueuedAt: Date;\n}\n\nexport interface ReservationGrant {\n reservationId: string;\n workspaceId?: string;\n labels: string[];\n count: number;\n expiresAt: Date;\n}\n\nexport interface PollDemandAndReserveParams {\n workspaceId: string;\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds?: number;\n}\n\nexport interface InstallationPollDemandAndReserveParams {\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds: number;\n eligibleWorkspaceIds: ReadonlySet<string>;\n signal?: AbortSignal;\n onReservations?: (reservations: ReservationGrant[]) => void;\n}\n\ninterface NormalizedTemplate {\n templateKey: string;\n labels: string[];\n remainingSlots: number;\n}\n\ninterface DemandRow {\n requiredLabels: string[];\n queued: number;\n oldestQueuedAt: Date;\n}\n\nexport async function pollDemandAndReserve(\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n return await db().transaction(async (tx) => {\n return await pollDemandAndReserveTx(tx, params);\n });\n}\n\nexport async function pollDemandAndReserveTx(\n tx: Tx,\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId}))`);\n return await pollDemandAndReserveLockedTx(tx, params);\n}\n\nexport async function pollInstallationDemandAndReserve(\n params: InstallationPollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n const candidateWorkspaceIds = await listInstallationDemandWorkspaceIds(\n params.eligibleWorkspaceIds,\n );\n const results: Array<{stats: DemandStat[]; reservations: ReservationGrant[]}> = [];\n let remainingMaxReservations = params.maxReservations;\n const remainingTemplates = params.templates.map((template) => ({\n ...template,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n for (const workspaceId of candidateWorkspaceIds) {\n if (params.signal?.aborted || remainingMaxReservations === 0) break;\n const result = await db().transaction(async (tx) => {\n const lockResult = await tx.execute<{locked: boolean}>(\n sql`select pg_try_advisory_xact_lock(hashtext(${workspaceId})) as locked`,\n );\n const locked = lockResult.rows[0];\n if (!locked?.locked) return {stats: [], reservations: []};\n return await pollDemandAndReserveLockedTx(tx, {\n workspaceId,\n provisionerId: params.provisionerId,\n maxReservations: remainingMaxReservations,\n ttlSeconds: params.ttlSeconds,\n templates: remainingTemplates.map((template) => ({\n ...template,\n availableSlots: template.remainingSlots,\n })),\n capabilityWindowSeconds: params.capabilityWindowSeconds,\n });\n });\n results.push(result);\n consumeInstallationTemplateSlots(remainingTemplates, result.reservations);\n params.onReservations?.(result.reservations);\n remainingMaxReservations -= result.reservations.reduce(\n (total, reservation) => total + reservation.count,\n 0,\n );\n }\n return {\n stats: results.flatMap((result) => result.stats),\n reservations: results.flatMap((result) => result.reservations),\n };\n}\n\nfunction consumeInstallationTemplateSlots(\n templates: NormalizedTemplate[],\n reservations: ReservationGrant[],\n): void {\n for (const reservation of reservations) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.labels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.count);\n }\n}\n\nasync function pollDemandAndReserveLockedTx(\n tx: Tx,\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n let demandRows = (\n await tx\n .select({\n requiredLabels: pendingJobExecutions.requiredLabels,\n queued: sql<number>`count(*)::int`,\n oldestQueuedAt: sql<Date | string>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(eq(pendingJobExecutions.workspaceId, params.workspaceId))\n .groupBy(pendingJobExecutions.requiredLabels)\n ).map((row) => ({\n ...row,\n oldestQueuedAt: new Date(row.oldestQueuedAt),\n }));\n if (params.capabilityWindowSeconds !== undefined) {\n const capabilityLabels = await listActiveWorkspaceCapabilityLabelsTx(tx, {\n workspaceId: params.workspaceId,\n windowSeconds: params.capabilityWindowSeconds,\n });\n demandRows = demandRows.filter(\n (demand) => !capabilityLabels.some((labels) => isSubset(demand.requiredLabels, labels)),\n );\n }\n\n const activeReservationRows = await tx\n .select({\n requiredLabels: reservations.requiredLabels,\n reserved: sql<number>`coalesce(sum(${reservations.count}), 0)::int`,\n })\n .from(reservations)\n .where(\n and(eq(reservations.workspaceId, params.workspaceId), gt(reservations.expiresAt, sql`now()`)),\n )\n .groupBy(reservations.requiredLabels);\n\n const reservedByLabels = new Map(\n activeReservationRows.map((row) => [labelKey(row.requiredLabels), row.reserved]),\n );\n const activeProvisionerReservationRows = await tx\n .select({\n requiredLabels: reservations.requiredLabels,\n reserved: sql<number>`coalesce(sum(${reservations.count}), 0)::int`,\n })\n .from(reservations)\n .where(\n and(\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .groupBy(reservations.requiredLabels);\n\n const templates = params.templates.map((template) => ({\n templateKey: template.templateKey,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n deductProvisionerReservations(templates, activeProvisionerReservationRows);\n const stats: DemandStat[] = [];\n const grants: ReservationGrant[] = [];\n let remainingMaxReservations = params.maxReservations;\n const workspaceIdField =\n params.capabilityWindowSeconds === undefined ? {} : {workspaceId: params.workspaceId};\n\n for (const demand of sortDemandRows(demandRows)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(demand.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n\n if (satisfyingTemplates.length === 0) continue;\n\n const reserved = reservedByLabels.get(labelKey(demand.requiredLabels)) ?? 0;\n const unreserved = Math.max(0, demand.queued - reserved);\n const capacity = satisfyingTemplates.reduce(\n (total, template) => total + template.remainingSlots,\n 0,\n );\n const grant = Math.min(unreserved, capacity, remainingMaxReservations);\n let reservedAfterGrant = reserved;\n\n if (grant > 0 && params.maxReservations > 0) {\n const [inserted] = await tx\n .insert(reservations)\n .values({\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: grant,\n expiresAt: sql`now() + (${params.ttlSeconds} || ' seconds')::interval`,\n })\n .returning({id: reservations.id, expiresAt: reservations.expiresAt});\n\n if (!inserted) throw new Error('Insert returned no rows');\n\n remainingMaxReservations -= grant;\n drawSlots(satisfyingTemplates, grant);\n await bindIdleRunnerInstancesTx(tx, {\n provisionerId: params.provisionerId,\n reservationId: inserted.id,\n requiredLabels: demand.requiredLabels,\n count: grant,\n workspaceId: params.workspaceId,\n });\n reservedAfterGrant += grant;\n grants.push({\n reservationId: inserted.id,\n ...workspaceIdField,\n labels: demand.requiredLabels,\n count: grant,\n expiresAt: inserted.expiresAt,\n });\n }\n\n stats.push({\n ...workspaceIdField,\n labels: demand.requiredLabels,\n queued: demand.queued,\n reserved: reservedAfterGrant,\n oldestQueuedAt: demand.oldestQueuedAt,\n });\n }\n\n return {stats, reservations: grants};\n}\n\nasync function bindIdleRunnerInstancesTx(\n tx: Tx,\n params: {\n provisionerId: string;\n reservationId: string;\n workspaceId: string;\n requiredLabels: string[];\n count: number;\n },\n): Promise<void> {\n const idleRunners = await tx\n .select({id: providerRunners.id})\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n isNull(providerRunners.workspaceId),\n isNull(providerRunners.reservationId),\n isNull(providerRunners.intendedReservationId),\n isNull(providerRunners.runnerSessionId),\n isNotNull(providerRunners.providerRunnerId),\n eq(providerRunners.state, 'running'),\n arrayContains(providerRunners.labels, params.requiredLabels),\n exists(\n tx\n .select({id: runnerControlSessions.id})\n .from(runnerControlSessions)\n .where(\n and(\n eq(runnerControlSessions.runnerInstanceId, providerRunners.id),\n eq(runnerControlSessions.provisionerId, params.provisionerId),\n isNull(runnerControlSessions.closedAt),\n gt(runnerControlSessions.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n )\n .orderBy(asc(providerRunners.createdAt), asc(providerRunners.id))\n .limit(params.count)\n .for('update');\n\n if (idleRunners.length === 0) return;\n\n await tx\n .update(providerRunners)\n .set({\n workspaceId: params.workspaceId,\n reservationId: params.reservationId,\n assignedAt: sql`now()`,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n inArray(\n providerRunners.id,\n idleRunners.map((runner) => runner.id),\n ),\n isNull(providerRunners.workspaceId),\n isNull(providerRunners.reservationId),\n isNull(providerRunners.runnerSessionId),\n ),\n );\n}\n\nasync function listInstallationDemandWorkspaceIds(eligibleWorkspaceIds: ReadonlySet<string>) {\n if (eligibleWorkspaceIds.size === 0) return [];\n const rows = await db()\n .select({\n workspaceId: pendingJobExecutions.workspaceId,\n oldestQueuedAt: sql<Date>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.workspaceId, [...eligibleWorkspaceIds]))\n .groupBy(pendingJobExecutions.workspaceId)\n .orderBy(\n asc(sql`min(${pendingJobExecutions.createdAt})`),\n asc(pendingJobExecutions.workspaceId),\n );\n return rows.map((row) => row.workspaceId);\n}\n\nexport async function listQueuedDemandWorkspaceIds(): Promise<string[]> {\n const rows = await db()\n .select({workspaceId: pendingJobExecutions.workspaceId})\n .from(pendingJobExecutions)\n .groupBy(pendingJobExecutions.workspaceId);\n return rows.map((row) => row.workspaceId);\n}\n\nasync function listActiveWorkspaceCapabilityLabelsTx(\n tx: Tx,\n params: {workspaceId: string; windowSeconds: number},\n): Promise<string[][]> {\n const rows = await tx\n .select({labels: provisionerCapabilitySnapshots.labels})\n .from(provisionerCapabilitySnapshots)\n .innerJoin(\n provisionerTokens,\n eq(provisionerTokens.id, provisionerCapabilitySnapshots.provisionerId),\n )\n .where(\n and(\n eq(provisionerCapabilitySnapshots.workspaceId, params.workspaceId),\n eq(provisionerTokens.workspaceId, params.workspaceId),\n eq(provisionerTokens.scope, 'workspace'),\n sql`${provisionerCapabilitySnapshots.advertisedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n sql`${provisionerTokens.revokedAt} is null`,\n sql`(${provisionerTokens.expiresAt} is null or ${provisionerTokens.expiresAt} > now())`,\n ),\n );\n return rows.map((row) => row.labels);\n}\n\nexport async function deleteExpiredReservations(params?: {limit?: number}): Promise<number> {\n return await db().transaction(async (tx) => {\n const expiredRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(lt(reservations.expiresAt, sql`now()`))\n .orderBy(asc(reservations.expiresAt))\n .limit(params?.limit ?? 1000);\n return await deleteReservationsWithCleanupTx(\n tx,\n expiredRows.map((reservation) => reservation.id),\n {expiredOnly: true},\n );\n });\n}\n\nexport async function deleteReservationsByIds(ids: string[]): Promise<number> {\n if (ids.length === 0) return 0;\n\n return await db().transaction(async (tx) => {\n return await deleteReservationsWithCleanupTx(tx, ids);\n });\n}\n\nasync function deleteReservationsWithCleanupTx(\n tx: Tx,\n ids: string[],\n params: {expiredOnly?: boolean} = {},\n): Promise<number> {\n if (ids.length === 0) return 0;\n\n const affectedRunners = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n runnerSessionId: providerRunners.runnerSessionId,\n reservationReleasedAt: providerRunners.reservationReleasedAt,\n })\n .from(providerRunners)\n .where(\n or(\n and(\n inArray(providerRunners.reservationId, ids),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n inArray(providerRunners.intendedReservationId, ids),\n ),\n )\n .orderBy(asc(providerRunners.id))\n .for('update');\n const reservationRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n inArray(reservations.id, ids),\n params.expiredOnly ? lt(reservations.expiresAt, sql`now()`) : undefined,\n ),\n )\n .for('update');\n const reservationIds = reservationRows.map((reservation) => reservation.id);\n\n if (reservationIds.length === 0) return 0;\n\n const assignedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.reservationId &&\n reservationIds.includes(runner.reservationId) &&\n !runner.runnerSessionId &&\n !runner.reservationReleasedAt,\n )\n .map((runner) => runner.id);\n const intendedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.intendedReservationId && reservationIds.includes(runner.intendedReservationId),\n )\n .map((runner) => runner.id);\n if (assignedRunnerIds.length > 0) {\n await tx\n .update(runnerActivationTokens)\n .set({revokedAt: sql`now()`})\n .where(\n and(\n inArray(runnerActivationTokens.runnerInstanceId, assignedRunnerIds),\n isNull(runnerActivationTokens.consumedAt),\n isNull(runnerActivationTokens.revokedAt),\n ),\n );\n await tx\n .update(providerRunners)\n .set({\n workspaceId: null,\n reservationId: null,\n assignedAt: null,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n inArray(providerRunners.id, assignedRunnerIds),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n );\n }\n if (intendedRunnerIds.length > 0) {\n await tx\n .update(providerRunners)\n .set({intendedReservationId: null, updatedAt: sql`now()`})\n .where(\n and(\n inArray(providerRunners.id, intendedRunnerIds),\n inArray(providerRunners.intendedReservationId, reservationIds),\n ),\n );\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(inArray(reservations.id, reservationIds))\n .returning({id: reservations.id});\n\n return deleted.length;\n}\n\nexport async function releaseReservationUnits(\n tx: Tx,\n params: {\n workspaceId: string;\n provisionerId: string;\n releases: Array<{reservationId: string; count: number}>;\n },\n): Promise<number> {\n let released = 0;\n\n for (const release of params.releases) {\n if (release.count <= 0) continue;\n\n const decremented = await tx\n .update(reservations)\n .set({count: sql`${reservations.count} - ${release.count}`})\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.count, release.count),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({id: reservations.id});\n\n if (decremented.length > 0) {\n released += release.count;\n continue;\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n sql`${reservations.count} <= ${release.count}`,\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({count: reservations.count});\n\n released += deleted.reduce((total, row) => total + row.count, 0);\n }\n\n return released;\n}\n\nfunction deductProvisionerReservations(\n templates: NormalizedTemplate[],\n activeReservations: {requiredLabels: string[]; reserved: number}[],\n): void {\n for (const reservation of sortReservationRows(activeReservations)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.reserved);\n }\n}\n\nfunction sortDemandRows(rows: DemandRow[]): DemandRow[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return a.oldestQueuedAt.getTime() - b.oldestQueuedAt.getTime();\n });\n}\n\nfunction sortReservationRows<T extends {requiredLabels: string[]}>(rows: T[]): T[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return labelKey(a.requiredLabels).localeCompare(labelKey(b.requiredLabels));\n });\n}\n\nfunction isSubset(requiredLabels: string[], availableLabels: string[]): boolean {\n return requiredLabels.every((label) => availableLabels.includes(label));\n}\n\nfunction drawSlots(templates: NormalizedTemplate[], count: number): void {\n let remaining = count;\n for (const template of templates) {\n if (remaining === 0) return;\n const used = Math.min(template.remainingSlots, remaining);\n template.remainingSlots -= used;\n remaining -= used;\n }\n}\n\nfunction labelKey(labels: string[]): string {\n return JSON.stringify(labels);\n}\n"],"names":["canonicalizeLabels","and","arrayContains","asc","eq","exists","gt","inArray","isNotNull","isNull","lt","or","sql","db","pendingJobExecutions","provisionerCapabilitySnapshots","provisionerTokens","reservations","runnerActivationTokens","runnerControlSessions","providerRunners","pollDemandAndReserve","params","transaction","tx","pollDemandAndReserveTx","execute","workspaceId","pollDemandAndReserveLockedTx","pollInstallationDemandAndReserve","candidateWorkspaceIds","listInstallationDemandWorkspaceIds","eligibleWorkspaceIds","results","remainingMaxReservations","maxReservations","remainingTemplates","templates","map","template","labels","remainingSlots","availableSlots","signal","aborted","result","lockResult","locked","rows","stats","provisionerId","ttlSeconds","capabilityWindowSeconds","push","consumeInstallationTemplateSlots","onReservations","reduce","total","reservation","count","flatMap","satisfyingTemplates","filter","isSubset","sort","a","b","length","templateKey","localeCompare","drawSlots","demandRows","select","requiredLabels","queued","oldestQueuedAt","createdAt","from","where","groupBy","row","Date","undefined","capabilityLabels","listActiveWorkspaceCapabilityLabelsTx","windowSeconds","demand","some","activeReservationRows","reserved","expiresAt","reservedByLabels","Map","labelKey","activeProvisionerReservationRows","deductProvisionerReservations","grants","workspaceIdField","sortDemandRows","get","unreserved","Math","max","capacity","grant","min","reservedAfterGrant","inserted","insert","values","returning","id","Error","bindIdleRunnerInstancesTx","reservationId","idleRunners","intendedReservationId","runnerSessionId","providerRunnerId","state","runnerInstanceId","closedAt","orderBy","limit","for","update","set","assignedAt","updatedAt","runner","size","listQueuedDemandWorkspaceIds","innerJoin","scope","advertisedAt","revokedAt","deleteExpiredReservations","expiredRows","deleteReservationsWithCleanupTx","expiredOnly","deleteReservationsByIds","ids","affectedRunners","reservationReleasedAt","reservationRows","reservationIds","assignedRunnerIds","includes","intendedRunnerIds","consumedAt","deleted","delete","releaseReservationUnits","released","release","releases","decremented","activeReservations","sortReservationRows","specificity","getTime","availableLabels","every","label","remaining","used","JSON","stringify"],"mappings":"AAAA,SAAQA,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,aAAa,EACbC,GAAG,EACHC,EAAE,EACFC,MAAM,EACNC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,EAAE,EACFC,GAAG,QACE,cAAc;AAErB,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,8BAA8B,QAAO,+CAA+C;AAC5F,SAAQC,iBAAiB,QAAO,iCAAiC;AACjE,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,sBAAsB,QAAO,uCAAuC;AAC5E,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAAQC,eAAe,QAAO,+BAA+B;AA0D7D,OAAO,eAAeC,qBACpBC,MAAkC;IAElC,OAAO,MAAMT,KAAKU,WAAW,CAAC,OAAOC;QACnC,OAAO,MAAMC,uBAAuBD,IAAIF;IAC1C;AACF;AAEA,OAAO,eAAeG,uBACpBD,EAAM,EACNF,MAAkC;IAElC,MAAME,GAAGE,OAAO,CAACd,GAAG,CAAC,sCAAsC,EAAEU,OAAOK,WAAW,CAAC,EAAE,CAAC;IACnF,OAAO,MAAMC,6BAA6BJ,IAAIF;AAChD;AAEA,OAAO,eAAeO,iCACpBP,MAA8C;IAE9C,MAAMQ,wBAAwB,MAAMC,mCAClCT,OAAOU,oBAAoB;IAE7B,MAAMC,UAA0E,EAAE;IAClF,IAAIC,2BAA2BZ,OAAOa,eAAe;IACrD,MAAMC,qBAAqBd,OAAOe,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YAC7D,GAAGA,QAAQ;YACXC,QAAQ;mBAAIxC,mBAAmBuC,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACA,KAAK,MAAMf,eAAeG,sBAAuB;QAC/C,IAAIR,OAAOqB,MAAM,EAAEC,WAAWV,6BAA6B,GAAG;QAC9D,MAAMW,SAAS,MAAMhC,KAAKU,WAAW,CAAC,OAAOC;YAC3C,MAAMsB,aAAa,MAAMtB,GAAGE,OAAO,CACjCd,GAAG,CAAC,0CAA0C,EAAEe,YAAY,YAAY,CAAC;YAE3E,MAAMoB,SAASD,WAAWE,IAAI,CAAC,EAAE;YACjC,IAAI,CAACD,QAAQA,QAAQ,OAAO;gBAACE,OAAO,EAAE;gBAAEhC,cAAc,EAAE;YAAA;YACxD,OAAO,MAAMW,6BAA6BJ,IAAI;gBAC5CG;gBACAuB,eAAe5B,OAAO4B,aAAa;gBACnCf,iBAAiBD;gBACjBiB,YAAY7B,OAAO6B,UAAU;gBAC7Bd,WAAWD,mBAAmBE,GAAG,CAAC,CAACC,WAAc,CAAA;wBAC/C,GAAGA,QAAQ;wBACXG,gBAAgBH,SAASE,cAAc;oBACzC,CAAA;gBACAW,yBAAyB9B,OAAO8B,uBAAuB;YACzD;QACF;QACAnB,QAAQoB,IAAI,CAACR;QACbS,iCAAiClB,oBAAoBS,OAAO5B,YAAY;QACxEK,OAAOiC,cAAc,GAAGV,OAAO5B,YAAY;QAC3CiB,4BAA4BW,OAAO5B,YAAY,CAACuC,MAAM,CACpD,CAACC,OAAOC,cAAgBD,QAAQC,YAAYC,KAAK,EACjD;IAEJ;IACA,OAAO;QACLV,OAAOhB,QAAQ2B,OAAO,CAAC,CAACf,SAAWA,OAAOI,KAAK;QAC/ChC,cAAcgB,QAAQ2B,OAAO,CAAC,CAACf,SAAWA,OAAO5B,YAAY;IAC/D;AACF;AAEA,SAASqC,iCACPjB,SAA+B,EAC/BpB,YAAgC;IAEhC,KAAK,MAAMyC,eAAezC,aAAc;QACtC,MAAM4C,sBAAsBxB,UACzByB,MAAM,CAAC,CAACvB,WAAawB,SAASL,YAAYlB,MAAM,EAAED,SAASC,MAAM,GACjEwB,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAEzB,MAAM,CAAC2B,MAAM,GAAGD,EAAE1B,MAAM,CAAC2B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBH,YAAYC,KAAK;IAClD;AACF;AAEA,eAAe/B,6BACbJ,EAAM,EACNF,MAAkC;IAElC,IAAIiD,aAAa,AACf,CAAA,MAAM/C,GACHgD,MAAM,CAAC;QACNC,gBAAgB3D,qBAAqB2D,cAAc;QACnDC,QAAQ9D,GAAW,CAAC,aAAa,CAAC;QAClC+D,gBAAgB/D,GAAkB,CAAC,IAAI,EAAEE,qBAAqB8D,SAAS,CAAC,CAAC,CAAC;IAC5E,GACCC,IAAI,CAAC/D,sBACLgE,KAAK,CAAC1E,GAAGU,qBAAqBa,WAAW,EAAEL,OAAOK,WAAW,GAC7DoD,OAAO,CAACjE,qBAAqB2D,cAAc,CAAA,EAC9CnC,GAAG,CAAC,CAAC0C,MAAS,CAAA;YACd,GAAGA,GAAG;YACNL,gBAAgB,IAAIM,KAAKD,IAAIL,cAAc;QAC7C,CAAA;IACA,IAAIrD,OAAO8B,uBAAuB,KAAK8B,WAAW;QAChD,MAAMC,mBAAmB,MAAMC,sCAAsC5D,IAAI;YACvEG,aAAaL,OAAOK,WAAW;YAC/B0D,eAAe/D,OAAO8B,uBAAuB;QAC/C;QACAmB,aAAaA,WAAWT,MAAM,CAC5B,CAACwB,SAAW,CAACH,iBAAiBI,IAAI,CAAC,CAAC/C,SAAWuB,SAASuB,OAAOb,cAAc,EAAEjC;IAEnF;IAEA,MAAMgD,wBAAwB,MAAMhE,GACjCgD,MAAM,CAAC;QACNC,gBAAgBxD,aAAawD,cAAc;QAC3CgB,UAAU7E,GAAW,CAAC,aAAa,EAAEK,aAAa0C,KAAK,CAAC,UAAU,CAAC;IACrE,GACCkB,IAAI,CAAC5D,cACL6D,KAAK,CACJ7E,IAAIG,GAAGa,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAAGrB,GAAGW,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,IAE5FmE,OAAO,CAAC9D,aAAawD,cAAc;IAEtC,MAAMkB,mBAAmB,IAAIC,IAC3BJ,sBAAsBlD,GAAG,CAAC,CAAC0C,MAAQ;YAACa,SAASb,IAAIP,cAAc;YAAGO,IAAIS,QAAQ;SAAC;IAEjF,MAAMK,mCAAmC,MAAMtE,GAC5CgD,MAAM,CAAC;QACNC,gBAAgBxD,aAAawD,cAAc;QAC3CgB,UAAU7E,GAAW,CAAC,aAAa,EAAEK,aAAa0C,KAAK,CAAC,UAAU,CAAC;IACrE,GACCkB,IAAI,CAAC5D,cACL6D,KAAK,CACJ7E,IACEG,GAAGa,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAC/CvB,GAAGa,aAAaiC,aAAa,EAAE5B,OAAO4B,aAAa,GACnD5C,GAAGW,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,IAGxCmE,OAAO,CAAC9D,aAAawD,cAAc;IAEtC,MAAMpC,YAAYf,OAAOe,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YACpD6B,aAAa7B,SAAS6B,WAAW;YACjC5B,QAAQ;mBAAIxC,mBAAmBuC,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACAqD,8BAA8B1D,WAAWyD;IACzC,MAAM7C,QAAsB,EAAE;IAC9B,MAAM+C,SAA6B,EAAE;IACrC,IAAI9D,2BAA2BZ,OAAOa,eAAe;IACrD,MAAM8D,mBACJ3E,OAAO8B,uBAAuB,KAAK8B,YAAY,CAAC,IAAI;QAACvD,aAAaL,OAAOK,WAAW;IAAA;IAEtF,KAAK,MAAM2D,UAAUY,eAAe3B,YAAa;QAC/C,MAAMV,sBAAsBxB,UACzByB,MAAM,CAAC,CAACvB,WAAawB,SAASuB,OAAOb,cAAc,EAAElC,SAASC,MAAM,GACpEwB,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAEzB,MAAM,CAAC2B,MAAM,GAAGD,EAAE1B,MAAM,CAAC2B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAG5F,IAAIP,oBAAoBM,MAAM,KAAK,GAAG;QAEtC,MAAMsB,WAAWE,iBAAiBQ,GAAG,CAACN,SAASP,OAAOb,cAAc,MAAM;QAC1E,MAAM2B,aAAaC,KAAKC,GAAG,CAAC,GAAGhB,OAAOZ,MAAM,GAAGe;QAC/C,MAAMc,WAAW1C,oBAAoBL,MAAM,CACzC,CAACC,OAAOlB,WAAakB,QAAQlB,SAASE,cAAc,EACpD;QAEF,MAAM+D,QAAQH,KAAKI,GAAG,CAACL,YAAYG,UAAUrE;QAC7C,IAAIwE,qBAAqBjB;QAEzB,IAAIe,QAAQ,KAAKlF,OAAOa,eAAe,GAAG,GAAG;YAC3C,MAAM,CAACwE,SAAS,GAAG,MAAMnF,GACtBoF,MAAM,CAAC3F,cACP4F,MAAM,CAAC;gBACNlF,aAAaL,OAAOK,WAAW;gBAC/BuB,eAAe5B,OAAO4B,aAAa;gBACnCuB,gBAAgBa,OAAOb,cAAc;gBACrCd,OAAO6C;gBACPd,WAAW9E,GAAG,CAAC,SAAS,EAAEU,OAAO6B,UAAU,CAAC,yBAAyB,CAAC;YACxE,GACC2D,SAAS,CAAC;gBAACC,IAAI9F,aAAa8F,EAAE;gBAAErB,WAAWzE,aAAayE,SAAS;YAAA;YAEpE,IAAI,CAACiB,UAAU,MAAM,IAAIK,MAAM;YAE/B9E,4BAA4BsE;YAC5BlC,UAAUT,qBAAqB2C;YAC/B,MAAMS,0BAA0BzF,IAAI;gBAClC0B,eAAe5B,OAAO4B,aAAa;gBACnCgE,eAAeP,SAASI,EAAE;gBAC1BtC,gBAAgBa,OAAOb,cAAc;gBACrCd,OAAO6C;gBACP7E,aAAaL,OAAOK,WAAW;YACjC;YACA+E,sBAAsBF;YACtBR,OAAO3C,IAAI,CAAC;gBACV6D,eAAeP,SAASI,EAAE;gBAC1B,GAAGd,gBAAgB;gBACnBzD,QAAQ8C,OAAOb,cAAc;gBAC7Bd,OAAO6C;gBACPd,WAAWiB,SAASjB,SAAS;YAC/B;QACF;QAEAzC,MAAMI,IAAI,CAAC;YACT,GAAG4C,gBAAgB;YACnBzD,QAAQ8C,OAAOb,cAAc;YAC7BC,QAAQY,OAAOZ,MAAM;YACrBe,UAAUiB;YACV/B,gBAAgBW,OAAOX,cAAc;QACvC;IACF;IAEA,OAAO;QAAC1B;QAAOhC,cAAc+E;IAAM;AACrC;AAEA,eAAeiB,0BACbzF,EAAM,EACNF,MAMC;IAED,MAAM6F,cAAc,MAAM3F,GACvBgD,MAAM,CAAC;QAACuC,IAAI3F,gBAAgB2F,EAAE;IAAA,GAC9BlC,IAAI,CAACzD,iBACL0D,KAAK,CACJ7E,IACEG,GAAGgB,gBAAgB8B,aAAa,EAAE5B,OAAO4B,aAAa,GACtDzC,OAAOW,gBAAgBO,WAAW,GAClClB,OAAOW,gBAAgB8F,aAAa,GACpCzG,OAAOW,gBAAgBgG,qBAAqB,GAC5C3G,OAAOW,gBAAgBiG,eAAe,GACtC7G,UAAUY,gBAAgBkG,gBAAgB,GAC1ClH,GAAGgB,gBAAgBmG,KAAK,EAAE,YAC1BrH,cAAckB,gBAAgBoB,MAAM,EAAElB,OAAOmD,cAAc,GAC3DpE,OACEmB,GACGgD,MAAM,CAAC;QAACuC,IAAI5F,sBAAsB4F,EAAE;IAAA,GACpClC,IAAI,CAAC1D,uBACL2D,KAAK,CACJ7E,IACEG,GAAGe,sBAAsBqG,gBAAgB,EAAEpG,gBAAgB2F,EAAE,GAC7D3G,GAAGe,sBAAsB+B,aAAa,EAAE5B,OAAO4B,aAAa,GAC5DzC,OAAOU,sBAAsBsG,QAAQ,GACrCnH,GAAGa,sBAAsBuE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,OAMzD8G,OAAO,CAACvH,IAAIiB,gBAAgBwD,SAAS,GAAGzE,IAAIiB,gBAAgB2F,EAAE,GAC9DY,KAAK,CAACrG,OAAOqC,KAAK,EAClBiE,GAAG,CAAC;IAEP,IAAIT,YAAYhD,MAAM,KAAK,GAAG;IAE9B,MAAM3C,GACHqG,MAAM,CAACzG,iBACP0G,GAAG,CAAC;QACHnG,aAAaL,OAAOK,WAAW;QAC/BuF,eAAe5F,OAAO4F,aAAa;QACnCa,YAAYnH,GAAG,CAAC,KAAK,CAAC;QACtBoH,WAAWpH,GAAG,CAAC,KAAK,CAAC;IACvB,GACCkE,KAAK,CACJ7E,IACEM,QACEa,gBAAgB2F,EAAE,EAClBI,YAAY7E,GAAG,CAAC,CAAC2F,SAAWA,OAAOlB,EAAE,IAEvCtG,OAAOW,gBAAgBO,WAAW,GAClClB,OAAOW,gBAAgB8F,aAAa,GACpCzG,OAAOW,gBAAgBiG,eAAe;AAG9C;AAEA,eAAetF,mCAAmCC,oBAAyC;IACzF,IAAIA,qBAAqBkG,IAAI,KAAK,GAAG,OAAO,EAAE;IAC9C,MAAMlF,OAAO,MAAMnC,KAChB2D,MAAM,CAAC;QACN7C,aAAab,qBAAqBa,WAAW;QAC7CgD,gBAAgB/D,GAAS,CAAC,IAAI,EAAEE,qBAAqB8D,SAAS,CAAC,CAAC,CAAC;IACnE,GACCC,IAAI,CAAC/D,sBACLgE,KAAK,CAACvE,QAAQO,qBAAqBa,WAAW,EAAE;WAAIK;KAAqB,GACzE+C,OAAO,CAACjE,qBAAqBa,WAAW,EACxC+F,OAAO,CACNvH,IAAIS,GAAG,CAAC,IAAI,EAAEE,qBAAqB8D,SAAS,CAAC,CAAC,CAAC,GAC/CzE,IAAIW,qBAAqBa,WAAW;IAExC,OAAOqB,KAAKV,GAAG,CAAC,CAAC0C,MAAQA,IAAIrD,WAAW;AAC1C;AAEA,OAAO,eAAewG;IACpB,MAAMnF,OAAO,MAAMnC,KAChB2D,MAAM,CAAC;QAAC7C,aAAab,qBAAqBa,WAAW;IAAA,GACrDkD,IAAI,CAAC/D,sBACLiE,OAAO,CAACjE,qBAAqBa,WAAW;IAC3C,OAAOqB,KAAKV,GAAG,CAAC,CAAC0C,MAAQA,IAAIrD,WAAW;AAC1C;AAEA,eAAeyD,sCACb5D,EAAM,EACNF,MAAoD;IAEpD,MAAM0B,OAAO,MAAMxB,GAChBgD,MAAM,CAAC;QAAChC,QAAQzB,+BAA+ByB,MAAM;IAAA,GACrDqC,IAAI,CAAC9D,gCACLqH,SAAS,CACRpH,mBACAZ,GAAGY,kBAAkB+F,EAAE,EAAEhG,+BAA+BmC,aAAa,GAEtE4B,KAAK,CACJ7E,IACEG,GAAGW,+BAA+BY,WAAW,EAAEL,OAAOK,WAAW,GACjEvB,GAAGY,kBAAkBW,WAAW,EAAEL,OAAOK,WAAW,GACpDvB,GAAGY,kBAAkBqH,KAAK,EAAE,cAC5BzH,GAAG,CAAC,EAAEG,+BAA+BuH,YAAY,CAAC,YAAY,EAAEhH,OAAO+D,aAAa,CAAC,yBAAyB,CAAC,EAC/GzE,GAAG,CAAC,EAAEI,kBAAkBuH,SAAS,CAAC,QAAQ,CAAC,EAC3C3H,GAAG,CAAC,CAAC,EAAEI,kBAAkB0E,SAAS,CAAC,YAAY,EAAE1E,kBAAkB0E,SAAS,CAAC,SAAS,CAAC;IAG7F,OAAO1C,KAAKV,GAAG,CAAC,CAAC0C,MAAQA,IAAIxC,MAAM;AACrC;AAEA,OAAO,eAAegG,0BAA0BlH,MAAyB;IACvE,OAAO,MAAMT,KAAKU,WAAW,CAAC,OAAOC;QACnC,MAAMiH,cAAc,MAAMjH,GACvBgD,MAAM,CAAC;YAACuC,IAAI9F,aAAa8F,EAAE;QAAA,GAC3BlC,IAAI,CAAC5D,cACL6D,KAAK,CAACpE,GAAGO,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,GAC3C8G,OAAO,CAACvH,IAAIc,aAAayE,SAAS,GAClCiC,KAAK,CAACrG,QAAQqG,SAAS;QAC1B,OAAO,MAAMe,gCACXlH,IACAiH,YAAYnG,GAAG,CAAC,CAACoB,cAAgBA,YAAYqD,EAAE,GAC/C;YAAC4B,aAAa;QAAI;IAEtB;AACF;AAEA,OAAO,eAAeC,wBAAwBC,GAAa;IACzD,IAAIA,IAAI1E,MAAM,KAAK,GAAG,OAAO;IAE7B,OAAO,MAAMtD,KAAKU,WAAW,CAAC,OAAOC;QACnC,OAAO,MAAMkH,gCAAgClH,IAAIqH;IACnD;AACF;AAEA,eAAeH,gCACblH,EAAM,EACNqH,GAAa,EACbvH,SAAkC,CAAC,CAAC;IAEpC,IAAIuH,IAAI1E,MAAM,KAAK,GAAG,OAAO;IAE7B,MAAM2E,kBAAkB,MAAMtH,GAC3BgD,MAAM,CAAC;QACNuC,IAAI3F,gBAAgB2F,EAAE;QACtBG,eAAe9F,gBAAgB8F,aAAa;QAC5CE,uBAAuBhG,gBAAgBgG,qBAAqB;QAC5DC,iBAAiBjG,gBAAgBiG,eAAe;QAChD0B,uBAAuB3H,gBAAgB2H,qBAAqB;IAC9D,GACClE,IAAI,CAACzD,iBACL0D,KAAK,CACJnE,GACEV,IACEM,QAAQa,gBAAgB8F,aAAa,EAAE2B,MACvCpI,OAAOW,gBAAgBiG,eAAe,GACtC5G,OAAOW,gBAAgB2H,qBAAqB,IAE9CxI,QAAQa,gBAAgBgG,qBAAqB,EAAEyB,OAGlDnB,OAAO,CAACvH,IAAIiB,gBAAgB2F,EAAE,GAC9Ba,GAAG,CAAC;IACP,MAAMoB,kBAAkB,MAAMxH,GAC3BgD,MAAM,CAAC;QAACuC,IAAI9F,aAAa8F,EAAE;IAAA,GAC3BlC,IAAI,CAAC5D,cACL6D,KAAK,CACJ7E,IACEM,QAAQU,aAAa8F,EAAE,EAAE8B,MACzBvH,OAAOqH,WAAW,GAAGjI,GAAGO,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,IAAIsE,YAGjE0C,GAAG,CAAC;IACP,MAAMqB,iBAAiBD,gBAAgB1G,GAAG,CAAC,CAACoB,cAAgBA,YAAYqD,EAAE;IAE1E,IAAIkC,eAAe9E,MAAM,KAAK,GAAG,OAAO;IAExC,MAAM+E,oBAAoBJ,gBACvBhF,MAAM,CACL,CAACmE,SACCA,OAAOf,aAAa,IACpB+B,eAAeE,QAAQ,CAAClB,OAAOf,aAAa,KAC5C,CAACe,OAAOZ,eAAe,IACvB,CAACY,OAAOc,qBAAqB,EAEhCzG,GAAG,CAAC,CAAC2F,SAAWA,OAAOlB,EAAE;IAC5B,MAAMqC,oBAAoBN,gBACvBhF,MAAM,CACL,CAACmE,SACCA,OAAOb,qBAAqB,IAAI6B,eAAeE,QAAQ,CAAClB,OAAOb,qBAAqB,GAEvF9E,GAAG,CAAC,CAAC2F,SAAWA,OAAOlB,EAAE;IAC5B,IAAImC,kBAAkB/E,MAAM,GAAG,GAAG;QAChC,MAAM3C,GACHqG,MAAM,CAAC3G,wBACP4G,GAAG,CAAC;YAACS,WAAW3H,GAAG,CAAC,KAAK,CAAC;QAAA,GAC1BkE,KAAK,CACJ7E,IACEM,QAAQW,uBAAuBsG,gBAAgB,EAAE0B,oBACjDzI,OAAOS,uBAAuBmI,UAAU,GACxC5I,OAAOS,uBAAuBqH,SAAS;QAG7C,MAAM/G,GACHqG,MAAM,CAACzG,iBACP0G,GAAG,CAAC;YACHnG,aAAa;YACbuF,eAAe;YACfa,YAAY;YACZC,WAAWpH,GAAG,CAAC,KAAK,CAAC;QACvB,GACCkE,KAAK,CACJ7E,IACEM,QAAQa,gBAAgB2F,EAAE,EAAEmC,oBAC5BzI,OAAOW,gBAAgBiG,eAAe,GACtC5G,OAAOW,gBAAgB2H,qBAAqB;IAGpD;IACA,IAAIK,kBAAkBjF,MAAM,GAAG,GAAG;QAChC,MAAM3C,GACHqG,MAAM,CAACzG,iBACP0G,GAAG,CAAC;YAACV,uBAAuB;YAAMY,WAAWpH,GAAG,CAAC,KAAK,CAAC;QAAA,GACvDkE,KAAK,CACJ7E,IACEM,QAAQa,gBAAgB2F,EAAE,EAAEqC,oBAC5B7I,QAAQa,gBAAgBgG,qBAAqB,EAAE6B;IAGvD;IAEA,MAAMK,UAAU,MAAM9H,GACnB+H,MAAM,CAACtI,cACP6D,KAAK,CAACvE,QAAQU,aAAa8F,EAAE,EAAEkC,iBAC/BnC,SAAS,CAAC;QAACC,IAAI9F,aAAa8F,EAAE;IAAA;IAEjC,OAAOuC,QAAQnF,MAAM;AACvB;AAEA,OAAO,eAAeqF,wBACpBhI,EAAM,EACNF,MAIC;IAED,IAAImI,WAAW;IAEf,KAAK,MAAMC,WAAWpI,OAAOqI,QAAQ,CAAE;QACrC,IAAID,QAAQ/F,KAAK,IAAI,GAAG;QAExB,MAAMiG,cAAc,MAAMpI,GACvBqG,MAAM,CAAC5G,cACP6G,GAAG,CAAC;YAACnE,OAAO/C,GAAG,CAAC,EAAEK,aAAa0C,KAAK,CAAC,GAAG,EAAE+F,QAAQ/F,KAAK,CAAC,CAAC;QAAA,GACzDmB,KAAK,CACJ7E,IACEG,GAAGa,aAAa8F,EAAE,EAAE2C,QAAQxC,aAAa,GACzC9G,GAAGa,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAC/CvB,GAAGa,aAAaiC,aAAa,EAAE5B,OAAO4B,aAAa,GACnD5C,GAAGW,aAAa0C,KAAK,EAAE+F,QAAQ/F,KAAK,GACpCrD,GAAGW,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,IAGxCkG,SAAS,CAAC;YAACC,IAAI9F,aAAa8F,EAAE;QAAA;QAEjC,IAAI6C,YAAYzF,MAAM,GAAG,GAAG;YAC1BsF,YAAYC,QAAQ/F,KAAK;YACzB;QACF;QAEA,MAAM2F,UAAU,MAAM9H,GACnB+H,MAAM,CAACtI,cACP6D,KAAK,CACJ7E,IACEG,GAAGa,aAAa8F,EAAE,EAAE2C,QAAQxC,aAAa,GACzC9G,GAAGa,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAC/CvB,GAAGa,aAAaiC,aAAa,EAAE5B,OAAO4B,aAAa,GACnDtC,GAAG,CAAC,EAAEK,aAAa0C,KAAK,CAAC,IAAI,EAAE+F,QAAQ/F,KAAK,CAAC,CAAC,EAC9CrD,GAAGW,aAAayE,SAAS,EAAE9E,GAAG,CAAC,KAAK,CAAC,IAGxCkG,SAAS,CAAC;YAACnD,OAAO1C,aAAa0C,KAAK;QAAA;QAEvC8F,YAAYH,QAAQ9F,MAAM,CAAC,CAACC,OAAOuB,MAAQvB,QAAQuB,IAAIrB,KAAK,EAAE;IAChE;IAEA,OAAO8F;AACT;AAEA,SAAS1D,8BACP1D,SAA+B,EAC/BwH,kBAAkE;IAElE,KAAK,MAAMnG,eAAeoG,oBAAoBD,oBAAqB;QACjE,MAAMhG,sBAAsBxB,UACzByB,MAAM,CAAC,CAACvB,WAAawB,SAASL,YAAYe,cAAc,EAAElC,SAASC,MAAM,GACzEwB,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAEzB,MAAM,CAAC2B,MAAM,GAAGD,EAAE1B,MAAM,CAAC2B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBH,YAAY+B,QAAQ;IACrD;AACF;AAEA,SAASS,eAAelD,IAAiB;IACvC,OAAO;WAAIA;KAAK,CAACgB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAM6F,cAAc7F,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAI4F,gBAAgB,GAAG,OAAOA;QAC9B,OAAO9F,EAAEU,cAAc,CAACqF,OAAO,KAAK9F,EAAES,cAAc,CAACqF,OAAO;IAC9D;AACF;AAEA,SAASF,oBAA0D9G,IAAS;IAC1E,OAAO;WAAIA;KAAK,CAACgB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAM6F,cAAc7F,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAI4F,gBAAgB,GAAG,OAAOA;QAC9B,OAAOlE,SAAS5B,EAAEQ,cAAc,EAAEJ,aAAa,CAACwB,SAAS3B,EAAEO,cAAc;IAC3E;AACF;AAEA,SAASV,SAASU,cAAwB,EAAEwF,eAAyB;IACnE,OAAOxF,eAAeyF,KAAK,CAAC,CAACC,QAAUF,gBAAgBd,QAAQ,CAACgB;AAClE;AAEA,SAAS7F,UAAUjC,SAA+B,EAAEsB,KAAa;IAC/D,IAAIyG,YAAYzG;IAChB,KAAK,MAAMpB,YAAYF,UAAW;QAChC,IAAI+H,cAAc,GAAG;QACrB,MAAMC,OAAOhE,KAAKI,GAAG,CAAClE,SAASE,cAAc,EAAE2H;QAC/C7H,SAASE,cAAc,IAAI4H;QAC3BD,aAAaC;IACf;AACF;AAEA,SAASxE,SAASrD,MAAgB;IAChC,OAAO8H,KAAKC,SAAS,CAAC/H;AACxB"}
1
+ {"version":3,"sources":["../../src/db/reservations.ts"],"sourcesContent":["import {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n and,\n arrayContains,\n asc,\n eq,\n exists,\n gt,\n inArray,\n isNotNull,\n isNull,\n lt,\n notExists,\n notInArray,\n or,\n sql,\n} from 'drizzle-orm';\nimport {recordProviderRunnerActivationOutcome} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';\nimport {terminalStates} from './runner-states.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {provisionerCapabilitySnapshots} from './schema/provisioner-capability-snapshots.js';\nimport {provisionerTokens} from './schema/provisioner-tokens.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerActivationTokens} from './schema/runner-activation-tokens.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {type providerRunnerLaunchKindEnum, providerRunners} from './schema/runner-instances.js';\n\nexport interface ReservationTemplate {\n templateKey: string;\n labels: string[];\n availableSlots: number;\n starting: number;\n running: number;\n}\n\nexport interface DemandStat {\n workspaceId?: string;\n labels: string[];\n queued: number;\n reserved: number;\n oldestQueuedAt: Date;\n}\n\nexport interface ReservationGrant {\n reservationId: string;\n workspaceId?: string;\n labels: string[];\n count: number;\n expiresAt: Date;\n}\n\nexport interface PollDemandAndReserveParams {\n workspaceId: string;\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n /** Lifetime of the short reservation that gives rebound runners time to activate. */\n activationGraceSeconds?: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds?: number;\n}\n\nexport interface InstallationPollDemandAndReserveParams {\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n /** Lifetime of the short reservation that gives rebound runners time to activate. */\n activationGraceSeconds?: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds: number;\n eligibleWorkspaceIds: ReadonlySet<string>;\n signal?: AbortSignal;\n onReservations?: (reservations: ReservationGrant[]) => void;\n}\n\ntype DemandScope = 'installation' | 'workspace';\n\ninterface IdleRunnerCandidate {\n id: string;\n launchKind: (typeof providerRunnerLaunchKindEnum.enumValues)[number];\n}\n\ninterface BindableRunnerParams {\n provisionerId: string;\n workspaceId: string;\n requiredLabels: string[];\n scope: DemandScope;\n}\n\ninterface IdleRunnerSelectionParams extends BindableRunnerParams {\n count: number;\n}\n\ntype PollDemandAndReserveLockedParams = PollDemandAndReserveParams & {\n scope: DemandScope;\n};\n\ninterface NormalizedTemplate {\n templateKey: string;\n labels: string[];\n remainingSlots: number;\n}\n\ninterface DemandRow {\n requiredLabels: string[];\n queued: number;\n oldestQueuedAt: Date;\n}\n\ninterface NewReservationUnits {\n labels: string[];\n count: number;\n}\n\ninterface PollDemandAndReserveResult {\n stats: DemandStat[];\n /** Only launch reservations are exposed to the provisioner. */\n reservations: ReservationGrant[];\n /** Internal allocation accounting for bound and launch rows together. */\n newlyReservedUnits: NewReservationUnits[];\n}\n\nexport async function pollDemandAndReserve(\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n const result = await db().transaction(async (tx) => {\n return await pollDemandAndReserveTx(tx, params);\n });\n return {stats: result.stats, reservations: result.reservations};\n}\n\nexport async function pollDemandAndReserveTx(\n tx: Tx,\n params: PollDemandAndReserveParams,\n): Promise<PollDemandAndReserveResult> {\n await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId}))`);\n return await pollDemandAndReserveLockedTx(tx, {...params, scope: 'workspace'});\n}\n\nexport async function pollInstallationDemandAndReserve(\n params: InstallationPollDemandAndReserveParams,\n): Promise<{\n stats: DemandStat[];\n reservations: ReservationGrant[];\n newlyReservedCount?: number;\n}> {\n const candidateWorkspaceIds = await listInstallationDemandWorkspaceIds(\n params.eligibleWorkspaceIds,\n );\n const results: PollDemandAndReserveResult[] = [];\n let remainingMaxReservations = params.maxReservations;\n const remainingTemplates = params.templates.map((template) => ({\n ...template,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n for (const workspaceId of candidateWorkspaceIds) {\n if (params.signal?.aborted || remainingMaxReservations === 0) break;\n const result = await db().transaction(async (tx) => {\n const lockResult = await tx.execute<{locked: boolean}>(\n sql`select pg_try_advisory_xact_lock(hashtext(${workspaceId})) as locked`,\n );\n const locked = lockResult.rows[0];\n if (!locked?.locked) return {stats: [], reservations: [], newlyReservedUnits: []};\n return await pollDemandAndReserveLockedTx(tx, {\n workspaceId,\n provisionerId: params.provisionerId,\n maxReservations: remainingMaxReservations,\n ttlSeconds: params.ttlSeconds,\n ...(params.activationGraceSeconds !== undefined\n ? {activationGraceSeconds: params.activationGraceSeconds}\n : {}),\n templates: remainingTemplates.map((template) => ({\n ...template,\n availableSlots: template.remainingSlots,\n })),\n capabilityWindowSeconds: params.capabilityWindowSeconds,\n scope: 'installation',\n });\n });\n results.push(result);\n consumeInstallationTemplateSlots(remainingTemplates, result.reservations);\n params.onReservations?.(result.reservations);\n remainingMaxReservations -= result.newlyReservedUnits.reduce(\n (total, reservation) => total + reservation.count,\n 0,\n );\n }\n const newlyReservedCount = results.reduce(\n (total, result) =>\n total +\n result.newlyReservedUnits.reduce((subtotal, reservation) => subtotal + reservation.count, 0),\n 0,\n );\n return {\n stats: results.flatMap((result) => result.stats),\n reservations: results.flatMap((result) => result.reservations),\n ...(newlyReservedCount > 0 ? {newlyReservedCount} : {}),\n };\n}\n\nfunction consumeInstallationTemplateSlots(\n templates: NormalizedTemplate[],\n launchGrants: ReservationGrant[],\n): void {\n // Adopted runners are already included in the running count behind availableSlots.\n // Only units that still need a launch consume advertised template capacity.\n for (const reservation of launchGrants) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.labels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.count);\n }\n}\n\nasync function pollDemandAndReserveLockedTx(\n tx: Tx,\n params: PollDemandAndReserveLockedParams,\n): Promise<PollDemandAndReserveResult> {\n let demandRows = (\n await tx\n .select({\n requiredLabels: pendingJobExecutions.requiredLabels,\n queued: sql<number>`count(*)::int`,\n oldestQueuedAt: sql<Date | string>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(eq(pendingJobExecutions.workspaceId, params.workspaceId))\n .groupBy(pendingJobExecutions.requiredLabels)\n ).map((row) => ({\n ...row,\n oldestQueuedAt: new Date(row.oldestQueuedAt),\n }));\n if (params.capabilityWindowSeconds !== undefined) {\n const capabilityLabels = await listActiveWorkspaceCapabilityLabelsTx(tx, {\n workspaceId: params.workspaceId,\n windowSeconds: params.capabilityWindowSeconds,\n });\n demandRows = demandRows.filter(\n (demand) => !capabilityLabels.some((labels) => isSubset(demand.requiredLabels, labels)),\n );\n }\n\n const activeReservationRows = await tx\n .select({\n requiredLabels: reservations.requiredLabels,\n reserved: sql<number>`coalesce(sum(${reservations.count}), 0)::int`,\n })\n .from(reservations)\n .where(\n and(eq(reservations.workspaceId, params.workspaceId), gt(reservations.expiresAt, sql`now()`)),\n )\n .groupBy(reservations.requiredLabels);\n\n const reservedByLabels = new Map(\n activeReservationRows.map((row) => [labelKey(row.requiredLabels), row.reserved]),\n );\n const activeProvisionerReservationRows = await listActiveProvisionerReservationRowsTx(tx, {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n });\n\n const templates = params.templates.map((template) => ({\n templateKey: template.templateKey,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n deductProvisionerReservations(templates, activeProvisionerReservationRows);\n const stats: DemandStat[] = [];\n const grants: ReservationGrant[] = [];\n const newlyReservedUnits: NewReservationUnits[] = [];\n let remainingMaxReservations = params.maxReservations;\n const workspaceIdField =\n params.capabilityWindowSeconds === undefined ? {} : {workspaceId: params.workspaceId};\n\n for (const demand of sortDemandRows(demandRows)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(demand.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n\n if (satisfyingTemplates.length === 0) continue;\n\n const reserved = reservedByLabels.get(labelKey(demand.requiredLabels)) ?? 0;\n const unreserved = Math.max(0, demand.queued - reserved);\n const capacity = satisfyingTemplates.reduce(\n (total, template) => total + template.remainingSlots,\n 0,\n );\n const grant = Math.min(unreserved, capacity, remainingMaxReservations);\n let reservedAfterGrant = reserved;\n\n if (grant > 0 && params.maxReservations > 0) {\n const idleRunners = await listIdleRunnerInstancesTx(tx, {\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: grant,\n workspaceId: params.workspaceId,\n scope: params.scope,\n });\n\n remainingMaxReservations -= grant;\n\n const boundCount = idleRunners.length;\n if (boundCount > 0) {\n const [boundReservation] = await tx\n .insert(reservations)\n .values({\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: boundCount,\n kind: 'bound',\n expiresAt: sql`now() + (${params.activationGraceSeconds ?? params.ttlSeconds} || ' seconds')::interval`,\n })\n .returning({id: reservations.id});\n\n if (!boundReservation) throw new Error('Insert returned no rows');\n\n await bindIdleRunnerInstancesTx(tx, {\n provisionerId: params.provisionerId,\n reservationId: boundReservation.id,\n workspaceId: params.workspaceId,\n requiredLabels: demand.requiredLabels,\n scope: params.scope,\n idleRunners,\n });\n }\n\n const launchCount = grant - boundCount;\n drawSlots(satisfyingTemplates, launchCount);\n let launchReservation: {id: string; expiresAt: Date} | undefined;\n if (launchCount > 0) {\n const [inserted] = await tx\n .insert(reservations)\n .values({\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: launchCount,\n kind: 'launch',\n expiresAt: sql`now() + (${params.ttlSeconds} || ' seconds')::interval`,\n })\n .returning({id: reservations.id, expiresAt: reservations.expiresAt});\n\n if (!inserted) throw new Error('Insert returned no rows');\n launchReservation = inserted;\n }\n\n newlyReservedUnits.push({labels: demand.requiredLabels, count: grant});\n reservedAfterGrant += grant;\n if (launchReservation) {\n grants.push({\n reservationId: launchReservation.id,\n ...workspaceIdField,\n labels: demand.requiredLabels,\n count: launchCount,\n expiresAt: launchReservation.expiresAt,\n });\n }\n }\n\n stats.push({\n ...workspaceIdField,\n labels: demand.requiredLabels,\n queued: demand.queued,\n reserved: reservedAfterGrant,\n oldestQueuedAt: demand.oldestQueuedAt,\n });\n }\n\n return {stats, reservations: grants, newlyReservedUnits};\n}\n\nasync function listActiveProvisionerReservationRowsTx(\n tx: Tx,\n params: {workspaceId: string; provisionerId: string},\n): Promise<Array<{requiredLabels: string[]; reserved: number}>> {\n const candidateRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n );\n const candidateIds = candidateRows.map((row) => row.id);\n if (candidateIds.length === 0) return [];\n\n // Assignment and provider-report transactions use this key before changing runner\n // links. Lock it before counting so an in-flight assignment cannot expose its unit twice.\n await lockRunnerReservationAdvisoryKeysTx(tx, {\n provisionerId: params.provisionerId,\n reservationIds: candidateIds,\n });\n\n const activeRows = await tx\n .select({\n id: reservations.id,\n requiredLabels: reservations.requiredLabels,\n count: reservations.count,\n })\n .from(reservations)\n .where(\n and(\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .orderBy(asc(reservations.id))\n .for('update');\n if (activeRows.length === 0) return [];\n\n const activeIds = activeRows.map((row) => row.id);\n const activeIdSet = new Set(activeIds);\n const usedRunnerRows = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n state: providerRunners.state,\n })\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n isNull(providerRunners.reservationReleasedAt),\n or(\n inArray(providerRunners.reservationId, activeIds),\n and(\n inArray(providerRunners.intendedReservationId, activeIds),\n notInArray(providerRunners.state, [...terminalStates]),\n ),\n ),\n ),\n );\n const usedByReservation = new Map<string, Set<string>>();\n for (const runner of usedRunnerRows) {\n if (runner.reservationId && activeIdSet.has(runner.reservationId)) {\n addUsedRunner(usedByReservation, runner.reservationId, runner.id);\n }\n if (\n runner.intendedReservationId &&\n activeIdSet.has(runner.intendedReservationId) &&\n !terminalStates.includes(runner.state as (typeof terminalStates)[number])\n ) {\n addUsedRunner(usedByReservation, runner.intendedReservationId, runner.id);\n }\n }\n\n return activeRows.flatMap((reservation) => {\n const used = usedByReservation.get(reservation.id)?.size ?? 0;\n const pending = Math.max(0, reservation.count - used);\n return pending > 0 ? [{requiredLabels: reservation.requiredLabels, reserved: pending}] : [];\n });\n}\n\nfunction addUsedRunner(\n usedByReservation: Map<string, Set<string>>,\n reservationId: string,\n runnerId: string,\n): void {\n const runnerIds = usedByReservation.get(reservationId);\n if (runnerIds) runnerIds.add(runnerId);\n else usedByReservation.set(reservationId, new Set([runnerId]));\n}\n\n// An expired or missing reservation is stale. A live reservation protects a runner that is\n// still booting or waiting for its activation grace period.\nfunction canBindRunner(tx: Tx, params: BindableRunnerParams) {\n return and(\n or(\n and(isNull(providerRunners.workspaceId), isNull(providerRunners.reservationId)),\n and(\n isNotNull(providerRunners.reservationId),\n params.scope === 'installation'\n ? undefined\n : or(\n isNull(providerRunners.workspaceId),\n eq(providerRunners.workspaceId, params.workspaceId),\n ),\n notExists(\n tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.id, providerRunners.reservationId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n ),\n or(\n isNull(providerRunners.intendedReservationId),\n notExists(\n tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.id, providerRunners.intendedReservationId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n isNull(providerRunners.reservationReleasedAt),\n isNull(providerRunners.runnerSessionId),\n );\n}\n\nfunction isBindableRunner(tx: Tx, params: BindableRunnerParams) {\n return and(\n eq(providerRunners.provisionerId, params.provisionerId),\n canBindRunner(tx, params),\n isNotNull(providerRunners.providerRunnerId),\n eq(providerRunners.state, 'running'),\n arrayContains(providerRunners.labels, params.requiredLabels),\n exists(\n tx\n .select({id: runnerControlSessions.id})\n .from(runnerControlSessions)\n .where(\n and(\n eq(runnerControlSessions.runnerInstanceId, providerRunners.id),\n eq(runnerControlSessions.provisionerId, params.provisionerId),\n isNull(runnerControlSessions.closedAt),\n gt(runnerControlSessions.expiresAt, sql`now()`),\n ),\n ),\n ),\n );\n}\n\nasync function listIdleRunnerInstancesTx(\n tx: Tx,\n params: IdleRunnerSelectionParams & {count: number},\n): Promise<IdleRunnerCandidate[]> {\n const lockedRunners = await selectIdleRunnerInstancesTx(tx, params);\n if (lockedRunners.length === 0) return lockedRunners;\n\n // A SELECT ... FOR UPDATE can wake with a fresh target row but the original\n // statement snapshot for reservation subqueries. Re-check the full predicate\n // in a new statement while retaining the row locks from the first query.\n return await selectIdleRunnerInstancesTx(tx, {\n ...params,\n runnerIds: lockedRunners.map((runner) => runner.id),\n count: lockedRunners.length,\n });\n}\n\nasync function selectIdleRunnerInstancesTx(\n tx: Tx,\n params: IdleRunnerSelectionParams & {count: number; runnerIds?: string[]},\n): Promise<IdleRunnerCandidate[]> {\n if (params.runnerIds) {\n // The first pass retains the row locks. Recheck the full predicate in a fresh statement so\n // reservation subqueries see the state that was current when the lock was acquired.\n return await tx\n .select({id: providerRunners.id, launchKind: providerRunners.launchKind})\n .from(providerRunners)\n .where(and(inArray(providerRunners.id, params.runnerIds), isBindableRunner(tx, params)))\n .orderBy(asc(providerRunners.createdAt), asc(providerRunners.id))\n .limit(params.count);\n }\n\n // Select the oldest candidates first, but leave row locking to a second pass. The cleanup\n // path locks runner rows by id, so the locking pass must use that same order. A skipped row\n // rolls back only this nested savepoint and retries the bounded candidate scan, allowing the\n // next-oldest eligible runner to refill the grant without retaining a partial lock set.\n const retrySelection = Symbol('retry bindable runner selection');\n const idleRunners = await (async () => {\n while (true) {\n try {\n return await tx.transaction(async (lockTx) => {\n const candidateRunners = await lockTx\n .select({id: providerRunners.id, launchKind: providerRunners.launchKind})\n .from(providerRunners)\n .where(isBindableRunner(lockTx, params))\n .orderBy(asc(providerRunners.createdAt), asc(providerRunners.id))\n .limit(params.count);\n\n if (candidateRunners.length === 0) return candidateRunners;\n\n // PostgreSQL can acquire row locks before sorting a multi-row FOR UPDATE result. Lock\n // each selected candidate individually so both polling and cleanup use id order.\n const lockedRunners: typeof candidateRunners = [];\n for (const candidate of [...candidateRunners].sort(compareRunnerIds)) {\n const [runner] = await lockTx\n .select({id: providerRunners.id, launchKind: providerRunners.launchKind})\n .from(providerRunners)\n .where(and(eq(providerRunners.id, candidate.id), isBindableRunner(lockTx, params)))\n .for('update');\n if (runner) lockedRunners.push(runner);\n }\n\n if (lockedRunners.length !== candidateRunners.length) throw retrySelection;\n return lockedRunners;\n });\n } catch (error) {\n if (error !== retrySelection) throw error;\n }\n }\n })();\n\n return idleRunners;\n}\n\nasync function bindIdleRunnerInstancesTx(\n tx: Tx,\n params: {\n provisionerId: string;\n reservationId: string;\n workspaceId: string;\n requiredLabels: string[];\n scope: DemandScope;\n idleRunners: IdleRunnerCandidate[];\n },\n): Promise<void> {\n const idleRunnerIds = params.idleRunners.map((runner) => runner.id);\n\n await tx\n .update(runnerActivationTokens)\n .set({revokedAt: sql`now()`})\n .where(\n and(\n inArray(runnerActivationTokens.runnerInstanceId, idleRunnerIds),\n isNull(runnerActivationTokens.consumedAt),\n isNull(runnerActivationTokens.revokedAt),\n ),\n );\n\n // listIdleRunnerInstancesTx re-checks the full eligibility predicate in a fresh statement\n // after acquiring row locks. Those locks prevent the validated set from changing before this\n // update runs.\n const boundRunners = await tx\n .update(providerRunners)\n .set({\n workspaceId: params.workspaceId,\n reservationId: params.reservationId,\n intendedReservationId: null,\n assignedAt: sql`now()`,\n updatedAt: sql`now()`,\n })\n .where(and(inArray(providerRunners.id, idleRunnerIds), isBindableRunner(tx, params)))\n .returning({id: providerRunners.id, launchKind: providerRunners.launchKind});\n\n if (boundRunners.length !== params.idleRunners.length)\n throw new Error('Locked idle runner set changed before binding');\n\n const reboundCount = boundRunners.filter((runner) => runner.launchKind === 'demand').length;\n if (reboundCount > 0)\n recordProviderRunnerActivationOutcome({outcome: 'rebound', count: reboundCount});\n}\n\nasync function listInstallationDemandWorkspaceIds(eligibleWorkspaceIds: ReadonlySet<string>) {\n if (eligibleWorkspaceIds.size === 0) return [];\n const rows = await db()\n .select({\n workspaceId: pendingJobExecutions.workspaceId,\n oldestQueuedAt: sql<Date>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.workspaceId, [...eligibleWorkspaceIds]))\n .groupBy(pendingJobExecutions.workspaceId)\n .orderBy(\n asc(sql`min(${pendingJobExecutions.createdAt})`),\n asc(pendingJobExecutions.workspaceId),\n );\n return rows.map((row) => row.workspaceId);\n}\n\nexport async function listQueuedDemandWorkspaceIds(): Promise<string[]> {\n const rows = await db()\n .select({workspaceId: pendingJobExecutions.workspaceId})\n .from(pendingJobExecutions)\n .groupBy(pendingJobExecutions.workspaceId);\n return rows.map((row) => row.workspaceId);\n}\n\nasync function listActiveWorkspaceCapabilityLabelsTx(\n tx: Tx,\n params: {workspaceId: string; windowSeconds: number},\n): Promise<string[][]> {\n const rows = await tx\n .select({labels: provisionerCapabilitySnapshots.labels})\n .from(provisionerCapabilitySnapshots)\n .innerJoin(\n provisionerTokens,\n eq(provisionerTokens.id, provisionerCapabilitySnapshots.provisionerId),\n )\n .where(\n and(\n eq(provisionerCapabilitySnapshots.workspaceId, params.workspaceId),\n eq(provisionerTokens.workspaceId, params.workspaceId),\n eq(provisionerTokens.scope, 'workspace'),\n sql`${provisionerCapabilitySnapshots.advertisedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n sql`${provisionerTokens.revokedAt} is null`,\n sql`(${provisionerTokens.expiresAt} is null or ${provisionerTokens.expiresAt} > now())`,\n ),\n );\n return rows.map((row) => row.labels);\n}\n\nexport async function deleteExpiredReservations(params?: {limit?: number}): Promise<number> {\n return await db().transaction(async (tx) => {\n const expiredRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(lt(reservations.expiresAt, sql`now()`))\n .orderBy(asc(reservations.expiresAt))\n .limit(params?.limit ?? 1000);\n return await deleteReservationsWithCleanupTx(\n tx,\n expiredRows.map((reservation) => reservation.id),\n {expiredOnly: true},\n );\n });\n}\n\nexport async function deleteReservationsByIds(ids: string[]): Promise<number> {\n if (ids.length === 0) return 0;\n\n return await db().transaction(async (tx) => {\n return await deleteReservationsWithCleanupTx(tx, ids);\n });\n}\n\nasync function deleteReservationsWithCleanupTx(\n tx: Tx,\n ids: string[],\n params: {expiredOnly?: boolean} = {},\n): Promise<number> {\n if (ids.length === 0) return 0;\n\n const isAffectedRunner = () =>\n or(\n and(\n inArray(providerRunners.reservationId, ids),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n inArray(providerRunners.intendedReservationId, ids),\n );\n const candidateRunners = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n runnerSessionId: providerRunners.runnerSessionId,\n reservationReleasedAt: providerRunners.reservationReleasedAt,\n })\n .from(providerRunners)\n .where(isAffectedRunner())\n .orderBy(asc(providerRunners.id));\n const affectedRunners: typeof candidateRunners = [];\n for (const candidate of [...candidateRunners].sort(compareRunnerIds)) {\n const [runner] = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n runnerSessionId: providerRunners.runnerSessionId,\n reservationReleasedAt: providerRunners.reservationReleasedAt,\n })\n .from(providerRunners)\n .where(and(eq(providerRunners.id, candidate.id), isAffectedRunner()))\n .for('update');\n if (runner) affectedRunners.push(runner);\n }\n const reservationRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n inArray(reservations.id, ids),\n params.expiredOnly ? lt(reservations.expiresAt, sql`now()`) : undefined,\n ),\n )\n .for('update');\n const reservationIds = reservationRows.map((reservation) => reservation.id);\n\n if (reservationIds.length === 0) return 0;\n\n const assignedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.reservationId &&\n reservationIds.includes(runner.reservationId) &&\n !runner.runnerSessionId &&\n !runner.reservationReleasedAt,\n )\n .map((runner) => runner.id);\n const intendedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.intendedReservationId && reservationIds.includes(runner.intendedReservationId),\n )\n .map((runner) => runner.id);\n if (assignedRunnerIds.length > 0) {\n await tx\n .update(runnerActivationTokens)\n .set({revokedAt: sql`now()`})\n .where(\n and(\n inArray(runnerActivationTokens.runnerInstanceId, assignedRunnerIds),\n isNull(runnerActivationTokens.consumedAt),\n isNull(runnerActivationTokens.revokedAt),\n ),\n );\n await tx\n .update(providerRunners)\n .set({\n workspaceId: null,\n reservationId: null,\n assignedAt: null,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n inArray(providerRunners.id, assignedRunnerIds),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n );\n }\n if (intendedRunnerIds.length > 0) {\n await tx\n .update(providerRunners)\n .set({intendedReservationId: null, updatedAt: sql`now()`})\n .where(\n and(\n inArray(providerRunners.id, intendedRunnerIds),\n inArray(providerRunners.intendedReservationId, reservationIds),\n ),\n );\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(inArray(reservations.id, reservationIds))\n .returning({id: reservations.id});\n\n return deleted.length;\n}\n\nexport async function releaseReservationUnits(\n tx: Tx,\n params: {\n workspaceId: string;\n provisionerId: string;\n releases: Array<{reservationId: string; count: number}>;\n },\n): Promise<number> {\n let released = 0;\n\n for (const release of params.releases) {\n if (release.count <= 0) continue;\n\n const decremented = await tx\n .update(reservations)\n .set({count: sql`${reservations.count} - ${release.count}`})\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.count, release.count),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({id: reservations.id});\n\n if (decremented.length > 0) {\n released += release.count;\n continue;\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n sql`${reservations.count} <= ${release.count}`,\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({count: reservations.count});\n\n released += deleted.reduce((total, row) => total + row.count, 0);\n }\n\n return released;\n}\n\nfunction deductProvisionerReservations(\n templates: NormalizedTemplate[],\n activeReservations: {requiredLabels: string[]; reserved: number}[],\n): void {\n for (const reservation of sortReservationRows(activeReservations)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.reserved);\n }\n}\n\nfunction compareRunnerIds(left: {id: string}, right: {id: string}): number {\n return left.id < right.id ? -1 : left.id > right.id ? 1 : 0;\n}\n\nfunction sortDemandRows(rows: DemandRow[]): DemandRow[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return a.oldestQueuedAt.getTime() - b.oldestQueuedAt.getTime();\n });\n}\n\nfunction sortReservationRows<T extends {requiredLabels: string[]}>(rows: T[]): T[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return labelKey(a.requiredLabels).localeCompare(labelKey(b.requiredLabels));\n });\n}\n\nfunction isSubset(requiredLabels: string[], availableLabels: string[]): boolean {\n return requiredLabels.every((label) => availableLabels.includes(label));\n}\n\nfunction drawSlots(templates: NormalizedTemplate[], count: number): void {\n let remaining = count;\n for (const template of templates) {\n if (remaining === 0) return;\n const used = Math.min(template.remainingSlots, remaining);\n template.remainingSlots -= used;\n remaining -= used;\n }\n}\n\nfunction labelKey(labels: string[]): string {\n return JSON.stringify(labels);\n}\n"],"names":["canonicalizeLabels","and","arrayContains","asc","eq","exists","gt","inArray","isNotNull","isNull","lt","notExists","notInArray","or","sql","recordProviderRunnerActivationOutcome","db","lockRunnerReservationAdvisoryKeysTx","terminalStates","pendingJobExecutions","provisionerCapabilitySnapshots","provisionerTokens","reservations","runnerActivationTokens","runnerControlSessions","providerRunners","pollDemandAndReserve","params","result","transaction","tx","pollDemandAndReserveTx","stats","execute","workspaceId","pollDemandAndReserveLockedTx","scope","pollInstallationDemandAndReserve","candidateWorkspaceIds","listInstallationDemandWorkspaceIds","eligibleWorkspaceIds","results","remainingMaxReservations","maxReservations","remainingTemplates","templates","map","template","labels","remainingSlots","availableSlots","signal","aborted","lockResult","locked","rows","newlyReservedUnits","provisionerId","ttlSeconds","activationGraceSeconds","undefined","capabilityWindowSeconds","push","consumeInstallationTemplateSlots","onReservations","reduce","total","reservation","count","newlyReservedCount","subtotal","flatMap","launchGrants","satisfyingTemplates","filter","isSubset","sort","a","b","length","templateKey","localeCompare","drawSlots","demandRows","select","requiredLabels","queued","oldestQueuedAt","createdAt","from","where","groupBy","row","Date","capabilityLabels","listActiveWorkspaceCapabilityLabelsTx","windowSeconds","demand","some","activeReservationRows","reserved","expiresAt","reservedByLabels","Map","labelKey","activeProvisionerReservationRows","listActiveProvisionerReservationRowsTx","deductProvisionerReservations","grants","workspaceIdField","sortDemandRows","get","unreserved","Math","max","capacity","grant","min","reservedAfterGrant","idleRunners","listIdleRunnerInstancesTx","boundCount","boundReservation","insert","values","kind","returning","id","Error","bindIdleRunnerInstancesTx","reservationId","launchCount","launchReservation","inserted","candidateRows","candidateIds","reservationIds","activeRows","orderBy","for","activeIds","activeIdSet","Set","usedRunnerRows","intendedReservationId","state","reservationReleasedAt","usedByReservation","runner","has","addUsedRunner","includes","used","size","pending","runnerId","runnerIds","add","set","canBindRunner","runnerSessionId","isBindableRunner","providerRunnerId","runnerInstanceId","closedAt","lockedRunners","selectIdleRunnerInstancesTx","launchKind","limit","retrySelection","Symbol","lockTx","candidateRunners","candidate","compareRunnerIds","error","idleRunnerIds","update","revokedAt","consumedAt","boundRunners","assignedAt","updatedAt","reboundCount","outcome","listQueuedDemandWorkspaceIds","innerJoin","advertisedAt","deleteExpiredReservations","expiredRows","deleteReservationsWithCleanupTx","expiredOnly","deleteReservationsByIds","ids","isAffectedRunner","affectedRunners","reservationRows","assignedRunnerIds","intendedRunnerIds","deleted","delete","releaseReservationUnits","released","release","releases","decremented","activeReservations","sortReservationRows","left","right","specificity","getTime","availableLabels","every","label","remaining","JSON","stringify"],"mappings":"AAAA,SAAQA,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,aAAa,EACbC,GAAG,EACHC,EAAE,EACFC,MAAM,EACNC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,SAAS,EACTC,UAAU,EACVC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SAAQC,qCAAqC,QAAO,uBAAuB;AAE3E,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,mCAAmC,QAAO,yBAAyB;AAC3E,SAAQC,cAAc,QAAO,qBAAqB;AAClD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,8BAA8B,QAAO,+CAA+C;AAC5F,SAAQC,iBAAiB,QAAO,iCAAiC;AACjE,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,sBAAsB,QAAO,uCAAuC;AAC5E,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAA2CC,eAAe,QAAO,+BAA+B;AAiGhG,OAAO,eAAeC,qBACpBC,MAAkC;IAElC,MAAMC,SAAS,MAAMZ,KAAKa,WAAW,CAAC,OAAOC;QAC3C,OAAO,MAAMC,uBAAuBD,IAAIH;IAC1C;IACA,OAAO;QAACK,OAAOJ,OAAOI,KAAK;QAAEV,cAAcM,OAAON,YAAY;IAAA;AAChE;AAEA,OAAO,eAAeS,uBACpBD,EAAM,EACNH,MAAkC;IAElC,MAAMG,GAAGG,OAAO,CAACnB,GAAG,CAAC,sCAAsC,EAAEa,OAAOO,WAAW,CAAC,EAAE,CAAC;IACnF,OAAO,MAAMC,6BAA6BL,IAAI;QAAC,GAAGH,MAAM;QAAES,OAAO;IAAW;AAC9E;AAEA,OAAO,eAAeC,iCACpBV,MAA8C;IAM9C,MAAMW,wBAAwB,MAAMC,mCAClCZ,OAAOa,oBAAoB;IAE7B,MAAMC,UAAwC,EAAE;IAChD,IAAIC,2BAA2Bf,OAAOgB,eAAe;IACrD,MAAMC,qBAAqBjB,OAAOkB,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YAC7D,GAAGA,QAAQ;YACXC,QAAQ;mBAAIhD,mBAAmB+C,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACA,KAAK,MAAMhB,eAAeI,sBAAuB;QAC/C,IAAIX,OAAOwB,MAAM,EAAEC,WAAWV,6BAA6B,GAAG;QAC9D,MAAMd,SAAS,MAAMZ,KAAKa,WAAW,CAAC,OAAOC;YAC3C,MAAMuB,aAAa,MAAMvB,GAAGG,OAAO,CACjCnB,GAAG,CAAC,0CAA0C,EAAEoB,YAAY,YAAY,CAAC;YAE3E,MAAMoB,SAASD,WAAWE,IAAI,CAAC,EAAE;YACjC,IAAI,CAACD,QAAQA,QAAQ,OAAO;gBAACtB,OAAO,EAAE;gBAAEV,cAAc,EAAE;gBAAEkC,oBAAoB,EAAE;YAAA;YAChF,OAAO,MAAMrB,6BAA6BL,IAAI;gBAC5CI;gBACAuB,eAAe9B,OAAO8B,aAAa;gBACnCd,iBAAiBD;gBACjBgB,YAAY/B,OAAO+B,UAAU;gBAC7B,GAAI/B,OAAOgC,sBAAsB,KAAKC,YAClC;oBAACD,wBAAwBhC,OAAOgC,sBAAsB;gBAAA,IACtD,CAAC,CAAC;gBACNd,WAAWD,mBAAmBE,GAAG,CAAC,CAACC,WAAc,CAAA;wBAC/C,GAAGA,QAAQ;wBACXG,gBAAgBH,SAASE,cAAc;oBACzC,CAAA;gBACAY,yBAAyBlC,OAAOkC,uBAAuB;gBACvDzB,OAAO;YACT;QACF;QACAK,QAAQqB,IAAI,CAAClC;QACbmC,iCAAiCnB,oBAAoBhB,OAAON,YAAY;QACxEK,OAAOqC,cAAc,GAAGpC,OAAON,YAAY;QAC3CoB,4BAA4Bd,OAAO4B,kBAAkB,CAACS,MAAM,CAC1D,CAACC,OAAOC,cAAgBD,QAAQC,YAAYC,KAAK,EACjD;IAEJ;IACA,MAAMC,qBAAqB5B,QAAQwB,MAAM,CACvC,CAACC,OAAOtC,SACNsC,QACAtC,OAAO4B,kBAAkB,CAACS,MAAM,CAAC,CAACK,UAAUH,cAAgBG,WAAWH,YAAYC,KAAK,EAAE,IAC5F;IAEF,OAAO;QACLpC,OAAOS,QAAQ8B,OAAO,CAAC,CAAC3C,SAAWA,OAAOI,KAAK;QAC/CV,cAAcmB,QAAQ8B,OAAO,CAAC,CAAC3C,SAAWA,OAAON,YAAY;QAC7D,GAAI+C,qBAAqB,IAAI;YAACA;QAAkB,IAAI,CAAC,CAAC;IACxD;AACF;AAEA,SAASN,iCACPlB,SAA+B,EAC/B2B,YAAgC;IAEhC,mFAAmF;IACnF,4EAA4E;IAC5E,KAAK,MAAML,eAAeK,aAAc;QACtC,MAAMC,sBAAsB5B,UACzB6B,MAAM,CAAC,CAAC3B,WAAa4B,SAASR,YAAYnB,MAAM,EAAED,SAASC,MAAM,GACjE4B,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAE7B,MAAM,CAAC+B,MAAM,GAAGD,EAAE9B,MAAM,CAAC+B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBN,YAAYC,KAAK;IAClD;AACF;AAEA,eAAejC,6BACbL,EAAM,EACNH,MAAwC;IAExC,IAAIwD,aAAa,AACf,CAAA,MAAMrD,GACHsD,MAAM,CAAC;QACNC,gBAAgBlE,qBAAqBkE,cAAc;QACnDC,QAAQxE,GAAW,CAAC,aAAa,CAAC;QAClCyE,gBAAgBzE,GAAkB,CAAC,IAAI,EAAEK,qBAAqBqE,SAAS,CAAC,CAAC,CAAC;IAC5E,GACCC,IAAI,CAACtE,sBACLuE,KAAK,CAACtF,GAAGe,qBAAqBe,WAAW,EAAEP,OAAOO,WAAW,GAC7DyD,OAAO,CAACxE,qBAAqBkE,cAAc,CAAA,EAC9CvC,GAAG,CAAC,CAAC8C,MAAS,CAAA;YACd,GAAGA,GAAG;YACNL,gBAAgB,IAAIM,KAAKD,IAAIL,cAAc;QAC7C,CAAA;IACA,IAAI5D,OAAOkC,uBAAuB,KAAKD,WAAW;QAChD,MAAMkC,mBAAmB,MAAMC,sCAAsCjE,IAAI;YACvEI,aAAaP,OAAOO,WAAW;YAC/B8D,eAAerE,OAAOkC,uBAAuB;QAC/C;QACAsB,aAAaA,WAAWT,MAAM,CAC5B,CAACuB,SAAW,CAACH,iBAAiBI,IAAI,CAAC,CAAClD,SAAW2B,SAASsB,OAAOZ,cAAc,EAAErC;IAEnF;IAEA,MAAMmD,wBAAwB,MAAMrE,GACjCsD,MAAM,CAAC;QACNC,gBAAgB/D,aAAa+D,cAAc;QAC3Ce,UAAUtF,GAAW,CAAC,aAAa,EAAEQ,aAAa8C,KAAK,CAAC,UAAU,CAAC;IACrE,GACCqB,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IAAIG,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAAG5B,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAE5F6E,OAAO,CAACrE,aAAa+D,cAAc;IAEtC,MAAMiB,mBAAmB,IAAIC,IAC3BJ,sBAAsBrD,GAAG,CAAC,CAAC8C,MAAQ;YAACY,SAASZ,IAAIP,cAAc;YAAGO,IAAIQ,QAAQ;SAAC;IAEjF,MAAMK,mCAAmC,MAAMC,uCAAuC5E,IAAI;QACxFI,aAAaP,OAAOO,WAAW;QAC/BuB,eAAe9B,OAAO8B,aAAa;IACrC;IAEA,MAAMZ,YAAYlB,OAAOkB,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YACpDiC,aAAajC,SAASiC,WAAW;YACjChC,QAAQ;mBAAIhD,mBAAmB+C,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACAyD,8BAA8B9D,WAAW4D;IACzC,MAAMzE,QAAsB,EAAE;IAC9B,MAAM4E,SAA6B,EAAE;IACrC,MAAMpD,qBAA4C,EAAE;IACpD,IAAId,2BAA2Bf,OAAOgB,eAAe;IACrD,MAAMkE,mBACJlF,OAAOkC,uBAAuB,KAAKD,YAAY,CAAC,IAAI;QAAC1B,aAAaP,OAAOO,WAAW;IAAA;IAEtF,KAAK,MAAM+D,UAAUa,eAAe3B,YAAa;QAC/C,MAAMV,sBAAsB5B,UACzB6B,MAAM,CAAC,CAAC3B,WAAa4B,SAASsB,OAAOZ,cAAc,EAAEtC,SAASC,MAAM,GACpE4B,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAE7B,MAAM,CAAC+B,MAAM,GAAGD,EAAE9B,MAAM,CAAC+B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAG5F,IAAIP,oBAAoBM,MAAM,KAAK,GAAG;QAEtC,MAAMqB,WAAWE,iBAAiBS,GAAG,CAACP,SAASP,OAAOZ,cAAc,MAAM;QAC1E,MAAM2B,aAAaC,KAAKC,GAAG,CAAC,GAAGjB,OAAOX,MAAM,GAAGc;QAC/C,MAAMe,WAAW1C,oBAAoBR,MAAM,CACzC,CAACC,OAAOnB,WAAamB,QAAQnB,SAASE,cAAc,EACpD;QAEF,MAAMmE,QAAQH,KAAKI,GAAG,CAACL,YAAYG,UAAUzE;QAC7C,IAAI4E,qBAAqBlB;QAEzB,IAAIgB,QAAQ,KAAKzF,OAAOgB,eAAe,GAAG,GAAG;YAC3C,MAAM4E,cAAc,MAAMC,0BAA0B1F,IAAI;gBACtD2B,eAAe9B,OAAO8B,aAAa;gBACnC4B,gBAAgBY,OAAOZ,cAAc;gBACrCjB,OAAOgD;gBACPlF,aAAaP,OAAOO,WAAW;gBAC/BE,OAAOT,OAAOS,KAAK;YACrB;YAEAM,4BAA4B0E;YAE5B,MAAMK,aAAaF,YAAYxC,MAAM;YACrC,IAAI0C,aAAa,GAAG;gBAClB,MAAM,CAACC,iBAAiB,GAAG,MAAM5F,GAC9B6F,MAAM,CAACrG,cACPsG,MAAM,CAAC;oBACN1F,aAAaP,OAAOO,WAAW;oBAC/BuB,eAAe9B,OAAO8B,aAAa;oBACnC4B,gBAAgBY,OAAOZ,cAAc;oBACrCjB,OAAOqD;oBACPI,MAAM;oBACNxB,WAAWvF,GAAG,CAAC,SAAS,EAAEa,OAAOgC,sBAAsB,IAAIhC,OAAO+B,UAAU,CAAC,yBAAyB,CAAC;gBACzG,GACCoE,SAAS,CAAC;oBAACC,IAAIzG,aAAayG,EAAE;gBAAA;gBAEjC,IAAI,CAACL,kBAAkB,MAAM,IAAIM,MAAM;gBAEvC,MAAMC,0BAA0BnG,IAAI;oBAClC2B,eAAe9B,OAAO8B,aAAa;oBACnCyE,eAAeR,iBAAiBK,EAAE;oBAClC7F,aAAaP,OAAOO,WAAW;oBAC/BmD,gBAAgBY,OAAOZ,cAAc;oBACrCjD,OAAOT,OAAOS,KAAK;oBACnBmF;gBACF;YACF;YAEA,MAAMY,cAAcf,QAAQK;YAC5BvC,UAAUT,qBAAqB0D;YAC/B,IAAIC;YACJ,IAAID,cAAc,GAAG;gBACnB,MAAM,CAACE,SAAS,GAAG,MAAMvG,GACtB6F,MAAM,CAACrG,cACPsG,MAAM,CAAC;oBACN1F,aAAaP,OAAOO,WAAW;oBAC/BuB,eAAe9B,OAAO8B,aAAa;oBACnC4B,gBAAgBY,OAAOZ,cAAc;oBACrCjB,OAAO+D;oBACPN,MAAM;oBACNxB,WAAWvF,GAAG,CAAC,SAAS,EAAEa,OAAO+B,UAAU,CAAC,yBAAyB,CAAC;gBACxE,GACCoE,SAAS,CAAC;oBAACC,IAAIzG,aAAayG,EAAE;oBAAE1B,WAAW/E,aAAa+E,SAAS;gBAAA;gBAEpE,IAAI,CAACgC,UAAU,MAAM,IAAIL,MAAM;gBAC/BI,oBAAoBC;YACtB;YAEA7E,mBAAmBM,IAAI,CAAC;gBAACd,QAAQiD,OAAOZ,cAAc;gBAAEjB,OAAOgD;YAAK;YACpEE,sBAAsBF;YACtB,IAAIgB,mBAAmB;gBACrBxB,OAAO9C,IAAI,CAAC;oBACVoE,eAAeE,kBAAkBL,EAAE;oBACnC,GAAGlB,gBAAgB;oBACnB7D,QAAQiD,OAAOZ,cAAc;oBAC7BjB,OAAO+D;oBACP9B,WAAW+B,kBAAkB/B,SAAS;gBACxC;YACF;QACF;QAEArE,MAAM8B,IAAI,CAAC;YACT,GAAG+C,gBAAgB;YACnB7D,QAAQiD,OAAOZ,cAAc;YAC7BC,QAAQW,OAAOX,MAAM;YACrBc,UAAUkB;YACV/B,gBAAgBU,OAAOV,cAAc;QACvC;IACF;IAEA,OAAO;QAACvD;QAAOV,cAAcsF;QAAQpD;IAAkB;AACzD;AAEA,eAAekD,uCACb5E,EAAM,EACNH,MAAoD;IAEpD,MAAM2G,gBAAgB,MAAMxG,GACzBsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnDnD,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC;IAG3C,MAAMyH,eAAeD,cAAcxF,GAAG,CAAC,CAAC8C,MAAQA,IAAImC,EAAE;IACtD,IAAIQ,aAAaxD,MAAM,KAAK,GAAG,OAAO,EAAE;IAExC,kFAAkF;IAClF,0FAA0F;IAC1F,MAAM9D,oCAAoCa,IAAI;QAC5C2B,eAAe9B,OAAO8B,aAAa;QACnC+E,gBAAgBD;IAClB;IAEA,MAAME,aAAa,MAAM3G,GACtBsD,MAAM,CAAC;QACN2C,IAAIzG,aAAayG,EAAE;QACnB1C,gBAAgB/D,aAAa+D,cAAc;QAC3CjB,OAAO9C,aAAa8C,KAAK;IAC3B,GACCqB,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnDnD,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAGxC4H,OAAO,CAACvI,IAAImB,aAAayG,EAAE,GAC3BY,GAAG,CAAC;IACP,IAAIF,WAAW1D,MAAM,KAAK,GAAG,OAAO,EAAE;IAEtC,MAAM6D,YAAYH,WAAW3F,GAAG,CAAC,CAAC8C,MAAQA,IAAImC,EAAE;IAChD,MAAMc,cAAc,IAAIC,IAAIF;IAC5B,MAAMG,iBAAiB,MAAMjH,GAC1BsD,MAAM,CAAC;QACN2C,IAAItG,gBAAgBsG,EAAE;QACtBG,eAAezG,gBAAgByG,aAAa;QAC5Cc,uBAAuBvH,gBAAgBuH,qBAAqB;QAC5DC,OAAOxH,gBAAgBwH,KAAK;IAC9B,GACCxD,IAAI,CAAChE,iBACLiE,KAAK,CACJzF,IACEG,GAAGqB,gBAAgBgC,aAAa,EAAE9B,OAAO8B,aAAa,GACtDhD,OAAOgB,gBAAgByH,qBAAqB,GAC5CrI,GACEN,QAAQkB,gBAAgByG,aAAa,EAAEU,YACvC3I,IACEM,QAAQkB,gBAAgBuH,qBAAqB,EAAEJ,YAC/ChI,WAAWa,gBAAgBwH,KAAK,EAAE;WAAI/H;KAAe;IAK/D,MAAMiI,oBAAoB,IAAI5C;IAC9B,KAAK,MAAM6C,UAAUL,eAAgB;QACnC,IAAIK,OAAOlB,aAAa,IAAIW,YAAYQ,GAAG,CAACD,OAAOlB,aAAa,GAAG;YACjEoB,cAAcH,mBAAmBC,OAAOlB,aAAa,EAAEkB,OAAOrB,EAAE;QAClE;QACA,IACEqB,OAAOJ,qBAAqB,IAC5BH,YAAYQ,GAAG,CAACD,OAAOJ,qBAAqB,KAC5C,CAAC9H,eAAeqI,QAAQ,CAACH,OAAOH,KAAK,GACrC;YACAK,cAAcH,mBAAmBC,OAAOJ,qBAAqB,EAAEI,OAAOrB,EAAE;QAC1E;IACF;IAEA,OAAOU,WAAWlE,OAAO,CAAC,CAACJ;QACzB,MAAMqF,OAAOL,kBAAkBpC,GAAG,CAAC5C,YAAY4D,EAAE,GAAG0B,QAAQ;QAC5D,MAAMC,UAAUzC,KAAKC,GAAG,CAAC,GAAG/C,YAAYC,KAAK,GAAGoF;QAChD,OAAOE,UAAU,IAAI;YAAC;gBAACrE,gBAAgBlB,YAAYkB,cAAc;gBAAEe,UAAUsD;YAAO;SAAE,GAAG,EAAE;IAC7F;AACF;AAEA,SAASJ,cACPH,iBAA2C,EAC3CjB,aAAqB,EACrByB,QAAgB;IAEhB,MAAMC,YAAYT,kBAAkBpC,GAAG,CAACmB;IACxC,IAAI0B,WAAWA,UAAUC,GAAG,CAACF;SACxBR,kBAAkBW,GAAG,CAAC5B,eAAe,IAAIY,IAAI;QAACa;KAAS;AAC9D;AAEA,2FAA2F;AAC3F,4DAA4D;AAC5D,SAASI,cAAcjI,EAAM,EAAEH,MAA4B;IACzD,OAAO1B,IACLY,GACEZ,IAAIQ,OAAOgB,gBAAgBS,WAAW,GAAGzB,OAAOgB,gBAAgByG,aAAa,IAC7EjI,IACEO,UAAUiB,gBAAgByG,aAAa,GACvCvG,OAAOS,KAAK,KAAK,iBACbwB,YACA/C,GACEJ,OAAOgB,gBAAgBS,WAAW,GAClC9B,GAAGqB,gBAAgBS,WAAW,EAAEP,OAAOO,WAAW,IAExDvB,UACEmB,GACGsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAEtG,gBAAgByG,aAAa,GACjD5H,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,QAMjDD,GACEJ,OAAOgB,gBAAgBuH,qBAAqB,GAC5CrI,UACEmB,GACGsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAEtG,gBAAgBuH,qBAAqB,GACzD1I,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,OAK/CL,OAAOgB,gBAAgByH,qBAAqB,GAC5CzI,OAAOgB,gBAAgBuI,eAAe;AAE1C;AAEA,SAASC,iBAAiBnI,EAAM,EAAEH,MAA4B;IAC5D,OAAO1B,IACLG,GAAGqB,gBAAgBgC,aAAa,EAAE9B,OAAO8B,aAAa,GACtDsG,cAAcjI,IAAIH,SAClBnB,UAAUiB,gBAAgByI,gBAAgB,GAC1C9J,GAAGqB,gBAAgBwH,KAAK,EAAE,YAC1B/I,cAAcuB,gBAAgBuB,MAAM,EAAErB,OAAO0D,cAAc,GAC3DhF,OACEyB,GACGsD,MAAM,CAAC;QAAC2C,IAAIvG,sBAAsBuG,EAAE;IAAA,GACpCtC,IAAI,CAACjE,uBACLkE,KAAK,CACJzF,IACEG,GAAGoB,sBAAsB2I,gBAAgB,EAAE1I,gBAAgBsG,EAAE,GAC7D3H,GAAGoB,sBAAsBiC,aAAa,EAAE9B,OAAO8B,aAAa,GAC5DhD,OAAOe,sBAAsB4I,QAAQ,GACrC9J,GAAGkB,sBAAsB6E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC;AAK1D;AAEA,eAAe0G,0BACb1F,EAAM,EACNH,MAAmD;IAEnD,MAAM0I,gBAAgB,MAAMC,4BAA4BxI,IAAIH;IAC5D,IAAI0I,cAActF,MAAM,KAAK,GAAG,OAAOsF;IAEvC,4EAA4E;IAC5E,6EAA6E;IAC7E,yEAAyE;IACzE,OAAO,MAAMC,4BAA4BxI,IAAI;QAC3C,GAAGH,MAAM;QACTiI,WAAWS,cAAcvH,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;QAClD3D,OAAOiG,cAActF,MAAM;IAC7B;AACF;AAEA,eAAeuF,4BACbxI,EAAM,EACNH,MAAyE;IAEzE,IAAIA,OAAOiI,SAAS,EAAE;QACpB,2FAA2F;QAC3F,oFAAoF;QACpF,OAAO,MAAM9H,GACVsD,MAAM,CAAC;YAAC2C,IAAItG,gBAAgBsG,EAAE;YAAEwC,YAAY9I,gBAAgB8I,UAAU;QAAA,GACtE9E,IAAI,CAAChE,iBACLiE,KAAK,CAACzF,IAAIM,QAAQkB,gBAAgBsG,EAAE,EAAEpG,OAAOiI,SAAS,GAAGK,iBAAiBnI,IAAIH,UAC9E+G,OAAO,CAACvI,IAAIsB,gBAAgB+D,SAAS,GAAGrF,IAAIsB,gBAAgBsG,EAAE,GAC9DyC,KAAK,CAAC7I,OAAOyC,KAAK;IACvB;IAEA,0FAA0F;IAC1F,4FAA4F;IAC5F,6FAA6F;IAC7F,wFAAwF;IACxF,MAAMqG,iBAAiBC,OAAO;IAC9B,MAAMnD,cAAc,MAAM,AAAC,CAAA;QACzB,MAAO,KAAM;YACX,IAAI;gBACF,OAAO,MAAMzF,GAAGD,WAAW,CAAC,OAAO8I;oBACjC,MAAMC,mBAAmB,MAAMD,OAC5BvF,MAAM,CAAC;wBAAC2C,IAAItG,gBAAgBsG,EAAE;wBAAEwC,YAAY9I,gBAAgB8I,UAAU;oBAAA,GACtE9E,IAAI,CAAChE,iBACLiE,KAAK,CAACuE,iBAAiBU,QAAQhJ,SAC/B+G,OAAO,CAACvI,IAAIsB,gBAAgB+D,SAAS,GAAGrF,IAAIsB,gBAAgBsG,EAAE,GAC9DyC,KAAK,CAAC7I,OAAOyC,KAAK;oBAErB,IAAIwG,iBAAiB7F,MAAM,KAAK,GAAG,OAAO6F;oBAE1C,sFAAsF;oBACtF,iFAAiF;oBACjF,MAAMP,gBAAyC,EAAE;oBACjD,KAAK,MAAMQ,aAAa;2BAAID;qBAAiB,CAAChG,IAAI,CAACkG,kBAAmB;wBACpE,MAAM,CAAC1B,OAAO,GAAG,MAAMuB,OACpBvF,MAAM,CAAC;4BAAC2C,IAAItG,gBAAgBsG,EAAE;4BAAEwC,YAAY9I,gBAAgB8I,UAAU;wBAAA,GACtE9E,IAAI,CAAChE,iBACLiE,KAAK,CAACzF,IAAIG,GAAGqB,gBAAgBsG,EAAE,EAAE8C,UAAU9C,EAAE,GAAGkC,iBAAiBU,QAAQhJ,UACzEgH,GAAG,CAAC;wBACP,IAAIS,QAAQiB,cAAcvG,IAAI,CAACsF;oBACjC;oBAEA,IAAIiB,cAActF,MAAM,KAAK6F,iBAAiB7F,MAAM,EAAE,MAAM0F;oBAC5D,OAAOJ;gBACT;YACF,EAAE,OAAOU,OAAO;gBACd,IAAIA,UAAUN,gBAAgB,MAAMM;YACtC;QACF;IACF,CAAA;IAEA,OAAOxD;AACT;AAEA,eAAeU,0BACbnG,EAAM,EACNH,MAOC;IAED,MAAMqJ,gBAAgBrJ,OAAO4F,WAAW,CAACzE,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;IAElE,MAAMjG,GACHmJ,MAAM,CAAC1J,wBACPuI,GAAG,CAAC;QAACoB,WAAWpK,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1B4E,KAAK,CACJzF,IACEM,QAAQgB,uBAAuB4I,gBAAgB,EAAEa,gBACjDvK,OAAOc,uBAAuB4J,UAAU,GACxC1K,OAAOc,uBAAuB2J,SAAS;IAI7C,0FAA0F;IAC1F,6FAA6F;IAC7F,eAAe;IACf,MAAME,eAAe,MAAMtJ,GACxBmJ,MAAM,CAACxJ,iBACPqI,GAAG,CAAC;QACH5H,aAAaP,OAAOO,WAAW;QAC/BgG,eAAevG,OAAOuG,aAAa;QACnCc,uBAAuB;QACvBqC,YAAYvK,GAAG,CAAC,KAAK,CAAC;QACtBwK,WAAWxK,GAAG,CAAC,KAAK,CAAC;IACvB,GACC4E,KAAK,CAACzF,IAAIM,QAAQkB,gBAAgBsG,EAAE,EAAEiD,gBAAgBf,iBAAiBnI,IAAIH,UAC3EmG,SAAS,CAAC;QAACC,IAAItG,gBAAgBsG,EAAE;QAAEwC,YAAY9I,gBAAgB8I,UAAU;IAAA;IAE5E,IAAIa,aAAarG,MAAM,KAAKpD,OAAO4F,WAAW,CAACxC,MAAM,EACnD,MAAM,IAAIiD,MAAM;IAElB,MAAMuD,eAAeH,aAAa1G,MAAM,CAAC,CAAC0E,SAAWA,OAAOmB,UAAU,KAAK,UAAUxF,MAAM;IAC3F,IAAIwG,eAAe,GACjBxK,sCAAsC;QAACyK,SAAS;QAAWpH,OAAOmH;IAAY;AAClF;AAEA,eAAehJ,mCAAmCC,oBAAyC;IACzF,IAAIA,qBAAqBiH,IAAI,KAAK,GAAG,OAAO,EAAE;IAC9C,MAAMlG,OAAO,MAAMvC,KAChBoE,MAAM,CAAC;QACNlD,aAAaf,qBAAqBe,WAAW;QAC7CqD,gBAAgBzE,GAAS,CAAC,IAAI,EAAEK,qBAAqBqE,SAAS,CAAC,CAAC,CAAC;IACnE,GACCC,IAAI,CAACtE,sBACLuE,KAAK,CAACnF,QAAQY,qBAAqBe,WAAW,EAAE;WAAIM;KAAqB,GACzEmD,OAAO,CAACxE,qBAAqBe,WAAW,EACxCwG,OAAO,CACNvI,IAAIW,GAAG,CAAC,IAAI,EAAEK,qBAAqBqE,SAAS,CAAC,CAAC,CAAC,GAC/CrF,IAAIgB,qBAAqBe,WAAW;IAExC,OAAOqB,KAAKT,GAAG,CAAC,CAAC8C,MAAQA,IAAI1D,WAAW;AAC1C;AAEA,OAAO,eAAeuJ;IACpB,MAAMlI,OAAO,MAAMvC,KAChBoE,MAAM,CAAC;QAAClD,aAAaf,qBAAqBe,WAAW;IAAA,GACrDuD,IAAI,CAACtE,sBACLwE,OAAO,CAACxE,qBAAqBe,WAAW;IAC3C,OAAOqB,KAAKT,GAAG,CAAC,CAAC8C,MAAQA,IAAI1D,WAAW;AAC1C;AAEA,eAAe6D,sCACbjE,EAAM,EACNH,MAAoD;IAEpD,MAAM4B,OAAO,MAAMzB,GAChBsD,MAAM,CAAC;QAACpC,QAAQ5B,+BAA+B4B,MAAM;IAAA,GACrDyC,IAAI,CAACrE,gCACLsK,SAAS,CACRrK,mBACAjB,GAAGiB,kBAAkB0G,EAAE,EAAE3G,+BAA+BqC,aAAa,GAEtEiC,KAAK,CACJzF,IACEG,GAAGgB,+BAA+Bc,WAAW,EAAEP,OAAOO,WAAW,GACjE9B,GAAGiB,kBAAkBa,WAAW,EAAEP,OAAOO,WAAW,GACpD9B,GAAGiB,kBAAkBe,KAAK,EAAE,cAC5BtB,GAAG,CAAC,EAAEM,+BAA+BuK,YAAY,CAAC,YAAY,EAAEhK,OAAOqE,aAAa,CAAC,yBAAyB,CAAC,EAC/GlF,GAAG,CAAC,EAAEO,kBAAkB6J,SAAS,CAAC,QAAQ,CAAC,EAC3CpK,GAAG,CAAC,CAAC,EAAEO,kBAAkBgF,SAAS,CAAC,YAAY,EAAEhF,kBAAkBgF,SAAS,CAAC,SAAS,CAAC;IAG7F,OAAO9C,KAAKT,GAAG,CAAC,CAAC8C,MAAQA,IAAI5C,MAAM;AACrC;AAEA,OAAO,eAAe4I,0BAA0BjK,MAAyB;IACvE,OAAO,MAAMX,KAAKa,WAAW,CAAC,OAAOC;QACnC,MAAM+J,cAAc,MAAM/J,GACvBsD,MAAM,CAAC;YAAC2C,IAAIzG,aAAayG,EAAE;QAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CAAChF,GAAGY,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,GAC3C4H,OAAO,CAACvI,IAAImB,aAAa+E,SAAS,GAClCmE,KAAK,CAAC7I,QAAQ6I,SAAS;QAC1B,OAAO,MAAMsB,gCACXhK,IACA+J,YAAY/I,GAAG,CAAC,CAACqB,cAAgBA,YAAY4D,EAAE,GAC/C;YAACgE,aAAa;QAAI;IAEtB;AACF;AAEA,OAAO,eAAeC,wBAAwBC,GAAa;IACzD,IAAIA,IAAIlH,MAAM,KAAK,GAAG,OAAO;IAE7B,OAAO,MAAM/D,KAAKa,WAAW,CAAC,OAAOC;QACnC,OAAO,MAAMgK,gCAAgChK,IAAImK;IACnD;AACF;AAEA,eAAeH,gCACbhK,EAAM,EACNmK,GAAa,EACbtK,SAAkC,CAAC,CAAC;IAEpC,IAAIsK,IAAIlH,MAAM,KAAK,GAAG,OAAO;IAE7B,MAAMmH,mBAAmB,IACvBrL,GACEZ,IACEM,QAAQkB,gBAAgByG,aAAa,EAAE+D,MACvCxL,OAAOgB,gBAAgBuI,eAAe,GACtCvJ,OAAOgB,gBAAgByH,qBAAqB,IAE9C3I,QAAQkB,gBAAgBuH,qBAAqB,EAAEiD;IAEnD,MAAMrB,mBAAmB,MAAM9I,GAC5BsD,MAAM,CAAC;QACN2C,IAAItG,gBAAgBsG,EAAE;QACtBG,eAAezG,gBAAgByG,aAAa;QAC5Cc,uBAAuBvH,gBAAgBuH,qBAAqB;QAC5DgB,iBAAiBvI,gBAAgBuI,eAAe;QAChDd,uBAAuBzH,gBAAgByH,qBAAqB;IAC9D,GACCzD,IAAI,CAAChE,iBACLiE,KAAK,CAACwG,oBACNxD,OAAO,CAACvI,IAAIsB,gBAAgBsG,EAAE;IACjC,MAAMoE,kBAA2C,EAAE;IACnD,KAAK,MAAMtB,aAAa;WAAID;KAAiB,CAAChG,IAAI,CAACkG,kBAAmB;QACpE,MAAM,CAAC1B,OAAO,GAAG,MAAMtH,GACpBsD,MAAM,CAAC;YACN2C,IAAItG,gBAAgBsG,EAAE;YACtBG,eAAezG,gBAAgByG,aAAa;YAC5Cc,uBAAuBvH,gBAAgBuH,qBAAqB;YAC5DgB,iBAAiBvI,gBAAgBuI,eAAe;YAChDd,uBAAuBzH,gBAAgByH,qBAAqB;QAC9D,GACCzD,IAAI,CAAChE,iBACLiE,KAAK,CAACzF,IAAIG,GAAGqB,gBAAgBsG,EAAE,EAAE8C,UAAU9C,EAAE,GAAGmE,qBAChDvD,GAAG,CAAC;QACP,IAAIS,QAAQ+C,gBAAgBrI,IAAI,CAACsF;IACnC;IACA,MAAMgD,kBAAkB,MAAMtK,GAC3BsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEM,QAAQe,aAAayG,EAAE,EAAEkE,MACzBtK,OAAOoK,WAAW,GAAGrL,GAAGY,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAAI8C,YAGjE+E,GAAG,CAAC;IACP,MAAMH,iBAAiB4D,gBAAgBtJ,GAAG,CAAC,CAACqB,cAAgBA,YAAY4D,EAAE;IAE1E,IAAIS,eAAezD,MAAM,KAAK,GAAG,OAAO;IAExC,MAAMsH,oBAAoBF,gBACvBzH,MAAM,CACL,CAAC0E,SACCA,OAAOlB,aAAa,IACpBM,eAAee,QAAQ,CAACH,OAAOlB,aAAa,KAC5C,CAACkB,OAAOY,eAAe,IACvB,CAACZ,OAAOF,qBAAqB,EAEhCpG,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;IAC5B,MAAMuE,oBAAoBH,gBACvBzH,MAAM,CACL,CAAC0E,SACCA,OAAOJ,qBAAqB,IAAIR,eAAee,QAAQ,CAACH,OAAOJ,qBAAqB,GAEvFlG,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;IAC5B,IAAIsE,kBAAkBtH,MAAM,GAAG,GAAG;QAChC,MAAMjD,GACHmJ,MAAM,CAAC1J,wBACPuI,GAAG,CAAC;YAACoB,WAAWpK,GAAG,CAAC,KAAK,CAAC;QAAA,GAC1B4E,KAAK,CACJzF,IACEM,QAAQgB,uBAAuB4I,gBAAgB,EAAEkC,oBACjD5L,OAAOc,uBAAuB4J,UAAU,GACxC1K,OAAOc,uBAAuB2J,SAAS;QAG7C,MAAMpJ,GACHmJ,MAAM,CAACxJ,iBACPqI,GAAG,CAAC;YACH5H,aAAa;YACbgG,eAAe;YACfmD,YAAY;YACZC,WAAWxK,GAAG,CAAC,KAAK,CAAC;QACvB,GACC4E,KAAK,CACJzF,IACEM,QAAQkB,gBAAgBsG,EAAE,EAAEsE,oBAC5B5L,OAAOgB,gBAAgBuI,eAAe,GACtCvJ,OAAOgB,gBAAgByH,qBAAqB;IAGpD;IACA,IAAIoD,kBAAkBvH,MAAM,GAAG,GAAG;QAChC,MAAMjD,GACHmJ,MAAM,CAACxJ,iBACPqI,GAAG,CAAC;YAACd,uBAAuB;YAAMsC,WAAWxK,GAAG,CAAC,KAAK,CAAC;QAAA,GACvD4E,KAAK,CACJzF,IACEM,QAAQkB,gBAAgBsG,EAAE,EAAEuE,oBAC5B/L,QAAQkB,gBAAgBuH,qBAAqB,EAAER;IAGvD;IAEA,MAAM+D,UAAU,MAAMzK,GACnB0K,MAAM,CAAClL,cACPoE,KAAK,CAACnF,QAAQe,aAAayG,EAAE,EAAES,iBAC/BV,SAAS,CAAC;QAACC,IAAIzG,aAAayG,EAAE;IAAA;IAEjC,OAAOwE,QAAQxH,MAAM;AACvB;AAEA,OAAO,eAAe0H,wBACpB3K,EAAM,EACNH,MAIC;IAED,IAAI+K,WAAW;IAEf,KAAK,MAAMC,WAAWhL,OAAOiL,QAAQ,CAAE;QACrC,IAAID,QAAQvI,KAAK,IAAI,GAAG;QAExB,MAAMyI,cAAc,MAAM/K,GACvBmJ,MAAM,CAAC3J,cACPwI,GAAG,CAAC;YAAC1F,OAAOtD,GAAG,CAAC,EAAEQ,aAAa8C,KAAK,CAAC,GAAG,EAAEuI,QAAQvI,KAAK,CAAC,CAAC;QAAA,GACzDsB,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAE4E,QAAQzE,aAAa,GACzC9H,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnDnD,GAAGgB,aAAa8C,KAAK,EAAEuI,QAAQvI,KAAK,GACpC9D,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAGxCgH,SAAS,CAAC;YAACC,IAAIzG,aAAayG,EAAE;QAAA;QAEjC,IAAI8E,YAAY9H,MAAM,GAAG,GAAG;YAC1B2H,YAAYC,QAAQvI,KAAK;YACzB;QACF;QAEA,MAAMmI,UAAU,MAAMzK,GACnB0K,MAAM,CAAClL,cACPoE,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAE4E,QAAQzE,aAAa,GACzC9H,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnD3C,GAAG,CAAC,EAAEQ,aAAa8C,KAAK,CAAC,IAAI,EAAEuI,QAAQvI,KAAK,CAAC,CAAC,EAC9C9D,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAGxCgH,SAAS,CAAC;YAAC1D,OAAO9C,aAAa8C,KAAK;QAAA;QAEvCsI,YAAYH,QAAQtI,MAAM,CAAC,CAACC,OAAO0B,MAAQ1B,QAAQ0B,IAAIxB,KAAK,EAAE;IAChE;IAEA,OAAOsI;AACT;AAEA,SAAS/F,8BACP9D,SAA+B,EAC/BiK,kBAAkE;IAElE,KAAK,MAAM3I,eAAe4I,oBAAoBD,oBAAqB;QACjE,MAAMrI,sBAAsB5B,UACzB6B,MAAM,CAAC,CAAC3B,WAAa4B,SAASR,YAAYkB,cAAc,EAAEtC,SAASC,MAAM,GACzE4B,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAE7B,MAAM,CAAC+B,MAAM,GAAGD,EAAE9B,MAAM,CAAC+B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBN,YAAYiC,QAAQ;IACrD;AACF;AAEA,SAAS0E,iBAAiBkC,IAAkB,EAAEC,KAAmB;IAC/D,OAAOD,KAAKjF,EAAE,GAAGkF,MAAMlF,EAAE,GAAG,CAAC,IAAIiF,KAAKjF,EAAE,GAAGkF,MAAMlF,EAAE,GAAG,IAAI;AAC5D;AAEA,SAASjB,eAAevD,IAAiB;IACvC,OAAO;WAAIA;KAAK,CAACqB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAMoI,cAAcpI,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAImI,gBAAgB,GAAG,OAAOA;QAC9B,OAAOrI,EAAEU,cAAc,CAAC4H,OAAO,KAAKrI,EAAES,cAAc,CAAC4H,OAAO;IAC9D;AACF;AAEA,SAASJ,oBAA0DxJ,IAAS;IAC1E,OAAO;WAAIA;KAAK,CAACqB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAMoI,cAAcpI,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAImI,gBAAgB,GAAG,OAAOA;QAC9B,OAAO1G,SAAS3B,EAAEQ,cAAc,EAAEJ,aAAa,CAACuB,SAAS1B,EAAEO,cAAc;IAC3E;AACF;AAEA,SAASV,SAASU,cAAwB,EAAE+H,eAAyB;IACnE,OAAO/H,eAAegI,KAAK,CAAC,CAACC,QAAUF,gBAAgB7D,QAAQ,CAAC+D;AAClE;AAEA,SAASpI,UAAUrC,SAA+B,EAAEuB,KAAa;IAC/D,IAAImJ,YAAYnJ;IAChB,KAAK,MAAMrB,YAAYF,UAAW;QAChC,IAAI0K,cAAc,GAAG;QACrB,MAAM/D,OAAOvC,KAAKI,GAAG,CAACtE,SAASE,cAAc,EAAEsK;QAC/CxK,SAASE,cAAc,IAAIuG;QAC3B+D,aAAa/D;IACf;AACF;AAEA,SAAShD,SAASxD,MAAgB;IAChC,OAAOwK,KAAKC,SAAS,CAACzK;AACxB"}