@shipfox/api-runners 12.3.0 → 12.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/.turbo/turbo-build.log +4 -4
  2. package/CHANGELOG.md +22 -0
  3. package/dist/config.d.ts +1 -0
  4. package/dist/config.d.ts.map +1 -1
  5. package/dist/config.js +9 -2
  6. package/dist/config.js.map +1 -1
  7. package/dist/core/demand.d.ts +4 -2
  8. package/dist/core/demand.d.ts.map +1 -1
  9. package/dist/core/demand.js +22 -6
  10. package/dist/core/demand.js.map +1 -1
  11. package/dist/core/entities/reservation.d.ts +1 -0
  12. package/dist/core/entities/reservation.d.ts.map +1 -1
  13. package/dist/core/entities/reservation.js.map +1 -1
  14. package/dist/core/entities/runner-instance.d.ts +3 -0
  15. package/dist/core/entities/runner-instance.d.ts.map +1 -1
  16. package/dist/core/entities/runner-instance.js.map +1 -1
  17. package/dist/core/runner-activation.d.ts +4 -1
  18. package/dist/core/runner-activation.d.ts.map +1 -1
  19. package/dist/core/runner-activation.js +32 -3
  20. package/dist/core/runner-activation.js.map +1 -1
  21. package/dist/core/runner-control-sessions.d.ts +8 -4
  22. package/dist/core/runner-control-sessions.d.ts.map +1 -1
  23. package/dist/core/runner-control-sessions.js +55 -9
  24. package/dist/core/runner-control-sessions.js.map +1 -1
  25. package/dist/core/runner-instances.d.ts +1 -0
  26. package/dist/core/runner-instances.d.ts.map +1 -1
  27. package/dist/core/runner-instances.js +1 -0
  28. package/dist/core/runner-instances.js.map +1 -1
  29. package/dist/db/db.d.ts +68 -0
  30. package/dist/db/db.d.ts.map +1 -1
  31. package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
  32. package/dist/db/ephemeral-registration-tokens.js +1 -1
  33. package/dist/db/ephemeral-registration-tokens.js.map +1 -1
  34. package/dist/db/reservation-locks.d.ts +6 -0
  35. package/dist/db/reservation-locks.d.ts.map +1 -0
  36. package/dist/db/reservation-locks.js +15 -0
  37. package/dist/db/reservation-locks.js.map +1 -0
  38. package/dist/db/reservations.d.ts +17 -3
  39. package/dist/db/reservations.d.ts.map +1 -1
  40. package/dist/db/reservations.js +246 -51
  41. package/dist/db/reservations.js.map +1 -1
  42. package/dist/db/runner-assignments.d.ts +17 -0
  43. package/dist/db/runner-assignments.d.ts.map +1 -1
  44. package/dist/db/runner-assignments.js +103 -8
  45. package/dist/db/runner-assignments.js.map +1 -1
  46. package/dist/db/runner-instances.d.ts +2 -2
  47. package/dist/db/runner-instances.d.ts.map +1 -1
  48. package/dist/db/runner-instances.js +124 -24
  49. package/dist/db/runner-instances.js.map +1 -1
  50. package/dist/db/runner-states.d.ts +2 -0
  51. package/dist/db/runner-states.d.ts.map +1 -0
  52. package/dist/db/runner-states.js +7 -0
  53. package/dist/db/runner-states.js.map +1 -0
  54. package/dist/db/schema/reservations.d.ts +18 -0
  55. package/dist/db/schema/reservations.d.ts.map +1 -1
  56. package/dist/db/schema/reservations.js +7 -1
  57. package/dist/db/schema/reservations.js.map +1 -1
  58. package/dist/db/schema/runner-instances.d.ts +18 -0
  59. package/dist/db/schema/runner-instances.d.ts.map +1 -1
  60. package/dist/db/schema/runner-instances.js +8 -0
  61. package/dist/db/schema/runner-instances.js.map +1 -1
  62. package/dist/metrics/index.d.ts +2 -2
  63. package/dist/metrics/index.d.ts.map +1 -1
  64. package/dist/metrics/index.js +1 -1
  65. package/dist/metrics/index.js.map +1 -1
  66. package/dist/metrics/instance.d.ts +24 -2
  67. package/dist/metrics/instance.d.ts.map +1 -1
  68. package/dist/metrics/instance.js +23 -0
  69. package/dist/metrics/instance.js.map +1 -1
  70. package/dist/presentation/dto/poll-demand.d.ts.map +1 -1
  71. package/dist/presentation/dto/poll-demand.js +3 -0
  72. package/dist/presentation/dto/poll-demand.js.map +1 -1
  73. package/dist/presentation/dto/runner-instances.d.ts +4 -1
  74. package/dist/presentation/dto/runner-instances.d.ts.map +1 -1
  75. package/dist/presentation/dto/runner-instances.js +4 -1
  76. package/dist/presentation/dto/runner-instances.js.map +1 -1
  77. package/dist/presentation/routes/assign-runner-instances.d.ts.map +1 -1
  78. package/dist/presentation/routes/assign-runner-instances.js +3 -3
  79. package/dist/presentation/routes/assign-runner-instances.js.map +1 -1
  80. package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
  81. package/dist/presentation/routes/poll-demand.js +1 -0
  82. package/dist/presentation/routes/poll-demand.js.map +1 -1
  83. package/dist/presentation/routes/reconcile-runner-instances.d.ts.map +1 -1
  84. package/dist/presentation/routes/reconcile-runner-instances.js +4 -2
  85. package/dist/presentation/routes/reconcile-runner-instances.js.map +1 -1
  86. package/dist/presentation/routes/runner-enrollment.d.ts +4 -1
  87. package/dist/presentation/routes/runner-enrollment.d.ts.map +1 -1
  88. package/dist/presentation/routes/runner-enrollment.js +11 -6
  89. package/dist/presentation/routes/runner-enrollment.js.map +1 -1
  90. package/dist/tsconfig.test.tsbuildinfo +1 -1
  91. package/drizzle/0005_previous_lenny_balinger.sql +5 -0
  92. package/drizzle/meta/0005_snapshot.json +2220 -0
  93. package/drizzle/meta/_journal.json +7 -0
  94. package/package.json +2 -2
  95. package/src/config.test.ts +22 -0
  96. package/src/config.ts +15 -2
  97. package/src/core/demand.test.ts +192 -3
  98. package/src/core/demand.ts +44 -4
  99. package/src/core/entities/reservation.ts +1 -0
  100. package/src/core/entities/runner-instance.ts +4 -0
  101. package/src/core/runner-activation.ts +48 -2
  102. package/src/core/runner-control-sessions.test.ts +64 -3
  103. package/src/core/runner-control-sessions.ts +62 -9
  104. package/src/core/runner-instances.test.ts +19 -0
  105. package/src/core/runner-instances.ts +2 -0
  106. package/src/core/runner-sessions.test.ts +121 -1
  107. package/src/db/ephemeral-registration-tokens.test.ts +20 -1
  108. package/src/db/ephemeral-registration-tokens.ts +1 -0
  109. package/src/db/reservation-locks.ts +22 -0
  110. package/src/db/reservations.test.ts +733 -22
  111. package/src/db/reservations.ts +408 -144
  112. package/src/db/runner-assignments.test.ts +105 -1
  113. package/src/db/runner-assignments.ts +178 -9
  114. package/src/db/runner-instances.test.ts +265 -2
  115. package/src/db/runner-instances.ts +215 -55
  116. package/src/db/runner-states.ts +7 -0
  117. package/src/db/schema/reservations.ts +5 -1
  118. package/src/db/schema/runner-instances.ts +9 -0
  119. package/src/metrics/index.ts +10 -1
  120. package/src/metrics/instance.ts +59 -2
  121. package/src/metrics/service.test.ts +8 -1
  122. package/src/presentation/dto/poll-demand.ts +3 -0
  123. package/src/presentation/dto/runner-instances.ts +8 -1
  124. package/src/presentation/routes/assign-runner-instances.ts +6 -2
  125. package/src/presentation/routes/late-runner-enrollment.test.ts +219 -2
  126. package/src/presentation/routes/poll-demand.test.ts +56 -0
  127. package/src/presentation/routes/poll-demand.ts +1 -0
  128. package/src/presentation/routes/reconcile-runner-instances.test.ts +34 -2
  129. package/src/presentation/routes/reconcile-runner-instances.ts +7 -1
  130. package/src/presentation/routes/runner-enrollment.test.ts +316 -3
  131. package/src/presentation/routes/runner-enrollment.ts +8 -5
  132. package/test/factories/reservation.ts +2 -0
  133. package/test/factories/runner-instance.ts +6 -0
  134. package/tsconfig.build.tsbuildinfo +1 -1
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/db/reservations.ts"],"sourcesContent":["import {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n and,\n arrayContains,\n asc,\n eq,\n exists,\n gt,\n inArray,\n isNotNull,\n isNull,\n lt,\n notExists,\n or,\n sql,\n} from 'drizzle-orm';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {provisionerCapabilitySnapshots} from './schema/provisioner-capability-snapshots.js';\nimport {provisionerTokens} from './schema/provisioner-tokens.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerActivationTokens} from './schema/runner-activation-tokens.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {providerRunners} from './schema/runner-instances.js';\n\nexport interface ReservationTemplate {\n templateKey: string;\n labels: string[];\n availableSlots: number;\n starting: number;\n running: number;\n}\n\nexport interface DemandStat {\n workspaceId?: string;\n labels: string[];\n queued: number;\n reserved: number;\n oldestQueuedAt: Date;\n}\n\nexport interface ReservationGrant {\n reservationId: string;\n workspaceId?: string;\n labels: string[];\n count: number;\n expiresAt: Date;\n}\n\nexport interface PollDemandAndReserveParams {\n workspaceId: string;\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds?: number;\n}\n\nexport interface InstallationPollDemandAndReserveParams {\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds: number;\n eligibleWorkspaceIds: ReadonlySet<string>;\n signal?: AbortSignal;\n onReservations?: (reservations: ReservationGrant[]) => void;\n}\n\ntype DemandScope = 'installation' | 'workspace';\n\ntype PollDemandAndReserveLockedParams = PollDemandAndReserveParams & {\n scope: DemandScope;\n};\n\ninterface NormalizedTemplate {\n templateKey: string;\n labels: string[];\n remainingSlots: number;\n}\n\ninterface DemandRow {\n requiredLabels: string[];\n queued: number;\n oldestQueuedAt: Date;\n}\n\nexport async function pollDemandAndReserve(\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n return await db().transaction(async (tx) => {\n return await pollDemandAndReserveTx(tx, params);\n });\n}\n\nexport async function pollDemandAndReserveTx(\n tx: Tx,\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId}))`);\n return await pollDemandAndReserveLockedTx(tx, {...params, scope: 'workspace'});\n}\n\nexport async function pollInstallationDemandAndReserve(\n params: InstallationPollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n const candidateWorkspaceIds = await listInstallationDemandWorkspaceIds(\n params.eligibleWorkspaceIds,\n );\n const results: Array<{stats: DemandStat[]; reservations: ReservationGrant[]}> = [];\n let remainingMaxReservations = params.maxReservations;\n const remainingTemplates = params.templates.map((template) => ({\n ...template,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n for (const workspaceId of candidateWorkspaceIds) {\n if (params.signal?.aborted || remainingMaxReservations === 0) break;\n const result = await db().transaction(async (tx) => {\n const lockResult = await tx.execute<{locked: boolean}>(\n sql`select pg_try_advisory_xact_lock(hashtext(${workspaceId})) as locked`,\n );\n const locked = lockResult.rows[0];\n if (!locked?.locked) return {stats: [], reservations: []};\n return await pollDemandAndReserveLockedTx(tx, {\n workspaceId,\n provisionerId: params.provisionerId,\n maxReservations: remainingMaxReservations,\n ttlSeconds: params.ttlSeconds,\n templates: remainingTemplates.map((template) => ({\n ...template,\n availableSlots: template.remainingSlots,\n })),\n capabilityWindowSeconds: params.capabilityWindowSeconds,\n scope: 'installation',\n });\n });\n results.push(result);\n consumeInstallationTemplateSlots(remainingTemplates, result.reservations);\n params.onReservations?.(result.reservations);\n remainingMaxReservations -= result.reservations.reduce(\n (total, reservation) => total + reservation.count,\n 0,\n );\n }\n return {\n stats: results.flatMap((result) => result.stats),\n reservations: results.flatMap((result) => result.reservations),\n };\n}\n\nfunction consumeInstallationTemplateSlots(\n templates: NormalizedTemplate[],\n reservations: ReservationGrant[],\n): void {\n for (const reservation of reservations) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.labels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.count);\n }\n}\n\nasync function pollDemandAndReserveLockedTx(\n tx: Tx,\n params: PollDemandAndReserveLockedParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n let demandRows = (\n await tx\n .select({\n requiredLabels: pendingJobExecutions.requiredLabels,\n queued: sql<number>`count(*)::int`,\n oldestQueuedAt: sql<Date | string>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(eq(pendingJobExecutions.workspaceId, params.workspaceId))\n .groupBy(pendingJobExecutions.requiredLabels)\n ).map((row) => ({\n ...row,\n oldestQueuedAt: new Date(row.oldestQueuedAt),\n }));\n if (params.capabilityWindowSeconds !== undefined) {\n const capabilityLabels = await listActiveWorkspaceCapabilityLabelsTx(tx, {\n workspaceId: params.workspaceId,\n windowSeconds: params.capabilityWindowSeconds,\n });\n demandRows = demandRows.filter(\n (demand) => !capabilityLabels.some((labels) => isSubset(demand.requiredLabels, labels)),\n );\n }\n\n const activeReservationRows = await tx\n .select({\n requiredLabels: reservations.requiredLabels,\n reserved: sql<number>`coalesce(sum(${reservations.count}), 0)::int`,\n })\n .from(reservations)\n .where(\n and(eq(reservations.workspaceId, params.workspaceId), gt(reservations.expiresAt, sql`now()`)),\n )\n .groupBy(reservations.requiredLabels);\n\n const reservedByLabels = new Map(\n activeReservationRows.map((row) => [labelKey(row.requiredLabels), row.reserved]),\n );\n const activeProvisionerReservationRows = await tx\n .select({\n requiredLabels: reservations.requiredLabels,\n reserved: sql<number>`coalesce(sum(${reservations.count}), 0)::int`,\n })\n .from(reservations)\n .where(\n and(\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .groupBy(reservations.requiredLabels);\n\n const templates = params.templates.map((template) => ({\n templateKey: template.templateKey,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n deductProvisionerReservations(templates, activeProvisionerReservationRows);\n const stats: DemandStat[] = [];\n const grants: ReservationGrant[] = [];\n let remainingMaxReservations = params.maxReservations;\n const workspaceIdField =\n params.capabilityWindowSeconds === undefined ? {} : {workspaceId: params.workspaceId};\n\n for (const demand of sortDemandRows(demandRows)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(demand.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n\n if (satisfyingTemplates.length === 0) continue;\n\n const reserved = reservedByLabels.get(labelKey(demand.requiredLabels)) ?? 0;\n const unreserved = Math.max(0, demand.queued - reserved);\n const capacity = satisfyingTemplates.reduce(\n (total, template) => total + template.remainingSlots,\n 0,\n );\n const grant = Math.min(unreserved, capacity, remainingMaxReservations);\n let reservedAfterGrant = reserved;\n\n if (grant > 0 && params.maxReservations > 0) {\n const [inserted] = await tx\n .insert(reservations)\n .values({\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: grant,\n expiresAt: sql`now() + (${params.ttlSeconds} || ' seconds')::interval`,\n })\n .returning({id: reservations.id, expiresAt: reservations.expiresAt});\n\n if (!inserted) throw new Error('Insert returned no rows');\n\n remainingMaxReservations -= grant;\n drawSlots(satisfyingTemplates, grant);\n await bindIdleRunnerInstancesTx(tx, {\n provisionerId: params.provisionerId,\n reservationId: inserted.id,\n requiredLabels: demand.requiredLabels,\n count: grant,\n workspaceId: params.workspaceId,\n scope: params.scope,\n });\n reservedAfterGrant += grant;\n grants.push({\n reservationId: inserted.id,\n ...workspaceIdField,\n labels: demand.requiredLabels,\n count: grant,\n expiresAt: inserted.expiresAt,\n });\n }\n\n stats.push({\n ...workspaceIdField,\n labels: demand.requiredLabels,\n queued: demand.queued,\n reserved: reservedAfterGrant,\n oldestQueuedAt: demand.oldestQueuedAt,\n });\n }\n\n return {stats, reservations: grants};\n}\n\nasync function bindIdleRunnerInstancesTx(\n tx: Tx,\n params: {\n provisionerId: string;\n reservationId: string;\n workspaceId: string;\n requiredLabels: string[];\n count: number;\n scope: DemandScope;\n },\n): Promise<void> {\n // Reservation expiry is the activation grace deadline. A live reservation\n // protects a runner that is still booting; an expired or missing row is stale.\n const canBindRunner = () =>\n and(\n or(\n and(isNull(providerRunners.workspaceId), isNull(providerRunners.reservationId)),\n and(\n isNotNull(providerRunners.reservationId),\n params.scope === 'installation'\n ? undefined\n : or(\n isNull(providerRunners.workspaceId),\n eq(providerRunners.workspaceId, params.workspaceId),\n ),\n notExists(\n tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.id, providerRunners.reservationId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n ),\n or(\n isNull(providerRunners.intendedReservationId),\n notExists(\n tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.id, providerRunners.intendedReservationId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n isNull(providerRunners.reservationReleasedAt),\n isNull(providerRunners.runnerSessionId),\n );\n\n const idleRunners = await tx\n .select({id: providerRunners.id})\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n canBindRunner(),\n isNotNull(providerRunners.providerRunnerId),\n eq(providerRunners.state, 'running'),\n arrayContains(providerRunners.labels, params.requiredLabels),\n exists(\n tx\n .select({id: runnerControlSessions.id})\n .from(runnerControlSessions)\n .where(\n and(\n eq(runnerControlSessions.runnerInstanceId, providerRunners.id),\n eq(runnerControlSessions.provisionerId, params.provisionerId),\n isNull(runnerControlSessions.closedAt),\n gt(runnerControlSessions.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n )\n .orderBy(asc(providerRunners.createdAt), asc(providerRunners.id))\n .limit(params.count)\n .for('update');\n\n if (idleRunners.length === 0) return;\n\n await tx\n .update(runnerActivationTokens)\n .set({revokedAt: sql`now()`})\n .where(\n and(\n inArray(\n runnerActivationTokens.runnerInstanceId,\n idleRunners.map((runner) => runner.id),\n ),\n isNull(runnerActivationTokens.consumedAt),\n isNull(runnerActivationTokens.revokedAt),\n ),\n );\n\n await tx\n .update(providerRunners)\n .set({\n workspaceId: params.workspaceId,\n reservationId: params.reservationId,\n intendedReservationId: null,\n assignedAt: sql`now()`,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n inArray(\n providerRunners.id,\n idleRunners.map((runner) => runner.id),\n ),\n canBindRunner(),\n ),\n );\n}\n\nasync function listInstallationDemandWorkspaceIds(eligibleWorkspaceIds: ReadonlySet<string>) {\n if (eligibleWorkspaceIds.size === 0) return [];\n const rows = await db()\n .select({\n workspaceId: pendingJobExecutions.workspaceId,\n oldestQueuedAt: sql<Date>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.workspaceId, [...eligibleWorkspaceIds]))\n .groupBy(pendingJobExecutions.workspaceId)\n .orderBy(\n asc(sql`min(${pendingJobExecutions.createdAt})`),\n asc(pendingJobExecutions.workspaceId),\n );\n return rows.map((row) => row.workspaceId);\n}\n\nexport async function listQueuedDemandWorkspaceIds(): Promise<string[]> {\n const rows = await db()\n .select({workspaceId: pendingJobExecutions.workspaceId})\n .from(pendingJobExecutions)\n .groupBy(pendingJobExecutions.workspaceId);\n return rows.map((row) => row.workspaceId);\n}\n\nasync function listActiveWorkspaceCapabilityLabelsTx(\n tx: Tx,\n params: {workspaceId: string; windowSeconds: number},\n): Promise<string[][]> {\n const rows = await tx\n .select({labels: provisionerCapabilitySnapshots.labels})\n .from(provisionerCapabilitySnapshots)\n .innerJoin(\n provisionerTokens,\n eq(provisionerTokens.id, provisionerCapabilitySnapshots.provisionerId),\n )\n .where(\n and(\n eq(provisionerCapabilitySnapshots.workspaceId, params.workspaceId),\n eq(provisionerTokens.workspaceId, params.workspaceId),\n eq(provisionerTokens.scope, 'workspace'),\n sql`${provisionerCapabilitySnapshots.advertisedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n sql`${provisionerTokens.revokedAt} is null`,\n sql`(${provisionerTokens.expiresAt} is null or ${provisionerTokens.expiresAt} > now())`,\n ),\n );\n return rows.map((row) => row.labels);\n}\n\nexport async function deleteExpiredReservations(params?: {limit?: number}): Promise<number> {\n return await db().transaction(async (tx) => {\n const expiredRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(lt(reservations.expiresAt, sql`now()`))\n .orderBy(asc(reservations.expiresAt))\n .limit(params?.limit ?? 1000);\n return await deleteReservationsWithCleanupTx(\n tx,\n expiredRows.map((reservation) => reservation.id),\n {expiredOnly: true},\n );\n });\n}\n\nexport async function deleteReservationsByIds(ids: string[]): Promise<number> {\n if (ids.length === 0) return 0;\n\n return await db().transaction(async (tx) => {\n return await deleteReservationsWithCleanupTx(tx, ids);\n });\n}\n\nasync function deleteReservationsWithCleanupTx(\n tx: Tx,\n ids: string[],\n params: {expiredOnly?: boolean} = {},\n): Promise<number> {\n if (ids.length === 0) return 0;\n\n const affectedRunners = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n runnerSessionId: providerRunners.runnerSessionId,\n reservationReleasedAt: providerRunners.reservationReleasedAt,\n })\n .from(providerRunners)\n .where(\n or(\n and(\n inArray(providerRunners.reservationId, ids),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n inArray(providerRunners.intendedReservationId, ids),\n ),\n )\n .orderBy(asc(providerRunners.id))\n .for('update');\n const reservationRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n inArray(reservations.id, ids),\n params.expiredOnly ? lt(reservations.expiresAt, sql`now()`) : undefined,\n ),\n )\n .for('update');\n const reservationIds = reservationRows.map((reservation) => reservation.id);\n\n if (reservationIds.length === 0) return 0;\n\n const assignedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.reservationId &&\n reservationIds.includes(runner.reservationId) &&\n !runner.runnerSessionId &&\n !runner.reservationReleasedAt,\n )\n .map((runner) => runner.id);\n const intendedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.intendedReservationId && reservationIds.includes(runner.intendedReservationId),\n )\n .map((runner) => runner.id);\n if (assignedRunnerIds.length > 0) {\n await tx\n .update(runnerActivationTokens)\n .set({revokedAt: sql`now()`})\n .where(\n and(\n inArray(runnerActivationTokens.runnerInstanceId, assignedRunnerIds),\n isNull(runnerActivationTokens.consumedAt),\n isNull(runnerActivationTokens.revokedAt),\n ),\n );\n await tx\n .update(providerRunners)\n .set({\n workspaceId: null,\n reservationId: null,\n assignedAt: null,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n inArray(providerRunners.id, assignedRunnerIds),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n );\n }\n if (intendedRunnerIds.length > 0) {\n await tx\n .update(providerRunners)\n .set({intendedReservationId: null, updatedAt: sql`now()`})\n .where(\n and(\n inArray(providerRunners.id, intendedRunnerIds),\n inArray(providerRunners.intendedReservationId, reservationIds),\n ),\n );\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(inArray(reservations.id, reservationIds))\n .returning({id: reservations.id});\n\n return deleted.length;\n}\n\nexport async function releaseReservationUnits(\n tx: Tx,\n params: {\n workspaceId: string;\n provisionerId: string;\n releases: Array<{reservationId: string; count: number}>;\n },\n): Promise<number> {\n let released = 0;\n\n for (const release of params.releases) {\n if (release.count <= 0) continue;\n\n const decremented = await tx\n .update(reservations)\n .set({count: sql`${reservations.count} - ${release.count}`})\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.count, release.count),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({id: reservations.id});\n\n if (decremented.length > 0) {\n released += release.count;\n continue;\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n sql`${reservations.count} <= ${release.count}`,\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({count: reservations.count});\n\n released += deleted.reduce((total, row) => total + row.count, 0);\n }\n\n return released;\n}\n\nfunction deductProvisionerReservations(\n templates: NormalizedTemplate[],\n activeReservations: {requiredLabels: string[]; reserved: number}[],\n): void {\n for (const reservation of sortReservationRows(activeReservations)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.reserved);\n }\n}\n\nfunction sortDemandRows(rows: DemandRow[]): DemandRow[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return a.oldestQueuedAt.getTime() - b.oldestQueuedAt.getTime();\n });\n}\n\nfunction sortReservationRows<T extends {requiredLabels: string[]}>(rows: T[]): T[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return labelKey(a.requiredLabels).localeCompare(labelKey(b.requiredLabels));\n });\n}\n\nfunction isSubset(requiredLabels: string[], availableLabels: string[]): boolean {\n return requiredLabels.every((label) => availableLabels.includes(label));\n}\n\nfunction drawSlots(templates: NormalizedTemplate[], count: number): void {\n let remaining = count;\n for (const template of templates) {\n if (remaining === 0) return;\n const used = Math.min(template.remainingSlots, remaining);\n template.remainingSlots -= used;\n remaining -= used;\n }\n}\n\nfunction labelKey(labels: string[]): string {\n return JSON.stringify(labels);\n}\n"],"names":["canonicalizeLabels","and","arrayContains","asc","eq","exists","gt","inArray","isNotNull","isNull","lt","notExists","or","sql","db","pendingJobExecutions","provisionerCapabilitySnapshots","provisionerTokens","reservations","runnerActivationTokens","runnerControlSessions","providerRunners","pollDemandAndReserve","params","transaction","tx","pollDemandAndReserveTx","execute","workspaceId","pollDemandAndReserveLockedTx","scope","pollInstallationDemandAndReserve","candidateWorkspaceIds","listInstallationDemandWorkspaceIds","eligibleWorkspaceIds","results","remainingMaxReservations","maxReservations","remainingTemplates","templates","map","template","labels","remainingSlots","availableSlots","signal","aborted","result","lockResult","locked","rows","stats","provisionerId","ttlSeconds","capabilityWindowSeconds","push","consumeInstallationTemplateSlots","onReservations","reduce","total","reservation","count","flatMap","satisfyingTemplates","filter","isSubset","sort","a","b","length","templateKey","localeCompare","drawSlots","demandRows","select","requiredLabels","queued","oldestQueuedAt","createdAt","from","where","groupBy","row","Date","undefined","capabilityLabels","listActiveWorkspaceCapabilityLabelsTx","windowSeconds","demand","some","activeReservationRows","reserved","expiresAt","reservedByLabels","Map","labelKey","activeProvisionerReservationRows","deductProvisionerReservations","grants","workspaceIdField","sortDemandRows","get","unreserved","Math","max","capacity","grant","min","reservedAfterGrant","inserted","insert","values","returning","id","Error","bindIdleRunnerInstancesTx","reservationId","canBindRunner","intendedReservationId","reservationReleasedAt","runnerSessionId","idleRunners","providerRunnerId","state","runnerInstanceId","closedAt","orderBy","limit","for","update","set","revokedAt","runner","consumedAt","assignedAt","updatedAt","size","listQueuedDemandWorkspaceIds","innerJoin","advertisedAt","deleteExpiredReservations","expiredRows","deleteReservationsWithCleanupTx","expiredOnly","deleteReservationsByIds","ids","affectedRunners","reservationRows","reservationIds","assignedRunnerIds","includes","intendedRunnerIds","deleted","delete","releaseReservationUnits","released","release","releases","decremented","activeReservations","sortReservationRows","specificity","getTime","availableLabels","every","label","remaining","used","JSON","stringify"],"mappings":"AAAA,SAAQA,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,aAAa,EACbC,GAAG,EACHC,EAAE,EACFC,MAAM,EACNC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,SAAS,EACTC,EAAE,EACFC,GAAG,QACE,cAAc;AAErB,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,8BAA8B,QAAO,+CAA+C;AAC5F,SAAQC,iBAAiB,QAAO,iCAAiC;AACjE,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,sBAAsB,QAAO,uCAAuC;AAC5E,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAAQC,eAAe,QAAO,+BAA+B;AAgE7D,OAAO,eAAeC,qBACpBC,MAAkC;IAElC,OAAO,MAAMT,KAAKU,WAAW,CAAC,OAAOC;QACnC,OAAO,MAAMC,uBAAuBD,IAAIF;IAC1C;AACF;AAEA,OAAO,eAAeG,uBACpBD,EAAM,EACNF,MAAkC;IAElC,MAAME,GAAGE,OAAO,CAACd,GAAG,CAAC,sCAAsC,EAAEU,OAAOK,WAAW,CAAC,EAAE,CAAC;IACnF,OAAO,MAAMC,6BAA6BJ,IAAI;QAAC,GAAGF,MAAM;QAAEO,OAAO;IAAW;AAC9E;AAEA,OAAO,eAAeC,iCACpBR,MAA8C;IAE9C,MAAMS,wBAAwB,MAAMC,mCAClCV,OAAOW,oBAAoB;IAE7B,MAAMC,UAA0E,EAAE;IAClF,IAAIC,2BAA2Bb,OAAOc,eAAe;IACrD,MAAMC,qBAAqBf,OAAOgB,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YAC7D,GAAGA,QAAQ;YACXC,QAAQ;mBAAI1C,mBAAmByC,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACA,KAAK,MAAMhB,eAAeI,sBAAuB;QAC/C,IAAIT,OAAOsB,MAAM,EAAEC,WAAWV,6BAA6B,GAAG;QAC9D,MAAMW,SAAS,MAAMjC,KAAKU,WAAW,CAAC,OAAOC;YAC3C,MAAMuB,aAAa,MAAMvB,GAAGE,OAAO,CACjCd,GAAG,CAAC,0CAA0C,EAAEe,YAAY,YAAY,CAAC;YAE3E,MAAMqB,SAASD,WAAWE,IAAI,CAAC,EAAE;YACjC,IAAI,CAACD,QAAQA,QAAQ,OAAO;gBAACE,OAAO,EAAE;gBAAEjC,cAAc,EAAE;YAAA;YACxD,OAAO,MAAMW,6BAA6BJ,IAAI;gBAC5CG;gBACAwB,eAAe7B,OAAO6B,aAAa;gBACnCf,iBAAiBD;gBACjBiB,YAAY9B,OAAO8B,UAAU;gBAC7Bd,WAAWD,mBAAmBE,GAAG,CAAC,CAACC,WAAc,CAAA;wBAC/C,GAAGA,QAAQ;wBACXG,gBAAgBH,SAASE,cAAc;oBACzC,CAAA;gBACAW,yBAAyB/B,OAAO+B,uBAAuB;gBACvDxB,OAAO;YACT;QACF;QACAK,QAAQoB,IAAI,CAACR;QACbS,iCAAiClB,oBAAoBS,OAAO7B,YAAY;QACxEK,OAAOkC,cAAc,GAAGV,OAAO7B,YAAY;QAC3CkB,4BAA4BW,OAAO7B,YAAY,CAACwC,MAAM,CACpD,CAACC,OAAOC,cAAgBD,QAAQC,YAAYC,KAAK,EACjD;IAEJ;IACA,OAAO;QACLV,OAAOhB,QAAQ2B,OAAO,CAAC,CAACf,SAAWA,OAAOI,KAAK;QAC/CjC,cAAciB,QAAQ2B,OAAO,CAAC,CAACf,SAAWA,OAAO7B,YAAY;IAC/D;AACF;AAEA,SAASsC,iCACPjB,SAA+B,EAC/BrB,YAAgC;IAEhC,KAAK,MAAM0C,eAAe1C,aAAc;QACtC,MAAM6C,sBAAsBxB,UACzByB,MAAM,CAAC,CAACvB,WAAawB,SAASL,YAAYlB,MAAM,EAAED,SAASC,MAAM,GACjEwB,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAEzB,MAAM,CAAC2B,MAAM,GAAGD,EAAE1B,MAAM,CAAC2B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBH,YAAYC,KAAK;IAClD;AACF;AAEA,eAAehC,6BACbJ,EAAM,EACNF,MAAwC;IAExC,IAAIkD,aAAa,AACf,CAAA,MAAMhD,GACHiD,MAAM,CAAC;QACNC,gBAAgB5D,qBAAqB4D,cAAc;QACnDC,QAAQ/D,GAAW,CAAC,aAAa,CAAC;QAClCgE,gBAAgBhE,GAAkB,CAAC,IAAI,EAAEE,qBAAqB+D,SAAS,CAAC,CAAC,CAAC;IAC5E,GACCC,IAAI,CAAChE,sBACLiE,KAAK,CAAC5E,GAAGW,qBAAqBa,WAAW,EAAEL,OAAOK,WAAW,GAC7DqD,OAAO,CAAClE,qBAAqB4D,cAAc,CAAA,EAC9CnC,GAAG,CAAC,CAAC0C,MAAS,CAAA;YACd,GAAGA,GAAG;YACNL,gBAAgB,IAAIM,KAAKD,IAAIL,cAAc;QAC7C,CAAA;IACA,IAAItD,OAAO+B,uBAAuB,KAAK8B,WAAW;QAChD,MAAMC,mBAAmB,MAAMC,sCAAsC7D,IAAI;YACvEG,aAAaL,OAAOK,WAAW;YAC/B2D,eAAehE,OAAO+B,uBAAuB;QAC/C;QACAmB,aAAaA,WAAWT,MAAM,CAC5B,CAACwB,SAAW,CAACH,iBAAiBI,IAAI,CAAC,CAAC/C,SAAWuB,SAASuB,OAAOb,cAAc,EAAEjC;IAEnF;IAEA,MAAMgD,wBAAwB,MAAMjE,GACjCiD,MAAM,CAAC;QACNC,gBAAgBzD,aAAayD,cAAc;QAC3CgB,UAAU9E,GAAW,CAAC,aAAa,EAAEK,aAAa2C,KAAK,CAAC,UAAU,CAAC;IACrE,GACCkB,IAAI,CAAC7D,cACL8D,KAAK,CACJ/E,IAAIG,GAAGc,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAAGtB,GAAGY,aAAa0E,SAAS,EAAE/E,GAAG,CAAC,KAAK,CAAC,IAE5FoE,OAAO,CAAC/D,aAAayD,cAAc;IAEtC,MAAMkB,mBAAmB,IAAIC,IAC3BJ,sBAAsBlD,GAAG,CAAC,CAAC0C,MAAQ;YAACa,SAASb,IAAIP,cAAc;YAAGO,IAAIS,QAAQ;SAAC;IAEjF,MAAMK,mCAAmC,MAAMvE,GAC5CiD,MAAM,CAAC;QACNC,gBAAgBzD,aAAayD,cAAc;QAC3CgB,UAAU9E,GAAW,CAAC,aAAa,EAAEK,aAAa2C,KAAK,CAAC,UAAU,CAAC;IACrE,GACCkB,IAAI,CAAC7D,cACL8D,KAAK,CACJ/E,IACEG,GAAGc,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAC/CxB,GAAGc,aAAakC,aAAa,EAAE7B,OAAO6B,aAAa,GACnD9C,GAAGY,aAAa0E,SAAS,EAAE/E,GAAG,CAAC,KAAK,CAAC,IAGxCoE,OAAO,CAAC/D,aAAayD,cAAc;IAEtC,MAAMpC,YAAYhB,OAAOgB,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YACpD6B,aAAa7B,SAAS6B,WAAW;YACjC5B,QAAQ;mBAAI1C,mBAAmByC,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACAqD,8BAA8B1D,WAAWyD;IACzC,MAAM7C,QAAsB,EAAE;IAC9B,MAAM+C,SAA6B,EAAE;IACrC,IAAI9D,2BAA2Bb,OAAOc,eAAe;IACrD,MAAM8D,mBACJ5E,OAAO+B,uBAAuB,KAAK8B,YAAY,CAAC,IAAI;QAACxD,aAAaL,OAAOK,WAAW;IAAA;IAEtF,KAAK,MAAM4D,UAAUY,eAAe3B,YAAa;QAC/C,MAAMV,sBAAsBxB,UACzByB,MAAM,CAAC,CAACvB,WAAawB,SAASuB,OAAOb,cAAc,EAAElC,SAASC,MAAM,GACpEwB,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAEzB,MAAM,CAAC2B,MAAM,GAAGD,EAAE1B,MAAM,CAAC2B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAG5F,IAAIP,oBAAoBM,MAAM,KAAK,GAAG;QAEtC,MAAMsB,WAAWE,iBAAiBQ,GAAG,CAACN,SAASP,OAAOb,cAAc,MAAM;QAC1E,MAAM2B,aAAaC,KAAKC,GAAG,CAAC,GAAGhB,OAAOZ,MAAM,GAAGe;QAC/C,MAAMc,WAAW1C,oBAAoBL,MAAM,CACzC,CAACC,OAAOlB,WAAakB,QAAQlB,SAASE,cAAc,EACpD;QAEF,MAAM+D,QAAQH,KAAKI,GAAG,CAACL,YAAYG,UAAUrE;QAC7C,IAAIwE,qBAAqBjB;QAEzB,IAAIe,QAAQ,KAAKnF,OAAOc,eAAe,GAAG,GAAG;YAC3C,MAAM,CAACwE,SAAS,GAAG,MAAMpF,GACtBqF,MAAM,CAAC5F,cACP6F,MAAM,CAAC;gBACNnF,aAAaL,OAAOK,WAAW;gBAC/BwB,eAAe7B,OAAO6B,aAAa;gBACnCuB,gBAAgBa,OAAOb,cAAc;gBACrCd,OAAO6C;gBACPd,WAAW/E,GAAG,CAAC,SAAS,EAAEU,OAAO8B,UAAU,CAAC,yBAAyB,CAAC;YACxE,GACC2D,SAAS,CAAC;gBAACC,IAAI/F,aAAa+F,EAAE;gBAAErB,WAAW1E,aAAa0E,SAAS;YAAA;YAEpE,IAAI,CAACiB,UAAU,MAAM,IAAIK,MAAM;YAE/B9E,4BAA4BsE;YAC5BlC,UAAUT,qBAAqB2C;YAC/B,MAAMS,0BAA0B1F,IAAI;gBAClC2B,eAAe7B,OAAO6B,aAAa;gBACnCgE,eAAeP,SAASI,EAAE;gBAC1BtC,gBAAgBa,OAAOb,cAAc;gBACrCd,OAAO6C;gBACP9E,aAAaL,OAAOK,WAAW;gBAC/BE,OAAOP,OAAOO,KAAK;YACrB;YACA8E,sBAAsBF;YACtBR,OAAO3C,IAAI,CAAC;gBACV6D,eAAeP,SAASI,EAAE;gBAC1B,GAAGd,gBAAgB;gBACnBzD,QAAQ8C,OAAOb,cAAc;gBAC7Bd,OAAO6C;gBACPd,WAAWiB,SAASjB,SAAS;YAC/B;QACF;QAEAzC,MAAMI,IAAI,CAAC;YACT,GAAG4C,gBAAgB;YACnBzD,QAAQ8C,OAAOb,cAAc;YAC7BC,QAAQY,OAAOZ,MAAM;YACrBe,UAAUiB;YACV/B,gBAAgBW,OAAOX,cAAc;QACvC;IACF;IAEA,OAAO;QAAC1B;QAAOjC,cAAcgF;IAAM;AACrC;AAEA,eAAeiB,0BACb1F,EAAM,EACNF,MAOC;IAED,0EAA0E;IAC1E,+EAA+E;IAC/E,MAAM8F,gBAAgB,IACpBpH,IACEW,GACEX,IAAIQ,OAAOY,gBAAgBO,WAAW,GAAGnB,OAAOY,gBAAgB+F,aAAa,IAC7EnH,IACEO,UAAUa,gBAAgB+F,aAAa,GACvC7F,OAAOO,KAAK,KAAK,iBACbsD,YACAxE,GACEH,OAAOY,gBAAgBO,WAAW,GAClCxB,GAAGiB,gBAAgBO,WAAW,EAAEL,OAAOK,WAAW,IAExDjB,UACEc,GACGiD,MAAM,CAAC;YAACuC,IAAI/F,aAAa+F,EAAE;QAAA,GAC3BlC,IAAI,CAAC7D,cACL8D,KAAK,CACJ/E,IACEG,GAAGc,aAAa+F,EAAE,EAAE5F,gBAAgB+F,aAAa,GACjD9G,GAAGY,aAAa0E,SAAS,EAAE/E,GAAG,CAAC,KAAK,CAAC,QAMjDD,GACEH,OAAOY,gBAAgBiG,qBAAqB,GAC5C3G,UACEc,GACGiD,MAAM,CAAC;YAACuC,IAAI/F,aAAa+F,EAAE;QAAA,GAC3BlC,IAAI,CAAC7D,cACL8D,KAAK,CACJ/E,IACEG,GAAGc,aAAa+F,EAAE,EAAE5F,gBAAgBiG,qBAAqB,GACzDhH,GAAGY,aAAa0E,SAAS,EAAE/E,GAAG,CAAC,KAAK,CAAC,OAK/CJ,OAAOY,gBAAgBkG,qBAAqB,GAC5C9G,OAAOY,gBAAgBmG,eAAe;IAG1C,MAAMC,cAAc,MAAMhG,GACvBiD,MAAM,CAAC;QAACuC,IAAI5F,gBAAgB4F,EAAE;IAAA,GAC9BlC,IAAI,CAAC1D,iBACL2D,KAAK,CACJ/E,IACEG,GAAGiB,gBAAgB+B,aAAa,EAAE7B,OAAO6B,aAAa,GACtDiE,iBACA7G,UAAUa,gBAAgBqG,gBAAgB,GAC1CtH,GAAGiB,gBAAgBsG,KAAK,EAAE,YAC1BzH,cAAcmB,gBAAgBqB,MAAM,EAAEnB,OAAOoD,cAAc,GAC3DtE,OACEoB,GACGiD,MAAM,CAAC;QAACuC,IAAI7F,sBAAsB6F,EAAE;IAAA,GACpClC,IAAI,CAAC3D,uBACL4D,KAAK,CACJ/E,IACEG,GAAGgB,sBAAsBwG,gBAAgB,EAAEvG,gBAAgB4F,EAAE,GAC7D7G,GAAGgB,sBAAsBgC,aAAa,EAAE7B,OAAO6B,aAAa,GAC5D3C,OAAOW,sBAAsByG,QAAQ,GACrCvH,GAAGc,sBAAsBwE,SAAS,EAAE/E,GAAG,CAAC,KAAK,CAAC,OAMzDiH,OAAO,CAAC3H,IAAIkB,gBAAgByD,SAAS,GAAG3E,IAAIkB,gBAAgB4F,EAAE,GAC9Dc,KAAK,CAACxG,OAAOsC,KAAK,EAClBmE,GAAG,CAAC;IAEP,IAAIP,YAAYpD,MAAM,KAAK,GAAG;IAE9B,MAAM5C,GACHwG,MAAM,CAAC9G,wBACP+G,GAAG,CAAC;QAACC,WAAWtH,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1BmE,KAAK,CACJ/E,IACEM,QACEY,uBAAuByG,gBAAgB,EACvCH,YAAYjF,GAAG,CAAC,CAAC4F,SAAWA,OAAOnB,EAAE,IAEvCxG,OAAOU,uBAAuBkH,UAAU,GACxC5H,OAAOU,uBAAuBgH,SAAS;IAI7C,MAAM1G,GACHwG,MAAM,CAAC5G,iBACP6G,GAAG,CAAC;QACHtG,aAAaL,OAAOK,WAAW;QAC/BwF,eAAe7F,OAAO6F,aAAa;QACnCE,uBAAuB;QACvBgB,YAAYzH,GAAG,CAAC,KAAK,CAAC;QACtB0H,WAAW1H,GAAG,CAAC,KAAK,CAAC;IACvB,GACCmE,KAAK,CACJ/E,IACEM,QACEc,gBAAgB4F,EAAE,EAClBQ,YAAYjF,GAAG,CAAC,CAAC4F,SAAWA,OAAOnB,EAAE,IAEvCI;AAGR;AAEA,eAAepF,mCAAmCC,oBAAyC;IACzF,IAAIA,qBAAqBsG,IAAI,KAAK,GAAG,OAAO,EAAE;IAC9C,MAAMtF,OAAO,MAAMpC,KAChB4D,MAAM,CAAC;QACN9C,aAAab,qBAAqBa,WAAW;QAC7CiD,gBAAgBhE,GAAS,CAAC,IAAI,EAAEE,qBAAqB+D,SAAS,CAAC,CAAC,CAAC;IACnE,GACCC,IAAI,CAAChE,sBACLiE,KAAK,CAACzE,QAAQQ,qBAAqBa,WAAW,EAAE;WAAIM;KAAqB,GACzE+C,OAAO,CAAClE,qBAAqBa,WAAW,EACxCkG,OAAO,CACN3H,IAAIU,GAAG,CAAC,IAAI,EAAEE,qBAAqB+D,SAAS,CAAC,CAAC,CAAC,GAC/C3E,IAAIY,qBAAqBa,WAAW;IAExC,OAAOsB,KAAKV,GAAG,CAAC,CAAC0C,MAAQA,IAAItD,WAAW;AAC1C;AAEA,OAAO,eAAe6G;IACpB,MAAMvF,OAAO,MAAMpC,KAChB4D,MAAM,CAAC;QAAC9C,aAAab,qBAAqBa,WAAW;IAAA,GACrDmD,IAAI,CAAChE,sBACLkE,OAAO,CAAClE,qBAAqBa,WAAW;IAC3C,OAAOsB,KAAKV,GAAG,CAAC,CAAC0C,MAAQA,IAAItD,WAAW;AAC1C;AAEA,eAAe0D,sCACb7D,EAAM,EACNF,MAAoD;IAEpD,MAAM2B,OAAO,MAAMzB,GAChBiD,MAAM,CAAC;QAAChC,QAAQ1B,+BAA+B0B,MAAM;IAAA,GACrDqC,IAAI,CAAC/D,gCACL0H,SAAS,CACRzH,mBACAb,GAAGa,kBAAkBgG,EAAE,EAAEjG,+BAA+BoC,aAAa,GAEtE4B,KAAK,CACJ/E,IACEG,GAAGY,+BAA+BY,WAAW,EAAEL,OAAOK,WAAW,GACjExB,GAAGa,kBAAkBW,WAAW,EAAEL,OAAOK,WAAW,GACpDxB,GAAGa,kBAAkBa,KAAK,EAAE,cAC5BjB,GAAG,CAAC,EAAEG,+BAA+B2H,YAAY,CAAC,YAAY,EAAEpH,OAAOgE,aAAa,CAAC,yBAAyB,CAAC,EAC/G1E,GAAG,CAAC,EAAEI,kBAAkBkH,SAAS,CAAC,QAAQ,CAAC,EAC3CtH,GAAG,CAAC,CAAC,EAAEI,kBAAkB2E,SAAS,CAAC,YAAY,EAAE3E,kBAAkB2E,SAAS,CAAC,SAAS,CAAC;IAG7F,OAAO1C,KAAKV,GAAG,CAAC,CAAC0C,MAAQA,IAAIxC,MAAM;AACrC;AAEA,OAAO,eAAekG,0BAA0BrH,MAAyB;IACvE,OAAO,MAAMT,KAAKU,WAAW,CAAC,OAAOC;QACnC,MAAMoH,cAAc,MAAMpH,GACvBiD,MAAM,CAAC;YAACuC,IAAI/F,aAAa+F,EAAE;QAAA,GAC3BlC,IAAI,CAAC7D,cACL8D,KAAK,CAACtE,GAAGQ,aAAa0E,SAAS,EAAE/E,GAAG,CAAC,KAAK,CAAC,GAC3CiH,OAAO,CAAC3H,IAAIe,aAAa0E,SAAS,GAClCmC,KAAK,CAACxG,QAAQwG,SAAS;QAC1B,OAAO,MAAMe,gCACXrH,IACAoH,YAAYrG,GAAG,CAAC,CAACoB,cAAgBA,YAAYqD,EAAE,GAC/C;YAAC8B,aAAa;QAAI;IAEtB;AACF;AAEA,OAAO,eAAeC,wBAAwBC,GAAa;IACzD,IAAIA,IAAI5E,MAAM,KAAK,GAAG,OAAO;IAE7B,OAAO,MAAMvD,KAAKU,WAAW,CAAC,OAAOC;QACnC,OAAO,MAAMqH,gCAAgCrH,IAAIwH;IACnD;AACF;AAEA,eAAeH,gCACbrH,EAAM,EACNwH,GAAa,EACb1H,SAAkC,CAAC,CAAC;IAEpC,IAAI0H,IAAI5E,MAAM,KAAK,GAAG,OAAO;IAE7B,MAAM6E,kBAAkB,MAAMzH,GAC3BiD,MAAM,CAAC;QACNuC,IAAI5F,gBAAgB4F,EAAE;QACtBG,eAAe/F,gBAAgB+F,aAAa;QAC5CE,uBAAuBjG,gBAAgBiG,qBAAqB;QAC5DE,iBAAiBnG,gBAAgBmG,eAAe;QAChDD,uBAAuBlG,gBAAgBkG,qBAAqB;IAC9D,GACCxC,IAAI,CAAC1D,iBACL2D,KAAK,CACJpE,GACEX,IACEM,QAAQc,gBAAgB+F,aAAa,EAAE6B,MACvCxI,OAAOY,gBAAgBmG,eAAe,GACtC/G,OAAOY,gBAAgBkG,qBAAqB,IAE9ChH,QAAQc,gBAAgBiG,qBAAqB,EAAE2B,OAGlDnB,OAAO,CAAC3H,IAAIkB,gBAAgB4F,EAAE,GAC9Be,GAAG,CAAC;IACP,MAAMmB,kBAAkB,MAAM1H,GAC3BiD,MAAM,CAAC;QAACuC,IAAI/F,aAAa+F,EAAE;IAAA,GAC3BlC,IAAI,CAAC7D,cACL8D,KAAK,CACJ/E,IACEM,QAAQW,aAAa+F,EAAE,EAAEgC,MACzB1H,OAAOwH,WAAW,GAAGrI,GAAGQ,aAAa0E,SAAS,EAAE/E,GAAG,CAAC,KAAK,CAAC,IAAIuE,YAGjE4C,GAAG,CAAC;IACP,MAAMoB,iBAAiBD,gBAAgB3G,GAAG,CAAC,CAACoB,cAAgBA,YAAYqD,EAAE;IAE1E,IAAImC,eAAe/E,MAAM,KAAK,GAAG,OAAO;IAExC,MAAMgF,oBAAoBH,gBACvBlF,MAAM,CACL,CAACoE,SACCA,OAAOhB,aAAa,IACpBgC,eAAeE,QAAQ,CAAClB,OAAOhB,aAAa,KAC5C,CAACgB,OAAOZ,eAAe,IACvB,CAACY,OAAOb,qBAAqB,EAEhC/E,GAAG,CAAC,CAAC4F,SAAWA,OAAOnB,EAAE;IAC5B,MAAMsC,oBAAoBL,gBACvBlF,MAAM,CACL,CAACoE,SACCA,OAAOd,qBAAqB,IAAI8B,eAAeE,QAAQ,CAAClB,OAAOd,qBAAqB,GAEvF9E,GAAG,CAAC,CAAC4F,SAAWA,OAAOnB,EAAE;IAC5B,IAAIoC,kBAAkBhF,MAAM,GAAG,GAAG;QAChC,MAAM5C,GACHwG,MAAM,CAAC9G,wBACP+G,GAAG,CAAC;YAACC,WAAWtH,GAAG,CAAC,KAAK,CAAC;QAAA,GAC1BmE,KAAK,CACJ/E,IACEM,QAAQY,uBAAuByG,gBAAgB,EAAEyB,oBACjD5I,OAAOU,uBAAuBkH,UAAU,GACxC5H,OAAOU,uBAAuBgH,SAAS;QAG7C,MAAM1G,GACHwG,MAAM,CAAC5G,iBACP6G,GAAG,CAAC;YACHtG,aAAa;YACbwF,eAAe;YACfkB,YAAY;YACZC,WAAW1H,GAAG,CAAC,KAAK,CAAC;QACvB,GACCmE,KAAK,CACJ/E,IACEM,QAAQc,gBAAgB4F,EAAE,EAAEoC,oBAC5B5I,OAAOY,gBAAgBmG,eAAe,GACtC/G,OAAOY,gBAAgBkG,qBAAqB;IAGpD;IACA,IAAIgC,kBAAkBlF,MAAM,GAAG,GAAG;QAChC,MAAM5C,GACHwG,MAAM,CAAC5G,iBACP6G,GAAG,CAAC;YAACZ,uBAAuB;YAAMiB,WAAW1H,GAAG,CAAC,KAAK,CAAC;QAAA,GACvDmE,KAAK,CACJ/E,IACEM,QAAQc,gBAAgB4F,EAAE,EAAEsC,oBAC5BhJ,QAAQc,gBAAgBiG,qBAAqB,EAAE8B;IAGvD;IAEA,MAAMI,UAAU,MAAM/H,GACnBgI,MAAM,CAACvI,cACP8D,KAAK,CAACzE,QAAQW,aAAa+F,EAAE,EAAEmC,iBAC/BpC,SAAS,CAAC;QAACC,IAAI/F,aAAa+F,EAAE;IAAA;IAEjC,OAAOuC,QAAQnF,MAAM;AACvB;AAEA,OAAO,eAAeqF,wBACpBjI,EAAM,EACNF,MAIC;IAED,IAAIoI,WAAW;IAEf,KAAK,MAAMC,WAAWrI,OAAOsI,QAAQ,CAAE;QACrC,IAAID,QAAQ/F,KAAK,IAAI,GAAG;QAExB,MAAMiG,cAAc,MAAMrI,GACvBwG,MAAM,CAAC/G,cACPgH,GAAG,CAAC;YAACrE,OAAOhD,GAAG,CAAC,EAAEK,aAAa2C,KAAK,CAAC,GAAG,EAAE+F,QAAQ/F,KAAK,CAAC,CAAC;QAAA,GACzDmB,KAAK,CACJ/E,IACEG,GAAGc,aAAa+F,EAAE,EAAE2C,QAAQxC,aAAa,GACzChH,GAAGc,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAC/CxB,GAAGc,aAAakC,aAAa,EAAE7B,OAAO6B,aAAa,GACnD9C,GAAGY,aAAa2C,KAAK,EAAE+F,QAAQ/F,KAAK,GACpCvD,GAAGY,aAAa0E,SAAS,EAAE/E,GAAG,CAAC,KAAK,CAAC,IAGxCmG,SAAS,CAAC;YAACC,IAAI/F,aAAa+F,EAAE;QAAA;QAEjC,IAAI6C,YAAYzF,MAAM,GAAG,GAAG;YAC1BsF,YAAYC,QAAQ/F,KAAK;YACzB;QACF;QAEA,MAAM2F,UAAU,MAAM/H,GACnBgI,MAAM,CAACvI,cACP8D,KAAK,CACJ/E,IACEG,GAAGc,aAAa+F,EAAE,EAAE2C,QAAQxC,aAAa,GACzChH,GAAGc,aAAaU,WAAW,EAAEL,OAAOK,WAAW,GAC/CxB,GAAGc,aAAakC,aAAa,EAAE7B,OAAO6B,aAAa,GACnDvC,GAAG,CAAC,EAAEK,aAAa2C,KAAK,CAAC,IAAI,EAAE+F,QAAQ/F,KAAK,CAAC,CAAC,EAC9CvD,GAAGY,aAAa0E,SAAS,EAAE/E,GAAG,CAAC,KAAK,CAAC,IAGxCmG,SAAS,CAAC;YAACnD,OAAO3C,aAAa2C,KAAK;QAAA;QAEvC8F,YAAYH,QAAQ9F,MAAM,CAAC,CAACC,OAAOuB,MAAQvB,QAAQuB,IAAIrB,KAAK,EAAE;IAChE;IAEA,OAAO8F;AACT;AAEA,SAAS1D,8BACP1D,SAA+B,EAC/BwH,kBAAkE;IAElE,KAAK,MAAMnG,eAAeoG,oBAAoBD,oBAAqB;QACjE,MAAMhG,sBAAsBxB,UACzByB,MAAM,CAAC,CAACvB,WAAawB,SAASL,YAAYe,cAAc,EAAElC,SAASC,MAAM,GACzEwB,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAEzB,MAAM,CAAC2B,MAAM,GAAGD,EAAE1B,MAAM,CAAC2B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBH,YAAY+B,QAAQ;IACrD;AACF;AAEA,SAASS,eAAelD,IAAiB;IACvC,OAAO;WAAIA;KAAK,CAACgB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAM6F,cAAc7F,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAI4F,gBAAgB,GAAG,OAAOA;QAC9B,OAAO9F,EAAEU,cAAc,CAACqF,OAAO,KAAK9F,EAAES,cAAc,CAACqF,OAAO;IAC9D;AACF;AAEA,SAASF,oBAA0D9G,IAAS;IAC1E,OAAO;WAAIA;KAAK,CAACgB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAM6F,cAAc7F,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAI4F,gBAAgB,GAAG,OAAOA;QAC9B,OAAOlE,SAAS5B,EAAEQ,cAAc,EAAEJ,aAAa,CAACwB,SAAS3B,EAAEO,cAAc;IAC3E;AACF;AAEA,SAASV,SAASU,cAAwB,EAAEwF,eAAyB;IACnE,OAAOxF,eAAeyF,KAAK,CAAC,CAACC,QAAUF,gBAAgBb,QAAQ,CAACe;AAClE;AAEA,SAAS7F,UAAUjC,SAA+B,EAAEsB,KAAa;IAC/D,IAAIyG,YAAYzG;IAChB,KAAK,MAAMpB,YAAYF,UAAW;QAChC,IAAI+H,cAAc,GAAG;QACrB,MAAMC,OAAOhE,KAAKI,GAAG,CAAClE,SAASE,cAAc,EAAE2H;QAC/C7H,SAASE,cAAc,IAAI4H;QAC3BD,aAAaC;IACf;AACF;AAEA,SAASxE,SAASrD,MAAgB;IAChC,OAAO8H,KAAKC,SAAS,CAAC/H;AACxB"}
1
+ {"version":3,"sources":["../../src/db/reservations.ts"],"sourcesContent":["import {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n and,\n arrayContains,\n asc,\n eq,\n exists,\n gt,\n inArray,\n isNotNull,\n isNull,\n lt,\n notExists,\n notInArray,\n or,\n sql,\n} from 'drizzle-orm';\nimport {recordProviderRunnerActivationOutcome} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';\nimport {terminalStates} from './runner-states.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {provisionerCapabilitySnapshots} from './schema/provisioner-capability-snapshots.js';\nimport {provisionerTokens} from './schema/provisioner-tokens.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerActivationTokens} from './schema/runner-activation-tokens.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {type providerRunnerLaunchKindEnum, providerRunners} from './schema/runner-instances.js';\n\nexport interface ReservationTemplate {\n templateKey: string;\n labels: string[];\n availableSlots: number;\n starting: number;\n running: number;\n}\n\nexport interface DemandStat {\n workspaceId?: string;\n labels: string[];\n queued: number;\n reserved: number;\n oldestQueuedAt: Date;\n}\n\nexport interface ReservationGrant {\n reservationId: string;\n workspaceId?: string;\n labels: string[];\n count: number;\n expiresAt: Date;\n}\n\nexport interface PollDemandAndReserveParams {\n workspaceId: string;\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n /** Lifetime of the short reservation that gives rebound runners time to activate. */\n activationGraceSeconds?: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds?: number;\n}\n\nexport interface InstallationPollDemandAndReserveParams {\n provisionerId: string;\n maxReservations: number;\n ttlSeconds: number;\n /** Lifetime of the short reservation that gives rebound runners time to activate. */\n activationGraceSeconds?: number;\n templates: ReservationTemplate[];\n capabilityWindowSeconds: number;\n eligibleWorkspaceIds: ReadonlySet<string>;\n signal?: AbortSignal;\n onReservations?: (reservations: ReservationGrant[]) => void;\n}\n\ntype DemandScope = 'installation' | 'workspace';\n\ninterface IdleRunnerCandidate {\n id: string;\n launchKind: (typeof providerRunnerLaunchKindEnum.enumValues)[number];\n}\n\ninterface BindableRunnerParams {\n provisionerId: string;\n workspaceId: string;\n requiredLabels: string[];\n scope: DemandScope;\n}\n\ninterface IdleRunnerSelectionParams extends BindableRunnerParams {\n count: number;\n}\n\ntype PollDemandAndReserveLockedParams = PollDemandAndReserveParams & {\n scope: DemandScope;\n};\n\ninterface NormalizedTemplate {\n templateKey: string;\n labels: string[];\n remainingSlots: number;\n}\n\ninterface DemandRow {\n requiredLabels: string[];\n queued: number;\n oldestQueuedAt: Date;\n}\n\ninterface NewReservationUnits {\n labels: string[];\n count: number;\n}\n\ninterface PollDemandAndReserveResult {\n stats: DemandStat[];\n /** Only launch reservations are exposed to the provisioner. */\n reservations: ReservationGrant[];\n /** Internal allocation accounting for bound and launch rows together. */\n newlyReservedUnits: NewReservationUnits[];\n}\n\nexport async function pollDemandAndReserve(\n params: PollDemandAndReserveParams,\n): Promise<{stats: DemandStat[]; reservations: ReservationGrant[]}> {\n const result = await db().transaction(async (tx) => {\n return await pollDemandAndReserveTx(tx, params);\n });\n return {stats: result.stats, reservations: result.reservations};\n}\n\nexport async function pollDemandAndReserveTx(\n tx: Tx,\n params: PollDemandAndReserveParams,\n): Promise<PollDemandAndReserveResult> {\n await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId}))`);\n return await pollDemandAndReserveLockedTx(tx, {...params, scope: 'workspace'});\n}\n\nexport async function pollInstallationDemandAndReserve(\n params: InstallationPollDemandAndReserveParams,\n): Promise<{\n stats: DemandStat[];\n reservations: ReservationGrant[];\n newlyReservedCount?: number;\n}> {\n const candidateWorkspaceIds = await listInstallationDemandWorkspaceIds(\n params.eligibleWorkspaceIds,\n );\n const results: PollDemandAndReserveResult[] = [];\n let remainingMaxReservations = params.maxReservations;\n const remainingTemplates = params.templates.map((template) => ({\n ...template,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n for (const workspaceId of candidateWorkspaceIds) {\n if (params.signal?.aborted || remainingMaxReservations === 0) break;\n const result = await db().transaction(async (tx) => {\n const lockResult = await tx.execute<{locked: boolean}>(\n sql`select pg_try_advisory_xact_lock(hashtext(${workspaceId})) as locked`,\n );\n const locked = lockResult.rows[0];\n if (!locked?.locked) return {stats: [], reservations: [], newlyReservedUnits: []};\n return await pollDemandAndReserveLockedTx(tx, {\n workspaceId,\n provisionerId: params.provisionerId,\n maxReservations: remainingMaxReservations,\n ttlSeconds: params.ttlSeconds,\n ...(params.activationGraceSeconds !== undefined\n ? {activationGraceSeconds: params.activationGraceSeconds}\n : {}),\n templates: remainingTemplates.map((template) => ({\n ...template,\n availableSlots: template.remainingSlots,\n })),\n capabilityWindowSeconds: params.capabilityWindowSeconds,\n scope: 'installation',\n });\n });\n results.push(result);\n consumeInstallationTemplateSlots(remainingTemplates, result.reservations);\n params.onReservations?.(result.reservations);\n remainingMaxReservations -= result.newlyReservedUnits.reduce(\n (total, reservation) => total + reservation.count,\n 0,\n );\n }\n const newlyReservedCount = results.reduce(\n (total, result) =>\n total +\n result.newlyReservedUnits.reduce((subtotal, reservation) => subtotal + reservation.count, 0),\n 0,\n );\n return {\n stats: results.flatMap((result) => result.stats),\n reservations: results.flatMap((result) => result.reservations),\n ...(newlyReservedCount > 0 ? {newlyReservedCount} : {}),\n };\n}\n\nfunction consumeInstallationTemplateSlots(\n templates: NormalizedTemplate[],\n launchGrants: ReservationGrant[],\n): void {\n // Adopted runners are already included in the running count behind availableSlots.\n // Only units that still need a launch consume advertised template capacity.\n for (const reservation of launchGrants) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.labels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.count);\n }\n}\n\nasync function pollDemandAndReserveLockedTx(\n tx: Tx,\n params: PollDemandAndReserveLockedParams,\n): Promise<PollDemandAndReserveResult> {\n let demandRows = (\n await tx\n .select({\n requiredLabels: pendingJobExecutions.requiredLabels,\n queued: sql<number>`count(*)::int`,\n oldestQueuedAt: sql<Date | string>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(eq(pendingJobExecutions.workspaceId, params.workspaceId))\n .groupBy(pendingJobExecutions.requiredLabels)\n ).map((row) => ({\n ...row,\n oldestQueuedAt: new Date(row.oldestQueuedAt),\n }));\n if (params.capabilityWindowSeconds !== undefined) {\n const capabilityLabels = await listActiveWorkspaceCapabilityLabelsTx(tx, {\n workspaceId: params.workspaceId,\n windowSeconds: params.capabilityWindowSeconds,\n });\n demandRows = demandRows.filter(\n (demand) => !capabilityLabels.some((labels) => isSubset(demand.requiredLabels, labels)),\n );\n }\n\n const activeReservationRows = await tx\n .select({\n requiredLabels: reservations.requiredLabels,\n reserved: sql<number>`coalesce(sum(${reservations.count}), 0)::int`,\n })\n .from(reservations)\n .where(\n and(eq(reservations.workspaceId, params.workspaceId), gt(reservations.expiresAt, sql`now()`)),\n )\n .groupBy(reservations.requiredLabels);\n\n const reservedByLabels = new Map(\n activeReservationRows.map((row) => [labelKey(row.requiredLabels), row.reserved]),\n );\n const activeProvisionerReservationRows = await listActiveProvisionerReservationRowsTx(tx, {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n });\n\n const templates = params.templates.map((template) => ({\n templateKey: template.templateKey,\n labels: [...canonicalizeLabels(template.labels)],\n remainingSlots: template.availableSlots,\n }));\n deductProvisionerReservations(templates, activeProvisionerReservationRows);\n const stats: DemandStat[] = [];\n const grants: ReservationGrant[] = [];\n const newlyReservedUnits: NewReservationUnits[] = [];\n let remainingMaxReservations = params.maxReservations;\n const workspaceIdField =\n params.capabilityWindowSeconds === undefined ? {} : {workspaceId: params.workspaceId};\n\n for (const demand of sortDemandRows(demandRows)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(demand.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n\n if (satisfyingTemplates.length === 0) continue;\n\n const reserved = reservedByLabels.get(labelKey(demand.requiredLabels)) ?? 0;\n const unreserved = Math.max(0, demand.queued - reserved);\n const capacity = satisfyingTemplates.reduce(\n (total, template) => total + template.remainingSlots,\n 0,\n );\n const grant = Math.min(unreserved, capacity, remainingMaxReservations);\n let reservedAfterGrant = reserved;\n\n if (grant > 0 && params.maxReservations > 0) {\n const idleRunners = await listIdleRunnerInstancesTx(tx, {\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: grant,\n workspaceId: params.workspaceId,\n scope: params.scope,\n });\n\n remainingMaxReservations -= grant;\n\n const boundCount = idleRunners.length;\n if (boundCount > 0) {\n const [boundReservation] = await tx\n .insert(reservations)\n .values({\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: boundCount,\n kind: 'bound',\n expiresAt: sql`now() + (${params.activationGraceSeconds ?? params.ttlSeconds} || ' seconds')::interval`,\n })\n .returning({id: reservations.id});\n\n if (!boundReservation) throw new Error('Insert returned no rows');\n\n await bindIdleRunnerInstancesTx(tx, {\n provisionerId: params.provisionerId,\n reservationId: boundReservation.id,\n workspaceId: params.workspaceId,\n requiredLabels: demand.requiredLabels,\n scope: params.scope,\n idleRunners,\n });\n }\n\n const launchCount = grant - boundCount;\n drawSlots(satisfyingTemplates, launchCount);\n let launchReservation: {id: string; expiresAt: Date} | undefined;\n if (launchCount > 0) {\n const [inserted] = await tx\n .insert(reservations)\n .values({\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n requiredLabels: demand.requiredLabels,\n count: launchCount,\n kind: 'launch',\n expiresAt: sql`now() + (${params.ttlSeconds} || ' seconds')::interval`,\n })\n .returning({id: reservations.id, expiresAt: reservations.expiresAt});\n\n if (!inserted) throw new Error('Insert returned no rows');\n launchReservation = inserted;\n }\n\n newlyReservedUnits.push({labels: demand.requiredLabels, count: grant});\n reservedAfterGrant += grant;\n if (launchReservation) {\n grants.push({\n reservationId: launchReservation.id,\n ...workspaceIdField,\n labels: demand.requiredLabels,\n count: launchCount,\n expiresAt: launchReservation.expiresAt,\n });\n }\n }\n\n stats.push({\n ...workspaceIdField,\n labels: demand.requiredLabels,\n queued: demand.queued,\n reserved: reservedAfterGrant,\n oldestQueuedAt: demand.oldestQueuedAt,\n });\n }\n\n return {stats, reservations: grants, newlyReservedUnits};\n}\n\nasync function listActiveProvisionerReservationRowsTx(\n tx: Tx,\n params: {workspaceId: string; provisionerId: string},\n): Promise<Array<{requiredLabels: string[]; reserved: number}>> {\n const candidateRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n );\n const candidateIds = candidateRows.map((row) => row.id);\n if (candidateIds.length === 0) return [];\n\n // Assignment and provider-report transactions use this key before changing runner\n // links. Lock it before counting so an in-flight assignment cannot expose its unit twice.\n await lockRunnerReservationAdvisoryKeysTx(tx, {\n provisionerId: params.provisionerId,\n reservationIds: candidateIds,\n });\n\n const activeRows = await tx\n .select({\n id: reservations.id,\n requiredLabels: reservations.requiredLabels,\n count: reservations.count,\n })\n .from(reservations)\n .where(\n and(\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .orderBy(asc(reservations.id))\n .for('update');\n if (activeRows.length === 0) return [];\n\n const activeIds = activeRows.map((row) => row.id);\n const activeIdSet = new Set(activeIds);\n const usedRunnerRows = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n state: providerRunners.state,\n })\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n isNull(providerRunners.reservationReleasedAt),\n or(\n inArray(providerRunners.reservationId, activeIds),\n and(\n inArray(providerRunners.intendedReservationId, activeIds),\n notInArray(providerRunners.state, [...terminalStates]),\n ),\n ),\n ),\n );\n const usedByReservation = new Map<string, Set<string>>();\n for (const runner of usedRunnerRows) {\n if (runner.reservationId && activeIdSet.has(runner.reservationId)) {\n addUsedRunner(usedByReservation, runner.reservationId, runner.id);\n }\n if (\n runner.intendedReservationId &&\n activeIdSet.has(runner.intendedReservationId) &&\n !terminalStates.includes(runner.state as (typeof terminalStates)[number])\n ) {\n addUsedRunner(usedByReservation, runner.intendedReservationId, runner.id);\n }\n }\n\n return activeRows.flatMap((reservation) => {\n const used = usedByReservation.get(reservation.id)?.size ?? 0;\n const pending = Math.max(0, reservation.count - used);\n return pending > 0 ? [{requiredLabels: reservation.requiredLabels, reserved: pending}] : [];\n });\n}\n\nfunction addUsedRunner(\n usedByReservation: Map<string, Set<string>>,\n reservationId: string,\n runnerId: string,\n): void {\n const runnerIds = usedByReservation.get(reservationId);\n if (runnerIds) runnerIds.add(runnerId);\n else usedByReservation.set(reservationId, new Set([runnerId]));\n}\n\n// An expired or missing reservation is stale. A live reservation protects a runner that is\n// still booting or waiting for its activation grace period.\nfunction canBindRunner(tx: Tx, params: BindableRunnerParams) {\n return and(\n or(\n and(isNull(providerRunners.workspaceId), isNull(providerRunners.reservationId)),\n and(\n isNotNull(providerRunners.reservationId),\n params.scope === 'installation'\n ? undefined\n : or(\n isNull(providerRunners.workspaceId),\n eq(providerRunners.workspaceId, params.workspaceId),\n ),\n notExists(\n tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.id, providerRunners.reservationId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n ),\n or(\n isNull(providerRunners.intendedReservationId),\n notExists(\n tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.id, providerRunners.intendedReservationId),\n gt(reservations.expiresAt, sql`now()`),\n ),\n ),\n ),\n ),\n isNull(providerRunners.reservationReleasedAt),\n isNull(providerRunners.runnerSessionId),\n );\n}\n\nfunction isBindableRunner(tx: Tx, params: BindableRunnerParams) {\n return and(\n eq(providerRunners.provisionerId, params.provisionerId),\n canBindRunner(tx, params),\n isNotNull(providerRunners.providerRunnerId),\n eq(providerRunners.state, 'running'),\n arrayContains(providerRunners.labels, params.requiredLabels),\n exists(\n tx\n .select({id: runnerControlSessions.id})\n .from(runnerControlSessions)\n .where(\n and(\n eq(runnerControlSessions.runnerInstanceId, providerRunners.id),\n eq(runnerControlSessions.provisionerId, params.provisionerId),\n isNull(runnerControlSessions.closedAt),\n gt(runnerControlSessions.expiresAt, sql`now()`),\n ),\n ),\n ),\n );\n}\n\nasync function listIdleRunnerInstancesTx(\n tx: Tx,\n params: IdleRunnerSelectionParams & {count: number},\n): Promise<IdleRunnerCandidate[]> {\n const lockedRunners = await selectIdleRunnerInstancesTx(tx, params);\n if (lockedRunners.length === 0) return lockedRunners;\n\n // A SELECT ... FOR UPDATE can wake with a fresh target row but the original\n // statement snapshot for reservation subqueries. Re-check the full predicate\n // in a new statement while retaining the row locks from the first query.\n return await selectIdleRunnerInstancesTx(tx, {\n ...params,\n runnerIds: lockedRunners.map((runner) => runner.id),\n count: lockedRunners.length,\n });\n}\n\nasync function selectIdleRunnerInstancesTx(\n tx: Tx,\n params: IdleRunnerSelectionParams & {count: number; runnerIds?: string[]},\n): Promise<IdleRunnerCandidate[]> {\n if (params.runnerIds) {\n // The first pass retains the row locks. Recheck the full predicate in a fresh statement so\n // reservation subqueries see the state that was current when the lock was acquired.\n return await tx\n .select({id: providerRunners.id, launchKind: providerRunners.launchKind})\n .from(providerRunners)\n .where(and(inArray(providerRunners.id, params.runnerIds), isBindableRunner(tx, params)))\n .orderBy(asc(providerRunners.createdAt), asc(providerRunners.id))\n .limit(params.count);\n }\n\n // Select the oldest candidates first, but leave row locking to a second pass. The cleanup\n // path locks runner rows by id, so the locking pass must use that same order. A skipped row\n // rolls back only this nested savepoint and retries the bounded candidate scan, allowing the\n // next-oldest eligible runner to refill the grant without retaining a partial lock set.\n const retrySelection = Symbol('retry bindable runner selection');\n const idleRunners = await (async () => {\n while (true) {\n try {\n return await tx.transaction(async (lockTx) => {\n const candidateRunners = await lockTx\n .select({id: providerRunners.id, launchKind: providerRunners.launchKind})\n .from(providerRunners)\n .where(isBindableRunner(lockTx, params))\n .orderBy(asc(providerRunners.createdAt), asc(providerRunners.id))\n .limit(params.count);\n\n if (candidateRunners.length === 0) return candidateRunners;\n\n // PostgreSQL can acquire row locks before sorting a multi-row FOR UPDATE result. Lock\n // each selected candidate individually so both polling and cleanup use id order.\n const lockedRunners: typeof candidateRunners = [];\n for (const candidate of [...candidateRunners].sort(compareRunnerIds)) {\n const [runner] = await lockTx\n .select({id: providerRunners.id, launchKind: providerRunners.launchKind})\n .from(providerRunners)\n .where(and(eq(providerRunners.id, candidate.id), isBindableRunner(lockTx, params)))\n .for('update');\n if (runner) lockedRunners.push(runner);\n }\n\n if (lockedRunners.length !== candidateRunners.length) throw retrySelection;\n return lockedRunners;\n });\n } catch (error) {\n if (error !== retrySelection) throw error;\n }\n }\n })();\n\n return idleRunners;\n}\n\nasync function bindIdleRunnerInstancesTx(\n tx: Tx,\n params: {\n provisionerId: string;\n reservationId: string;\n workspaceId: string;\n requiredLabels: string[];\n scope: DemandScope;\n idleRunners: IdleRunnerCandidate[];\n },\n): Promise<void> {\n const idleRunnerIds = params.idleRunners.map((runner) => runner.id);\n\n await tx\n .update(runnerActivationTokens)\n .set({revokedAt: sql`now()`})\n .where(\n and(\n inArray(runnerActivationTokens.runnerInstanceId, idleRunnerIds),\n isNull(runnerActivationTokens.consumedAt),\n isNull(runnerActivationTokens.revokedAt),\n ),\n );\n\n // listIdleRunnerInstancesTx re-checks the full eligibility predicate in a fresh statement\n // after acquiring row locks. Those locks prevent the validated set from changing before this\n // update runs.\n const boundRunners = await tx\n .update(providerRunners)\n .set({\n workspaceId: params.workspaceId,\n reservationId: params.reservationId,\n intendedReservationId: null,\n assignedAt: sql`now()`,\n updatedAt: sql`now()`,\n })\n .where(and(inArray(providerRunners.id, idleRunnerIds), isBindableRunner(tx, params)))\n .returning({id: providerRunners.id, launchKind: providerRunners.launchKind});\n\n if (boundRunners.length !== params.idleRunners.length)\n throw new Error('Locked idle runner set changed before binding');\n\n const reboundCount = boundRunners.filter((runner) => runner.launchKind === 'demand').length;\n if (reboundCount > 0)\n recordProviderRunnerActivationOutcome({outcome: 'rebound', count: reboundCount});\n}\n\nasync function listInstallationDemandWorkspaceIds(eligibleWorkspaceIds: ReadonlySet<string>) {\n if (eligibleWorkspaceIds.size === 0) return [];\n const rows = await db()\n .select({\n workspaceId: pendingJobExecutions.workspaceId,\n oldestQueuedAt: sql<Date>`min(${pendingJobExecutions.createdAt})`,\n })\n .from(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.workspaceId, [...eligibleWorkspaceIds]))\n .groupBy(pendingJobExecutions.workspaceId)\n .orderBy(\n asc(sql`min(${pendingJobExecutions.createdAt})`),\n asc(pendingJobExecutions.workspaceId),\n );\n return rows.map((row) => row.workspaceId);\n}\n\nexport async function listQueuedDemandWorkspaceIds(): Promise<string[]> {\n const rows = await db()\n .select({workspaceId: pendingJobExecutions.workspaceId})\n .from(pendingJobExecutions)\n .groupBy(pendingJobExecutions.workspaceId);\n return rows.map((row) => row.workspaceId);\n}\n\nasync function listActiveWorkspaceCapabilityLabelsTx(\n tx: Tx,\n params: {workspaceId: string; windowSeconds: number},\n): Promise<string[][]> {\n const rows = await tx\n .select({labels: provisionerCapabilitySnapshots.labels})\n .from(provisionerCapabilitySnapshots)\n .innerJoin(\n provisionerTokens,\n eq(provisionerTokens.id, provisionerCapabilitySnapshots.provisionerId),\n )\n .where(\n and(\n eq(provisionerCapabilitySnapshots.workspaceId, params.workspaceId),\n eq(provisionerTokens.workspaceId, params.workspaceId),\n eq(provisionerTokens.scope, 'workspace'),\n sql`${provisionerCapabilitySnapshots.advertisedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n sql`${provisionerTokens.revokedAt} is null`,\n sql`(${provisionerTokens.expiresAt} is null or ${provisionerTokens.expiresAt} > now())`,\n ),\n );\n return rows.map((row) => row.labels);\n}\n\nexport async function deleteExpiredReservations(params?: {limit?: number}): Promise<number> {\n return await db().transaction(async (tx) => {\n const expiredRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(lt(reservations.expiresAt, sql`now()`))\n .orderBy(asc(reservations.expiresAt))\n .limit(params?.limit ?? 1000);\n return await deleteReservationsWithCleanupTx(\n tx,\n expiredRows.map((reservation) => reservation.id),\n {expiredOnly: true},\n );\n });\n}\n\nexport async function deleteReservationsByIds(ids: string[]): Promise<number> {\n if (ids.length === 0) return 0;\n\n return await db().transaction(async (tx) => {\n return await deleteReservationsWithCleanupTx(tx, ids);\n });\n}\n\nasync function deleteReservationsWithCleanupTx(\n tx: Tx,\n ids: string[],\n params: {expiredOnly?: boolean} = {},\n): Promise<number> {\n if (ids.length === 0) return 0;\n\n const isAffectedRunner = () =>\n or(\n and(\n inArray(providerRunners.reservationId, ids),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n inArray(providerRunners.intendedReservationId, ids),\n );\n const candidateRunners = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n runnerSessionId: providerRunners.runnerSessionId,\n reservationReleasedAt: providerRunners.reservationReleasedAt,\n })\n .from(providerRunners)\n .where(isAffectedRunner())\n .orderBy(asc(providerRunners.id));\n const affectedRunners: typeof candidateRunners = [];\n for (const candidate of [...candidateRunners].sort(compareRunnerIds)) {\n const [runner] = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n runnerSessionId: providerRunners.runnerSessionId,\n reservationReleasedAt: providerRunners.reservationReleasedAt,\n })\n .from(providerRunners)\n .where(and(eq(providerRunners.id, candidate.id), isAffectedRunner()))\n .for('update');\n if (runner) affectedRunners.push(runner);\n }\n const reservationRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n inArray(reservations.id, ids),\n params.expiredOnly ? lt(reservations.expiresAt, sql`now()`) : undefined,\n ),\n )\n .for('update');\n const reservationIds = reservationRows.map((reservation) => reservation.id);\n\n if (reservationIds.length === 0) return 0;\n\n const assignedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.reservationId &&\n reservationIds.includes(runner.reservationId) &&\n !runner.runnerSessionId &&\n !runner.reservationReleasedAt,\n )\n .map((runner) => runner.id);\n const intendedRunnerIds = affectedRunners\n .filter(\n (runner) =>\n runner.intendedReservationId && reservationIds.includes(runner.intendedReservationId),\n )\n .map((runner) => runner.id);\n if (assignedRunnerIds.length > 0) {\n await tx\n .update(runnerActivationTokens)\n .set({revokedAt: sql`now()`})\n .where(\n and(\n inArray(runnerActivationTokens.runnerInstanceId, assignedRunnerIds),\n isNull(runnerActivationTokens.consumedAt),\n isNull(runnerActivationTokens.revokedAt),\n ),\n );\n await tx\n .update(providerRunners)\n .set({\n workspaceId: null,\n reservationId: null,\n assignedAt: null,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n inArray(providerRunners.id, assignedRunnerIds),\n isNull(providerRunners.runnerSessionId),\n isNull(providerRunners.reservationReleasedAt),\n ),\n );\n }\n if (intendedRunnerIds.length > 0) {\n await tx\n .update(providerRunners)\n .set({intendedReservationId: null, updatedAt: sql`now()`})\n .where(\n and(\n inArray(providerRunners.id, intendedRunnerIds),\n inArray(providerRunners.intendedReservationId, reservationIds),\n ),\n );\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(inArray(reservations.id, reservationIds))\n .returning({id: reservations.id});\n\n return deleted.length;\n}\n\nexport async function releaseReservationUnits(\n tx: Tx,\n params: {\n workspaceId: string;\n provisionerId: string;\n releases: Array<{reservationId: string; count: number}>;\n },\n): Promise<number> {\n let released = 0;\n\n for (const release of params.releases) {\n if (release.count <= 0) continue;\n\n const decremented = await tx\n .update(reservations)\n .set({count: sql`${reservations.count} - ${release.count}`})\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n gt(reservations.count, release.count),\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({id: reservations.id});\n\n if (decremented.length > 0) {\n released += release.count;\n continue;\n }\n\n const deleted = await tx\n .delete(reservations)\n .where(\n and(\n eq(reservations.id, release.reservationId),\n eq(reservations.workspaceId, params.workspaceId),\n eq(reservations.provisionerId, params.provisionerId),\n sql`${reservations.count} <= ${release.count}`,\n gt(reservations.expiresAt, sql`now()`),\n ),\n )\n .returning({count: reservations.count});\n\n released += deleted.reduce((total, row) => total + row.count, 0);\n }\n\n return released;\n}\n\nfunction deductProvisionerReservations(\n templates: NormalizedTemplate[],\n activeReservations: {requiredLabels: string[]; reserved: number}[],\n): void {\n for (const reservation of sortReservationRows(activeReservations)) {\n const satisfyingTemplates = templates\n .filter((template) => isSubset(reservation.requiredLabels, template.labels))\n .sort(\n (a, b) => a.labels.length - b.labels.length || a.templateKey.localeCompare(b.templateKey),\n );\n drawSlots(satisfyingTemplates, reservation.reserved);\n }\n}\n\nfunction compareRunnerIds(left: {id: string}, right: {id: string}): number {\n return left.id < right.id ? -1 : left.id > right.id ? 1 : 0;\n}\n\nfunction sortDemandRows(rows: DemandRow[]): DemandRow[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return a.oldestQueuedAt.getTime() - b.oldestQueuedAt.getTime();\n });\n}\n\nfunction sortReservationRows<T extends {requiredLabels: string[]}>(rows: T[]): T[] {\n return [...rows].sort((a, b) => {\n const specificity = b.requiredLabels.length - a.requiredLabels.length;\n if (specificity !== 0) return specificity;\n return labelKey(a.requiredLabels).localeCompare(labelKey(b.requiredLabels));\n });\n}\n\nfunction isSubset(requiredLabels: string[], availableLabels: string[]): boolean {\n return requiredLabels.every((label) => availableLabels.includes(label));\n}\n\nfunction drawSlots(templates: NormalizedTemplate[], count: number): void {\n let remaining = count;\n for (const template of templates) {\n if (remaining === 0) return;\n const used = Math.min(template.remainingSlots, remaining);\n template.remainingSlots -= used;\n remaining -= used;\n }\n}\n\nfunction labelKey(labels: string[]): string {\n return JSON.stringify(labels);\n}\n"],"names":["canonicalizeLabels","and","arrayContains","asc","eq","exists","gt","inArray","isNotNull","isNull","lt","notExists","notInArray","or","sql","recordProviderRunnerActivationOutcome","db","lockRunnerReservationAdvisoryKeysTx","terminalStates","pendingJobExecutions","provisionerCapabilitySnapshots","provisionerTokens","reservations","runnerActivationTokens","runnerControlSessions","providerRunners","pollDemandAndReserve","params","result","transaction","tx","pollDemandAndReserveTx","stats","execute","workspaceId","pollDemandAndReserveLockedTx","scope","pollInstallationDemandAndReserve","candidateWorkspaceIds","listInstallationDemandWorkspaceIds","eligibleWorkspaceIds","results","remainingMaxReservations","maxReservations","remainingTemplates","templates","map","template","labels","remainingSlots","availableSlots","signal","aborted","lockResult","locked","rows","newlyReservedUnits","provisionerId","ttlSeconds","activationGraceSeconds","undefined","capabilityWindowSeconds","push","consumeInstallationTemplateSlots","onReservations","reduce","total","reservation","count","newlyReservedCount","subtotal","flatMap","launchGrants","satisfyingTemplates","filter","isSubset","sort","a","b","length","templateKey","localeCompare","drawSlots","demandRows","select","requiredLabels","queued","oldestQueuedAt","createdAt","from","where","groupBy","row","Date","capabilityLabels","listActiveWorkspaceCapabilityLabelsTx","windowSeconds","demand","some","activeReservationRows","reserved","expiresAt","reservedByLabels","Map","labelKey","activeProvisionerReservationRows","listActiveProvisionerReservationRowsTx","deductProvisionerReservations","grants","workspaceIdField","sortDemandRows","get","unreserved","Math","max","capacity","grant","min","reservedAfterGrant","idleRunners","listIdleRunnerInstancesTx","boundCount","boundReservation","insert","values","kind","returning","id","Error","bindIdleRunnerInstancesTx","reservationId","launchCount","launchReservation","inserted","candidateRows","candidateIds","reservationIds","activeRows","orderBy","for","activeIds","activeIdSet","Set","usedRunnerRows","intendedReservationId","state","reservationReleasedAt","usedByReservation","runner","has","addUsedRunner","includes","used","size","pending","runnerId","runnerIds","add","set","canBindRunner","runnerSessionId","isBindableRunner","providerRunnerId","runnerInstanceId","closedAt","lockedRunners","selectIdleRunnerInstancesTx","launchKind","limit","retrySelection","Symbol","lockTx","candidateRunners","candidate","compareRunnerIds","error","idleRunnerIds","update","revokedAt","consumedAt","boundRunners","assignedAt","updatedAt","reboundCount","outcome","listQueuedDemandWorkspaceIds","innerJoin","advertisedAt","deleteExpiredReservations","expiredRows","deleteReservationsWithCleanupTx","expiredOnly","deleteReservationsByIds","ids","isAffectedRunner","affectedRunners","reservationRows","assignedRunnerIds","intendedRunnerIds","deleted","delete","releaseReservationUnits","released","release","releases","decremented","activeReservations","sortReservationRows","left","right","specificity","getTime","availableLabels","every","label","remaining","JSON","stringify"],"mappings":"AAAA,SAAQA,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,aAAa,EACbC,GAAG,EACHC,EAAE,EACFC,MAAM,EACNC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,SAAS,EACTC,UAAU,EACVC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SAAQC,qCAAqC,QAAO,uBAAuB;AAE3E,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,mCAAmC,QAAO,yBAAyB;AAC3E,SAAQC,cAAc,QAAO,qBAAqB;AAClD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,8BAA8B,QAAO,+CAA+C;AAC5F,SAAQC,iBAAiB,QAAO,iCAAiC;AACjE,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,sBAAsB,QAAO,uCAAuC;AAC5E,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAA2CC,eAAe,QAAO,+BAA+B;AAiGhG,OAAO,eAAeC,qBACpBC,MAAkC;IAElC,MAAMC,SAAS,MAAMZ,KAAKa,WAAW,CAAC,OAAOC;QAC3C,OAAO,MAAMC,uBAAuBD,IAAIH;IAC1C;IACA,OAAO;QAACK,OAAOJ,OAAOI,KAAK;QAAEV,cAAcM,OAAON,YAAY;IAAA;AAChE;AAEA,OAAO,eAAeS,uBACpBD,EAAM,EACNH,MAAkC;IAElC,MAAMG,GAAGG,OAAO,CAACnB,GAAG,CAAC,sCAAsC,EAAEa,OAAOO,WAAW,CAAC,EAAE,CAAC;IACnF,OAAO,MAAMC,6BAA6BL,IAAI;QAAC,GAAGH,MAAM;QAAES,OAAO;IAAW;AAC9E;AAEA,OAAO,eAAeC,iCACpBV,MAA8C;IAM9C,MAAMW,wBAAwB,MAAMC,mCAClCZ,OAAOa,oBAAoB;IAE7B,MAAMC,UAAwC,EAAE;IAChD,IAAIC,2BAA2Bf,OAAOgB,eAAe;IACrD,MAAMC,qBAAqBjB,OAAOkB,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YAC7D,GAAGA,QAAQ;YACXC,QAAQ;mBAAIhD,mBAAmB+C,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACA,KAAK,MAAMhB,eAAeI,sBAAuB;QAC/C,IAAIX,OAAOwB,MAAM,EAAEC,WAAWV,6BAA6B,GAAG;QAC9D,MAAMd,SAAS,MAAMZ,KAAKa,WAAW,CAAC,OAAOC;YAC3C,MAAMuB,aAAa,MAAMvB,GAAGG,OAAO,CACjCnB,GAAG,CAAC,0CAA0C,EAAEoB,YAAY,YAAY,CAAC;YAE3E,MAAMoB,SAASD,WAAWE,IAAI,CAAC,EAAE;YACjC,IAAI,CAACD,QAAQA,QAAQ,OAAO;gBAACtB,OAAO,EAAE;gBAAEV,cAAc,EAAE;gBAAEkC,oBAAoB,EAAE;YAAA;YAChF,OAAO,MAAMrB,6BAA6BL,IAAI;gBAC5CI;gBACAuB,eAAe9B,OAAO8B,aAAa;gBACnCd,iBAAiBD;gBACjBgB,YAAY/B,OAAO+B,UAAU;gBAC7B,GAAI/B,OAAOgC,sBAAsB,KAAKC,YAClC;oBAACD,wBAAwBhC,OAAOgC,sBAAsB;gBAAA,IACtD,CAAC,CAAC;gBACNd,WAAWD,mBAAmBE,GAAG,CAAC,CAACC,WAAc,CAAA;wBAC/C,GAAGA,QAAQ;wBACXG,gBAAgBH,SAASE,cAAc;oBACzC,CAAA;gBACAY,yBAAyBlC,OAAOkC,uBAAuB;gBACvDzB,OAAO;YACT;QACF;QACAK,QAAQqB,IAAI,CAAClC;QACbmC,iCAAiCnB,oBAAoBhB,OAAON,YAAY;QACxEK,OAAOqC,cAAc,GAAGpC,OAAON,YAAY;QAC3CoB,4BAA4Bd,OAAO4B,kBAAkB,CAACS,MAAM,CAC1D,CAACC,OAAOC,cAAgBD,QAAQC,YAAYC,KAAK,EACjD;IAEJ;IACA,MAAMC,qBAAqB5B,QAAQwB,MAAM,CACvC,CAACC,OAAOtC,SACNsC,QACAtC,OAAO4B,kBAAkB,CAACS,MAAM,CAAC,CAACK,UAAUH,cAAgBG,WAAWH,YAAYC,KAAK,EAAE,IAC5F;IAEF,OAAO;QACLpC,OAAOS,QAAQ8B,OAAO,CAAC,CAAC3C,SAAWA,OAAOI,KAAK;QAC/CV,cAAcmB,QAAQ8B,OAAO,CAAC,CAAC3C,SAAWA,OAAON,YAAY;QAC7D,GAAI+C,qBAAqB,IAAI;YAACA;QAAkB,IAAI,CAAC,CAAC;IACxD;AACF;AAEA,SAASN,iCACPlB,SAA+B,EAC/B2B,YAAgC;IAEhC,mFAAmF;IACnF,4EAA4E;IAC5E,KAAK,MAAML,eAAeK,aAAc;QACtC,MAAMC,sBAAsB5B,UACzB6B,MAAM,CAAC,CAAC3B,WAAa4B,SAASR,YAAYnB,MAAM,EAAED,SAASC,MAAM,GACjE4B,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAE7B,MAAM,CAAC+B,MAAM,GAAGD,EAAE9B,MAAM,CAAC+B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBN,YAAYC,KAAK;IAClD;AACF;AAEA,eAAejC,6BACbL,EAAM,EACNH,MAAwC;IAExC,IAAIwD,aAAa,AACf,CAAA,MAAMrD,GACHsD,MAAM,CAAC;QACNC,gBAAgBlE,qBAAqBkE,cAAc;QACnDC,QAAQxE,GAAW,CAAC,aAAa,CAAC;QAClCyE,gBAAgBzE,GAAkB,CAAC,IAAI,EAAEK,qBAAqBqE,SAAS,CAAC,CAAC,CAAC;IAC5E,GACCC,IAAI,CAACtE,sBACLuE,KAAK,CAACtF,GAAGe,qBAAqBe,WAAW,EAAEP,OAAOO,WAAW,GAC7DyD,OAAO,CAACxE,qBAAqBkE,cAAc,CAAA,EAC9CvC,GAAG,CAAC,CAAC8C,MAAS,CAAA;YACd,GAAGA,GAAG;YACNL,gBAAgB,IAAIM,KAAKD,IAAIL,cAAc;QAC7C,CAAA;IACA,IAAI5D,OAAOkC,uBAAuB,KAAKD,WAAW;QAChD,MAAMkC,mBAAmB,MAAMC,sCAAsCjE,IAAI;YACvEI,aAAaP,OAAOO,WAAW;YAC/B8D,eAAerE,OAAOkC,uBAAuB;QAC/C;QACAsB,aAAaA,WAAWT,MAAM,CAC5B,CAACuB,SAAW,CAACH,iBAAiBI,IAAI,CAAC,CAAClD,SAAW2B,SAASsB,OAAOZ,cAAc,EAAErC;IAEnF;IAEA,MAAMmD,wBAAwB,MAAMrE,GACjCsD,MAAM,CAAC;QACNC,gBAAgB/D,aAAa+D,cAAc;QAC3Ce,UAAUtF,GAAW,CAAC,aAAa,EAAEQ,aAAa8C,KAAK,CAAC,UAAU,CAAC;IACrE,GACCqB,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IAAIG,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAAG5B,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAE5F6E,OAAO,CAACrE,aAAa+D,cAAc;IAEtC,MAAMiB,mBAAmB,IAAIC,IAC3BJ,sBAAsBrD,GAAG,CAAC,CAAC8C,MAAQ;YAACY,SAASZ,IAAIP,cAAc;YAAGO,IAAIQ,QAAQ;SAAC;IAEjF,MAAMK,mCAAmC,MAAMC,uCAAuC5E,IAAI;QACxFI,aAAaP,OAAOO,WAAW;QAC/BuB,eAAe9B,OAAO8B,aAAa;IACrC;IAEA,MAAMZ,YAAYlB,OAAOkB,SAAS,CAACC,GAAG,CAAC,CAACC,WAAc,CAAA;YACpDiC,aAAajC,SAASiC,WAAW;YACjChC,QAAQ;mBAAIhD,mBAAmB+C,SAASC,MAAM;aAAE;YAChDC,gBAAgBF,SAASG,cAAc;QACzC,CAAA;IACAyD,8BAA8B9D,WAAW4D;IACzC,MAAMzE,QAAsB,EAAE;IAC9B,MAAM4E,SAA6B,EAAE;IACrC,MAAMpD,qBAA4C,EAAE;IACpD,IAAId,2BAA2Bf,OAAOgB,eAAe;IACrD,MAAMkE,mBACJlF,OAAOkC,uBAAuB,KAAKD,YAAY,CAAC,IAAI;QAAC1B,aAAaP,OAAOO,WAAW;IAAA;IAEtF,KAAK,MAAM+D,UAAUa,eAAe3B,YAAa;QAC/C,MAAMV,sBAAsB5B,UACzB6B,MAAM,CAAC,CAAC3B,WAAa4B,SAASsB,OAAOZ,cAAc,EAAEtC,SAASC,MAAM,GACpE4B,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAE7B,MAAM,CAAC+B,MAAM,GAAGD,EAAE9B,MAAM,CAAC+B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAG5F,IAAIP,oBAAoBM,MAAM,KAAK,GAAG;QAEtC,MAAMqB,WAAWE,iBAAiBS,GAAG,CAACP,SAASP,OAAOZ,cAAc,MAAM;QAC1E,MAAM2B,aAAaC,KAAKC,GAAG,CAAC,GAAGjB,OAAOX,MAAM,GAAGc;QAC/C,MAAMe,WAAW1C,oBAAoBR,MAAM,CACzC,CAACC,OAAOnB,WAAamB,QAAQnB,SAASE,cAAc,EACpD;QAEF,MAAMmE,QAAQH,KAAKI,GAAG,CAACL,YAAYG,UAAUzE;QAC7C,IAAI4E,qBAAqBlB;QAEzB,IAAIgB,QAAQ,KAAKzF,OAAOgB,eAAe,GAAG,GAAG;YAC3C,MAAM4E,cAAc,MAAMC,0BAA0B1F,IAAI;gBACtD2B,eAAe9B,OAAO8B,aAAa;gBACnC4B,gBAAgBY,OAAOZ,cAAc;gBACrCjB,OAAOgD;gBACPlF,aAAaP,OAAOO,WAAW;gBAC/BE,OAAOT,OAAOS,KAAK;YACrB;YAEAM,4BAA4B0E;YAE5B,MAAMK,aAAaF,YAAYxC,MAAM;YACrC,IAAI0C,aAAa,GAAG;gBAClB,MAAM,CAACC,iBAAiB,GAAG,MAAM5F,GAC9B6F,MAAM,CAACrG,cACPsG,MAAM,CAAC;oBACN1F,aAAaP,OAAOO,WAAW;oBAC/BuB,eAAe9B,OAAO8B,aAAa;oBACnC4B,gBAAgBY,OAAOZ,cAAc;oBACrCjB,OAAOqD;oBACPI,MAAM;oBACNxB,WAAWvF,GAAG,CAAC,SAAS,EAAEa,OAAOgC,sBAAsB,IAAIhC,OAAO+B,UAAU,CAAC,yBAAyB,CAAC;gBACzG,GACCoE,SAAS,CAAC;oBAACC,IAAIzG,aAAayG,EAAE;gBAAA;gBAEjC,IAAI,CAACL,kBAAkB,MAAM,IAAIM,MAAM;gBAEvC,MAAMC,0BAA0BnG,IAAI;oBAClC2B,eAAe9B,OAAO8B,aAAa;oBACnCyE,eAAeR,iBAAiBK,EAAE;oBAClC7F,aAAaP,OAAOO,WAAW;oBAC/BmD,gBAAgBY,OAAOZ,cAAc;oBACrCjD,OAAOT,OAAOS,KAAK;oBACnBmF;gBACF;YACF;YAEA,MAAMY,cAAcf,QAAQK;YAC5BvC,UAAUT,qBAAqB0D;YAC/B,IAAIC;YACJ,IAAID,cAAc,GAAG;gBACnB,MAAM,CAACE,SAAS,GAAG,MAAMvG,GACtB6F,MAAM,CAACrG,cACPsG,MAAM,CAAC;oBACN1F,aAAaP,OAAOO,WAAW;oBAC/BuB,eAAe9B,OAAO8B,aAAa;oBACnC4B,gBAAgBY,OAAOZ,cAAc;oBACrCjB,OAAO+D;oBACPN,MAAM;oBACNxB,WAAWvF,GAAG,CAAC,SAAS,EAAEa,OAAO+B,UAAU,CAAC,yBAAyB,CAAC;gBACxE,GACCoE,SAAS,CAAC;oBAACC,IAAIzG,aAAayG,EAAE;oBAAE1B,WAAW/E,aAAa+E,SAAS;gBAAA;gBAEpE,IAAI,CAACgC,UAAU,MAAM,IAAIL,MAAM;gBAC/BI,oBAAoBC;YACtB;YAEA7E,mBAAmBM,IAAI,CAAC;gBAACd,QAAQiD,OAAOZ,cAAc;gBAAEjB,OAAOgD;YAAK;YACpEE,sBAAsBF;YACtB,IAAIgB,mBAAmB;gBACrBxB,OAAO9C,IAAI,CAAC;oBACVoE,eAAeE,kBAAkBL,EAAE;oBACnC,GAAGlB,gBAAgB;oBACnB7D,QAAQiD,OAAOZ,cAAc;oBAC7BjB,OAAO+D;oBACP9B,WAAW+B,kBAAkB/B,SAAS;gBACxC;YACF;QACF;QAEArE,MAAM8B,IAAI,CAAC;YACT,GAAG+C,gBAAgB;YACnB7D,QAAQiD,OAAOZ,cAAc;YAC7BC,QAAQW,OAAOX,MAAM;YACrBc,UAAUkB;YACV/B,gBAAgBU,OAAOV,cAAc;QACvC;IACF;IAEA,OAAO;QAACvD;QAAOV,cAAcsF;QAAQpD;IAAkB;AACzD;AAEA,eAAekD,uCACb5E,EAAM,EACNH,MAAoD;IAEpD,MAAM2G,gBAAgB,MAAMxG,GACzBsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnDnD,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC;IAG3C,MAAMyH,eAAeD,cAAcxF,GAAG,CAAC,CAAC8C,MAAQA,IAAImC,EAAE;IACtD,IAAIQ,aAAaxD,MAAM,KAAK,GAAG,OAAO,EAAE;IAExC,kFAAkF;IAClF,0FAA0F;IAC1F,MAAM9D,oCAAoCa,IAAI;QAC5C2B,eAAe9B,OAAO8B,aAAa;QACnC+E,gBAAgBD;IAClB;IAEA,MAAME,aAAa,MAAM3G,GACtBsD,MAAM,CAAC;QACN2C,IAAIzG,aAAayG,EAAE;QACnB1C,gBAAgB/D,aAAa+D,cAAc;QAC3CjB,OAAO9C,aAAa8C,KAAK;IAC3B,GACCqB,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnDnD,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAGxC4H,OAAO,CAACvI,IAAImB,aAAayG,EAAE,GAC3BY,GAAG,CAAC;IACP,IAAIF,WAAW1D,MAAM,KAAK,GAAG,OAAO,EAAE;IAEtC,MAAM6D,YAAYH,WAAW3F,GAAG,CAAC,CAAC8C,MAAQA,IAAImC,EAAE;IAChD,MAAMc,cAAc,IAAIC,IAAIF;IAC5B,MAAMG,iBAAiB,MAAMjH,GAC1BsD,MAAM,CAAC;QACN2C,IAAItG,gBAAgBsG,EAAE;QACtBG,eAAezG,gBAAgByG,aAAa;QAC5Cc,uBAAuBvH,gBAAgBuH,qBAAqB;QAC5DC,OAAOxH,gBAAgBwH,KAAK;IAC9B,GACCxD,IAAI,CAAChE,iBACLiE,KAAK,CACJzF,IACEG,GAAGqB,gBAAgBgC,aAAa,EAAE9B,OAAO8B,aAAa,GACtDhD,OAAOgB,gBAAgByH,qBAAqB,GAC5CrI,GACEN,QAAQkB,gBAAgByG,aAAa,EAAEU,YACvC3I,IACEM,QAAQkB,gBAAgBuH,qBAAqB,EAAEJ,YAC/ChI,WAAWa,gBAAgBwH,KAAK,EAAE;WAAI/H;KAAe;IAK/D,MAAMiI,oBAAoB,IAAI5C;IAC9B,KAAK,MAAM6C,UAAUL,eAAgB;QACnC,IAAIK,OAAOlB,aAAa,IAAIW,YAAYQ,GAAG,CAACD,OAAOlB,aAAa,GAAG;YACjEoB,cAAcH,mBAAmBC,OAAOlB,aAAa,EAAEkB,OAAOrB,EAAE;QAClE;QACA,IACEqB,OAAOJ,qBAAqB,IAC5BH,YAAYQ,GAAG,CAACD,OAAOJ,qBAAqB,KAC5C,CAAC9H,eAAeqI,QAAQ,CAACH,OAAOH,KAAK,GACrC;YACAK,cAAcH,mBAAmBC,OAAOJ,qBAAqB,EAAEI,OAAOrB,EAAE;QAC1E;IACF;IAEA,OAAOU,WAAWlE,OAAO,CAAC,CAACJ;QACzB,MAAMqF,OAAOL,kBAAkBpC,GAAG,CAAC5C,YAAY4D,EAAE,GAAG0B,QAAQ;QAC5D,MAAMC,UAAUzC,KAAKC,GAAG,CAAC,GAAG/C,YAAYC,KAAK,GAAGoF;QAChD,OAAOE,UAAU,IAAI;YAAC;gBAACrE,gBAAgBlB,YAAYkB,cAAc;gBAAEe,UAAUsD;YAAO;SAAE,GAAG,EAAE;IAC7F;AACF;AAEA,SAASJ,cACPH,iBAA2C,EAC3CjB,aAAqB,EACrByB,QAAgB;IAEhB,MAAMC,YAAYT,kBAAkBpC,GAAG,CAACmB;IACxC,IAAI0B,WAAWA,UAAUC,GAAG,CAACF;SACxBR,kBAAkBW,GAAG,CAAC5B,eAAe,IAAIY,IAAI;QAACa;KAAS;AAC9D;AAEA,2FAA2F;AAC3F,4DAA4D;AAC5D,SAASI,cAAcjI,EAAM,EAAEH,MAA4B;IACzD,OAAO1B,IACLY,GACEZ,IAAIQ,OAAOgB,gBAAgBS,WAAW,GAAGzB,OAAOgB,gBAAgByG,aAAa,IAC7EjI,IACEO,UAAUiB,gBAAgByG,aAAa,GACvCvG,OAAOS,KAAK,KAAK,iBACbwB,YACA/C,GACEJ,OAAOgB,gBAAgBS,WAAW,GAClC9B,GAAGqB,gBAAgBS,WAAW,EAAEP,OAAOO,WAAW,IAExDvB,UACEmB,GACGsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAEtG,gBAAgByG,aAAa,GACjD5H,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,QAMjDD,GACEJ,OAAOgB,gBAAgBuH,qBAAqB,GAC5CrI,UACEmB,GACGsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAEtG,gBAAgBuH,qBAAqB,GACzD1I,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,OAK/CL,OAAOgB,gBAAgByH,qBAAqB,GAC5CzI,OAAOgB,gBAAgBuI,eAAe;AAE1C;AAEA,SAASC,iBAAiBnI,EAAM,EAAEH,MAA4B;IAC5D,OAAO1B,IACLG,GAAGqB,gBAAgBgC,aAAa,EAAE9B,OAAO8B,aAAa,GACtDsG,cAAcjI,IAAIH,SAClBnB,UAAUiB,gBAAgByI,gBAAgB,GAC1C9J,GAAGqB,gBAAgBwH,KAAK,EAAE,YAC1B/I,cAAcuB,gBAAgBuB,MAAM,EAAErB,OAAO0D,cAAc,GAC3DhF,OACEyB,GACGsD,MAAM,CAAC;QAAC2C,IAAIvG,sBAAsBuG,EAAE;IAAA,GACpCtC,IAAI,CAACjE,uBACLkE,KAAK,CACJzF,IACEG,GAAGoB,sBAAsB2I,gBAAgB,EAAE1I,gBAAgBsG,EAAE,GAC7D3H,GAAGoB,sBAAsBiC,aAAa,EAAE9B,OAAO8B,aAAa,GAC5DhD,OAAOe,sBAAsB4I,QAAQ,GACrC9J,GAAGkB,sBAAsB6E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC;AAK1D;AAEA,eAAe0G,0BACb1F,EAAM,EACNH,MAAmD;IAEnD,MAAM0I,gBAAgB,MAAMC,4BAA4BxI,IAAIH;IAC5D,IAAI0I,cAActF,MAAM,KAAK,GAAG,OAAOsF;IAEvC,4EAA4E;IAC5E,6EAA6E;IAC7E,yEAAyE;IACzE,OAAO,MAAMC,4BAA4BxI,IAAI;QAC3C,GAAGH,MAAM;QACTiI,WAAWS,cAAcvH,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;QAClD3D,OAAOiG,cAActF,MAAM;IAC7B;AACF;AAEA,eAAeuF,4BACbxI,EAAM,EACNH,MAAyE;IAEzE,IAAIA,OAAOiI,SAAS,EAAE;QACpB,2FAA2F;QAC3F,oFAAoF;QACpF,OAAO,MAAM9H,GACVsD,MAAM,CAAC;YAAC2C,IAAItG,gBAAgBsG,EAAE;YAAEwC,YAAY9I,gBAAgB8I,UAAU;QAAA,GACtE9E,IAAI,CAAChE,iBACLiE,KAAK,CAACzF,IAAIM,QAAQkB,gBAAgBsG,EAAE,EAAEpG,OAAOiI,SAAS,GAAGK,iBAAiBnI,IAAIH,UAC9E+G,OAAO,CAACvI,IAAIsB,gBAAgB+D,SAAS,GAAGrF,IAAIsB,gBAAgBsG,EAAE,GAC9DyC,KAAK,CAAC7I,OAAOyC,KAAK;IACvB;IAEA,0FAA0F;IAC1F,4FAA4F;IAC5F,6FAA6F;IAC7F,wFAAwF;IACxF,MAAMqG,iBAAiBC,OAAO;IAC9B,MAAMnD,cAAc,MAAM,AAAC,CAAA;QACzB,MAAO,KAAM;YACX,IAAI;gBACF,OAAO,MAAMzF,GAAGD,WAAW,CAAC,OAAO8I;oBACjC,MAAMC,mBAAmB,MAAMD,OAC5BvF,MAAM,CAAC;wBAAC2C,IAAItG,gBAAgBsG,EAAE;wBAAEwC,YAAY9I,gBAAgB8I,UAAU;oBAAA,GACtE9E,IAAI,CAAChE,iBACLiE,KAAK,CAACuE,iBAAiBU,QAAQhJ,SAC/B+G,OAAO,CAACvI,IAAIsB,gBAAgB+D,SAAS,GAAGrF,IAAIsB,gBAAgBsG,EAAE,GAC9DyC,KAAK,CAAC7I,OAAOyC,KAAK;oBAErB,IAAIwG,iBAAiB7F,MAAM,KAAK,GAAG,OAAO6F;oBAE1C,sFAAsF;oBACtF,iFAAiF;oBACjF,MAAMP,gBAAyC,EAAE;oBACjD,KAAK,MAAMQ,aAAa;2BAAID;qBAAiB,CAAChG,IAAI,CAACkG,kBAAmB;wBACpE,MAAM,CAAC1B,OAAO,GAAG,MAAMuB,OACpBvF,MAAM,CAAC;4BAAC2C,IAAItG,gBAAgBsG,EAAE;4BAAEwC,YAAY9I,gBAAgB8I,UAAU;wBAAA,GACtE9E,IAAI,CAAChE,iBACLiE,KAAK,CAACzF,IAAIG,GAAGqB,gBAAgBsG,EAAE,EAAE8C,UAAU9C,EAAE,GAAGkC,iBAAiBU,QAAQhJ,UACzEgH,GAAG,CAAC;wBACP,IAAIS,QAAQiB,cAAcvG,IAAI,CAACsF;oBACjC;oBAEA,IAAIiB,cAActF,MAAM,KAAK6F,iBAAiB7F,MAAM,EAAE,MAAM0F;oBAC5D,OAAOJ;gBACT;YACF,EAAE,OAAOU,OAAO;gBACd,IAAIA,UAAUN,gBAAgB,MAAMM;YACtC;QACF;IACF,CAAA;IAEA,OAAOxD;AACT;AAEA,eAAeU,0BACbnG,EAAM,EACNH,MAOC;IAED,MAAMqJ,gBAAgBrJ,OAAO4F,WAAW,CAACzE,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;IAElE,MAAMjG,GACHmJ,MAAM,CAAC1J,wBACPuI,GAAG,CAAC;QAACoB,WAAWpK,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1B4E,KAAK,CACJzF,IACEM,QAAQgB,uBAAuB4I,gBAAgB,EAAEa,gBACjDvK,OAAOc,uBAAuB4J,UAAU,GACxC1K,OAAOc,uBAAuB2J,SAAS;IAI7C,0FAA0F;IAC1F,6FAA6F;IAC7F,eAAe;IACf,MAAME,eAAe,MAAMtJ,GACxBmJ,MAAM,CAACxJ,iBACPqI,GAAG,CAAC;QACH5H,aAAaP,OAAOO,WAAW;QAC/BgG,eAAevG,OAAOuG,aAAa;QACnCc,uBAAuB;QACvBqC,YAAYvK,GAAG,CAAC,KAAK,CAAC;QACtBwK,WAAWxK,GAAG,CAAC,KAAK,CAAC;IACvB,GACC4E,KAAK,CAACzF,IAAIM,QAAQkB,gBAAgBsG,EAAE,EAAEiD,gBAAgBf,iBAAiBnI,IAAIH,UAC3EmG,SAAS,CAAC;QAACC,IAAItG,gBAAgBsG,EAAE;QAAEwC,YAAY9I,gBAAgB8I,UAAU;IAAA;IAE5E,IAAIa,aAAarG,MAAM,KAAKpD,OAAO4F,WAAW,CAACxC,MAAM,EACnD,MAAM,IAAIiD,MAAM;IAElB,MAAMuD,eAAeH,aAAa1G,MAAM,CAAC,CAAC0E,SAAWA,OAAOmB,UAAU,KAAK,UAAUxF,MAAM;IAC3F,IAAIwG,eAAe,GACjBxK,sCAAsC;QAACyK,SAAS;QAAWpH,OAAOmH;IAAY;AAClF;AAEA,eAAehJ,mCAAmCC,oBAAyC;IACzF,IAAIA,qBAAqBiH,IAAI,KAAK,GAAG,OAAO,EAAE;IAC9C,MAAMlG,OAAO,MAAMvC,KAChBoE,MAAM,CAAC;QACNlD,aAAaf,qBAAqBe,WAAW;QAC7CqD,gBAAgBzE,GAAS,CAAC,IAAI,EAAEK,qBAAqBqE,SAAS,CAAC,CAAC,CAAC;IACnE,GACCC,IAAI,CAACtE,sBACLuE,KAAK,CAACnF,QAAQY,qBAAqBe,WAAW,EAAE;WAAIM;KAAqB,GACzEmD,OAAO,CAACxE,qBAAqBe,WAAW,EACxCwG,OAAO,CACNvI,IAAIW,GAAG,CAAC,IAAI,EAAEK,qBAAqBqE,SAAS,CAAC,CAAC,CAAC,GAC/CrF,IAAIgB,qBAAqBe,WAAW;IAExC,OAAOqB,KAAKT,GAAG,CAAC,CAAC8C,MAAQA,IAAI1D,WAAW;AAC1C;AAEA,OAAO,eAAeuJ;IACpB,MAAMlI,OAAO,MAAMvC,KAChBoE,MAAM,CAAC;QAAClD,aAAaf,qBAAqBe,WAAW;IAAA,GACrDuD,IAAI,CAACtE,sBACLwE,OAAO,CAACxE,qBAAqBe,WAAW;IAC3C,OAAOqB,KAAKT,GAAG,CAAC,CAAC8C,MAAQA,IAAI1D,WAAW;AAC1C;AAEA,eAAe6D,sCACbjE,EAAM,EACNH,MAAoD;IAEpD,MAAM4B,OAAO,MAAMzB,GAChBsD,MAAM,CAAC;QAACpC,QAAQ5B,+BAA+B4B,MAAM;IAAA,GACrDyC,IAAI,CAACrE,gCACLsK,SAAS,CACRrK,mBACAjB,GAAGiB,kBAAkB0G,EAAE,EAAE3G,+BAA+BqC,aAAa,GAEtEiC,KAAK,CACJzF,IACEG,GAAGgB,+BAA+Bc,WAAW,EAAEP,OAAOO,WAAW,GACjE9B,GAAGiB,kBAAkBa,WAAW,EAAEP,OAAOO,WAAW,GACpD9B,GAAGiB,kBAAkBe,KAAK,EAAE,cAC5BtB,GAAG,CAAC,EAAEM,+BAA+BuK,YAAY,CAAC,YAAY,EAAEhK,OAAOqE,aAAa,CAAC,yBAAyB,CAAC,EAC/GlF,GAAG,CAAC,EAAEO,kBAAkB6J,SAAS,CAAC,QAAQ,CAAC,EAC3CpK,GAAG,CAAC,CAAC,EAAEO,kBAAkBgF,SAAS,CAAC,YAAY,EAAEhF,kBAAkBgF,SAAS,CAAC,SAAS,CAAC;IAG7F,OAAO9C,KAAKT,GAAG,CAAC,CAAC8C,MAAQA,IAAI5C,MAAM;AACrC;AAEA,OAAO,eAAe4I,0BAA0BjK,MAAyB;IACvE,OAAO,MAAMX,KAAKa,WAAW,CAAC,OAAOC;QACnC,MAAM+J,cAAc,MAAM/J,GACvBsD,MAAM,CAAC;YAAC2C,IAAIzG,aAAayG,EAAE;QAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CAAChF,GAAGY,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,GAC3C4H,OAAO,CAACvI,IAAImB,aAAa+E,SAAS,GAClCmE,KAAK,CAAC7I,QAAQ6I,SAAS;QAC1B,OAAO,MAAMsB,gCACXhK,IACA+J,YAAY/I,GAAG,CAAC,CAACqB,cAAgBA,YAAY4D,EAAE,GAC/C;YAACgE,aAAa;QAAI;IAEtB;AACF;AAEA,OAAO,eAAeC,wBAAwBC,GAAa;IACzD,IAAIA,IAAIlH,MAAM,KAAK,GAAG,OAAO;IAE7B,OAAO,MAAM/D,KAAKa,WAAW,CAAC,OAAOC;QACnC,OAAO,MAAMgK,gCAAgChK,IAAImK;IACnD;AACF;AAEA,eAAeH,gCACbhK,EAAM,EACNmK,GAAa,EACbtK,SAAkC,CAAC,CAAC;IAEpC,IAAIsK,IAAIlH,MAAM,KAAK,GAAG,OAAO;IAE7B,MAAMmH,mBAAmB,IACvBrL,GACEZ,IACEM,QAAQkB,gBAAgByG,aAAa,EAAE+D,MACvCxL,OAAOgB,gBAAgBuI,eAAe,GACtCvJ,OAAOgB,gBAAgByH,qBAAqB,IAE9C3I,QAAQkB,gBAAgBuH,qBAAqB,EAAEiD;IAEnD,MAAMrB,mBAAmB,MAAM9I,GAC5BsD,MAAM,CAAC;QACN2C,IAAItG,gBAAgBsG,EAAE;QACtBG,eAAezG,gBAAgByG,aAAa;QAC5Cc,uBAAuBvH,gBAAgBuH,qBAAqB;QAC5DgB,iBAAiBvI,gBAAgBuI,eAAe;QAChDd,uBAAuBzH,gBAAgByH,qBAAqB;IAC9D,GACCzD,IAAI,CAAChE,iBACLiE,KAAK,CAACwG,oBACNxD,OAAO,CAACvI,IAAIsB,gBAAgBsG,EAAE;IACjC,MAAMoE,kBAA2C,EAAE;IACnD,KAAK,MAAMtB,aAAa;WAAID;KAAiB,CAAChG,IAAI,CAACkG,kBAAmB;QACpE,MAAM,CAAC1B,OAAO,GAAG,MAAMtH,GACpBsD,MAAM,CAAC;YACN2C,IAAItG,gBAAgBsG,EAAE;YACtBG,eAAezG,gBAAgByG,aAAa;YAC5Cc,uBAAuBvH,gBAAgBuH,qBAAqB;YAC5DgB,iBAAiBvI,gBAAgBuI,eAAe;YAChDd,uBAAuBzH,gBAAgByH,qBAAqB;QAC9D,GACCzD,IAAI,CAAChE,iBACLiE,KAAK,CAACzF,IAAIG,GAAGqB,gBAAgBsG,EAAE,EAAE8C,UAAU9C,EAAE,GAAGmE,qBAChDvD,GAAG,CAAC;QACP,IAAIS,QAAQ+C,gBAAgBrI,IAAI,CAACsF;IACnC;IACA,MAAMgD,kBAAkB,MAAMtK,GAC3BsD,MAAM,CAAC;QAAC2C,IAAIzG,aAAayG,EAAE;IAAA,GAC3BtC,IAAI,CAACnE,cACLoE,KAAK,CACJzF,IACEM,QAAQe,aAAayG,EAAE,EAAEkE,MACzBtK,OAAOoK,WAAW,GAAGrL,GAAGY,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAAI8C,YAGjE+E,GAAG,CAAC;IACP,MAAMH,iBAAiB4D,gBAAgBtJ,GAAG,CAAC,CAACqB,cAAgBA,YAAY4D,EAAE;IAE1E,IAAIS,eAAezD,MAAM,KAAK,GAAG,OAAO;IAExC,MAAMsH,oBAAoBF,gBACvBzH,MAAM,CACL,CAAC0E,SACCA,OAAOlB,aAAa,IACpBM,eAAee,QAAQ,CAACH,OAAOlB,aAAa,KAC5C,CAACkB,OAAOY,eAAe,IACvB,CAACZ,OAAOF,qBAAqB,EAEhCpG,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;IAC5B,MAAMuE,oBAAoBH,gBACvBzH,MAAM,CACL,CAAC0E,SACCA,OAAOJ,qBAAqB,IAAIR,eAAee,QAAQ,CAACH,OAAOJ,qBAAqB,GAEvFlG,GAAG,CAAC,CAACsG,SAAWA,OAAOrB,EAAE;IAC5B,IAAIsE,kBAAkBtH,MAAM,GAAG,GAAG;QAChC,MAAMjD,GACHmJ,MAAM,CAAC1J,wBACPuI,GAAG,CAAC;YAACoB,WAAWpK,GAAG,CAAC,KAAK,CAAC;QAAA,GAC1B4E,KAAK,CACJzF,IACEM,QAAQgB,uBAAuB4I,gBAAgB,EAAEkC,oBACjD5L,OAAOc,uBAAuB4J,UAAU,GACxC1K,OAAOc,uBAAuB2J,SAAS;QAG7C,MAAMpJ,GACHmJ,MAAM,CAACxJ,iBACPqI,GAAG,CAAC;YACH5H,aAAa;YACbgG,eAAe;YACfmD,YAAY;YACZC,WAAWxK,GAAG,CAAC,KAAK,CAAC;QACvB,GACC4E,KAAK,CACJzF,IACEM,QAAQkB,gBAAgBsG,EAAE,EAAEsE,oBAC5B5L,OAAOgB,gBAAgBuI,eAAe,GACtCvJ,OAAOgB,gBAAgByH,qBAAqB;IAGpD;IACA,IAAIoD,kBAAkBvH,MAAM,GAAG,GAAG;QAChC,MAAMjD,GACHmJ,MAAM,CAACxJ,iBACPqI,GAAG,CAAC;YAACd,uBAAuB;YAAMsC,WAAWxK,GAAG,CAAC,KAAK,CAAC;QAAA,GACvD4E,KAAK,CACJzF,IACEM,QAAQkB,gBAAgBsG,EAAE,EAAEuE,oBAC5B/L,QAAQkB,gBAAgBuH,qBAAqB,EAAER;IAGvD;IAEA,MAAM+D,UAAU,MAAMzK,GACnB0K,MAAM,CAAClL,cACPoE,KAAK,CAACnF,QAAQe,aAAayG,EAAE,EAAES,iBAC/BV,SAAS,CAAC;QAACC,IAAIzG,aAAayG,EAAE;IAAA;IAEjC,OAAOwE,QAAQxH,MAAM;AACvB;AAEA,OAAO,eAAe0H,wBACpB3K,EAAM,EACNH,MAIC;IAED,IAAI+K,WAAW;IAEf,KAAK,MAAMC,WAAWhL,OAAOiL,QAAQ,CAAE;QACrC,IAAID,QAAQvI,KAAK,IAAI,GAAG;QAExB,MAAMyI,cAAc,MAAM/K,GACvBmJ,MAAM,CAAC3J,cACPwI,GAAG,CAAC;YAAC1F,OAAOtD,GAAG,CAAC,EAAEQ,aAAa8C,KAAK,CAAC,GAAG,EAAEuI,QAAQvI,KAAK,CAAC,CAAC;QAAA,GACzDsB,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAE4E,QAAQzE,aAAa,GACzC9H,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnDnD,GAAGgB,aAAa8C,KAAK,EAAEuI,QAAQvI,KAAK,GACpC9D,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAGxCgH,SAAS,CAAC;YAACC,IAAIzG,aAAayG,EAAE;QAAA;QAEjC,IAAI8E,YAAY9H,MAAM,GAAG,GAAG;YAC1B2H,YAAYC,QAAQvI,KAAK;YACzB;QACF;QAEA,MAAMmI,UAAU,MAAMzK,GACnB0K,MAAM,CAAClL,cACPoE,KAAK,CACJzF,IACEG,GAAGkB,aAAayG,EAAE,EAAE4E,QAAQzE,aAAa,GACzC9H,GAAGkB,aAAaY,WAAW,EAAEP,OAAOO,WAAW,GAC/C9B,GAAGkB,aAAamC,aAAa,EAAE9B,OAAO8B,aAAa,GACnD3C,GAAG,CAAC,EAAEQ,aAAa8C,KAAK,CAAC,IAAI,EAAEuI,QAAQvI,KAAK,CAAC,CAAC,EAC9C9D,GAAGgB,aAAa+E,SAAS,EAAEvF,GAAG,CAAC,KAAK,CAAC,IAGxCgH,SAAS,CAAC;YAAC1D,OAAO9C,aAAa8C,KAAK;QAAA;QAEvCsI,YAAYH,QAAQtI,MAAM,CAAC,CAACC,OAAO0B,MAAQ1B,QAAQ0B,IAAIxB,KAAK,EAAE;IAChE;IAEA,OAAOsI;AACT;AAEA,SAAS/F,8BACP9D,SAA+B,EAC/BiK,kBAAkE;IAElE,KAAK,MAAM3I,eAAe4I,oBAAoBD,oBAAqB;QACjE,MAAMrI,sBAAsB5B,UACzB6B,MAAM,CAAC,CAAC3B,WAAa4B,SAASR,YAAYkB,cAAc,EAAEtC,SAASC,MAAM,GACzE4B,IAAI,CACH,CAACC,GAAGC,IAAMD,EAAE7B,MAAM,CAAC+B,MAAM,GAAGD,EAAE9B,MAAM,CAAC+B,MAAM,IAAIF,EAAEG,WAAW,CAACC,aAAa,CAACH,EAAEE,WAAW;QAE5FE,UAAUT,qBAAqBN,YAAYiC,QAAQ;IACrD;AACF;AAEA,SAAS0E,iBAAiBkC,IAAkB,EAAEC,KAAmB;IAC/D,OAAOD,KAAKjF,EAAE,GAAGkF,MAAMlF,EAAE,GAAG,CAAC,IAAIiF,KAAKjF,EAAE,GAAGkF,MAAMlF,EAAE,GAAG,IAAI;AAC5D;AAEA,SAASjB,eAAevD,IAAiB;IACvC,OAAO;WAAIA;KAAK,CAACqB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAMoI,cAAcpI,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAImI,gBAAgB,GAAG,OAAOA;QAC9B,OAAOrI,EAAEU,cAAc,CAAC4H,OAAO,KAAKrI,EAAES,cAAc,CAAC4H,OAAO;IAC9D;AACF;AAEA,SAASJ,oBAA0DxJ,IAAS;IAC1E,OAAO;WAAIA;KAAK,CAACqB,IAAI,CAAC,CAACC,GAAGC;QACxB,MAAMoI,cAAcpI,EAAEO,cAAc,CAACN,MAAM,GAAGF,EAAEQ,cAAc,CAACN,MAAM;QACrE,IAAImI,gBAAgB,GAAG,OAAOA;QAC9B,OAAO1G,SAAS3B,EAAEQ,cAAc,EAAEJ,aAAa,CAACuB,SAAS1B,EAAEO,cAAc;IAC3E;AACF;AAEA,SAASV,SAASU,cAAwB,EAAE+H,eAAyB;IACnE,OAAO/H,eAAegI,KAAK,CAAC,CAACC,QAAUF,gBAAgB7D,QAAQ,CAAC+D;AAClE;AAEA,SAASpI,UAAUrC,SAA+B,EAAEuB,KAAa;IAC/D,IAAImJ,YAAYnJ;IAChB,KAAK,MAAMrB,YAAYF,UAAW;QAChC,IAAI0K,cAAc,GAAG;QACrB,MAAM/D,OAAOvC,KAAKI,GAAG,CAACtE,SAASE,cAAc,EAAEsK;QAC/CxK,SAASE,cAAc,IAAIuG;QAC3B+D,aAAa/D;IACf;AACF;AAEA,SAAShD,SAASxD,MAAgB;IAChC,OAAOwK,KAAKC,SAAS,CAACzK;AACxB"}
@@ -10,4 +10,21 @@ export declare function assignRunnerInstancesTx(tx: Tx, params: {
10
10
  reservationId: string;
11
11
  runnerInstanceIds: string[];
12
12
  }): Promise<string[]>;
13
+ export type RunnerReservationCapacityFailureReason = 'reservation-not-found' | 'reservation-kind-mismatch' | 'reservation-expired' | 'capacity-exhausted';
14
+ export interface RunnerReservationCapacityValidation {
15
+ acceptedByReservation: Map<string, number>;
16
+ unavailableByReservation: Map<string, {
17
+ reason: RunnerReservationCapacityFailureReason;
18
+ count: number;
19
+ }>;
20
+ }
21
+ export declare function validateRunnerReservationCapacityTx(tx: Tx, params: {
22
+ provisionerId: string;
23
+ requests: readonly {
24
+ reservationId: string;
25
+ count: number;
26
+ }[];
27
+ }, options?: {
28
+ advisoryLocksHeld?: boolean;
29
+ }): Promise<RunnerReservationCapacityValidation>;
13
30
  //# sourceMappingURL=runner-assignments.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"runner-assignments.d.ts","sourceRoot":"","sources":["../../src/db/runner-assignments.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAOhC,gGAAgG;AAChG,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAEpB;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GACA,OAAO,CAAC,MAAM,EAAE,CAAC,CAoInB"}
1
+ {"version":3,"file":"runner-assignments.d.ts","sourceRoot":"","sources":["../../src/db/runner-assignments.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAQhC,gGAAgG;AAChG,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAEpB;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GACA,OAAO,CAAC,MAAM,EAAE,CAAC,CAkJnB;AAED,MAAM,MAAM,sCAAsC,GAC9C,uBAAuB,GACvB,2BAA2B,GAC3B,qBAAqB,GACrB,oBAAoB,CAAC;AAEzB,MAAM,WAAW,mCAAmC;IAClD,qBAAqB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,wBAAwB,EAAE,GAAG,CAC3B,MAAM,EACN;QAAC,MAAM,EAAE,sCAAsC,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAChE,CAAC;CACH;AAED,wBAAsB,mCAAmC,CACvD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,SAAS;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,EAAE,CAAC;CAC7D,EACD,OAAO,GAAE;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;CAAM,GAC1C,OAAO,CAAC,mCAAmC,CAAC,CAmI9C"}
@@ -1,7 +1,8 @@
1
- import { and, eq, inArray, isNull, notInArray, or, sql } from 'drizzle-orm';
1
+ import { and, eq, inArray, isNull, ne, notInArray, or, sql } from 'drizzle-orm';
2
2
  import { ReservationExpiredError, ReservationNotFoundError, RunnerInstanceAlreadyAssignedError, RunnerInstanceNotAssignableError } from '#core/errors.js';
3
3
  import { db } from './db.js';
4
- import { terminalStates } from './runner-instances.js';
4
+ import { lockRunnerReservationAdvisoryKeysTx } from './reservation-locks.js';
5
+ import { terminalStates } from './runner-states.js';
5
6
  import { reservations } from './schema/reservations.js';
6
7
  import { runnerControlSessions } from './schema/runner-control-sessions.js';
7
8
  import { providerRunners } from './schema/runner-instances.js';
@@ -12,26 +13,32 @@ export async function assignRunnerInstancesTx(tx, params) {
12
13
  const runnerInstanceIds = [
13
14
  ...params.runnerInstanceIds
14
15
  ].sort();
15
- await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${`runners_assignment:${params.provisionerId}:${params.reservationId}`}))`);
16
+ await lockRunnerReservationAdvisoryKeysTx(tx, {
17
+ provisionerId: params.provisionerId,
18
+ reservationIds: [
19
+ params.reservationId
20
+ ]
21
+ });
16
22
  const runnerRows = await tx.select({
17
23
  id: providerRunners.id,
18
24
  reservationId: providerRunners.reservationId,
19
25
  intendedReservationId: providerRunners.intendedReservationId,
20
26
  workspaceId: providerRunners.workspaceId,
27
+ assignedAt: providerRunners.assignedAt,
21
28
  providerRunnerId: providerRunners.providerRunnerId,
22
29
  labels: providerRunners.labels,
23
30
  state: providerRunners.state
24
31
  }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.id, runnerInstanceIds))).for('update');
25
32
  // The assignment outlives its short reservation row. A retry after maintenance
26
- // cleanup is complete when every owned runner already carries the requested assignment.
27
- if (runnerInstanceIds.length > 0 && runnerRows.length === runnerInstanceIds.length && runnerRows.every((runner)=>runner.reservationId === params.reservationId)) {
33
+ // cleanup is complete when every owned runner already carries the committed assignment.
34
+ if (runnerInstanceIds.length > 0 && runnerRows.length === runnerInstanceIds.length && runnerRows.every((runner)=>runner.reservationId === params.reservationId && runner.assignedAt !== null && runner.workspaceId !== null)) {
28
35
  await tx.update(providerRunners).set({
29
36
  intendedReservationId: null,
30
37
  updatedAt: sql`now()`
31
38
  }).where(and(eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.id, runnerInstanceIds)));
32
39
  return params.runnerInstanceIds;
33
40
  }
34
- const [reservation] = await tx.select().from(reservations).where(and(eq(reservations.id, params.reservationId), eq(reservations.provisionerId, params.provisionerId))).limit(1).for('update');
41
+ const [reservation] = await tx.select().from(reservations).where(and(eq(reservations.id, params.reservationId), eq(reservations.provisionerId, params.provisionerId), eq(reservations.kind, 'launch'))).limit(1).for('update');
35
42
  if (!reservation) throw new ReservationNotFoundError(params.reservationId);
36
43
  if (reservation.expiresAt <= new Date()) throw new ReservationExpiredError(params.reservationId);
37
44
  if (runnerRows.length !== runnerInstanceIds.length) throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '');
@@ -45,10 +52,11 @@ export async function assignRunnerInstancesTx(tx, params) {
45
52
  }));
46
53
  const alreadyAssigned = runners.filter((runner)=>runner.reservationId !== null);
47
54
  if (alreadyAssigned.some((runner)=>runner.reservationId !== reservation.id)) throw new RunnerInstanceAlreadyAssignedError(alreadyAssigned[0]?.id ?? '');
48
- const newRunners = runners.filter((runner)=>runner.reservationId === null);
55
+ const newRunners = runners.filter((runner)=>runner.reservationId === null || runner.reservationId === reservation.id && (runner.assignedAt === null || runner.workspaceId === null));
56
+ const newRunnerIds = newRunners.map((runner)=>runner.id);
49
57
  const assignedCount = await tx.select({
50
58
  count: sql`count(*)::int`
51
- }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), or(and(eq(providerRunners.reservationId, reservation.id), isNull(providerRunners.reservationReleasedAt)), and(eq(providerRunners.intendedReservationId, reservation.id), notInArray(providerRunners.id, runnerInstanceIds), isNull(providerRunners.reservationReleasedAt), notInArray(providerRunners.state, [
59
+ }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), notInArray(providerRunners.id, newRunnerIds), or(and(eq(providerRunners.reservationId, reservation.id), isNull(providerRunners.reservationReleasedAt)), and(eq(providerRunners.intendedReservationId, reservation.id), isNull(providerRunners.reservationReleasedAt), notInArray(providerRunners.state, [
52
60
  ...terminalStates
53
61
  ])))));
54
62
  if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count) throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '');
@@ -66,5 +74,92 @@ export async function assignRunnerInstancesTx(tx, params) {
66
74
  }
67
75
  return params.runnerInstanceIds;
68
76
  }
77
+ export async function validateRunnerReservationCapacityTx(tx, params, options = {}) {
78
+ const requestedByReservation = new Map();
79
+ for (const request of params.requests){
80
+ if (request.count <= 0) continue;
81
+ requestedByReservation.set(request.reservationId, (requestedByReservation.get(request.reservationId) ?? 0) + request.count);
82
+ }
83
+ const reservationIds = [
84
+ ...requestedByReservation.keys()
85
+ ].sort();
86
+ if (reservationIds.length === 0) return {
87
+ acceptedByReservation: new Map(),
88
+ unavailableByReservation: new Map()
89
+ };
90
+ if (!options.advisoryLocksHeld) await lockRunnerReservationAdvisoryKeysTx(tx, {
91
+ provisionerId: params.provisionerId,
92
+ reservationIds
93
+ });
94
+ const reservationRows = await tx.select({
95
+ id: reservations.id,
96
+ count: reservations.count,
97
+ expiresAt: reservations.expiresAt,
98
+ isExpired: sql`${reservations.expiresAt} <= now()`
99
+ }).from(reservations).where(and(eq(reservations.provisionerId, params.provisionerId), inArray(reservations.id, reservationIds), eq(reservations.kind, 'launch'))).for('update');
100
+ const nonLaunchReservationRows = await tx.select({
101
+ id: reservations.id
102
+ }).from(reservations).where(and(eq(reservations.provisionerId, params.provisionerId), inArray(reservations.id, reservationIds), ne(reservations.kind, 'launch'))).for('update');
103
+ const reservationsById = new Map(reservationRows.map((reservation)=>[
104
+ reservation.id,
105
+ reservation
106
+ ]));
107
+ const nonLaunchReservationIds = new Set(nonLaunchReservationRows.map((reservation)=>reservation.id));
108
+ const requestedReservationIds = new Set(reservationIds);
109
+ const usedRunnerIdsByReservation = new Map();
110
+ const usedRunnerRows = await tx.select({
111
+ id: providerRunners.id,
112
+ reservationId: providerRunners.reservationId,
113
+ intendedReservationId: providerRunners.intendedReservationId,
114
+ state: providerRunners.state
115
+ }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), isNull(providerRunners.reservationReleasedAt), or(inArray(providerRunners.reservationId, reservationIds), and(inArray(providerRunners.intendedReservationId, reservationIds), notInArray(providerRunners.state, [
116
+ ...terminalStates
117
+ ])))));
118
+ for (const runner of usedRunnerRows){
119
+ if (runner.reservationId && requestedReservationIds.has(runner.reservationId)) {
120
+ const usedRunnerIds = usedRunnerIdsByReservation.get(runner.reservationId) ?? new Set();
121
+ usedRunnerIds.add(runner.id);
122
+ usedRunnerIdsByReservation.set(runner.reservationId, usedRunnerIds);
123
+ }
124
+ if (runner.intendedReservationId && requestedReservationIds.has(runner.intendedReservationId) && !terminalStates.includes(runner.state)) {
125
+ const usedRunnerIds = usedRunnerIdsByReservation.get(runner.intendedReservationId) ?? new Set();
126
+ usedRunnerIds.add(runner.id);
127
+ usedRunnerIdsByReservation.set(runner.intendedReservationId, usedRunnerIds);
128
+ }
129
+ }
130
+ const acceptedByReservation = new Map();
131
+ const unavailableByReservation = new Map();
132
+ for (const reservationId of reservationIds){
133
+ const requested = requestedByReservation.get(reservationId) ?? 0;
134
+ const reservation = reservationsById.get(reservationId);
135
+ if (!reservation) {
136
+ unavailableByReservation.set(reservationId, {
137
+ reason: nonLaunchReservationIds.has(reservationId) ? 'reservation-kind-mismatch' : 'reservation-not-found',
138
+ count: requested
139
+ });
140
+ continue;
141
+ }
142
+ if (reservation.isExpired) {
143
+ unavailableByReservation.set(reservationId, {
144
+ reason: 'reservation-expired',
145
+ count: requested
146
+ });
147
+ continue;
148
+ }
149
+ const used = usedRunnerIdsByReservation.get(reservationId)?.size ?? 0;
150
+ const accepted = Math.min(requested, Math.max(0, reservation.count - used));
151
+ acceptedByReservation.set(reservationId, accepted);
152
+ if (accepted < requested) {
153
+ unavailableByReservation.set(reservationId, {
154
+ reason: 'capacity-exhausted',
155
+ count: requested - accepted
156
+ });
157
+ }
158
+ }
159
+ return {
160
+ acceptedByReservation,
161
+ unavailableByReservation
162
+ };
163
+ }
69
164
 
70
165
  //# sourceMappingURL=runner-assignments.js.map
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/db/runner-assignments.ts"],"sourcesContent":["import {and, eq, inArray, isNull, notInArray, or, sql} from 'drizzle-orm';\nimport {\n ReservationExpiredError,\n ReservationNotFoundError,\n RunnerInstanceAlreadyAssignedError,\n RunnerInstanceNotAssignableError,\n} from '#core/errors.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {terminalStates} from './runner-instances.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {providerRunners} from './schema/runner-instances.js';\n\n/** Atomically consumes reservation units by writing the immutable assignment on each runner. */\nexport async function assignRunnerInstances(params: {\n provisionerId: string;\n reservationId: string;\n runnerInstanceIds: string[];\n}): Promise<string[]> {\n return await db().transaction(async (tx) => assignRunnerInstancesTx(tx, params));\n}\n\nexport async function assignRunnerInstancesTx(\n tx: Tx,\n params: {\n provisionerId: string;\n reservationId: string;\n runnerInstanceIds: string[];\n },\n): Promise<string[]> {\n const runnerInstanceIds = [...params.runnerInstanceIds].sort();\n await tx.execute(\n sql`select pg_advisory_xact_lock(hashtext(${`runners_assignment:${params.provisionerId}:${params.reservationId}`}))`,\n );\n const runnerRows = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n workspaceId: providerRunners.workspaceId,\n providerRunnerId: providerRunners.providerRunnerId,\n labels: providerRunners.labels,\n state: providerRunners.state,\n })\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n inArray(providerRunners.id, runnerInstanceIds),\n ),\n )\n .for('update');\n\n // The assignment outlives its short reservation row. A retry after maintenance\n // cleanup is complete when every owned runner already carries the requested assignment.\n if (\n runnerInstanceIds.length > 0 &&\n runnerRows.length === runnerInstanceIds.length &&\n runnerRows.every((runner) => runner.reservationId === params.reservationId)\n ) {\n await tx\n .update(providerRunners)\n .set({intendedReservationId: null, updatedAt: sql`now()`})\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n inArray(providerRunners.id, runnerInstanceIds),\n ),\n );\n return params.runnerInstanceIds;\n }\n\n const [reservation] = await tx\n .select()\n .from(reservations)\n .where(\n and(\n eq(reservations.id, params.reservationId),\n eq(reservations.provisionerId, params.provisionerId),\n ),\n )\n .limit(1)\n .for('update');\n if (!reservation) throw new ReservationNotFoundError(params.reservationId);\n if (reservation.expiresAt <= new Date()) throw new ReservationExpiredError(params.reservationId);\n if (runnerRows.length !== runnerInstanceIds.length)\n throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '');\n\n const activeControlSessions = await tx\n .select({runnerInstanceId: runnerControlSessions.runnerInstanceId})\n .from(runnerControlSessions)\n .where(\n and(\n inArray(\n runnerControlSessions.runnerInstanceId,\n runnerRows.map((runner) => runner.id),\n ),\n isNull(runnerControlSessions.closedAt),\n ),\n );\n const runnerInstanceIdsWithControlSession = new Set(\n activeControlSessions.map((session) => session.runnerInstanceId),\n );\n const runners = runnerRows.map((runner) => ({\n ...runner,\n controlSessionId: runnerInstanceIdsWithControlSession.has(runner.id) ? runner.id : null,\n }));\n\n const alreadyAssigned = runners.filter((runner) => runner.reservationId !== null);\n if (alreadyAssigned.some((runner) => runner.reservationId !== reservation.id))\n throw new RunnerInstanceAlreadyAssignedError(alreadyAssigned[0]?.id ?? '');\n const newRunners = runners.filter((runner) => runner.reservationId === null);\n const assignedCount = await tx\n .select({count: sql<number>`count(*)::int`})\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n or(\n and(\n eq(providerRunners.reservationId, reservation.id),\n isNull(providerRunners.reservationReleasedAt),\n ),\n and(\n eq(providerRunners.intendedReservationId, reservation.id),\n notInArray(providerRunners.id, runnerInstanceIds),\n isNull(providerRunners.reservationReleasedAt),\n notInArray(providerRunners.state, [...terminalStates]),\n ),\n ),\n ),\n );\n if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count)\n throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '');\n for (const runner of newRunners) {\n if (\n runner.state !== 'running' ||\n !runner.providerRunnerId ||\n !runner.controlSessionId ||\n !reservation.requiredLabels.every((label) => runner.labels.includes(label))\n )\n throw new RunnerInstanceNotAssignableError(runner.id);\n }\n if (newRunners.length > 0) {\n await tx\n .update(providerRunners)\n .set({\n workspaceId: reservation.workspaceId,\n reservationId: reservation.id,\n intendedReservationId: null,\n assignedAt: sql`now()`,\n updatedAt: sql`now()`,\n })\n .where(\n inArray(\n providerRunners.id,\n newRunners.map((runner) => runner.id),\n ),\n );\n }\n return params.runnerInstanceIds;\n}\n"],"names":["and","eq","inArray","isNull","notInArray","or","sql","ReservationExpiredError","ReservationNotFoundError","RunnerInstanceAlreadyAssignedError","RunnerInstanceNotAssignableError","db","terminalStates","reservations","runnerControlSessions","providerRunners","assignRunnerInstances","params","transaction","tx","assignRunnerInstancesTx","runnerInstanceIds","sort","execute","provisionerId","reservationId","runnerRows","select","id","intendedReservationId","workspaceId","providerRunnerId","labels","state","from","where","for","length","every","runner","update","set","updatedAt","reservation","limit","expiresAt","Date","activeControlSessions","runnerInstanceId","map","closedAt","runnerInstanceIdsWithControlSession","Set","session","runners","controlSessionId","has","alreadyAssigned","filter","some","newRunners","assignedCount","count","reservationReleasedAt","requiredLabels","label","includes","assignedAt"],"mappings":"AAAA,SAAQA,GAAG,EAAEC,EAAE,EAAEC,OAAO,EAAEC,MAAM,EAAEC,UAAU,EAAEC,EAAE,EAAEC,GAAG,QAAO,cAAc;AAC1E,SACEC,uBAAuB,EACvBC,wBAAwB,EACxBC,kCAAkC,EAClCC,gCAAgC,QAC3B,kBAAkB;AAEzB,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,cAAc,QAAO,wBAAwB;AACrD,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAAQC,eAAe,QAAO,+BAA+B;AAE7D,8FAA8F,GAC9F,OAAO,eAAeC,sBAAsBC,MAI3C;IACC,OAAO,MAAMN,KAAKO,WAAW,CAAC,OAAOC,KAAOC,wBAAwBD,IAAIF;AAC1E;AAEA,OAAO,eAAeG,wBACpBD,EAAM,EACNF,MAIC;IAED,MAAMI,oBAAoB;WAAIJ,OAAOI,iBAAiB;KAAC,CAACC,IAAI;IAC5D,MAAMH,GAAGI,OAAO,CACdjB,GAAG,CAAC,sCAAsC,EAAE,CAAC,mBAAmB,EAAEW,OAAOO,aAAa,CAAC,CAAC,EAAEP,OAAOQ,aAAa,EAAE,CAAC,EAAE,CAAC;IAEtH,MAAMC,aAAa,MAAMP,GACtBQ,MAAM,CAAC;QACNC,IAAIb,gBAAgBa,EAAE;QACtBH,eAAeV,gBAAgBU,aAAa;QAC5CI,uBAAuBd,gBAAgBc,qBAAqB;QAC5DC,aAAaf,gBAAgBe,WAAW;QACxCC,kBAAkBhB,gBAAgBgB,gBAAgB;QAClDC,QAAQjB,gBAAgBiB,MAAM;QAC9BC,OAAOlB,gBAAgBkB,KAAK;IAC9B,GACCC,IAAI,CAACnB,iBACLoB,KAAK,CACJnC,IACEC,GAAGc,gBAAgBS,aAAa,EAAEP,OAAOO,aAAa,GACtDtB,QAAQa,gBAAgBa,EAAE,EAAEP,qBAG/Be,GAAG,CAAC;IAEP,+EAA+E;IAC/E,wFAAwF;IACxF,IACEf,kBAAkBgB,MAAM,GAAG,KAC3BX,WAAWW,MAAM,KAAKhB,kBAAkBgB,MAAM,IAC9CX,WAAWY,KAAK,CAAC,CAACC,SAAWA,OAAOd,aAAa,KAAKR,OAAOQ,aAAa,GAC1E;QACA,MAAMN,GACHqB,MAAM,CAACzB,iBACP0B,GAAG,CAAC;YAACZ,uBAAuB;YAAMa,WAAWpC,GAAG,CAAC,KAAK,CAAC;QAAA,GACvD6B,KAAK,CACJnC,IACEC,GAAGc,gBAAgBS,aAAa,EAAEP,OAAOO,aAAa,GACtDtB,QAAQa,gBAAgBa,EAAE,EAAEP;QAGlC,OAAOJ,OAAOI,iBAAiB;IACjC;IAEA,MAAM,CAACsB,YAAY,GAAG,MAAMxB,GACzBQ,MAAM,GACNO,IAAI,CAACrB,cACLsB,KAAK,CACJnC,IACEC,GAAGY,aAAae,EAAE,EAAEX,OAAOQ,aAAa,GACxCxB,GAAGY,aAAaW,aAAa,EAAEP,OAAOO,aAAa,IAGtDoB,KAAK,CAAC,GACNR,GAAG,CAAC;IACP,IAAI,CAACO,aAAa,MAAM,IAAInC,yBAAyBS,OAAOQ,aAAa;IACzE,IAAIkB,YAAYE,SAAS,IAAI,IAAIC,QAAQ,MAAM,IAAIvC,wBAAwBU,OAAOQ,aAAa;IAC/F,IAAIC,WAAWW,MAAM,KAAKhB,kBAAkBgB,MAAM,EAChD,MAAM,IAAI3B,iCAAiCW,iBAAiB,CAAC,EAAE,IAAI;IAErE,MAAM0B,wBAAwB,MAAM5B,GACjCQ,MAAM,CAAC;QAACqB,kBAAkBlC,sBAAsBkC,gBAAgB;IAAA,GAChEd,IAAI,CAACpB,uBACLqB,KAAK,CACJnC,IACEE,QACEY,sBAAsBkC,gBAAgB,EACtCtB,WAAWuB,GAAG,CAAC,CAACV,SAAWA,OAAOX,EAAE,IAEtCzB,OAAOW,sBAAsBoC,QAAQ;IAG3C,MAAMC,sCAAsC,IAAIC,IAC9CL,sBAAsBE,GAAG,CAAC,CAACI,UAAYA,QAAQL,gBAAgB;IAEjE,MAAMM,UAAU5B,WAAWuB,GAAG,CAAC,CAACV,SAAY,CAAA;YAC1C,GAAGA,MAAM;YACTgB,kBAAkBJ,oCAAoCK,GAAG,CAACjB,OAAOX,EAAE,IAAIW,OAAOX,EAAE,GAAG;QACrF,CAAA;IAEA,MAAM6B,kBAAkBH,QAAQI,MAAM,CAAC,CAACnB,SAAWA,OAAOd,aAAa,KAAK;IAC5E,IAAIgC,gBAAgBE,IAAI,CAAC,CAACpB,SAAWA,OAAOd,aAAa,KAAKkB,YAAYf,EAAE,GAC1E,MAAM,IAAInB,mCAAmCgD,eAAe,CAAC,EAAE,EAAE7B,MAAM;IACzE,MAAMgC,aAAaN,QAAQI,MAAM,CAAC,CAACnB,SAAWA,OAAOd,aAAa,KAAK;IACvE,MAAMoC,gBAAgB,MAAM1C,GACzBQ,MAAM,CAAC;QAACmC,OAAOxD,GAAW,CAAC,aAAa,CAAC;IAAA,GACzC4B,IAAI,CAACnB,iBACLoB,KAAK,CACJnC,IACEC,GAAGc,gBAAgBS,aAAa,EAAEP,OAAOO,aAAa,GACtDnB,GACEL,IACEC,GAAGc,gBAAgBU,aAAa,EAAEkB,YAAYf,EAAE,GAChDzB,OAAOY,gBAAgBgD,qBAAqB,IAE9C/D,IACEC,GAAGc,gBAAgBc,qBAAqB,EAAEc,YAAYf,EAAE,GACxDxB,WAAWW,gBAAgBa,EAAE,EAAEP,oBAC/BlB,OAAOY,gBAAgBgD,qBAAqB,GAC5C3D,WAAWW,gBAAgBkB,KAAK,EAAE;WAAIrB;KAAe;IAK/D,IAAI,AAACiD,CAAAA,aAAa,CAAC,EAAE,EAAEC,SAAS,CAAA,IAAKF,WAAWvB,MAAM,GAAGM,YAAYmB,KAAK,EACxE,MAAM,IAAIpD,iCAAiCkD,UAAU,CAAC,EAAE,EAAEhC,MAAM;IAClE,KAAK,MAAMW,UAAUqB,WAAY;QAC/B,IACErB,OAAON,KAAK,KAAK,aACjB,CAACM,OAAOR,gBAAgB,IACxB,CAACQ,OAAOgB,gBAAgB,IACxB,CAACZ,YAAYqB,cAAc,CAAC1B,KAAK,CAAC,CAAC2B,QAAU1B,OAAOP,MAAM,CAACkC,QAAQ,CAACD,SAEpE,MAAM,IAAIvD,iCAAiC6B,OAAOX,EAAE;IACxD;IACA,IAAIgC,WAAWvB,MAAM,GAAG,GAAG;QACzB,MAAMlB,GACHqB,MAAM,CAACzB,iBACP0B,GAAG,CAAC;YACHX,aAAaa,YAAYb,WAAW;YACpCL,eAAekB,YAAYf,EAAE;YAC7BC,uBAAuB;YACvBsC,YAAY7D,GAAG,CAAC,KAAK,CAAC;YACtBoC,WAAWpC,GAAG,CAAC,KAAK,CAAC;QACvB,GACC6B,KAAK,CACJjC,QACEa,gBAAgBa,EAAE,EAClBgC,WAAWX,GAAG,CAAC,CAACV,SAAWA,OAAOX,EAAE;IAG5C;IACA,OAAOX,OAAOI,iBAAiB;AACjC"}
1
+ {"version":3,"sources":["../../src/db/runner-assignments.ts"],"sourcesContent":["import {and, eq, inArray, isNull, ne, notInArray, or, sql} from 'drizzle-orm';\nimport {\n ReservationExpiredError,\n ReservationNotFoundError,\n RunnerInstanceAlreadyAssignedError,\n RunnerInstanceNotAssignableError,\n} from '#core/errors.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';\nimport {terminalStates} from './runner-states.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {providerRunners} from './schema/runner-instances.js';\n\n/** Atomically consumes reservation units by writing the immutable assignment on each runner. */\nexport async function assignRunnerInstances(params: {\n provisionerId: string;\n reservationId: string;\n runnerInstanceIds: string[];\n}): Promise<string[]> {\n return await db().transaction(async (tx) => assignRunnerInstancesTx(tx, params));\n}\n\nexport async function assignRunnerInstancesTx(\n tx: Tx,\n params: {\n provisionerId: string;\n reservationId: string;\n runnerInstanceIds: string[];\n },\n): Promise<string[]> {\n const runnerInstanceIds = [...params.runnerInstanceIds].sort();\n await lockRunnerReservationAdvisoryKeysTx(tx, {\n provisionerId: params.provisionerId,\n reservationIds: [params.reservationId],\n });\n const runnerRows = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n workspaceId: providerRunners.workspaceId,\n assignedAt: providerRunners.assignedAt,\n providerRunnerId: providerRunners.providerRunnerId,\n labels: providerRunners.labels,\n state: providerRunners.state,\n })\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n inArray(providerRunners.id, runnerInstanceIds),\n ),\n )\n .for('update');\n\n // The assignment outlives its short reservation row. A retry after maintenance\n // cleanup is complete when every owned runner already carries the committed assignment.\n if (\n runnerInstanceIds.length > 0 &&\n runnerRows.length === runnerInstanceIds.length &&\n runnerRows.every(\n (runner) =>\n runner.reservationId === params.reservationId &&\n runner.assignedAt !== null &&\n runner.workspaceId !== null,\n )\n ) {\n await tx\n .update(providerRunners)\n .set({intendedReservationId: null, updatedAt: sql`now()`})\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n inArray(providerRunners.id, runnerInstanceIds),\n ),\n );\n return params.runnerInstanceIds;\n }\n\n const [reservation] = await tx\n .select()\n .from(reservations)\n .where(\n and(\n eq(reservations.id, params.reservationId),\n eq(reservations.provisionerId, params.provisionerId),\n eq(reservations.kind, 'launch'),\n ),\n )\n .limit(1)\n .for('update');\n if (!reservation) throw new ReservationNotFoundError(params.reservationId);\n if (reservation.expiresAt <= new Date()) throw new ReservationExpiredError(params.reservationId);\n if (runnerRows.length !== runnerInstanceIds.length)\n throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '');\n\n const activeControlSessions = await tx\n .select({runnerInstanceId: runnerControlSessions.runnerInstanceId})\n .from(runnerControlSessions)\n .where(\n and(\n inArray(\n runnerControlSessions.runnerInstanceId,\n runnerRows.map((runner) => runner.id),\n ),\n isNull(runnerControlSessions.closedAt),\n ),\n );\n const runnerInstanceIdsWithControlSession = new Set(\n activeControlSessions.map((session) => session.runnerInstanceId),\n );\n const runners = runnerRows.map((runner) => ({\n ...runner,\n controlSessionId: runnerInstanceIdsWithControlSession.has(runner.id) ? runner.id : null,\n }));\n\n const alreadyAssigned = runners.filter((runner) => runner.reservationId !== null);\n if (alreadyAssigned.some((runner) => runner.reservationId !== reservation.id))\n throw new RunnerInstanceAlreadyAssignedError(alreadyAssigned[0]?.id ?? '');\n const newRunners = runners.filter(\n (runner) =>\n runner.reservationId === null ||\n (runner.reservationId === reservation.id &&\n (runner.assignedAt === null || runner.workspaceId === null)),\n );\n const newRunnerIds = newRunners.map((runner) => runner.id);\n const assignedCount = await tx\n .select({count: sql<number>`count(*)::int`})\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n notInArray(providerRunners.id, newRunnerIds),\n or(\n and(\n eq(providerRunners.reservationId, reservation.id),\n isNull(providerRunners.reservationReleasedAt),\n ),\n and(\n eq(providerRunners.intendedReservationId, reservation.id),\n isNull(providerRunners.reservationReleasedAt),\n notInArray(providerRunners.state, [...terminalStates]),\n ),\n ),\n ),\n );\n if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count)\n throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '');\n for (const runner of newRunners) {\n if (\n runner.state !== 'running' ||\n !runner.providerRunnerId ||\n !runner.controlSessionId ||\n !reservation.requiredLabels.every((label) => runner.labels.includes(label))\n )\n throw new RunnerInstanceNotAssignableError(runner.id);\n }\n if (newRunners.length > 0) {\n await tx\n .update(providerRunners)\n .set({\n workspaceId: reservation.workspaceId,\n reservationId: reservation.id,\n intendedReservationId: null,\n assignedAt: sql`now()`,\n updatedAt: sql`now()`,\n })\n .where(\n inArray(\n providerRunners.id,\n newRunners.map((runner) => runner.id),\n ),\n );\n }\n return params.runnerInstanceIds;\n}\n\nexport type RunnerReservationCapacityFailureReason =\n | 'reservation-not-found'\n | 'reservation-kind-mismatch'\n | 'reservation-expired'\n | 'capacity-exhausted';\n\nexport interface RunnerReservationCapacityValidation {\n acceptedByReservation: Map<string, number>;\n unavailableByReservation: Map<\n string,\n {reason: RunnerReservationCapacityFailureReason; count: number}\n >;\n}\n\nexport async function validateRunnerReservationCapacityTx(\n tx: Tx,\n params: {\n provisionerId: string;\n requests: readonly {reservationId: string; count: number}[];\n },\n options: {advisoryLocksHeld?: boolean} = {},\n): Promise<RunnerReservationCapacityValidation> {\n const requestedByReservation = new Map<string, number>();\n for (const request of params.requests) {\n if (request.count <= 0) continue;\n requestedByReservation.set(\n request.reservationId,\n (requestedByReservation.get(request.reservationId) ?? 0) + request.count,\n );\n }\n const reservationIds = [...requestedByReservation.keys()].sort();\n if (reservationIds.length === 0)\n return {acceptedByReservation: new Map(), unavailableByReservation: new Map()};\n\n if (!options.advisoryLocksHeld)\n await lockRunnerReservationAdvisoryKeysTx(tx, {\n provisionerId: params.provisionerId,\n reservationIds,\n });\n\n const reservationRows = await tx\n .select({\n id: reservations.id,\n count: reservations.count,\n expiresAt: reservations.expiresAt,\n isExpired: sql<boolean>`${reservations.expiresAt} <= now()`,\n })\n .from(reservations)\n .where(\n and(\n eq(reservations.provisionerId, params.provisionerId),\n inArray(reservations.id, reservationIds),\n eq(reservations.kind, 'launch'),\n ),\n )\n .for('update');\n const nonLaunchReservationRows = await tx\n .select({id: reservations.id})\n .from(reservations)\n .where(\n and(\n eq(reservations.provisionerId, params.provisionerId),\n inArray(reservations.id, reservationIds),\n ne(reservations.kind, 'launch'),\n ),\n )\n .for('update');\n const reservationsById = new Map(\n reservationRows.map((reservation) => [reservation.id, reservation]),\n );\n const nonLaunchReservationIds = new Set(\n nonLaunchReservationRows.map((reservation) => reservation.id),\n );\n const requestedReservationIds = new Set(reservationIds);\n const usedRunnerIdsByReservation = new Map<string, Set<string>>();\n const usedRunnerRows = await tx\n .select({\n id: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n state: providerRunners.state,\n })\n .from(providerRunners)\n .where(\n and(\n eq(providerRunners.provisionerId, params.provisionerId),\n isNull(providerRunners.reservationReleasedAt),\n or(\n inArray(providerRunners.reservationId, reservationIds),\n and(\n inArray(providerRunners.intendedReservationId, reservationIds),\n notInArray(providerRunners.state, [...terminalStates]),\n ),\n ),\n ),\n );\n for (const runner of usedRunnerRows) {\n if (runner.reservationId && requestedReservationIds.has(runner.reservationId)) {\n const usedRunnerIds =\n usedRunnerIdsByReservation.get(runner.reservationId) ?? new Set<string>();\n usedRunnerIds.add(runner.id);\n usedRunnerIdsByReservation.set(runner.reservationId, usedRunnerIds);\n }\n if (\n runner.intendedReservationId &&\n requestedReservationIds.has(runner.intendedReservationId) &&\n !terminalStates.includes(runner.state as (typeof terminalStates)[number])\n ) {\n const usedRunnerIds =\n usedRunnerIdsByReservation.get(runner.intendedReservationId) ?? new Set<string>();\n usedRunnerIds.add(runner.id);\n usedRunnerIdsByReservation.set(runner.intendedReservationId, usedRunnerIds);\n }\n }\n\n const acceptedByReservation = new Map<string, number>();\n const unavailableByReservation = new Map<\n string,\n {reason: RunnerReservationCapacityFailureReason; count: number}\n >();\n for (const reservationId of reservationIds) {\n const requested = requestedByReservation.get(reservationId) ?? 0;\n const reservation = reservationsById.get(reservationId);\n if (!reservation) {\n unavailableByReservation.set(reservationId, {\n reason: nonLaunchReservationIds.has(reservationId)\n ? 'reservation-kind-mismatch'\n : 'reservation-not-found',\n count: requested,\n });\n continue;\n }\n if (reservation.isExpired) {\n unavailableByReservation.set(reservationId, {\n reason: 'reservation-expired',\n count: requested,\n });\n continue;\n }\n\n const used = usedRunnerIdsByReservation.get(reservationId)?.size ?? 0;\n const accepted = Math.min(requested, Math.max(0, reservation.count - used));\n acceptedByReservation.set(reservationId, accepted);\n if (accepted < requested) {\n unavailableByReservation.set(reservationId, {\n reason: 'capacity-exhausted',\n count: requested - accepted,\n });\n }\n }\n\n return {acceptedByReservation, unavailableByReservation};\n}\n"],"names":["and","eq","inArray","isNull","ne","notInArray","or","sql","ReservationExpiredError","ReservationNotFoundError","RunnerInstanceAlreadyAssignedError","RunnerInstanceNotAssignableError","db","lockRunnerReservationAdvisoryKeysTx","terminalStates","reservations","runnerControlSessions","providerRunners","assignRunnerInstances","params","transaction","tx","assignRunnerInstancesTx","runnerInstanceIds","sort","provisionerId","reservationIds","reservationId","runnerRows","select","id","intendedReservationId","workspaceId","assignedAt","providerRunnerId","labels","state","from","where","for","length","every","runner","update","set","updatedAt","reservation","kind","limit","expiresAt","Date","activeControlSessions","runnerInstanceId","map","closedAt","runnerInstanceIdsWithControlSession","Set","session","runners","controlSessionId","has","alreadyAssigned","filter","some","newRunners","newRunnerIds","assignedCount","count","reservationReleasedAt","requiredLabels","label","includes","validateRunnerReservationCapacityTx","options","requestedByReservation","Map","request","requests","get","keys","acceptedByReservation","unavailableByReservation","advisoryLocksHeld","reservationRows","isExpired","nonLaunchReservationRows","reservationsById","nonLaunchReservationIds","requestedReservationIds","usedRunnerIdsByReservation","usedRunnerRows","usedRunnerIds","add","requested","reason","used","size","accepted","Math","min","max"],"mappings":"AAAA,SAAQA,GAAG,EAAEC,EAAE,EAAEC,OAAO,EAAEC,MAAM,EAAEC,EAAE,EAAEC,UAAU,EAAEC,EAAE,EAAEC,GAAG,QAAO,cAAc;AAC9E,SACEC,uBAAuB,EACvBC,wBAAwB,EACxBC,kCAAkC,EAClCC,gCAAgC,QAC3B,kBAAkB;AAEzB,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,mCAAmC,QAAO,yBAAyB;AAC3E,SAAQC,cAAc,QAAO,qBAAqB;AAClD,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAAQC,eAAe,QAAO,+BAA+B;AAE7D,8FAA8F,GAC9F,OAAO,eAAeC,sBAAsBC,MAI3C;IACC,OAAO,MAAMP,KAAKQ,WAAW,CAAC,OAAOC,KAAOC,wBAAwBD,IAAIF;AAC1E;AAEA,OAAO,eAAeG,wBACpBD,EAAM,EACNF,MAIC;IAED,MAAMI,oBAAoB;WAAIJ,OAAOI,iBAAiB;KAAC,CAACC,IAAI;IAC5D,MAAMX,oCAAoCQ,IAAI;QAC5CI,eAAeN,OAAOM,aAAa;QACnCC,gBAAgB;YAACP,OAAOQ,aAAa;SAAC;IACxC;IACA,MAAMC,aAAa,MAAMP,GACtBQ,MAAM,CAAC;QACNC,IAAIb,gBAAgBa,EAAE;QACtBH,eAAeV,gBAAgBU,aAAa;QAC5CI,uBAAuBd,gBAAgBc,qBAAqB;QAC5DC,aAAaf,gBAAgBe,WAAW;QACxCC,YAAYhB,gBAAgBgB,UAAU;QACtCC,kBAAkBjB,gBAAgBiB,gBAAgB;QAClDC,QAAQlB,gBAAgBkB,MAAM;QAC9BC,OAAOnB,gBAAgBmB,KAAK;IAC9B,GACCC,IAAI,CAACpB,iBACLqB,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDvB,QAAQe,gBAAgBa,EAAE,EAAEP,qBAG/BgB,GAAG,CAAC;IAEP,+EAA+E;IAC/E,wFAAwF;IACxF,IACEhB,kBAAkBiB,MAAM,GAAG,KAC3BZ,WAAWY,MAAM,KAAKjB,kBAAkBiB,MAAM,IAC9CZ,WAAWa,KAAK,CACd,CAACC,SACCA,OAAOf,aAAa,KAAKR,OAAOQ,aAAa,IAC7Ce,OAAOT,UAAU,KAAK,QACtBS,OAAOV,WAAW,KAAK,OAE3B;QACA,MAAMX,GACHsB,MAAM,CAAC1B,iBACP2B,GAAG,CAAC;YAACb,uBAAuB;YAAMc,WAAWtC,GAAG,CAAC,KAAK,CAAC;QAAA,GACvD+B,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDvB,QAAQe,gBAAgBa,EAAE,EAAEP;QAGlC,OAAOJ,OAAOI,iBAAiB;IACjC;IAEA,MAAM,CAACuB,YAAY,GAAG,MAAMzB,GACzBQ,MAAM,GACNQ,IAAI,CAACtB,cACLuB,KAAK,CACJtC,IACEC,GAAGc,aAAae,EAAE,EAAEX,OAAOQ,aAAa,GACxC1B,GAAGc,aAAaU,aAAa,EAAEN,OAAOM,aAAa,GACnDxB,GAAGc,aAAagC,IAAI,EAAE,YAGzBC,KAAK,CAAC,GACNT,GAAG,CAAC;IACP,IAAI,CAACO,aAAa,MAAM,IAAIrC,yBAAyBU,OAAOQ,aAAa;IACzE,IAAImB,YAAYG,SAAS,IAAI,IAAIC,QAAQ,MAAM,IAAI1C,wBAAwBW,OAAOQ,aAAa;IAC/F,IAAIC,WAAWY,MAAM,KAAKjB,kBAAkBiB,MAAM,EAChD,MAAM,IAAI7B,iCAAiCY,iBAAiB,CAAC,EAAE,IAAI;IAErE,MAAM4B,wBAAwB,MAAM9B,GACjCQ,MAAM,CAAC;QAACuB,kBAAkBpC,sBAAsBoC,gBAAgB;IAAA,GAChEf,IAAI,CAACrB,uBACLsB,KAAK,CACJtC,IACEE,QACEc,sBAAsBoC,gBAAgB,EACtCxB,WAAWyB,GAAG,CAAC,CAACX,SAAWA,OAAOZ,EAAE,IAEtC3B,OAAOa,sBAAsBsC,QAAQ;IAG3C,MAAMC,sCAAsC,IAAIC,IAC9CL,sBAAsBE,GAAG,CAAC,CAACI,UAAYA,QAAQL,gBAAgB;IAEjE,MAAMM,UAAU9B,WAAWyB,GAAG,CAAC,CAACX,SAAY,CAAA;YAC1C,GAAGA,MAAM;YACTiB,kBAAkBJ,oCAAoCK,GAAG,CAAClB,OAAOZ,EAAE,IAAIY,OAAOZ,EAAE,GAAG;QACrF,CAAA;IAEA,MAAM+B,kBAAkBH,QAAQI,MAAM,CAAC,CAACpB,SAAWA,OAAOf,aAAa,KAAK;IAC5E,IAAIkC,gBAAgBE,IAAI,CAAC,CAACrB,SAAWA,OAAOf,aAAa,KAAKmB,YAAYhB,EAAE,GAC1E,MAAM,IAAIpB,mCAAmCmD,eAAe,CAAC,EAAE,EAAE/B,MAAM;IACzE,MAAMkC,aAAaN,QAAQI,MAAM,CAC/B,CAACpB,SACCA,OAAOf,aAAa,KAAK,QACxBe,OAAOf,aAAa,KAAKmB,YAAYhB,EAAE,IACrCY,CAAAA,OAAOT,UAAU,KAAK,QAAQS,OAAOV,WAAW,KAAK,IAAG;IAE/D,MAAMiC,eAAeD,WAAWX,GAAG,CAAC,CAACX,SAAWA,OAAOZ,EAAE;IACzD,MAAMoC,gBAAgB,MAAM7C,GACzBQ,MAAM,CAAC;QAACsC,OAAO5D,GAAW,CAAC,aAAa,CAAC;IAAA,GACzC8B,IAAI,CAACpB,iBACLqB,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDpB,WAAWY,gBAAgBa,EAAE,EAAEmC,eAC/B3D,GACEN,IACEC,GAAGgB,gBAAgBU,aAAa,EAAEmB,YAAYhB,EAAE,GAChD3B,OAAOc,gBAAgBmD,qBAAqB,IAE9CpE,IACEC,GAAGgB,gBAAgBc,qBAAqB,EAAEe,YAAYhB,EAAE,GACxD3B,OAAOc,gBAAgBmD,qBAAqB,GAC5C/D,WAAWY,gBAAgBmB,KAAK,EAAE;WAAItB;KAAe;IAK/D,IAAI,AAACoD,CAAAA,aAAa,CAAC,EAAE,EAAEC,SAAS,CAAA,IAAKH,WAAWxB,MAAM,GAAGM,YAAYqB,KAAK,EACxE,MAAM,IAAIxD,iCAAiCqD,UAAU,CAAC,EAAE,EAAElC,MAAM;IAClE,KAAK,MAAMY,UAAUsB,WAAY;QAC/B,IACEtB,OAAON,KAAK,KAAK,aACjB,CAACM,OAAOR,gBAAgB,IACxB,CAACQ,OAAOiB,gBAAgB,IACxB,CAACb,YAAYuB,cAAc,CAAC5B,KAAK,CAAC,CAAC6B,QAAU5B,OAAOP,MAAM,CAACoC,QAAQ,CAACD,SAEpE,MAAM,IAAI3D,iCAAiC+B,OAAOZ,EAAE;IACxD;IACA,IAAIkC,WAAWxB,MAAM,GAAG,GAAG;QACzB,MAAMnB,GACHsB,MAAM,CAAC1B,iBACP2B,GAAG,CAAC;YACHZ,aAAac,YAAYd,WAAW;YACpCL,eAAemB,YAAYhB,EAAE;YAC7BC,uBAAuB;YACvBE,YAAY1B,GAAG,CAAC,KAAK,CAAC;YACtBsC,WAAWtC,GAAG,CAAC,KAAK,CAAC;QACvB,GACC+B,KAAK,CACJpC,QACEe,gBAAgBa,EAAE,EAClBkC,WAAWX,GAAG,CAAC,CAACX,SAAWA,OAAOZ,EAAE;IAG5C;IACA,OAAOX,OAAOI,iBAAiB;AACjC;AAgBA,OAAO,eAAeiD,oCACpBnD,EAAM,EACNF,MAGC,EACDsD,UAAyC,CAAC,CAAC;IAE3C,MAAMC,yBAAyB,IAAIC;IACnC,KAAK,MAAMC,WAAWzD,OAAO0D,QAAQ,CAAE;QACrC,IAAID,QAAQT,KAAK,IAAI,GAAG;QACxBO,uBAAuB9B,GAAG,CACxBgC,QAAQjD,aAAa,EACrB,AAAC+C,CAAAA,uBAAuBI,GAAG,CAACF,QAAQjD,aAAa,KAAK,CAAA,IAAKiD,QAAQT,KAAK;IAE5E;IACA,MAAMzC,iBAAiB;WAAIgD,uBAAuBK,IAAI;KAAG,CAACvD,IAAI;IAC9D,IAAIE,eAAec,MAAM,KAAK,GAC5B,OAAO;QAACwC,uBAAuB,IAAIL;QAAOM,0BAA0B,IAAIN;IAAK;IAE/E,IAAI,CAACF,QAAQS,iBAAiB,EAC5B,MAAMrE,oCAAoCQ,IAAI;QAC5CI,eAAeN,OAAOM,aAAa;QACnCC;IACF;IAEF,MAAMyD,kBAAkB,MAAM9D,GAC3BQ,MAAM,CAAC;QACNC,IAAIf,aAAae,EAAE;QACnBqC,OAAOpD,aAAaoD,KAAK;QACzBlB,WAAWlC,aAAakC,SAAS;QACjCmC,WAAW7E,GAAY,CAAC,EAAEQ,aAAakC,SAAS,CAAC,SAAS,CAAC;IAC7D,GACCZ,IAAI,CAACtB,cACLuB,KAAK,CACJtC,IACEC,GAAGc,aAAaU,aAAa,EAAEN,OAAOM,aAAa,GACnDvB,QAAQa,aAAae,EAAE,EAAEJ,iBACzBzB,GAAGc,aAAagC,IAAI,EAAE,YAGzBR,GAAG,CAAC;IACP,MAAM8C,2BAA2B,MAAMhE,GACpCQ,MAAM,CAAC;QAACC,IAAIf,aAAae,EAAE;IAAA,GAC3BO,IAAI,CAACtB,cACLuB,KAAK,CACJtC,IACEC,GAAGc,aAAaU,aAAa,EAAEN,OAAOM,aAAa,GACnDvB,QAAQa,aAAae,EAAE,EAAEJ,iBACzBtB,GAAGW,aAAagC,IAAI,EAAE,YAGzBR,GAAG,CAAC;IACP,MAAM+C,mBAAmB,IAAIX,IAC3BQ,gBAAgB9B,GAAG,CAAC,CAACP,cAAgB;YAACA,YAAYhB,EAAE;YAAEgB;SAAY;IAEpE,MAAMyC,0BAA0B,IAAI/B,IAClC6B,yBAAyBhC,GAAG,CAAC,CAACP,cAAgBA,YAAYhB,EAAE;IAE9D,MAAM0D,0BAA0B,IAAIhC,IAAI9B;IACxC,MAAM+D,6BAA6B,IAAId;IACvC,MAAMe,iBAAiB,MAAMrE,GAC1BQ,MAAM,CAAC;QACNC,IAAIb,gBAAgBa,EAAE;QACtBH,eAAeV,gBAAgBU,aAAa;QAC5CI,uBAAuBd,gBAAgBc,qBAAqB;QAC5DK,OAAOnB,gBAAgBmB,KAAK;IAC9B,GACCC,IAAI,CAACpB,iBACLqB,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDtB,OAAOc,gBAAgBmD,qBAAqB,GAC5C9D,GACEJ,QAAQe,gBAAgBU,aAAa,EAAED,iBACvC1B,IACEE,QAAQe,gBAAgBc,qBAAqB,EAAEL,iBAC/CrB,WAAWY,gBAAgBmB,KAAK,EAAE;WAAItB;KAAe;IAK/D,KAAK,MAAM4B,UAAUgD,eAAgB;QACnC,IAAIhD,OAAOf,aAAa,IAAI6D,wBAAwB5B,GAAG,CAAClB,OAAOf,aAAa,GAAG;YAC7E,MAAMgE,gBACJF,2BAA2BX,GAAG,CAACpC,OAAOf,aAAa,KAAK,IAAI6B;YAC9DmC,cAAcC,GAAG,CAAClD,OAAOZ,EAAE;YAC3B2D,2BAA2B7C,GAAG,CAACF,OAAOf,aAAa,EAAEgE;QACvD;QACA,IACEjD,OAAOX,qBAAqB,IAC5ByD,wBAAwB5B,GAAG,CAAClB,OAAOX,qBAAqB,KACxD,CAACjB,eAAeyD,QAAQ,CAAC7B,OAAON,KAAK,GACrC;YACA,MAAMuD,gBACJF,2BAA2BX,GAAG,CAACpC,OAAOX,qBAAqB,KAAK,IAAIyB;YACtEmC,cAAcC,GAAG,CAAClD,OAAOZ,EAAE;YAC3B2D,2BAA2B7C,GAAG,CAACF,OAAOX,qBAAqB,EAAE4D;QAC/D;IACF;IAEA,MAAMX,wBAAwB,IAAIL;IAClC,MAAMM,2BAA2B,IAAIN;IAIrC,KAAK,MAAMhD,iBAAiBD,eAAgB;QAC1C,MAAMmE,YAAYnB,uBAAuBI,GAAG,CAACnD,kBAAkB;QAC/D,MAAMmB,cAAcwC,iBAAiBR,GAAG,CAACnD;QACzC,IAAI,CAACmB,aAAa;YAChBmC,yBAAyBrC,GAAG,CAACjB,eAAe;gBAC1CmE,QAAQP,wBAAwB3B,GAAG,CAACjC,iBAChC,8BACA;gBACJwC,OAAO0B;YACT;YACA;QACF;QACA,IAAI/C,YAAYsC,SAAS,EAAE;YACzBH,yBAAyBrC,GAAG,CAACjB,eAAe;gBAC1CmE,QAAQ;gBACR3B,OAAO0B;YACT;YACA;QACF;QAEA,MAAME,OAAON,2BAA2BX,GAAG,CAACnD,gBAAgBqE,QAAQ;QACpE,MAAMC,WAAWC,KAAKC,GAAG,CAACN,WAAWK,KAAKE,GAAG,CAAC,GAAGtD,YAAYqB,KAAK,GAAG4B;QACrEf,sBAAsBpC,GAAG,CAACjB,eAAesE;QACzC,IAAIA,WAAWJ,WAAW;YACxBZ,yBAAyBrC,GAAG,CAACjB,eAAe;gBAC1CmE,QAAQ;gBACR3B,OAAO0B,YAAYI;YACrB;QACF;IACF;IAEA,OAAO;QAACjB;QAAuBC;IAAwB;AACzD"}
@@ -1,13 +1,13 @@
1
1
  import type { RunnerInstance, RunnerInstanceState } from '#core/entities/runner-instance.js';
2
2
  import type { Tx } from './db.js';
3
3
  import { type RunnerInstanceBoundJobExecution } from './job-executions.js';
4
- export declare const terminalStates: readonly ["stopped", "failed", "terminated"];
5
4
  export declare const activeStates: readonly ["starting", "running", "stopping"];
6
5
  export declare const divergenceCountStates: readonly ["starting", "running"];
7
- export type RunnerInstanceTerminateIntentReason = 'job-cancelled';
6
+ export type RunnerInstanceTerminateIntentReason = 'activation-timeout' | 'job-cancelled';
8
7
  export interface RunnerInstanceTerminateIntent {
9
8
  providerRunnerId: string;
10
9
  reason: RunnerInstanceTerminateIntentReason;
10
+ activationTimeoutRetry?: boolean;
11
11
  }
12
12
  export interface ActiveRunnerInstanceTemplateCount {
13
13
  templateKey: string;
@@ -1 +1 @@
1
- {"version":3,"file":"runner-instances.d.ts","sourceRoot":"","sources":["../../src/db/runner-instances.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,mCAAmC,CAAC;AAE3F,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAEhC,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,qBAAqB,CAAC;AAU7B,eAAO,MAAM,cAAc,8CAIwB,CAAC;AACpD,eAAO,MAAM,YAAY,8CAI0B,CAAC;AACpD,eAAO,MAAM,qBAAqB,kCAG/B,CAAC;AAEJ,MAAM,MAAM,mCAAmC,GAAG,eAAe,CAAC;AAElE,MAAM,WAAW,6BAA6B;IAC5C,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,mCAAmC,CAAC;CAC7C;AAED,MAAM,WAAW,iCAAiC;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,cAAc,GAAG,WAAW,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,yBAAyB,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,8BAA8B;IAC7C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,gCAAgC;IAC/C,YAAY,EAAE,cAAc,EAAE,CAAC;IAC/B,oCAAoC,EAAE,GAAG,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAiBD,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,2BAA2B,GAAG,OAAO,CAAC;IACxF,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,6BAA6B,EAAE,CAAC;CAC1D,CAAC,CA+ED;AAED,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAC,GACnD,OAAO,CAAC,iCAAiC,EAAE,CAAC,CAuB9C;AAED,wBAAsB,iCAAiC,CAAC,MAAM,EAAE;IAC9D,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0BlB;AAED,wBAAsB,yBAAyB,CAAC,MAAM,EAAE;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAe5B;AAED,wBAAsB,+BAA+B,CAAC,MAAM,EAAE;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAKpB;AAED,wBAAsB,oCAAoC,CACxD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GACA,OAAO,CAAC,6BAA6B,EAAE,CAAC,CAsB1C;AAqFD,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,8BAA8B,GACrC,OAAO,CAAC,gCAAgC,CAAC,CAqG3C;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,8BAA8B,CAAC,CA4D1C;AAkcD,wBAAgB,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAEnE"}
1
+ {"version":3,"file":"runner-instances.d.ts","sourceRoot":"","sources":["../../src/db/runner-instances.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,mCAAmC,CAAC;AAG3F,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAEhC,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,qBAAqB,CAAC;AAa7B,eAAO,MAAM,YAAY,8CAI0B,CAAC;AACpD,eAAO,MAAM,qBAAqB,kCAG/B,CAAC;AAEJ,MAAM,MAAM,mCAAmC,GAAG,oBAAoB,GAAG,eAAe,CAAC;AAEzF,MAAM,WAAW,6BAA6B;IAC5C,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,mCAAmC,CAAC;IAC5C,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,iCAAiC;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,cAAc,GAAG,WAAW,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,yBAAyB,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,8BAA8B;IAC7C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,gCAAgC;IAC/C,YAAY,EAAE,cAAc,EAAE,CAAC;IAC/B,oCAAoC,EAAE,GAAG,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAiBD,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,2BAA2B,GAAG,OAAO,CAAC;IACxF,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,6BAA6B,EAAE,CAAC;CAC1D,CAAC,CAoFD;AA6FD,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAC,GACnD,OAAO,CAAC,iCAAiC,EAAE,CAAC,CAuB9C;AAED,wBAAsB,iCAAiC,CAAC,MAAM,EAAE;IAC9D,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0BlB;AAED,wBAAsB,yBAAyB,CAAC,MAAM,EAAE;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAe5B;AAED,wBAAsB,+BAA+B,CAAC,MAAM,EAAE;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAKpB;AAED,wBAAsB,oCAAoC,CACxD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GACA,OAAO,CAAC,6BAA6B,EAAE,CAAC,CAqC1C;AAqID,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,8BAA8B,GACrC,OAAO,CAAC,gCAAgC,CAAC,CAqG3C;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,8BAA8B,CAAC,CA4D1C;AAkcD,wBAAgB,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAEnE"}