@shipfox/api-runners 12.2.0 → 12.4.0

package/dist/db/runner-assignments.d.ts

@@ -10,4 +10,21 @@ export declare function assignRunnerInstancesTx(tx: Tx, params: {
     reservationId: string;
     runnerInstanceIds: string[];
 }): Promise<string[]>;
+export type RunnerReservationCapacityFailureReason = 'reservation-not-found' | 'reservation-kind-mismatch' | 'reservation-expired' | 'capacity-exhausted';
+export interface RunnerReservationCapacityValidation {
+    acceptedByReservation: Map<string, number>;
+    unavailableByReservation: Map<string, {
+        reason: RunnerReservationCapacityFailureReason;
+        count: number;
+    }>;
+}
+export declare function validateRunnerReservationCapacityTx(tx: Tx, params: {
+    provisionerId: string;
+    requests: readonly {
+        reservationId: string;
+        count: number;
+    }[];
+}, options?: {
+    advisoryLocksHeld?: boolean;
+}): Promise<RunnerReservationCapacityValidation>;
 //# sourceMappingURL=runner-assignments.d.ts.map

package/dist/db/runner-assignments.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner-assignments.d.ts","sourceRoot":"","sources":["../../src/db/runner-assignments.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAOhC,gGAAgG;AAChG,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAEpB;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GACA,OAAO,CAAC,MAAM,EAAE,CAAC,CAoInB"}
+{"version":3,"file":"runner-assignments.d.ts","sourceRoot":"","sources":["../../src/db/runner-assignments.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAQhC,gGAAgG;AAChG,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAEpB;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GACA,OAAO,CAAC,MAAM,EAAE,CAAC,CAkJnB;AAED,MAAM,MAAM,sCAAsC,GAC9C,uBAAuB,GACvB,2BAA2B,GAC3B,qBAAqB,GACrB,oBAAoB,CAAC;AAEzB,MAAM,WAAW,mCAAmC;IAClD,qBAAqB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,wBAAwB,EAAE,GAAG,CAC3B,MAAM,EACN;QAAC,MAAM,EAAE,sCAAsC,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAChE,CAAC;CACH;AAED,wBAAsB,mCAAmC,CACvD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,SAAS;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,EAAE,CAAC;CAC7D,EACD,OAAO,GAAE;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;CAAM,GAC1C,OAAO,CAAC,mCAAmC,CAAC,CAmI9C"}

package/dist/db/runner-assignments.js

@@ -1,7 +1,8 @@
-import { and, eq, inArray, isNull, notInArray, or, sql } from 'drizzle-orm';
+import { and, eq, inArray, isNull, ne, notInArray, or, sql } from 'drizzle-orm';
 import { ReservationExpiredError, ReservationNotFoundError, RunnerInstanceAlreadyAssignedError, RunnerInstanceNotAssignableError } from '#core/errors.js';
 import { db } from './db.js';
-import { terminalStates } from './runner-instances.js';
+import { lockRunnerReservationAdvisoryKeysTx } from './reservation-locks.js';
+import { terminalStates } from './runner-states.js';
 import { reservations } from './schema/reservations.js';
 import { runnerControlSessions } from './schema/runner-control-sessions.js';
 import { providerRunners } from './schema/runner-instances.js';
@@ -12,26 +13,32 @@ export async function assignRunnerInstancesTx(tx, params) {
     const runnerInstanceIds = [
         ...params.runnerInstanceIds
     ].sort();
-    await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${`runners_assignment:${params.provisionerId}:${params.reservationId}`}))`);
+    await lockRunnerReservationAdvisoryKeysTx(tx, {
+        provisionerId: params.provisionerId,
+        reservationIds: [
+            params.reservationId
+        ]
+    });
     const runnerRows = await tx.select({
         id: providerRunners.id,
         reservationId: providerRunners.reservationId,
         intendedReservationId: providerRunners.intendedReservationId,
         workspaceId: providerRunners.workspaceId,
+        assignedAt: providerRunners.assignedAt,
         providerRunnerId: providerRunners.providerRunnerId,
         labels: providerRunners.labels,
         state: providerRunners.state
     }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.id, runnerInstanceIds))).for('update');
     // The assignment outlives its short reservation row. A retry after maintenance
-    // cleanup is complete when every owned runner already carries the requested assignment.
-    if (runnerInstanceIds.length > 0 && runnerRows.length === runnerInstanceIds.length && runnerRows.every((runner)=>runner.reservationId === params.reservationId)) {
+    // cleanup is complete when every owned runner already carries the committed assignment.
+    if (runnerInstanceIds.length > 0 && runnerRows.length === runnerInstanceIds.length && runnerRows.every((runner)=>runner.reservationId === params.reservationId && runner.assignedAt !== null && runner.workspaceId !== null)) {
         await tx.update(providerRunners).set({
             intendedReservationId: null,
             updatedAt: sql`now()`
         }).where(and(eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.id, runnerInstanceIds)));
         return params.runnerInstanceIds;
     }
-    const [reservation] = await tx.select().from(reservations).where(and(eq(reservations.id, params.reservationId), eq(reservations.provisionerId, params.provisionerId))).limit(1).for('update');
+    const [reservation] = await tx.select().from(reservations).where(and(eq(reservations.id, params.reservationId), eq(reservations.provisionerId, params.provisionerId), eq(reservations.kind, 'launch'))).limit(1).for('update');
     if (!reservation) throw new ReservationNotFoundError(params.reservationId);
     if (reservation.expiresAt <= new Date()) throw new ReservationExpiredError(params.reservationId);
     if (runnerRows.length !== runnerInstanceIds.length) throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '');
@@ -45,10 +52,11 @@ export async function assignRunnerInstancesTx(tx, params) {
         }));
     const alreadyAssigned = runners.filter((runner)=>runner.reservationId !== null);
     if (alreadyAssigned.some((runner)=>runner.reservationId !== reservation.id)) throw new RunnerInstanceAlreadyAssignedError(alreadyAssigned[0]?.id ?? '');
-    const newRunners = runners.filter((runner)=>runner.reservationId === null);
+    const newRunners = runners.filter((runner)=>runner.reservationId === null || runner.reservationId === reservation.id && (runner.assignedAt === null || runner.workspaceId === null));
+    const newRunnerIds = newRunners.map((runner)=>runner.id);
     const assignedCount = await tx.select({
         count: sql`count(*)::int`
-    }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), or(and(eq(providerRunners.reservationId, reservation.id), isNull(providerRunners.reservationReleasedAt)), and(eq(providerRunners.intendedReservationId, reservation.id), notInArray(providerRunners.id, runnerInstanceIds), isNull(providerRunners.reservationReleasedAt), notInArray(providerRunners.state, [
+    }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), notInArray(providerRunners.id, newRunnerIds), or(and(eq(providerRunners.reservationId, reservation.id), isNull(providerRunners.reservationReleasedAt)), and(eq(providerRunners.intendedReservationId, reservation.id), isNull(providerRunners.reservationReleasedAt), notInArray(providerRunners.state, [
         ...terminalStates
     ])))));
     if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count) throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '');
@@ -66,5 +74,92 @@ export async function assignRunnerInstancesTx(tx, params) {
     }
     return params.runnerInstanceIds;
 }
+export async function validateRunnerReservationCapacityTx(tx, params, options = {}) {
+    const requestedByReservation = new Map();
+    for (const request of params.requests){
+        if (request.count <= 0) continue;
+        requestedByReservation.set(request.reservationId, (requestedByReservation.get(request.reservationId) ?? 0) + request.count);
+    }
+    const reservationIds = [
+        ...requestedByReservation.keys()
+    ].sort();
+    if (reservationIds.length === 0) return {
+        acceptedByReservation: new Map(),
+        unavailableByReservation: new Map()
+    };
+    if (!options.advisoryLocksHeld) await lockRunnerReservationAdvisoryKeysTx(tx, {
+        provisionerId: params.provisionerId,
+        reservationIds
+    });
+    const reservationRows = await tx.select({
+        id: reservations.id,
+        count: reservations.count,
+        expiresAt: reservations.expiresAt,
+        isExpired: sql`${reservations.expiresAt} <= now()`
+    }).from(reservations).where(and(eq(reservations.provisionerId, params.provisionerId), inArray(reservations.id, reservationIds), eq(reservations.kind, 'launch'))).for('update');
+    const nonLaunchReservationRows = await tx.select({
+        id: reservations.id
+    }).from(reservations).where(and(eq(reservations.provisionerId, params.provisionerId), inArray(reservations.id, reservationIds), ne(reservations.kind, 'launch'))).for('update');
+    const reservationsById = new Map(reservationRows.map((reservation)=>[
+            reservation.id,
+            reservation
+        ]));
+    const nonLaunchReservationIds = new Set(nonLaunchReservationRows.map((reservation)=>reservation.id));
+    const requestedReservationIds = new Set(reservationIds);
+    const usedRunnerIdsByReservation = new Map();
+    const usedRunnerRows = await tx.select({
+        id: providerRunners.id,
+        reservationId: providerRunners.reservationId,
+        intendedReservationId: providerRunners.intendedReservationId,
+        state: providerRunners.state
+    }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), isNull(providerRunners.reservationReleasedAt), or(inArray(providerRunners.reservationId, reservationIds), and(inArray(providerRunners.intendedReservationId, reservationIds), notInArray(providerRunners.state, [
+        ...terminalStates
+    ])))));
+    for (const runner of usedRunnerRows){
+        if (runner.reservationId && requestedReservationIds.has(runner.reservationId)) {
+            const usedRunnerIds = usedRunnerIdsByReservation.get(runner.reservationId) ?? new Set();
+            usedRunnerIds.add(runner.id);
+            usedRunnerIdsByReservation.set(runner.reservationId, usedRunnerIds);
+        }
+        if (runner.intendedReservationId && requestedReservationIds.has(runner.intendedReservationId) && !terminalStates.includes(runner.state)) {
+            const usedRunnerIds = usedRunnerIdsByReservation.get(runner.intendedReservationId) ?? new Set();
+            usedRunnerIds.add(runner.id);
+            usedRunnerIdsByReservation.set(runner.intendedReservationId, usedRunnerIds);
+        }
+    }
+    const acceptedByReservation = new Map();
+    const unavailableByReservation = new Map();
+    for (const reservationId of reservationIds){
+        const requested = requestedByReservation.get(reservationId) ?? 0;
+        const reservation = reservationsById.get(reservationId);
+        if (!reservation) {
+            unavailableByReservation.set(reservationId, {
+                reason: nonLaunchReservationIds.has(reservationId) ? 'reservation-kind-mismatch' : 'reservation-not-found',
+                count: requested
+            });
+            continue;
+        }
+        if (reservation.isExpired) {
+            unavailableByReservation.set(reservationId, {
+                reason: 'reservation-expired',
+                count: requested
+            });
+            continue;
+        }
+        const used = usedRunnerIdsByReservation.get(reservationId)?.size ?? 0;
+        const accepted = Math.min(requested, Math.max(0, reservation.count - used));
+        acceptedByReservation.set(reservationId, accepted);
+        if (accepted < requested) {
+            unavailableByReservation.set(reservationId, {
+                reason: 'capacity-exhausted',
+                count: requested - accepted
+            });
+        }
+    }
+    return {
+        acceptedByReservation,
+        unavailableByReservation
+    };
+}
 
 //# sourceMappingURL=runner-assignments.js.map

package/dist/db/runner-assignments.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/db/runner-assignments.ts"],"sourcesContent":["import {and, eq, inArray, isNull, notInArray, or, sql} from 'drizzle-orm';\nimport {\n  ReservationExpiredError,\n  ReservationNotFoundError,\n  RunnerInstanceAlreadyAssignedError,\n  RunnerInstanceNotAssignableError,\n} from '#core/errors.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {terminalStates} from './runner-instances.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {providerRunners} from './schema/runner-instances.js';\n\n/** Atomically consumes reservation units by writing the immutable assignment on each runner. */\nexport async function assignRunnerInstances(params: {\n  provisionerId: string;\n  reservationId: string;\n  runnerInstanceIds: string[];\n}): Promise<string[]> {\n  return await db().transaction(async (tx) => assignRunnerInstancesTx(tx, params));\n}\n\nexport async function assignRunnerInstancesTx(\n  tx: Tx,\n  params: {\n    provisionerId: string;\n    reservationId: string;\n    runnerInstanceIds: string[];\n  },\n): Promise<string[]> {\n  const runnerInstanceIds = [...params.runnerInstanceIds].sort();\n  await tx.execute(\n    sql`select pg_advisory_xact_lock(hashtext(${`runners_assignment:${params.provisionerId}:${params.reservationId}`}))`,\n  );\n  const runnerRows = await tx\n    .select({\n      id: providerRunners.id,\n      reservationId: providerRunners.reservationId,\n      intendedReservationId: providerRunners.intendedReservationId,\n      workspaceId: providerRunners.workspaceId,\n      providerRunnerId: providerRunners.providerRunnerId,\n      labels: providerRunners.labels,\n      state: providerRunners.state,\n    })\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        inArray(providerRunners.id, runnerInstanceIds),\n      ),\n    )\n    .for('update');\n\n  // The assignment outlives its short reservation row. A retry after maintenance\n  // cleanup is complete when every owned runner already carries the requested assignment.\n  if (\n    runnerInstanceIds.length > 0 &&\n    runnerRows.length === runnerInstanceIds.length &&\n    runnerRows.every((runner) => runner.reservationId === params.reservationId)\n  ) {\n    await tx\n      .update(providerRunners)\n      .set({intendedReservationId: null, updatedAt: sql`now()`})\n      .where(\n        and(\n          eq(providerRunners.provisionerId, params.provisionerId),\n          inArray(providerRunners.id, runnerInstanceIds),\n        ),\n      );\n    return params.runnerInstanceIds;\n  }\n\n  const [reservation] = await tx\n    .select()\n    .from(reservations)\n    .where(\n      and(\n        eq(reservations.id, params.reservationId),\n        eq(reservations.provisionerId, params.provisionerId),\n      ),\n    )\n    .limit(1)\n    .for('update');\n  if (!reservation) throw new ReservationNotFoundError(params.reservationId);\n  if (reservation.expiresAt <= new Date()) throw new ReservationExpiredError(params.reservationId);\n  if (runnerRows.length !== runnerInstanceIds.length)\n    throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '');\n\n  const activeControlSessions = await tx\n    .select({runnerInstanceId: runnerControlSessions.runnerInstanceId})\n    .from(runnerControlSessions)\n    .where(\n      and(\n        inArray(\n          runnerControlSessions.runnerInstanceId,\n          runnerRows.map((runner) => runner.id),\n        ),\n        isNull(runnerControlSessions.closedAt),\n      ),\n    );\n  const runnerInstanceIdsWithControlSession = new Set(\n    activeControlSessions.map((session) => session.runnerInstanceId),\n  );\n  const runners = runnerRows.map((runner) => ({\n    ...runner,\n    controlSessionId: runnerInstanceIdsWithControlSession.has(runner.id) ? runner.id : null,\n  }));\n\n  const alreadyAssigned = runners.filter((runner) => runner.reservationId !== null);\n  if (alreadyAssigned.some((runner) => runner.reservationId !== reservation.id))\n    throw new RunnerInstanceAlreadyAssignedError(alreadyAssigned[0]?.id ?? '');\n  const newRunners = runners.filter((runner) => runner.reservationId === null);\n  const assignedCount = await tx\n    .select({count: sql<number>`count(*)::int`})\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        or(\n          and(\n            eq(providerRunners.reservationId, reservation.id),\n            isNull(providerRunners.reservationReleasedAt),\n          ),\n          and(\n            eq(providerRunners.intendedReservationId, reservation.id),\n            notInArray(providerRunners.id, runnerInstanceIds),\n            isNull(providerRunners.reservationReleasedAt),\n            notInArray(providerRunners.state, [...terminalStates]),\n          ),\n        ),\n      ),\n    );\n  if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count)\n    throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '');\n  for (const runner of newRunners) {\n    if (\n      runner.state !== 'running' ||\n      !runner.providerRunnerId ||\n      !runner.controlSessionId ||\n      !reservation.requiredLabels.every((label) => runner.labels.includes(label))\n    )\n      throw new RunnerInstanceNotAssignableError(runner.id);\n  }\n  if (newRunners.length > 0) {\n    await tx\n      .update(providerRunners)\n      .set({\n        workspaceId: reservation.workspaceId,\n        reservationId: reservation.id,\n        intendedReservationId: null,\n        assignedAt: sql`now()`,\n        updatedAt: sql`now()`,\n      })\n      .where(\n        inArray(\n          providerRunners.id,\n          newRunners.map((runner) => runner.id),\n        ),\n      );\n  }\n  return params.runnerInstanceIds;\n}\n"],"names":["and","eq","inArray","isNull","notInArray","or","sql","ReservationExpiredError","ReservationNotFoundError","RunnerInstanceAlreadyAssignedError","RunnerInstanceNotAssignableError","db","terminalStates","reservations","runnerControlSessions","providerRunners","assignRunnerInstances","params","transaction","tx","assignRunnerInstancesTx","runnerInstanceIds","sort","execute","provisionerId","reservationId","runnerRows","select","id","intendedReservationId","workspaceId","providerRunnerId","labels","state","from","where","for","length","every","runner","update","set","updatedAt","reservation","limit","expiresAt","Date","activeControlSessions","runnerInstanceId","map","closedAt","runnerInstanceIdsWithControlSession","Set","session","runners","controlSessionId","has","alreadyAssigned","filter","some","newRunners","assignedCount","count","reservationReleasedAt","requiredLabels","label","includes","assignedAt"],"mappings":"AAAA,SAAQA,GAAG,EAAEC,EAAE,EAAEC,OAAO,EAAEC,MAAM,EAAEC,UAAU,EAAEC,EAAE,EAAEC,GAAG,QAAO,cAAc;AAC1E,SACEC,uBAAuB,EACvBC,wBAAwB,EACxBC,kCAAkC,EAClCC,gCAAgC,QAC3B,kBAAkB;AAEzB,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,cAAc,QAAO,wBAAwB;AACrD,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAAQC,eAAe,QAAO,+BAA+B;AAE7D,8FAA8F,GAC9F,OAAO,eAAeC,sBAAsBC,MAI3C;IACC,OAAO,MAAMN,KAAKO,WAAW,CAAC,OAAOC,KAAOC,wBAAwBD,IAAIF;AAC1E;AAEA,OAAO,eAAeG,wBACpBD,EAAM,EACNF,MAIC;IAED,MAAMI,oBAAoB;WAAIJ,OAAOI,iBAAiB;KAAC,CAACC,IAAI;IAC5D,MAAMH,GAAGI,OAAO,CACdjB,GAAG,CAAC,sCAAsC,EAAE,CAAC,mBAAmB,EAAEW,OAAOO,aAAa,CAAC,CAAC,EAAEP,OAAOQ,aAAa,EAAE,CAAC,EAAE,CAAC;IAEtH,MAAMC,aAAa,MAAMP,GACtBQ,MAAM,CAAC;QACNC,IAAIb,gBAAgBa,EAAE;QACtBH,eAAeV,gBAAgBU,aAAa;QAC5CI,uBAAuBd,gBAAgBc,qBAAqB;QAC5DC,aAAaf,gBAAgBe,WAAW;QACxCC,kBAAkBhB,gBAAgBgB,gBAAgB;QAClDC,QAAQjB,gBAAgBiB,MAAM;QAC9BC,OAAOlB,gBAAgBkB,KAAK;IAC9B,GACCC,IAAI,CAACnB,iBACLoB,KAAK,CACJnC,IACEC,GAAGc,gBAAgBS,aAAa,EAAEP,OAAOO,aAAa,GACtDtB,QAAQa,gBAAgBa,EAAE,EAAEP,qBAG/Be,GAAG,CAAC;IAEP,+EAA+E;IAC/E,wFAAwF;IACxF,IACEf,kBAAkBgB,MAAM,GAAG,KAC3BX,WAAWW,MAAM,KAAKhB,kBAAkBgB,MAAM,IAC9CX,WAAWY,KAAK,CAAC,CAACC,SAAWA,OAAOd,aAAa,KAAKR,OAAOQ,aAAa,GAC1E;QACA,MAAMN,GACHqB,MAAM,CAACzB,iBACP0B,GAAG,CAAC;YAACZ,uBAAuB;YAAMa,WAAWpC,GAAG,CAAC,KAAK,CAAC;QAAA,GACvD6B,KAAK,CACJnC,IACEC,GAAGc,gBAAgBS,aAAa,EAAEP,OAAOO,aAAa,GACtDtB,QAAQa,gBAAgBa,EAAE,EAAEP;QAGlC,OAAOJ,OAAOI,iBAAiB;IACjC;IAEA,MAAM,CAACsB,YAAY,GAAG,MAAMxB,GACzBQ,MAAM,GACNO,IAAI,CAACrB,cACLsB,KAAK,CACJnC,IACEC,GAAGY,aAAae,EAAE,EAAEX,OAAOQ,aAAa,GACxCxB,GAAGY,aAAaW,aAAa,EAAEP,OAAOO,aAAa,IAGtDoB,KAAK,CAAC,GACNR,GAAG,CAAC;IACP,IAAI,CAACO,aAAa,MAAM,IAAInC,yBAAyBS,OAAOQ,aAAa;IACzE,IAAIkB,YAAYE,SAAS,IAAI,IAAIC,QAAQ,MAAM,IAAIvC,wBAAwBU,OAAOQ,aAAa;IAC/F,IAAIC,WAAWW,MAAM,KAAKhB,kBAAkBgB,MAAM,EAChD,MAAM,IAAI3B,iCAAiCW,iBAAiB,CAAC,EAAE,IAAI;IAErE,MAAM0B,wBAAwB,MAAM5B,GACjCQ,MAAM,CAAC;QAACqB,kBAAkBlC,sBAAsBkC,gBAAgB;IAAA,GAChEd,IAAI,CAACpB,uBACLqB,KAAK,CACJnC,IACEE,QACEY,sBAAsBkC,gBAAgB,EACtCtB,WAAWuB,GAAG,CAAC,CAACV,SAAWA,OAAOX,EAAE,IAEtCzB,OAAOW,sBAAsBoC,QAAQ;IAG3C,MAAMC,sCAAsC,IAAIC,IAC9CL,sBAAsBE,GAAG,CAAC,CAACI,UAAYA,QAAQL,gBAAgB;IAEjE,MAAMM,UAAU5B,WAAWuB,GAAG,CAAC,CAACV,SAAY,CAAA;YAC1C,GAAGA,MAAM;YACTgB,kBAAkBJ,oCAAoCK,GAAG,CAACjB,OAAOX,EAAE,IAAIW,OAAOX,EAAE,GAAG;QACrF,CAAA;IAEA,MAAM6B,kBAAkBH,QAAQI,MAAM,CAAC,CAACnB,SAAWA,OAAOd,aAAa,KAAK;IAC5E,IAAIgC,gBAAgBE,IAAI,CAAC,CAACpB,SAAWA,OAAOd,aAAa,KAAKkB,YAAYf,EAAE,GAC1E,MAAM,IAAInB,mCAAmCgD,eAAe,CAAC,EAAE,EAAE7B,MAAM;IACzE,MAAMgC,aAAaN,QAAQI,MAAM,CAAC,CAACnB,SAAWA,OAAOd,aAAa,KAAK;IACvE,MAAMoC,gBAAgB,MAAM1C,GACzBQ,MAAM,CAAC;QAACmC,OAAOxD,GAAW,CAAC,aAAa,CAAC;IAAA,GACzC4B,IAAI,CAACnB,iBACLoB,KAAK,CACJnC,IACEC,GAAGc,gBAAgBS,aAAa,EAAEP,OAAOO,aAAa,GACtDnB,GACEL,IACEC,GAAGc,gBAAgBU,aAAa,EAAEkB,YAAYf,EAAE,GAChDzB,OAAOY,gBAAgBgD,qBAAqB,IAE9C/D,IACEC,GAAGc,gBAAgBc,qBAAqB,EAAEc,YAAYf,EAAE,GACxDxB,WAAWW,gBAAgBa,EAAE,EAAEP,oBAC/BlB,OAAOY,gBAAgBgD,qBAAqB,GAC5C3D,WAAWW,gBAAgBkB,KAAK,EAAE;WAAIrB;KAAe;IAK/D,IAAI,AAACiD,CAAAA,aAAa,CAAC,EAAE,EAAEC,SAAS,CAAA,IAAKF,WAAWvB,MAAM,GAAGM,YAAYmB,KAAK,EACxE,MAAM,IAAIpD,iCAAiCkD,UAAU,CAAC,EAAE,EAAEhC,MAAM;IAClE,KAAK,MAAMW,UAAUqB,WAAY;QAC/B,IACErB,OAAON,KAAK,KAAK,aACjB,CAACM,OAAOR,gBAAgB,IACxB,CAACQ,OAAOgB,gBAAgB,IACxB,CAACZ,YAAYqB,cAAc,CAAC1B,KAAK,CAAC,CAAC2B,QAAU1B,OAAOP,MAAM,CAACkC,QAAQ,CAACD,SAEpE,MAAM,IAAIvD,iCAAiC6B,OAAOX,EAAE;IACxD;IACA,IAAIgC,WAAWvB,MAAM,GAAG,GAAG;QACzB,MAAMlB,GACHqB,MAAM,CAACzB,iBACP0B,GAAG,CAAC;YACHX,aAAaa,YAAYb,WAAW;YACpCL,eAAekB,YAAYf,EAAE;YAC7BC,uBAAuB;YACvBsC,YAAY7D,GAAG,CAAC,KAAK,CAAC;YACtBoC,WAAWpC,GAAG,CAAC,KAAK,CAAC;QACvB,GACC6B,KAAK,CACJjC,QACEa,gBAAgBa,EAAE,EAClBgC,WAAWX,GAAG,CAAC,CAACV,SAAWA,OAAOX,EAAE;IAG5C;IACA,OAAOX,OAAOI,iBAAiB;AACjC"}
+{"version":3,"sources":["../../src/db/runner-assignments.ts"],"sourcesContent":["import {and, eq, inArray, isNull, ne, notInArray, or, sql} from 'drizzle-orm';\nimport {\n  ReservationExpiredError,\n  ReservationNotFoundError,\n  RunnerInstanceAlreadyAssignedError,\n  RunnerInstanceNotAssignableError,\n} from '#core/errors.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';\nimport {terminalStates} from './runner-states.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {providerRunners} from './schema/runner-instances.js';\n\n/** Atomically consumes reservation units by writing the immutable assignment on each runner. */\nexport async function assignRunnerInstances(params: {\n  provisionerId: string;\n  reservationId: string;\n  runnerInstanceIds: string[];\n}): Promise<string[]> {\n  return await db().transaction(async (tx) => assignRunnerInstancesTx(tx, params));\n}\n\nexport async function assignRunnerInstancesTx(\n  tx: Tx,\n  params: {\n    provisionerId: string;\n    reservationId: string;\n    runnerInstanceIds: string[];\n  },\n): Promise<string[]> {\n  const runnerInstanceIds = [...params.runnerInstanceIds].sort();\n  await lockRunnerReservationAdvisoryKeysTx(tx, {\n    provisionerId: params.provisionerId,\n    reservationIds: [params.reservationId],\n  });\n  const runnerRows = await tx\n    .select({\n      id: providerRunners.id,\n      reservationId: providerRunners.reservationId,\n      intendedReservationId: providerRunners.intendedReservationId,\n      workspaceId: providerRunners.workspaceId,\n      assignedAt: providerRunners.assignedAt,\n      providerRunnerId: providerRunners.providerRunnerId,\n      labels: providerRunners.labels,\n      state: providerRunners.state,\n    })\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        inArray(providerRunners.id, runnerInstanceIds),\n      ),\n    )\n    .for('update');\n\n  // The assignment outlives its short reservation row. A retry after maintenance\n  // cleanup is complete when every owned runner already carries the committed assignment.\n  if (\n    runnerInstanceIds.length > 0 &&\n    runnerRows.length === runnerInstanceIds.length &&\n    runnerRows.every(\n      (runner) =>\n        runner.reservationId === params.reservationId &&\n        runner.assignedAt !== null &&\n        runner.workspaceId !== null,\n    )\n  ) {\n    await tx\n      .update(providerRunners)\n      .set({intendedReservationId: null, updatedAt: sql`now()`})\n      .where(\n        and(\n          eq(providerRunners.provisionerId, params.provisionerId),\n          inArray(providerRunners.id, runnerInstanceIds),\n        ),\n      );\n    return params.runnerInstanceIds;\n  }\n\n  const [reservation] = await tx\n    .select()\n    .from(reservations)\n    .where(\n      and(\n        eq(reservations.id, params.reservationId),\n        eq(reservations.provisionerId, params.provisionerId),\n        eq(reservations.kind, 'launch'),\n      ),\n    )\n    .limit(1)\n    .for('update');\n  if (!reservation) throw new ReservationNotFoundError(params.reservationId);\n  if (reservation.expiresAt <= new Date()) throw new ReservationExpiredError(params.reservationId);\n  if (runnerRows.length !== runnerInstanceIds.length)\n    throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '');\n\n  const activeControlSessions = await tx\n    .select({runnerInstanceId: runnerControlSessions.runnerInstanceId})\n    .from(runnerControlSessions)\n    .where(\n      and(\n        inArray(\n          runnerControlSessions.runnerInstanceId,\n          runnerRows.map((runner) => runner.id),\n        ),\n        isNull(runnerControlSessions.closedAt),\n      ),\n    );\n  const runnerInstanceIdsWithControlSession = new Set(\n    activeControlSessions.map((session) => session.runnerInstanceId),\n  );\n  const runners = runnerRows.map((runner) => ({\n    ...runner,\n    controlSessionId: runnerInstanceIdsWithControlSession.has(runner.id) ? runner.id : null,\n  }));\n\n  const alreadyAssigned = runners.filter((runner) => runner.reservationId !== null);\n  if (alreadyAssigned.some((runner) => runner.reservationId !== reservation.id))\n    throw new RunnerInstanceAlreadyAssignedError(alreadyAssigned[0]?.id ?? '');\n  const newRunners = runners.filter(\n    (runner) =>\n      runner.reservationId === null ||\n      (runner.reservationId === reservation.id &&\n        (runner.assignedAt === null || runner.workspaceId === null)),\n  );\n  const newRunnerIds = newRunners.map((runner) => runner.id);\n  const assignedCount = await tx\n    .select({count: sql<number>`count(*)::int`})\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        notInArray(providerRunners.id, newRunnerIds),\n        or(\n          and(\n            eq(providerRunners.reservationId, reservation.id),\n            isNull(providerRunners.reservationReleasedAt),\n          ),\n          and(\n            eq(providerRunners.intendedReservationId, reservation.id),\n            isNull(providerRunners.reservationReleasedAt),\n            notInArray(providerRunners.state, [...terminalStates]),\n          ),\n        ),\n      ),\n    );\n  if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count)\n    throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '');\n  for (const runner of newRunners) {\n    if (\n      runner.state !== 'running' ||\n      !runner.providerRunnerId ||\n      !runner.controlSessionId ||\n      !reservation.requiredLabels.every((label) => runner.labels.includes(label))\n    )\n      throw new RunnerInstanceNotAssignableError(runner.id);\n  }\n  if (newRunners.length > 0) {\n    await tx\n      .update(providerRunners)\n      .set({\n        workspaceId: reservation.workspaceId,\n        reservationId: reservation.id,\n        intendedReservationId: null,\n        assignedAt: sql`now()`,\n        updatedAt: sql`now()`,\n      })\n      .where(\n        inArray(\n          providerRunners.id,\n          newRunners.map((runner) => runner.id),\n        ),\n      );\n  }\n  return params.runnerInstanceIds;\n}\n\nexport type RunnerReservationCapacityFailureReason =\n  | 'reservation-not-found'\n  | 'reservation-kind-mismatch'\n  | 'reservation-expired'\n  | 'capacity-exhausted';\n\nexport interface RunnerReservationCapacityValidation {\n  acceptedByReservation: Map<string, number>;\n  unavailableByReservation: Map<\n    string,\n    {reason: RunnerReservationCapacityFailureReason; count: number}\n  >;\n}\n\nexport async function validateRunnerReservationCapacityTx(\n  tx: Tx,\n  params: {\n    provisionerId: string;\n    requests: readonly {reservationId: string; count: number}[];\n  },\n  options: {advisoryLocksHeld?: boolean} = {},\n): Promise<RunnerReservationCapacityValidation> {\n  const requestedByReservation = new Map<string, number>();\n  for (const request of params.requests) {\n    if (request.count <= 0) continue;\n    requestedByReservation.set(\n      request.reservationId,\n      (requestedByReservation.get(request.reservationId) ?? 0) + request.count,\n    );\n  }\n  const reservationIds = [...requestedByReservation.keys()].sort();\n  if (reservationIds.length === 0)\n    return {acceptedByReservation: new Map(), unavailableByReservation: new Map()};\n\n  if (!options.advisoryLocksHeld)\n    await lockRunnerReservationAdvisoryKeysTx(tx, {\n      provisionerId: params.provisionerId,\n      reservationIds,\n    });\n\n  const reservationRows = await tx\n    .select({\n      id: reservations.id,\n      count: reservations.count,\n      expiresAt: reservations.expiresAt,\n      isExpired: sql<boolean>`${reservations.expiresAt} <= now()`,\n    })\n    .from(reservations)\n    .where(\n      and(\n        eq(reservations.provisionerId, params.provisionerId),\n        inArray(reservations.id, reservationIds),\n        eq(reservations.kind, 'launch'),\n      ),\n    )\n    .for('update');\n  const nonLaunchReservationRows = await tx\n    .select({id: reservations.id})\n    .from(reservations)\n    .where(\n      and(\n        eq(reservations.provisionerId, params.provisionerId),\n        inArray(reservations.id, reservationIds),\n        ne(reservations.kind, 'launch'),\n      ),\n    )\n    .for('update');\n  const reservationsById = new Map(\n    reservationRows.map((reservation) => [reservation.id, reservation]),\n  );\n  const nonLaunchReservationIds = new Set(\n    nonLaunchReservationRows.map((reservation) => reservation.id),\n  );\n  const requestedReservationIds = new Set(reservationIds);\n  const usedRunnerIdsByReservation = new Map<string, Set<string>>();\n  const usedRunnerRows = await tx\n    .select({\n      id: providerRunners.id,\n      reservationId: providerRunners.reservationId,\n      intendedReservationId: providerRunners.intendedReservationId,\n      state: providerRunners.state,\n    })\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        isNull(providerRunners.reservationReleasedAt),\n        or(\n          inArray(providerRunners.reservationId, reservationIds),\n          and(\n            inArray(providerRunners.intendedReservationId, reservationIds),\n            notInArray(providerRunners.state, [...terminalStates]),\n          ),\n        ),\n      ),\n    );\n  for (const runner of usedRunnerRows) {\n    if (runner.reservationId && requestedReservationIds.has(runner.reservationId)) {\n      const usedRunnerIds =\n        usedRunnerIdsByReservation.get(runner.reservationId) ?? new Set<string>();\n      usedRunnerIds.add(runner.id);\n      usedRunnerIdsByReservation.set(runner.reservationId, usedRunnerIds);\n    }\n    if (\n      runner.intendedReservationId &&\n      requestedReservationIds.has(runner.intendedReservationId) &&\n      !terminalStates.includes(runner.state as (typeof terminalStates)[number])\n    ) {\n      const usedRunnerIds =\n        usedRunnerIdsByReservation.get(runner.intendedReservationId) ?? new Set<string>();\n      usedRunnerIds.add(runner.id);\n      usedRunnerIdsByReservation.set(runner.intendedReservationId, usedRunnerIds);\n    }\n  }\n\n  const acceptedByReservation = new Map<string, number>();\n  const unavailableByReservation = new Map<\n    string,\n    {reason: RunnerReservationCapacityFailureReason; count: number}\n  >();\n  for (const reservationId of reservationIds) {\n    const requested = requestedByReservation.get(reservationId) ?? 0;\n    const reservation = reservationsById.get(reservationId);\n    if (!reservation) {\n      unavailableByReservation.set(reservationId, {\n        reason: nonLaunchReservationIds.has(reservationId)\n          ? 'reservation-kind-mismatch'\n          : 'reservation-not-found',\n        count: requested,\n      });\n      continue;\n    }\n    if (reservation.isExpired) {\n      unavailableByReservation.set(reservationId, {\n        reason: 'reservation-expired',\n        count: requested,\n      });\n      continue;\n    }\n\n    const used = usedRunnerIdsByReservation.get(reservationId)?.size ?? 0;\n    const accepted = Math.min(requested, Math.max(0, reservation.count - used));\n    acceptedByReservation.set(reservationId, accepted);\n    if (accepted < requested) {\n      unavailableByReservation.set(reservationId, {\n        reason: 'capacity-exhausted',\n        count: requested - accepted,\n      });\n    }\n  }\n\n  return {acceptedByReservation, unavailableByReservation};\n}\n"],"names":["and","eq","inArray","isNull","ne","notInArray","or","sql","ReservationExpiredError","ReservationNotFoundError","RunnerInstanceAlreadyAssignedError","RunnerInstanceNotAssignableError","db","lockRunnerReservationAdvisoryKeysTx","terminalStates","reservations","runnerControlSessions","providerRunners","assignRunnerInstances","params","transaction","tx","assignRunnerInstancesTx","runnerInstanceIds","sort","provisionerId","reservationIds","reservationId","runnerRows","select","id","intendedReservationId","workspaceId","assignedAt","providerRunnerId","labels","state","from","where","for","length","every","runner","update","set","updatedAt","reservation","kind","limit","expiresAt","Date","activeControlSessions","runnerInstanceId","map","closedAt","runnerInstanceIdsWithControlSession","Set","session","runners","controlSessionId","has","alreadyAssigned","filter","some","newRunners","newRunnerIds","assignedCount","count","reservationReleasedAt","requiredLabels","label","includes","validateRunnerReservationCapacityTx","options","requestedByReservation","Map","request","requests","get","keys","acceptedByReservation","unavailableByReservation","advisoryLocksHeld","reservationRows","isExpired","nonLaunchReservationRows","reservationsById","nonLaunchReservationIds","requestedReservationIds","usedRunnerIdsByReservation","usedRunnerRows","usedRunnerIds","add","requested","reason","used","size","accepted","Math","min","max"],"mappings":"AAAA,SAAQA,GAAG,EAAEC,EAAE,EAAEC,OAAO,EAAEC,MAAM,EAAEC,EAAE,EAAEC,UAAU,EAAEC,EAAE,EAAEC,GAAG,QAAO,cAAc;AAC9E,SACEC,uBAAuB,EACvBC,wBAAwB,EACxBC,kCAAkC,EAClCC,gCAAgC,QAC3B,kBAAkB;AAEzB,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,mCAAmC,QAAO,yBAAyB;AAC3E,SAAQC,cAAc,QAAO,qBAAqB;AAClD,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAAQC,eAAe,QAAO,+BAA+B;AAE7D,8FAA8F,GAC9F,OAAO,eAAeC,sBAAsBC,MAI3C;IACC,OAAO,MAAMP,KAAKQ,WAAW,CAAC,OAAOC,KAAOC,wBAAwBD,IAAIF;AAC1E;AAEA,OAAO,eAAeG,wBACpBD,EAAM,EACNF,MAIC;IAED,MAAMI,oBAAoB;WAAIJ,OAAOI,iBAAiB;KAAC,CAACC,IAAI;IAC5D,MAAMX,oCAAoCQ,IAAI;QAC5CI,eAAeN,OAAOM,aAAa;QACnCC,gBAAgB;YAACP,OAAOQ,aAAa;SAAC;IACxC;IACA,MAAMC,aAAa,MAAMP,GACtBQ,MAAM,CAAC;QACNC,IAAIb,gBAAgBa,EAAE;QACtBH,eAAeV,gBAAgBU,aAAa;QAC5CI,uBAAuBd,gBAAgBc,qBAAqB;QAC5DC,aAAaf,gBAAgBe,WAAW;QACxCC,YAAYhB,gBAAgBgB,UAAU;QACtCC,kBAAkBjB,gBAAgBiB,gBAAgB;QAClDC,QAAQlB,gBAAgBkB,MAAM;QAC9BC,OAAOnB,gBAAgBmB,KAAK;IAC9B,GACCC,IAAI,CAACpB,iBACLqB,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDvB,QAAQe,gBAAgBa,EAAE,EAAEP,qBAG/BgB,GAAG,CAAC;IAEP,+EAA+E;IAC/E,wFAAwF;IACxF,IACEhB,kBAAkBiB,MAAM,GAAG,KAC3BZ,WAAWY,MAAM,KAAKjB,kBAAkBiB,MAAM,IAC9CZ,WAAWa,KAAK,CACd,CAACC,SACCA,OAAOf,aAAa,KAAKR,OAAOQ,aAAa,IAC7Ce,OAAOT,UAAU,KAAK,QACtBS,OAAOV,WAAW,KAAK,OAE3B;QACA,MAAMX,GACHsB,MAAM,CAAC1B,iBACP2B,GAAG,CAAC;YAACb,uBAAuB;YAAMc,WAAWtC,GAAG,CAAC,KAAK,CAAC;QAAA,GACvD+B,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDvB,QAAQe,gBAAgBa,EAAE,EAAEP;QAGlC,OAAOJ,OAAOI,iBAAiB;IACjC;IAEA,MAAM,CAACuB,YAAY,GAAG,MAAMzB,GACzBQ,MAAM,GACNQ,IAAI,CAACtB,cACLuB,KAAK,CACJtC,IACEC,GAAGc,aAAae,EAAE,EAAEX,OAAOQ,aAAa,GACxC1B,GAAGc,aAAaU,aAAa,EAAEN,OAAOM,aAAa,GACnDxB,GAAGc,aAAagC,IAAI,EAAE,YAGzBC,KAAK,CAAC,GACNT,GAAG,CAAC;IACP,IAAI,CAACO,aAAa,MAAM,IAAIrC,yBAAyBU,OAAOQ,aAAa;IACzE,IAAImB,YAAYG,SAAS,IAAI,IAAIC,QAAQ,MAAM,IAAI1C,wBAAwBW,OAAOQ,aAAa;IAC/F,IAAIC,WAAWY,MAAM,KAAKjB,kBAAkBiB,MAAM,EAChD,MAAM,IAAI7B,iCAAiCY,iBAAiB,CAAC,EAAE,IAAI;IAErE,MAAM4B,wBAAwB,MAAM9B,GACjCQ,MAAM,CAAC;QAACuB,kBAAkBpC,sBAAsBoC,gBAAgB;IAAA,GAChEf,IAAI,CAACrB,uBACLsB,KAAK,CACJtC,IACEE,QACEc,sBAAsBoC,gBAAgB,EACtCxB,WAAWyB,GAAG,CAAC,CAACX,SAAWA,OAAOZ,EAAE,IAEtC3B,OAAOa,sBAAsBsC,QAAQ;IAG3C,MAAMC,sCAAsC,IAAIC,IAC9CL,sBAAsBE,GAAG,CAAC,CAACI,UAAYA,QAAQL,gBAAgB;IAEjE,MAAMM,UAAU9B,WAAWyB,GAAG,CAAC,CAACX,SAAY,CAAA;YAC1C,GAAGA,MAAM;YACTiB,kBAAkBJ,oCAAoCK,GAAG,CAAClB,OAAOZ,EAAE,IAAIY,OAAOZ,EAAE,GAAG;QACrF,CAAA;IAEA,MAAM+B,kBAAkBH,QAAQI,MAAM,CAAC,CAACpB,SAAWA,OAAOf,aAAa,KAAK;IAC5E,IAAIkC,gBAAgBE,IAAI,CAAC,CAACrB,SAAWA,OAAOf,aAAa,KAAKmB,YAAYhB,EAAE,GAC1E,MAAM,IAAIpB,mCAAmCmD,eAAe,CAAC,EAAE,EAAE/B,MAAM;IACzE,MAAMkC,aAAaN,QAAQI,MAAM,CAC/B,CAACpB,SACCA,OAAOf,aAAa,KAAK,QACxBe,OAAOf,aAAa,KAAKmB,YAAYhB,EAAE,IACrCY,CAAAA,OAAOT,UAAU,KAAK,QAAQS,OAAOV,WAAW,KAAK,IAAG;IAE/D,MAAMiC,eAAeD,WAAWX,GAAG,CAAC,CAACX,SAAWA,OAAOZ,EAAE;IACzD,MAAMoC,gBAAgB,MAAM7C,GACzBQ,MAAM,CAAC;QAACsC,OAAO5D,GAAW,CAAC,aAAa,CAAC;IAAA,GACzC8B,IAAI,CAACpB,iBACLqB,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDpB,WAAWY,gBAAgBa,EAAE,EAAEmC,eAC/B3D,GACEN,IACEC,GAAGgB,gBAAgBU,aAAa,EAAEmB,YAAYhB,EAAE,GAChD3B,OAAOc,gBAAgBmD,qBAAqB,IAE9CpE,IACEC,GAAGgB,gBAAgBc,qBAAqB,EAAEe,YAAYhB,EAAE,GACxD3B,OAAOc,gBAAgBmD,qBAAqB,GAC5C/D,WAAWY,gBAAgBmB,KAAK,EAAE;WAAItB;KAAe;IAK/D,IAAI,AAACoD,CAAAA,aAAa,CAAC,EAAE,EAAEC,SAAS,CAAA,IAAKH,WAAWxB,MAAM,GAAGM,YAAYqB,KAAK,EACxE,MAAM,IAAIxD,iCAAiCqD,UAAU,CAAC,EAAE,EAAElC,MAAM;IAClE,KAAK,MAAMY,UAAUsB,WAAY;QAC/B,IACEtB,OAAON,KAAK,KAAK,aACjB,CAACM,OAAOR,gBAAgB,IACxB,CAACQ,OAAOiB,gBAAgB,IACxB,CAACb,YAAYuB,cAAc,CAAC5B,KAAK,CAAC,CAAC6B,QAAU5B,OAAOP,MAAM,CAACoC,QAAQ,CAACD,SAEpE,MAAM,IAAI3D,iCAAiC+B,OAAOZ,EAAE;IACxD;IACA,IAAIkC,WAAWxB,MAAM,GAAG,GAAG;QACzB,MAAMnB,GACHsB,MAAM,CAAC1B,iBACP2B,GAAG,CAAC;YACHZ,aAAac,YAAYd,WAAW;YACpCL,eAAemB,YAAYhB,EAAE;YAC7BC,uBAAuB;YACvBE,YAAY1B,GAAG,CAAC,KAAK,CAAC;YACtBsC,WAAWtC,GAAG,CAAC,KAAK,CAAC;QACvB,GACC+B,KAAK,CACJpC,QACEe,gBAAgBa,EAAE,EAClBkC,WAAWX,GAAG,CAAC,CAACX,SAAWA,OAAOZ,EAAE;IAG5C;IACA,OAAOX,OAAOI,iBAAiB;AACjC;AAgBA,OAAO,eAAeiD,oCACpBnD,EAAM,EACNF,MAGC,EACDsD,UAAyC,CAAC,CAAC;IAE3C,MAAMC,yBAAyB,IAAIC;IACnC,KAAK,MAAMC,WAAWzD,OAAO0D,QAAQ,CAAE;QACrC,IAAID,QAAQT,KAAK,IAAI,GAAG;QACxBO,uBAAuB9B,GAAG,CACxBgC,QAAQjD,aAAa,EACrB,AAAC+C,CAAAA,uBAAuBI,GAAG,CAACF,QAAQjD,aAAa,KAAK,CAAA,IAAKiD,QAAQT,KAAK;IAE5E;IACA,MAAMzC,iBAAiB;WAAIgD,uBAAuBK,IAAI;KAAG,CAACvD,IAAI;IAC9D,IAAIE,eAAec,MAAM,KAAK,GAC5B,OAAO;QAACwC,uBAAuB,IAAIL;QAAOM,0BAA0B,IAAIN;IAAK;IAE/E,IAAI,CAACF,QAAQS,iBAAiB,EAC5B,MAAMrE,oCAAoCQ,IAAI;QAC5CI,eAAeN,OAAOM,aAAa;QACnCC;IACF;IAEF,MAAMyD,kBAAkB,MAAM9D,GAC3BQ,MAAM,CAAC;QACNC,IAAIf,aAAae,EAAE;QACnBqC,OAAOpD,aAAaoD,KAAK;QACzBlB,WAAWlC,aAAakC,SAAS;QACjCmC,WAAW7E,GAAY,CAAC,EAAEQ,aAAakC,SAAS,CAAC,SAAS,CAAC;IAC7D,GACCZ,IAAI,CAACtB,cACLuB,KAAK,CACJtC,IACEC,GAAGc,aAAaU,aAAa,EAAEN,OAAOM,aAAa,GACnDvB,QAAQa,aAAae,EAAE,EAAEJ,iBACzBzB,GAAGc,aAAagC,IAAI,EAAE,YAGzBR,GAAG,CAAC;IACP,MAAM8C,2BAA2B,MAAMhE,GACpCQ,MAAM,CAAC;QAACC,IAAIf,aAAae,EAAE;IAAA,GAC3BO,IAAI,CAACtB,cACLuB,KAAK,CACJtC,IACEC,GAAGc,aAAaU,aAAa,EAAEN,OAAOM,aAAa,GACnDvB,QAAQa,aAAae,EAAE,EAAEJ,iBACzBtB,GAAGW,aAAagC,IAAI,EAAE,YAGzBR,GAAG,CAAC;IACP,MAAM+C,mBAAmB,IAAIX,IAC3BQ,gBAAgB9B,GAAG,CAAC,CAACP,cAAgB;YAACA,YAAYhB,EAAE;YAAEgB;SAAY;IAEpE,MAAMyC,0BAA0B,IAAI/B,IAClC6B,yBAAyBhC,GAAG,CAAC,CAACP,cAAgBA,YAAYhB,EAAE;IAE9D,MAAM0D,0BAA0B,IAAIhC,IAAI9B;IACxC,MAAM+D,6BAA6B,IAAId;IACvC,MAAMe,iBAAiB,MAAMrE,GAC1BQ,MAAM,CAAC;QACNC,IAAIb,gBAAgBa,EAAE;QACtBH,eAAeV,gBAAgBU,aAAa;QAC5CI,uBAAuBd,gBAAgBc,qBAAqB;QAC5DK,OAAOnB,gBAAgBmB,KAAK;IAC9B,GACCC,IAAI,CAACpB,iBACLqB,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDtB,OAAOc,gBAAgBmD,qBAAqB,GAC5C9D,GACEJ,QAAQe,gBAAgBU,aAAa,EAAED,iBACvC1B,IACEE,QAAQe,gBAAgBc,qBAAqB,EAAEL,iBAC/CrB,WAAWY,gBAAgBmB,KAAK,EAAE;WAAItB;KAAe;IAK/D,KAAK,MAAM4B,UAAUgD,eAAgB;QACnC,IAAIhD,OAAOf,aAAa,IAAI6D,wBAAwB5B,GAAG,CAAClB,OAAOf,aAAa,GAAG;YAC7E,MAAMgE,gBACJF,2BAA2BX,GAAG,CAACpC,OAAOf,aAAa,KAAK,IAAI6B;YAC9DmC,cAAcC,GAAG,CAAClD,OAAOZ,EAAE;YAC3B2D,2BAA2B7C,GAAG,CAACF,OAAOf,aAAa,EAAEgE;QACvD;QACA,IACEjD,OAAOX,qBAAqB,IAC5ByD,wBAAwB5B,GAAG,CAAClB,OAAOX,qBAAqB,KACxD,CAACjB,eAAeyD,QAAQ,CAAC7B,OAAON,KAAK,GACrC;YACA,MAAMuD,gBACJF,2BAA2BX,GAAG,CAACpC,OAAOX,qBAAqB,KAAK,IAAIyB;YACtEmC,cAAcC,GAAG,CAAClD,OAAOZ,EAAE;YAC3B2D,2BAA2B7C,GAAG,CAACF,OAAOX,qBAAqB,EAAE4D;QAC/D;IACF;IAEA,MAAMX,wBAAwB,IAAIL;IAClC,MAAMM,2BAA2B,IAAIN;IAIrC,KAAK,MAAMhD,iBAAiBD,eAAgB;QAC1C,MAAMmE,YAAYnB,uBAAuBI,GAAG,CAACnD,kBAAkB;QAC/D,MAAMmB,cAAcwC,iBAAiBR,GAAG,CAACnD;QACzC,IAAI,CAACmB,aAAa;YAChBmC,yBAAyBrC,GAAG,CAACjB,eAAe;gBAC1CmE,QAAQP,wBAAwB3B,GAAG,CAACjC,iBAChC,8BACA;gBACJwC,OAAO0B;YACT;YACA;QACF;QACA,IAAI/C,YAAYsC,SAAS,EAAE;YACzBH,yBAAyBrC,GAAG,CAACjB,eAAe;gBAC1CmE,QAAQ;gBACR3B,OAAO0B;YACT;YACA;QACF;QAEA,MAAME,OAAON,2BAA2BX,GAAG,CAACnD,gBAAgBqE,QAAQ;QACpE,MAAMC,WAAWC,KAAKC,GAAG,CAACN,WAAWK,KAAKE,GAAG,CAAC,GAAGtD,YAAYqB,KAAK,GAAG4B;QACrEf,sBAAsBpC,GAAG,CAACjB,eAAesE;QACzC,IAAIA,WAAWJ,WAAW;YACxBZ,yBAAyBrC,GAAG,CAACjB,eAAe;gBAC1CmE,QAAQ;gBACR3B,OAAO0B,YAAYI;YACrB;QACF;IACF;IAEA,OAAO;QAACjB;QAAuBC;IAAwB;AACzD"}

package/dist/db/runner-instances.d.ts

@@ -1,13 +1,13 @@
 import type { RunnerInstance, RunnerInstanceState } from '#core/entities/runner-instance.js';
 import type { Tx } from './db.js';
 import { type RunnerInstanceBoundJobExecution } from './job-executions.js';
-export declare const terminalStates: readonly ["stopped", "failed", "terminated"];
 export declare const activeStates: readonly ["starting", "running", "stopping"];
 export declare const divergenceCountStates: readonly ["starting", "running"];
-export type RunnerInstanceTerminateIntentReason = 'job-cancelled';
+export type RunnerInstanceTerminateIntentReason = 'activation-timeout' | 'job-cancelled';
 export interface RunnerInstanceTerminateIntent {
     providerRunnerId: string;
     reason: RunnerInstanceTerminateIntentReason;
+    activationTimeoutRetry?: boolean;
 }
 export interface ActiveRunnerInstanceTemplateCount {
     templateKey: string;
@@ -61,6 +61,9 @@ export declare function listActiveRunnerInstanceCountsByTemplateTx(tx: Tx, param
     workspaceId: string;
     provisionerId: string;
 }): Promise<ActiveRunnerInstanceTemplateCount[]>;
+export declare function countStaleEnrolledRunnerInstances(params: {
+    graceSeconds: number;
+}): Promise<number>;
 export declare function listActiveRunnerInstances(params: {
     workspaceId: string;
     windowSeconds: number;

package/dist/db/runner-instances.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner-instances.d.ts","sourceRoot":"","sources":["../../src/db/runner-instances.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,mCAAmC,CAAC;AAE3F,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAEhC,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,qBAAqB,CAAC;AAS7B,eAAO,MAAM,cAAc,8CAIwB,CAAC;AACpD,eAAO,MAAM,YAAY,8CAI0B,CAAC;AACpD,eAAO,MAAM,qBAAqB,kCAG/B,CAAC;AAEJ,MAAM,MAAM,mCAAmC,GAAG,eAAe,CAAC;AAElE,MAAM,WAAW,6BAA6B;IAC5C,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,mCAAmC,CAAC;CAC7C;AAED,MAAM,WAAW,iCAAiC;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,cAAc,GAAG,WAAW,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,yBAAyB,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,8BAA8B;IAC7C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,gCAAgC;IAC/C,YAAY,EAAE,cAAc,EAAE,CAAC;IAC/B,oCAAoC,EAAE,GAAG,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAiBD,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,2BAA2B,GAAG,OAAO,CAAC;IACxF,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,6BAA6B,EAAE,CAAC;CAC1D,CAAC,CA+ED;AAED,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAC,GACnD,OAAO,CAAC,iCAAiC,EAAE,CAAC,CAuB9C;AAED,wBAAsB,yBAAyB,CAAC,MAAM,EAAE;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAe5B;AAED,wBAAsB,+BAA+B,CAAC,MAAM,EAAE;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAKpB;AAED,wBAAsB,oCAAoC,CACxD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GACA,OAAO,CAAC,6BAA6B,EAAE,CAAC,CAsB1C;AAqFD,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,8BAA8B,GACrC,OAAO,CAAC,gCAAgC,CAAC,CAqG3C;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,8BAA8B,CAAC,CA4D1C;AAkcD,wBAAgB,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAEnE"}
+{"version":3,"file":"runner-instances.d.ts","sourceRoot":"","sources":["../../src/db/runner-instances.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,mCAAmC,CAAC;AAG3F,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAEhC,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,qBAAqB,CAAC;AAa7B,eAAO,MAAM,YAAY,8CAI0B,CAAC;AACpD,eAAO,MAAM,qBAAqB,kCAG/B,CAAC;AAEJ,MAAM,MAAM,mCAAmC,GAAG,oBAAoB,GAAG,eAAe,CAAC;AAEzF,MAAM,WAAW,6BAA6B;IAC5C,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,mCAAmC,CAAC;IAC5C,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,iCAAiC;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,cAAc,GAAG,WAAW,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,yBAAyB,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,8BAA8B;IAC7C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,gCAAgC;IAC/C,YAAY,EAAE,cAAc,EAAE,CAAC;IAC/B,oCAAoC,EAAE,GAAG,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAiBD,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,2BAA2B,GAAG,OAAO,CAAC;IACxF,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,6BAA6B,EAAE,CAAC;CAC1D,CAAC,CAoFD;AA6FD,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAC,GACnD,OAAO,CAAC,iCAAiC,EAAE,CAAC,CAuB9C;AAED,wBAAsB,iCAAiC,CAAC,MAAM,EAAE;IAC9D,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0BlB;AAED,wBAAsB,yBAAyB,CAAC,MAAM,EAAE;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAe5B;AAED,wBAAsB,+BAA+B,CAAC,MAAM,EAAE;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAKpB;AAED,wBAAsB,oCAAoC,CACxD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GACA,OAAO,CAAC,6BAA6B,EAAE,CAAC,CAqC1C;AAqID,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,8BAA8B,GACrC,OAAO,CAAC,gCAAgC,CAAC,CAqG3C;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,8BAA8B,CAAC,CA4D1C;AAkcD,wBAAgB,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAEnE"}

package/dist/db/runner-instances.js

@@ -1,21 +1,22 @@
 import { logger } from '@shipfox/node-opentelemetry';
 import { and, asc, desc, eq, exists, gt, inArray, isNotNull, isNull, lt, notExists, notInArray, or, sql } from 'drizzle-orm';
 import { alias } from 'drizzle-orm/pg-core';
+import { config } from '#config.js';
 import { sanitizeRunnerLabels } from '#core/runner-labels.js';
+import { recordRunnerReservationCapacityFailure } from '#metrics/index.js';
 import { db } from './db.js';
 import { listRunningJobExecutionsByRunnerInstanceTx } from './job-executions.js';
+import { lockRunnerReservationAdvisoryKeysTx } from './reservation-locks.js';
 import { releaseReservationUnits } from './reservations.js';
+import { validateRunnerReservationCapacityTx } from './runner-assignments.js';
+import { terminalStates } from './runner-states.js';
 import { ephemeralRegistrationTokens } from './schema/ephemeral-registration-tokens.js';
 import { provisionerTokens } from './schema/provisioner-tokens.js';
 import { reservations } from './schema/reservations.js';
+import { runnerControlSessions } from './schema/runner-control-sessions.js';
 import { providerRunners, toRunnerInstance } from './schema/runner-instances.js';
 import { runnerSessions } from './schema/runner-sessions.js';
 import { runningJobExecutions } from './schema/running-job-executions.js';
-export const terminalStates = [
-    'stopped',
-    'failed',
-    'terminated'
-];
 export const activeStates = [
     'starting',
     'running',
@@ -50,8 +51,9 @@ export async function reportRunnerInstances(params) {
             await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId ?? params.provisionerId}))`);
         }
         const events = await hydrateRunnerSessionIdsFromConsumedTokens(tx, params, aggregatedEvents);
-        const terminateIntentsHonored = await listTerminateIntentsHonoredByTerminatedReportsTx(tx, params, events);
-        const values = events.map((event)=>({
+        const reservationSafeEvents = await guardReportedReservationIdsTx(tx, params, events);
+        const terminateIntentsHonored = await listTerminateIntentsHonoredByTerminatedReportsTx(tx, params, reservationSafeEvents);
+        const values = reservationSafeEvents.map((event)=>({
                 workspaceId: params.workspaceId,
                 provisionerId: params.provisionerId,
                 providerRunnerId: event.providerRunnerId,
@@ -80,7 +82,7 @@ export async function reportRunnerInstances(params) {
             ],
             targetWhere: isNotNull(providerRunners.providerRunnerId),
             set: {
-                reservationId: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(excluded.reservation_id, ${providerRunners.reservationId}) ELSE ${providerRunners.reservationId} END`,
+                reservationId: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(${providerRunners.reservationId}, excluded.reservation_id) ELSE ${providerRunners.reservationId} END`,
                 templateKey: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(excluded.template_key, ${providerRunners.templateKey}) ELSE ${providerRunners.templateKey} END`,
                 labels: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.labels ELSE ${providerRunners.labels} END`,
                 state: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.state ELSE ${providerRunners.state} END`,
@@ -100,14 +102,92 @@ export async function reportRunnerInstances(params) {
           OR ${providerRunnerMilestoneUpdateCondition()}
         `
         });
-        const reservationsReleased = hasTerminalEvent ? await releaseTerminalRunnerInstanceReservations(tx, params, events) : 0;
+        const reservationsReleased = hasTerminalEvent ? await releaseTerminalRunnerInstanceReservations(tx, params, reservationSafeEvents) : 0;
         return {
-            accepted: events.length,
+            accepted: reservationSafeEvents.length,
             reservationsReleased,
             terminateIntentsHonored
         };
     });
 }
+async function guardReportedReservationIdsTx(tx, params, events) {
+    const reservationIds = [
+        ...new Set(events.flatMap((event)=>event.reservationId ? [
+                event.reservationId
+            ] : []))
+    ].sort();
+    if (reservationIds.length === 0) return events;
+    await lockRunnerReservationAdvisoryKeysTx(tx, {
+        provisionerId: params.provisionerId,
+        reservationIds
+    });
+    const providerRunnerIds = events.map((event)=>event.providerRunnerId);
+    const existingRows = await tx.select({
+        providerRunnerId: providerRunners.providerRunnerId,
+        reservationId: providerRunners.reservationId,
+        intendedReservationId: providerRunners.intendedReservationId,
+        reservationReleasedAt: providerRunners.reservationReleasedAt
+    }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.providerRunnerId, providerRunnerIds))).for('update');
+    const existingByProviderRunnerId = new Map(existingRows.flatMap((row)=>row.providerRunnerId ? [
+            [
+                row.providerRunnerId,
+                row
+            ]
+        ] : []));
+    const candidateReservationByProviderRunnerId = new Map();
+    for (const event of events){
+        if (!event.reservationId) continue;
+        if (isTerminalState(event.state)) continue;
+        const existing = existingByProviderRunnerId.get(event.providerRunnerId);
+        if (existing?.reservationId || existing?.intendedReservationId === event.reservationId) continue;
+        if (existing?.intendedReservationId || existing?.reservationReleasedAt) continue;
+        candidateReservationByProviderRunnerId.set(event.providerRunnerId, event.reservationId);
+    }
+    const validation = await validateRunnerReservationCapacityTx(tx, {
+        provisionerId: params.provisionerId,
+        requests: [
+            ...candidateReservationByProviderRunnerId.values()
+        ].map((reservationId)=>({
+                reservationId,
+                count: 1
+            }))
+    }, {
+        advisoryLocksHeld: true
+    });
+    for (const { reason, count } of validation.unavailableByReservation.values())recordRunnerReservationCapacityFailure(reason, count);
+    const remainingAcceptedByReservation = new Map(validation.acceptedByReservation);
+    return events.map((event)=>{
+        const existing = existingByProviderRunnerId.get(event.providerRunnerId);
+        const reservationId = candidateReservationByProviderRunnerId.get(event.providerRunnerId);
+        if (!reservationId) {
+            if (event.reservationId && isTerminalState(event.state)) {
+                // Terminal cleanup releases the IDs stored on the projection row. An arbitrary
+                // terminal report must not consume reservation capacity during that cleanup.
+                const matchesExistingReservation = existing !== undefined && (existing.reservationId === event.reservationId || existing.intendedReservationId === event.reservationId);
+                if (!matchesExistingReservation) return {
+                    ...event,
+                    reservationId: null
+                };
+            }
+            if (event.reservationId && existing?.intendedReservationId && existing.intendedReservationId !== event.reservationId) return {
+                ...event,
+                reservationId: null
+            };
+            if (event.reservationId && existing?.reservationReleasedAt) return {
+                ...event,
+                reservationId: null
+            };
+            return event;
+        }
+        const remaining = remainingAcceptedByReservation.get(reservationId) ?? 0;
+        if (remaining === 0) return {
+            ...event,
+            reservationId: null
+        };
+        remainingAcceptedByReservation.set(reservationId, remaining - 1);
+        return event;
+    });
+}
 export async function listActiveRunnerInstanceCountsByTemplateTx(tx, params) {
     const rows = await tx.select({
         templateKey: providerRunners.templateKey,
@@ -122,6 +202,14 @@ export async function listActiveRunnerInstanceCountsByTemplateTx(tx, params) {
             }
         ] : []);
 }
+export async function countStaleEnrolledRunnerInstances(params) {
+    const [row] = await db().select({
+        count: sql`count(*)::int`
+    }).from(providerRunners).where(and(eq(providerRunners.state, 'running'), isNull(providerRunners.workspaceId), isNull(providerRunners.runnerSessionId), lt(providerRunners.reportedAt, staleRunnerInstanceCutoff(params.graceSeconds)), exists(db().select({
+        id: runnerControlSessions.id
+    }).from(runnerControlSessions).where(and(eq(runnerControlSessions.runnerInstanceId, providerRunners.id), isNull(runnerControlSessions.closedAt), gt(runnerControlSessions.expiresAt, sql`now()`))))));
+    return row?.count ?? 0;
+}
 export async function listActiveRunnerInstances(params) {
     const rows = await db().select().from(providerRunners).where(and(eq(providerRunners.workspaceId, params.workspaceId), inArray(providerRunners.state, activeStates), sql`${providerRunners.updatedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`)).orderBy(desc(providerRunners.updatedAt), desc(providerRunners.id)).limit(params.limit ?? 1000);
     return rows.map(toRunnerInstance);
@@ -144,21 +232,47 @@ export async function listProvisionerTerminateIntentRowsTx(tx, params) {
             returnedCount: returnedRows.length
         }, 'provisioner terminate intents truncated by poll-demand limit');
     }
-    return returnedRows.flatMap((row)=>row.providerRunnerId ? [
-            {
-                ...row,
-                providerRunnerId: row.providerRunnerId
-            }
+    const activationTimeoutRunnerIds = returnedRows.flatMap((row)=>row.reason === 'activation-timeout' && row.providerRunnerId ? [
+            row.providerRunnerId
         ] : []);
+    if (activationTimeoutRunnerIds.length > 0) {
+        await tx.update(providerRunners).set({
+            reservationReleasedAt: sql`now()`,
+            updatedAt: sql`now()`
+        }).where(and(eq(providerRunners.workspaceId, params.workspaceId), eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.providerRunnerId, activationTimeoutRunnerIds), isNull(providerRunners.reservationReleasedAt)));
+    }
+    return returnedRows.flatMap((row)=>toRunnerInstanceTerminateIntent(row));
+}
+function toRunnerInstanceTerminateIntent(row) {
+    if (!row.providerRunnerId) return [];
+    return [
+        {
+            providerRunnerId: row.providerRunnerId,
+            reason: row.reason,
+            ...row.reason === 'activation-timeout' && row.activationTimeoutRetry ? {
+                activationTimeoutRetry: true
+            } : {}
+        }
+    ];
 }
 function provisionerTerminateIntentsQuery(tx, params) {
     const newerRunningJobExecutions = alias(runningJobExecutions, 'newer_running_jobs');
+    const latestCancelledJob = exists(tx.select({
+        id: runningJobExecutions.id
+    }).from(runningJobExecutions).where(and(eq(runningJobExecutions.workspaceId, params.workspaceId), eq(runningJobExecutions.provisionerId, params.provisionerId), eq(runningJobExecutions.providerRunnerId, providerRunners.providerRunnerId), isNotNull(runningJobExecutions.cancellationRequestedAt), notExists(tx.select({
+        id: newerRunningJobExecutions.id
+    }).from(newerRunningJobExecutions).where(and(eq(newerRunningJobExecutions.workspaceId, runningJobExecutions.workspaceId), eq(newerRunningJobExecutions.provisionerId, runningJobExecutions.provisionerId), eq(newerRunningJobExecutions.providerRunnerId, runningJobExecutions.providerRunnerId), or(gt(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt), and(eq(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt), gt(newerRunningJobExecutions.jobExecutionId, runningJobExecutions.jobExecutionId)))))))));
+    const activationTimeout = and(eq(providerRunners.launchKind, 'demand'), isNull(providerRunners.runnerSessionId), lt(providerRunners.createdAt, sql`now() - (${config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS} || ' seconds')::interval`), notExists(tx.select({
+        id: reservations.id
+    }).from(reservations).where(and(or(eq(reservations.id, providerRunners.reservationId), eq(reservations.id, providerRunners.intendedReservationId)), eq(reservations.workspaceId, params.workspaceId), eq(reservations.provisionerId, params.provisionerId), gt(reservations.expiresAt, sql`now()`)))));
     return tx.select({
         providerRunnerId: providerRunners.providerRunnerId,
-        reason: sql`'job-cancelled'`
-    }).from(runningJobExecutions).innerJoin(providerRunners, and(eq(providerRunners.workspaceId, runningJobExecutions.workspaceId), eq(providerRunners.provisionerId, runningJobExecutions.provisionerId), eq(providerRunners.providerRunnerId, runningJobExecutions.providerRunnerId))).where(and(eq(runningJobExecutions.workspaceId, params.workspaceId), eq(runningJobExecutions.provisionerId, params.provisionerId), isNotNull(providerRunners.providerRunnerId), isNotNull(runningJobExecutions.cancellationRequestedAt), inArray(providerRunners.state, activeStates), params.providerRunnerIds && params.providerRunnerIds.length > 0 ? inArray(providerRunners.providerRunnerId, params.providerRunnerIds) : undefined, notExists(tx.select({
-        id: newerRunningJobExecutions.id
-    }).from(newerRunningJobExecutions).where(and(eq(newerRunningJobExecutions.workspaceId, runningJobExecutions.workspaceId), eq(newerRunningJobExecutions.provisionerId, runningJobExecutions.provisionerId), eq(newerRunningJobExecutions.providerRunnerId, runningJobExecutions.providerRunnerId), or(gt(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt), and(eq(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt), gt(newerRunningJobExecutions.jobExecutionId, runningJobExecutions.jobExecutionId)))))))).groupBy(providerRunners.providerRunnerId);
+        activationTimeoutRetry: sql`${providerRunners.reservationReleasedAt} is not null`,
+        reason: sql`case
+        when ${activationTimeout} then 'activation-timeout'
+        else 'job-cancelled'
+      end`
+    }).from(providerRunners).where(and(eq(providerRunners.workspaceId, params.workspaceId), eq(providerRunners.provisionerId, params.provisionerId), isNotNull(providerRunners.providerRunnerId), inArray(providerRunners.state, activeStates), params.providerRunnerIds && params.providerRunnerIds.length > 0 ? inArray(providerRunners.providerRunnerId, params.providerRunnerIds) : undefined, or(latestCancelledJob, activationTimeout)));
 }
 async function listTerminateIntentsHonoredByTerminatedReportsTx(tx, params, events) {
     if (!params.workspaceId) return [];
@@ -171,12 +285,7 @@ async function listTerminateIntentsHonoredByTerminatedReportsTx(tx, params, even
         provisionerId: params.provisionerId,
         providerRunnerIds: terminatedRunnerInstanceIds
     }).orderBy(asc(providerRunners.providerRunnerId));
-    return rows.flatMap((row)=>row.providerRunnerId ? [
-            {
-                ...row,
-                providerRunnerId: row.providerRunnerId
-            }
-        ] : []);
+    return rows.flatMap((row)=>toRunnerInstanceTerminateIntent(row));
 }
 export async function reconcileRunnerInstances(params) {
     const observedRunnerInstanceIds = [