@shipfox/api-runners 12.3.0 → 12.4.1

package/dist/db/runner-instances.d.ts

@@ -1,13 +1,13 @@
 import type { RunnerInstance, RunnerInstanceState } from '#core/entities/runner-instance.js';
 import type { Tx } from './db.js';
 import { type RunnerInstanceBoundJobExecution } from './job-executions.js';
-export declare const terminalStates: readonly ["stopped", "failed", "terminated"];
 export declare const activeStates: readonly ["starting", "running", "stopping"];
 export declare const divergenceCountStates: readonly ["starting", "running"];
-export type RunnerInstanceTerminateIntentReason = 'job-cancelled';
+export type RunnerInstanceTerminateIntentReason = 'activation-timeout' | 'job-cancelled';
 export interface RunnerInstanceTerminateIntent {
     providerRunnerId: string;
     reason: RunnerInstanceTerminateIntentReason;
+    activationTimeoutRetry?: boolean;
 }
 export interface ActiveRunnerInstanceTemplateCount {
     templateKey: string;
@@ -64,6 +64,15 @@ export declare function listActiveRunnerInstanceCountsByTemplateTx(tx: Tx, param
 export declare function countStaleEnrolledRunnerInstances(params: {
     graceSeconds: number;
 }): Promise<number>;
+export type ProviderRunnerPhase = 'control_session' | 'enrollment' | 'assignment' | 'activation' | 'idle';
+export interface ProviderRunnerPhaseMetric {
+    phase: ProviderRunnerPhase;
+    provider: string;
+    launchKind: 'demand' | 'warm' | 'manual';
+    count: number;
+    oldestAgeMilliseconds: number;
+}
+export declare function listProviderRunnerByPhaseMetrics(): Promise<ProviderRunnerPhaseMetric[]>;
 export declare function listActiveRunnerInstances(params: {
     workspaceId: string;
     windowSeconds: number;

package/dist/db/runner-instances.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner-instances.d.ts","sourceRoot":"","sources":["../../src/db/runner-instances.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,mCAAmC,CAAC;AAE3F,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAEhC,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,qBAAqB,CAAC;AAU7B,eAAO,MAAM,cAAc,8CAIwB,CAAC;AACpD,eAAO,MAAM,YAAY,8CAI0B,CAAC;AACpD,eAAO,MAAM,qBAAqB,kCAG/B,CAAC;AAEJ,MAAM,MAAM,mCAAmC,GAAG,eAAe,CAAC;AAElE,MAAM,WAAW,6BAA6B;IAC5C,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,mCAAmC,CAAC;CAC7C;AAED,MAAM,WAAW,iCAAiC;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,cAAc,GAAG,WAAW,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,yBAAyB,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,8BAA8B;IAC7C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,gCAAgC;IAC/C,YAAY,EAAE,cAAc,EAAE,CAAC;IAC/B,oCAAoC,EAAE,GAAG,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAiBD,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,2BAA2B,GAAG,OAAO,CAAC;IACxF,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,6BAA6B,EAAE,CAAC;CAC1D,CAAC,CA+ED;AAED,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAC,GACnD,OAAO,CAAC,iCAAiC,EAAE,CAAC,CAuB9C;AAED,wBAAsB,iCAAiC,CAAC,MAAM,EAAE;IAC9D,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0BlB;AAED,wBAAsB,yBAAyB,CAAC,MAAM,EAAE;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAe5B;AAED,wBAAsB,+BAA+B,CAAC,MAAM,EAAE;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAKpB;AAED,wBAAsB,oCAAoC,CACxD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GACA,OAAO,CAAC,6BAA6B,EAAE,CAAC,CAsB1C;AAqFD,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,8BAA8B,GACrC,OAAO,CAAC,gCAAgC,CAAC,CAqG3C;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,8BAA8B,CAAC,CA4D1C;AAkcD,wBAAgB,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAEnE"}
+{"version":3,"file":"runner-instances.d.ts","sourceRoot":"","sources":["../../src/db/runner-instances.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,mCAAmC,CAAC;AAG3F,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAEhC,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,qBAAqB,CAAC;AAa7B,eAAO,MAAM,YAAY,8CAI0B,CAAC;AACpD,eAAO,MAAM,qBAAqB,kCAG/B,CAAC;AAEJ,MAAM,MAAM,mCAAmC,GAAG,oBAAoB,GAAG,eAAe,CAAC;AAEzF,MAAM,WAAW,6BAA6B;IAC5C,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,mCAAmC,CAAC;IAC5C,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,iCAAiC;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,cAAc,GAAG,WAAW,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,yBAAyB,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,8BAA8B;IAC7C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,gCAAgC;IAC/C,YAAY,EAAE,cAAc,EAAE,CAAC;IAC/B,oCAAoC,EAAE,GAAG,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAiBD,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,2BAA2B,GAAG,OAAO,CAAC;IACxF,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,6BAA6B,EAAE,CAAC;CAC1D,CAAC,CAoFD;AA6FD,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAC,GACnD,OAAO,CAAC,iCAAiC,EAAE,CAAC,CAuB9C;AAED,wBAAsB,iCAAiC,CAAC,MAAM,EAAE;IAC9D,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0BlB;AAED,MAAM,MAAM,mBAAmB,GAC3B,iBAAiB,GACjB,YAAY,GACZ,YAAY,GACZ,YAAY,GACZ,MAAM,CAAC;AAEX,MAAM,WAAW,yBAAyB;IACxC,KAAK,EAAE,mBAAmB,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,GAAG,MAAM,GAAG,QAAQ,CAAC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,wBAAsB,gCAAgC,IAAI,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAyD7F;AAED,wBAAsB,yBAAyB,CAAC,MAAM,EAAE;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAe5B;AAED,wBAAsB,+BAA+B,CAAC,MAAM,EAAE;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAKpB;AAED,wBAAsB,oCAAoC,CACxD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GACA,OAAO,CAAC,6BAA6B,EAAE,CAAC,CAqC1C;AAqID,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,8BAA8B,GACrC,OAAO,CAAC,gCAAgC,CAAC,CAqG3C;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,8BAA8B,CAAC,CA4D1C;AAkcD,wBAAgB,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAEnE"}

package/dist/db/runner-instances.js

@@ -1,10 +1,15 @@
 import { logger } from '@shipfox/node-opentelemetry';
 import { and, asc, desc, eq, exists, gt, inArray, isNotNull, isNull, lt, notExists, notInArray, or, sql } from 'drizzle-orm';
 import { alias } from 'drizzle-orm/pg-core';
+import { config } from '#config.js';
 import { sanitizeRunnerLabels } from '#core/runner-labels.js';
+import { recordRunnerReservationCapacityFailure } from '#metrics/index.js';
 import { db } from './db.js';
 import { listRunningJobExecutionsByRunnerInstanceTx } from './job-executions.js';
+import { lockRunnerReservationAdvisoryKeysTx } from './reservation-locks.js';
 import { releaseReservationUnits } from './reservations.js';
+import { validateRunnerReservationCapacityTx } from './runner-assignments.js';
+import { terminalStates } from './runner-states.js';
 import { ephemeralRegistrationTokens } from './schema/ephemeral-registration-tokens.js';
 import { provisionerTokens } from './schema/provisioner-tokens.js';
 import { reservations } from './schema/reservations.js';
@@ -12,11 +17,6 @@ import { runnerControlSessions } from './schema/runner-control-sessions.js';
 import { providerRunners, toRunnerInstance } from './schema/runner-instances.js';
 import { runnerSessions } from './schema/runner-sessions.js';
 import { runningJobExecutions } from './schema/running-job-executions.js';
-export const terminalStates = [
-    'stopped',
-    'failed',
-    'terminated'
-];
 export const activeStates = [
     'starting',
     'running',
@@ -51,8 +51,9 @@ export async function reportRunnerInstances(params) {
             await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId ?? params.provisionerId}))`);
         }
         const events = await hydrateRunnerSessionIdsFromConsumedTokens(tx, params, aggregatedEvents);
-        const terminateIntentsHonored = await listTerminateIntentsHonoredByTerminatedReportsTx(tx, params, events);
-        const values = events.map((event)=>({
+        const reservationSafeEvents = await guardReportedReservationIdsTx(tx, params, events);
+        const terminateIntentsHonored = await listTerminateIntentsHonoredByTerminatedReportsTx(tx, params, reservationSafeEvents);
+        const values = reservationSafeEvents.map((event)=>({
                 workspaceId: params.workspaceId,
                 provisionerId: params.provisionerId,
                 providerRunnerId: event.providerRunnerId,
@@ -101,14 +102,92 @@ export async function reportRunnerInstances(params) {
           OR ${providerRunnerMilestoneUpdateCondition()}
         `
         });
-        const reservationsReleased = hasTerminalEvent ? await releaseTerminalRunnerInstanceReservations(tx, params, events) : 0;
+        const reservationsReleased = hasTerminalEvent ? await releaseTerminalRunnerInstanceReservations(tx, params, reservationSafeEvents) : 0;
         return {
-            accepted: events.length,
+            accepted: reservationSafeEvents.length,
             reservationsReleased,
             terminateIntentsHonored
         };
     });
 }
+async function guardReportedReservationIdsTx(tx, params, events) {
+    const reservationIds = [
+        ...new Set(events.flatMap((event)=>event.reservationId ? [
+                event.reservationId
+            ] : []))
+    ].sort();
+    if (reservationIds.length === 0) return events;
+    await lockRunnerReservationAdvisoryKeysTx(tx, {
+        provisionerId: params.provisionerId,
+        reservationIds
+    });
+    const providerRunnerIds = events.map((event)=>event.providerRunnerId);
+    const existingRows = await tx.select({
+        providerRunnerId: providerRunners.providerRunnerId,
+        reservationId: providerRunners.reservationId,
+        intendedReservationId: providerRunners.intendedReservationId,
+        reservationReleasedAt: providerRunners.reservationReleasedAt
+    }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.providerRunnerId, providerRunnerIds))).for('update');
+    const existingByProviderRunnerId = new Map(existingRows.flatMap((row)=>row.providerRunnerId ? [
+            [
+                row.providerRunnerId,
+                row
+            ]
+        ] : []));
+    const candidateReservationByProviderRunnerId = new Map();
+    for (const event of events){
+        if (!event.reservationId) continue;
+        if (isTerminalState(event.state)) continue;
+        const existing = existingByProviderRunnerId.get(event.providerRunnerId);
+        if (existing?.reservationId || existing?.intendedReservationId === event.reservationId) continue;
+        if (existing?.intendedReservationId || existing?.reservationReleasedAt) continue;
+        candidateReservationByProviderRunnerId.set(event.providerRunnerId, event.reservationId);
+    }
+    const validation = await validateRunnerReservationCapacityTx(tx, {
+        provisionerId: params.provisionerId,
+        requests: [
+            ...candidateReservationByProviderRunnerId.values()
+        ].map((reservationId)=>({
+                reservationId,
+                count: 1
+            }))
+    }, {
+        advisoryLocksHeld: true
+    });
+    for (const { reason, count } of validation.unavailableByReservation.values())recordRunnerReservationCapacityFailure(reason, count);
+    const remainingAcceptedByReservation = new Map(validation.acceptedByReservation);
+    return events.map((event)=>{
+        const existing = existingByProviderRunnerId.get(event.providerRunnerId);
+        const reservationId = candidateReservationByProviderRunnerId.get(event.providerRunnerId);
+        if (!reservationId) {
+            if (event.reservationId && isTerminalState(event.state)) {
+                // Terminal cleanup releases the IDs stored on the projection row. An arbitrary
+                // terminal report must not consume reservation capacity during that cleanup.
+                const matchesExistingReservation = existing !== undefined && (existing.reservationId === event.reservationId || existing.intendedReservationId === event.reservationId);
+                if (!matchesExistingReservation) return {
+                    ...event,
+                    reservationId: null
+                };
+            }
+            if (event.reservationId && existing?.intendedReservationId && existing.intendedReservationId !== event.reservationId) return {
+                ...event,
+                reservationId: null
+            };
+            if (event.reservationId && existing?.reservationReleasedAt) return {
+                ...event,
+                reservationId: null
+            };
+            return event;
+        }
+        const remaining = remainingAcceptedByReservation.get(reservationId) ?? 0;
+        if (remaining === 0) return {
+            ...event,
+            reservationId: null
+        };
+        remainingAcceptedByReservation.set(reservationId, remaining - 1);
+        return event;
+    });
+}
 export async function listActiveRunnerInstanceCountsByTemplateTx(tx, params) {
     const rows = await tx.select({
         templateKey: providerRunners.templateKey,
@@ -131,6 +210,49 @@ export async function countStaleEnrolledRunnerInstances(params) {
     }).from(runnerControlSessions).where(and(eq(runnerControlSessions.runnerInstanceId, providerRunners.id), isNull(runnerControlSessions.closedAt), gt(runnerControlSessions.expiresAt, sql`now()`))))));
     return row?.count ?? 0;
 }
+export async function listProviderRunnerByPhaseMetrics() {
+    const phase = sql`case
+    when ${runnerControlSessions.id} is null then 'control_session'
+    when ${providerRunners.state} <> 'running' then 'enrollment'
+    when ${providerRunners.intendedReservationId} is not null
+      and ${providerRunners.workspaceId} is null then 'assignment'
+    when ${providerRunners.workspaceId} is not null then 'activation'
+    else 'idle'
+  end`;
+    const startedAt = sql`case
+    when ${runnerControlSessions.id} is null then ${providerRunners.createdAt}
+    when ${providerRunners.state} <> 'running' then ${runnerControlSessions.createdAt}
+    when ${providerRunners.intendedReservationId} is not null
+      and ${providerRunners.workspaceId} is null then ${runnerControlSessions.createdAt}
+    when ${providerRunners.workspaceId} is not null
+      then coalesce(${providerRunners.assignedAt}, ${runnerControlSessions.createdAt})
+    else coalesce(
+      ${runnerControlSessions.createdAt},
+      ${providerRunners.assignedAt},
+      ${providerRunners.createdAt}
+    )
+  end`;
+    const rows = await db().select({
+        phase,
+        provider: sql`coalesce(${providerRunners.providerKind}, 'unknown')`,
+        launchKind: providerRunners.launchKind,
+        count: sql`count(*)::int`,
+        oldestAgeMilliseconds: sql`coalesce(
+        max(extract(epoch from (now() - (${startedAt}))) * 1000),
+        0
+      )::double precision`
+    }).from(providerRunners).leftJoin(runnerControlSessions, and(eq(runnerControlSessions.runnerInstanceId, providerRunners.id), isNull(runnerControlSessions.closedAt), gt(runnerControlSessions.expiresAt, sql`now()`))).where(and(inArray(providerRunners.state, [
+        'starting',
+        'running'
+    ]), isNull(providerRunners.runnerSessionId))).groupBy(phase, providerRunners.providerKind, providerRunners.launchKind);
+    return rows.map((row)=>({
+            phase: row.phase,
+            provider: row.provider,
+            launchKind: row.launchKind,
+            count: row.count,
+            oldestAgeMilliseconds: Math.max(0, row.oldestAgeMilliseconds)
+        }));
+}
 export async function listActiveRunnerInstances(params) {
     const rows = await db().select().from(providerRunners).where(and(eq(providerRunners.workspaceId, params.workspaceId), inArray(providerRunners.state, activeStates), sql`${providerRunners.updatedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`)).orderBy(desc(providerRunners.updatedAt), desc(providerRunners.id)).limit(params.limit ?? 1000);
     return rows.map(toRunnerInstance);
@@ -153,21 +275,47 @@ export async function listProvisionerTerminateIntentRowsTx(tx, params) {
             returnedCount: returnedRows.length
         }, 'provisioner terminate intents truncated by poll-demand limit');
     }
-    return returnedRows.flatMap((row)=>row.providerRunnerId ? [
-            {
-                ...row,
-                providerRunnerId: row.providerRunnerId
-            }
+    const activationTimeoutRunnerIds = returnedRows.flatMap((row)=>row.reason === 'activation-timeout' && row.providerRunnerId ? [
+            row.providerRunnerId
         ] : []);
+    if (activationTimeoutRunnerIds.length > 0) {
+        await tx.update(providerRunners).set({
+            reservationReleasedAt: sql`now()`,
+            updatedAt: sql`now()`
+        }).where(and(eq(providerRunners.workspaceId, params.workspaceId), eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.providerRunnerId, activationTimeoutRunnerIds), isNull(providerRunners.reservationReleasedAt)));
+    }
+    return returnedRows.flatMap((row)=>toRunnerInstanceTerminateIntent(row));
+}
+function toRunnerInstanceTerminateIntent(row) {
+    if (!row.providerRunnerId) return [];
+    return [
+        {
+            providerRunnerId: row.providerRunnerId,
+            reason: row.reason,
+            ...row.reason === 'activation-timeout' && row.activationTimeoutRetry ? {
+                activationTimeoutRetry: true
+            } : {}
+        }
+    ];
 }
 function provisionerTerminateIntentsQuery(tx, params) {
     const newerRunningJobExecutions = alias(runningJobExecutions, 'newer_running_jobs');
+    const latestCancelledJob = exists(tx.select({
+        id: runningJobExecutions.id
+    }).from(runningJobExecutions).where(and(eq(runningJobExecutions.workspaceId, params.workspaceId), eq(runningJobExecutions.provisionerId, params.provisionerId), eq(runningJobExecutions.providerRunnerId, providerRunners.providerRunnerId), isNotNull(runningJobExecutions.cancellationRequestedAt), notExists(tx.select({
+        id: newerRunningJobExecutions.id
+    }).from(newerRunningJobExecutions).where(and(eq(newerRunningJobExecutions.workspaceId, runningJobExecutions.workspaceId), eq(newerRunningJobExecutions.provisionerId, runningJobExecutions.provisionerId), eq(newerRunningJobExecutions.providerRunnerId, runningJobExecutions.providerRunnerId), or(gt(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt), and(eq(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt), gt(newerRunningJobExecutions.jobExecutionId, runningJobExecutions.jobExecutionId)))))))));
+    const activationTimeout = and(eq(providerRunners.launchKind, 'demand'), isNull(providerRunners.runnerSessionId), lt(providerRunners.createdAt, sql`now() - (${config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS} || ' seconds')::interval`), notExists(tx.select({
+        id: reservations.id
+    }).from(reservations).where(and(or(eq(reservations.id, providerRunners.reservationId), eq(reservations.id, providerRunners.intendedReservationId)), eq(reservations.workspaceId, params.workspaceId), eq(reservations.provisionerId, params.provisionerId), gt(reservations.expiresAt, sql`now()`)))));
     return tx.select({
         providerRunnerId: providerRunners.providerRunnerId,
-        reason: sql`'job-cancelled'`
-    }).from(runningJobExecutions).innerJoin(providerRunners, and(eq(providerRunners.workspaceId, runningJobExecutions.workspaceId), eq(providerRunners.provisionerId, runningJobExecutions.provisionerId), eq(providerRunners.providerRunnerId, runningJobExecutions.providerRunnerId))).where(and(eq(runningJobExecutions.workspaceId, params.workspaceId), eq(runningJobExecutions.provisionerId, params.provisionerId), isNotNull(providerRunners.providerRunnerId), isNotNull(runningJobExecutions.cancellationRequestedAt), inArray(providerRunners.state, activeStates), params.providerRunnerIds && params.providerRunnerIds.length > 0 ? inArray(providerRunners.providerRunnerId, params.providerRunnerIds) : undefined, notExists(tx.select({
-        id: newerRunningJobExecutions.id
-    }).from(newerRunningJobExecutions).where(and(eq(newerRunningJobExecutions.workspaceId, runningJobExecutions.workspaceId), eq(newerRunningJobExecutions.provisionerId, runningJobExecutions.provisionerId), eq(newerRunningJobExecutions.providerRunnerId, runningJobExecutions.providerRunnerId), or(gt(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt), and(eq(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt), gt(newerRunningJobExecutions.jobExecutionId, runningJobExecutions.jobExecutionId)))))))).groupBy(providerRunners.providerRunnerId);
+        activationTimeoutRetry: sql`${providerRunners.reservationReleasedAt} is not null`,
+        reason: sql`case
+        when ${activationTimeout} then 'activation-timeout'
+        else 'job-cancelled'
+      end`
+    }).from(providerRunners).where(and(eq(providerRunners.workspaceId, params.workspaceId), eq(providerRunners.provisionerId, params.provisionerId), isNotNull(providerRunners.providerRunnerId), inArray(providerRunners.state, activeStates), params.providerRunnerIds && params.providerRunnerIds.length > 0 ? inArray(providerRunners.providerRunnerId, params.providerRunnerIds) : undefined, or(latestCancelledJob, activationTimeout)));
 }
 async function listTerminateIntentsHonoredByTerminatedReportsTx(tx, params, events) {
     if (!params.workspaceId) return [];
@@ -180,12 +328,7 @@ async function listTerminateIntentsHonoredByTerminatedReportsTx(tx, params, even
         provisionerId: params.provisionerId,
         providerRunnerIds: terminatedRunnerInstanceIds
     }).orderBy(asc(providerRunners.providerRunnerId));
-    return rows.flatMap((row)=>row.providerRunnerId ? [
-            {
-                ...row,
-                providerRunnerId: row.providerRunnerId
-            }
-        ] : []);
+    return rows.flatMap((row)=>toRunnerInstanceTerminateIntent(row));
 }
 export async function reconcileRunnerInstances(params) {
     const observedRunnerInstanceIds = [