@shipfox/api-runners 12.2.0 → 12.4.0

package/src/db/runner-instances.ts

@@ -17,27 +17,28 @@ import {
   sql,
 } from 'drizzle-orm';
 import {alias} from 'drizzle-orm/pg-core';
+import {config} from '#config.js';
 import type {RunnerInstance, RunnerInstanceState} from '#core/entities/runner-instance.js';
 import {sanitizeRunnerLabels} from '#core/runner-labels.js';
+import {recordRunnerReservationCapacityFailure} from '#metrics/index.js';
 import type {Tx} from './db.js';
 import {db} from './db.js';
 import {
   listRunningJobExecutionsByRunnerInstanceTx,
   type RunnerInstanceBoundJobExecution,
 } from './job-executions.js';
+import {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';
 import {releaseReservationUnits} from './reservations.js';
+import {validateRunnerReservationCapacityTx} from './runner-assignments.js';
+import {terminalStates} from './runner-states.js';
 import {ephemeralRegistrationTokens} from './schema/ephemeral-registration-tokens.js';
 import {provisionerTokens} from './schema/provisioner-tokens.js';
 import {reservations} from './schema/reservations.js';
+import {runnerControlSessions} from './schema/runner-control-sessions.js';
 import {providerRunners, toRunnerInstance} from './schema/runner-instances.js';
 import {runnerSessions} from './schema/runner-sessions.js';
 import {runningJobExecutions} from './schema/running-job-executions.js';
 
-export const terminalStates = [
-  'stopped',
-  'failed',
-  'terminated',
-] as const satisfies readonly RunnerInstanceState[];
 export const activeStates = [
   'starting',
   'running',
@@ -48,11 +49,12 @@ export const divergenceCountStates = ['starting', 'running'] as const satisfies
   'starting' | 'running'
 >[];
 
-export type RunnerInstanceTerminateIntentReason = 'job-cancelled';
+export type RunnerInstanceTerminateIntentReason = 'activation-timeout' | 'job-cancelled';
 
 export interface RunnerInstanceTerminateIntent {
   providerRunnerId: string;
   reason: RunnerInstanceTerminateIntentReason;
+  activationTimeoutRetry?: boolean;
 }
 
 export interface ActiveRunnerInstanceTemplateCount {
@@ -154,13 +156,14 @@ export async function reportRunnerInstances(params: ReportRunnerInstancesParams)
     }
 
     const events = await hydrateRunnerSessionIdsFromConsumedTokens(tx, params, aggregatedEvents);
+    const reservationSafeEvents = await guardReportedReservationIdsTx(tx, params, events);
     const terminateIntentsHonored = await listTerminateIntentsHonoredByTerminatedReportsTx(
       tx,
       params,
-      events,
+      reservationSafeEvents,
     );
 
-    const values = events.map((event) => ({
+    const values = reservationSafeEvents.map((event) => ({
       workspaceId: params.workspaceId,
       provisionerId: params.provisionerId,
       providerRunnerId: event.providerRunnerId,
@@ -190,7 +193,7 @@ export async function reportRunnerInstances(params: ReportRunnerInstancesParams)
         target: [providerRunners.provisionerId, providerRunners.providerRunnerId],
         targetWhere: isNotNull(providerRunners.providerRunnerId),
         set: {
-          reservationId: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(excluded.reservation_id, ${providerRunners.reservationId}) ELSE ${providerRunners.reservationId} END`,
+          reservationId: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(${providerRunners.reservationId}, excluded.reservation_id) ELSE ${providerRunners.reservationId} END`,
           templateKey: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(excluded.template_key, ${providerRunners.templateKey}) ELSE ${providerRunners.templateKey} END`,
           labels: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.labels ELSE ${providerRunners.labels} END`,
           state: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.state ELSE ${providerRunners.state} END`,
@@ -212,10 +215,105 @@ export async function reportRunnerInstances(params: ReportRunnerInstancesParams)
       });
 
     const reservationsReleased = hasTerminalEvent
-      ? await releaseTerminalRunnerInstanceReservations(tx, params, events)
+      ? await releaseTerminalRunnerInstanceReservations(tx, params, reservationSafeEvents)
       : 0;
 
-    return {accepted: events.length, reservationsReleased, terminateIntentsHonored};
+    return {
+      accepted: reservationSafeEvents.length,
+      reservationsReleased,
+      terminateIntentsHonored,
+    };
+  });
+}
+
+async function guardReportedReservationIdsTx(
+  tx: Tx,
+  params: ReportRunnerInstancesParams,
+  events: RunnerInstanceReportRow[],
+): Promise<RunnerInstanceReportRow[]> {
+  const reservationIds = [
+    ...new Set(events.flatMap((event) => (event.reservationId ? [event.reservationId] : []))),
+  ].sort();
+  if (reservationIds.length === 0) return events;
+
+  await lockRunnerReservationAdvisoryKeysTx(tx, {
+    provisionerId: params.provisionerId,
+    reservationIds,
+  });
+  const providerRunnerIds = events.map((event) => event.providerRunnerId);
+  const existingRows = await tx
+    .select({
+      providerRunnerId: providerRunners.providerRunnerId,
+      reservationId: providerRunners.reservationId,
+      intendedReservationId: providerRunners.intendedReservationId,
+      reservationReleasedAt: providerRunners.reservationReleasedAt,
+    })
+    .from(providerRunners)
+    .where(
+      and(
+        eq(providerRunners.provisionerId, params.provisionerId),
+        inArray(providerRunners.providerRunnerId, providerRunnerIds),
+      ),
+    )
+    .for('update');
+  const existingByProviderRunnerId = new Map(
+    existingRows.flatMap((row) =>
+      row.providerRunnerId ? [[row.providerRunnerId, row] as const] : [],
+    ),
+  );
+  const candidateReservationByProviderRunnerId = new Map<string, string>();
+  for (const event of events) {
+    if (!event.reservationId) continue;
+    if (isTerminalState(event.state)) continue;
+    const existing = existingByProviderRunnerId.get(event.providerRunnerId);
+    if (existing?.reservationId || existing?.intendedReservationId === event.reservationId)
+      continue;
+    if (existing?.intendedReservationId || existing?.reservationReleasedAt) continue;
+    candidateReservationByProviderRunnerId.set(event.providerRunnerId, event.reservationId);
+  }
+
+  const validation = await validateRunnerReservationCapacityTx(
+    tx,
+    {
+      provisionerId: params.provisionerId,
+      requests: [...candidateReservationByProviderRunnerId.values()].map((reservationId) => ({
+        reservationId,
+        count: 1,
+      })),
+    },
+    {advisoryLocksHeld: true},
+  );
+  for (const {reason, count} of validation.unavailableByReservation.values())
+    recordRunnerReservationCapacityFailure(reason, count);
+
+  const remainingAcceptedByReservation = new Map(validation.acceptedByReservation);
+  return events.map((event) => {
+    const existing = existingByProviderRunnerId.get(event.providerRunnerId);
+    const reservationId = candidateReservationByProviderRunnerId.get(event.providerRunnerId);
+    if (!reservationId) {
+      if (event.reservationId && isTerminalState(event.state)) {
+        // Terminal cleanup releases the IDs stored on the projection row. An arbitrary
+        // terminal report must not consume reservation capacity during that cleanup.
+        const matchesExistingReservation =
+          existing !== undefined &&
+          (existing.reservationId === event.reservationId ||
+            existing.intendedReservationId === event.reservationId);
+        if (!matchesExistingReservation) return {...event, reservationId: null};
+      }
+      if (
+        event.reservationId &&
+        existing?.intendedReservationId &&
+        existing.intendedReservationId !== event.reservationId
+      )
+        return {...event, reservationId: null};
+      if (event.reservationId && existing?.reservationReleasedAt)
+        return {...event, reservationId: null};
+      return event;
+    }
+    const remaining = remainingAcceptedByReservation.get(reservationId) ?? 0;
+    if (remaining === 0) return {...event, reservationId: null};
+    remainingAcceptedByReservation.set(reservationId, remaining - 1);
+    return event;
   });
 }
 
@@ -247,6 +345,36 @@ export async function listActiveRunnerInstanceCountsByTemplateTx(
   );
 }
 
+export async function countStaleEnrolledRunnerInstances(params: {
+  graceSeconds: number;
+}): Promise<number> {
+  const [row] = await db()
+    .select({count: sql<number>`count(*)::int`})
+    .from(providerRunners)
+    .where(
+      and(
+        eq(providerRunners.state, 'running'),
+        isNull(providerRunners.workspaceId),
+        isNull(providerRunners.runnerSessionId),
+        lt(providerRunners.reportedAt, staleRunnerInstanceCutoff(params.graceSeconds)),
+        exists(
+          db()
+            .select({id: runnerControlSessions.id})
+            .from(runnerControlSessions)
+            .where(
+              and(
+                eq(runnerControlSessions.runnerInstanceId, providerRunners.id),
+                isNull(runnerControlSessions.closedAt),
+                gt(runnerControlSessions.expiresAt, sql`now()`),
+              ),
+            ),
+        ),
+      ),
+    );
+
+  return row?.count ?? 0;
+}
+
 export async function listActiveRunnerInstances(params: {
   workspaceId: string;
   windowSeconds: number;
@@ -305,9 +433,42 @@ export async function listProvisionerTerminateIntentRowsTx(
     );
   }
 
-  return returnedRows.flatMap((row) =>
-    row.providerRunnerId ? [{...row, providerRunnerId: row.providerRunnerId}] : [],
+  const activationTimeoutRunnerIds = returnedRows.flatMap((row) =>
+    row.reason === 'activation-timeout' && row.providerRunnerId ? [row.providerRunnerId] : [],
   );
+  if (activationTimeoutRunnerIds.length > 0) {
+    await tx
+      .update(providerRunners)
+      .set({reservationReleasedAt: sql`now()`, updatedAt: sql`now()`})
+      .where(
+        and(
+          eq(providerRunners.workspaceId, params.workspaceId),
+          eq(providerRunners.provisionerId, params.provisionerId),
+          inArray(providerRunners.providerRunnerId, activationTimeoutRunnerIds),
+          isNull(providerRunners.reservationReleasedAt),
+        ),
+      );
+  }
+
+  return returnedRows.flatMap((row) => toRunnerInstanceTerminateIntent(row));
+}
+
+function toRunnerInstanceTerminateIntent(row: {
+  providerRunnerId: string | null;
+  reason: RunnerInstanceTerminateIntentReason;
+  activationTimeoutRetry: boolean;
+}): RunnerInstanceTerminateIntent[] {
+  if (!row.providerRunnerId) return [];
+
+  return [
+    {
+      providerRunnerId: row.providerRunnerId,
+      reason: row.reason,
+      ...(row.reason === 'activation-timeout' && row.activationTimeoutRetry
+        ? {activationTimeoutRetry: true}
+        : {}),
+    },
+  ];
 }
 
 function provisionerTerminateIntentsQuery(
@@ -315,59 +476,91 @@ function provisionerTerminateIntentsQuery(
   params: {workspaceId: string; provisionerId: string; providerRunnerIds?: string[]},
 ) {
   const newerRunningJobExecutions = alias(runningJobExecutions, 'newer_running_jobs');
+  const latestCancelledJob = exists(
+    tx
+      .select({id: runningJobExecutions.id})
+      .from(runningJobExecutions)
+      .where(
+        and(
+          eq(runningJobExecutions.workspaceId, params.workspaceId),
+          eq(runningJobExecutions.provisionerId, params.provisionerId),
+          eq(runningJobExecutions.providerRunnerId, providerRunners.providerRunnerId),
+          isNotNull(runningJobExecutions.cancellationRequestedAt),
+          notExists(
+            tx
+              .select({id: newerRunningJobExecutions.id})
+              .from(newerRunningJobExecutions)
+              .where(
+                and(
+                  eq(newerRunningJobExecutions.workspaceId, runningJobExecutions.workspaceId),
+                  eq(newerRunningJobExecutions.provisionerId, runningJobExecutions.provisionerId),
+                  eq(
+                    newerRunningJobExecutions.providerRunnerId,
+                    runningJobExecutions.providerRunnerId,
+                  ),
+                  or(
+                    gt(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt),
+                    and(
+                      eq(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt),
+                      gt(
+                        newerRunningJobExecutions.jobExecutionId,
+                        runningJobExecutions.jobExecutionId,
+                      ),
+                    ),
+                  ),
+                ),
+              ),
+          ),
+        ),
+      ),
+  );
+  const activationTimeout = and(
+    eq(providerRunners.launchKind, 'demand'),
+    isNull(providerRunners.runnerSessionId),
+    lt(
+      providerRunners.createdAt,
+      sql`now() - (${config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS} || ' seconds')::interval`,
+    ),
+    notExists(
+      tx
+        .select({id: reservations.id})
+        .from(reservations)
+        .where(
+          and(
+            or(
+              eq(reservations.id, providerRunners.reservationId),
+              eq(reservations.id, providerRunners.intendedReservationId),
+            ),
+            eq(reservations.workspaceId, params.workspaceId),
+            eq(reservations.provisionerId, params.provisionerId),
+            gt(reservations.expiresAt, sql`now()`),
+          ),
+        ),
+    ),
+  );
 
   return tx
     .select({
       providerRunnerId: providerRunners.providerRunnerId,
-      reason: sql<RunnerInstanceTerminateIntentReason>`'job-cancelled'`,
+      activationTimeoutRetry: sql<boolean>`${providerRunners.reservationReleasedAt} is not null`,
+      reason: sql<RunnerInstanceTerminateIntentReason>`case
+        when ${activationTimeout} then 'activation-timeout'
+        else 'job-cancelled'
+      end`,
     })
-    .from(runningJobExecutions)
-    .innerJoin(
-      providerRunners,
-      and(
-        eq(providerRunners.workspaceId, runningJobExecutions.workspaceId),
-        eq(providerRunners.provisionerId, runningJobExecutions.provisionerId),
-        eq(providerRunners.providerRunnerId, runningJobExecutions.providerRunnerId),
-      ),
-    )
+    .from(providerRunners)
     .where(
       and(
-        eq(runningJobExecutions.workspaceId, params.workspaceId),
-        eq(runningJobExecutions.provisionerId, params.provisionerId),
+        eq(providerRunners.workspaceId, params.workspaceId),
+        eq(providerRunners.provisionerId, params.provisionerId),
         isNotNull(providerRunners.providerRunnerId),
-        isNotNull(runningJobExecutions.cancellationRequestedAt),
         inArray(providerRunners.state, activeStates),
         params.providerRunnerIds && params.providerRunnerIds.length > 0
           ? inArray(providerRunners.providerRunnerId, params.providerRunnerIds)
           : undefined,
-        notExists(
-          tx
-            .select({id: newerRunningJobExecutions.id})
-            .from(newerRunningJobExecutions)
-            .where(
-              and(
-                eq(newerRunningJobExecutions.workspaceId, runningJobExecutions.workspaceId),
-                eq(newerRunningJobExecutions.provisionerId, runningJobExecutions.provisionerId),
-                eq(
-                  newerRunningJobExecutions.providerRunnerId,
-                  runningJobExecutions.providerRunnerId,
-                ),
-                or(
-                  gt(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt),
-                  and(
-                    eq(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt),
-                    gt(
-                      newerRunningJobExecutions.jobExecutionId,
-                      runningJobExecutions.jobExecutionId,
-                    ),
-                  ),
-                ),
-              ),
-            ),
-        ),
+        or(latestCancelledJob, activationTimeout),
       ),
-    )
-    .groupBy(providerRunners.providerRunnerId);
+    );
 }
 
 async function listTerminateIntentsHonoredByTerminatedReportsTx(
@@ -388,9 +581,7 @@ async function listTerminateIntentsHonoredByTerminatedReportsTx(
     provisionerId: params.provisionerId,
     providerRunnerIds: terminatedRunnerInstanceIds,
   }).orderBy(asc(providerRunners.providerRunnerId));
-  return rows.flatMap((row) =>
-    row.providerRunnerId ? [{...row, providerRunnerId: row.providerRunnerId}] : [],
-  );
+  return rows.flatMap((row) => toRunnerInstanceTerminateIntent(row));
 }
 
 export async function reconcileRunnerInstances(

package/src/db/runner-sessions.ts

@@ -59,33 +59,44 @@ export async function createRunnerSessionConsumingActivationToken(params: {
   toolCapabilities?: RunnerToolCapabilitiesDto | null;
 }) {
   return await db().transaction(async (tx) => {
-    const [token] = await tx
+    const [runner] = await tx
       .select({
-        id: runnerActivationTokens.id,
-        runnerInstanceId: runnerActivationTokens.runnerInstanceId,
+        activationTokenId: runnerActivationTokens.id,
+        runnerInstanceId: providerRunners.id,
         workspaceId: providerRunners.workspaceId,
         provisionerId: providerRunners.provisionerId,
         providerRunnerId: providerRunners.providerRunnerId,
+        runnerSessionId: providerRunners.runnerSessionId,
       })
       .from(runnerActivationTokens)
       .innerJoin(providerRunners, eq(providerRunners.id, runnerActivationTokens.runnerInstanceId))
+      .where(eq(runnerActivationTokens.id, params.activationTokenId))
+      .limit(1)
+      .for('update', {of: providerRunners});
+    if (!runner?.workspaceId || !runner.providerRunnerId || runner.runnerSessionId)
+      throw new Error('Runner activation token is invalid, expired, or has already been used');
+
+    const [activationToken] = await tx
+      .select({id: runnerActivationTokens.id})
+      .from(runnerActivationTokens)
       .where(
         and(
-          eq(runnerActivationTokens.id, params.activationTokenId),
+          eq(runnerActivationTokens.id, runner.activationTokenId),
+          eq(runnerActivationTokens.runnerInstanceId, runner.runnerInstanceId),
           isNull(runnerActivationTokens.consumedAt),
           isNull(runnerActivationTokens.revokedAt),
           gt(runnerActivationTokens.expiresAt, sql`now()`),
-          isNull(providerRunners.runnerSessionId),
         ),
       )
       .limit(1)
       .for('update');
-    if (!token?.workspaceId || !token.providerRunnerId)
+    if (!activationToken)
       throw new Error('Runner activation token is invalid, expired, or has already been used');
+
     const [provisioner] = await tx
       .select({scope: provisionerTokens.scope})
       .from(provisionerTokens)
-      .where(eq(provisionerTokens.id, token.provisionerId))
+      .where(eq(provisionerTokens.id, runner.provisionerId))
       .limit(1);
     const labels = sanitizeRunnerLabelsOrThrow(params.labels, {
       scope: provisioner?.scope ?? 'workspace',
@@ -95,13 +106,13 @@ export async function createRunnerSessionConsumingActivationToken(params: {
     const [session] = await tx
       .insert(runnerSessions)
       .values({
-        workspaceId: token.workspaceId,
+        workspaceId: runner.workspaceId,
         scope: 'workspace',
-        registrationTokenId: token.id,
+        registrationTokenId: activationToken.id,
         registrationTokenKind: 'activation',
-        runnerInstanceId: token.runnerInstanceId,
-        provisionerId: token.provisionerId,
-        providerRunnerId: token.providerRunnerId,
+        runnerInstanceId: runner.runnerInstanceId,
+        provisionerId: runner.provisionerId,
+        providerRunnerId: runner.providerRunnerId,
         labels,
         toolCapabilities: params.toolCapabilities ?? null,
         toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,
@@ -113,13 +124,13 @@ export async function createRunnerSessionConsumingActivationToken(params: {
     await tx
       .update(runnerActivationTokens)
       .set({consumedAt: sql`now()`, consumedSessionId: session.id})
-      .where(eq(runnerActivationTokens.id, token.id));
+      .where(eq(runnerActivationTokens.id, activationToken.id));
     await tx
       .update(providerRunners)
       .set({runnerSessionId: session.id, updatedAt: sql`now()`})
       .where(
         and(
-          eq(providerRunners.id, token.runnerInstanceId),
+          eq(providerRunners.id, runner.runnerInstanceId),
           isNull(providerRunners.runnerSessionId),
         ),
       );
@@ -128,7 +139,7 @@ export async function createRunnerSessionConsumingActivationToken(params: {
       .set({closedAt: sql`now()`, closeReason: 'activated'})
       .where(
         and(
-          eq(runnerControlSessions.runnerInstanceId, token.runnerInstanceId),
+          eq(runnerControlSessions.runnerInstanceId, runner.runnerInstanceId),
           isNull(runnerControlSessions.closedAt),
         ),
       );

package/src/db/runner-states.ts

@@ -0,0 +1,7 @@
+import type {RunnerInstanceState} from '#core/entities/runner-instance.js';
+
+export const terminalStates = [
+  'stopped',
+  'failed',
+  'terminated',
+] as const satisfies readonly RunnerInstanceState[];

package/src/db/schema/reservations.ts

@@ -1,9 +1,11 @@
 import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';
 import {sql} from 'drizzle-orm';
-import {check, index, integer, text, timestamp, uuid} from 'drizzle-orm/pg-core';
+import {check, index, integer, pgEnum, text, timestamp, uuid} from 'drizzle-orm/pg-core';
 import type {Reservation} from '#core/entities/reservation.js';
 import {pgTable} from './common.js';
 
+export const reservationKindEnum = pgEnum('runners_reservation_kind', ['bound', 'launch']);
+
 export const reservations = pgTable(
   'reservations',
   {
@@ -12,6 +14,7 @@ export const reservations = pgTable(
     provisionerId: uuid('provisioner_id').notNull(),
     requiredLabels: text('required_labels').array().notNull(),
     count: integer('count').notNull(),
+    kind: reservationKindEnum('kind').notNull().default('launch'),
     createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
     expiresAt: timestamp('expires_at', {withTimezone: true}).notNull(),
   },
@@ -32,6 +35,7 @@ export function toReservation(row: ReservationDb): Reservation {
     provisionerId: row.provisionerId,
     requiredLabels: row.requiredLabels,
     count: row.count,
+    kind: row.kind,
     createdAt: row.createdAt,
     expiresAt: row.expiresAt,
   };

package/src/db/schema/runner-instances.ts

@@ -14,6 +14,12 @@ export const providerRunnerStateEnum = pgEnum('runners_provider_runner_state', [
   'terminated',
 ]);
 
+export const providerRunnerLaunchKindEnum = pgEnum('runners_provider_runner_launch_kind', [
+  'demand',
+  'warm',
+  'manual',
+]);
+
 export const providerRunners = pgTable(
   'runner_instances',
   {
@@ -25,6 +31,7 @@ export const providerRunners = pgTable(
     providerRunnerId: text('provider_runner_id'),
     intendedReservationId: uuid('intended_reservation_id'),
     reservationId: uuid('reservation_id'),
+    launchKind: providerRunnerLaunchKindEnum('launch_kind').notNull().default('manual'),
     assignedAt: timestamp('assigned_at', {withTimezone: true}),
     templateKey: text('template_key'),
     labels: text('labels').array().notNull().default([]),
@@ -82,7 +89,9 @@ export function toRunnerInstance(row: RunnerInstanceDb): RunnerInstance {
     workspaceId: row.workspaceId,
     provisionerId: row.provisionerId,
     providerRunnerId: row.providerRunnerId,
+    intendedReservationId: row.intendedReservationId,
     reservationId: row.reservationId,
+    launchKind: row.launchKind,
     assignedAt: row.assignedAt,
     templateKey: row.templateKey,
     labels: row.labels,

package/src/metrics/index.ts

@@ -1,13 +1,22 @@
-export type {RunnerReservationPromotionFailureReason} from './instance.js';
+export type {
+  RunnerActivationTokenNotIssuedReason,
+  RunnerActivationTokenNotIssuedSurface,
+  RunnerReservationCapacityFailureReason,
+  RunnerReservationPromotionFailureReason,
+} from './instance.js';
 export {
   jobExecutionClaimedCount,
   jobExecutionEnqueuedCount,
   jobExecutionLeaseExpiredCount,
   providerRunnerAbsentTerminatedCount,
+  providerRunnerActivationOutcomeCount,
   providerRunnerCountDivergenceCount,
   providerRunnerReconcileCallCount,
   providerRunnerTerminateIntentHonoredCount,
   providerRunnerTerminateIntentIssuedCount,
+  recordProviderRunnerActivationOutcome,
+  recordRunnerActivationTokenNotIssued,
+  recordRunnerReservationCapacityFailure,
   recordRunnerReservationPromotionFailure,
   recordRunnersRateLimitCheck,
   recordRunnersRateLimitPruneFailure,