@shipfox/api-runners 12.4.0 → 12.5.0

package/src/db/runner-assignments.ts

@@ -5,6 +5,11 @@ import {
   RunnerInstanceAlreadyAssignedError,
   RunnerInstanceNotAssignableError,
 } from '#core/errors.js';
+import {
+  type ProviderRunnerAssignmentObservation,
+  type RunnerAssignmentSurface,
+  recordProviderRunnerControlSessionToAssignment,
+} from '#metrics/instance.js';
 import type {Tx} from './db.js';
 import {db} from './db.js';
 import {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';
@@ -19,7 +24,17 @@ export async function assignRunnerInstances(params: {
   reservationId: string;
   runnerInstanceIds: string[];
 }): Promise<string[]> {
-  return await db().transaction(async (tx) => assignRunnerInstancesTx(tx, params));
+  const result = await db().transaction(async (tx) =>
+    assignRunnerInstancesTx(tx, {...params, surface: 'provisioner'}),
+  );
+  for (const observation of result.controlSessionToAssignment)
+    recordProviderRunnerControlSessionToAssignment(observation);
+  return result.runnerInstanceIds;
+}
+
+interface AssignRunnerInstancesTxResult {
+  runnerInstanceIds: string[];
+  controlSessionToAssignment: ProviderRunnerAssignmentObservation[];
 }
 
 export async function assignRunnerInstancesTx(
@@ -28,8 +43,9 @@ export async function assignRunnerInstancesTx(
     provisionerId: string;
     reservationId: string;
     runnerInstanceIds: string[];
+    surface: RunnerAssignmentSurface;
   },
-): Promise<string[]> {
+): Promise<AssignRunnerInstancesTxResult> {
   const runnerInstanceIds = [...params.runnerInstanceIds].sort();
   await lockRunnerReservationAdvisoryKeysTx(tx, {
     provisionerId: params.provisionerId,
@@ -43,6 +59,8 @@ export async function assignRunnerInstancesTx(
       workspaceId: providerRunners.workspaceId,
       assignedAt: providerRunners.assignedAt,
       providerRunnerId: providerRunners.providerRunnerId,
+      provider: providerRunners.providerKind,
+      launchKind: providerRunners.launchKind,
       labels: providerRunners.labels,
       state: providerRunners.state,
     })
@@ -76,7 +94,10 @@ export async function assignRunnerInstancesTx(
           inArray(providerRunners.id, runnerInstanceIds),
         ),
       );
-    return params.runnerInstanceIds;
+    return {
+      runnerInstanceIds: params.runnerInstanceIds,
+      controlSessionToAssignment: [],
+    };
   }
 
   const [reservation] = await tx
@@ -94,10 +115,13 @@ export async function assignRunnerInstancesTx(
   if (!reservation) throw new ReservationNotFoundError(params.reservationId);
   if (reservation.expiresAt <= new Date()) throw new ReservationExpiredError(params.reservationId);
   if (runnerRows.length !== runnerInstanceIds.length)
-    throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '');
+    throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '', 'runner-not-found');
 
   const activeControlSessions = await tx
-    .select({runnerInstanceId: runnerControlSessions.runnerInstanceId})
+    .select({
+      runnerInstanceId: runnerControlSessions.runnerInstanceId,
+      createdAt: runnerControlSessions.createdAt,
+    })
     .from(runnerControlSessions)
     .where(
       and(
@@ -111,9 +135,13 @@ export async function assignRunnerInstancesTx(
   const runnerInstanceIdsWithControlSession = new Set(
     activeControlSessions.map((session) => session.runnerInstanceId),
   );
+  const controlSessionCreatedAtByRunner = new Map(
+    activeControlSessions.map((session) => [session.runnerInstanceId, session.createdAt]),
+  );
   const runners = runnerRows.map((runner) => ({
     ...runner,
     controlSessionId: runnerInstanceIdsWithControlSession.has(runner.id) ? runner.id : null,
+    controlSessionCreatedAt: controlSessionCreatedAtByRunner.get(runner.id) ?? null,
   }));
 
   const alreadyAssigned = runners.filter((runner) => runner.reservationId !== null);
@@ -147,18 +175,19 @@ export async function assignRunnerInstancesTx(
       ),
     );
   if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count)
-    throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '');
+    throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '', 'capacity-exhausted');
   for (const runner of newRunners) {
-    if (
-      runner.state !== 'running' ||
-      !runner.providerRunnerId ||
-      !runner.controlSessionId ||
-      !reservation.requiredLabels.every((label) => runner.labels.includes(label))
-    )
-      throw new RunnerInstanceNotAssignableError(runner.id);
+    if (runner.state !== 'running')
+      throw new RunnerInstanceNotAssignableError(runner.id, 'runner-not-running');
+    if (!runner.providerRunnerId)
+      throw new RunnerInstanceNotAssignableError(runner.id, 'provider-identity-missing');
+    if (!runner.controlSessionId)
+      throw new RunnerInstanceNotAssignableError(runner.id, 'control-session-not-active');
+    if (!reservation.requiredLabels.every((label) => runner.labels.includes(label)))
+      throw new RunnerInstanceNotAssignableError(runner.id, 'labels-mismatch');
   }
   if (newRunners.length > 0) {
-    await tx
+    const assignedRows = await tx
       .update(providerRunners)
       .set({
         workspaceId: reservation.workspaceId,
@@ -172,9 +201,31 @@ export async function assignRunnerInstancesTx(
           providerRunners.id,
           newRunners.map((runner) => runner.id),
         ),
-      );
+      )
+      .returning({id: providerRunners.id, assignedAt: providerRunners.assignedAt});
+    const runnersById = new Map(newRunners.map((runner) => [runner.id, runner]));
+    const controlSessionToAssignment: ProviderRunnerAssignmentObservation[] = [];
+    for (const assigned of assignedRows) {
+      const runner = runnersById.get(assigned.id);
+      const controlSessionCreatedAt = runner?.controlSessionCreatedAt;
+      if (!runner || !assigned.assignedAt || !controlSessionCreatedAt) continue;
+      controlSessionToAssignment.push({
+        durationMilliseconds: assigned.assignedAt.getTime() - controlSessionCreatedAt.getTime(),
+        provider: runner.provider,
+        launchKind: runner.launchKind,
+        surface: params.surface,
+        runnerInstanceId: runner.id,
+      });
+    }
+    return {
+      runnerInstanceIds: params.runnerInstanceIds,
+      controlSessionToAssignment,
+    };
   }
-  return params.runnerInstanceIds;
+  return {
+    runnerInstanceIds: params.runnerInstanceIds,
+    controlSessionToAssignment: [],
+  };
 }
 
 export type RunnerReservationCapacityFailureReason =

package/src/db/runner-instances.test.ts

@@ -7,6 +7,7 @@ import {
   countStaleEnrolledRunnerInstances,
   listActiveRunnerInstanceCountsByTemplateTx,
   listActiveRunnerInstances,
+  listProviderRunnerByPhaseMetrics,
   listProvisionerTerminateIntentRowsTx,
   listProvisionerTerminateIntents,
   reapStaleRunnerInstances,
@@ -2540,6 +2541,97 @@ describe('countStaleEnrolledRunnerInstances', () => {
   }
 });
 
+describe('listProviderRunnerByPhaseMetrics', () => {
+  it('groups runners by the lifecycle phase that is currently blocking activation', async () => {
+    const provisionerId = crypto.randomUUID();
+    const provider = `metrics-test-${crypto.randomUUID()}`;
+    const reservationId = crypto.randomUUID();
+    const workspaceId = crypto.randomUUID();
+    const old = new Date(Date.now() - 120_000);
+
+    const createRunner = async (params: {
+      state: 'starting' | 'running';
+      launchKind: 'demand' | 'warm' | 'manual';
+      intendedReservationId?: string | null;
+      workspaceId?: string | null;
+      reservationId?: string | null;
+      assignedAt?: Date | null;
+      createdAt?: Date;
+    }) => {
+      const [runner] = await db()
+        .insert(providerRunners)
+        .values({
+          provisionerId,
+          providerRunnerId: crypto.randomUUID(),
+          providerKind: provider,
+          launchKind: params.launchKind,
+          intendedReservationId: params.intendedReservationId ?? null,
+          workspaceId: params.workspaceId ?? null,
+          reservationId: params.reservationId ?? null,
+          assignedAt: params.assignedAt ?? null,
+          state: params.state,
+          labels: ['linux'],
+          reportedAt: new Date(),
+          createdAt: params.createdAt ?? old,
+        })
+        .returning({id: providerRunners.id});
+      if (!runner) throw new Error('Expected runner instance');
+      return runner.id;
+    };
+
+    await createRunner({state: 'starting', launchKind: 'demand'});
+    const enrollmentRunnerId = await createRunner({state: 'starting', launchKind: 'warm'});
+    const assignmentRunnerId = await createRunner({
+      state: 'running',
+      launchKind: 'demand',
+      intendedReservationId: reservationId,
+    });
+    const activationRunnerId = await createRunner({
+      state: 'running',
+      launchKind: 'manual',
+      workspaceId,
+      reservationId,
+      assignedAt: old,
+    });
+    const idleRunnerId = await createRunner({state: 'running', launchKind: 'warm'});
+    const demandIdleRunnerId = await createRunner({state: 'running', launchKind: 'demand'});
+
+    await db()
+      .insert(runnerControlSessions)
+      .values(
+        [
+          enrollmentRunnerId,
+          assignmentRunnerId,
+          activationRunnerId,
+          idleRunnerId,
+          demandIdleRunnerId,
+        ].map((runnerInstanceId) => ({
+          runnerInstanceId,
+          provisionerId,
+          hashedToken: crypto.randomUUID(),
+          prefix: 'metrics-test',
+          expiresAt: new Date(Date.now() + 60_000),
+          createdAt: old,
+        })),
+      );
+
+    const metrics = await listProviderRunnerByPhaseMetrics();
+    const ownMetrics = metrics.filter((metric) => metric.provider === provider);
+
+    expect(ownMetrics).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({phase: 'control_session', launchKind: 'demand', count: 1}),
+        expect.objectContaining({phase: 'enrollment', launchKind: 'warm', count: 1}),
+        expect.objectContaining({phase: 'assignment', launchKind: 'demand', count: 1}),
+        expect.objectContaining({phase: 'activation', launchKind: 'manual', count: 1}),
+        expect.objectContaining({phase: 'idle', launchKind: 'warm', count: 1}),
+        expect.objectContaining({phase: 'idle', launchKind: 'demand', count: 1}),
+      ]),
+    );
+    expect(ownMetrics).toHaveLength(6);
+  });
+});
+
 describe('runner instance provider attachment', () => {
   it('updates the pre-created runner instance before its first lifecycle report', async () => {
     const provisionerId = crypto.randomUUID();

package/src/db/runner-instances.ts

@@ -375,6 +375,80 @@ export async function countStaleEnrolledRunnerInstances(params: {
   return row?.count ?? 0;
 }
 
+export type ProviderRunnerPhase =
+  | 'control_session'
+  | 'enrollment'
+  | 'assignment'
+  | 'activation'
+  | 'idle';
+
+export interface ProviderRunnerPhaseMetric {
+  phase: ProviderRunnerPhase;
+  provider: string;
+  launchKind: 'demand' | 'warm' | 'manual';
+  count: number;
+  oldestAgeMilliseconds: number;
+}
+
+export async function listProviderRunnerByPhaseMetrics(): Promise<ProviderRunnerPhaseMetric[]> {
+  const phase = sql<ProviderRunnerPhase>`case
+    when ${runnerControlSessions.id} is null then 'control_session'
+    when ${providerRunners.state} <> 'running' then 'enrollment'
+    when ${providerRunners.intendedReservationId} is not null
+      and ${providerRunners.workspaceId} is null then 'assignment'
+    when ${providerRunners.workspaceId} is not null then 'activation'
+    else 'idle'
+  end`;
+  const startedAt = sql<Date | null>`case
+    when ${runnerControlSessions.id} is null then ${providerRunners.createdAt}
+    when ${providerRunners.state} <> 'running' then ${runnerControlSessions.createdAt}
+    when ${providerRunners.intendedReservationId} is not null
+      and ${providerRunners.workspaceId} is null then ${runnerControlSessions.createdAt}
+    when ${providerRunners.workspaceId} is not null
+      then coalesce(${providerRunners.assignedAt}, ${runnerControlSessions.createdAt})
+    else coalesce(
+      ${runnerControlSessions.createdAt},
+      ${providerRunners.assignedAt},
+      ${providerRunners.createdAt}
+    )
+  end`;
+  const rows = await db()
+    .select({
+      phase,
+      provider: sql<string>`coalesce(${providerRunners.providerKind}, 'unknown')`,
+      launchKind: providerRunners.launchKind,
+      count: sql<number>`count(*)::int`,
+      oldestAgeMilliseconds: sql<number>`coalesce(
+        max(extract(epoch from (now() - (${startedAt}))) * 1000),
+        0
+      )::double precision`,
+    })
+    .from(providerRunners)
+    .leftJoin(
+      runnerControlSessions,
+      and(
+        eq(runnerControlSessions.runnerInstanceId, providerRunners.id),
+        isNull(runnerControlSessions.closedAt),
+        gt(runnerControlSessions.expiresAt, sql`now()`),
+      ),
+    )
+    .where(
+      and(
+        inArray(providerRunners.state, ['starting', 'running']),
+        isNull(providerRunners.runnerSessionId),
+      ),
+    )
+    .groupBy(phase, providerRunners.providerKind, providerRunners.launchKind);
+
+  return rows.map((row) => ({
+    phase: row.phase,
+    provider: row.provider,
+    launchKind: row.launchKind,
+    count: row.count,
+    oldestAgeMilliseconds: Math.max(0, row.oldestAgeMilliseconds),
+  }));
+}
+
 export async function listActiveRunnerInstances(params: {
   workspaceId: string;
   windowSeconds: number;

package/src/db/runner-sessions.ts

@@ -3,6 +3,10 @@ import {and, asc, eq, gt, inArray, isNull, lt, notExists, or, sql} from 'drizzle
 import type {RunnerSession} from '#core/entities/runner-session.js';
 import {EmptyRunnerLabelsError} from '#core/errors.js';
 import {sanitizeRunnerLabelsOrThrow} from '#core/runner-labels.js';
+import {
+  type ProviderRunnerLifecycleObservation,
+  recordProviderRunnerAssignmentToActivation,
+} from '#metrics/instance.js';
 import {db} from './db.js';
 import {provisionerTokens} from './schema/provisioner-tokens.js';
 import {runnerActivationTokens} from './schema/runner-activation-tokens.js';
@@ -58,7 +62,8 @@ export async function createRunnerSessionConsumingActivationToken(params: {
   labels: string[];
   toolCapabilities?: RunnerToolCapabilitiesDto | null;
 }) {
-  return await db().transaction(async (tx) => {
+  let assignmentToActivationObservation: ProviderRunnerLifecycleObservation | null = null;
+  const session = await db().transaction(async (tx) => {
     const [runner] = await tx
       .select({
         activationTokenId: runnerActivationTokens.id,
@@ -66,6 +71,9 @@ export async function createRunnerSessionConsumingActivationToken(params: {
         workspaceId: providerRunners.workspaceId,
         provisionerId: providerRunners.provisionerId,
         providerRunnerId: providerRunners.providerRunnerId,
+        assignedAt: providerRunners.assignedAt,
+        provider: providerRunners.providerKind,
+        launchKind: providerRunners.launchKind,
         runnerSessionId: providerRunners.runnerSessionId,
       })
       .from(runnerActivationTokens)
@@ -143,8 +151,18 @@ export async function createRunnerSessionConsumingActivationToken(params: {
           isNull(runnerControlSessions.closedAt),
         ),
       );
+    if (runner.assignedAt)
+      assignmentToActivationObservation = {
+        durationMilliseconds: session.createdAt.getTime() - runner.assignedAt.getTime(),
+        provider: runner.provider,
+        launchKind: runner.launchKind,
+        runnerInstanceId: runner.runnerInstanceId,
+      };
     return toRunnerSession(session);
   });
+  if (assignmentToActivationObservation)
+    recordProviderRunnerAssignmentToActivation(assignmentToActivationObservation);
+  return session;
 }
 
 export interface DeleteExpiredRunnerSessionsParams {

package/src/metrics/index.ts

@@ -1,6 +1,8 @@
 export type {
   RunnerActivationTokenNotIssuedReason,
   RunnerActivationTokenNotIssuedSurface,
+  RunnerAssignmentSurface,
+  RunnerLaunchKind,
   RunnerReservationCapacityFailureReason,
   RunnerReservationPromotionFailureReason,
 } from './instance.js';
@@ -8,13 +10,25 @@ export {
   jobExecutionClaimedCount,
   jobExecutionEnqueuedCount,
   jobExecutionLeaseExpiredCount,
+  jobExecutionQueueTimeDuration,
   providerRunnerAbsentTerminatedCount,
   providerRunnerActivationOutcomeCount,
+  providerRunnerActivationToFirstClaimDuration,
+  providerRunnerAssignmentRejectedCount,
+  providerRunnerAssignmentToActivationDuration,
+  providerRunnerControlSessionToAssignmentDuration,
   providerRunnerCountDivergenceCount,
+  providerRunnerCreatedToControlSessionDuration,
   providerRunnerReconcileCallCount,
   providerRunnerTerminateIntentHonoredCount,
   providerRunnerTerminateIntentIssuedCount,
+  recordJobExecutionQueueTime,
   recordProviderRunnerActivationOutcome,
+  recordProviderRunnerActivationToFirstClaim,
+  recordProviderRunnerAssignmentRejected,
+  recordProviderRunnerAssignmentToActivation,
+  recordProviderRunnerControlSessionToAssignment,
+  recordProviderRunnerCreatedToControlSession,
   recordRunnerActivationTokenNotIssued,
   recordRunnerReservationCapacityFailure,
   recordRunnerReservationPromotionFailure,

package/src/metrics/instance.test.ts

@@ -0,0 +1,118 @@
+const metricMocks = vi.hoisted(() => {
+  const histograms = new Map<string, {record: ReturnType<typeof vi.fn>}>();
+  const counters = new Map<string, {add: ReturnType<typeof vi.fn>}>();
+  const createHistogram = vi.fn((name: string) => {
+    const histogram = {record: vi.fn()};
+    histograms.set(name, histogram);
+    return histogram;
+  });
+  const createCounter = vi.fn((name: string) => {
+    const counter = {add: vi.fn()};
+    counters.set(name, counter);
+    return counter;
+  });
+
+  return {counters, createCounter, createHistogram, histograms};
+});
+
+vi.mock('@shipfox/node-opentelemetry', () => ({
+  instanceMetrics: {
+    getMeter: () => ({
+      createCounter: metricMocks.createCounter,
+      createHistogram: metricMocks.createHistogram,
+    }),
+  },
+  logger: () => ({debug: vi.fn()}),
+}));
+
+let metrics: typeof import('./instance.js');
+
+beforeAll(async () => {
+  vi.resetModules();
+  metricMocks.counters.clear();
+  metricMocks.createCounter.mockClear();
+  metricMocks.createHistogram.mockClear();
+  metricMocks.histograms.clear();
+  metrics = await import('./instance.js');
+});
+
+describe('runner lifecycle metrics', () => {
+  it('defines provider-runner histograms with millisecond units and buckets', () => {
+    const calls = metricMocks.createHistogram.mock.calls as unknown as Array<
+      [string, {unit?: string; advice?: {explicitBucketBoundaries?: number[]}}]
+    >;
+
+    expect(
+      calls
+        .filter(([name]) => name.startsWith('runners_provider_runner_'))
+        .map(([name, options]) => ({
+          name,
+          unit: options.unit,
+          buckets: options.advice?.explicitBucketBoundaries,
+        })),
+    ).toEqual([
+      {
+        name: 'runners_provider_runner_created_to_control_session',
+        unit: 'ms',
+        buckets: [
+          100, 500, 1_000, 5_000, 10_000, 15_000, 20_000, 30_000, 45_000, 60_000, 90_000, 120_000,
+          300_000, 600_000,
+        ],
+      },
+      {
+        name: 'runners_provider_runner_control_session_to_assignment',
+        unit: 'ms',
+        buckets: [
+          100, 500, 1_000, 5_000, 10_000, 15_000, 20_000, 30_000, 45_000, 60_000, 90_000, 120_000,
+          300_000, 600_000,
+        ],
+      },
+      {
+        name: 'runners_provider_runner_assignment_to_activation',
+        unit: 'ms',
+        buckets: [10, 25, 50, 100, 250, 500, 1_000, 5_000, 10_000],
+      },
+      {
+        name: 'runners_provider_runner_activation_to_first_claim',
+        unit: 'ms',
+        buckets: [10, 25, 50, 100, 250, 500, 1_000, 5_000, 10_000],
+      },
+    ]);
+  });
+
+  it('records lifecycle durations without converting milliseconds', () => {
+    metrics.recordProviderRunnerCreatedToControlSession({
+      durationMilliseconds: 1_234,
+      provider: 'ec2',
+      launchKind: 'demand',
+    });
+
+    expect(
+      metricMocks.histograms.get('runners_provider_runner_created_to_control_session')?.record,
+    ).toHaveBeenCalledWith(1_234, {provider: 'ec2', launch_kind: 'demand'});
+  });
+});
+
+describe('job execution queue time metrics', () => {
+  it('records milliseconds with bounded labels and ignores negative durations', () => {
+    const record = metricMocks.histograms.get('runners_job_execution_queue_time')?.record;
+    expect(record).toBeDefined();
+
+    metrics.recordJobExecutionQueueTime({
+      durationMilliseconds: 1_234,
+      provider: null,
+      launchKind: 'manual',
+    });
+    metrics.recordJobExecutionQueueTime({
+      durationMilliseconds: -1,
+      provider: 'ec2',
+      launchKind: 'demand',
+    });
+
+    expect(record).toHaveBeenCalledTimes(1);
+    expect(record).toHaveBeenCalledWith(1_234, {
+      provider: 'unknown',
+      launch_kind: 'manual',
+    });
+  });
+});