@shipfox/api-runners 12.4.0 → 12.5.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shipfox/api-runners",
   "license": "MIT",
-  "version": "12.4.0",
+  "version": "12.5.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ShipfoxHQ/shipfox.git",
@@ -25,7 +25,7 @@
     "@shipfox/api-auth-context": "12.2.0",
     "@shipfox/api-auth-dto": "12.0.0",
     "@shipfox/api-runners-dto": "12.4.0",
-    "@shipfox/api-workflows-dto": "12.3.0",
+    "@shipfox/api-workflows-dto": "12.5.0",
     "@shipfox/config": "1.2.4",
     "@shipfox/inter-module": "0.2.3",
     "@shipfox/node-drizzle": "0.3.5",

package/src/core/errors.ts

@@ -109,18 +109,47 @@ export class ReservationExpiredError extends Error {
   }
 }
 
+export type RunnerInstanceNotAssignableReason =
+  | 'runner-not-found'
+  | 'runner-not-running'
+  | 'provider-identity-missing'
+  | 'control-session-not-active'
+  | 'labels-mismatch'
+  | 'capacity-exhausted';
+
+export type RunnerAssignmentRejectionReason =
+  | 'reservation-not-found'
+  | 'reservation-expired'
+  | 'already-assigned'
+  | RunnerInstanceNotAssignableReason;
+
 export class RunnerInstanceNotAssignableError extends Error {
-  constructor(public readonly runnerInstanceId: string) {
+  constructor(
+    public readonly runnerInstanceId: string,
+    public readonly reason: RunnerInstanceNotAssignableReason,
+  ) {
     super(`Runner instance cannot be assigned: ${runnerInstanceId}`);
     this.name = 'RunnerInstanceNotAssignableError';
   }
 }
+
 export class RunnerInstanceAlreadyAssignedError extends Error {
   constructor(public readonly runnerInstanceId: string) {
     super(`Runner instance is already assigned: ${runnerInstanceId}`);
     this.name = 'RunnerInstanceAlreadyAssignedError';
   }
 }
+
+export function getRunnerAssignmentRejectionReason(
+  error: unknown,
+): RunnerAssignmentRejectionReason | null {
+  if (error instanceof ReservationNotFoundError) return 'reservation-not-found';
+  if (error instanceof ReservationExpiredError) return 'reservation-expired';
+  if (error instanceof RunnerInstanceAlreadyAssignedError) return 'already-assigned';
+  if (error instanceof RunnerInstanceNotAssignableError) return error.reason;
+  return null;
+}
+
 export class ReservationAlreadyAssignedError extends Error {
   constructor(public readonly reservationId: string) {
     super(`Reservation is already assigned: ${reservationId}`);

package/src/core/runner-control-sessions.test.ts

@@ -359,6 +359,77 @@ describe('enrollRunnerControlSession', () => {
       vi.resetModules();
     }
   });
+
+  it('records a rolled-back assignment rejection without recording its assignment timing', async () => {
+    const provisionerId = crypto.randomUUID();
+    const reservation = await createReservation({provisionerId});
+    const runnerInstanceId = await createRunner({
+      provisionerId,
+      intendedReservationId: reservation.id,
+    });
+    const assignmentTimingRecord = vi.fn();
+    const assignmentRejectionRecord = vi.fn();
+    const promotionFailureRecord = vi.fn();
+
+    vi.resetModules();
+    const {RunnerInstanceNotAssignableError} = await import('#core/errors.js');
+    const activationIssueError = new RunnerInstanceNotAssignableError(
+      runnerInstanceId,
+      'capacity-exhausted',
+    );
+    vi.doMock('#metrics/index.js', async (importOriginal) => ({
+      ...(await importOriginal<typeof import('#metrics/index.js')>()),
+      recordProviderRunnerControlSessionToAssignment: assignmentTimingRecord,
+      recordProviderRunnerAssignmentRejected: assignmentRejectionRecord,
+      recordRunnerReservationPromotionFailure: promotionFailureRecord,
+    }));
+    vi.doMock('./runner-activation.js', async (importOriginal) => ({
+      ...(await importOriginal<typeof import('./runner-activation.js')>()),
+      issueRunnerActivationTokenTx: vi.fn().mockRejectedValue(activationIssueError),
+    }));
+
+    const {closePostgresClient, createPostgresClient} = await import('@shipfox/node-postgres');
+    createPostgresClient();
+
+    try {
+      const {enrollRunnerControlSession: enroll} = await import('./runner-control-sessions.js');
+      const result = await enroll({
+        runnerInstanceId,
+        provisionerId,
+        labels: ['linux'],
+        providerKind: 'docker',
+        protocolVersion: '1',
+      });
+      const [runner] = await db()
+        .select({
+          intendedReservationId: providerRunners.intendedReservationId,
+          reservationId: providerRunners.reservationId,
+          workspaceId: providerRunners.workspaceId,
+          assignedAt: providerRunners.assignedAt,
+        })
+        .from(providerRunners)
+        .where(eq(providerRunners.id, runnerInstanceId));
+
+      expect(result).toBeNull();
+      expect(runner).toEqual({
+        intendedReservationId: reservation.id,
+        reservationId: null,
+        workspaceId: null,
+        assignedAt: null,
+      });
+      expect(assignmentTimingRecord).not.toHaveBeenCalled();
+      expect(assignmentRejectionRecord).toHaveBeenCalledWith({
+        reason: 'capacity-exhausted',
+        surface: 'enrollment',
+      });
+      expect(promotionFailureRecord).toHaveBeenCalledWith('not-assignable');
+    } finally {
+      await closePostgresClient();
+      vi.doUnmock('#metrics/index.js');
+      vi.doUnmock('./runner-activation.js');
+      vi.resetModules();
+    }
+  });
 });
 
 async function expectPromotionFailure(params: {

package/src/core/runner-control-sessions.ts

@@ -5,6 +5,7 @@ import {extractDisplayPrefix, generateOpaqueToken, hashOpaqueToken} from '@shipf
 import {and, eq, gt, isNull, notInArray, or, sql} from 'drizzle-orm';
 import {config} from '#config.js';
 import {
+  getRunnerAssignmentRejectionReason,
   ReservationExpiredError,
   ReservationNotFoundError,
   RunnerInstanceAlreadyAssignedError,
@@ -22,6 +23,9 @@ import {runnerBootstrapTokens, runnerControlSessions} from '#db/schema/runner-co
 import {providerRunners} from '#db/schema/runner-instances.js';
 import {
   type RunnerReservationPromotionFailureReason,
+  recordProviderRunnerAssignmentRejected,
+  recordProviderRunnerControlSessionToAssignment,
+  recordProviderRunnerCreatedToControlSession,
   recordRunnerReservationCapacityFailure,
   recordRunnerReservationPromotionFailure,
 } from '#metrics/index.js';
@@ -137,7 +141,7 @@ export async function exchangeRunnerBootstrapToken(params: {
 }): Promise<{runnerInstanceId: string; controlSessionToken: string; expiresAt: Date}> {
   const controlSessionToken = generateOpaqueToken('runnerControlSession');
   const expiresAt = new Date(Date.now() + params.ttlSeconds * 1000);
-  return await db().transaction(async (tx) => {
+  const result = await db().transaction(async (tx) => {
     const [bootstrap] = await tx
       .update(runnerBootstrapTokens)
       .set({consumedAt: sql`now()`})
@@ -151,6 +155,21 @@ export async function exchangeRunnerBootstrapToken(params: {
       )
       .returning();
     if (!bootstrap) throw new RunnerBootstrapTokenInvalidError();
+    const [runner] = await tx
+      .select({
+        runnerInstanceId: providerRunners.id,
+        createdAt: providerRunners.createdAt,
+        provider: providerRunners.providerKind,
+        launchKind: providerRunners.launchKind,
+      })
+      .from(providerRunners)
+      .where(
+        and(
+          eq(providerRunners.id, bootstrap.runnerInstanceId),
+          eq(providerRunners.provisionerId, bootstrap.provisionerId),
+        ),
+      )
+      .limit(1);
     const [session] = await tx
       .insert(runnerControlSessions)
       .values({
@@ -160,10 +179,28 @@ export async function exchangeRunnerBootstrapToken(params: {
         prefix: extractDisplayPrefix(controlSessionToken),
         expiresAt,
       })
-      .returning({id: runnerControlSessions.id});
+      .returning({createdAt: runnerControlSessions.createdAt});
     if (!session) throw new Error('Runner control session insert returned no row');
-    return {runnerInstanceId: bootstrap.runnerInstanceId, controlSessionToken, expiresAt};
+    return {
+      runner,
+      session,
+      runnerInstanceId: bootstrap.runnerInstanceId,
+      controlSessionToken,
+      expiresAt,
+    };
   });
+  if (result.runner)
+    recordProviderRunnerCreatedToControlSession({
+      durationMilliseconds: result.session.createdAt.getTime() - result.runner.createdAt.getTime(),
+      provider: result.runner.provider,
+      launchKind: result.runner.launchKind,
+      runnerInstanceId: result.runner.runnerInstanceId,
+    });
+  return {
+    runnerInstanceId: result.runnerInstanceId,
+    controlSessionToken: result.controlSessionToken,
+    expiresAt: result.expiresAt,
+  };
 }
 
 export async function resolveRunnerControlSession(rawToken: string) {
@@ -195,7 +232,7 @@ export async function enrollRunnerControlSession(params: {
   providerKind: string;
   protocolVersion: string;
 }): Promise<string | null> {
-  return await db().transaction(async (tx) => {
+  const result = await db().transaction(async (tx) => {
     const [current] = await tx
       .select({
         intendedReservationId: providerRunners.intendedReservationId,
@@ -266,23 +303,38 @@ export async function enrollRunnerControlSession(params: {
     await updateRunnerControlSessionLastSeen(tx, params.runnerInstanceId, params.provisionerId);
 
     const intendedReservationId = updated.intendedReservationId;
-    if (!intendedReservationId) return null;
+    if (!intendedReservationId)
+      return {
+        activationToken: null,
+        controlSessionToAssignment: [],
+        assignmentRejectedReason: null,
+        promotionFailureReason: null,
+      };
 
     try {
-      return await tx.transaction(async (promotionTx) => {
-        await assignRunnerInstancesTx(promotionTx, {
+      const promotion = await tx.transaction(async (promotionTx) => {
+        const assignment = await assignRunnerInstancesTx(promotionTx, {
           provisionerId: params.provisionerId,
           reservationId: intendedReservationId,
           runnerInstanceIds: [params.runnerInstanceId],
+          surface: 'enrollment',
         });
-        return await issueRunnerActivationTokenTx(promotionTx, {
+        const activationToken = await issueRunnerActivationTokenTx(promotionTx, {
           runnerInstanceId: params.runnerInstanceId,
           provisionerId: params.provisionerId,
           ttlSeconds: config.RUNNER_ACTIVATION_TOKEN_TTL_SECONDS,
           surface: 'enrollment',
         });
+        return {activationToken, controlSessionToAssignment: assignment.controlSessionToAssignment};
       });
+      return {
+        activationToken: promotion.activationToken,
+        controlSessionToAssignment: promotion.controlSessionToAssignment,
+        assignmentRejectedReason: null,
+        promotionFailureReason: null,
+      };
     } catch (error) {
+      const assignmentRejectionReason = getRunnerAssignmentRejectionReason(error);
       const expectedFailureReason = getRunnerReservationPromotionFailureReason(error);
       const details = {
         err: error,
@@ -291,7 +343,6 @@ export async function enrollRunnerControlSession(params: {
         reservationId: intendedReservationId,
       };
       if (expectedFailureReason) {
-        recordRunnerReservationPromotionFailure(expectedFailureReason);
         logger().debug(details, 'Runner reservation could not be promoted during enrollment');
       } else {
         logger().error(
@@ -299,9 +350,27 @@ export async function enrollRunnerControlSession(params: {
           'Unexpected failure promoting runner reservation during enrollment',
         );
       }
-      return null;
+      return {
+        activationToken: null,
+        controlSessionToAssignment: [],
+        assignmentRejectedReason: assignmentRejectionReason,
+        promotionFailureReason: expectedFailureReason,
+      };
     }
   });
+  for (const observation of result.controlSessionToAssignment)
+    recordProviderRunnerControlSessionToAssignment(observation);
+  // These counters intentionally overlap: assignment_rejected records the assignment-facing
+  // reason, while reservation_promotion_failure records the enrollment recovery outcome. They
+  // answer different questions and must not be summed.
+  if (result.assignmentRejectedReason)
+    recordProviderRunnerAssignmentRejected({
+      reason: result.assignmentRejectedReason,
+      surface: 'enrollment',
+    });
+  if (result.promotionFailureReason)
+    recordRunnerReservationPromotionFailure(result.promotionFailureReason);
+  return result.activationToken;
 }
 
 function getRunnerReservationPromotionFailureReason(

package/src/db/job-executions.test.ts

@@ -8,6 +8,7 @@ import {eq, sql} from 'drizzle-orm';
 import {EmptyRequiredLabelsError, RunnerSessionExhaustedError} from '#core/errors.js';
 import {claimJobExecution} from '#core/job-executions.js';
 import {detectAndExpireStuckJobs} from '#core/maintenance.js';
+import * as runnerMetrics from '#metrics/instance.js';
 import {
   getLeaseTokenClaims,
   pendingJobFactory,
@@ -242,20 +243,95 @@ describe('claimPendingJobExecution', () => {
     expect(new Date(payload.claimedAt).getTime()).toBe(running?.startedAt.getTime());
   });
 
+  it('records queue time from the pending row creation to the runner claim', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+    await db()
+      .update(pendingJobExecutions)
+      .set({createdAt: sql`now() - interval '60 seconds'`})
+      .where(eq(pendingJobExecutions.jobExecutionId, created.jobExecutionId));
+    const [pending] = await db()
+      .select({createdAt: pendingJobExecutions.createdAt})
+      .from(pendingJobExecutions)
+      .where(eq(pendingJobExecutions.jobExecutionId, created.jobExecutionId));
+    const recordQueueTime = vi.spyOn(runnerMetrics, 'recordJobExecutionQueueTime');
+
+    try {
+      await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+      const [running] = await db()
+        .select({startedAt: runningJobExecutions.startedAt})
+        .from(runningJobExecutions)
+        .where(eq(runningJobExecutions.jobExecutionId, created.jobExecutionId));
+      const observation = recordQueueTime.mock.calls[0]?.[0];
+      if (!pending || !running) throw new Error('Expected pending and running job rows');
+      expect(recordQueueTime).toHaveBeenCalledTimes(1);
+      expect(observation).toMatchObject({provider: null, launchKind: 'manual'});
+      expect(observation?.durationMilliseconds).toBe(
+        running.startedAt.getTime() - pending.createdAt.getTime(),
+      );
+      expect(observation?.durationMilliseconds).toBeGreaterThanOrEqual(60_000);
+      expect(observation?.durationMilliseconds).toBeLessThan(120_000);
+    } finally {
+      recordQueueTime.mockRestore();
+    }
+  });
+
+  it('records bounded queue labels for a provisioned runner claim', async () => {
+    const provisionerId = crypto.randomUUID();
+    const providerRunnerId = `provisioned-runner-${crypto.randomUUID()}`;
+    await db()
+      .update(runnerSessions)
+      .set({registrationTokenKind: 'ephemeral', maxClaims: 1, provisionerId, providerRunnerId})
+      .where(eq(runnerSessions.id, runnerSessionId));
+    await db().insert(providerRunners).values({
+      workspaceId,
+      provisionerId,
+      providerRunnerId,
+      providerKind: 'ec2',
+      launchKind: 'demand',
+      labels: sessionLabels,
+      state: 'running',
+      reportedAt: new Date(),
+    });
+    await pendingJobFactory.create({workspaceId});
+    const recordQueueTime = vi.spyOn(runnerMetrics, 'recordJobExecutionQueueTime');
+
+    try {
+      await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: 1});
+
+      expect(recordQueueTime).toHaveBeenCalledWith(
+        expect.objectContaining({provider: 'ec2', launchKind: 'demand'}),
+      );
+      expect(recordQueueTime.mock.calls[0]?.[0].durationMilliseconds).toBeGreaterThanOrEqual(0);
+    } finally {
+      recordQueueTime.mockRestore();
+    }
+  });
+
   it('emits no claimed event when there is nothing to claim', async () => {
+    const recordQueueTime = vi.spyOn(runnerMetrics, 'recordJobExecutionQueueTime');
     const before = await db()
       .select()
       .from(runnersOutbox)
       .where(eq(runnersOutbox.eventType, RUNNER_JOB_CLAIMED));
 
-    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    try {
+      const claimed = await claimPendingJobExecution({
+        workspaceId,
+        runnerSessionId,
+        maxClaims: null,
+      });
 
-    const after = await db()
-      .select()
-      .from(runnersOutbox)
-      .where(eq(runnersOutbox.eventType, RUNNER_JOB_CLAIMED));
-    expect(claimed).toBeNull();
-    expect(after).toHaveLength(before.length);
+      const after = await db()
+        .select()
+        .from(runnersOutbox)
+        .where(eq(runnersOutbox.eventType, RUNNER_JOB_CLAIMED));
+      expect(claimed).toBeNull();
+      expect(after).toHaveLength(before.length);
+      expect(recordQueueTime).not.toHaveBeenCalled();
+    } finally {
+      recordQueueTime.mockRestore();
+    }
   });
 
   it('emits no claimed event when dropping an orphan pending row', async () => {
@@ -273,13 +349,23 @@ describe('claimPendingJobExecution', () => {
     });
     // Clear the initial claim's events so this assertion only covers the orphan claim.
     const beforeOrphanClaim = await outboxEventsForJob(RUNNER_JOB_CLAIMED, created.jobId);
+    const recordQueueTime = vi.spyOn(runnerMetrics, 'recordJobExecutionQueueTime');
 
-    const second = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    try {
+      const second = await claimPendingJobExecution({
+        workspaceId,
+        runnerSessionId,
+        maxClaims: null,
+      });
 
-    expect(second).toBeNull();
-    expect(await outboxEventsForJob(RUNNER_JOB_CLAIMED, created.jobId)).toHaveLength(
-      beforeOrphanClaim.length,
-    );
+      expect(second).toBeNull();
+      expect(await outboxEventsForJob(RUNNER_JOB_CLAIMED, created.jobId)).toHaveLength(
+        beforeOrphanClaim.length,
+      );
+      expect(recordQueueTime).not.toHaveBeenCalled();
+    } finally {
+      recordQueueTime.mockRestore();
+    }
   });
 
   it('returns the job ids when a job is available', async () => {

package/src/db/job-executions.ts

@@ -29,7 +29,14 @@ import {
   RunnerSessionExhaustedError,
   RunningJobExecutionNotFoundError,
 } from '#core/errors.js';
-import {jobExecutionEnqueuedCount, jobExecutionLeaseExpiredCount} from '#metrics/instance.js';
+import {
+  type JobExecutionQueueTimeObservation,
+  jobExecutionEnqueuedCount,
+  jobExecutionLeaseExpiredCount,
+  type ProviderRunnerLifecycleObservation,
+  recordJobExecutionQueueTime,
+  recordProviderRunnerActivationToFirstClaim,
+} from '#metrics/instance.js';
 import type {Tx} from './db.js';
 import {db} from './db.js';
 import {runnersOutbox} from './schema/outbox.js';
@@ -192,7 +199,9 @@ export async function claimPendingJobExecution(params: {
 
   if (params.sessionLabels.length === 0) return null;
 
-  return await db().transaction(async (tx) => {
+  let activationToFirstClaimObservation: ProviderRunnerLifecycleObservation | null = null;
+  let queueTimeObservation: JobExecutionQueueTimeObservation | null = null;
+  const result = await db().transaction(async (tx) => {
     let provisionerId: string | null = null;
     let providerRunnerId: string | null = null;
     let runnerInstanceId: string | null = null;
@@ -288,6 +297,12 @@ export async function claimPendingJobExecution(params: {
     const claimed = inserted[0];
     if (!claimed) return null;
 
+    queueTimeObservation = {
+      durationMilliseconds: claimed.claimedAt.getTime() - row.createdAt.getTime(),
+      provider: null,
+      launchKind: params.maxClaims === null ? 'manual' : 'unknown',
+    };
+
     const runnerInstanceCondition = runnerInstanceId
       ? eq(providerRunners.id, runnerInstanceId)
       : provisionerId && providerRunnerId
@@ -297,13 +312,42 @@ export async function claimPendingJobExecution(params: {
           )
         : null;
     if (runnerInstanceCondition) {
-      await tx
+      const [claimedRunner] = await tx
         .update(providerRunners)
         .set({
           firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,
           updatedAt: sql`now()`,
         })
-        .where(runnerInstanceCondition);
+        .where(runnerInstanceCondition)
+        .returning({
+          firstClaimedAt: providerRunners.firstClaimedAt,
+          isFirstClaim: sql<boolean>`${providerRunners.firstClaimedAt} = ${claimed.claimedAt}`,
+          provider: providerRunners.providerKind,
+          launchKind: providerRunners.launchKind,
+          runnerInstanceId: providerRunners.id,
+          sessionCreatedAtEpochMs: sql<number | null>`(
+            select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000
+            from ${runnerSessions}
+            where ${runnerSessions.id} = ${params.runnerSessionId}
+          )`,
+        });
+      if (claimedRunner && queueTimeObservation) {
+        queueTimeObservation.provider = claimedRunner.provider;
+        queueTimeObservation.launchKind = claimedRunner.launchKind;
+      }
+      if (
+        claimedRunner?.isFirstClaim &&
+        claimedRunner.firstClaimedAt !== null &&
+        claimedRunner.sessionCreatedAtEpochMs !== null &&
+        claimedRunner.sessionCreatedAtEpochMs !== undefined
+      )
+        activationToFirstClaimObservation = {
+          durationMilliseconds:
+            claimedRunner.firstClaimedAt.getTime() - claimedRunner.sessionCreatedAtEpochMs,
+          provider: claimedRunner.provider,
+          launchKind: claimedRunner.launchKind,
+          runnerInstanceId: claimedRunner.runnerInstanceId,
+        };
     }
 
     if (params.maxClaims !== null) {
@@ -335,6 +379,10 @@ export async function claimPendingJobExecution(params: {
       projectId: row.projectId,
     };
   });
+  if (queueTimeObservation) recordJobExecutionQueueTime(queueTimeObservation);
+  if (activationToFirstClaimObservation)
+    recordProviderRunnerActivationToFirstClaim(activationToFirstClaimObservation);
+  return result;
 }
 
 async function touchRunnerSessionLiveness(params: {