@shipfox/api-runners 12.2.0 → 12.4.0

package/src/presentation/routes/runner-enrollment.test.ts

@@ -6,6 +6,7 @@ import {
   createApp,
   extractBearerToken,
 } from '@shipfox/node-fastify';
+import {vi} from '@shipfox/vitest/vi';
 import {eq} from 'drizzle-orm';
 import type {FastifyInstance, FastifyRequest} from 'fastify';
 import {db} from '#db/db.js';
@@ -13,6 +14,7 @@ import {reservations} from '#db/schema/reservations.js';
 import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
 import {runnerBootstrapTokens, runnerControlSessions} from '#db/schema/runner-control-sessions.js';
 import {providerRunners} from '#db/schema/runner-instances.js';
+import {runnerReservationCapacityFailureCount} from '#metrics/instance.js';
 import {
   createRunnerControlSessionAuthMethod,
   createRunnerRegistrationTokenAuthMethod,
@@ -159,6 +161,7 @@ describe('runner enrollment control plane', () => {
       provisionerId,
       labels: ['linux', 'shipfox-managed'],
       providerRunnerId: 'container-1',
+      launchKind: 'warm',
       state: 'running',
       protocolVersion: '1',
     });
@@ -210,7 +213,7 @@ describe('runner enrollment control plane', () => {
     expect(instance?.labels).toEqual(['linux']);
   });
 
-  it('promotes an intended reservation during enrollment and returns its activation token', async () => {
+  it('repairs an intended reservation after a provider report and returns its activation token', async () => {
     const workspaceId = crypto.randomUUID();
     const [reservation] = await db()
       .insert(reservations)
@@ -240,14 +243,49 @@ describe('runner enrollment control plane', () => {
     });
     const controlToken = exchanged.json().control_session_token;
 
+    const providerRunnerId = 'container-intended-assignment';
     const attached = await app.inject({
       method: 'POST',
       url: `/provisioners/runner-instances/${runner.runner_instance_id}/provider-runner`,
       headers: {authorization: `Bearer ${token}`},
-      payload: {provider_runner_id: 'container-intended-assignment'},
+      payload: {provider_runner_id: providerRunnerId},
     });
     expect(attached.json()).toEqual({attached: true});
 
+    const reported = await app.inject({
+      method: 'POST',
+      url: '/provisioners/runner-instances/report',
+      headers: {authorization: `Bearer ${token}`},
+      payload: {
+        events: [
+          {
+            provider_runner_id: providerRunnerId,
+            reservation_id: reservation.id,
+            template_key: 'linux',
+            labels: ['linux'],
+            state: 'starting',
+            reported_at: new Date().toISOString(),
+            provider_kind: 'docker',
+          },
+        ],
+      },
+    });
+    expect(reported.statusCode).toBe(200);
+    expect(reported.json()).toEqual({accepted: 1, reservations_released: 0});
+
+    const [reportedInstance] = await db()
+      .select()
+      .from(providerRunners)
+      .where(eq(providerRunners.id, runner.runner_instance_id));
+    expect(reportedInstance).toMatchObject({
+      workspaceId: null,
+      reservationId: reservation.id,
+      intendedReservationId: reservation.id,
+      assignedAt: null,
+      state: 'starting',
+      providerRunnerId,
+    });
+
     const enrolled = await app.inject({
       method: 'POST',
       url: '/runner-control/enrollment',
@@ -264,13 +302,288 @@ describe('runner enrollment control plane', () => {
     expect(instance).toMatchObject({
       workspaceId,
       reservationId: reservation.id,
+      launchKind: 'demand',
       intendedReservationId: null,
       state: 'running',
-      providerRunnerId: 'container-intended-assignment',
+      providerRunnerId,
       assignedAt: expect.any(Date),
     });
   });
 
+  it('does not create a duplicate row or bootstrap token after a reservation is consumed', async () => {
+    const workspaceId = crypto.randomUUID();
+    const [reservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 1,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning();
+    if (!reservation) throw new Error('Reservation insert returned no row');
+
+    const first = await app.inject({
+      method: 'POST',
+      url: '/provisioners/runner-instances/batch',
+      headers: {authorization: `Bearer ${token}`},
+      payload: {runner_instances: [{reservation_id: reservation.id}]},
+    });
+    const firstRunner = first.json().runner_instances[0];
+    expect(first.statusCode).toBe(200);
+    expect(first.json().reservation_unavailable).toBeUndefined();
+
+    const second = await app.inject({
+      method: 'POST',
+      url: '/provisioners/runner-instances/batch',
+      headers: {authorization: `Bearer ${token}`},
+      payload: {runner_instances: [{reservation_id: reservation.id}]},
+    });
+
+    expect(second.statusCode).toBe(200);
+    expect(second.json()).toMatchObject({
+      runner_instances: [],
+      reservation_unavailable: true,
+    });
+    const runners = await db()
+      .select()
+      .from(providerRunners)
+      .where(eq(providerRunners.intendedReservationId, reservation.id));
+    const bootstrapTokens = await db()
+      .select()
+      .from(runnerBootstrapTokens)
+      .where(eq(runnerBootstrapTokens.runnerInstanceId, firstRunner.runner_instance_id));
+    expect(runners).toHaveLength(1);
+    expect(bootstrapTokens).toHaveLength(1);
+  });
+
+  it('returns a classified shortfall for expired and foreign reservations', async () => {
+    const otherProvisioner = await provisionerTokenFactory.create({scope: 'installation'});
+    const runnersBefore = await db()
+      .select()
+      .from(providerRunners)
+      .where(eq(providerRunners.provisionerId, provisionerId));
+    const bootstrapTokensBefore = await db()
+      .select()
+      .from(runnerBootstrapTokens)
+      .where(eq(runnerBootstrapTokens.provisionerId, provisionerId));
+    const expiredReservation = await db()
+      .insert(reservations)
+      .values({
+        workspaceId: crypto.randomUUID(),
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 1,
+        expiresAt: new Date(Date.now() - 1_000),
+      })
+      .returning({id: reservations.id});
+    const foreignReservation = await db()
+      .insert(reservations)
+      .values({
+        workspaceId: crypto.randomUUID(),
+        provisionerId: otherProvisioner.id,
+        requiredLabels: ['linux'],
+        count: 1,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    const boundReservation = await db()
+      .insert(reservations)
+      .values({
+        workspaceId: crypto.randomUUID(),
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 1,
+        kind: 'bound',
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    const reservationIds = [
+      expiredReservation[0]?.id,
+      foreignReservation[0]?.id,
+      boundReservation[0]?.id,
+    ];
+    if (reservationIds.some((reservationId) => !reservationId))
+      throw new Error('Reservation insert returned no row');
+
+    const failureSpy = vi.spyOn(runnerReservationCapacityFailureCount, 'add');
+    try {
+      for (const reservationId of reservationIds) {
+        const response = await app.inject({
+          method: 'POST',
+          url: '/provisioners/runner-instances/batch',
+          headers: {authorization: `Bearer ${token}`},
+          payload: {runner_instances: [{reservation_id: reservationId}]},
+        });
+
+        expect(response.statusCode).toBe(200);
+        expect(response.json()).toMatchObject({
+          runner_instances: [],
+          reservation_unavailable: true,
+        });
+      }
+
+      expect(failureSpy).toHaveBeenCalledWith(1, {reason: 'reservation-expired'});
+      expect(failureSpy).toHaveBeenCalledWith(1, {reason: 'reservation-not-found'});
+      expect(failureSpy).toHaveBeenCalledWith(1, {reason: 'reservation-kind-mismatch'});
+    } finally {
+      failureSpy.mockRestore();
+    }
+
+    const runners = await db()
+      .select()
+      .from(providerRunners)
+      .where(eq(providerRunners.provisionerId, provisionerId));
+    const bootstrapTokens = await db()
+      .select()
+      .from(runnerBootstrapTokens)
+      .where(eq(runnerBootstrapTokens.provisionerId, provisionerId));
+    expect(runners).toHaveLength(runnersBefore.length);
+    expect(bootstrapTokens).toHaveLength(bootstrapTokensBefore.length);
+  });
+
+  it('returns the remaining reservation capacity with request indexes', async () => {
+    const workspaceId = crypto.randomUUID();
+    const [reservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 2,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning();
+    if (!reservation) throw new Error('Reservation insert returned no row');
+
+    const first = await app.inject({
+      method: 'POST',
+      url: '/provisioners/runner-instances/batch',
+      headers: {authorization: `Bearer ${token}`},
+      payload: {runner_instances: [{reservation_id: reservation.id}]},
+    });
+    expect(first.statusCode).toBe(200);
+
+    const second = await app.inject({
+      method: 'POST',
+      url: '/provisioners/runner-instances/batch',
+      headers: {authorization: `Bearer ${token}`},
+      payload: {
+        runner_instances: [
+          {template_key: 'linux', reservation_id: reservation.id},
+          {template_key: 'linux', reservation_id: reservation.id},
+        ],
+      },
+    });
+
+    expect(second.statusCode).toBe(200);
+    expect(second.json()).toMatchObject({
+      runner_instances: [
+        {
+          request_index: 0,
+          runner_instance_id: expect.any(String),
+          bootstrap_token: expect.any(String),
+        },
+      ],
+      reservation_unavailable: true,
+    });
+  });
+
+  it('preserves request indexes when a mixed reservation batch has a shortfall', async () => {
+    const [exhaustedReservation, availableReservation] = await db()
+      .insert(reservations)
+      .values([
+        {
+          workspaceId: crypto.randomUUID(),
+          provisionerId,
+          requiredLabels: ['linux'],
+          count: 1,
+          expiresAt: new Date(Date.now() + 60_000),
+        },
+        {
+          workspaceId: crypto.randomUUID(),
+          provisionerId,
+          requiredLabels: ['linux'],
+          count: 1,
+          expiresAt: new Date(Date.now() + 60_000),
+        },
+      ])
+      .returning();
+    if (!exhaustedReservation || !availableReservation)
+      throw new Error('Reservation insert returned no rows');
+
+    const consumed = await app.inject({
+      method: 'POST',
+      url: '/provisioners/runner-instances/batch',
+      headers: {authorization: `Bearer ${token}`},
+      payload: {runner_instances: [{reservation_id: exhaustedReservation.id}]},
+    });
+    expect(consumed.statusCode).toBe(200);
+
+    const mixed = await app.inject({
+      method: 'POST',
+      url: '/provisioners/runner-instances/batch',
+      headers: {authorization: `Bearer ${token}`},
+      payload: {
+        runner_instances: [
+          {template_key: 'linux', reservation_id: exhaustedReservation.id},
+          {template_key: 'linux', reservation_id: availableReservation.id},
+        ],
+      },
+    });
+
+    expect(mixed.statusCode).toBe(200);
+    expect(mixed.json()).toMatchObject({
+      runner_instances: [
+        {
+          request_index: 1,
+          runner_instance_id: expect.any(String),
+          bootstrap_token: expect.any(String),
+        },
+      ],
+      reservation_unavailable: true,
+    });
+  });
+
+  it('rejects assignment for a provider-reported runner before enrollment', async () => {
+    const workspaceId = crypto.randomUUID();
+    const [reservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 1,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning();
+    if (!reservation) throw new Error('Reservation insert returned no row');
+    const [runner] = await db()
+      .insert(providerRunners)
+      .values({
+        provisionerId,
+        intendedReservationId: reservation.id,
+        reservationId: reservation.id,
+        providerRunnerId: crypto.randomUUID(),
+        labels: ['linux'],
+        state: 'starting',
+        reportedAt: new Date(),
+      })
+      .returning({id: providerRunners.id});
+    if (!runner) throw new Error('Runner instance insert returned no row');
+
+    const assigned = await app.inject({
+      method: 'POST',
+      url: '/provisioners/runner-instances/assignments',
+      headers: {authorization: `Bearer ${token}`},
+      payload: {reservation_id: reservation.id, runner_instance_ids: [runner.id]},
+    });
+
+    expect(assigned.statusCode).toBe(409);
+    expect(assigned.json()).toMatchObject({code: 'runner-instance-not-assignable'});
+  });
+
   it('keeps enrollment successful when an intended reservation cannot be promoted', async () => {
     const workspaceId = crypto.randomUUID();
     const [reservation] = await db()

package/src/presentation/routes/runner-enrollment.ts

@@ -42,7 +42,7 @@ export const createRunnerInstancesRoute = defineRoute({
   },
   handler: async (request) => {
     const {provisionerTokenId} = requireProvisionerContext(request);
-    const results = await createRunnerInstancesWithBootstrapTokens({
+    const result = await createRunnerInstancesWithBootstrapTokens({
       provisionerId: provisionerTokenId,
       ...(request.body.provider_kind ? {providerKind: request.body.provider_kind} : {}),
       runnerInstances: request.body.runner_instances.map((runner) => ({
@@ -52,10 +52,12 @@ export const createRunnerInstancesRoute = defineRoute({
       ttlSeconds: config.RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS,
     });
     return {
-      runner_instances: results.map((result) => ({
-        runner_instance_id: result.runnerInstanceId,
-        bootstrap_token: result.bootstrapToken,
+      runner_instances: result.runnerInstances.map((runnerInstance) => ({
+        runner_instance_id: runnerInstance.runnerInstanceId,
+        bootstrap_token: runnerInstance.bootstrapToken,
+        request_index: runnerInstance.requestIndex,
       })),
+      ...(result.reservationUnavailable ? {reservation_unavailable: true as const} : {}),
     };
   },
 });
@@ -173,7 +175,7 @@ type RunnerAssignmentPollParams = {
   deadline: number;
   intervalMs: number;
   signal: AbortSignal;
-  getAssignment: () => Promise<unknown | null>;
+  getAssignment: () => Promise<{workspaceId: string | null; runnerSessionId: string | null} | null>;
   issueActivationToken: () => Promise<string | null>;
 };
 
@@ -229,6 +231,7 @@ export const runnerAssignmentPollRoute = defineRoute({
         issueRunnerActivationToken({
           ...session,
           ttlSeconds: config.RUNNER_ACTIVATION_TOKEN_TTL_SECONDS,
+          surface: 'poll',
         }),
     });
     return {activation_token: activationToken};

package/test/factories/reservation.ts

@@ -7,6 +7,7 @@ interface ReservationAttrs {
   provisionerId: string;
   requiredLabels: string[];
   count: number;
+  kind?: 'bound' | 'launch';
   expiresAt: Date;
 }
 
@@ -19,6 +20,7 @@ export const reservationFactory = Factory.define<ReservationAttrs>(({onCreate})
       provisionerId: row.provisionerId,
       requiredLabels: row.requiredLabels,
       count: row.count,
+      kind: row.kind,
       expiresAt: row.expiresAt,
     };
   });

package/test/factories/runner-instance.ts

@@ -11,7 +11,9 @@ export const providerRunnerFactory = Factory.define<RunnerInstance>(({onCreate})
         workspaceId: providerRunner.workspaceId,
         provisionerId: providerRunner.provisionerId,
         providerRunnerId: providerRunner.providerRunnerId,
+        intendedReservationId: providerRunner.intendedReservationId,
         reservationId: providerRunner.reservationId,
+        launchKind: providerRunner.launchKind,
         templateKey: providerRunner.templateKey,
         labels: providerRunner.labels,
         state: providerRunner.state,
@@ -25,6 +27,8 @@ export const providerRunnerFactory = Factory.define<RunnerInstance>(({onCreate})
         failedAt: providerRunner.failedAt,
         terminatedAt: providerRunner.terminatedAt,
         reservationReleasedAt: providerRunner.reservationReleasedAt,
+        createdAt: providerRunner.createdAt,
+        updatedAt: providerRunner.updatedAt,
       })
       .returning();
 
@@ -37,7 +41,9 @@ export const providerRunnerFactory = Factory.define<RunnerInstance>(({onCreate})
     workspaceId: crypto.randomUUID(),
     provisionerId: crypto.randomUUID(),
     providerRunnerId: crypto.randomUUID(),
+    intendedReservationId: null,
     reservationId: null,
+    launchKind: 'manual',
     templateKey: 'linux',
     labels: ['linux'],
     state: 'running',

package/test/fixtures/late-runner-enrollment.ts

@@ -0,0 +1,103 @@
+import {extractDisplayPrefix, generateOpaqueToken, hashOpaqueToken} from '@shipfox/node-tokens';
+import {db} from '#db/db.js';
+import {deleteReservationsByIds} from '#db/reservations.js';
+import {reservations} from '#db/schema/reservations.js';
+import {runnerControlSessions} from '#db/schema/runner-control-sessions.js';
+import {providerRunners} from '#db/schema/runner-instances.js';
+
+export interface LateRunnerEnrollmentArrangement {
+  controlSessionToken: string;
+  provisionerId: string;
+  reservationId: string;
+  runnerInstanceId: string;
+  workspaceId: string;
+}
+
+export async function arrangeExpiredRunnerEnrollment(params: {
+  provisionerId: string;
+  workspaceId: string;
+}): Promise<LateRunnerEnrollmentArrangement> {
+  const reservationId = await insertReservation({
+    ...params,
+    expiresAt: new Date(Date.now() - 1_000),
+  });
+
+  return await createRunnerEnrollmentArrangement({...params, reservationId});
+}
+
+export async function arrangeDeletedRunnerEnrollment(params: {
+  provisionerId: string;
+  workspaceId: string;
+}): Promise<LateRunnerEnrollmentArrangement> {
+  const reservationId = await insertReservation({
+    ...params,
+    expiresAt: new Date(Date.now() + 60_000),
+  });
+  const deleted = await deleteReservationsByIds([reservationId]);
+  if (deleted !== 1) throw new Error('Expected reservation cleanup to delete one row');
+
+  // The provider can still enroll from a launch payload carrying the deleted reservation id.
+  return await createRunnerEnrollmentArrangement({...params, reservationId});
+}
+
+async function insertReservation(params: {
+  expiresAt: Date;
+  provisionerId: string;
+  workspaceId: string;
+}): Promise<string> {
+  const [reservation] = await db()
+    .insert(reservations)
+    .values({
+      workspaceId: params.workspaceId,
+      provisionerId: params.provisionerId,
+      requiredLabels: ['linux'],
+      count: 1,
+      expiresAt: params.expiresAt,
+    })
+    .returning({id: reservations.id});
+  if (!reservation) throw new Error('Expected reservation');
+
+  return reservation.id;
+}
+
+async function createRunnerEnrollmentArrangement(params: {
+  provisionerId: string;
+  reservationId: string;
+  workspaceId: string;
+}): Promise<LateRunnerEnrollmentArrangement> {
+  const providerRunnerId = `late-runner-${crypto.randomUUID()}`;
+  const controlSessionToken = generateOpaqueToken('runnerControlSession');
+  const [runner] = await db()
+    .insert(providerRunners)
+    .values({
+      provisionerId: params.provisionerId,
+      providerRunnerId,
+      intendedReservationId: params.reservationId,
+      templateKey: 'linux',
+      labels: ['linux'],
+      state: 'starting',
+      providerKind: 'docker',
+      protocolVersion: '1',
+      reportedAt: new Date(),
+    })
+    .returning({id: providerRunners.id});
+  if (!runner) throw new Error('Expected runner instance');
+
+  await db()
+    .insert(runnerControlSessions)
+    .values({
+      runnerInstanceId: runner.id,
+      provisionerId: params.provisionerId,
+      hashedToken: hashOpaqueToken(controlSessionToken),
+      prefix: extractDisplayPrefix(controlSessionToken),
+      expiresAt: new Date(Date.now() + 60_000),
+    });
+
+  return {
+    controlSessionToken,
+    provisionerId: params.provisionerId,
+    reservationId: params.reservationId,
+    runnerInstanceId: runner.id,
+    workspaceId: params.workspaceId,
+  };
+}

package/test/index.ts

@@ -23,3 +23,8 @@ export {
   mintRunnerSessionToken,
 } from './fixtures/auth.js';
 export {runnersTestAuthClient} from './fixtures/auth-inter-module.js';
+export {
+  arrangeDeletedRunnerEnrollment,
+  arrangeExpiredRunnerEnrollment,
+  type LateRunnerEnrollmentArrangement,
+} from './fixtures/late-runner-enrollment.js';