@shipfox/api-runners 12.3.0 → 12.4.1

package/src/db/runner-instances.test.ts

@@ -7,6 +7,7 @@ import {
   countStaleEnrolledRunnerInstances,
   listActiveRunnerInstanceCountsByTemplateTx,
   listActiveRunnerInstances,
+  listProviderRunnerByPhaseMetrics,
   listProvisionerTerminateIntentRowsTx,
   listProvisionerTerminateIntents,
   reapStaleRunnerInstances,
@@ -146,6 +147,28 @@ describe('reportRunnerInstances', () => {
     expect(rows[0]?.state).toBe('running');
   });
 
+  it('preserves launch kind when a provider report updates the runner row', async () => {
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'demand-runner',
+      launchKind: 'demand',
+    });
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [event({providerRunnerId: 'demand-runner', state: 'running'})],
+    });
+
+    const [row] = await db()
+      .select({launchKind: providerRunners.launchKind})
+      .from(providerRunners)
+      .where(eq(providerRunners.providerRunnerId, 'demand-runner'));
+    expect(row?.launchKind).toBe('demand');
+  });
+
   it('uses state progression to dedupe equal-timestamp provisioned runner reports', async () => {
     const reportedAt = new Date();
 
@@ -366,6 +389,58 @@ describe('reportRunnerInstances', () => {
     expect(rows[0]?.reservationId).toBe(reservationId);
   });
 
+  it('does not adopt a bound reservation id from an unclaimed report', async () => {
+    const reservationId = await createReservation(1, {kind: 'bound'});
+    await db()
+      .insert(providerRunners)
+      .values({
+        workspaceId,
+        provisionerId,
+        providerRunnerId: 'bound-reservation-runner',
+        state: 'starting',
+        reportedAt: new Date('2025-01-01T00:00:00.000Z'),
+      });
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({
+          providerRunnerId: 'bound-reservation-runner',
+          reservationId,
+          state: 'running',
+          reportedAt: new Date('2025-01-01T00:01:00.000Z'),
+        }),
+      ],
+    });
+
+    const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(rows[0]?.reservationId).toBeNull();
+  });
+
+  it('does not let reports exceed reservation capacity when creating projection rows', async () => {
+    const reservationId = await createReservation(1);
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({providerRunnerId: 'provisioned-runner-1', reservationId}),
+        event({providerRunnerId: 'provisioned-runner-2', reservationId}),
+      ],
+    });
+
+    const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(rows.find((row) => row.providerRunnerId === 'provisioned-runner-1')?.reservationId).toBe(
+      reservationId,
+    );
+    expect(
+      rows.find((row) => row.providerRunnerId === 'provisioned-runner-2')?.reservationId,
+    ).toBeNull();
+  });
+
   it('does not let equal-timestamp lower-priority reports flip terminal state', async () => {
     const reservationId = await createReservation(1);
     const reportedAt = new Date('2025-01-01T00:00:00.000Z');
@@ -554,8 +629,15 @@ describe('reportRunnerInstances', () => {
     expect(active).toEqual([]);
   });
 
-  it('releases one reservation unit for a terminal unclaimed provisioned runner', async () => {
+  it('releases one reservation unit for a terminal runner with an intended reservation', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -571,6 +653,30 @@ describe('reportRunnerInstances', () => {
     expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
   });
 
+  it('does not let an unrecognized terminal report consume reservation capacity', async () => {
+    const reservationId = await createReservation(1);
+
+    const result = await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [event({providerRunnerId: 'unrecognized-runner', reservationId, state: 'failed'})],
+    });
+
+    const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
+    const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
+    expect(reservationRows[0]?.count).toBe(1);
+    expect(providerRunnerRows).toHaveLength(1);
+    expect(providerRunnerRows[0]).toMatchObject({
+      providerRunnerId: 'unrecognized-runner',
+      reservationId: null,
+      intendedReservationId: null,
+      reservationReleasedAt: null,
+      state: 'failed',
+    });
+  });
+
   it('releases an intended reservation for a runner that dies before enrollment', async () => {
     const reservationId = await createReservation(2);
     const [runner] = await db()
@@ -741,6 +847,13 @@ describe('reportRunnerInstances', () => {
 
   it('releases a reservation only once across repeated terminal reports', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     await reportRunnerInstances({
       scope: 'workspace',
@@ -762,6 +875,13 @@ describe('reportRunnerInstances', () => {
 
   it('does not let a newer running report revive a terminal provisioned runner', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
     const failedAt = new Date();
     await reportRunnerInstances({
       scope: 'workspace',
@@ -800,6 +920,13 @@ describe('reportRunnerInstances', () => {
 
   it('tracks provider cleanup as terminated', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -821,6 +948,20 @@ describe('reportRunnerInstances', () => {
 
   it('releases multiple units from the same reservation in one batch', async () => {
     const reservationId = await createReservation(3);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-2',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -839,6 +980,13 @@ describe('reportRunnerInstances', () => {
 
   it('deletes a one-unit reservation instead of violating the positive count check', async () => {
     const reservationId = await createReservation(1);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -862,6 +1010,13 @@ describe('reportRunnerInstances', () => {
     });
     const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
     if (!reservation) throw new Error('Expected reservation');
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservation.id,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -1012,13 +1167,14 @@ describe('reportRunnerInstances', () => {
 
   async function createReservation(
     count: number,
-    overrides?: {workspaceId?: string; provisionerId?: string},
+    overrides?: {kind?: 'bound' | 'launch'; workspaceId?: string; provisionerId?: string},
   ): Promise<string> {
     await reservationFactory.create({
       workspaceId: overrides?.workspaceId ?? workspaceId,
       provisionerId: overrides?.provisionerId ?? provisionerId,
       requiredLabels: ['linux'],
       count,
+      kind: overrides?.kind ?? 'launch',
       expiresAt: new Date(Date.now() + 60_000),
     });
     const [reservation] = await db()
@@ -1157,6 +1313,106 @@ describe('listProvisionerTerminateIntents', () => {
     expect(result).toEqual([{providerRunnerId: 'provisioned-runner-1', reason: 'job-cancelled'}]);
   });
 
+  it('returns an activation-timeout intent for a stale demand-backed runner', async () => {
+    await createRunnerInstance({
+      providerRunnerId: 'stale-demand-runner',
+      launchKind: 'demand',
+      createdAt: new Date(Date.now() - 301_000),
+    });
+
+    const result = await db().transaction((tx) =>
+      listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
+    );
+    const [runner] = await providerRunnerRowsFor({workspaceId, provisionerId});
+
+    expect(result).toEqual([
+      {providerRunnerId: 'stale-demand-runner', reason: 'activation-timeout'},
+    ]);
+    expect(runner?.reservationReleasedAt).toBeInstanceOf(Date);
+  });
+
+  it('does not reclaim warm, manual, active, or already-activated runners', async () => {
+    const [liveReservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 1,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    if (!liveReservation) throw new Error('Expected live reservation');
+
+    const staleCreatedAt = new Date(Date.now() - 301_000);
+    await createRunnerInstance({
+      providerRunnerId: 'stale-warm-runner',
+      launchKind: 'warm',
+      createdAt: staleCreatedAt,
+    });
+    await createRunnerInstance({
+      providerRunnerId: 'stale-manual-runner',
+      launchKind: 'manual',
+      createdAt: staleCreatedAt,
+    });
+    await createRunnerInstance({
+      providerRunnerId: 'live-demand-runner',
+      launchKind: 'demand',
+      reservationId: liveReservation.id,
+      createdAt: staleCreatedAt,
+    });
+    await db()
+      .insert(providerRunners)
+      .values({
+        workspaceId,
+        provisionerId,
+        intendedReservationId: liveReservation.id,
+        providerRunnerId: 'live-intended-demand-runner',
+        launchKind: 'demand',
+        state: 'running',
+        labels: ['linux'],
+        reportedAt: staleCreatedAt,
+        createdAt: staleCreatedAt,
+        updatedAt: staleCreatedAt,
+      });
+    await createRunnerInstance({
+      providerRunnerId: 'activated-demand-runner',
+      launchKind: 'demand',
+      runnerSessionId: crypto.randomUUID(),
+      createdAt: staleCreatedAt,
+    });
+
+    const result = await listProvisionerTerminateIntents({workspaceId, provisionerId, limit: 1000});
+
+    expect(result).toEqual([]);
+  });
+
+  it('marks activation-timeout retries after releasing the runner reservation', async () => {
+    await createRunnerInstance({
+      providerRunnerId: 'retryable-demand-runner',
+      launchKind: 'demand',
+      createdAt: new Date(Date.now() - 301_000),
+    });
+
+    const first = await db().transaction((tx) =>
+      listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
+    );
+    const second = await db().transaction((tx) =>
+      listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
+    );
+
+    expect(first).toEqual([
+      {providerRunnerId: 'retryable-demand-runner', reason: 'activation-timeout'},
+    ]);
+    expect(second).toEqual([
+      {
+        providerRunnerId: 'retryable-demand-runner',
+        reason: 'activation-timeout',
+        activationTimeoutRetry: true,
+      },
+    ]);
+  });
+
   it('excludes active provisioned runners whose latest bound job is healthy', async () => {
     await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
     await insertRunningJobRow({
@@ -1272,12 +1528,20 @@ describe('listProvisionerTerminateIntents', () => {
     providerRunnerId: string;
     provisionerId?: string;
     state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
+    launchKind?: 'demand' | 'warm' | 'manual';
+    createdAt?: Date;
+    reservationId?: string | null;
+    runnerSessionId?: string | null;
   }) {
     return await providerRunnerFactory.create({
       workspaceId,
       provisionerId: params.provisionerId ?? provisionerId,
       providerRunnerId: params.providerRunnerId,
       state: params.state ?? 'running',
+      launchKind: params.launchKind ?? 'manual',
+      createdAt: params.createdAt ?? new Date(),
+      reservationId: params.reservationId ?? null,
+      runnerSessionId: params.runnerSessionId ?? null,
     });
   }
 });
@@ -2277,6 +2541,97 @@ describe('countStaleEnrolledRunnerInstances', () => {
   }
 });
 
+describe('listProviderRunnerByPhaseMetrics', () => {
+  it('groups runners by the lifecycle phase that is currently blocking activation', async () => {
+    const provisionerId = crypto.randomUUID();
+    const provider = `metrics-test-${crypto.randomUUID()}`;
+    const reservationId = crypto.randomUUID();
+    const workspaceId = crypto.randomUUID();
+    const old = new Date(Date.now() - 120_000);
+
+    const createRunner = async (params: {
+      state: 'starting' | 'running';
+      launchKind: 'demand' | 'warm' | 'manual';
+      intendedReservationId?: string | null;
+      workspaceId?: string | null;
+      reservationId?: string | null;
+      assignedAt?: Date | null;
+      createdAt?: Date;
+    }) => {
+      const [runner] = await db()
+        .insert(providerRunners)
+        .values({
+          provisionerId,
+          providerRunnerId: crypto.randomUUID(),
+          providerKind: provider,
+          launchKind: params.launchKind,
+          intendedReservationId: params.intendedReservationId ?? null,
+          workspaceId: params.workspaceId ?? null,
+          reservationId: params.reservationId ?? null,
+          assignedAt: params.assignedAt ?? null,
+          state: params.state,
+          labels: ['linux'],
+          reportedAt: new Date(),
+          createdAt: params.createdAt ?? old,
+        })
+        .returning({id: providerRunners.id});
+      if (!runner) throw new Error('Expected runner instance');
+      return runner.id;
+    };
+
+    await createRunner({state: 'starting', launchKind: 'demand'});
+    const enrollmentRunnerId = await createRunner({state: 'starting', launchKind: 'warm'});
+    const assignmentRunnerId = await createRunner({
+      state: 'running',
+      launchKind: 'demand',
+      intendedReservationId: reservationId,
+    });
+    const activationRunnerId = await createRunner({
+      state: 'running',
+      launchKind: 'manual',
+      workspaceId,
+      reservationId,
+      assignedAt: old,
+    });
+    const idleRunnerId = await createRunner({state: 'running', launchKind: 'warm'});
+    const demandIdleRunnerId = await createRunner({state: 'running', launchKind: 'demand'});
+
+    await db()
+      .insert(runnerControlSessions)
+      .values(
+        [
+          enrollmentRunnerId,
+          assignmentRunnerId,
+          activationRunnerId,
+          idleRunnerId,
+          demandIdleRunnerId,
+        ].map((runnerInstanceId) => ({
+          runnerInstanceId,
+          provisionerId,
+          hashedToken: crypto.randomUUID(),
+          prefix: 'metrics-test',
+          expiresAt: new Date(Date.now() + 60_000),
+          createdAt: old,
+        })),
+      );
+
+    const metrics = await listProviderRunnerByPhaseMetrics();
+    const ownMetrics = metrics.filter((metric) => metric.provider === provider);
+
+    expect(ownMetrics).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({phase: 'control_session', launchKind: 'demand', count: 1}),
+        expect.objectContaining({phase: 'enrollment', launchKind: 'warm', count: 1}),
+        expect.objectContaining({phase: 'assignment', launchKind: 'demand', count: 1}),
+        expect.objectContaining({phase: 'activation', launchKind: 'manual', count: 1}),
+        expect.objectContaining({phase: 'idle', launchKind: 'warm', count: 1}),
+        expect.objectContaining({phase: 'idle', launchKind: 'demand', count: 1}),
+      ]),
+    );
+    expect(ownMetrics).toHaveLength(6);
+  });
+});
+
 describe('runner instance provider attachment', () => {
   it('updates the pre-created runner instance before its first lifecycle report', async () => {
     const provisionerId = crypto.randomUUID();