@shipfox/api-runners 12.2.0 → 12.4.0

package/src/db/runner-instances.test.ts

@@ -4,6 +4,7 @@ import {db} from '#db/db.js';
 import {createRunnerSessionConsumingEphemeralToken} from '#db/ephemeral-registration-tokens.js';
 import {
   attachRunnerInstanceProviderId,
+  countStaleEnrolledRunnerInstances,
   listActiveRunnerInstanceCountsByTemplateTx,
   listActiveRunnerInstances,
   listProvisionerTerminateIntentRowsTx,
@@ -14,6 +15,7 @@ import {
 } from '#db/runner-instances.js';
 import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
 import {reservations} from '#db/schema/reservations.js';
+import {runnerControlSessions} from '#db/schema/runner-control-sessions.js';
 import {providerRunners} from '#db/schema/runner-instances.js';
 import {runnerSessions} from '#db/schema/runner-sessions.js';
 import {runningJobExecutions} from '#db/schema/running-job-executions.js';
@@ -144,6 +146,28 @@ describe('reportRunnerInstances', () => {
     expect(rows[0]?.state).toBe('running');
   });
 
+  it('preserves launch kind when a provider report updates the runner row', async () => {
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'demand-runner',
+      launchKind: 'demand',
+    });
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [event({providerRunnerId: 'demand-runner', state: 'running'})],
+    });
+
+    const [row] = await db()
+      .select({launchKind: providerRunners.launchKind})
+      .from(providerRunners)
+      .where(eq(providerRunners.providerRunnerId, 'demand-runner'));
+    expect(row?.launchKind).toBe('demand');
+  });
+
   it('uses state progression to dedupe equal-timestamp provisioned runner reports', async () => {
     const reportedAt = new Date();
 
@@ -231,6 +255,191 @@ describe('reportRunnerInstances', () => {
     expect(rows[0]?.reason).toBe('fresh');
   });
 
+  it('preserves a rebound reservation through stale reports and releases it on termination', async () => {
+    const staleReservationId = await createReservation(1);
+    await db()
+      .update(reservations)
+      .set({expiresAt: new Date(Date.now() - 60_000)})
+      .where(eq(reservations.id, staleReservationId));
+    const reboundReservationId = await createReservation(2);
+    const initialReportedAt = new Date(Date.now() - 120_000);
+    const runningReportedAt = new Date(Date.now() - 60_000);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'rebound-runner',
+      reservationId: reboundReservationId,
+      state: 'running',
+      reportedAt: initialReportedAt,
+    });
+
+    const runningReport = await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({
+          providerRunnerId: 'rebound-runner',
+          reservationId: staleReservationId,
+          reportedAt: runningReportedAt,
+        }),
+      ],
+    });
+    const terminalReport = await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({
+          providerRunnerId: 'rebound-runner',
+          reservationId: staleReservationId,
+          state: 'failed',
+          reportedAt: new Date(runningReportedAt.getTime() + 1_000),
+        }),
+      ],
+    });
+
+    const [runner] = await db()
+      .select()
+      .from(providerRunners)
+      .where(eq(providerRunners.providerRunnerId, 'rebound-runner'));
+    const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
+    expect(runningReport).toEqual({
+      accepted: 1,
+      reservationsReleased: 0,
+      terminateIntentsHonored: [],
+    });
+    expect(terminalReport).toEqual({
+      accepted: 1,
+      reservationsReleased: 1,
+      terminateIntentsHonored: [],
+    });
+    expect(runner).toMatchObject({
+      reservationId: reboundReservationId,
+      reservationReleasedAt: expect.any(Date),
+    });
+    expect(
+      reservationRows.find((reservation) => reservation.id === reboundReservationId)?.count,
+    ).toBe(1);
+    expect(
+      reservationRows.find((reservation) => reservation.id === staleReservationId)?.count,
+    ).toBe(1);
+  });
+
+  it('keeps the stored reservation when a report carries a stale reservation id', async () => {
+    const staleReservationId = await createReservation(1);
+    const boundReservationId = await createReservation(1);
+    await db()
+      .insert(providerRunners)
+      .values({
+        workspaceId,
+        provisionerId,
+        reservationId: boundReservationId,
+        providerRunnerId: 'provisioned-runner-1',
+        state: 'running',
+        reportedAt: new Date('2025-01-01T00:00:00.000Z'),
+      });
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({
+          providerRunnerId: 'provisioned-runner-1',
+          reservationId: staleReservationId,
+          state: 'running',
+          reportedAt: new Date('2025-01-01T00:01:00.000Z'),
+        }),
+      ],
+    });
+
+    const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(rows[0]?.reservationId).toBe(boundReservationId);
+  });
+
+  it('adopts a reported reservation id when the runner carries none', async () => {
+    const reservationId = await createReservation(1);
+    await db()
+      .insert(providerRunners)
+      .values({
+        workspaceId,
+        provisionerId,
+        providerRunnerId: 'provisioned-runner-1',
+        state: 'starting',
+        reportedAt: new Date('2025-01-01T00:00:00.000Z'),
+      });
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({
+          providerRunnerId: 'provisioned-runner-1',
+          reservationId,
+          state: 'running',
+          reportedAt: new Date('2025-01-01T00:01:00.000Z'),
+        }),
+      ],
+    });
+
+    const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(rows[0]?.reservationId).toBe(reservationId);
+  });
+
+  it('does not adopt a bound reservation id from an unclaimed report', async () => {
+    const reservationId = await createReservation(1, {kind: 'bound'});
+    await db()
+      .insert(providerRunners)
+      .values({
+        workspaceId,
+        provisionerId,
+        providerRunnerId: 'bound-reservation-runner',
+        state: 'starting',
+        reportedAt: new Date('2025-01-01T00:00:00.000Z'),
+      });
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({
+          providerRunnerId: 'bound-reservation-runner',
+          reservationId,
+          state: 'running',
+          reportedAt: new Date('2025-01-01T00:01:00.000Z'),
+        }),
+      ],
+    });
+
+    const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(rows[0]?.reservationId).toBeNull();
+  });
+
+  it('does not let reports exceed reservation capacity when creating projection rows', async () => {
+    const reservationId = await createReservation(1);
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({providerRunnerId: 'provisioned-runner-1', reservationId}),
+        event({providerRunnerId: 'provisioned-runner-2', reservationId}),
+      ],
+    });
+
+    const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(rows.find((row) => row.providerRunnerId === 'provisioned-runner-1')?.reservationId).toBe(
+      reservationId,
+    );
+    expect(
+      rows.find((row) => row.providerRunnerId === 'provisioned-runner-2')?.reservationId,
+    ).toBeNull();
+  });
+
   it('does not let equal-timestamp lower-priority reports flip terminal state', async () => {
     const reservationId = await createReservation(1);
     const reportedAt = new Date('2025-01-01T00:00:00.000Z');
@@ -419,8 +628,15 @@ describe('reportRunnerInstances', () => {
     expect(active).toEqual([]);
   });
 
-  it('releases one reservation unit for a terminal unclaimed provisioned runner', async () => {
+  it('releases one reservation unit for a terminal runner with an intended reservation', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -436,6 +652,30 @@ describe('reportRunnerInstances', () => {
     expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
   });
 
+  it('does not let an unrecognized terminal report consume reservation capacity', async () => {
+    const reservationId = await createReservation(1);
+
+    const result = await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [event({providerRunnerId: 'unrecognized-runner', reservationId, state: 'failed'})],
+    });
+
+    const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
+    const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
+    expect(reservationRows[0]?.count).toBe(1);
+    expect(providerRunnerRows).toHaveLength(1);
+    expect(providerRunnerRows[0]).toMatchObject({
+      providerRunnerId: 'unrecognized-runner',
+      reservationId: null,
+      intendedReservationId: null,
+      reservationReleasedAt: null,
+      state: 'failed',
+    });
+  });
+
   it('releases an intended reservation for a runner that dies before enrollment', async () => {
     const reservationId = await createReservation(2);
     const [runner] = await db()
@@ -606,6 +846,13 @@ describe('reportRunnerInstances', () => {
 
   it('releases a reservation only once across repeated terminal reports', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     await reportRunnerInstances({
       scope: 'workspace',
@@ -627,6 +874,13 @@ describe('reportRunnerInstances', () => {
 
   it('does not let a newer running report revive a terminal provisioned runner', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
     const failedAt = new Date();
     await reportRunnerInstances({
       scope: 'workspace',
@@ -665,6 +919,13 @@ describe('reportRunnerInstances', () => {
 
   it('tracks provider cleanup as terminated', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -686,6 +947,20 @@ describe('reportRunnerInstances', () => {
 
   it('releases multiple units from the same reservation in one batch', async () => {
     const reservationId = await createReservation(3);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-2',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -704,6 +979,13 @@ describe('reportRunnerInstances', () => {
 
   it('deletes a one-unit reservation instead of violating the positive count check', async () => {
     const reservationId = await createReservation(1);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -727,6 +1009,13 @@ describe('reportRunnerInstances', () => {
     });
     const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
     if (!reservation) throw new Error('Expected reservation');
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservation.id,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -877,13 +1166,14 @@ describe('reportRunnerInstances', () => {
 
   async function createReservation(
     count: number,
-    overrides?: {workspaceId?: string; provisionerId?: string},
+    overrides?: {kind?: 'bound' | 'launch'; workspaceId?: string; provisionerId?: string},
   ): Promise<string> {
     await reservationFactory.create({
       workspaceId: overrides?.workspaceId ?? workspaceId,
       provisionerId: overrides?.provisionerId ?? provisionerId,
       requiredLabels: ['linux'],
       count,
+      kind: overrides?.kind ?? 'launch',
       expiresAt: new Date(Date.now() + 60_000),
     });
     const [reservation] = await db()
@@ -1022,6 +1312,106 @@ describe('listProvisionerTerminateIntents', () => {
     expect(result).toEqual([{providerRunnerId: 'provisioned-runner-1', reason: 'job-cancelled'}]);
   });
 
+  it('returns an activation-timeout intent for a stale demand-backed runner', async () => {
+    await createRunnerInstance({
+      providerRunnerId: 'stale-demand-runner',
+      launchKind: 'demand',
+      createdAt: new Date(Date.now() - 301_000),
+    });
+
+    const result = await db().transaction((tx) =>
+      listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
+    );
+    const [runner] = await providerRunnerRowsFor({workspaceId, provisionerId});
+
+    expect(result).toEqual([
+      {providerRunnerId: 'stale-demand-runner', reason: 'activation-timeout'},
+    ]);
+    expect(runner?.reservationReleasedAt).toBeInstanceOf(Date);
+  });
+
+  it('does not reclaim warm, manual, active, or already-activated runners', async () => {
+    const [liveReservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 1,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    if (!liveReservation) throw new Error('Expected live reservation');
+
+    const staleCreatedAt = new Date(Date.now() - 301_000);
+    await createRunnerInstance({
+      providerRunnerId: 'stale-warm-runner',
+      launchKind: 'warm',
+      createdAt: staleCreatedAt,
+    });
+    await createRunnerInstance({
+      providerRunnerId: 'stale-manual-runner',
+      launchKind: 'manual',
+      createdAt: staleCreatedAt,
+    });
+    await createRunnerInstance({
+      providerRunnerId: 'live-demand-runner',
+      launchKind: 'demand',
+      reservationId: liveReservation.id,
+      createdAt: staleCreatedAt,
+    });
+    await db()
+      .insert(providerRunners)
+      .values({
+        workspaceId,
+        provisionerId,
+        intendedReservationId: liveReservation.id,
+        providerRunnerId: 'live-intended-demand-runner',
+        launchKind: 'demand',
+        state: 'running',
+        labels: ['linux'],
+        reportedAt: staleCreatedAt,
+        createdAt: staleCreatedAt,
+        updatedAt: staleCreatedAt,
+      });
+    await createRunnerInstance({
+      providerRunnerId: 'activated-demand-runner',
+      launchKind: 'demand',
+      runnerSessionId: crypto.randomUUID(),
+      createdAt: staleCreatedAt,
+    });
+
+    const result = await listProvisionerTerminateIntents({workspaceId, provisionerId, limit: 1000});
+
+    expect(result).toEqual([]);
+  });
+
+  it('marks activation-timeout retries after releasing the runner reservation', async () => {
+    await createRunnerInstance({
+      providerRunnerId: 'retryable-demand-runner',
+      launchKind: 'demand',
+      createdAt: new Date(Date.now() - 301_000),
+    });
+
+    const first = await db().transaction((tx) =>
+      listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
+    );
+    const second = await db().transaction((tx) =>
+      listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
+    );
+
+    expect(first).toEqual([
+      {providerRunnerId: 'retryable-demand-runner', reason: 'activation-timeout'},
+    ]);
+    expect(second).toEqual([
+      {
+        providerRunnerId: 'retryable-demand-runner',
+        reason: 'activation-timeout',
+        activationTimeoutRetry: true,
+      },
+    ]);
+  });
+
   it('excludes active provisioned runners whose latest bound job is healthy', async () => {
     await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
     await insertRunningJobRow({
@@ -1137,12 +1527,20 @@ describe('listProvisionerTerminateIntents', () => {
     providerRunnerId: string;
     provisionerId?: string;
     state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
+    launchKind?: 'demand' | 'warm' | 'manual';
+    createdAt?: Date;
+    reservationId?: string | null;
+    runnerSessionId?: string | null;
   }) {
     return await providerRunnerFactory.create({
       workspaceId,
       provisionerId: params.provisionerId ?? provisionerId,
       providerRunnerId: params.providerRunnerId,
       state: params.state ?? 'running',
+      launchKind: params.launchKind ?? 'manual',
+      createdAt: params.createdAt ?? new Date(),
+      reservationId: params.reservationId ?? null,
+      runnerSessionId: params.runnerSessionId ?? null,
     });
   }
 });
@@ -2046,6 +2444,102 @@ describe('reconcileRunnerInstances', () => {
   }
 });
 
+describe('countStaleEnrolledRunnerInstances', () => {
+  let provisionerId: string;
+
+  beforeEach(() => {
+    provisionerId = crypto.randomUUID();
+  });
+
+  function staleAt(): Date {
+    return new Date(Date.now() - 120_000);
+  }
+
+  it('counts only stale reported runners with a live control session and no assignment', async () => {
+    const baseline = await countStaleEnrolledRunnerInstances({graceSeconds: 60});
+    const stale = await createRunner({updatedAt: staleAt()});
+    await createControlSession(stale);
+
+    const fresh = await createRunner({updatedAt: new Date()});
+    await createControlSession(fresh);
+
+    const recentlyMutated = await createRunner({reportedAt: staleAt(), updatedAt: new Date()});
+    await createControlSession(recentlyMutated);
+
+    const freshReport = await createRunner({reportedAt: new Date(), updatedAt: staleAt()});
+    await createControlSession(freshReport);
+
+    const assigned = await createRunner({workspaceId: crypto.randomUUID(), updatedAt: staleAt()});
+    await createControlSession(assigned);
+
+    const activated = await createRunner({
+      runnerSessionId: crypto.randomUUID(),
+      updatedAt: staleAt(),
+    });
+    await createControlSession(activated);
+
+    await createRunner({updatedAt: staleAt()});
+    const expiredControlSession = await createRunner({updatedAt: staleAt()});
+    await createControlSession(expiredControlSession, {
+      expiresAt: new Date(Date.now() - 1_000),
+    });
+    const closedControlSession = await createRunner({updatedAt: staleAt()});
+    await createControlSession(closedControlSession, {
+      closedAt: new Date(),
+    });
+    const stopped = await createRunner({state: 'stopped', updatedAt: staleAt()});
+    await createControlSession(stopped);
+
+    const count = await countStaleEnrolledRunnerInstances({graceSeconds: 60});
+
+    expect(count - baseline).toBe(2);
+  });
+
+  async function createRunner(params: {
+    state?: 'running' | 'stopped';
+    workspaceId?: string | null;
+    runnerSessionId?: string | null;
+    reportedAt?: Date;
+    updatedAt?: Date;
+  }) {
+    const reportedAt = params.reportedAt ?? params.updatedAt ?? new Date();
+    const [runner] = await db()
+      .insert(providerRunners)
+      .values({
+        workspaceId: params.workspaceId ?? null,
+        provisionerId,
+        providerRunnerId: crypto.randomUUID(),
+        templateKey: 'linux',
+        labels: ['linux'],
+        state: params.state ?? 'running',
+        runnerSessionId: params.runnerSessionId ?? null,
+        providerKind: 'docker',
+        reportedAt,
+        updatedAt: params.updatedAt ?? reportedAt,
+      })
+      .returning({id: providerRunners.id});
+    if (!runner) throw new Error('Expected runner instance');
+    return runner.id;
+  }
+
+  async function createControlSession(
+    runnerInstanceId: string,
+    params: {expiresAt?: Date; closedAt?: Date} = {},
+  ) {
+    await db()
+      .insert(runnerControlSessions)
+      .values({
+        runnerInstanceId,
+        provisionerId,
+        hashedToken: crypto.randomUUID(),
+        prefix: 'test',
+        expiresAt: params.expiresAt ?? new Date(Date.now() + 60_000),
+        closedAt: params.closedAt,
+        closeReason: params.closedAt ? 'test' : null,
+      });
+  }
+});
+
 describe('runner instance provider attachment', () => {
   it('updates the pre-created runner instance before its first lifecycle report', async () => {
     const provisionerId = crypto.randomUUID();