@shipfox/api-runners 12.3.0 → 12.4.0

package/src/db/runner-instances.test.ts

@@ -146,6 +146,28 @@ describe('reportRunnerInstances', () => {
     expect(rows[0]?.state).toBe('running');
   });
 
+  it('preserves launch kind when a provider report updates the runner row', async () => {
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'demand-runner',
+      launchKind: 'demand',
+    });
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [event({providerRunnerId: 'demand-runner', state: 'running'})],
+    });
+
+    const [row] = await db()
+      .select({launchKind: providerRunners.launchKind})
+      .from(providerRunners)
+      .where(eq(providerRunners.providerRunnerId, 'demand-runner'));
+    expect(row?.launchKind).toBe('demand');
+  });
+
   it('uses state progression to dedupe equal-timestamp provisioned runner reports', async () => {
     const reportedAt = new Date();
 
@@ -366,6 +388,58 @@ describe('reportRunnerInstances', () => {
     expect(rows[0]?.reservationId).toBe(reservationId);
   });
 
+  it('does not adopt a bound reservation id from an unclaimed report', async () => {
+    const reservationId = await createReservation(1, {kind: 'bound'});
+    await db()
+      .insert(providerRunners)
+      .values({
+        workspaceId,
+        provisionerId,
+        providerRunnerId: 'bound-reservation-runner',
+        state: 'starting',
+        reportedAt: new Date('2025-01-01T00:00:00.000Z'),
+      });
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({
+          providerRunnerId: 'bound-reservation-runner',
+          reservationId,
+          state: 'running',
+          reportedAt: new Date('2025-01-01T00:01:00.000Z'),
+        }),
+      ],
+    });
+
+    const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(rows[0]?.reservationId).toBeNull();
+  });
+
+  it('does not let reports exceed reservation capacity when creating projection rows', async () => {
+    const reservationId = await createReservation(1);
+
+    await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({providerRunnerId: 'provisioned-runner-1', reservationId}),
+        event({providerRunnerId: 'provisioned-runner-2', reservationId}),
+      ],
+    });
+
+    const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(rows.find((row) => row.providerRunnerId === 'provisioned-runner-1')?.reservationId).toBe(
+      reservationId,
+    );
+    expect(
+      rows.find((row) => row.providerRunnerId === 'provisioned-runner-2')?.reservationId,
+    ).toBeNull();
+  });
+
   it('does not let equal-timestamp lower-priority reports flip terminal state', async () => {
     const reservationId = await createReservation(1);
     const reportedAt = new Date('2025-01-01T00:00:00.000Z');
@@ -554,8 +628,15 @@ describe('reportRunnerInstances', () => {
     expect(active).toEqual([]);
   });
 
-  it('releases one reservation unit for a terminal unclaimed provisioned runner', async () => {
+  it('releases one reservation unit for a terminal runner with an intended reservation', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -571,6 +652,30 @@ describe('reportRunnerInstances', () => {
     expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
   });
 
+  it('does not let an unrecognized terminal report consume reservation capacity', async () => {
+    const reservationId = await createReservation(1);
+
+    const result = await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [event({providerRunnerId: 'unrecognized-runner', reservationId, state: 'failed'})],
+    });
+
+    const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
+    const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
+    expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
+    expect(reservationRows[0]?.count).toBe(1);
+    expect(providerRunnerRows).toHaveLength(1);
+    expect(providerRunnerRows[0]).toMatchObject({
+      providerRunnerId: 'unrecognized-runner',
+      reservationId: null,
+      intendedReservationId: null,
+      reservationReleasedAt: null,
+      state: 'failed',
+    });
+  });
+
   it('releases an intended reservation for a runner that dies before enrollment', async () => {
     const reservationId = await createReservation(2);
     const [runner] = await db()
@@ -741,6 +846,13 @@ describe('reportRunnerInstances', () => {
 
   it('releases a reservation only once across repeated terminal reports', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     await reportRunnerInstances({
       scope: 'workspace',
@@ -762,6 +874,13 @@ describe('reportRunnerInstances', () => {
 
   it('does not let a newer running report revive a terminal provisioned runner', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
     const failedAt = new Date();
     await reportRunnerInstances({
       scope: 'workspace',
@@ -800,6 +919,13 @@ describe('reportRunnerInstances', () => {
 
   it('tracks provider cleanup as terminated', async () => {
     const reservationId = await createReservation(2);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -821,6 +947,20 @@ describe('reportRunnerInstances', () => {
 
   it('releases multiple units from the same reservation in one batch', async () => {
     const reservationId = await createReservation(3);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-2',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -839,6 +979,13 @@ describe('reportRunnerInstances', () => {
 
   it('deletes a one-unit reservation instead of violating the positive count check', async () => {
     const reservationId = await createReservation(1);
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservationId,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -862,6 +1009,13 @@ describe('reportRunnerInstances', () => {
     });
     const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
     if (!reservation) throw new Error('Expected reservation');
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'provisioned-runner-1',
+      intendedReservationId: reservation.id,
+      state: 'running',
+    });
 
     const result = await reportRunnerInstances({
       scope: 'workspace',
@@ -1012,13 +1166,14 @@ describe('reportRunnerInstances', () => {
 
   async function createReservation(
     count: number,
-    overrides?: {workspaceId?: string; provisionerId?: string},
+    overrides?: {kind?: 'bound' | 'launch'; workspaceId?: string; provisionerId?: string},
   ): Promise<string> {
     await reservationFactory.create({
       workspaceId: overrides?.workspaceId ?? workspaceId,
       provisionerId: overrides?.provisionerId ?? provisionerId,
       requiredLabels: ['linux'],
       count,
+      kind: overrides?.kind ?? 'launch',
       expiresAt: new Date(Date.now() + 60_000),
     });
     const [reservation] = await db()
@@ -1157,6 +1312,106 @@ describe('listProvisionerTerminateIntents', () => {
     expect(result).toEqual([{providerRunnerId: 'provisioned-runner-1', reason: 'job-cancelled'}]);
   });
 
+  it('returns an activation-timeout intent for a stale demand-backed runner', async () => {
+    await createRunnerInstance({
+      providerRunnerId: 'stale-demand-runner',
+      launchKind: 'demand',
+      createdAt: new Date(Date.now() - 301_000),
+    });
+
+    const result = await db().transaction((tx) =>
+      listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
+    );
+    const [runner] = await providerRunnerRowsFor({workspaceId, provisionerId});
+
+    expect(result).toEqual([
+      {providerRunnerId: 'stale-demand-runner', reason: 'activation-timeout'},
+    ]);
+    expect(runner?.reservationReleasedAt).toBeInstanceOf(Date);
+  });
+
+  it('does not reclaim warm, manual, active, or already-activated runners', async () => {
+    const [liveReservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 1,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    if (!liveReservation) throw new Error('Expected live reservation');
+
+    const staleCreatedAt = new Date(Date.now() - 301_000);
+    await createRunnerInstance({
+      providerRunnerId: 'stale-warm-runner',
+      launchKind: 'warm',
+      createdAt: staleCreatedAt,
+    });
+    await createRunnerInstance({
+      providerRunnerId: 'stale-manual-runner',
+      launchKind: 'manual',
+      createdAt: staleCreatedAt,
+    });
+    await createRunnerInstance({
+      providerRunnerId: 'live-demand-runner',
+      launchKind: 'demand',
+      reservationId: liveReservation.id,
+      createdAt: staleCreatedAt,
+    });
+    await db()
+      .insert(providerRunners)
+      .values({
+        workspaceId,
+        provisionerId,
+        intendedReservationId: liveReservation.id,
+        providerRunnerId: 'live-intended-demand-runner',
+        launchKind: 'demand',
+        state: 'running',
+        labels: ['linux'],
+        reportedAt: staleCreatedAt,
+        createdAt: staleCreatedAt,
+        updatedAt: staleCreatedAt,
+      });
+    await createRunnerInstance({
+      providerRunnerId: 'activated-demand-runner',
+      launchKind: 'demand',
+      runnerSessionId: crypto.randomUUID(),
+      createdAt: staleCreatedAt,
+    });
+
+    const result = await listProvisionerTerminateIntents({workspaceId, provisionerId, limit: 1000});
+
+    expect(result).toEqual([]);
+  });
+
+  it('marks activation-timeout retries after releasing the runner reservation', async () => {
+    await createRunnerInstance({
+      providerRunnerId: 'retryable-demand-runner',
+      launchKind: 'demand',
+      createdAt: new Date(Date.now() - 301_000),
+    });
+
+    const first = await db().transaction((tx) =>
+      listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
+    );
+    const second = await db().transaction((tx) =>
+      listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
+    );
+
+    expect(first).toEqual([
+      {providerRunnerId: 'retryable-demand-runner', reason: 'activation-timeout'},
+    ]);
+    expect(second).toEqual([
+      {
+        providerRunnerId: 'retryable-demand-runner',
+        reason: 'activation-timeout',
+        activationTimeoutRetry: true,
+      },
+    ]);
+  });
+
   it('excludes active provisioned runners whose latest bound job is healthy', async () => {
     await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
     await insertRunningJobRow({
@@ -1272,12 +1527,20 @@ describe('listProvisionerTerminateIntents', () => {
     providerRunnerId: string;
     provisionerId?: string;
     state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
+    launchKind?: 'demand' | 'warm' | 'manual';
+    createdAt?: Date;
+    reservationId?: string | null;
+    runnerSessionId?: string | null;
   }) {
     return await providerRunnerFactory.create({
       workspaceId,
       provisionerId: params.provisionerId ?? provisionerId,
       providerRunnerId: params.providerRunnerId,
       state: params.state ?? 'running',
+      launchKind: params.launchKind ?? 'manual',
+      createdAt: params.createdAt ?? new Date(),
+      reservationId: params.reservationId ?? null,
+      runnerSessionId: params.runnerSessionId ?? null,
     });
   }
 });