@shipfox/api-runners 12.5.0 → 12.7.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shipfox/api-runners",
   "license": "MIT",
-  "version": "12.5.0",
+  "version": "12.7.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ShipfoxHQ/shipfox.git",
@@ -25,7 +25,7 @@
     "@shipfox/api-auth-context": "12.2.0",
     "@shipfox/api-auth-dto": "12.0.0",
     "@shipfox/api-runners-dto": "12.4.0",
-    "@shipfox/api-workflows-dto": "12.5.0",
+    "@shipfox/api-workflows-dto": "12.7.0",
     "@shipfox/config": "1.2.4",
     "@shipfox/inter-module": "0.2.3",
     "@shipfox/node-drizzle": "0.3.5",

package/src/db/job-executions.test.ts

@@ -29,6 +29,7 @@ import {
 } from './job-executions.js';
 import {runnersOutbox} from './schema/outbox.js';
 import {pendingJobExecutions} from './schema/pending-job-executions.js';
+import {reservations} from './schema/reservations.js';
 import {providerRunners} from './schema/runner-instances.js';
 import {runnerSessions} from './schema/runner-sessions.js';
 import {runningJobExecutions} from './schema/running-job-executions.js';
@@ -969,6 +970,124 @@ describe('releaseJobExecution', () => {
     ).resolves.toBeUndefined();
   });
 
+  it('releases a terminal runner reservation after deleting its final lease', async () => {
+    const provisionerId = crypto.randomUUID();
+    const providerRunnerId = `provisioned-runner-${crypto.randomUUID()}`;
+    const [reservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: sessionLabels,
+        count: 1,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    if (!reservation) throw new Error('Expected reservation');
+
+    await db()
+      .update(runnerSessions)
+      .set({registrationTokenKind: 'ephemeral', maxClaims: 1, provisionerId, providerRunnerId})
+      .where(eq(runnerSessions.id, runnerSessionId));
+    await db().insert(providerRunners).values({
+      workspaceId,
+      provisionerId,
+      providerRunnerId,
+      reservationId: reservation.id,
+      runnerSessionId,
+      state: 'terminated',
+      reportedAt: new Date(),
+      terminatedAt: new Date(),
+      labels: sessionLabels,
+    });
+
+    const created = await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: 1});
+    expect(claimed?.jobExecutionId).toBe(created.jobExecutionId);
+
+    await releaseJobExecution({jobExecutionId: created.jobExecutionId});
+
+    expect(
+      await db().select().from(reservations).where(eq(reservations.id, reservation.id)),
+    ).toHaveLength(0);
+    const [runner] = await db()
+      .select({reservationReleasedAt: providerRunners.reservationReleasedAt})
+      .from(providerRunners)
+      .where(eq(providerRunners.providerRunnerId, providerRunnerId));
+    expect(runner?.reservationReleasedAt).toEqual(expect.any(Date));
+  });
+
+  it('rechecks terminal state after a concurrent terminal projection commits', async () => {
+    const provisionerId = crypto.randomUUID();
+    const providerRunnerId = `provisioned-runner-${crypto.randomUUID()}`;
+    const [reservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: sessionLabels,
+        count: 1,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    if (!reservation) throw new Error('Expected reservation');
+
+    await db()
+      .update(runnerSessions)
+      .set({registrationTokenKind: 'ephemeral', maxClaims: 1, provisionerId, providerRunnerId})
+      .where(eq(runnerSessions.id, runnerSessionId));
+    await db().insert(providerRunners).values({
+      workspaceId,
+      provisionerId,
+      providerRunnerId,
+      reservationId: reservation.id,
+      runnerSessionId,
+      state: 'running',
+      reportedAt: new Date(),
+      labels: sessionLabels,
+    });
+
+    const created = await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: 1});
+    expect(claimed?.jobExecutionId).toBe(created.jobExecutionId);
+
+    const terminalProjectionReady = deferred<void>();
+    const releaseTerminalProjection = deferred<void>();
+    const terminalProjection = db().transaction(async (tx) => {
+      await tx
+        .update(providerRunners)
+        .set({state: 'failed', failedAt: new Date(), updatedAt: new Date()})
+        .where(eq(providerRunners.providerRunnerId, providerRunnerId));
+      terminalProjectionReady.resolve();
+      await releaseTerminalProjection.promise;
+    });
+
+    await terminalProjectionReady.promise;
+    const cleanup = releaseJobExecution({jobExecutionId: created.jobExecutionId});
+    try {
+      await waitForLockWait({queryLike: '%runner_instances%'});
+
+      expect(
+        await db().select().from(reservations).where(eq(reservations.id, reservation.id)),
+      ).toHaveLength(1);
+    } finally {
+      releaseTerminalProjection.resolve();
+      await Promise.all([terminalProjection, cleanup]);
+    }
+
+    expect(
+      await db().select().from(reservations).where(eq(reservations.id, reservation.id)),
+    ).toHaveLength(0);
+    const [runner] = await db()
+      .select({
+        reservationReleasedAt: providerRunners.reservationReleasedAt,
+        state: providerRunners.state,
+      })
+      .from(providerRunners)
+      .where(eq(providerRunners.providerRunnerId, providerRunnerId));
+    expect(runner).toMatchObject({state: 'failed', reservationReleasedAt: expect.any(Date)});
+  });
+
   it('releases regardless of which session holds the lease', async () => {
     await pendingJobFactory.create({workspaceId});
     const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});

package/src/db/job-executions.ts

@@ -39,6 +39,7 @@ import {
 } from '#metrics/instance.js';
 import type {Tx} from './db.js';
 import {db} from './db.js';
+import {releaseTerminalRunnerInstanceReservationsByIds} from './reservations.js';
 import {runnersOutbox} from './schema/outbox.js';
 import {pendingJobExecutions} from './schema/pending-job-executions.js';
 import {providerRunners} from './schema/runner-instances.js';
@@ -405,6 +406,9 @@ async function touchRunnerSessionLiveness(params: {
 /**
  * Releases a job execution's lease when the orchestration workflow finalizes it: deletes the
  * running-job-execution row AND any lingering pending row for the same execution, in one tx.
+ * When the deleted lease was the last one for a terminal provider runner, it also releases the
+ * runner's reservation in the same runner-owned transaction. Terminal reports and workflow lease
+ * finalization remain independently retryable; they do not share a cross-module scope lock.
  * Idempotent (0-row no-op), no token scope (the workflow is authoritative), and
  * emits no event: the workflow already owns the outcome. Sweeping the pending row
  * too closes the at-least-once window where an enqueue retry left an orphan that a
@@ -421,9 +425,27 @@ export async function releaseJobExecution(params: {jobExecutionId: string}): Pro
     await tx
       .delete(pendingJobExecutions)
       .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));
-    await tx
+    const deletedRunningRows = await tx
       .delete(runningJobExecutions)
-      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));
+      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))
+      .returning({
+        workspaceId: runningJobExecutions.workspaceId,
+        provisionerId: runningJobExecutions.provisionerId,
+        providerRunnerId: runningJobExecutions.providerRunnerId,
+      });
+
+    const deletedRunningRow = deletedRunningRows[0];
+    if (deletedRunningRow?.provisionerId && deletedRunningRow.providerRunnerId) {
+      await releaseTerminalRunnerInstanceReservationsByIds(tx, {
+        workspaceId: null,
+        provisionerId: deletedRunningRow.provisionerId,
+        providerRunnerIds: [deletedRunningRow.providerRunnerId],
+        requireUnlinkedSession: false,
+        // Lock and re-check the runner row locally so terminal reporting remains retryable
+        // without a workflow/runner scope lock.
+        requireTerminalState: false,
+      });
+    }
   });
 }
 

package/src/db/provisioner-tokens.test.ts

@@ -10,6 +10,7 @@ import {
   touchProvisionerLastSeen,
 } from '#db/provisioner-tokens.js';
 import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
+import {reservations} from '#db/schema/reservations.js';
 import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
 import {runnerBootstrapTokens, runnerControlSessions} from '#db/schema/runner-control-sessions.js';
 import {providerRunners} from '#db/schema/runner-instances.js';
@@ -105,10 +106,22 @@ describe('provisioner token db', () => {
   it("cascades revocation to the provisioner's unclaimed runner credentials and sessions", async () => {
     const workspaceId = crypto.randomUUID();
     const provisioner = await provisionerTokenFactory.create({workspaceId});
+    const [reservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId: provisioner.id,
+        requiredLabels: ['linux'],
+        count: 1,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning();
+    if (!reservation) throw new Error('Expected reservation');
     const runner = await providerRunnerFactory.create({
       workspaceId,
       provisionerId: provisioner.id,
       runnerSessionId: null,
+      reservationId: reservation.id,
     });
     const future = new Date(Date.now() + 60_000);
     const tokenValue = (name: string) => hashOpaqueToken(`${name}-${crypto.randomUUID()}`);
@@ -192,6 +205,10 @@ describe('provisioner token db', () => {
       .select()
       .from(providerRunners)
       .where(eq(providerRunners.id, runner.id));
+    const [releasedReservation] = await db()
+      .select()
+      .from(reservations)
+      .where(eq(reservations.id, reservation.id));
     const [revokedSession] = await db()
       .select()
       .from(runnerSessions)
@@ -207,6 +224,8 @@ describe('provisioner token db', () => {
     expect(activation?.revokedAt).toBeInstanceOf(Date);
     expect(terminatedRunner).toMatchObject({state: 'terminated'});
     expect(terminatedRunner?.terminatedAt).toBeInstanceOf(Date);
+    expect(terminatedRunner?.reservationReleasedAt).toBeInstanceOf(Date);
+    expect(releasedReservation).toBeUndefined();
     expect(revokedSession?.revokedAt).toBeInstanceOf(Date);
     expect(preservedSession?.revokedAt).toBeNull();
   });

package/src/db/provisioner-tokens.ts

@@ -12,6 +12,7 @@ import type {
 } from '#core/entities/provisioner-token.js';
 import {ProvisionerAdminIdempotencyKeyReuseError} from '#core/errors.js';
 import {db} from './db.js';
+import {releaseTerminalRunnerInstanceReservationsByIds} from './reservations.js';
 import {runnersAdminCommandResults} from './schema/admin-command-results.js';
 import {runnersOutbox} from './schema/outbox.js';
 import {provisionerTokens, toProvisionerToken} from './schema/provisioner-tokens.js';
@@ -62,7 +63,7 @@ async function cascadeProvisionerRevocation(tx: Tx, provisionerId: string) {
         ),
       ),
     );
-  await tx
+  const terminatedRows = await tx
     .update(providerRunners)
     .set({state: 'terminated', terminatedAt: sql`now()`, updatedAt: sql`now()`})
     .where(
@@ -70,7 +71,17 @@ async function cascadeProvisionerRevocation(tx: Tx, provisionerId: string) {
         eq(providerRunners.provisionerId, provisionerId),
         isNull(providerRunners.runnerSessionId),
       ),
-    );
+    )
+    .returning({id: providerRunners.id});
+  const runnerInstanceIds = terminatedRows.map((row) => row.id);
+  if (runnerInstanceIds.length > 0) {
+    await releaseTerminalRunnerInstanceReservationsByIds(tx, {
+      workspaceId: null,
+      provisionerId,
+      runnerInstanceIds,
+      requireUnlinkedSession: false,
+    });
+  }
   await tx
     .update(runnerSessions)
     .set({revokedAt: sql`now()`})

package/src/db/reservations.test.ts

@@ -535,6 +535,66 @@ describe('pollDemandAndReserve', () => {
     expect(result.reservations).toEqual([]);
   });
 
+  it('deducts a terminal assigned runner reservation unit as pending', async () => {
+    const reservation = await createIntendedReservation({
+      workspaceId,
+      expiresAt: new Date(Date.now() + 60_000),
+    });
+    await createIdleRunner({
+      labels: ['linux'],
+      reservationId: reservation.id,
+      state: 'terminated',
+    });
+    await createPendingJobs(1, ['linux', 'gpu']);
+
+    const result = await pollDemandAndReserve({
+      workspaceId,
+      provisionerId,
+      maxReservations: 1,
+      ttlSeconds: 60,
+      templates: [template('linux-gpu', ['linux', 'gpu'], 1)],
+    });
+
+    expect(result.reservations).toEqual([]);
+  });
+
+  it('counts only active assigned runners when deducting provisioner reservations', async () => {
+    const [reservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 2,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    if (!reservation) throw new Error('Expected reservation');
+    await createIdleRunner({
+      labels: ['linux'],
+      reservationId: reservation.id,
+      state: 'running',
+    });
+    await createIdleRunner({
+      labels: ['linux'],
+      reservationId: reservation.id,
+      state: 'failed',
+    });
+    await createPendingJobs(2, ['linux', 'gpu']);
+
+    const result = await pollDemandAndReserve({
+      workspaceId,
+      provisionerId,
+      maxReservations: 2,
+      ttlSeconds: 60,
+      templates: [template('linux-gpu', ['linux', 'gpu'], 2)],
+    });
+
+    expect(result.reservations).toHaveLength(1);
+    expect(result.reservations[0]?.count).toBe(1);
+    expect(result.stats[0]).toMatchObject({queued: 2, reserved: 1});
+  });
+
   it('binds a runner whose intended reservation has passed its activation grace period', async () => {
     const intendedReservation = await createIntendedReservation({
       workspaceId,
@@ -1716,6 +1776,84 @@ describe('pollDemandAndReserve', () => {
     expect(rows[0]?.count).toBe(1);
   });
 
+  it('releases units from multiple reservations in one transaction', async () => {
+    const [first] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 3,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    const [second] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: ['linux', 'gpu'],
+        count: 2,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    if (!first || !second) throw new Error('Expected reservations');
+
+    const released = await db().transaction((tx) =>
+      releaseReservationUnits(tx, {
+        workspaceId,
+        provisionerId,
+        releases: [
+          {reservationId: first.id, count: 2},
+          {reservationId: second.id, count: 1},
+        ],
+      }),
+    );
+
+    const rows = await reservationsForTest();
+    expect(released).toBe(3);
+    expect(rows).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({id: first.id, count: 1}),
+        expect.objectContaining({id: second.id, count: 1}),
+      ]),
+    );
+  });
+
+  it('does not strand units when concurrent releases drain one reservation', async () => {
+    const [reservation] = await db()
+      .insert(reservations)
+      .values({
+        workspaceId,
+        provisionerId,
+        requiredLabels: ['linux'],
+        count: 2,
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({id: reservations.id});
+    if (!reservation) throw new Error('Expected reservation');
+
+    const [firstReleased, secondReleased] = await Promise.all([
+      db().transaction((tx) =>
+        releaseReservationUnits(tx, {
+          workspaceId,
+          provisionerId,
+          releases: [{reservationId: reservation.id, count: 1}],
+        }),
+      ),
+      db().transaction((tx) =>
+        releaseReservationUnits(tx, {
+          workspaceId,
+          provisionerId,
+          releases: [{reservationId: reservation.id, count: 1}],
+        }),
+      ),
+    ]);
+
+    expect(firstReleased + secondReleased).toBe(2);
+    expect(await reservationsForTest()).toHaveLength(0);
+  });
+
   it('deletes reservations when releasing all remaining units', async () => {
     await reservationFactory.create({
       workspaceId,