@shipfox/api-runners 12.7.0 → 13.1.0

package/src/db/job-executions.ts

@@ -1,7 +1,6 @@
 import {
   RUNNER_JOB_CLAIMED,
   RUNNER_JOB_LEASE_EXPIRED,
-  RUNNER_JOB_QUEUED,
   type RunnersEventMap,
   type RunnerToolCapabilitiesDto,
 } from '@shipfox/api-runners-dto';
@@ -39,9 +38,14 @@ import {
 } from '#metrics/instance.js';
 import type {Tx} from './db.js';
 import {db} from './db.js';
-import {releaseTerminalRunnerInstanceReservationsByIds} from './reservations.js';
+import {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';
+import {
+  releaseReservationUnits,
+  releaseTerminalRunnerInstanceReservationsByIds,
+} from './reservations.js';
 import {runnersOutbox} from './schema/outbox.js';
 import {pendingJobExecutions} from './schema/pending-job-executions.js';
+import {reservations} from './schema/reservations.js';
 import {providerRunners} from './schema/runner-instances.js';
 import {runnerSessions} from './schema/runner-sessions.js';
 import {runningJobExecutions} from './schema/running-job-executions.js';
@@ -54,6 +58,39 @@ async function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {
   );
 }
 
+async function releaseReservationsForTerminalRunningRows(
+  tx: Tx,
+  rows: ReadonlyArray<{
+    provisionerId: string | null;
+    providerRunnerId: string | null;
+  }>,
+): Promise<void> {
+  // Converge after the lease fact changes. The reservation helper locks each provider runner
+  // and releases only when the runner is terminal and no uncancelled lease remains.
+  const providerRunnerIdsByProvisionerId = new Map<string, Set<string>>();
+  for (const row of rows) {
+    if (row.provisionerId === null || row.providerRunnerId === null) continue;
+    const providerRunnerIds =
+      providerRunnerIdsByProvisionerId.get(row.provisionerId) ?? new Set<string>();
+    providerRunnerIds.add(row.providerRunnerId);
+    providerRunnerIdsByProvisionerId.set(row.provisionerId, providerRunnerIds);
+  }
+
+  for (const provisionerId of [...providerRunnerIdsByProvisionerId.keys()].sort()) {
+    const providerRunnerIds = providerRunnerIdsByProvisionerId.get(provisionerId);
+    if (!providerRunnerIds) continue;
+    await releaseTerminalRunnerInstanceReservationsByIds(tx, {
+      workspaceId: null,
+      provisionerId,
+      providerRunnerIds: [...providerRunnerIds].sort(),
+      requireUnlinkedSession: false,
+      // Lock and re-check the runner row locally so lease finalization remains retryable
+      // without a workflow/runner scope lock.
+      requireTerminalState: false,
+    });
+  }
+}
+
 export interface EnqueueJobExecutionParams {
   workspaceId: string;
   workflowRunId: string;
@@ -62,6 +99,7 @@ export interface EnqueueJobExecutionParams {
   jobExecutionId: string;
   projectId: string;
   requiredLabels: string[];
+  queuedAt: Date;
 }
 
 export async function getWorkspaceJobCounts(params: {
@@ -92,11 +130,9 @@ export async function getWorkspaceJobCounts(params: {
   }));
 }
 
-// Idempotent while the job execution is still pending: a duplicate jobExecutionId already in
-// `runners_pending_jobs` is a no-op. Temporal retries the enqueue activity at-least-once, so a
-// unique-violation throw on a retry-after-lost-result would permanently fail a healthy execution.
-// The per-execution advisory lock serializes retries with lease expiry/reconciliation, and the
-// durable lease-expired event prevents a retry from re-queueing an execution already reaped.
+// The workflows outbox delivers queue facts at least once. The per-execution advisory lock and
+// pending/running checks make replay a no-op across the claim transition. A durable lease-expired
+// event also prevents a delayed replay from resurrecting an execution already reaped locally.
 export async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {
   const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];
   if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();
@@ -104,6 +140,13 @@ export async function enqueueJobExecution(params: EnqueueJobExecutionParams): Pr
   const enqueued = await db().transaction(async (tx) => {
     await lockJobExecution(tx, params.jobExecutionId);
 
+    const [running] = await tx
+      .select({jobExecutionId: runningJobExecutions.jobExecutionId})
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))
+      .limit(1);
+    if (running) return false;
+
     const [leaseExpired] = await tx
       .select({id: runnersOutbox.id})
       .from(runnersOutbox)
@@ -126,26 +169,12 @@ export async function enqueueJobExecution(params: EnqueueJobExecutionParams): Pr
         jobExecutionId: params.jobExecutionId,
         projectId: params.projectId,
         requiredLabels,
+        createdAt: params.queuedAt,
       })
       .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})
       .returning({createdAt: pendingJobExecutions.createdAt});
 
-    // A retry that hits the conflict inserts nothing: the first enqueue already
-    // emitted the queued event (durably, in the outbox), so re-emitting would
-    // only add a redundant row the subscriber coalesces away. Skip it.
-    if (!inserted) return false;
-
-    await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {
-      type: RUNNER_JOB_QUEUED,
-      payload: {
-        workflowRunId: params.workflowRunId,
-        workflowRunAttemptId: params.workflowRunAttemptId,
-        jobId: params.jobId,
-        jobExecutionId: params.jobExecutionId,
-        queuedAt: inserted.createdAt.toISOString(),
-      },
-    });
-    return true;
+    return inserted !== undefined;
   });
 
   if (enqueued) jobExecutionEnqueuedCount.add(1);
@@ -238,6 +267,32 @@ export async function claimPendingJobExecution(params: {
       providerRunnerId = session.providerRunnerId;
     }
 
+    const runnerInstanceCondition = runnerInstanceId
+      ? eq(providerRunners.id, runnerInstanceId)
+      : provisionerId && providerRunnerId
+        ? and(
+            eq(providerRunners.provisionerId, provisionerId),
+            eq(providerRunners.providerRunnerId, providerRunnerId),
+          )
+        : null;
+    if (runnerInstanceCondition && provisionerId) {
+      const [runner] = await tx
+        .select({
+          reservationId: providerRunners.reservationId,
+          intendedReservationId: providerRunners.intendedReservationId,
+        })
+        .from(providerRunners)
+        .where(runnerInstanceCondition)
+        .limit(1);
+      await lockRunnerReservationAdvisoryKeysTx(tx, {
+        provisionerId,
+        reservationIds: [runner?.reservationId, runner?.intendedReservationId].filter(
+          (reservationId): reservationId is string =>
+            reservationId !== null && reservationId !== undefined,
+        ),
+      });
+    }
+
     // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker
     // for rows sharing a created_at within a batch. Lock only the FIFO candidate before
     // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this
@@ -304,14 +359,6 @@ export async function claimPendingJobExecution(params: {
       launchKind: params.maxClaims === null ? 'manual' : 'unknown',
     };
 
-    const runnerInstanceCondition = runnerInstanceId
-      ? eq(providerRunners.id, runnerInstanceId)
-      : provisionerId && providerRunnerId
-        ? and(
-            eq(providerRunners.provisionerId, provisionerId),
-            eq(providerRunners.providerRunnerId, providerRunnerId),
-          )
-        : null;
     if (runnerInstanceCondition) {
       const [claimedRunner] = await tx
         .update(providerRunners)
@@ -326,6 +373,8 @@ export async function claimPendingJobExecution(params: {
           provider: providerRunners.providerKind,
           launchKind: providerRunners.launchKind,
           runnerInstanceId: providerRunners.id,
+          reservationId: providerRunners.reservationId,
+          intendedReservationId: providerRunners.intendedReservationId,
           sessionCreatedAtEpochMs: sql<number | null>`(
             select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000
             from ${runnerSessions}
@@ -336,6 +385,47 @@ export async function claimPendingJobExecution(params: {
         queueTimeObservation.provider = claimedRunner.provider;
         queueTimeObservation.launchKind = claimedRunner.launchKind;
       }
+      if (claimedRunner?.isFirstClaim) {
+        const [releasedRunner] = await tx
+          .update(providerRunners)
+          .set({
+            intendedReservationId: null,
+            reservationReleasedAt: sql`now()`,
+            updatedAt: sql`now()`,
+          })
+          .where(
+            and(
+              eq(providerRunners.id, claimedRunner.runnerInstanceId),
+              or(
+                isNotNull(providerRunners.reservationId),
+                isNotNull(providerRunners.intendedReservationId),
+              ),
+              isNull(providerRunners.reservationReleasedAt),
+            ),
+          )
+          .returning({
+            provisionerId: providerRunners.provisionerId,
+          });
+        const reservationId = claimedRunner.intendedReservationId ?? claimedRunner.reservationId;
+        if (releasedRunner?.provisionerId && reservationId) {
+          const [reservation] = await tx
+            .select({workspaceId: reservations.workspaceId})
+            .from(reservations)
+            .where(
+              and(
+                eq(reservations.id, reservationId),
+                eq(reservations.provisionerId, releasedRunner.provisionerId),
+              ),
+            )
+            .limit(1);
+          if (reservation)
+            await releaseReservationUnits(tx, {
+              workspaceId: reservation.workspaceId,
+              provisionerId: releasedRunner.provisionerId,
+              releases: [{reservationId, count: 1}],
+            });
+        }
+      }
       if (
         claimedRunner?.isFirstClaim &&
         claimedRunner.firstClaimedAt !== null &&
@@ -403,64 +493,18 @@ async function touchRunnerSessionLiveness(params: {
     );
 }
 
-/**
- * Releases a job execution's lease when the orchestration workflow finalizes it: deletes the
- * running-job-execution row AND any lingering pending row for the same execution, in one tx.
- * When the deleted lease was the last one for a terminal provider runner, it also releases the
- * runner's reservation in the same runner-owned transaction. Terminal reports and workflow lease
- * finalization remain independently retryable; they do not share a cross-module scope lock.
- * Idempotent (0-row no-op), no token scope (the workflow is authoritative), and
- * emits no event: the workflow already owns the outcome. Sweeping the pending row
- * too closes the at-least-once window where an enqueue retry left an orphan that a
- * later claim would otherwise pick up for an already-finished job execution.
- */
-export async function releaseJobExecution(params: {jobExecutionId: string}): Promise<void> {
-  await db().transaction(async (tx) => {
-    await lockJobExecution(tx, params.jobExecutionId);
-
-    // Delete pending before running to match `claimPendingJobExecution`'s lock-acquisition
-    // order (it locks the pending row first, then the running row). A concurrent
-    // claim picking up an orphan pending row for this same job execution would otherwise
-    // deadlock against the reverse order here.
-    await tx
-      .delete(pendingJobExecutions)
-      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));
-    const deletedRunningRows = await tx
-      .delete(runningJobExecutions)
-      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))
-      .returning({
-        workspaceId: runningJobExecutions.workspaceId,
-        provisionerId: runningJobExecutions.provisionerId,
-        providerRunnerId: runningJobExecutions.providerRunnerId,
-      });
-
-    const deletedRunningRow = deletedRunningRows[0];
-    if (deletedRunningRow?.provisionerId && deletedRunningRow.providerRunnerId) {
-      await releaseTerminalRunnerInstanceReservationsByIds(tx, {
-        workspaceId: null,
-        provisionerId: deletedRunningRow.provisionerId,
-        providerRunnerIds: [deletedRunningRow.providerRunnerId],
-        requireUnlinkedSession: false,
-        // Lock and re-check the runner row locally so terminal reporting remains retryable
-        // without a workflow/runner scope lock.
-        requireTerminalState: false,
-      });
-    }
-  });
-}
-
 /**
  * Reaps stale leases (bounded by `limit`), emitting one
  * `runners.job.lease_expired` event per reaped job execution.
  *
  * The cutoff is re-checked in the DELETE, not just the locking subquery, so a
  * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its
- * pending row: a failed best-effort `releaseJobExecution` would otherwise leave an orphan
- * that a later claim re-runs as an already-finished job execution.
+ * pending row so an at-least-once queue replay cannot leave an orphan.
  *
- * Locks pending-then-running to match `claimPendingJobExecution`, `releaseJobExecution`, and
- * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock running
- * rows first; the running-row DELETE re-checks the stale predicate after the pending-row sweep.
+ * Locks pending-then-running to match `claimPendingJobExecution` and
+ * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock
+ * running rows first; the running-row DELETE re-checks the stale predicate after the
+ * pending-row sweep.
  */
 export async function expireStuckJobExecutions(params: {
   thresholdSeconds: number;
@@ -537,10 +581,14 @@ export async function expireStuckJobExecutions(params: {
         workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,
         jobId: runningJobExecutions.jobId,
         jobExecutionId: runningJobExecutions.jobExecutionId,
+        provisionerId: runningJobExecutions.provisionerId,
+        providerRunnerId: runningJobExecutions.providerRunnerId,
       });
 
     if (deleted.length === 0) return [];
 
+    await releaseReservationsForTerminalRunningRows(tx, deleted);
+
     await writeOutboxEvents<RunnersEventMap>(
       tx,
       runnersOutbox,
@@ -555,7 +603,12 @@ export async function expireStuckJobExecutions(params: {
       })),
     );
 
-    return deleted;
+    return deleted.map(({workflowRunId, workflowRunAttemptId, jobId, jobExecutionId}) => ({
+      workflowRunId,
+      workflowRunAttemptId,
+      jobId,
+      jobExecutionId,
+    }));
   });
 
   if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);
@@ -803,8 +856,8 @@ export async function recordHeartbeat(params: {
 
 /**
  * Reconciles a terminal job execution with runner state in one transaction: removes its pending
- * queue row and requests cancellation on its running lease, if either exists. The operation is
- * idempotent and preserves the first cancellation-request timestamp.
+ * queue row, requests cancellation on its running lease, and converges any linked reservation.
+ * The operation is idempotent and preserves the first cancellation-request timestamp.
  */
 export async function reconcileTerminalJobExecution(params: {
   jobExecutionId: string;
@@ -818,25 +871,17 @@ export async function reconcileTerminalJobExecution(params: {
     await tx
       .delete(pendingJobExecutions)
       .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));
-    await tx
+    const cancelledRunningRows = await tx
       .update(runningJobExecutions)
       .set({
         cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,
       })
-      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));
-  });
-}
-
-export async function cancelRunnerJobs(params: {jobIds: string[]}): Promise<void> {
-  if (params.jobIds.length === 0) return;
+      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))
+      .returning({
+        provisionerId: runningJobExecutions.provisionerId,
+        providerRunnerId: runningJobExecutions.providerRunnerId,
+      });
 
-  await db().transaction(async (tx) => {
-    await tx.delete(pendingJobExecutions).where(inArray(pendingJobExecutions.jobId, params.jobIds));
-    await tx
-      .update(runningJobExecutions)
-      .set({
-        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,
-      })
-      .where(inArray(runningJobExecutions.jobId, params.jobIds));
+    await releaseReservationsForTerminalRunningRows(tx, cancelledRunningRows);
   });
 }

package/src/db/reservations.ts

@@ -939,7 +939,9 @@ export async function releaseTerminalRunnerInstanceReservationsByIds(
     params.workspaceId === null
       ? sql``
       : sql`and ${eq(reservations.workspaceId, params.workspaceId)}`;
-  const noRunningJobPredicate = notExists(
+  // Cancellation keeps the lease row so the provisioner can observe its terminate intent.
+  // Once the runner is terminal, only an uncancelled lease should block reservation release.
+  const noUncancelledRunningJobPredicate = notExists(
     tx
       .select({id: runningJobExecutions.id})
       .from(runningJobExecutions)
@@ -947,6 +949,7 @@ export async function releaseTerminalRunnerInstanceReservationsByIds(
         and(
           eq(runningJobExecutions.provisionerId, params.provisionerId),
           eq(runningJobExecutions.providerRunnerId, providerRunners.providerRunnerId),
+          isNull(runningJobExecutions.cancellationRequestedAt),
           params.workspaceId === null
             ? undefined
             : eq(runningJobExecutions.workspaceId, params.workspaceId),
@@ -1103,7 +1106,7 @@ export async function releaseTerminalRunnerInstanceReservationsByIds(
         params.requireUnlinkedSession === false
           ? undefined
           : isNull(providerRunners.runnerSessionId),
-        noRunningJobPredicate,
+        noUncancelledRunningJobPredicate,
         reportFreshnessPredicate,
         isNull(providerRunners.reservationReleasedAt),
       ),
@@ -1137,7 +1140,7 @@ export async function releaseTerminalRunnerInstanceReservationsByIds(
         params.requireUnlinkedSession === false
           ? undefined
           : isNull(providerRunners.runnerSessionId),
-        noRunningJobPredicate,
+        noUncancelledRunningJobPredicate,
         reportFreshnessPredicate,
         isNull(providerRunners.reservationReleasedAt),
       ),

package/src/db/runner-instances.test.ts

@@ -1145,6 +1145,106 @@ describe('reportRunnerInstances', () => {
     expect(reservation?.count).toBe(1);
   });
 
+  it('keeps a reservation when a terminal runner has cancelled and uncancelled jobs', async () => {
+    const reservationId = await createReservation(1);
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'mixed-terminal-runner',
+      reservationId,
+      runnerSessionId: runnerSession.id,
+      state: 'running',
+    });
+    await insertRunningJobRow({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'mixed-terminal-runner',
+      startedAt: new Date('2025-01-01T00:00:00.000Z'),
+      cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
+    });
+    await insertRunningJobRow({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'mixed-terminal-runner',
+      startedAt: new Date('2025-01-01T00:02:00.000Z'),
+    });
+
+    const result = await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({
+          providerRunnerId: 'mixed-terminal-runner',
+          reservationId,
+          state: 'terminated',
+          runnerSessionId: runnerSession.id,
+        }),
+      ],
+    });
+
+    const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
+    const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
+    expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
+    expect(providerRunner).toMatchObject({state: 'terminated', reservationReleasedAt: null});
+    expect(reservation?.count).toBe(1);
+  });
+
+  it('releases a reservation when a terminal runner only has a cancelled job', async () => {
+    const reservationId = await createReservation(1);
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    await providerRunnerFactory.create({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'cancelled-terminal-runner',
+      reservationId,
+      runnerSessionId: runnerSession.id,
+      state: 'running',
+    });
+    await insertRunningJobRow({
+      workspaceId,
+      provisionerId,
+      providerRunnerId: 'cancelled-terminal-runner',
+      cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
+    });
+
+    const result = await reportRunnerInstances({
+      scope: 'workspace',
+      workspaceId,
+      provisionerId,
+      events: [
+        event({
+          providerRunnerId: 'cancelled-terminal-runner',
+          reservationId,
+          state: 'terminated',
+          runnerSessionId: runnerSession.id,
+        }),
+      ],
+    });
+
+    const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
+    const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
+    const runningJobRows = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.providerRunnerId, 'cancelled-terminal-runner'));
+    expect(result).toEqual({
+      accepted: 1,
+      reservationsReleased: 1,
+      terminateIntentsHonored: [
+        {providerRunnerId: 'cancelled-terminal-runner', reason: 'job-cancelled'},
+      ],
+    });
+    expect(providerRunner).toMatchObject({
+      state: 'terminated',
+      reservationReleasedAt: expect.any(Date),
+    });
+    expect(reservationRows).toHaveLength(0);
+    expect(runningJobRows).toHaveLength(1);
+    expect(runningJobRows[0]?.cancellationRequestedAt).toBeInstanceOf(Date);
+  });
+
   it('uses the consumed ephemeral token session before releasing a terminal report', async () => {
     const reservationId = await createReservation(1);
     const token = await ephemeralRegistrationTokenFactory.create({

package/src/index.test.ts

@@ -10,6 +10,7 @@ describe('createRunnersModule', () => {
     expect(module.name).toBe('runners');
     expect(module.auth).toHaveLength(3);
     expect(module.routes).toHaveLength(13);
+    expect(module.subscribers).toHaveLength(2);
     expect(module.workers).toHaveLength(1);
   });
 

package/src/index.ts

@@ -4,7 +4,8 @@ import type {AuthInterModuleClient} from '@shipfox/api-auth-dto/inter-module';
 import {administrationActionEventSchemas} from '@shipfox/api-common-dto';
 import {runnersEventSchemas} from '@shipfox/api-runners-dto';
 import {
-  WORKFLOWS_JOB_EXECUTION_TIMED_OUT,
+  WORKFLOWS_JOB_EXECUTION_QUEUED,
+  WORKFLOWS_JOB_EXECUTION_TERMINATED,
   type WorkflowsEventMapDto,
 } from '@shipfox/api-workflows-dto';
 import {type ShipfoxModule, subscriberFactory} from '@shipfox/node-module';
@@ -16,7 +17,8 @@ import {
   createRunnerControlSessionAuthMethod,
   createRunnerRegistrationTokenAuthMethod,
   createRunnerRoutes,
-  onWorkflowsJobExecutionTimedOut,
+  onWorkflowsJobExecutionQueued,
+  onWorkflowsJobExecutionTerminated,
 } from '#presentation/index.js';
 import {createRunnersInterModulePresentation} from '#presentation/inter-module.js';
 import {createRunnersMaintenanceActivities} from '#temporal/activities/index.js';
@@ -30,12 +32,8 @@ export {
   unadvertisedRunnerTools,
 } from '#core/runner-tool-capabilities.js';
 export {
-  cancelRunnerJobs,
-  type EnqueueJobExecutionParams,
-  enqueueJobExecution,
   getWorkspaceJobCounts,
   isJobLeaseActive,
-  releaseJobExecution,
 } from '#db/index.js';
 export type {
   CreateRunnersModuleOptions,
@@ -64,7 +62,10 @@ export function createRunnersModule({
     publishers: [
       {name: 'runners', table: runnersOutbox, db, eventSchemas: runnersPublisherEventSchemas},
     ],
-    subscribers: [subscriber(WORKFLOWS_JOB_EXECUTION_TIMED_OUT, onWorkflowsJobExecutionTimedOut)],
+    subscribers: [
+      subscriber(WORKFLOWS_JOB_EXECUTION_QUEUED, onWorkflowsJobExecutionQueued),
+      subscriber(WORKFLOWS_JOB_EXECUTION_TERMINATED, onWorkflowsJobExecutionTerminated),
+    ],
     workers: [
       {
         taskQueue: RUNNERS_MAINTENANCE_TASK_QUEUE,

package/src/presentation/index.ts

@@ -4,4 +4,5 @@ export {
   createRunnerRegistrationTokenAuthMethod,
 } from './auth/index.js';
 export {createRunnerRoutes, createRunnerRoutes as routes} from './routes/index.js';
-export {onWorkflowsJobExecutionTimedOut} from './subscribers/on-workflows-job-execution-timed-out.js';
+export {onWorkflowsJobExecutionQueued} from './subscribers/on-workflows-job-execution-queued.js';
+export {onWorkflowsJobExecutionTerminated} from './subscribers/on-workflows-job-execution-terminated.js';

package/src/presentation/inter-module.ts

@@ -1,39 +1,12 @@
 import {runnersInterModuleContract} from '@shipfox/api-runners-dto/inter-module';
-import {
-  createInterModuleKnownError,
-  defineInterModulePresentation,
-  type InterModulePresentation,
-} from '@shipfox/inter-module';
-import {EmptyRequiredLabelsError} from '#core/errors.js';
+import {defineInterModulePresentation, type InterModulePresentation} from '@shipfox/inter-module';
 import {getEffectiveRunnerToolCapabilities} from '#core/runner-tool-capabilities.js';
-import {
-  cancelRunnerJobs,
-  enqueueJobExecution,
-  getWorkspaceJobCounts,
-  isJobLeaseActive,
-  releaseJobExecution,
-} from '#db/job-executions.js';
+import {getWorkspaceJobCounts, isJobLeaseActive} from '#db/job-executions.js';
 
 export function createRunnersInterModulePresentation(): InterModulePresentation<
   typeof runnersInterModuleContract
 > {
   return defineInterModulePresentation(runnersInterModuleContract, {
-    enqueueJobExecution: async (input) => {
-      try {
-        await enqueueJobExecution(input);
-        return {};
-      } catch (error) {
-        throw toEnqueueJobExecutionKnownError(error);
-      }
-    },
-    releaseJobExecution: async (input) => {
-      await releaseJobExecution(input);
-      return {};
-    },
-    cancelJobs: async (input) => {
-      await cancelRunnerJobs(input);
-      return {};
-    },
     getLeaseState: async (input) => ({active: await isJobLeaseActive(input)}),
     getEffectiveRunnerToolCapabilities: async (input) => {
       const result = await getEffectiveRunnerToolCapabilities(input);
@@ -44,14 +17,3 @@ export function createRunnersInterModulePresentation(): InterModulePresentation<
     }),
   });
 }
-
-export function toEnqueueJobExecutionKnownError(error: unknown): unknown {
-  if (error instanceof EmptyRequiredLabelsError) {
-    return createInterModuleKnownError(
-      runnersInterModuleContract.methods.enqueueJobExecution,
-      'empty-required-labels',
-      {},
-    );
-  }
-  return error;
-}