@shipfox/api-runners 2.0.0

package/src/db/job-executions.test.ts

@@ -0,0 +1,1500 @@
+import {verifyJobLeaseToken} from '@shipfox/api-auth';
+import {
+  RUNNER_JOB_CLAIMED,
+  RUNNER_JOB_LEASE_EXPIRED,
+  RUNNER_JOB_QUEUED,
+} from '@shipfox/api-runners-dto';
+import {eq, sql} from 'drizzle-orm';
+import {EmptyRequiredLabelsError, RunnerSessionExhaustedError} from '#core/errors.js';
+import {claimJobExecution} from '#core/job-executions.js';
+import {detectAndExpireStuckJobs} from '#core/maintenance.js';
+import {pendingJobFactory, runnerSessionFactory} from '#test/index.js';
+import {db} from './db.js';
+import {
+  cancelRunnerJobs,
+  claimPendingJobExecution as claimPendingJobExecutionDb,
+  enqueueJobExecution,
+  expireStuckJobExecutions,
+  getJobExecutionQueueDepth,
+  isJobLeaseActive,
+  recordHeartbeat,
+  releaseJobExecution,
+  requestJobExecutionCancellation,
+} from './job-executions.js';
+import {runnersOutbox} from './schema/outbox.js';
+import {pendingJobExecutions} from './schema/pending-job-executions.js';
+import {runnerSessions} from './schema/runner-sessions.js';
+import {runningJobExecutions} from './schema/running-job-executions.js';
+
+const sessionLabels = ['linux', 'x64'];
+
+function claimPendingJobExecution(
+  params: Omit<
+    Parameters<typeof claimPendingJobExecutionDb>[0],
+    'maxClaims' | 'sessionLabels' | 'runnerSessionLivenessThrottleSeconds'
+  > & {
+    maxClaims?: number | null;
+    sessionLabels?: string[];
+    runnerSessionLivenessThrottleSeconds?: number;
+  },
+) {
+  return claimPendingJobExecutionDb({
+    ...params,
+    maxClaims: params.maxClaims ?? null,
+    sessionLabels: params.sessionLabels ?? sessionLabels,
+    runnerSessionLivenessThrottleSeconds: params.runnerSessionLivenessThrottleSeconds ?? 10,
+  });
+}
+
+async function outboxEventsForJob(eventType: string, jobId: string) {
+  const rows = await db()
+    .select()
+    .from(runnersOutbox)
+    .where(eq(runnersOutbox.eventType, eventType));
+  return rows.filter((row) => (row.payload as {jobId?: string}).jobId === jobId);
+}
+
+describe('enqueueJobExecution', () => {
+  it('stores a pending assignment row', async () => {
+    const jobId = crypto.randomUUID();
+    const jobExecutionId = crypto.randomUUID();
+    const workflowRunId = crypto.randomUUID();
+    const workflowRunAttemptId = crypto.randomUUID();
+    const workspaceId = crypto.randomUUID();
+    const projectId = crypto.randomUUID();
+
+    await enqueueJobExecution({
+      workspaceId,
+      workflowRunId,
+      jobId,
+      jobExecutionId,
+      workflowRunAttemptId,
+      projectId,
+      requiredLabels: ['linux'],
+    });
+
+    const rows = await db()
+      .select()
+      .from(pendingJobExecutions)
+      .where(eq(pendingJobExecutions.jobExecutionId, jobExecutionId));
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.jobId).toBe(jobId);
+    expect(rows[0]?.jobExecutionId).toBe(jobExecutionId);
+    expect(rows[0]?.workflowRunId).toBe(workflowRunId);
+    expect(rows[0]?.workflowRunAttemptId).toBe(workflowRunAttemptId);
+    expect(rows[0]?.projectId).toBe(projectId);
+    expect(rows[0]?.workspaceId).toBe(workspaceId);
+    expect(rows[0]?.requiredLabels).toEqual(['linux']);
+    expect(rows[0]).not.toHaveProperty('payload');
+  });
+
+  it('stores canonical required labels', async () => {
+    const jobId = crypto.randomUUID();
+    const jobExecutionId = crypto.randomUUID();
+
+    await enqueueJobExecution({
+      workspaceId: crypto.randomUUID(),
+      jobId,
+      jobExecutionId,
+      workflowRunId: crypto.randomUUID(),
+      workflowRunAttemptId: crypto.randomUUID(),
+      projectId: crypto.randomUUID(),
+      requiredLabels: ['Ubuntu22', ' ubuntu22 ', 'LINUX'],
+    });
+
+    const rows = await db()
+      .select()
+      .from(pendingJobExecutions)
+      .where(eq(pendingJobExecutions.jobId, jobId));
+    expect(rows[0]?.requiredLabels).toEqual(['linux', 'ubuntu22']);
+  });
+
+  it('rejects empty required labels', async () => {
+    await expect(
+      enqueueJobExecution({
+        workspaceId: crypto.randomUUID(),
+        jobId: crypto.randomUUID(),
+        jobExecutionId: crypto.randomUUID(),
+        workflowRunId: crypto.randomUUID(),
+        workflowRunAttemptId: crypto.randomUUID(),
+        projectId: crypto.randomUUID(),
+        requiredLabels: [],
+      }),
+    ).rejects.toBeInstanceOf(EmptyRequiredLabelsError);
+  });
+
+  it('is idempotent: scheduling the same jobId twice is a no-op', async () => {
+    const jobId = crypto.randomUUID();
+    const params = {
+      workspaceId: crypto.randomUUID(),
+      workflowRunId: crypto.randomUUID(),
+      workflowRunAttemptId: crypto.randomUUID(),
+      projectId: crypto.randomUUID(),
+      jobId,
+      jobExecutionId: crypto.randomUUID(),
+      requiredLabels: ['linux'],
+    };
+
+    await enqueueJobExecution(params);
+    await expect(enqueueJobExecution(params)).resolves.toBeUndefined();
+
+    const rows = await db()
+      .select()
+      .from(pendingJobExecutions)
+      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));
+    expect(rows).toHaveLength(1);
+  });
+
+  it('emits runners.job.queued carrying the pending row created_at', async () => {
+    const jobId = crypto.randomUUID();
+    const workflowRunId = crypto.randomUUID();
+    const workflowRunAttemptId = crypto.randomUUID();
+
+    await enqueueJobExecution({
+      workspaceId: crypto.randomUUID(),
+      workflowRunId,
+      jobId,
+      jobExecutionId: crypto.randomUUID(),
+      workflowRunAttemptId,
+      projectId: crypto.randomUUID(),
+      requiredLabels: ['linux'],
+    });
+
+    const [pending] = await db()
+      .select()
+      .from(pendingJobExecutions)
+      .where(eq(pendingJobExecutions.jobId, jobId));
+    const outbox = await outboxEventsForJob(RUNNER_JOB_QUEUED, jobId);
+    expect(outbox).toHaveLength(1);
+    expect(outbox[0]?.eventType).toBe(RUNNER_JOB_QUEUED);
+    const payload = outbox[0]?.payload as {
+      jobId: string;
+      workflowRunId: string;
+      workflowRunAttemptId: string;
+      queuedAt: string;
+    };
+    expect(payload.jobId).toBe(jobId);
+    expect(payload.workflowRunId).toBe(workflowRunId);
+    expect(payload.workflowRunAttemptId).toBe(workflowRunAttemptId);
+    expect(new Date(payload.queuedAt).getTime()).toBe(pending?.createdAt.getTime());
+  });
+
+  it('does not double-emit queued when the same jobId is re-enqueued (idempotency regression)', async () => {
+    const params = {
+      workspaceId: crypto.randomUUID(),
+      workflowRunId: crypto.randomUUID(),
+      workflowRunAttemptId: crypto.randomUUID(),
+      projectId: crypto.randomUUID(),
+      jobId: crypto.randomUUID(),
+      jobExecutionId: crypto.randomUUID(),
+      requiredLabels: ['linux'],
+    };
+
+    await enqueueJobExecution(params);
+    await enqueueJobExecution(params);
+
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId)),
+    ).toHaveLength(1);
+    expect(await outboxEventsForJob(RUNNER_JOB_QUEUED, params.jobId)).toHaveLength(1);
+  });
+});
+
+describe('claimPendingJobExecution', () => {
+  let workspaceId: string;
+  let runnerSessionId: string;
+
+  beforeEach(async () => {
+    workspaceId = crypto.randomUUID();
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    runnerSessionId = runnerSession.id;
+  });
+
+  it('emits runners.job.claimed carrying the claim instant on a real claim', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    const [running] = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    const outbox = await outboxEventsForJob(RUNNER_JOB_CLAIMED, created.jobId);
+    expect(outbox).toHaveLength(1);
+    const payload = outbox[0]?.payload as {
+      jobId: string;
+      workflowRunId: string;
+      workflowRunAttemptId: string;
+      claimedAt: string;
+    };
+    expect(payload.jobId).toBe(created.jobId);
+    expect(payload.workflowRunId).toBe(created.workflowRunId);
+    expect(payload.workflowRunAttemptId).toBe(created.workflowRunAttemptId);
+    expect(new Date(payload.claimedAt).getTime()).toBe(running?.startedAt.getTime());
+  });
+
+  it('emits no claimed event when there is nothing to claim', async () => {
+    const before = await db()
+      .select()
+      .from(runnersOutbox)
+      .where(eq(runnersOutbox.eventType, RUNNER_JOB_CLAIMED));
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    const after = await db()
+      .select()
+      .from(runnersOutbox)
+      .where(eq(runnersOutbox.eventType, RUNNER_JOB_CLAIMED));
+    expect(claimed).toBeNull();
+    expect(after).toHaveLength(before.length);
+  });
+
+  it('emits no claimed event when dropping an orphan pending row', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+    const first = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    if (!first) throw new Error('Expected pending job to be claimed');
+    await db().insert(pendingJobExecutions).values({
+      workspaceId,
+      workflowRunId: created.workflowRunId,
+      jobId: created.jobId,
+      jobExecutionId: first.jobExecutionId,
+      workflowRunAttemptId: created.workflowRunAttemptId,
+      projectId: created.projectId,
+      requiredLabels: created.requiredLabels,
+    });
+    // Clear the initial claim's events so this assertion only covers the orphan claim.
+    const beforeOrphanClaim = await outboxEventsForJob(RUNNER_JOB_CLAIMED, created.jobId);
+
+    const second = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    expect(second).toBeNull();
+    expect(await outboxEventsForJob(RUNNER_JOB_CLAIMED, created.jobId)).toHaveLength(
+      beforeOrphanClaim.length,
+    );
+  });
+
+  it('returns the job ids when a job is available', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    expect(claimed).not.toBeNull();
+    expect(claimed?.jobId).toBe(created.jobId);
+    expect(claimed?.workflowRunAttemptId).toBe(created.workflowRunAttemptId);
+    expect(claimed?.projectId).toBe(created.projectId);
+  });
+
+  it('starts a claimed job without a first heartbeat marker', async () => {
+    await pendingJobFactory.create({workspaceId});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    const [running] = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobExecutionId, claimed?.jobExecutionId as string));
+    expect(running?.firstHeartbeatAt).toBeNull();
+  });
+
+  it('reports an active lease only for the session that claimed the job', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+    const otherRunnerSession = await runnerSessionFactory.create({workspaceId});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    const active = await isJobLeaseActive({
+      jobId: created.jobId,
+      jobExecutionId: claimed?.jobExecutionId as string,
+      runnerSessionId,
+    });
+    const stale = await isJobLeaseActive({
+      jobId: created.jobId,
+      jobExecutionId: created.jobExecutionId,
+      runnerSessionId: otherRunnerSession.id,
+    });
+    const mismatchedJob = await isJobLeaseActive({
+      jobId: crypto.randomUUID(),
+      jobExecutionId: claimed?.jobExecutionId as string,
+      runnerSessionId,
+    });
+
+    expect(active).toBe(true);
+    expect(stale).toBe(false);
+    expect(mismatchedJob).toBe(false);
+  });
+
+  it('returns null when no jobs are pending', async () => {
+    await db()
+      .update(runnerSessions)
+      .set({updatedAt: new Date('2025-01-01T00:00:00.000Z')})
+      .where(eq(runnerSessions.id, runnerSessionId));
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    const [session] = await db()
+      .select({updatedAt: runnerSessions.updatedAt})
+      .from(runnerSessions)
+      .where(eq(runnerSessions.id, runnerSessionId));
+    expect(claimed).toBeNull();
+    expect(session?.updatedAt.getTime()).toBeGreaterThan(
+      new Date('2025-01-01T00:00:00.000Z').getTime(),
+    );
+  });
+
+  it('touches runner session liveness when a job is claimed', async () => {
+    const staleUpdatedAt = new Date('2025-01-01T00:00:00.000Z');
+    await db()
+      .update(runnerSessions)
+      .set({updatedAt: staleUpdatedAt})
+      .where(eq(runnerSessions.id, runnerSessionId));
+    await pendingJobFactory.create({workspaceId});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    const [session] = await db()
+      .select({updatedAt: runnerSessions.updatedAt})
+      .from(runnerSessions)
+      .where(eq(runnerSessions.id, runnerSessionId));
+    expect(claimed).not.toBeNull();
+    expect(session?.updatedAt.getTime()).toBeGreaterThan(staleUpdatedAt.getTime());
+  });
+
+  it('does not touch runner session liveness inside the throttle window', async () => {
+    const freshUpdatedAt = new Date();
+    await db()
+      .update(runnerSessions)
+      .set({updatedAt: freshUpdatedAt})
+      .where(eq(runnerSessions.id, runnerSessionId));
+
+    const claimed = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId,
+      maxClaims: null,
+      runnerSessionLivenessThrottleSeconds: 10,
+    });
+
+    const [session] = await db()
+      .select({updatedAt: runnerSessions.updatedAt})
+      .from(runnerSessions)
+      .where(eq(runnerSessions.id, runnerSessionId));
+    expect(claimed).toBeNull();
+    expect(session?.updatedAt.getTime()).toBe(freshUpdatedAt.getTime());
+  });
+
+  it('enforces a non-null session claim cap from the database', async () => {
+    const provisionerId = crypto.randomUUID();
+    const provisionedRunnerId = `provisioned-runner-${crypto.randomUUID()}`;
+    await db()
+      .update(runnerSessions)
+      .set({registrationTokenKind: 'ephemeral', maxClaims: 1, provisionerId, provisionedRunnerId})
+      .where(eq(runnerSessions.id, runnerSessionId));
+    await pendingJobFactory.create({workspaceId});
+    await pendingJobFactory.create({workspaceId});
+
+    const first = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: 1});
+
+    expect(first).not.toBeNull();
+    await expect(
+      claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: 1}),
+    ).rejects.toBeInstanceOf(RunnerSessionExhaustedError);
+  });
+
+  it('does not spend a claim when a capped session polls an empty queue', async () => {
+    const provisionerId = crypto.randomUUID();
+    const provisionedRunnerId = `provisioned-runner-${crypto.randomUUID()}`;
+    await db()
+      .update(runnerSessions)
+      .set({registrationTokenKind: 'ephemeral', maxClaims: 1, provisionerId, provisionedRunnerId})
+      .where(eq(runnerSessions.id, runnerSessionId));
+
+    const empty = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: 1});
+
+    const [afterEmpty] = await db()
+      .select({claimsUsed: runnerSessions.claimsUsed})
+      .from(runnerSessions)
+      .where(eq(runnerSessions.id, runnerSessionId));
+    expect(empty).toBeNull();
+    expect(afterEmpty?.claimsUsed).toBe(0);
+
+    const created = await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: 1});
+
+    const [afterClaim] = await db()
+      .select({claimsUsed: runnerSessions.claimsUsed})
+      .from(runnerSessions)
+      .where(eq(runnerSessions.id, runnerSessionId));
+    expect(claimed?.jobId).toBe(created.jobId);
+    expect(afterClaim?.claimsUsed).toBe(1);
+  });
+
+  it('allows a manual session to claim repeatedly', async () => {
+    const first = await pendingJobFactory.create({workspaceId});
+    const second = await pendingJobFactory.create({workspaceId});
+
+    const firstClaim = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId,
+      maxClaims: null,
+    });
+    const secondClaim = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId,
+      maxClaims: null,
+    });
+
+    expect(firstClaim?.jobId).toBe(first.jobId);
+    expect(secondClaim?.jobId).toBe(second.jobId);
+  });
+
+  it('only one caller wins when two claim concurrently', async () => {
+    const otherRunnerSession = await runnerSessionFactory.create({workspaceId});
+    await pendingJobFactory.create({workspaceId});
+
+    const [claim1, claim2] = await Promise.all([
+      claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null}),
+      claimPendingJobExecution({
+        workspaceId,
+        runnerSessionId: otherRunnerSession.id,
+        maxClaims: null,
+      }),
+    ]);
+
+    const claimed = [claim1, claim2].filter(Boolean);
+    expect(claimed).toHaveLength(1);
+  });
+
+  it('claims the oldest job first', async () => {
+    const older = await pendingJobFactory.create({workspaceId});
+    await pendingJobFactory.create({workspaceId});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    expect(claimed?.jobId).toBe(older.jobId);
+  });
+
+  it('moves the job from pending to running', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+
+    await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    const pending = await db()
+      .select()
+      .from(pendingJobExecutions)
+      .where(eq(pendingJobExecutions.workspaceId, workspaceId));
+    const running = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.workspaceId, workspaceId));
+    expect(pending).toHaveLength(0);
+    expect(running).toHaveLength(1);
+    expect(running[0]?.runnerSessionId).toBe(runnerSessionId);
+    expect(running[0]?.projectId).toBe(created.projectId);
+    expect(running[0]?.requiredLabels).toEqual(created.requiredLabels);
+    expect(running[0]?.runnerLabels).toEqual(sessionLabels);
+    expect(running[0]?.provisionerId).toBeNull();
+    expect(running[0]?.provisionedRunnerId).toBeNull();
+  });
+
+  it('copies an ephemeral session provisioned-runner link onto the running job', async () => {
+    const provisionerId = crypto.randomUUID();
+    const provisionedRunnerId = `provisioned-runner-${crypto.randomUUID()}`;
+    await db()
+      .update(runnerSessions)
+      .set({registrationTokenKind: 'ephemeral', maxClaims: 1, provisionerId, provisionedRunnerId})
+      .where(eq(runnerSessions.id, runnerSessionId));
+    const created = await pendingJobFactory.create({workspaceId});
+
+    await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: 1});
+
+    const [running] = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, created.jobId));
+    expect(running?.provisionerId).toBe(provisionerId);
+    expect(running?.provisionedRunnerId).toBe(provisionedRunnerId);
+  });
+
+  it('rejects a running job row with a partial provisioned-runner link', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+
+    await expect(
+      db().insert(runningJobExecutions).values({
+        workspaceId,
+        workflowRunId: created.workflowRunId,
+        jobId: created.jobId,
+        jobExecutionId: created.jobExecutionId,
+        workflowRunAttemptId: created.workflowRunAttemptId,
+        projectId: created.projectId,
+        runnerSessionId,
+        provisionerId: crypto.randomUUID(),
+        requiredLabels: created.requiredLabels,
+        runnerLabels: sessionLabels,
+      }),
+    ).rejects.toThrow();
+  });
+
+  it('claims a job whose required labels are a subset of the session labels', async () => {
+    const created = await pendingJobFactory.create({
+      workspaceId,
+      requiredLabels: ['linux'],
+    });
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId});
+
+    expect(claimed?.jobId).toBe(created.jobId);
+  });
+
+  it('claims a job whose required labels exactly match the session labels', async () => {
+    const created = await pendingJobFactory.create({
+      workspaceId,
+      requiredLabels: ['linux', 'x64'],
+    });
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId});
+
+    expect(claimed?.jobId).toBe(created.jobId);
+  });
+
+  it('claims by labels only when runner tool capabilities differ', async () => {
+    const matchingRunner = await runnerSessionFactory.create({
+      workspaceId,
+      labels: sessionLabels,
+      toolCapabilities: {harnesses: {pi: {tools: ['read']}}},
+    });
+    const underProvisionedRunner = await runnerSessionFactory.create({
+      workspaceId,
+      labels: sessionLabels,
+      toolCapabilities: {harnesses: {pi: {tools: []}}},
+    });
+    const firstJob = await pendingJobFactory.create({
+      workspaceId,
+      requiredLabels: ['linux'],
+    });
+    const secondJob = await pendingJobFactory.create({
+      workspaceId,
+      requiredLabels: ['linux'],
+    });
+
+    const firstClaim = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId: underProvisionedRunner.id,
+    });
+    const secondClaim = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId: matchingRunner.id,
+    });
+
+    expect([firstClaim?.jobId, secondClaim?.jobId].sort()).toEqual(
+      [firstJob.jobId, secondJob.jobId].sort(),
+    );
+  });
+
+  it('skips an older incompatible job and claims the oldest compatible job', async () => {
+    await pendingJobFactory.create({workspaceId, requiredLabels: ['macos']});
+    const compatible = await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
+    await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId});
+
+    expect(claimed?.jobId).toBe(compatible.jobId);
+  });
+
+  it('claims the older matching job before newer matching jobs', async () => {
+    await pendingJobFactory.create({workspaceId, requiredLabels: ['macos']});
+    const olderMatching = await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
+    await pendingJobFactory.create({workspaceId, requiredLabels: ['x64']});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId});
+
+    expect(claimed?.jobId).toBe(olderMatching.jobId);
+  });
+
+  it('returns null when no compatible job is pending', async () => {
+    await pendingJobFactory.create({workspaceId, requiredLabels: ['macos']});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId});
+
+    expect(claimed).toBeNull();
+  });
+
+  it('returns null for an empty session label set', async () => {
+    await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
+
+    const claimed = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId,
+      sessionLabels: [],
+    });
+
+    expect(claimed).toBeNull();
+  });
+
+  it('claims the compatible row from a mixed-label queue', async () => {
+    for (let index = 0; index < 8; index += 1) {
+      await pendingJobFactory.create({workspaceId, requiredLabels: [`gpu-${index}`]});
+    }
+    const compatible = await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId});
+
+    expect(claimed?.jobId).toBe(compatible.jobId);
+  });
+
+  it('does not claim jobs from another workspace', async () => {
+    await pendingJobFactory.create({workspaceId: crypto.randomUUID()});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    expect(claimed).toBeNull();
+  });
+
+  it('drops an orphan pending row whose job is already running, without a poison loop', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+    const first = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    if (!first) throw new Error('Expected pending job to be claimed');
+    expect(first.jobId).toBe(created.jobId);
+
+    // Simulate an enqueue retry that re-inserts a pending row after the claim.
+    await db().insert(pendingJobExecutions).values({
+      workspaceId,
+      workflowRunId: created.workflowRunId,
+      jobId: created.jobId,
+      jobExecutionId: first.jobExecutionId,
+      workflowRunAttemptId: created.workflowRunAttemptId,
+      projectId: created.projectId,
+      requiredLabels: created.requiredLabels,
+    });
+
+    const second = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    expect(second).toBeNull();
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.workspaceId, workspaceId)),
+    ).toHaveLength(0);
+    const running = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.workspaceId, workspaceId));
+    expect(running).toHaveLength(1);
+    expect(running[0]?.jobId).toBe(created.jobId);
+  });
+
+  it('leaves a non-matching orphan unclaimed until release sweeps it', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+    const first = await claimPendingJobExecution({workspaceId, runnerSessionId});
+    if (!first) throw new Error('Expected pending job to be claimed');
+    expect(first.jobId).toBe(created.jobId);
+    await db().insert(pendingJobExecutions).values({
+      workspaceId,
+      workflowRunId: created.workflowRunId,
+      jobId: created.jobId,
+      jobExecutionId: first.jobExecutionId,
+      workflowRunAttemptId: created.workflowRunAttemptId,
+      projectId: created.projectId,
+      requiredLabels: created.requiredLabels,
+    });
+
+    const second = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId,
+      sessionLabels: ['macos'],
+    });
+    await releaseJobExecution({jobExecutionId: created.jobExecutionId});
+
+    expect(second).toBeNull();
+    expect(
+      await db()
+        .select()
+        .from(runningJobExecutions)
+        .where(eq(runningJobExecutions.workspaceId, workspaceId)),
+    ).toHaveLength(0);
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.workspaceId, workspaceId)),
+    ).toHaveLength(0);
+  });
+
+  it('claims a real pending job ahead of a newer orphan', async () => {
+    const alreadyRunning = await pendingJobFactory.create({workspaceId});
+    const first = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    if (!first) throw new Error('Expected pending job to be claimed');
+
+    // A genuinely new pending job (older), then an orphan re-insert for the running job (newer).
+    const real = await pendingJobFactory.create({workspaceId});
+    await db().insert(pendingJobExecutions).values({
+      workspaceId,
+      workflowRunId: alreadyRunning.workflowRunId,
+      jobId: alreadyRunning.jobId,
+      jobExecutionId: first.jobExecutionId,
+      workflowRunAttemptId: alreadyRunning.workflowRunAttemptId,
+      projectId: alreadyRunning.projectId,
+      requiredLabels: alreadyRunning.requiredLabels,
+    });
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    expect(claimed?.jobId).toBe(real.jobId);
+  });
+});
+
+describe('claimJobExecution', () => {
+  let workspaceId: string;
+  let runnerSessionId: string;
+
+  beforeEach(async () => {
+    workspaceId = crypto.randomUUID();
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    runnerSessionId = runnerSession.id;
+  });
+
+  it('mints a lease token whose claims match the claimed job', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+
+    const claimed = await claimJobExecution({
+      workspaceId,
+      runnerSessionId,
+      sessionLabels,
+      maxClaims: null,
+    });
+
+    expect(claimed).not.toBeNull();
+    expect(claimed?.jobId).toBe(created.jobId);
+    expect(claimed?.workflowRunAttemptId).toBe(created.workflowRunAttemptId);
+    expect(claimed).not.toHaveProperty('steps');
+
+    const claims = await verifyJobLeaseToken(claimed?.leaseToken as string);
+    expect(claims).toMatchObject({
+      jobId: created.jobId,
+      workflowRunAttemptId: created.workflowRunAttemptId,
+      projectId: created.projectId,
+      workspaceId,
+      runnerSessionId,
+    });
+  });
+
+  it('returns null and mints no token when the queue is empty', async () => {
+    const claimed = await claimJobExecution({
+      workspaceId,
+      runnerSessionId,
+      sessionLabels,
+      maxClaims: null,
+    });
+
+    expect(claimed).toBeNull();
+  });
+});
+
+describe('releaseJobExecution', () => {
+  let workspaceId: string;
+  let runnerSessionId: string;
+
+  beforeEach(async () => {
+    workspaceId = crypto.randomUUID();
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    runnerSessionId = runnerSession.id;
+  });
+
+  it('deletes the running row and writes no outbox event', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    const before = await outboxEventsForJob(RUNNER_JOB_CLAIMED, claimed?.jobId as string);
+
+    await releaseJobExecution({jobExecutionId: claimed?.jobExecutionId as string});
+
+    expect(
+      await db()
+        .select()
+        .from(runningJobExecutions)
+        .where(eq(runningJobExecutions.jobExecutionId, claimed?.jobExecutionId as string)),
+    ).toHaveLength(0);
+    expect(await outboxEventsForJob(RUNNER_JOB_CLAIMED, claimed?.jobId as string)).toHaveLength(
+      before.length,
+    );
+  });
+
+  it('is a no-op when the job is absent (idempotent)', async () => {
+    await expect(
+      releaseJobExecution({jobExecutionId: crypto.randomUUID()}),
+    ).resolves.toBeUndefined();
+  });
+
+  it('releases regardless of which session holds the lease', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    // No token is passed: the workflow is authoritative over the lease.
+    await releaseJobExecution({jobExecutionId: claimed?.jobExecutionId as string});
+
+    expect(
+      await db()
+        .select()
+        .from(runningJobExecutions)
+        .where(eq(runningJobExecutions.jobExecutionId, claimed?.jobExecutionId as string)),
+    ).toHaveLength(0);
+  });
+
+  it('also sweeps a lingering pending row for the same job', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    // An orphan pending row left by a post-claim enqueue retry.
+    await db()
+      .insert(pendingJobExecutions)
+      .values({
+        workspaceId,
+        workflowRunId: claimed?.workflowRunId as string,
+        jobId: claimed?.jobId as string,
+        jobExecutionId: claimed?.jobExecutionId as string,
+        workflowRunAttemptId: claimed?.workflowRunAttemptId as string,
+        projectId: claimed?.projectId as string,
+        requiredLabels: ['linux'],
+      });
+
+    await releaseJobExecution({jobExecutionId: claimed?.jobExecutionId as string});
+
+    expect(
+      await db()
+        .select()
+        .from(runningJobExecutions)
+        .where(eq(runningJobExecutions.jobExecutionId, claimed?.jobExecutionId as string)),
+    ).toHaveLength(0);
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.jobExecutionId, claimed?.jobExecutionId as string)),
+    ).toHaveLength(0);
+  });
+});
+
+describe('recordHeartbeat', () => {
+  let workspaceId: string;
+  let runnerSessionId: string;
+
+  beforeEach(async () => {
+    workspaceId = crypto.randomUUID();
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    runnerSessionId = runnerSession.id;
+  });
+
+  it('returns cancel:false on a fresh row and records the first heartbeat', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    const before = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    // Force last_heartbeat_at into the past so we can observe the update.
+    await db()
+      .update(runningJobExecutions)
+      .set({lastHeartbeatAt: sql`now() - interval '1 hour'`})
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+
+    const result = await recordHeartbeat({
+      jobExecutionId: claimed?.jobExecutionId as string,
+      runnerSessionId,
+    });
+
+    expect(result).toMatchObject({
+      cancellationRequested: false,
+      runningJobExecution: {
+        jobId: claimed?.jobId,
+        jobExecutionId: claimed?.jobExecutionId,
+        runnerSessionId,
+      },
+    });
+
+    const after = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    expect(after[0]?.firstHeartbeatAt).toBeInstanceOf(Date);
+    expect(after[0]?.lastHeartbeatAt.getTime()).toBeGreaterThan(
+      (before[0]?.lastHeartbeatAt.getTime() ?? 0) - 1,
+    );
+  });
+
+  it('preserves first_heartbeat_at on later heartbeats', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    await recordHeartbeat({
+      jobExecutionId: claimed?.jobExecutionId as string,
+      runnerSessionId,
+    });
+    const [afterFirst] = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    await db()
+      .update(runningJobExecutions)
+      .set({lastHeartbeatAt: sql`now() - interval '1 hour'`})
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+
+    await recordHeartbeat({
+      jobExecutionId: claimed?.jobExecutionId as string,
+      runnerSessionId,
+    });
+
+    const [afterSecond] = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    expect(afterSecond?.firstHeartbeatAt?.getTime()).toBe(afterFirst?.firstHeartbeatAt?.getTime());
+    expect(afterSecond?.lastHeartbeatAt.getTime()).toBeGreaterThan(
+      afterFirst?.firstHeartbeatAt?.getTime() ?? 0,
+    );
+  });
+
+  it('returns cancel:true after requestJobExecutionCancellation', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    await requestJobExecutionCancellation({jobExecutionId: claimed?.jobExecutionId as string});
+
+    const result = await recordHeartbeat({
+      jobExecutionId: claimed?.jobExecutionId as string,
+      runnerSessionId,
+    });
+
+    expect(result).toMatchObject({
+      cancellationRequested: true,
+      runningJobExecution: {
+        jobId: claimed?.jobId,
+        jobExecutionId: claimed?.jobExecutionId,
+        runnerSessionId,
+      },
+    });
+  });
+
+  it('throws RunningJobExecutionNotFoundError when jobId is unknown', async () => {
+    await expect(
+      recordHeartbeat({jobExecutionId: crypto.randomUUID(), runnerSessionId}),
+    ).rejects.toThrow('Running job execution not found');
+  });
+
+  it('throws when jobId belongs to a different session', async () => {
+    const otherRunnerSession = await runnerSessionFactory.create({workspaceId});
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    await expect(
+      recordHeartbeat({
+        jobExecutionId: claimed?.jobExecutionId as string,
+        runnerSessionId: otherRunnerSession.id,
+      }),
+    ).rejects.toThrow('Running job execution not found');
+  });
+});
+
+describe('requestJobExecutionCancellation', () => {
+  let workspaceId: string;
+  let runnerSessionId: string;
+
+  beforeEach(async () => {
+    workspaceId = crypto.randomUUID();
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    runnerSessionId = runnerSession.id;
+  });
+
+  it('sets cancellation_requested_at on a fresh row', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    await requestJobExecutionCancellation({jobExecutionId: claimed?.jobExecutionId as string});
+
+    const rows = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    expect(rows[0]?.cancellationRequestedAt).not.toBeNull();
+  });
+
+  it('is idempotent: second call preserves the first timestamp', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    await requestJobExecutionCancellation({jobExecutionId: claimed?.jobExecutionId as string});
+    const after1 = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    const firstTs = after1[0]?.cancellationRequestedAt;
+
+    await new Promise((r) => setTimeout(r, 10));
+    await requestJobExecutionCancellation({jobExecutionId: claimed?.jobExecutionId as string});
+
+    const after2 = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    expect(after2[0]?.cancellationRequestedAt?.getTime()).toBe(firstTs?.getTime());
+  });
+
+  it('is a no-op when the job execution is missing (does not throw)', async () => {
+    await expect(
+      requestJobExecutionCancellation({jobExecutionId: crypto.randomUUID()}),
+    ).resolves.toBeUndefined();
+  });
+});
+
+describe('cancelRunnerJobs', () => {
+  let workspaceId: string;
+  let runnerSessionId: string;
+
+  beforeEach(async () => {
+    workspaceId = crypto.randomUUID();
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    runnerSessionId = runnerSession.id;
+  });
+
+  it('deletes queued jobs and requests cancellation for running jobs', async () => {
+    const running = await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    const queued = await pendingJobFactory.create({workspaceId});
+
+    await cancelRunnerJobs({jobIds: [queued.jobId, claimed?.jobId as string]});
+
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.workspaceId, workspaceId)),
+    ).toHaveLength(0);
+    const rows = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.workspaceId, workspaceId));
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.jobId).toBe(running.jobId);
+    expect(rows[0]?.cancellationRequestedAt).not.toBeNull();
+  });
+
+  it('is idempotent and no-ops for absent jobs', async () => {
+    const queued = await pendingJobFactory.create({workspaceId});
+
+    await cancelRunnerJobs({jobIds: [queued.jobId, crypto.randomUUID()]});
+    await cancelRunnerJobs({jobIds: [queued.jobId, crypto.randomUUID()]});
+
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.workspaceId, workspaceId)),
+    ).toHaveLength(0);
+    expect(
+      await db()
+        .select()
+        .from(runningJobExecutions)
+        .where(eq(runningJobExecutions.workspaceId, workspaceId)),
+    ).toHaveLength(0);
+  });
+
+  it('prevents a cancelled queued job from being claimed', async () => {
+    const queued = await pendingJobFactory.create({workspaceId});
+
+    await cancelRunnerJobs({jobIds: [queued.jobId]});
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    expect(claimed).toBeNull();
+  });
+});
+
+describe('detectAndExpireStuckJobs', () => {
+  let workspaceId: string;
+  let runnerSessionId: string;
+
+  beforeEach(async () => {
+    workspaceId = crypto.randomUUID();
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    runnerSessionId = runnerSession.id;
+  });
+
+  async function makeStaleJob(staleSeconds: number): Promise<{
+    jobId: string;
+    jobExecutionId: string;
+    workflowRunId: string;
+    workflowRunAttemptId: string;
+    projectId: string;
+  }> {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    await db()
+      .update(runningJobExecutions)
+      .set({
+        firstHeartbeatAt: sql`now() - (${staleSeconds} || ' seconds')::interval`,
+        lastHeartbeatAt: sql`now() - (${staleSeconds} || ' seconds')::interval`,
+      })
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    return {
+      jobId: claimed?.jobId as string,
+      jobExecutionId: claimed?.jobExecutionId as string,
+      workflowRunId: claimed?.workflowRunId as string,
+      workflowRunAttemptId: claimed?.workflowRunAttemptId as string,
+      projectId: claimed?.projectId as string,
+    };
+  }
+
+  async function makeNoFirstHeartbeatJob(ageSeconds: number): Promise<{
+    jobId: string;
+    jobExecutionId: string;
+    workflowRunId: string;
+    workflowRunAttemptId: string;
+    projectId: string;
+  }> {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    await db()
+      .update(runningJobExecutions)
+      .set({
+        startedAt: sql`now() - (${ageSeconds} || ' seconds')::interval`,
+        lastHeartbeatAt: sql`now() - (${ageSeconds} || ' seconds')::interval`,
+      })
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    return {
+      jobId: claimed?.jobId as string,
+      jobExecutionId: claimed?.jobExecutionId as string,
+      workflowRunId: claimed?.workflowRunId as string,
+      workflowRunAttemptId: claimed?.workflowRunAttemptId as string,
+      projectId: claimed?.projectId as string,
+    };
+  }
+
+  async function runningJobsForTest() {
+    return await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.workspaceId, workspaceId));
+  }
+
+  async function outboxForJobs(jobIds: string[]) {
+    const all = await db().select().from(runnersOutbox);
+    return all.filter((row) => {
+      // The same job ids can also have queued and claimed events from setup.
+      if (row.eventType !== RUNNER_JOB_LEASE_EXPIRED) return false;
+      const payload = row.payload as {jobId?: string};
+      return payload.jobId !== undefined && jobIds.includes(payload.jobId);
+    });
+  }
+
+  it('expires a stuck job and writes a runners.job.lease_expired event', async () => {
+    const {jobId, workflowRunId, workflowRunAttemptId} = await makeStaleJob(600);
+
+    const result = await detectAndExpireStuckJobs({thresholdSeconds: 180});
+
+    expect(result.expired).toBeGreaterThanOrEqual(1);
+    expect(await runningJobsForTest()).toHaveLength(0);
+
+    const outbox = await outboxForJobs([jobId]);
+    expect(outbox).toHaveLength(1);
+    expect(outbox[0]?.eventType).toBe(RUNNER_JOB_LEASE_EXPIRED);
+    const payload = outbox[0]?.payload as Record<string, unknown>;
+    expect(payload.jobId).toBe(jobId);
+    expect(payload.workflowRunId).toBe(workflowRunId);
+    expect(payload.workflowRunAttemptId).toBe(workflowRunAttemptId);
+    // The lease-expired event carries only the assignment identifiers.
+    expect(payload.status).toBeUndefined();
+    expect(payload.steps).toBeUndefined();
+  });
+
+  it('expires a job that never sent a first heartbeat after the startup grace', async () => {
+    const {jobId, workflowRunId, workflowRunAttemptId} = await makeNoFirstHeartbeatJob(90);
+
+    const result = await detectAndExpireStuckJobs({
+      noFirstHeartbeatGraceSeconds: 60,
+      thresholdSeconds: 180,
+    });
+
+    expect(result.expired).toBeGreaterThanOrEqual(1);
+    expect(await runningJobsForTest()).toHaveLength(0);
+
+    const outbox = await outboxForJobs([jobId]);
+    expect(outbox).toHaveLength(1);
+    const payload = outbox[0]?.payload as Record<string, unknown>;
+    expect(payload.jobId).toBe(jobId);
+    expect(payload.workflowRunId).toBe(workflowRunId);
+    expect(payload.workflowRunAttemptId).toBe(workflowRunAttemptId);
+  });
+
+  it('does not expire a job that is still inside the first heartbeat grace', async () => {
+    const {jobId} = await makeNoFirstHeartbeatJob(30);
+
+    await detectAndExpireStuckJobs({
+      noFirstHeartbeatGraceSeconds: 60,
+      thresholdSeconds: 180,
+    });
+
+    expect(await runningJobsForTest()).toHaveLength(1);
+    expect(await outboxForJobs([jobId])).toHaveLength(0);
+  });
+
+  it('does not expire a heartbeated job through the first heartbeat grace path', async () => {
+    const {jobId} = await makeStaleJob(90);
+
+    await detectAndExpireStuckJobs({
+      noFirstHeartbeatGraceSeconds: 60,
+      thresholdSeconds: 180,
+    });
+
+    expect(await runningJobsForTest()).toHaveLength(1);
+    expect(await outboxForJobs([jobId])).toHaveLength(0);
+  });
+
+  it('uses the stale-heartbeat threshold for upgraded rows that heartbeated before first heartbeat tracking', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    await db()
+      .update(runningJobExecutions)
+      .set({
+        startedAt: sql`now() - interval '90 seconds'`,
+        firstHeartbeatAt: null,
+        lastHeartbeatAt: sql`now() - interval '30 seconds'`,
+      })
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+
+    await detectAndExpireStuckJobs({
+      noFirstHeartbeatGraceSeconds: 60,
+      thresholdSeconds: 180,
+    });
+
+    expect(await runningJobsForTest()).toHaveLength(1);
+    expect(await outboxForJobs([claimed?.jobId as string])).toHaveLength(0);
+  });
+
+  it('expires upgraded heartbeated rows only after their stale-heartbeat threshold', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    await db()
+      .update(runningJobExecutions)
+      .set({
+        startedAt: sql`now() - interval '900 seconds'`,
+        firstHeartbeatAt: null,
+        lastHeartbeatAt: sql`now() - interval '600 seconds'`,
+      })
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+
+    await detectAndExpireStuckJobs({
+      noFirstHeartbeatGraceSeconds: 60,
+      thresholdSeconds: 180,
+    });
+
+    expect(await runningJobsForTest()).toHaveLength(0);
+    expect(await outboxForJobs([claimed?.jobId as string])).toHaveLength(1);
+  });
+
+  it('does not expire a job whose heartbeat is still inside the threshold window', async () => {
+    const {jobId} = await makeStaleJob(60);
+
+    await detectAndExpireStuckJobs({thresholdSeconds: 180});
+
+    expect(await runningJobsForTest()).toHaveLength(1);
+    expect(await outboxForJobs([jobId])).toHaveLength(0);
+  });
+
+  it('only expires the stuck rows in a mixed batch', async () => {
+    const stuck1 = await makeStaleJob(600);
+    const stuck2 = await makeStaleJob(600);
+    const fresh = await makeStaleJob(30);
+
+    await detectAndExpireStuckJobs({thresholdSeconds: 180});
+
+    const remaining = await runningJobsForTest();
+    expect(remaining.map((r) => r.jobId)).toEqual([fresh.jobId]);
+    expect(await outboxForJobs([stuck1.jobId, stuck2.jobId, fresh.jobId])).toHaveLength(2);
+  });
+
+  it('returns zero when there are no stuck jobs', async () => {
+    const result = await detectAndExpireStuckJobs({thresholdSeconds: 180});
+    expect(result.expired).toBe(0);
+  });
+
+  it('skips a row whose heartbeat refreshed before the atomic DELETE re-evaluates the predicate', async () => {
+    // Pre-stale, then refresh, then run — the cutoff is folded into the DELETE's
+    // WHERE so the live row survives even though the iteration SELECT saw it stale.
+    const {jobId} = await makeStaleJob(600);
+    await db()
+      .update(runningJobExecutions)
+      .set({
+        firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,
+        lastHeartbeatAt: sql`now()`,
+      })
+      .where(eq(runningJobExecutions.jobId, jobId));
+
+    await detectAndExpireStuckJobs({thresholdSeconds: 180});
+
+    expect(await runningJobsForTest()).toHaveLength(1);
+    expect(await outboxForJobs([jobId])).toHaveLength(0);
+  });
+
+  it('double-expiring the same stuck job emits exactly one event', async () => {
+    const {jobId} = await makeStaleJob(600);
+
+    await detectAndExpireStuckJobs({thresholdSeconds: 180});
+    await detectAndExpireStuckJobs({thresholdSeconds: 180});
+
+    expect(
+      await db().select().from(runningJobExecutions).where(eq(runningJobExecutions.jobId, jobId)),
+    ).toHaveLength(0);
+    expect(await outboxForJobs([jobId])).toHaveLength(1);
+  });
+
+  it('sweeps an orphan pending row for the job it reaps (best-effort release may have failed)', async () => {
+    const {jobId, jobExecutionId, workflowRunId, workflowRunAttemptId, projectId} =
+      await makeStaleJob(600);
+    // A post-claim enqueue retry left a pending row whose job is already running;
+    // without this sweep it would stay re-claimable for an already-finished job.
+    await db()
+      .insert(pendingJobExecutions)
+      .values({
+        workspaceId,
+        workflowRunId,
+        jobId,
+        jobExecutionId,
+        workflowRunAttemptId,
+        projectId,
+        requiredLabels: ['linux'],
+      });
+
+    await detectAndExpireStuckJobs({thresholdSeconds: 180});
+
+    expect(await runningJobsForTest()).toHaveLength(0);
+    expect(
+      await db().select().from(pendingJobExecutions).where(eq(pendingJobExecutions.jobId, jobId)),
+    ).toHaveLength(0);
+  });
+
+  it('leaves the orphan pending row alone when the running row is not stale enough to reap', async () => {
+    const {jobId, jobExecutionId, workflowRunId, workflowRunAttemptId, projectId} =
+      await makeStaleJob(60);
+    await db()
+      .insert(pendingJobExecutions)
+      .values({
+        workspaceId,
+        workflowRunId,
+        jobId,
+        jobExecutionId,
+        workflowRunAttemptId,
+        projectId,
+        requiredLabels: ['linux'],
+      });
+
+    await detectAndExpireStuckJobs({thresholdSeconds: 180});
+
+    // The sweep is gated on actually reaping a running row, so a live job's
+    // pending row is untouched.
+    expect(
+      await db().select().from(pendingJobExecutions).where(eq(pendingJobExecutions.jobId, jobId)),
+    ).toHaveLength(1);
+  });
+
+  it('returns the reaped workflow/job identifiers per row without leaking the internal id', async () => {
+    const {jobId, jobExecutionId, workflowRunId, workflowRunAttemptId} = await makeStaleJob(600);
+
+    const reaped = await expireStuckJobExecutions({
+      noFirstHeartbeatGraceSeconds: 60,
+      thresholdSeconds: 180,
+    });
+
+    const mine = reaped.find((row) => row.jobId === jobId);
+    expect(mine).toEqual({jobId, jobExecutionId, workflowRunId, workflowRunAttemptId});
+    expect(mine).not.toHaveProperty('id');
+  });
+
+  it('writes one lease_expired event per reaped job in a single bulk insert', async () => {
+    const stuck1 = await makeStaleJob(600);
+    const stuck2 = await makeStaleJob(600);
+
+    await detectAndExpireStuckJobs({thresholdSeconds: 180});
+
+    const outbox = await outboxForJobs([stuck1.jobId, stuck2.jobId]);
+    expect(outbox).toHaveLength(2);
+    expect(outbox.every((row) => row.eventType === RUNNER_JOB_LEASE_EXPIRED)).toBe(true);
+  });
+
+  it('two concurrent ticks reap each stuck job exactly once (no double-emit)', async () => {
+    const stuck1 = await makeStaleJob(600);
+    const stuck2 = await makeStaleJob(600);
+
+    await Promise.all([
+      detectAndExpireStuckJobs({thresholdSeconds: 180}),
+      detectAndExpireStuckJobs({thresholdSeconds: 180}),
+    ]);
+
+    expect(await runningJobsForTest()).toHaveLength(0);
+    expect(await outboxForJobs([stuck1.jobId, stuck2.jobId])).toHaveLength(2);
+  });
+
+  it('a reaper tick and a concurrent claim of the same orphan-pending job leave consistent state', async () => {
+    const {jobId, jobExecutionId, workflowRunId, workflowRunAttemptId, projectId} =
+      await makeStaleJob(600);
+    // Orphan pending row from a post-claim enqueue retry for an already-running job.
+    await db()
+      .insert(pendingJobExecutions)
+      .values({
+        workspaceId,
+        workflowRunId,
+        jobId,
+        jobExecutionId,
+        workflowRunAttemptId,
+        projectId,
+        requiredLabels: ['linux'],
+      });
+
+    // The reaper locks running-then-pending while the claim locks pending-then-running;
+    // a deadlock loser rolls back, so either side may settle as rejected.
+    await Promise.allSettled([
+      detectAndExpireStuckJobs({thresholdSeconds: 180}),
+      claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null}),
+    ]);
+
+    // A follow-up tick finishes any reap that lost a deadlock race.
+    await detectAndExpireStuckJobs({thresholdSeconds: 180});
+
+    // The expired job is gone and not re-claimable; its orphan pending row is swept.
+    expect(await runningJobsForTest()).toHaveLength(0);
+    expect(
+      await db().select().from(pendingJobExecutions).where(eq(pendingJobExecutions.jobId, jobId)),
+    ).toHaveLength(0);
+    expect(
+      await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null}),
+    ).toBeNull();
+  });
+});
+
+describe('getJobExecutionQueueDepth', () => {
+  let workspaceId: string;
+  let runnerSessionId: string;
+
+  beforeEach(async () => {
+    workspaceId = crypto.randomUUID();
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    runnerSessionId = runnerSession.id;
+  });
+
+  it('reports queue depth counters', async () => {
+    const depth = await getJobExecutionQueueDepth();
+
+    expect(depth.pendingJobExecutions).toBeGreaterThanOrEqual(0);
+    expect(depth.runningJobExecutions).toBeGreaterThanOrEqual(0);
+  });
+
+  it('counts pending and running jobs separately', async () => {
+    const baseline = await getJobExecutionQueueDepth();
+    await pendingJobFactory.create({workspaceId});
+    await pendingJobFactory.create({workspaceId});
+    await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+
+    const depth = await getJobExecutionQueueDepth();
+
+    expect(depth).toEqual({
+      pendingJobExecutions: baseline.pendingJobExecutions + 1,
+      runningJobExecutions: baseline.runningJobExecutions + 1,
+    });
+  });
+});