@shipfox/api-runners 10.2.0 → 12.0.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shipfox/api-runners",
   "license": "MIT",
-  "version": "10.2.0",
+  "version": "12.0.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ShipfoxHQ/shipfox.git",
@@ -21,25 +21,25 @@
     "@temporalio/workflow": "1.18.1",
     "drizzle-orm": "^0.45.2",
     "zod": "^4.4.3",
-    "@shipfox/api-common-dto": "9.2.0",
-    "@shipfox/api-runners-dto": "10.2.0",
-    "@shipfox/api-workflows-dto": "10.0.0",
-    "@shipfox/inter-module": "0.2.2",
+    "@shipfox/api-common-dto": "12.0.0",
+    "@shipfox/api-auth-context": "12.0.0",
+    "@shipfox/api-auth-dto": "12.0.0",
+    "@shipfox/api-runners-dto": "12.0.0",
+    "@shipfox/api-workflows-dto": "12.0.0",
     "@shipfox/config": "1.2.4",
-    "@shipfox/node-drizzle": "0.3.4",
+    "@shipfox/inter-module": "0.2.3",
+    "@shipfox/node-drizzle": "0.3.5",
     "@shipfox/node-error-monitoring": "0.3.0",
-    "@shipfox/node-module": "1.0.4",
     "@shipfox/node-auth-root-key": "0.2.3",
-    "@shipfox/node-fastify": "0.4.0",
+    "@shipfox/node-fastify": "0.4.1",
+    "@shipfox/node-module": "1.0.5",
     "@shipfox/node-opentelemetry": "0.6.3",
     "@shipfox/node-outbox": "0.2.6",
-    "@shipfox/node-postgres": "0.4.4",
-    "@shipfox/node-rate-limit": "0.3.2",
+    "@shipfox/node-postgres": "0.5.0",
+    "@shipfox/node-rate-limit": "0.4.0",
     "@shipfox/node-temporal": "0.4.4",
     "@shipfox/node-tokens": "0.3.2",
-    "@shipfox/runner-labels": "0.1.3",
-    "@shipfox/api-auth-dto": "10.2.0",
-    "@shipfox/api-auth-context": "10.2.0"
+    "@shipfox/runner-labels": "0.1.3"
   },
   "imports": {
     "#*": "./dist/*"

package/src/db/job-executions.test.ts

@@ -3,6 +3,7 @@ import {
   RUNNER_JOB_LEASE_EXPIRED,
   RUNNER_JOB_QUEUED,
 } from '@shipfox/api-runners-dto';
+import {pgClient} from '@shipfox/node-postgres';
 import {eq, sql} from 'drizzle-orm';
 import {EmptyRequiredLabelsError, RunnerSessionExhaustedError} from '#core/errors.js';
 import {claimJobExecution} from '#core/job-executions.js';
@@ -21,9 +22,9 @@ import {
   expireStuckJobExecutions,
   getJobExecutionQueueDepth,
   isJobLeaseActive,
+  reconcileTerminalJobExecution,
   recordHeartbeat,
   releaseJobExecution,
-  requestJobExecutionCancellation,
 } from './job-executions.js';
 import {runnersOutbox} from './schema/outbox.js';
 import {pendingJobExecutions} from './schema/pending-job-executions.js';
@@ -486,6 +487,40 @@ describe('claimPendingJobExecution', () => {
     expect(claimed).toHaveLength(1);
   });
 
+  it('locks only the FIFO candidate before acquiring its execution advisory lock', async () => {
+    const first = await pendingJobFactory.create({workspaceId});
+    const second = await pendingJobFactory.create({workspaceId});
+    const releaseLock = deferred<void>();
+    const lockReady = deferred<void>();
+    const lockHolder = db().transaction(async (tx) => {
+      await tx.execute(
+        sql`select pg_advisory_xact_lock(hashtext(${`runners_job_execution:${first.jobExecutionId}`}))`,
+      );
+      lockReady.resolve();
+      await releaseLock.promise;
+    });
+
+    try {
+      await lockReady.promise;
+      expect(
+        await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null}),
+      ).toBeNull();
+      expect(
+        await db()
+          .select()
+          .from(pendingJobExecutions)
+          .where(eq(pendingJobExecutions.workspaceId, workspaceId)),
+      ).toHaveLength(2);
+    } finally {
+      releaseLock.resolve();
+      await lockHolder;
+    }
+
+    const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    expect(claimed?.jobExecutionId).toBe(first.jobExecutionId);
+    expect(second.jobExecutionId).not.toBe(claimed?.jobExecutionId);
+  });
+
   it('claims the oldest job first', async () => {
     const older = await pendingJobFactory.create({workspaceId});
     await pendingJobFactory.create({workspaceId});
@@ -975,11 +1010,11 @@ describe('recordHeartbeat', () => {
     );
   });
 
-  it('returns cancel:true after requestJobExecutionCancellation', async () => {
+  it('returns cancel:true after reconcileTerminalJobExecution', async () => {
     await pendingJobFactory.create({workspaceId});
     const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
 
-    await requestJobExecutionCancellation({jobExecutionId: claimed?.jobExecutionId as string});
+    await reconcileTerminalJobExecution({jobExecutionId: claimed?.jobExecutionId as string});
 
     const result = await recordHeartbeat({
       jobExecutionId: claimed?.jobExecutionId as string,
@@ -1016,7 +1051,7 @@ describe('recordHeartbeat', () => {
   });
 });
 
-describe('requestJobExecutionCancellation', () => {
+describe('reconcileTerminalJobExecution', () => {
   let workspaceId: string;
   let runnerSessionId: string;
 
@@ -1026,47 +1061,194 @@ describe('requestJobExecutionCancellation', () => {
     runnerSessionId = runnerSession.id;
   });
 
-  it('sets cancellation_requested_at on a fresh row', async () => {
-    await pendingJobFactory.create({workspaceId});
+  it('deletes a pending execution', async () => {
+    const pending = await pendingJobFactory.create({workspaceId});
+
+    await reconcileTerminalJobExecution({jobExecutionId: pending.jobExecutionId});
+
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.jobExecutionId, pending.jobExecutionId)),
+    ).toHaveLength(0);
+  });
+
+  it('sets cancellation_requested_at on a running execution', async () => {
+    const pending = await pendingJobFactory.create({workspaceId});
     const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    expect(claimed?.jobExecutionId).toBe(pending.jobExecutionId);
 
-    await requestJobExecutionCancellation({jobExecutionId: claimed?.jobExecutionId as string});
+    await reconcileTerminalJobExecution({jobExecutionId: pending.jobExecutionId});
 
     const rows = await db()
       .select()
       .from(runningJobExecutions)
-      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+      .where(eq(runningJobExecutions.jobExecutionId, pending.jobExecutionId));
     expect(rows[0]?.cancellationRequestedAt).not.toBeNull();
   });
 
   it('is idempotent: second call preserves the first timestamp', async () => {
-    await pendingJobFactory.create({workspaceId});
+    const pending = await pendingJobFactory.create({workspaceId});
     const claimed = await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+    expect(claimed?.jobExecutionId).toBe(pending.jobExecutionId);
 
-    await requestJobExecutionCancellation({jobExecutionId: claimed?.jobExecutionId as string});
+    await reconcileTerminalJobExecution({jobExecutionId: pending.jobExecutionId});
     const after1 = await db()
       .select()
       .from(runningJobExecutions)
-      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+      .where(eq(runningJobExecutions.jobExecutionId, pending.jobExecutionId));
     const firstTs = after1[0]?.cancellationRequestedAt;
 
     await new Promise((r) => setTimeout(r, 10));
-    await requestJobExecutionCancellation({jobExecutionId: claimed?.jobExecutionId as string});
+    await reconcileTerminalJobExecution({jobExecutionId: pending.jobExecutionId});
 
     const after2 = await db()
       .select()
       .from(runningJobExecutions)
-      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+      .where(eq(runningJobExecutions.jobExecutionId, pending.jobExecutionId));
     expect(after2[0]?.cancellationRequestedAt?.getTime()).toBe(firstTs?.getTime());
   });
 
-  it('is a no-op when the job execution is missing (does not throw)', async () => {
+  it('is a no-op when the job execution is missing', async () => {
     await expect(
-      requestJobExecutionCancellation({jobExecutionId: crypto.randomUUID()}),
+      reconcileTerminalJobExecution({jobExecutionId: crypto.randomUUID()}),
     ).resolves.toBeUndefined();
   });
+
+  it('leaves a pending sibling for the same job untouched', async () => {
+    const target = await pendingJobFactory.create({workspaceId});
+    const sibling = await pendingJobFactory.create({workspaceId, jobId: target.jobId});
+
+    await reconcileTerminalJobExecution({jobExecutionId: target.jobExecutionId});
+
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.jobExecutionId, target.jobExecutionId)),
+    ).toHaveLength(0);
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.jobExecutionId, sibling.jobExecutionId)),
+    ).toHaveLength(1);
+  });
+
+  it('leaves a running sibling for the same job uncancelled', async () => {
+    const target = await pendingJobFactory.create({workspaceId});
+    const sibling = await pendingJobFactory.create({workspaceId, jobId: target.jobId});
+    const targetClaim = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId,
+      maxClaims: null,
+    });
+    const siblingClaim = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId,
+      maxClaims: null,
+    });
+    expect(targetClaim?.jobExecutionId).toBe(target.jobExecutionId);
+    expect(siblingClaim?.jobExecutionId).toBe(sibling.jobExecutionId);
+
+    await reconcileTerminalJobExecution({jobExecutionId: target.jobExecutionId});
+
+    const rows = await db()
+      .select({
+        jobExecutionId: runningJobExecutions.jobExecutionId,
+        cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,
+      })
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, target.jobId));
+    const byJobExecutionId = new Map(
+      rows.map((row) => [row.jobExecutionId, row.cancellationRequestedAt]),
+    );
+    expect(byJobExecutionId.get(target.jobExecutionId)).not.toBeNull();
+    expect(byJobExecutionId.get(sibling.jobExecutionId)).toBeNull();
+  });
+
+  it('cancels the lease when reconciliation races a claim that has already acquired the row', async () => {
+    const pending = await pendingJobFactory.create({workspaceId});
+    const releaseClaim = deferred<void>();
+    const claimTransactionReady = deferred<void>();
+    const lockHolder = db().transaction(async (tx) => {
+      await tx.execute(sql`LOCK TABLE runners_outbox IN ACCESS EXCLUSIVE MODE`);
+      claimTransactionReady.resolve();
+      await releaseClaim.promise;
+    });
+
+    let claim: ReturnType<typeof claimPendingJobExecution> | undefined;
+    let reconciliation: Promise<void> | undefined;
+    try {
+      await claimTransactionReady.promise;
+      claim = claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null});
+      await waitForLockWait({queryLike: '%runners_outbox%'});
+
+      reconciliation = reconcileTerminalJobExecution({jobExecutionId: pending.jobExecutionId});
+      // Claim has already deleted the pending row and inserted the lease, but cannot commit its
+      // outbox event while the table lock is held. Reconciliation must wait on that same claim's
+      // execution advisory lock.
+      await waitForLockWait({queryLike: '%pg_advisory_xact_lock%'});
+    } finally {
+      releaseClaim.resolve();
+      await Promise.allSettled([
+        lockHolder,
+        claim ?? Promise.resolve(null),
+        reconciliation ?? Promise.resolve(),
+      ]);
+    }
+
+    if (!claim || !reconciliation) throw new Error('Claim and reconciliation must both start');
+    const [claimed] = await Promise.all([claim, reconciliation, lockHolder]);
+    expect(claimed?.jobExecutionId).toBe(pending.jobExecutionId);
+
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.jobExecutionId, pending.jobExecutionId)),
+    ).toHaveLength(0);
+    const runningRows = await db()
+      .select({cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt})
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobExecutionId, pending.jobExecutionId));
+    expect(runningRows).toHaveLength(1);
+    expect(runningRows[0]?.cancellationRequestedAt).not.toBeNull();
+  });
 });
 
+function deferred<T>() {
+  let resolve!: (value: T | PromiseLike<T>) => void;
+  let reject!: (reason?: unknown) => void;
+  const promise = new Promise<T>((innerResolve, innerReject) => {
+    resolve = innerResolve;
+    reject = innerReject;
+  });
+  return {promise, resolve, reject};
+}
+
+async function waitForLockWait(params: {queryLike: string}) {
+  const deadline = Date.now() + 2_000;
+  while (Date.now() < deadline) {
+    const result = await pgClient().query<{count: number}>(
+      `
+        SELECT count(*)::int AS count
+        FROM pg_stat_activity
+        WHERE datname = current_database()
+          AND pid <> pg_backend_pid()
+          AND state = 'active'
+          AND wait_event_type = 'Lock'
+          AND query ILIKE $1
+      `,
+      [params.queryLike],
+    );
+    if ((result.rows[0]?.count ?? 0) > 0) return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  throw new Error(`Timed out waiting for lock waiter matching ${params.queryLike}`);
+}
+
 describe('cancelRunnerJobs', () => {
   let workspaceId: string;
   let runnerSessionId: string;
@@ -1226,6 +1408,32 @@ describe('detectAndExpireStuckJobs', () => {
     expect(payload.steps).toBeUndefined();
   });
 
+  it('does not requeue an execution after its lease has expired', async () => {
+    const stale = await makeStaleJob(600);
+
+    await expireStuckJobExecutions({
+      noFirstHeartbeatGraceSeconds: 60,
+      thresholdSeconds: 180,
+    });
+
+    await enqueueJobExecution({
+      workspaceId,
+      workflowRunId: stale.workflowRunId,
+      workflowRunAttemptId: stale.workflowRunAttemptId,
+      jobId: stale.jobId,
+      jobExecutionId: stale.jobExecutionId,
+      projectId: stale.projectId,
+      requiredLabels: ['linux'],
+    });
+
+    expect(
+      await db()
+        .select()
+        .from(pendingJobExecutions)
+        .where(eq(pendingJobExecutions.jobExecutionId, stale.jobExecutionId)),
+    ).toHaveLength(0);
+  });
+
   it('expires a job that never sent a first heartbeat after the startup grace', async () => {
     const {jobId, workflowRunId, workflowRunAttemptId} = await makeNoFirstHeartbeatJob(90);
 
@@ -1338,7 +1546,7 @@ describe('detectAndExpireStuckJobs', () => {
   });
 
   it('skips a row whose heartbeat refreshed before the atomic DELETE re-evaluates the predicate', async () => {
-    // Pre-stale, then refresh, then run — the cutoff is folded into the DELETE's
+    // Pre-stale, then refresh, then run: the cutoff is folded into the DELETE's
     // WHERE so the live row survives even though the iteration SELECT saw it stale.
     const {jobId} = await makeStaleJob(600);
     await db()
@@ -1453,7 +1661,43 @@ describe('detectAndExpireStuckJobs', () => {
     expect(await outboxForJobs([stuck1.jobId, stuck2.jobId])).toHaveLength(2);
   });
 
-  it('a reaper tick and a concurrent claim of the same orphan-pending job leave consistent state', async () => {
+  it('skips an execution whose advisory lock is held and reaps the next stale page', async () => {
+    const first = await makeStaleJob(600);
+    const second = await makeStaleJob(600);
+    const releaseLock = deferred<void>();
+    const lockReady = deferred<void>();
+    const lockHolder = db().transaction(async (tx) => {
+      await tx.execute(
+        sql`select pg_advisory_xact_lock(hashtext(${`runners_job_execution:${first.jobExecutionId}`}))`,
+      );
+      lockReady.resolve();
+      await releaseLock.promise;
+    });
+
+    try {
+      await lockReady.promise;
+      const reaped = await expireStuckJobExecutions({
+        noFirstHeartbeatGraceSeconds: 60,
+        thresholdSeconds: 180,
+        limit: 1,
+      });
+
+      expect(reaped).toHaveLength(1);
+      expect(reaped[0]?.jobExecutionId).toBe(second.jobExecutionId);
+    } finally {
+      releaseLock.resolve();
+      await lockHolder;
+    }
+
+    const remaining = await expireStuckJobExecutions({
+      noFirstHeartbeatGraceSeconds: 60,
+      thresholdSeconds: 180,
+      limit: 1,
+    });
+    expect(remaining.map((row) => row.jobExecutionId)).toContain(first.jobExecutionId);
+  });
+
+  it('a reaper tick and a concurrent claim of the same orphan-pending job do not deadlock', async () => {
     const {jobId, jobExecutionId, workflowRunId, workflowRunAttemptId, projectId} =
       await makeStaleJob(600);
     // Orphan pending row from a post-claim enqueue retry for an already-running job.
@@ -1469,21 +1713,25 @@ describe('detectAndExpireStuckJobs', () => {
         requiredLabels: ['linux'],
       });
 
-    // The reaper locks running-then-pending while the claim locks pending-then-running;
-    // a deadlock loser rolls back, so either side may settle as rejected.
-    await Promise.allSettled([
+    // The reaper may acquire the execution lock first, or it may skip the row while the claim
+    // transaction holds it. Either interleaving must complete without deadlocking.
+    const [firstReap, claimed] = await Promise.all([
       detectAndExpireStuckJobs({thresholdSeconds: 180}),
       claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null}),
     ]);
 
-    // A follow-up tick finishes any reap that lost a deadlock race.
-    await detectAndExpireStuckJobs({thresholdSeconds: 180});
+    // Once the contending claim settles, the next tick reaps a row that the non-blocking
+    // advisory-lock scan intentionally skipped.
+    const secondReap = await detectAndExpireStuckJobs({thresholdSeconds: 180});
 
+    expect(claimed).toBeNull();
+    expect(firstReap.expired + secondReap.expired).toBeGreaterThanOrEqual(1);
     // The expired job is gone and not re-claimable; its orphan pending row is swept.
     expect(await runningJobsForTest()).toHaveLength(0);
     expect(
       await db().select().from(pendingJobExecutions).where(eq(pendingJobExecutions.jobId, jobId)),
     ).toHaveLength(0);
+    expect(await outboxForJobs([jobId])).toHaveLength(1);
     expect(
       await claimPendingJobExecution({workspaceId, runnerSessionId, maxClaims: null}),
     ).toBeNull();

package/src/db/job-executions.ts

@@ -38,6 +38,14 @@ import {providerRunners} from './schema/runner-instances.js';
 import {runnerSessions} from './schema/runner-sessions.js';
 import {runningJobExecutions} from './schema/running-job-executions.js';
 
+const runnerJobExecutionLockPrefix = 'runners_job_execution:';
+
+async function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {
+  await tx.execute(
+    sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`,
+  );
+}
+
 export interface EnqueueJobExecutionParams {
   workspaceId: string;
   workflowRunId: string;
@@ -77,17 +85,29 @@ export async function getWorkspaceJobCounts(params: {
 }
 
 // Idempotent while the job execution is still pending: a duplicate jobExecutionId already in
-// `runners_pending_jobs` is a no-op. Temporal retries the enqueue activity
-// at-least-once, so a unique-violation throw on a retry-after-lost-result
-// would permanently fail a healthy job execution's workflow. The guard does not extend
-// past the claim: once the job execution has moved to `runners_running_jobs`, a retry
-// can reinsert an orphan pending row, which a later `claimPendingJobExecution` drops
-// via the running-execution unique constraint (onConflictDoNothing) instead of failing.
+// `runners_pending_jobs` is a no-op. Temporal retries the enqueue activity at-least-once, so a
+// unique-violation throw on a retry-after-lost-result would permanently fail a healthy execution.
+// The per-execution advisory lock serializes retries with lease expiry/reconciliation, and the
+// durable lease-expired event prevents a retry from re-queueing an execution already reaped.
 export async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {
   const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];
   if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();
 
   const enqueued = await db().transaction(async (tx) => {
+    await lockJobExecution(tx, params.jobExecutionId);
+
+    const [leaseExpired] = await tx
+      .select({id: runnersOutbox.id})
+      .from(runnersOutbox)
+      .where(
+        and(
+          eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED),
+          sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`,
+        ),
+      )
+      .limit(1);
+    if (leaseExpired) return false;
+
     const [inserted] = await tx
       .insert(pendingJobExecutions)
       .values({
@@ -209,8 +229,10 @@ export async function claimPendingJobExecution(params: {
     }
 
     // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker
-    // for rows sharing a created_at within a batch.
-    const [row] = await tx
+    // for rows sharing a created_at within a batch. Lock only the FIFO candidate before
+    // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this
+    // predicate would evaluate it while scanning and temporarily lock many queue entries.
+    const candidate = tx
       .select()
       .from(pendingJobExecutions)
       .where(
@@ -221,7 +243,19 @@ export async function claimPendingJobExecution(params: {
       )
       .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))
       .limit(1)
-      .for('update', {skipLocked: true});
+      .for('update', {skipLocked: true})
+      .as('pending_candidate');
+
+    const [row] = await tx
+      .select()
+      .from(candidate)
+      .where(
+        sql`
+          pg_try_advisory_xact_lock(
+            hashtext(${runnerJobExecutionLockPrefix} || ${candidate.jobExecutionId}::text)
+          )
+        `,
+      );
 
     if (!row) return null;
 
@@ -324,12 +358,14 @@ async function touchRunnerSessionLiveness(params: {
  * Releases a job execution's lease when the orchestration workflow finalizes it: deletes the
  * running-job-execution row AND any lingering pending row for the same execution, in one tx.
  * Idempotent (0-row no-op), no token scope (the workflow is authoritative), and
- * emits no event — the workflow already owns the outcome. Sweeping the pending row
+ * emits no event: the workflow already owns the outcome. Sweeping the pending row
  * too closes the at-least-once window where an enqueue retry left an orphan that a
  * later claim would otherwise pick up for an already-finished job execution.
  */
 export async function releaseJobExecution(params: {jobExecutionId: string}): Promise<void> {
   await db().transaction(async (tx) => {
+    await lockJobExecution(tx, params.jobExecutionId);
+
     // Delete pending before running to match `claimPendingJobExecution`'s lock-acquisition
     // order (it locks the pending row first, then the running row). A concurrent
     // claim picking up an orphan pending row for this same job execution would otherwise
@@ -352,9 +388,9 @@ export async function releaseJobExecution(params: {jobExecutionId: string}): Pro
  * pending row: a failed best-effort `releaseJobExecution` would otherwise leave an orphan
  * that a later claim re-runs as an already-finished job execution.
  *
- * Locks running-then-pending, the inverse of `claimPendingJobExecution` / `releaseJobExecution`.
- * That pre-existing asymmetry opens a narrow deadlock window against a concurrent
- * claim of the same orphan-pending job execution; Postgres breaks it and the cron retries.
+ * Locks pending-then-running to match `claimPendingJobExecution`, `releaseJobExecution`, and
+ * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock running
+ * rows first; the running-row DELETE re-checks the stale predicate after the pending-row sweep.
  */
 export async function expireStuckJobExecutions(params: {
   thresholdSeconds: number;
@@ -386,17 +422,42 @@ export async function expireStuckJobExecutions(params: {
       ),
     );
 
-    const staleIds = tx
-      .select({id: runningJobExecutions.id})
+    const staleRows = await tx
+      .select({
+        id: runningJobExecutions.id,
+        jobExecutionId: runningJobExecutions.jobExecutionId,
+      })
       .from(runningJobExecutions)
-      .where(stalePredicate)
+      .where(
+        and(
+          stalePredicate,
+          sql`
+            pg_try_advisory_xact_lock(
+              hashtext(${runnerJobExecutionLockPrefix} || ${runningJobExecutions.jobExecutionId}::text)
+            )
+          `,
+        ),
+      )
       .orderBy(
         asc(
           sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,
         ),
+        asc(runningJobExecutions.id),
       )
-      .limit(params.limit ?? 100)
-      .for('update', {skipLocked: true});
+      .limit(params.limit ?? 100);
+
+    if (staleRows.length === 0) return [];
+
+    const staleIds = staleRows.map((row) => row.id);
+    const staleJobExecutionIds = staleRows.map((row) => row.jobExecutionId);
+
+    // Sweep pending rows first so this transaction cannot hold a running-row lock while waiting
+    // for a pending-row lock held by reconciliation or a claim of an orphan pending row. The
+    // advisory lock acquired in the candidate scan also makes enqueue retries wait until this
+    // transaction has either reaped the execution or released its lock without selecting it.
+    await tx
+      .delete(pendingJobExecutions)
+      .where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));
 
     const deleted = await tx
       .delete(runningJobExecutions)
@@ -410,13 +471,6 @@ export async function expireStuckJobExecutions(params: {
 
     if (deleted.length === 0) return [];
 
-    await tx.delete(pendingJobExecutions).where(
-      inArray(
-        pendingJobExecutions.jobExecutionId,
-        deleted.map((row) => row.jobExecutionId),
-      ),
-    );
-
     await writeOutboxEvents<RunnersEventMap>(
       tx,
       runnersOutbox,
@@ -677,15 +731,30 @@ export async function recordHeartbeat(params: {
   };
 }
 
-export async function requestJobExecutionCancellation(params: {
+/**
+ * Reconciles a terminal job execution with runner state in one transaction: removes its pending
+ * queue row and requests cancellation on its running lease, if either exists. The operation is
+ * idempotent and preserves the first cancellation-request timestamp.
+ */
+export async function reconcileTerminalJobExecution(params: {
   jobExecutionId: string;
 }): Promise<void> {
-  await db()
-    .update(runningJobExecutions)
-    .set({
-      cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,
-    })
-    .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));
+  await db().transaction(async (tx) => {
+    await lockJobExecution(tx, params.jobExecutionId);
+
+    // Delete pending before updating running to match claim/release lock order. Claim locks
+    // pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes
+    // a concurrent terminal reconciliation either win before claim or cancel the new lease.
+    await tx
+      .delete(pendingJobExecutions)
+      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));
+    await tx
+      .update(runningJobExecutions)
+      .set({
+        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,
+      })
+      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));
+  });
 }
 
 export async function cancelRunnerJobs(params: {jobIds: string[]}): Promise<void> {