@shipfox/api-workflows 12.7.0 → 13.1.0

package/src/temporal/workflows/job-execution-orchestration.ts

@@ -26,14 +26,14 @@ import {remainingMs} from './deadline.js';
  * execution pending while it is queued, then performs the versioned pending → running
  * transition after the current claim. One deadline spans both waits; claim never resets
  * it. Signals can arrive in any order, so the precedence is finished > lease expired >
- * claimed > timeout. Lease cleanup remains best-effort: a runners DB outage must never
- * block the child workflow from returning the job outcome to run-orchestration.
+ * claimed > timeout. Workflows commits queue and terminal facts with its state transitions;
+ * runners reconciles its local queue, lease, and reservation projections asynchronously.
  */
 
 const {
   setJobStatus,
   setJobExecutionStatus,
-  enqueueJobExecutionForRunner,
+  queueJobExecutionActivity,
   bulkSetStepStatuses,
   failJobExecutionAsTimedOutActivity,
   resolveLeaseExpiredJobExecutionActivity,
@@ -48,14 +48,6 @@ const {resolveJobStatusFromJobExecutionsActivity} = proxyActivities<
   retry: {maximumAttempts: 5},
 });
 
-// Lease cleanup gets a bounded retry policy of its own: after a few attempts the
-// workflow stops waiting on it (see releaseLeaseBestEffort) rather than blocking
-// the DAG on runners-side availability.
-const {releaseLeaseActivity} = proxyActivities<ReturnType<typeof createOrchestrationActivities>>({
-  startToCloseTimeout: '30s',
-  retry: {maximumAttempts: 5},
-});
-
 const DEFAULT_EXECUTION_MAX_DURATION_MS = 6 * 60 * 60 * 1000;
 
 export const jobFinishedSignal =
@@ -71,12 +63,9 @@ export interface JobClaimedSignalPayload {
 export const jobClaimedSignal = defineSignal<[JobClaimedSignalPayload]>(JOB_CLAIMED_SIGNAL);
 
 export interface JobExecutionOrchestrationInput {
-  workspaceId: string;
-  workflowRunId: string;
   runAttemptId: string;
   jobId: string;
   jobExecutionId: string;
-  projectId: string;
   jobVersion: number;
   executionVersion: number;
   executionTimeoutMs?: number | null | undefined;
@@ -89,17 +78,6 @@ export interface JobExecutionOrchestrationResult {
   jobVersion: number;
 }
 
-async function releaseLeaseBestEffort(jobExecutionId: string): Promise<void> {
-  try {
-    await releaseLeaseActivity({jobExecutionId});
-  } catch (err) {
-    log.warn('lease release failed; stuck detector will reap the row', {
-      jobExecutionId,
-      error: String(err),
-    });
-  }
-}
-
 async function resolveJobStatusOrFailClosed(
   input: JobExecutionOrchestrationInput,
 ): Promise<{status: RuntimeCompletionStatus; jobVersion: number}> {
@@ -121,7 +99,7 @@ async function resolveJobStatusOrFailClosed(
   }
 }
 
-async function enqueueJobExecution(input: JobExecutionOrchestrationInput): Promise<void> {
+async function queueJobExecution(input: JobExecutionOrchestrationInput) {
   if (hasNoRequiredRunnerLabels(input.requiredLabels)) {
     throw ApplicationFailure.nonRetryable(
       `Job ${input.jobId} has no required runner labels`,
@@ -129,15 +107,12 @@ async function enqueueJobExecution(input: JobExecutionOrchestrationInput): Promi
     );
   }
 
-  await enqueueJobExecutionForRunner({
-    workspaceId: input.workspaceId,
-    workflowRunId: input.workflowRunId,
-    runAttemptId: input.runAttemptId,
-    jobId: input.jobId,
-    jobExecutionId: input.jobExecutionId,
-    projectId: input.projectId,
-    requiredLabels: input.requiredLabels,
-  });
+  return jobExecutionStartOutcome(
+    await queueJobExecutionActivity({
+      jobId: input.jobId,
+      jobExecutionId: input.jobExecutionId,
+    }),
+  );
 }
 
 async function markJobExecutionRunning(
@@ -218,7 +193,6 @@ async function resolveFinishedJobExecution({
       terminationReason: 'finished',
       status,
     });
-    await releaseLeaseBestEffort(input.jobExecutionId);
     return {status, jobVersion: input.jobVersion};
   }
   const resolved = await resolveJobStatusOrFailClosed(input);
@@ -228,7 +202,6 @@ async function resolveFinishedJobExecution({
     terminationReason: 'finished',
     status: resolved.status,
   });
-  await releaseLeaseBestEffort(input.jobExecutionId);
   return resolved;
 }
 
@@ -257,7 +230,6 @@ async function resolveLeaseExpiredJobExecution({
       terminationReason: 'lease_expired',
       status: leaseExpired.status,
     });
-    await releaseLeaseBestEffort(input.jobExecutionId);
     return {status: leaseExpired.status, jobVersion: input.jobVersion};
   }
   const {status, jobVersion} = await resolveJobStatusOrFailClosed(input);
@@ -267,13 +239,12 @@ async function resolveLeaseExpiredJobExecution({
     terminationReason: 'lease_expired',
     status,
   });
-  await releaseLeaseBestEffort(input.jobExecutionId);
   return {status, jobVersion};
 }
 
-// Timeout backstop. The activity atomically fails the execution, marks
-// `timed_out_at`, and enqueues WORKFLOWS_JOB_EXECUTION_TIMED_OUT; the runners subscriber
-// then asks the runner to cancel. The lease is intentionally NOT released here.
+// Timeout backstop. The activity atomically fails the execution, marks `timed_out_at`, and
+// enqueues the generic terminal execution fact. The runners subscriber then asks the runner to
+// cancel. The lease is intentionally NOT released here.
 async function resolveTimedOutJobExecution({
   input,
   runningVersion,
@@ -314,7 +285,10 @@ export async function jobExecutionOrchestration(
   // Register every signal before enqueue can block or publish a claim/outcome event. The
   // handlers retain signals that arrive while the enqueue activity is in flight.
   registerJobExecutionSignalHandlers(input.jobExecutionId, signals);
-  await enqueueJobExecution(input);
+  const queued = await queueJobExecution(input);
+  if (queued.kind === 'terminal') {
+    return {status: queued.result.status, jobVersion: input.jobVersion};
+  }
 
   const timeoutMs = input.executionTimeoutMs ?? DEFAULT_EXECUTION_MAX_DURATION_MS;
   const deadline = Date.now() + timeoutMs;

package/src/temporal/workflows/job-listener-orchestration-continuation.test.ts

@@ -63,9 +63,6 @@ describe('listener continue-as-new guard', () => {
 
   it('carries listener loop state into the continued run', () => {
     const input: JobListenerOrchestrationInput = {
-      workspaceId: 'workspace-1',
-      workflowRunId: 'run-1',
-      projectId: 'project-1',
       runAttemptId: 'attempt-1',
       jobId: 'job-1',
       jobVersion: 3,

package/src/temporal/workflows/job-listener-orchestration.test.ts

@@ -197,7 +197,7 @@ describe('jobListenerOrchestration', () => {
 
       expect(listenerFiringOutcomeCalls()).toHaveLength(0);
       expect(resolveJobListenerCalls().map((c) => c.params.reason)).toEqual(['until']);
-      expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(0);
+      expect(callsNamed('queueJobExecutionActivity')).toHaveLength(0);
     });
 
     test('resolves with timeout when the listening deadline elapses', async () => {
@@ -233,9 +233,10 @@ describe('jobListenerOrchestration', () => {
 
       await runListener({maxExecutions: 1});
 
-      expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(1);
-      expect(callsNamed('enqueueJobExecutionForRunner')[0]?.params).toMatchObject({
-        requiredLabels: ['gpu', 'linux'],
+      expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
+      expect(callsNamed('queueJobExecutionActivity')[0]?.params).toMatchObject({
+        jobId,
+        jobExecutionId: 'exec-1',
       });
       expect(listenerFiringOutcomeCalls().map((c) => c.params.outcome)).toEqual(['succeeded']);
     });
@@ -250,7 +251,7 @@ describe('jobListenerOrchestration', () => {
       await runListener({maxExecutions: 1});
 
       // The child actually ran (enqueued), then returned failed.
-      expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(1);
+      expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
       expect(listenerFiringOutcomeCalls().map((c) => c.params.outcome)).toEqual(['failed']);
       expect(resolveJobListenerCalls().map((c) => c.params.reason)).toEqual(['max_executions']);
     });
@@ -265,7 +266,7 @@ describe('jobListenerOrchestration', () => {
       await runListener({maxExecutions: 2});
 
       // Sequence 1 failed at materialization (no child enqueued); sequence 2 ran and succeeded.
-      expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(1);
+      expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
       expect(listenerFiringOutcomeCalls().map((c) => c.params.outcome)).toEqual([
         'failed',
         'succeeded',
@@ -369,15 +370,12 @@ describe('jobListenerOrchestration', () => {
       });
 
       const handle = await startListener({onResolve: 'cancel'});
-      await waitForActivity('enqueueJobExecutionForRunner');
+      await waitForActivity('queueJobExecutionActivity');
       await handle.signal(LISTENER_RESOLVE_SIGNAL);
       await handle.result();
 
       expect(settleListenerCalls().map((c) => c.params.status)).toEqual(['cancelled']);
       expect(listenerFiringOutcomeCalls().map((c) => c.params.outcome)).toEqual(['cancelled']);
-      expect(callsNamed('cancelRunnerJobsActivity')).toEqual([
-        {name: 'cancelRunnerJobsActivity', params: {jobIds: [jobId]}},
-      ]);
       expect(resolveJobListenerCalls().map((c) => c.params.reason)).toEqual(['until']);
     });
 
@@ -390,7 +388,7 @@ describe('jobListenerOrchestration', () => {
       });
 
       const handle = await startListener({onResolve: 'finish'});
-      await waitForActivity('enqueueJobExecutionForRunner');
+      await waitForActivity('queueJobExecutionActivity');
       // Resolution arrives mid-firing; finish mode must wait for the child, not cancel it.
       await handle.signal(LISTENER_RESOLVE_SIGNAL);
       const child = testEnv.client.workflow.getHandle(`job:${jobId}`);
@@ -398,7 +396,6 @@ describe('jobListenerOrchestration', () => {
       await handle.result();
 
       expect(settleListenerCalls()).toHaveLength(0);
-      expect(callsNamed('cancelRunnerJobsActivity')).toHaveLength(0);
       expect(listenerFiringOutcomeCalls().map((c) => c.params.outcome)).toEqual(['succeeded']);
       expect(resolveJobListenerCalls().map((c) => c.params.reason)).toEqual(['until']);
     });

package/src/temporal/workflows/job-listener-orchestration.ts

@@ -25,7 +25,6 @@ const {
   resolveJobListenerActivity,
   settleListenerJobExecutionActivity,
   recordListenerFiringOutcomeActivity,
-  cancelRunnerJobsActivity,
 } = proxyActivities<ReturnType<typeof createOrchestrationActivities>>({
   startToCloseTimeout: '30s',
 });
@@ -34,9 +33,6 @@ export const listenerEventsAvailableSignal = defineSignal<[]>(LISTENER_EVENTS_AV
 export const listenerResolveSignal = defineSignal<[]>(LISTENER_RESOLVE_SIGNAL);
 
 export interface JobListenerOrchestrationInput {
-  workspaceId: string;
-  workflowRunId: string;
-  projectId: string;
   runAttemptId: string;
   jobId: string;
   jobVersion: number;
@@ -413,9 +409,6 @@ async function runListenerExecution(params: {
       workflowIdReusePolicy: 'ALLOW_DUPLICATE',
       args: [
         {
-          workspaceId: params.input.workspaceId,
-          workflowRunId: params.input.workflowRunId,
-          projectId: params.input.projectId,
           jobId: params.input.jobId,
           jobExecutionId: params.jobExecutionId,
           runAttemptId: params.input.runAttemptId,
@@ -447,7 +440,6 @@ async function runListenerExecution(params: {
         status: 'cancelled',
       });
       await recordListenerFiringOutcomeActivity({outcome: 'cancelled'});
-      await cancelRunnerJobsActivity({jobIds: [params.input.jobId]});
       return;
     }
   }

package/src/temporal/workflows/run-orchestration.test.ts

@@ -59,17 +59,13 @@ describe('runOrchestration', () => {
 
     const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
     expect(runAttemptStatuses).toEqual(['running', 'succeeded']);
-    const enqueueCalls = callsNamed('enqueueJobExecutionForRunner');
+    const enqueueCalls = callsNamed('queueJobExecutionActivity');
     expect(enqueueCalls).toHaveLength(3);
-    // The lease tuple is sourced from the loaded dag (workspace/project/run together).
-    for (const call of enqueueCalls) {
-      expect(call.params).toMatchObject({
-        workspaceId: 'workspace-1',
-        projectId: 'project-1',
-        runAttemptId: 'r1-attempt-1',
-        requiredLabels: ['ubuntu22'],
-      });
-    }
+    expect(enqueueCalls.map((call) => call.params)).toEqual([
+      {jobId: 'j1', jobExecutionId: 'j1'},
+      {jobId: 'j2', jobExecutionId: 'j2'},
+      {jobId: 'j3', jobExecutionId: 'j3'},
+    ]);
   });
 
   test('already-succeeded upstream jobs are not re-enqueued', async () => {
@@ -83,7 +79,7 @@ describe('runOrchestration', () => {
 
     const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
     expect(runAttemptStatuses).toEqual(['running', 'succeeded']);
-    const enqueueCalls = callsNamed('enqueueJobExecutionForRunner');
+    const enqueueCalls = callsNamed('queueJobExecutionActivity');
     expect(enqueueCalls).toHaveLength(1);
     expect(enqueueCalls[0]?.params).toMatchObject({jobId: 'j2'});
   });
@@ -99,7 +95,7 @@ describe('runOrchestration', () => {
 
     const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
     expect(runAttemptStatuses).toEqual(['running', 'failed']);
-    const enqueueCalls = callsNamed('enqueueJobExecutionForRunner');
+    const enqueueCalls = callsNamed('queueJobExecutionActivity');
     expect(enqueueCalls).toHaveLength(1);
     expect(enqueueCalls[0]?.params).toMatchObject({jobId: 'j2'});
   });
@@ -116,7 +112,7 @@ describe('runOrchestration', () => {
 
     const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
     expect(runAttemptStatuses).toEqual(['running', 'succeeded']);
-    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(3);
+    expect(callsNamed('queueJobExecutionActivity')).toHaveLength(3);
   });
 
   test('single job fails, dependents are skipped', async () => {
@@ -132,7 +128,7 @@ describe('runOrchestration', () => {
     const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
     expect(runAttemptStatuses).toEqual(['running', 'failed']);
 
-    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(1);
+    expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
     const jobStatuses = setJobStatusCalls().map((c) => ({
       id: c.params.jobId,
       status: c.params.status,
@@ -162,7 +158,7 @@ describe('runOrchestration', () => {
 
     const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
     expect(runAttemptStatuses).toEqual(['running', 'failed']);
-    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(1);
+    expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
 
     const jobStatuses = setJobStatusCalls().map((c) => ({
       id: c.params.jobId,
@@ -199,7 +195,7 @@ describe('runOrchestration', () => {
         },
       },
     ]);
-    expect(callsNamed('enqueueJobExecutionForRunner').map((call) => call.params)).toEqual([
+    expect(callsNamed('queueJobExecutionActivity').map((call) => call.params)).toEqual([
       expect.objectContaining({jobId: 'j1'}),
       expect.objectContaining({jobId: 'j2'}),
     ]);
@@ -221,7 +217,7 @@ describe('runOrchestration', () => {
 
     await executeRun();
 
-    expect(callsNamed('enqueueJobExecutionForRunner').map((call) => call.params)).toEqual([
+    expect(callsNamed('queueJobExecutionActivity').map((call) => call.params)).toEqual([
       expect.objectContaining({jobId: 'j1'}),
     ]);
     expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual([
@@ -249,7 +245,7 @@ describe('runOrchestration', () => {
         },
       },
     ]);
-    const enqueuedJobs = callsNamed('enqueueJobExecutionForRunner').map((call) => call.params);
+    const enqueuedJobs = callsNamed('queueJobExecutionActivity').map((call) => call.params);
     expect(enqueuedJobs).toHaveLength(3);
     expect(enqueuedJobs).toEqual(
       expect.arrayContaining([
@@ -273,7 +269,7 @@ describe('runOrchestration', () => {
 
     await executeRun();
 
-    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(0);
+    expect(callsNamed('queueJobExecutionActivity')).toHaveLength(0);
     expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual([
       'running',
       'succeeded',
@@ -310,7 +306,7 @@ describe('runOrchestration', () => {
     expect(finalRunAttemptStatus?.params.version).toBeGreaterThan(0);
   });
 
-  test('cancel signal stops scheduling and fans out runner cancellation', async () => {
+  test('cancel signal stops scheduling without calling runners directly', async () => {
     const jobs = [dagJob('j1', 'build'), dagJob('j2', 'deploy', ['build'])];
     setCfg({dag: makeDag(jobs, 'r-cancel'), jobResults: new Map(), skipSignal: true});
 
@@ -319,16 +315,13 @@ describe('runOrchestration', () => {
       workflowId: `run:${workflowRunId}`,
       args: [{workflowRunId, workspaceId}],
     });
-    await waitForActivity('enqueueJobExecutionForRunner');
+    await waitForActivity('queueJobExecutionActivity');
 
     await handle.signal('run-cancel');
     await handle.result();
 
     expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual(['running']);
-    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(1);
-    expect(callsNamed('cancelRunnerJobsActivity')).toEqual([
-      {name: 'cancelRunnerJobsActivity', params: {jobIds: ['j1', 'j2']}},
-    ]);
+    expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
   });
 
   test('starts listening jobs and cancels them with the run', async () => {
@@ -347,10 +340,7 @@ describe('runOrchestration', () => {
     await handle.result();
 
     expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual(['running']);
-    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(0);
-    expect(callsNamed('cancelRunnerJobsActivity')).toEqual([
-      {name: 'cancelRunnerJobsActivity', params: {jobIds: ['j1']}},
-    ]);
+    expect(callsNamed('queueJobExecutionActivity')).toHaveLength(0);
   });
 
   test('aborts early when the initial running write reports an already-terminal run', async () => {
@@ -364,8 +354,7 @@ describe('runOrchestration', () => {
     await executeRun();
 
     expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual(['running']);
-    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(0);
-    expect(callsNamed('cancelRunnerJobsActivity')).toHaveLength(0);
+    expect(callsNamed('queueJobExecutionActivity')).toHaveLength(0);
   });
 
   test('diamond DAG with partial failure', async () => {
@@ -383,7 +372,7 @@ describe('runOrchestration', () => {
     expect(runAttemptStatuses).toEqual(['running', 'failed']);
 
     // A, B, C enqueued: D skipped because B failed
-    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(3);
+    expect(callsNamed('queueJobExecutionActivity')).toHaveLength(3);
     const jobStatuses = setJobStatusCalls().map((c) => ({
       id: c.params.jobId,
       status: c.params.status,
@@ -401,7 +390,7 @@ describe('runOrchestration', () => {
     setCfg({
       dag: makeDag(jobs, 'r7'),
       jobResults: new Map(),
-      enqueueError: 'Runner service unavailable',
+      queueError: 'Runner service unavailable',
     });
 
     await expect(executeRun()).rejects.toThrow();

package/src/temporal/workflows/run-orchestration.ts

@@ -9,7 +9,6 @@ import {
 } from '@temporalio/workflow';
 import {
   createRuntimeRunProgress,
-  nonCompletedRuntimeJobIds,
   type RuntimeRunProgress,
   recordRuntimeJobResult,
   recordSkippedRuntimeJob,
@@ -26,15 +25,10 @@ import {deadlineReached, remainingMs} from './deadline.js';
 import {jobExecutionOrchestration} from './job-execution-orchestration.js';
 import {jobListenerOrchestration} from './job-listener-orchestration.js';
 
-const {
-  loadRunAttemptDag,
-  evaluateJobActivationsActivity,
-  setRunAttemptStatus,
-  setJobStatus,
-  cancelRunnerJobsActivity,
-} = proxyActivities<ReturnType<typeof createOrchestrationActivities>>({
-  startToCloseTimeout: '30s',
-});
+const {loadRunAttemptDag, evaluateJobActivationsActivity, setRunAttemptStatus, setJobStatus} =
+  proxyActivities<ReturnType<typeof createOrchestrationActivities>>({
+    startToCloseTimeout: '30s',
+  });
 
 const {failRunAsTimedOutActivity} = proxyActivities<
   ReturnType<typeof createOrchestrationActivities>
@@ -74,12 +68,10 @@ export async function runOrchestration(input: RunOrchestrationInput): Promise<vo
 
   while (true) {
     if (cancelRequested) {
-      await cancelNonCompletedRunnerJobs(dag, progress);
       return;
     }
     if (deadlineReached(runDeadline)) {
       await failRunAsTimedOutActivity({runAttemptId: input.runAttemptId});
-      await cancelNonCompletedRunnerJobs(dag, progress);
       return;
     }
 
@@ -144,12 +136,10 @@ export async function runOrchestration(input: RunOrchestrationInput): Promise<vo
 
     const settled = await waitForNextSettlement(inFlight, () => cancelRequested, runDeadline);
     if (settled.kind === 'cancelled') {
-      await cancelNonCompletedRunnerJobs(dag, progress);
       return;
     }
     if (settled.kind === 'timed-out') {
       await failRunAsTimedOutActivity({runAttemptId: input.runAttemptId});
-      await cancelNonCompletedRunnerJobs(dag, progress);
       return;
     }
     inFlight.delete(settled.job.key);
@@ -187,9 +177,6 @@ function launchJob(
       workflowId: `job-listener:${job.id}`,
       args: [
         {
-          workspaceId: run.workspaceId,
-          workflowRunId: run.workflowRunId,
-          projectId: run.projectId,
           jobId: job.id,
           runAttemptId: run.runAttemptId,
           jobVersion: runtimeJobVersion(job, progress),
@@ -235,9 +222,6 @@ function launchJob(
     workflowId: `job:${job.id}`,
     args: [
       {
-        workspaceId: run.workspaceId,
-        workflowRunId: run.workflowRunId,
-        projectId: run.projectId,
         jobId: job.id,
         jobExecutionId: job.jobExecutionId,
         runAttemptId: run.runAttemptId,
@@ -273,11 +257,3 @@ async function waitForNextSettlement(
   );
   return await Promise.race([childSettled, cancel]);
 }
-
-async function cancelNonCompletedRunnerJobs(
-  dag: RunDag,
-  progress: RuntimeRunProgress,
-): Promise<void> {
-  const jobIds = nonCompletedRuntimeJobIds(dag.jobs, progress);
-  await cancelRunnerJobsActivity({jobIds});
-}

package/src/temporal/workflows/test-env.ts

@@ -25,11 +25,11 @@ export interface TestConfig {
   dag: RunDag;
   /** Map of jobId → status the mock runner should signal back */
   jobResults: Map<string, RuntimeCompletionStatus>;
-  /** If set, enqueueJobExecutionForRunner will throw with this message instead of signaling */
-  enqueueError?: string;
-  /** If true, enqueueJobExecutionForRunner sends two job-finished signals (for dedup testing) */
+  /** If set, queueJobExecutionActivity will throw with this message instead of signaling */
+  queueError?: string;
+  /** If true, queueJobExecutionActivity sends two job-finished signals (for dedup testing) */
   duplicateSignal?: boolean;
-  /** If true, enqueueJobExecutionForRunner does nothing without a signal for timeout testing */
+  /** If true, queueJobExecutionActivity does nothing without a signal for timeout testing */
   skipSignal?: boolean;
   /** If true, emit the claim but hold back the terminal outcome (for deadline testing) */
   skipOutcomeSignal?: boolean;
@@ -43,8 +43,6 @@ export interface TestConfig {
   leaseExpiredStatus?: RuntimeCompletionStatus;
   /** If set, resolveJobStatusFromJobExecutionsActivity throws with this message */
   resolveJobStatusError?: string;
-  /** If set, releaseLeaseActivity throws (non-retryable) to prove the workflow still returns */
-  releaseLeaseError?: string;
   /** Scripted job activation decisions keyed by job id; defaults to start */
   activationDecisions?: Map<string, JobActivationDecision>;
   /** If set, failJobExecutionAsTimedOutActivity throws (for timeout error-path testing) */
@@ -289,30 +287,19 @@ function createMockActivities() {
       calls.push({name: 'bulkSetStepStatuses', params});
     },
 
-    cancelRunnerJobsActivity: (params: {jobIds: string[]}) => {
-      calls.push({name: 'cancelRunnerJobsActivity', params});
-    },
-
     // Scheduling is step-less. The mock runner reports the job outcome by signalling
     // the per-step terminal-completion signal (job-finished) and/or the lease-expiry
     // signal, reproducing the outbox → subscriber → signal rail.
-    enqueueJobExecutionForRunner: async (params: {
-      workspaceId: string;
-      workflowRunId: string;
-      runAttemptId: string;
-      jobId: string;
-      jobExecutionId: string;
-      projectId: string;
-      requiredLabels: string[];
-    }) => {
-      calls.push({name: 'enqueueJobExecutionForRunner', params});
+    queueJobExecutionActivity: async (params: {jobId: string; jobExecutionId: string}) => {
+      calls.push({name: 'queueJobExecutionActivity', params});
 
-      if (cfg.enqueueError) {
+      if (cfg.queueError) {
         const {ApplicationFailure} = await import('@temporalio/common');
-        throw ApplicationFailure.nonRetryable(cfg.enqueueError);
+        throw ApplicationFailure.nonRetryable(cfg.queueError);
       }
 
-      if (cfg.skipSignal) return;
+      const queued = {newVersion: nextVersion(), status: 'pending' as const};
+      if (cfg.skipSignal) return queued;
 
       const status = cfg.jobResults.get(params.jobId) ?? 'succeeded';
       const handle = testEnv.client.workflow.getHandle(`job:${params.jobId}`);
@@ -321,17 +308,17 @@ function createMockActivities() {
         claimedAt: cfg.claimedAt ?? '2026-06-22T10:05:00.000Z',
       });
 
-      if (cfg.skipOutcomeSignal) return;
+      if (cfg.skipOutcomeSignal) return queued;
 
       if (cfg.signalLeaseExpired) {
         await handle.signal(JOB_LEASE_EXPIRED_SIGNAL, {jobExecutionId: params.jobExecutionId});
-        return;
+        return queued;
       }
 
       if (cfg.signalBoth) {
         await handle.signal(JOB_FINISHED_SIGNAL, {status, jobExecutionId: params.jobExecutionId});
         await handle.signal(JOB_LEASE_EXPIRED_SIGNAL, {jobExecutionId: params.jobExecutionId});
-        return;
+        return queued;
       }
 
       // The terminal outbox is independent from the claim, but the mock delivers it from the
@@ -342,6 +329,7 @@ function createMockActivities() {
         status,
         duplicateSignal: cfg.duplicateSignal ?? false,
       });
+      return queued;
     },
 
     resolveLeaseExpiredJobExecutionActivity: (params: {
@@ -369,14 +357,6 @@ function createMockActivities() {
       };
     },
 
-    releaseLeaseActivity: async (params: {jobExecutionId: string}) => {
-      calls.push({name: 'releaseLeaseActivity', params});
-      if (cfg.releaseLeaseError) {
-        const {ApplicationFailure} = await import('@temporalio/common');
-        throw ApplicationFailure.nonRetryable(cfg.releaseLeaseError);
-      }
-    },
-
     failJobExecutionAsTimedOutActivity: async (params: {
       jobExecutionId: string;
       runAttemptId: string;

package/test/fixtures/runners-inter-module.ts

@@ -35,9 +35,6 @@ export function resetRunnersTestClient(): void {
 }
 
 export const runnersTestClient: RunnersInterModuleClient = {
-  enqueueJobExecution: async () => ({}),
-  releaseJobExecution: async () => ({}),
-  cancelJobs: async () => ({}),
   getLeaseState: async (params) => ({active: activeLeases.has(leaseKey(params))}),
   getEffectiveRunnerToolCapabilities: async ({runnerSessionId}) =>
     toolCapabilities.get(runnerSessionId) ?? {capabilities: {harnesses: {}}, reportFresh: false},

package/test/helpers/workflow-runs.ts

@@ -1,4 +1,5 @@
 import {
+  WORKFLOWS_JOB_EXECUTION_TERMINATED,
   WORKFLOWS_JOB_TERMINATED,
   WORKFLOWS_STEP_ATTEMPT_TERMINATED,
   WORKFLOWS_WORKFLOW_RUN_ATTEMPT_CREATED,
@@ -137,6 +138,30 @@ export async function jobTerminatedEvents(jobId: string) {
   );
 }
 
+export async function jobExecutionTerminatedEvents(jobExecutionId: string) {
+  const rows = await db()
+    .select({payload: workflowsOutbox.payload})
+    .from(workflowsOutbox)
+    .where(
+      and(
+        eq(workflowsOutbox.eventType, WORKFLOWS_JOB_EXECUTION_TERMINATED),
+        sql`${workflowsOutbox.payload}->>'jobExecutionId' = ${jobExecutionId}`,
+      ),
+    );
+  return rows.map(
+    (row) =>
+      row.payload as {
+        jobId: string;
+        jobExecutionId: string;
+        workflowRunId: string;
+        workflowRunAttemptId: string;
+        status: string;
+        statusReason: string | null;
+        statusReasonMessage?: string | null;
+      },
+  );
+}
+
 export async function runTerminatedEvents(workflowRunId: string) {
   const rows = await db()
     .select({payload: workflowsOutbox.payload})