@shipfox/api-workflows 2.0.0

package/src/temporal/workflows/run-orchestration.test.ts

@@ -0,0 +1,409 @@
+import {randomUUID} from 'node:crypto';
+import {
+  callsNamed,
+  dagJob,
+  makeDag,
+  resetCalls,
+  setCfg,
+  setJobStatusCalls,
+  setRunAttemptStatusCalls,
+  setupEnv,
+  TASK_QUEUE,
+  teardownEnv,
+  testEnv,
+} from './test-env.js';
+
+beforeAll(async () => {
+  await setupEnv();
+}, 60_000);
+
+afterAll(async () => {
+  await teardownEnv();
+}, 15_000);
+
+let workflowRunId: string;
+let workspaceId: string;
+
+beforeEach(() => {
+  resetCalls();
+  workflowRunId = `run-${randomUUID()}`;
+  workspaceId = `workspace-${randomUUID()}`;
+});
+
+async function executeRun(): Promise<void> {
+  await testEnv.client.workflow.execute('runOrchestration', {
+    taskQueue: TASK_QUEUE,
+    workflowId: `workflow-run-attempt:${workflowRunId}-attempt-1`,
+    args: [{workflowRunId, runAttemptId: `${workflowRunId}-attempt-1`, workspaceId}],
+  });
+}
+
+async function waitForActivity(name: string): Promise<void> {
+  for (let attempt = 0; attempt < 40; attempt += 1) {
+    if (callsNamed(name).length > 0) return;
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+  throw new Error(`Timed out waiting for ${name}`);
+}
+
+describe('runOrchestration', () => {
+  test('all-pending linear DAG enqueues every job and succeeds', async () => {
+    const jobs = [
+      dagJob('j1', 'build'),
+      dagJob('j2', 'test', ['build']),
+      dagJob('j3', 'deploy', ['test']),
+    ];
+    setCfg({dag: makeDag(jobs, 'r1'), jobResults: new Map()});
+
+    await executeRun();
+
+    const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
+    expect(runAttemptStatuses).toEqual(['running', 'succeeded']);
+    const enqueueCalls = callsNamed('enqueueJobExecutionForRunner');
+    expect(enqueueCalls).toHaveLength(3);
+    // The lease tuple is sourced from the loaded dag (workspace/project/run together).
+    for (const call of enqueueCalls) {
+      expect(call.params).toMatchObject({
+        workspaceId: 'workspace-1',
+        projectId: 'project-1',
+        runAttemptId: 'r1-attempt-1',
+        requiredLabels: ['ubuntu22'],
+      });
+    }
+  });
+
+  test('already-succeeded upstream jobs are not re-enqueued', async () => {
+    const jobs = [
+      dagJob('j1', 'build', [], {status: 'succeeded'}),
+      dagJob('j2', 'test', ['build']),
+    ];
+    setCfg({dag: makeDag(jobs, 'r1-carried'), jobResults: new Map([['j2', 'succeeded']])});
+
+    await executeRun();
+
+    const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
+    expect(runAttemptStatuses).toEqual(['running', 'succeeded']);
+    const enqueueCalls = callsNamed('enqueueJobExecutionForRunner');
+    expect(enqueueCalls).toHaveLength(1);
+    expect(enqueueCalls[0]?.params).toMatchObject({jobId: 'j2'});
+  });
+
+  test('already-succeeded upstream jobs can unblock a failed retry', async () => {
+    const jobs = [
+      dagJob('j1', 'build', [], {status: 'succeeded'}),
+      dagJob('j2', 'test', ['build']),
+    ];
+    setCfg({dag: makeDag(jobs, 'r1-carried-failed'), jobResults: new Map([['j2', 'failed']])});
+
+    await executeRun();
+
+    const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
+    expect(runAttemptStatuses).toEqual(['running', 'failed']);
+    const enqueueCalls = callsNamed('enqueueJobExecutionForRunner');
+    expect(enqueueCalls).toHaveLength(1);
+    expect(enqueueCalls[0]?.params).toMatchObject({jobId: 'j2'});
+  });
+
+  test('parallel roots both succeed', async () => {
+    const jobs = [
+      dagJob('j1', 'lint'),
+      dagJob('j2', 'test'),
+      dagJob('j3', 'deploy', ['lint', 'test']),
+    ];
+    setCfg({dag: makeDag(jobs, 'r2'), jobResults: new Map()});
+
+    await executeRun();
+
+    const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
+    expect(runAttemptStatuses).toEqual(['running', 'succeeded']);
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(3);
+  });
+
+  test('single job fails, dependents are skipped', async () => {
+    const jobs = [
+      dagJob('j1', 'build'),
+      dagJob('j2', 'test', ['build']),
+      dagJob('j3', 'deploy', ['test']),
+    ];
+    setCfg({dag: makeDag(jobs, 'r3'), jobResults: new Map([['j1', 'failed']])});
+
+    await executeRun();
+
+    const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
+    expect(runAttemptStatuses).toEqual(['running', 'failed']);
+
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(1);
+    const jobStatuses = setJobStatusCalls().map((c) => ({
+      id: c.params.jobId,
+      status: c.params.status,
+      statusReason: c.params.statusReason ?? null,
+    }));
+    expect(jobStatuses).toContainEqual({
+      id: 'j2',
+      status: 'skipped',
+      statusReason: 'default_gate_rejected',
+    });
+    expect(jobStatuses).toContainEqual({
+      id: 'j3',
+      status: 'skipped',
+      statusReason: 'default_gate_rejected',
+    });
+  });
+
+  test('first job fails, all downstream skipped', async () => {
+    const jobs = [
+      dagJob('j1', 'build'),
+      dagJob('j2', 'a', ['build']),
+      dagJob('j3', 'b', ['build']),
+    ];
+    setCfg({dag: makeDag(jobs, 'r4'), jobResults: new Map([['j1', 'failed']])});
+
+    await executeRun();
+
+    const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
+    expect(runAttemptStatuses).toEqual(['running', 'failed']);
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(1);
+
+    const jobStatuses = setJobStatusCalls().map((c) => ({
+      id: c.params.jobId,
+      status: c.params.status,
+      statusReason: c.params.statusReason ?? null,
+    }));
+    expect(jobStatuses).toContainEqual({
+      id: 'j2',
+      status: 'skipped',
+      statusReason: 'default_gate_rejected',
+    });
+    expect(jobStatuses).toContainEqual({
+      id: 'j3',
+      status: 'skipped',
+      statusReason: 'default_gate_rejected',
+    });
+  });
+
+  test('explicit-if failure handler runs after a failed dependency', async () => {
+    const jobs = [
+      dagJob('j1', 'build'),
+      dagJob('j2', 'notify', ['build'], {hasActivationCondition: true}),
+    ];
+    setCfg({dag: makeDag(jobs, 'r-job-if-failed'), jobResults: new Map([['j1', 'failed']])});
+
+    await executeRun();
+
+    expect(callsNamed('evaluateJobActivationsActivity')).toEqual([
+      {
+        name: 'evaluateJobActivationsActivity',
+        params: {
+          runAttemptId: `${workflowRunId}-attempt-1`,
+          jobs: [{jobId: 'j2', expectedVersion: 1}],
+        },
+      },
+    ]);
+    expect(callsNamed('enqueueJobExecutionForRunner').map((call) => call.params)).toEqual([
+      expect.objectContaining({jobId: 'j1'}),
+      expect.objectContaining({jobId: 'j2'}),
+    ]);
+    expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual(['running', 'failed']);
+  });
+
+  test('explicit-if job skips when activation evaluation rejects it', async () => {
+    const jobs = [
+      dagJob('j1', 'build'),
+      dagJob('j2', 'notify', ['build'], {hasActivationCondition: true}),
+    ];
+    setCfg({
+      dag: makeDag(jobs, 'r-job-if-rejected'),
+      jobResults: new Map(),
+      activationDecisions: new Map([
+        ['j2', {kind: 'terminal-job', jobId: 'j2', status: 'skipped', jobVersion: 7}],
+      ]),
+    });
+
+    await executeRun();
+
+    expect(callsNamed('enqueueJobExecutionForRunner').map((call) => call.params)).toEqual([
+      expect.objectContaining({jobId: 'j1'}),
+    ]);
+    expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual([
+      'running',
+      'succeeded',
+    ]);
+  });
+
+  test('fan-in explicit-if job activates after every need is terminal', async () => {
+    const jobs = [
+      dagJob('j1', 'build'),
+      dagJob('j2', 'lint'),
+      dagJob('j3', 'notify', ['build', 'lint'], {hasActivationCondition: true}),
+    ];
+    setCfg({dag: makeDag(jobs, 'r-job-if-fanin'), jobResults: new Map([['j1', 'failed']])});
+
+    await executeRun();
+
+    expect(callsNamed('evaluateJobActivationsActivity')).toEqual([
+      {
+        name: 'evaluateJobActivationsActivity',
+        params: {
+          runAttemptId: `${workflowRunId}-attempt-1`,
+          jobs: [{jobId: 'j3', expectedVersion: 1}],
+        },
+      },
+    ]);
+    const enqueuedJobs = callsNamed('enqueueJobExecutionForRunner').map((call) => call.params);
+    expect(enqueuedJobs).toHaveLength(3);
+    expect(enqueuedJobs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({jobId: 'j1'}),
+        expect.objectContaining({jobId: 'j2'}),
+        expect.objectContaining({jobId: 'j3'}),
+      ]),
+    );
+    expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual(['running', 'failed']);
+  });
+
+  test('all-skipped run completes succeeded', async () => {
+    const jobs = [dagJob('j1', 'notify', [], {hasActivationCondition: true})];
+    setCfg({
+      dag: makeDag(jobs, 'r-all-skipped'),
+      jobResults: new Map(),
+      activationDecisions: new Map([
+        ['j1', {kind: 'terminal-job', jobId: 'j1', status: 'skipped', jobVersion: 4}],
+      ]),
+    });
+
+    await executeRun();
+
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(0);
+    expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual([
+      'running',
+      'succeeded',
+    ]);
+  });
+
+  test('explicit-if listening job skips before listener registration', async () => {
+    const jobs = [dagJob('j1', 'listen', [], {mode: 'listening', hasActivationCondition: true})];
+    setCfg({
+      dag: makeDag(jobs, 'r-listen-skipped'),
+      jobResults: new Map(),
+      activationDecisions: new Map([
+        ['j1', {kind: 'terminal-job', jobId: 'j1', status: 'skipped', jobVersion: 4}],
+      ]),
+    });
+
+    await executeRun();
+
+    expect(callsNamed('activateJobListenerActivity')).toHaveLength(0);
+    expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual([
+      'running',
+      'succeeded',
+    ]);
+  });
+
+  test('version tracking flows through correctly', async () => {
+    const jobs = [dagJob('j1', 'build')];
+    setCfg({dag: makeDag(jobs, 'r5'), jobResults: new Map()});
+
+    await executeRun();
+
+    // The final attempt status update should use the version returned by the first update.
+    const finalRunAttemptStatus = setRunAttemptStatusCalls().at(-1);
+    expect(finalRunAttemptStatus?.params.version).toBeGreaterThan(0);
+  });
+
+  test('cancel signal stops scheduling and fans out runner cancellation', async () => {
+    const jobs = [dagJob('j1', 'build'), dagJob('j2', 'deploy', ['build'])];
+    setCfg({dag: makeDag(jobs, 'r-cancel'), jobResults: new Map(), skipSignal: true});
+
+    const handle = await testEnv.client.workflow.start('runOrchestration', {
+      taskQueue: TASK_QUEUE,
+      workflowId: `run:${workflowRunId}`,
+      args: [{workflowRunId, workspaceId}],
+    });
+    await waitForActivity('enqueueJobExecutionForRunner');
+
+    await handle.signal('run-cancel');
+    await handle.result();
+
+    expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual(['running']);
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(1);
+    expect(callsNamed('cancelRunnerJobsActivity')).toEqual([
+      {name: 'cancelRunnerJobsActivity', params: {jobIds: ['j1', 'j2']}},
+    ]);
+  });
+
+  test('starts listening jobs and cancels them with the run', async () => {
+    const jobs = [dagJob('j1', 'listen', [], {mode: 'listening'})];
+    setCfg({dag: makeDag(jobs, 'r-listen'), jobResults: new Map()});
+    const runAttemptId = `${workflowRunId}-attempt-listen`;
+
+    const handle = await testEnv.client.workflow.start('runOrchestration', {
+      taskQueue: TASK_QUEUE,
+      workflowId: `workflow-run-attempt:${runAttemptId}`,
+      args: [{workflowRunId, runAttemptId, workspaceId}],
+    });
+    await waitForActivity('setRunAttemptStatus');
+
+    await handle.signal('run-cancel');
+    await handle.result();
+
+    expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual(['running']);
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(0);
+    expect(callsNamed('cancelRunnerJobsActivity')).toEqual([
+      {name: 'cancelRunnerJobsActivity', params: {jobIds: ['j1']}},
+    ]);
+  });
+
+  test('aborts early when the initial running write reports an already-terminal run', async () => {
+    const jobs = [dagJob('j1', 'build')];
+    setCfg({
+      dag: makeDag(jobs, 'r-terminal'),
+      jobResults: new Map(),
+      initialRunStatus: 'cancelled',
+    });
+
+    await executeRun();
+
+    expect(setRunAttemptStatusCalls().map((c) => c.params.status)).toEqual(['running']);
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(0);
+    expect(callsNamed('cancelRunnerJobsActivity')).toHaveLength(0);
+  });
+
+  test('diamond DAG with partial failure', async () => {
+    const jobs = [
+      dagJob('j1', 'A'),
+      dagJob('j2', 'B', ['A']),
+      dagJob('j3', 'C', ['A']),
+      dagJob('j4', 'D', ['B', 'C']),
+    ];
+    setCfg({dag: makeDag(jobs, 'r6'), jobResults: new Map([['j2', 'failed']])});
+
+    await executeRun();
+
+    const runAttemptStatuses = setRunAttemptStatusCalls().map((c) => c.params.status);
+    expect(runAttemptStatuses).toEqual(['running', 'failed']);
+
+    // A, B, C enqueued — D skipped because B failed
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(3);
+    const jobStatuses = setJobStatusCalls().map((c) => ({
+      id: c.params.jobId,
+      status: c.params.status,
+      statusReason: c.params.statusReason ?? null,
+    }));
+    expect(jobStatuses).toContainEqual({
+      id: 'j4',
+      status: 'skipped',
+      statusReason: 'default_gate_rejected',
+    });
+  });
+
+  test('child workflow crash propagates to parent', async () => {
+    const jobs = [dagJob('j1', 'build')];
+    setCfg({
+      dag: makeDag(jobs, 'r7'),
+      jobResults: new Map(),
+      enqueueError: 'Runner service unavailable',
+    });
+
+    await expect(executeRun()).rejects.toThrow();
+  });
+});

package/src/temporal/workflows/run-orchestration.ts

@@ -0,0 +1,283 @@
+import {
+  condition,
+  defineSignal,
+  executeChild,
+  log,
+  ParentClosePolicy,
+  proxyActivities,
+  setHandler,
+} from '@temporalio/workflow';
+import {
+  createRuntimeRunProgress,
+  nonCompletedRuntimeJobIds,
+  type RuntimeRunProgress,
+  recordRuntimeJobResult,
+  recordSkippedRuntimeJob,
+  runtimeJobVersion,
+  shouldContinueStartedRun,
+} from '#core/workflow-scheduling/run-progress.js';
+import type {RuntimeCompletionStatus} from '#core/workflow-scheduling/runtime-dag.js';
+import {scheduleRuntimeDag} from '#core/workflow-scheduling/schedule-runtime-dag.js';
+
+import type {createOrchestrationActivities} from '../activities/index.js';
+import type {DagJob, RunDag} from '../activities/orchestration-activities.js';
+import {RUN_CANCEL_SIGNAL} from '../constants.js';
+import {deadlineReached, remainingMs} from './deadline.js';
+import {jobExecutionOrchestration} from './job-execution-orchestration.js';
+import {jobListenerOrchestration} from './job-listener-orchestration.js';
+
+const {
+  loadRunAttemptDag,
+  evaluateJobActivationsActivity,
+  setRunAttemptStatus,
+  setJobStatus,
+  cancelRunnerJobsActivity,
+} = proxyActivities<ReturnType<typeof createOrchestrationActivities>>({
+  startToCloseTimeout: '30s',
+});
+
+const {failRunAsTimedOutActivity} = proxyActivities<
+  ReturnType<typeof createOrchestrationActivities>
+>({
+  startToCloseTimeout: '30s',
+  retry: {maximumAttempts: 5},
+});
+
+export const runCancelSignal = defineSignal<[]>(RUN_CANCEL_SIGNAL);
+
+export interface RunOrchestrationInput {
+  workflowRunId: string;
+  runAttemptId: string;
+  workspaceId: string;
+}
+
+export async function runOrchestration(input: RunOrchestrationInput): Promise<void> {
+  let cancelRequested = false;
+  setHandler(runCancelSignal, () => {
+    cancelRequested = true;
+  });
+
+  const dag = await loadRunAttemptDag(input.runAttemptId);
+  const runDeadline = Date.now() + dag.runTimeoutMs;
+
+  let runVersion = dag.runVersion;
+  const {newVersion, status} = await setRunAttemptStatus({
+    runAttemptId: input.runAttemptId,
+    status: 'running',
+    version: runVersion,
+  });
+  runVersion = newVersion;
+  if (!shouldContinueStartedRun(status)) return;
+
+  const progress = createRuntimeRunProgress(dag.jobs);
+  const inFlight = new Map<string, Promise<{job: DagJob; result: LaunchResult}>>();
+
+  while (true) {
+    if (cancelRequested) {
+      await cancelNonCompletedRunnerJobs(dag, progress);
+      return;
+    }
+    if (deadlineReached(runDeadline)) {
+      await failRunAsTimedOutActivity({runAttemptId: input.runAttemptId});
+      await cancelNonCompletedRunnerJobs(dag, progress);
+      return;
+    }
+
+    const commands = scheduleRuntimeDag({
+      jobs: dag.jobs,
+      completed: progress.completed,
+      running: new Set(inFlight.keys()),
+    });
+    const completeRun = commands.find((command) => command.kind === 'complete-run');
+    const startJobs = new Map<string, DagJob>();
+
+    for (const command of commands) {
+      if (command.kind !== 'skip-job') continue;
+      await skipJob(command.job, progress, command.statusReason);
+    }
+
+    for (const command of commands) {
+      if (command.kind !== 'evaluate-job-activation') continue;
+      const activationJobsById = new Map(command.jobs.map((job) => [job.id, job]));
+      const decisions = await evaluateJobActivationsActivity({
+        runAttemptId: input.runAttemptId,
+        jobs: command.jobs.map((job) => ({
+          jobId: job.id,
+          expectedVersion: runtimeJobVersion(job, progress),
+        })),
+      });
+      for (const decision of decisions) {
+        const job = activationJobsById.get(decision.jobId);
+        if (!job) continue;
+        if (decision.kind === 'start-job') {
+          startJobs.set(job.key, job);
+          continue;
+        }
+        recordRuntimeJobResult(job, progress, {
+          status: decision.status,
+          jobVersion: decision.jobVersion,
+        });
+      }
+    }
+
+    for (const command of commands) {
+      if (command.kind !== 'start-job') continue;
+      startJobs.set(command.job.key, command.job);
+    }
+
+    for (const job of startJobs.values()) {
+      if (!inFlight.has(job.key)) {
+        inFlight.set(job.key, launchJob(job, dag, progress));
+      }
+    }
+
+    if (completeRun) {
+      await setRunAttemptStatus({
+        runAttemptId: input.runAttemptId,
+        status: completeRun.status,
+        version: runVersion,
+      });
+      return;
+    }
+
+    if (startJobs.size === 0 && inFlight.size === 0) continue;
+
+    const settled = await waitForNextSettlement(inFlight, () => cancelRequested, runDeadline);
+    if (settled.kind === 'cancelled') {
+      await cancelNonCompletedRunnerJobs(dag, progress);
+      return;
+    }
+    if (settled.kind === 'timed-out') {
+      await failRunAsTimedOutActivity({runAttemptId: input.runAttemptId});
+      await cancelNonCompletedRunnerJobs(dag, progress);
+      return;
+    }
+    inFlight.delete(settled.job.key);
+    recordRuntimeJobResult(settled.job, progress, settled.result);
+  }
+}
+
+async function skipJob(
+  job: DagJob,
+  progress: RuntimeRunProgress,
+  statusReason: 'default_gate_rejected',
+): Promise<void> {
+  const version = runtimeJobVersion(job, progress);
+  const {newVersion} = await setJobStatus({
+    jobId: job.id,
+    status: 'skipped',
+    version,
+    statusReason,
+  });
+  recordSkippedRuntimeJob(job, progress, newVersion);
+}
+
+interface LaunchResult {
+  status: RuntimeCompletionStatus;
+  jobVersion: number;
+}
+
+function launchJob(
+  job: DagJob,
+  run: RunDag,
+  progress: RuntimeRunProgress,
+): Promise<{job: DagJob; result: LaunchResult}> {
+  if (job.mode === 'listening') {
+    return executeChild(jobListenerOrchestration, {
+      workflowId: `job-listener:${job.id}`,
+      args: [
+        {
+          workspaceId: run.workspaceId,
+          workflowRunId: run.workflowRunId,
+          projectId: run.projectId,
+          jobId: job.id,
+          runAttemptId: run.runAttemptId,
+          jobVersion: runtimeJobVersion(job, progress),
+          ...(job.executionTimeoutMs === undefined
+            ? {}
+            : {executionTimeoutMs: job.executionTimeoutMs}),
+          ...(job.listeningTimeoutMs === undefined
+            ? {}
+            : {listeningTimeoutMs: job.listeningTimeoutMs}),
+          ...(job.maxExecutions === undefined ? {} : {maxExecutions: job.maxExecutions}),
+          ...(job.onResolve === undefined ? {} : {onResolve: job.onResolve}),
+          ...(job.batchDebounceMs === undefined ? {} : {batchDebounceMs: job.batchDebounceMs}),
+          ...(job.batchMaxSize === undefined ? {} : {batchMaxSize: job.batchMaxSize}),
+          ...(job.batchMaxWaitMs === undefined ? {} : {batchMaxWaitMs: job.batchMaxWaitMs}),
+          requiredLabels: job.runner,
+        },
+      ],
+      parentClosePolicy: ParentClosePolicy.TERMINATE,
+    })
+      .then((result) => ({job, result}))
+      .catch(async (error) => {
+        log.warn('listener child failed; marking runtime job failed', {
+          jobId: job.id,
+          error: String(error),
+        });
+        const failed = await setJobStatus({
+          jobId: job.id,
+          status: 'failed',
+          version: runtimeJobVersion(job, progress),
+          statusReason: 'unknown',
+        });
+        return {
+          job,
+          result: {status: 'failed' as const, jobVersion: failed.newVersion},
+        };
+      });
+  }
+
+  if (job.jobExecutionId === undefined) {
+    throw new Error(`Cannot start job without an execution: ${job.id}`);
+  }
+  return executeChild(jobExecutionOrchestration, {
+    workflowId: `job:${job.id}`,
+    args: [
+      {
+        workspaceId: run.workspaceId,
+        workflowRunId: run.workflowRunId,
+        projectId: run.projectId,
+        jobId: job.id,
+        jobExecutionId: job.jobExecutionId,
+        runAttemptId: run.runAttemptId,
+        jobVersion: runtimeJobVersion(job, progress),
+        executionVersion: job.executionVersion ?? runtimeJobVersion(job, progress),
+        ...(job.executionTimeoutMs === undefined
+          ? {}
+          : {executionTimeoutMs: job.executionTimeoutMs}),
+        requiredLabels: job.runner,
+      },
+    ],
+    parentClosePolicy: ParentClosePolicy.TERMINATE,
+  }).then((result) => ({
+    job,
+    result: {status: result.status, jobVersion: result.jobVersion},
+  }));
+}
+
+async function waitForNextSettlement(
+  inFlight: ReadonlyMap<string, Promise<{job: DagJob; result: LaunchResult}>>,
+  isCancelRequested: () => boolean,
+  runDeadline: number,
+): Promise<
+  {kind: 'settled'; job: DagJob; result: LaunchResult} | {kind: 'cancelled'} | {kind: 'timed-out'}
+> {
+  const remaining = remainingMs(runDeadline) ?? 0;
+  const childSettled = Promise.race([...inFlight.values()]).then((settled) => ({
+    kind: 'settled' as const,
+    ...settled,
+  }));
+  const cancel = condition(isCancelRequested, remaining).then((woke) =>
+    woke ? {kind: 'cancelled' as const} : {kind: 'timed-out' as const},
+  );
+  return await Promise.race([childSettled, cancel]);
+}
+
+async function cancelNonCompletedRunnerJobs(
+  dag: RunDag,
+  progress: RuntimeRunProgress,
+): Promise<void> {
+  const jobIds = nonCompletedRuntimeJobIds(dag.jobs, progress);
+  await cancelRunnerJobsActivity({jobIds});
+}