@shipfox/api-workflows 2.0.0

package/src/temporal/workflows/job-execution-orchestration-timeout.test.ts

@@ -0,0 +1,165 @@
+import {resolve} from 'node:path';
+import {TestWorkflowEnvironment} from '@temporalio/testing';
+import {Worker} from '@temporalio/worker';
+
+const TASK_QUEUE = 'test-orchestration-timeout';
+const WORKFLOWS_PATH = resolve(import.meta.dirname, '../../../dist/temporal/workflows/index.js');
+
+interface ActivityCall {
+  name: string;
+  params: unknown;
+}
+
+let testEnv: TestWorkflowEnvironment;
+let worker: Worker;
+let workerRunPromise: Promise<void> | undefined;
+
+let calls: ActivityCall[] = [];
+let versionSeq = 0;
+let failJobExecutionAsTimedOutShouldThrow = false;
+
+function nextVersion(): number {
+  return ++versionSeq;
+}
+
+function callsNamed(name: string): ActivityCall[] {
+  return calls.filter((c) => c.name === name);
+}
+
+beforeAll(async () => {
+  // Time-skipping environment lets us advance virtual time past the execution
+  // timeout without waiting in real wall time.
+  testEnv = await TestWorkflowEnvironment.createTimeSkipping();
+
+  worker = await Worker.create({
+    connection: testEnv.nativeConnection,
+    taskQueue: TASK_QUEUE,
+    workflowsPath: WORKFLOWS_PATH,
+    activities: {
+      loadRunAttemptDag: () => ({
+        workflowRunId: 'run-1',
+        runAttemptId: 'run-attempt-1',
+        workspaceId: 'workspace-1',
+        projectId: 'project-1',
+        runVersion: 1,
+        jobs: [],
+      }),
+      setRunAttemptStatus: (params: unknown) => {
+        calls.push({name: 'setRunAttemptStatus', params});
+        return {newVersion: nextVersion()};
+      },
+      setJobStatus: (params: unknown) => {
+        calls.push({name: 'setJobStatus', params});
+        return {newVersion: nextVersion()};
+      },
+      setJobExecutionStatus: (params: unknown) => {
+        calls.push({name: 'setJobExecutionStatus', params});
+        return {newVersion: nextVersion(), status: (params as {status?: string}).status};
+      },
+      bulkSetStepStatuses: (params: unknown) => {
+        calls.push({name: 'bulkSetStepStatuses', params});
+      },
+      resolveLeaseExpiredJobExecutionActivity: (params: unknown) => {
+        calls.push({name: 'resolveLeaseExpiredJobExecutionActivity', params});
+        return {status: 'failed', executionVersion: nextVersion()};
+      },
+      resolveJobStatusFromJobExecutionsActivity: (params: unknown) => {
+        calls.push({name: 'resolveJobStatusFromJobExecutionsActivity', params});
+        return {status: 'failed', jobVersion: nextVersion()};
+      },
+      releaseLeaseActivity: (params: unknown) => {
+        calls.push({name: 'releaseLeaseActivity', params});
+      },
+      enqueueJobExecutionForRunner: (params: unknown) => {
+        // No-op — we want the workflow to block on the signal until the timeout fires.
+        calls.push({name: 'enqueueJobExecutionForRunner', params});
+      },
+      failJobExecutionAsTimedOutActivity: async (params: unknown) => {
+        calls.push({name: 'failJobExecutionAsTimedOutActivity', params});
+        if (failJobExecutionAsTimedOutShouldThrow) {
+          const {ApplicationFailure} = await import('@temporalio/common');
+          throw ApplicationFailure.nonRetryable('simulated DB outage');
+        }
+        return {newVersion: nextVersion()};
+      },
+    },
+  });
+
+  workerRunPromise = worker.run();
+  workerRunPromise.catch(() => {
+    // Suppress unhandled rejection on worker shutdown.
+  });
+}, 60_000);
+
+afterAll(async () => {
+  worker?.shutdown();
+  await workerRunPromise;
+  await testEnv?.teardown();
+}, 15_000);
+
+beforeEach(() => {
+  calls = [];
+  versionSeq = 0;
+  failJobExecutionAsTimedOutShouldThrow = false;
+});
+
+const defaultJobInput = {
+  workspaceId: 'workspace-1',
+  jobId: 'job-timeout',
+  runAttemptId: 'run-attempt-1',
+  projectId: 'project-1',
+  jobVersion: 1,
+  jobExecutionId: 'job-timeout',
+  executionVersion: 1,
+  requiredLabels: ['ubuntu22'],
+};
+
+function executeJob(input: typeof defaultJobInput): Promise<{status: string; jobVersion: number}> {
+  const normalized = {
+    ...input,
+    jobExecutionId:
+      input.jobExecutionId === defaultJobInput.jobExecutionId &&
+      input.jobId !== defaultJobInput.jobId
+        ? input.jobId
+        : input.jobExecutionId,
+  };
+  return testEnv.client.workflow.execute('jobExecutionOrchestration', {
+    taskQueue: TASK_QUEUE,
+    workflowId: `job:${input.jobId}-${Math.random()}`,
+    args: [normalized],
+  });
+}
+
+describe('jobExecutionOrchestration timeout path', () => {
+  test('times out after JOB_MAX_DURATION; calls failJobExecutionAsTimedOutActivity then bulkSetStepStatuses', async () => {
+    const result = await executeJob(defaultJobInput);
+
+    expect(result.status).toBe('failed');
+
+    expect(callsNamed('failJobExecutionAsTimedOutActivity')).toHaveLength(1);
+
+    const setJobStatuses = callsNamed('setJobExecutionStatus').map(
+      (c) => (c.params as {status: string}).status,
+    );
+    expect(setJobStatuses).toEqual(['running']);
+
+    // Timeout path sweeps steps with the bulk activity (no per-step detail).
+    const stepCall = callsNamed('bulkSetStepStatuses')[0];
+    expect((stepCall?.params as {status: string})?.status).toBe('failed');
+    // The lease is intentionally NOT released on the timeout path (the TIMED_OUT
+    // event drives cooperative cancel; the stuck detector reaps the row).
+    expect(callsNamed('releaseLeaseActivity')).toHaveLength(0);
+    expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(0);
+  }, 60_000);
+
+  test('failJobExecutionAsTimedOutActivity throws → workflow surfaces the error', async () => {
+    failJobExecutionAsTimedOutShouldThrow = true;
+
+    await expect(executeJob({...defaultJobInput, jobId: 'job-fail-error'})).rejects.toThrow();
+
+    const setJobStatuses = callsNamed('setJobExecutionStatus').map(
+      (c) => (c.params as {status: string}).status,
+    );
+    expect(setJobStatuses).toEqual(['running']);
+  }, 60_000);
+});

package/src/temporal/workflows/job-execution-orchestration.test.ts

@@ -0,0 +1,311 @@
+import {
+  callsNamed,
+  dagJob,
+  makeDag,
+  resetCalls,
+  setCfg,
+  setExecutionStatusCalls,
+  setJobStatusCalls,
+  setupEnv,
+  TASK_QUEUE,
+  teardownEnv,
+  testEnv,
+} from './test-env.js';
+
+beforeAll(async () => {
+  await setupEnv();
+}, 60_000);
+
+afterAll(async () => {
+  await teardownEnv();
+}, 15_000);
+
+beforeEach(() => {
+  resetCalls();
+});
+
+const defaultJobInput = {
+  workspaceId: 'workspace-1',
+  jobId: 'job-1',
+  workflowRunId: 'run-1',
+  projectId: 'project-1',
+  jobVersion: 1,
+  jobExecutionId: 'job-1',
+  executionVersion: 1,
+  requiredLabels: ['ubuntu22'],
+};
+
+function executeJob(input: typeof defaultJobInput): Promise<{status: string; jobVersion: number}> {
+  const normalized = {
+    ...input,
+    jobExecutionId:
+      input.jobExecutionId === defaultJobInput.jobExecutionId &&
+      input.jobId !== defaultJobInput.jobId
+        ? input.jobId
+        : input.jobExecutionId,
+  };
+  return testEnv.client.workflow.execute('jobExecutionOrchestration', {
+    taskQueue: TASK_QUEUE,
+    workflowId: `job:${input.jobId}`,
+    args: [normalized],
+  });
+}
+
+function finalStatusesFor(jobId: string): string[] {
+  return setExecutionStatusCalls()
+    .filter((c) => c.params.jobExecutionId === jobId)
+    .map((c) => c.params.status);
+}
+
+function terminalSetJobCall(jobId: string) {
+  return setExecutionStatusCalls().find(
+    (call) => call.params.jobExecutionId === jobId && call.params.status !== 'running',
+  );
+}
+
+async function expectEmptyRequiredLabelsFailure(input: typeof defaultJobInput): Promise<void> {
+  let error: unknown;
+
+  try {
+    await executeJob(input);
+  } catch (err) {
+    error = err;
+  }
+
+  expect(error).toMatchObject({
+    cause: expect.objectContaining({
+      message: `Job ${input.jobId} has no required runner labels`,
+      nonRetryable: true,
+      type: 'EmptyRequiredLabelsError',
+    }),
+  });
+}
+
+describe('jobExecutionOrchestration', () => {
+  test('finished signal (succeeded) flips status and releases the lease', async () => {
+    setCfg({
+      dag: makeDag([dagJob('job-1', 'build')]),
+      jobResults: new Map([['job-1', 'succeeded']]),
+    });
+
+    const result = await executeJob(defaultJobInput);
+
+    expect(result.status).toBe('succeeded');
+    expect(finalStatusesFor('job-1')).toEqual(['running', 'succeeded']);
+    expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
+    expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(0);
+    expect(callsNamed('bulkSetStepStatuses')).toHaveLength(0);
+  });
+
+  test('empty required labels fail before the job is marked running', async () => {
+    setCfg({
+      dag: makeDag([dagJob('job-empty-labels', 'build')]),
+      jobResults: new Map([['job-empty-labels', 'succeeded']]),
+    });
+
+    await expectEmptyRequiredLabelsFailure({
+      ...defaultJobInput,
+      jobId: 'job-empty-labels',
+      requiredLabels: [],
+    });
+
+    expect(setExecutionStatusCalls()).toHaveLength(0);
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(0);
+  });
+
+  test('whitespace-only required labels fail before the job is marked running', async () => {
+    setCfg({
+      dag: makeDag([dagJob('job-blank-labels', 'build')]),
+      jobResults: new Map([['job-blank-labels', 'succeeded']]),
+    });
+
+    await expectEmptyRequiredLabelsFailure({
+      ...defaultJobInput,
+      jobId: 'job-blank-labels',
+      requiredLabels: ['  '],
+    });
+
+    expect(setExecutionStatusCalls()).toHaveLength(0);
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(0);
+  });
+
+  test('finished signal (failed) flips status without sweeping steps', async () => {
+    setCfg({dag: makeDag([dagJob('job-2', 'build')]), jobResults: new Map([['job-2', 'failed']])});
+
+    const result = await executeJob({...defaultJobInput, jobId: 'job-2'});
+
+    expect(result.status).toBe('failed');
+    expect(finalStatusesFor('job-2')).toEqual(['running', 'failed']);
+    expect(terminalSetJobCall('job-2')?.params.statusReason).toBe('step_failed');
+    expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
+    expect(callsNamed('bulkSetStepStatuses')).toHaveLength(0);
+  });
+
+  test('job status resolution failure fails the job closed', async () => {
+    setCfg({
+      dag: makeDag([dagJob('job-resolve-fail', 'build')]),
+      jobResults: new Map([['job-resolve-fail', 'succeeded']]),
+      resolveJobStatusError: 'invalid job success expression',
+    });
+
+    const result = await executeJob({...defaultJobInput, jobId: 'job-resolve-fail'});
+
+    expect(result.status).toBe('failed');
+    expect(finalStatusesFor('job-resolve-fail')).toEqual(['running', 'succeeded']);
+    expect(setJobStatusCalls()).toContainEqual({
+      name: 'setJobStatus',
+      params: {
+        jobId: 'job-resolve-fail',
+        status: 'failed',
+        version: 1,
+        statusReason: 'step_failed',
+      },
+    });
+    expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
+  });
+
+  test('lease-expired signal resolves via the activity and releases the lease', async () => {
+    setCfg({
+      dag: makeDag([]),
+      jobResults: new Map(),
+      signalLeaseExpired: true,
+      leaseExpiredStatus: 'failed',
+    });
+
+    const result = await executeJob({...defaultJobInput, jobId: 'job-le'});
+
+    expect(result.status).toBe('failed');
+    expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(1);
+    expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
+    expect(callsNamed('failJobExecutionAsTimedOutActivity')).toHaveLength(0);
+  });
+
+  test('lease-expired adoption: resolver reports succeeded → job succeeds (server state wins)', async () => {
+    setCfg({
+      dag: makeDag([]),
+      jobResults: new Map(),
+      signalLeaseExpired: true,
+      leaseExpiredStatus: 'succeeded',
+    });
+
+    const result = await executeJob({...defaultJobInput, jobId: 'job-le2'});
+
+    expect(result.status).toBe('succeeded');
+    expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(1);
+  });
+
+  test('both signals: finished wins, lease-expiry is ignored', async () => {
+    setCfg({
+      dag: makeDag([]),
+      jobResults: new Map([['job-both', 'succeeded']]),
+      signalBoth: true,
+    });
+
+    const result = await executeJob({...defaultJobInput, jobId: 'job-both'});
+
+    expect(result.status).toBe('succeeded');
+    expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(0);
+  });
+
+  test('both signals with finished=failed: fails via the finished path, one terminal setJobStatus', async () => {
+    setCfg({
+      dag: makeDag([]),
+      jobResults: new Map([['job-bf', 'failed']]),
+      signalBoth: true,
+    });
+
+    const result = await executeJob({...defaultJobInput, jobId: 'job-bf'});
+
+    expect(result.status).toBe('failed');
+    expect(finalStatusesFor('job-bf').filter((s) => s !== 'running')).toEqual(['failed']);
+    expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(0);
+  });
+
+  test('duplicate finished signal: first wins', async () => {
+    setCfg({
+      dag: makeDag([]),
+      jobResults: new Map([['job-dup', 'succeeded']]),
+      duplicateSignal: true,
+    });
+
+    const result = await executeJob({...defaultJobInput, jobId: 'job-dup'});
+
+    expect(result.status).toBe('succeeded');
+    expect(finalStatusesFor('job-dup').filter((s) => s !== 'running')).toEqual(['succeeded']);
+  });
+
+  test('ignores finished signals for a stale job execution', async () => {
+    setCfg({dag: makeDag([]), jobResults: new Map(), skipSignal: true});
+
+    const handle = await testEnv.client.workflow.start('jobExecutionOrchestration', {
+      taskQueue: TASK_QUEUE,
+      workflowId: 'job:job-stale-finished',
+      args: [
+        {
+          ...defaultJobInput,
+          jobId: 'job-stale-finished',
+          jobExecutionId: 'execution-current',
+        },
+      ],
+    });
+    await handle.signal('job-finished', {status: 'failed', jobExecutionId: 'execution-old'});
+    await new Promise((r) => setTimeout(r, 1000));
+    const description = await handle.describe();
+
+    await handle.signal('job-finished', {status: 'succeeded', jobExecutionId: 'execution-current'});
+    const result = await handle.result();
+
+    expect(description.status.name).toBe('RUNNING');
+    expect(result.status).toBe('succeeded');
+    expect(finalStatusesFor('execution-current').filter((s) => s !== 'running')).toEqual([
+      'succeeded',
+    ]);
+  }, 15_000);
+
+  test('already-terminal job is not enqueued', async () => {
+    setCfg({
+      dag: makeDag([]),
+      jobResults: new Map(),
+      runningJobStatus: 'cancelled',
+    });
+
+    const result = await executeJob({...defaultJobInput, jobId: 'job-cancelled'});
+
+    expect(result).toMatchObject({status: 'cancelled'});
+    expect(callsNamed('enqueueJobExecutionForRunner')).toHaveLength(0);
+    expect(callsNamed('releaseLeaseActivity')).toHaveLength(0);
+  });
+
+  test('releaseLease failure does not block the result (best-effort)', async () => {
+    setCfg({
+      dag: makeDag([]),
+      jobResults: new Map([['job-rl', 'succeeded']]),
+      releaseLeaseError: 'runners db down',
+    });
+
+    const result = await executeJob({...defaultJobInput, jobId: 'job-rl'});
+
+    expect(result.status).toBe('succeeded');
+    expect(callsNamed('releaseLeaseActivity').length).toBeGreaterThanOrEqual(1);
+  });
+
+  test('no signal — workflow stays blocked indefinitely', async () => {
+    setCfg({dag: makeDag([]), jobResults: new Map(), skipSignal: true});
+
+    const handle = await testEnv.client.workflow.start('jobExecutionOrchestration', {
+      taskQueue: TASK_QUEUE,
+      workflowId: 'job:job-stuck',
+      args: [{...defaultJobInput, jobId: 'job-stuck'}],
+    });
+
+    await new Promise((r) => setTimeout(r, 2000));
+
+    const description = await handle.describe();
+    expect(description.status.name).toBe('RUNNING');
+
+    // Clean up: signal it so it completes and does not leak.
+    setCfg({dag: makeDag([]), jobResults: new Map()});
+    await handle.signal('job-finished', {status: 'failed'});
+    await handle.result();
+  }, 15_000);
+});