@shipfox/api-runners 2.0.0

package/src/presentation/subscribers/on-workflows-job-execution-timed-out.test.ts

@@ -0,0 +1,147 @@
+import type {WorkflowsJobExecutionTimedOutEventDto} from '@shipfox/api-workflows-dto';
+import {eq} from 'drizzle-orm';
+import {db} from '#db/db.js';
+import {claimPendingJobExecution} from '#db/job-executions.js';
+import {runningJobExecutions} from '#db/schema/running-job-executions.js';
+import {pendingJobFactory, runnerSessionFactory} from '#test/index.js';
+import {onWorkflowsJobExecutionTimedOut} from './on-workflows-job-execution-timed-out.js';
+
+function buildPayload(
+  jobId: string,
+  jobExecutionId: string,
+  workflowRunAttemptId: string,
+): WorkflowsJobExecutionTimedOutEventDto {
+  return {jobId, jobExecutionId, workflowRunAttemptId};
+}
+
+describe('onWorkflowsJobExecutionTimedOut', () => {
+  let workspaceId: string;
+  let runnerSessionId: string;
+
+  beforeEach(async () => {
+    workspaceId = crypto.randomUUID();
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    runnerSessionId = runnerSession.id;
+  });
+
+  it('sets cancellation_requested_at on the matching running_jobs row', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId,
+      sessionLabels: ['linux', 'x64'],
+      maxClaims: null,
+      runnerSessionLivenessThrottleSeconds: 10,
+    });
+    expect(claimed).not.toBeNull();
+
+    await onWorkflowsJobExecutionTimedOut(
+      buildPayload(
+        claimed?.jobId as string,
+        claimed?.jobExecutionId as string,
+        claimed?.workflowRunAttemptId as string,
+      ),
+    );
+
+    const rows = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    expect(rows[0]?.cancellationRequestedAt).not.toBeNull();
+  });
+
+  it('does not cancel another running execution for the same job', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId,
+      sessionLabels: ['linux', 'x64'],
+      maxClaims: null,
+      runnerSessionLivenessThrottleSeconds: 10,
+    });
+    expect(claimed).not.toBeNull();
+    const siblingJobExecutionId = crypto.randomUUID();
+    const siblingRunnerSession = await runnerSessionFactory.create({workspaceId});
+    await db()
+      .insert(runningJobExecutions)
+      .values({
+        workspaceId,
+        workflowRunId: claimed?.workflowRunId as string,
+        jobId: claimed?.jobId as string,
+        jobExecutionId: siblingJobExecutionId,
+        workflowRunAttemptId: claimed?.workflowRunAttemptId as string,
+        projectId: claimed?.projectId as string,
+        runnerSessionId: siblingRunnerSession.id,
+        requiredLabels: ['linux'],
+        runnerLabels: ['linux', 'x64'],
+      });
+
+    await onWorkflowsJobExecutionTimedOut(
+      buildPayload(
+        claimed?.jobId as string,
+        claimed?.jobExecutionId as string,
+        claimed?.workflowRunAttemptId as string,
+      ),
+    );
+
+    const rows = await db()
+      .select({
+        jobExecutionId: runningJobExecutions.jobExecutionId,
+        cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,
+      })
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    const byJobExecutionId = new Map(
+      rows.map((row) => [row.jobExecutionId, row.cancellationRequestedAt]),
+    );
+    expect(byJobExecutionId.get(claimed?.jobExecutionId as string)).not.toBeNull();
+    expect(byJobExecutionId.get(siblingJobExecutionId)).toBeNull();
+  });
+
+  it('idempotent under double delivery: second call preserves the first timestamp', async () => {
+    await pendingJobFactory.create({workspaceId});
+    const claimed = await claimPendingJobExecution({
+      workspaceId,
+      runnerSessionId,
+      sessionLabels: ['linux', 'x64'],
+      maxClaims: null,
+      runnerSessionLivenessThrottleSeconds: 10,
+    });
+
+    await onWorkflowsJobExecutionTimedOut(
+      buildPayload(
+        claimed?.jobId as string,
+        claimed?.jobExecutionId as string,
+        claimed?.workflowRunAttemptId as string,
+      ),
+    );
+    const after1 = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    const firstTs = after1[0]?.cancellationRequestedAt;
+
+    await new Promise((r) => setTimeout(r, 10));
+    await onWorkflowsJobExecutionTimedOut(
+      buildPayload(
+        claimed?.jobId as string,
+        claimed?.jobExecutionId as string,
+        claimed?.workflowRunAttemptId as string,
+      ),
+    );
+
+    const after2 = await db()
+      .select()
+      .from(runningJobExecutions)
+      .where(eq(runningJobExecutions.jobId, claimed?.jobId as string));
+    expect(after2[0]?.cancellationRequestedAt?.getTime()).toBe(firstTs?.getTime());
+  });
+
+  it('no-op when the running_jobs row is gone (already finalized)', async () => {
+    await expect(
+      onWorkflowsJobExecutionTimedOut(
+        buildPayload(crypto.randomUUID(), crypto.randomUUID(), crypto.randomUUID()),
+      ),
+    ).resolves.toBeUndefined();
+  });
+});

package/src/presentation/subscribers/on-workflows-job-execution-timed-out.ts

@@ -0,0 +1,17 @@
+import type {WorkflowsJobExecutionTimedOutEventDto} from '@shipfox/api-workflows-dto';
+import {logger} from '@shipfox/node-opentelemetry';
+import {requestJobExecutionCancellation} from '#db/job-executions.js';
+
+export async function onWorkflowsJobExecutionTimedOut(
+  payload: WorkflowsJobExecutionTimedOutEventDto,
+): Promise<void> {
+  logger().info(
+    {
+      jobId: payload.jobId,
+      jobExecutionId: payload.jobExecutionId,
+      workflowRunAttemptId: payload.workflowRunAttemptId,
+    },
+    'Requesting runner cancellation for timed-out job execution',
+  );
+  await requestJobExecutionCancellation({jobExecutionId: payload.jobExecutionId});
+}

package/src/temporal/activities/index.ts

@@ -0,0 +1,17 @@
+import {
+  deleteExpiredEphemeralRegistrationTokensActivity,
+  deleteExpiredReservationsActivity,
+  deleteExpiredRunnerSessionsActivity,
+  detectAndExpireStuckJobsActivity,
+  reapStaleProvisionedRunnersActivity,
+} from './maintenance-activities.js';
+
+export function createRunnersMaintenanceActivities() {
+  return {
+    deleteExpiredEphemeralRegistrationTokensActivity,
+    deleteExpiredReservationsActivity,
+    deleteExpiredRunnerSessionsActivity,
+    detectAndExpireStuckJobsActivity,
+    reapStaleProvisionedRunnersActivity,
+  };
+}

package/src/temporal/activities/maintenance-activities.test.ts

@@ -0,0 +1,104 @@
+import {
+  deleteExpiredEphemeralRegistrationTokens,
+  deleteExpiredRunnerReservations,
+  deleteExpiredRunnerSessions,
+  detectAndExpireStuckJobs,
+  reapStaleProvisionedRunners,
+} from '#core/maintenance.js';
+import {
+  deleteExpiredEphemeralRegistrationTokensActivity,
+  deleteExpiredReservationsActivity,
+  deleteExpiredRunnerSessionsActivity,
+  detectAndExpireStuckJobsActivity,
+  reapStaleProvisionedRunnersActivity,
+} from './maintenance-activities.js';
+
+vi.mock('#core/maintenance.js', () => ({
+  deleteExpiredEphemeralRegistrationTokens: vi.fn(),
+  deleteExpiredRunnerReservations: vi.fn(),
+  deleteExpiredRunnerSessions: vi.fn(),
+  detectAndExpireStuckJobs: vi.fn(),
+  reapStaleProvisionedRunners: vi.fn(),
+}));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe('detectAndExpireStuckJobsActivity', () => {
+  it('delegates to core maintenance', async () => {
+    vi.mocked(detectAndExpireStuckJobs).mockResolvedValueOnce({expired: 2});
+
+    const result = await detectAndExpireStuckJobsActivity({thresholdSeconds: 180});
+
+    expect(result).toEqual({expired: 2});
+    expect(detectAndExpireStuckJobs).toHaveBeenCalledWith({thresholdSeconds: 180});
+  });
+});
+
+describe('deleteExpiredReservationsActivity', () => {
+  it('delegates to core maintenance', async () => {
+    vi.mocked(deleteExpiredRunnerReservations).mockResolvedValueOnce({deleted: 3});
+
+    const result = await deleteExpiredReservationsActivity({limit: 50});
+
+    expect(result).toEqual({deleted: 3});
+    expect(deleteExpiredRunnerReservations).toHaveBeenCalledWith({limit: 50});
+  });
+});
+
+describe('reapStaleProvisionedRunnersActivity', () => {
+  it('delegates to core maintenance', async () => {
+    vi.mocked(reapStaleProvisionedRunners).mockResolvedValueOnce({
+      reaped: 4,
+      reservationsReleased: 2,
+    });
+
+    const result = await reapStaleProvisionedRunnersActivity({
+      thresholdSeconds: 300,
+      limit: 100,
+    });
+
+    expect(result).toEqual({reaped: 4, reservationsReleased: 2});
+    expect(reapStaleProvisionedRunners).toHaveBeenCalledWith({
+      thresholdSeconds: 300,
+      limit: 100,
+    });
+  });
+});
+
+describe('deleteExpiredRunnerSessionsActivity', () => {
+  it('delegates to core maintenance', async () => {
+    vi.mocked(deleteExpiredRunnerSessions).mockResolvedValueOnce({deleted: 4});
+
+    const result = await deleteExpiredRunnerSessionsActivity({
+      manualRetentionDays: 30,
+      ephemeralRetentionDays: 7,
+      limit: 25,
+    });
+
+    expect(result).toEqual({deleted: 4});
+    expect(deleteExpiredRunnerSessions).toHaveBeenCalledWith({
+      manualRetentionDays: 30,
+      ephemeralRetentionDays: 7,
+      limit: 25,
+    });
+  });
+});
+
+describe('deleteExpiredEphemeralRegistrationTokensActivity', () => {
+  it('delegates to core maintenance', async () => {
+    vi.mocked(deleteExpiredEphemeralRegistrationTokens).mockResolvedValueOnce({deleted: 6});
+
+    const result = await deleteExpiredEphemeralRegistrationTokensActivity({
+      retentionDays: 7,
+      limit: 25,
+    });
+
+    expect(result).toEqual({deleted: 6});
+    expect(deleteExpiredEphemeralRegistrationTokens).toHaveBeenCalledWith({
+      retentionDays: 7,
+      limit: 25,
+    });
+  });
+});

package/src/temporal/activities/maintenance-activities.ts

@@ -0,0 +1,41 @@
+import {
+  deleteExpiredEphemeralRegistrationTokens,
+  deleteExpiredRunnerReservations,
+  deleteExpiredRunnerSessions,
+  detectAndExpireStuckJobs,
+  reapStaleProvisionedRunners,
+} from '#core/maintenance.js';
+
+export function detectAndExpireStuckJobsActivity(params: {
+  thresholdSeconds: number;
+}): Promise<{expired: number}> {
+  return detectAndExpireStuckJobs(params);
+}
+
+export function deleteExpiredReservationsActivity(params?: {
+  limit?: number;
+}): Promise<{deleted: number}> {
+  return deleteExpiredRunnerReservations(params);
+}
+
+export function reapStaleProvisionedRunnersActivity(params?: {
+  thresholdSeconds: number;
+  limit: number;
+}): Promise<{reaped: number; reservationsReleased: number}> {
+  return reapStaleProvisionedRunners(params);
+}
+
+export function deleteExpiredRunnerSessionsActivity(params?: {
+  manualRetentionDays?: number;
+  ephemeralRetentionDays?: number;
+  limit?: number;
+}): Promise<{deleted: number}> {
+  return deleteExpiredRunnerSessions(params);
+}
+
+export function deleteExpiredEphemeralRegistrationTokensActivity(params?: {
+  retentionDays?: number;
+  limit?: number;
+}): Promise<{deleted: number}> {
+  return deleteExpiredEphemeralRegistrationTokens(params);
+}

package/src/temporal/constants.ts

@@ -0,0 +1 @@
+export const RUNNERS_MAINTENANCE_TASK_QUEUE = 'runners-maintenance';

package/src/temporal/workflows/index.ts

@@ -0,0 +1 @@
+export {stuckJobDetector} from './stuck-job-detector.js';

package/src/temporal/workflows/stuck-job-detector.test.ts

@@ -0,0 +1,93 @@
+const mocks = vi.hoisted(() => ({
+  deleteExpiredEphemeralRegistrationTokensActivity: vi.fn(),
+  deleteExpiredReservationsActivity: vi.fn(),
+  deleteExpiredRunnerSessionsActivity: vi.fn(),
+  detectAndExpireStuckJobsActivity: vi.fn(),
+  reapStaleProvisionedRunnersActivity: vi.fn(),
+  info: vi.fn(),
+  warn: vi.fn(),
+}));
+
+vi.mock('@temporalio/workflow', () => ({
+  log: {
+    info: mocks.info,
+    warn: mocks.warn,
+  },
+  proxyActivities: vi.fn(() => ({
+    deleteExpiredEphemeralRegistrationTokensActivity:
+      mocks.deleteExpiredEphemeralRegistrationTokensActivity,
+    deleteExpiredReservationsActivity: mocks.deleteExpiredReservationsActivity,
+    deleteExpiredRunnerSessionsActivity: mocks.deleteExpiredRunnerSessionsActivity,
+    detectAndExpireStuckJobsActivity: mocks.detectAndExpireStuckJobsActivity,
+    reapStaleProvisionedRunnersActivity: mocks.reapStaleProvisionedRunnersActivity,
+  })),
+}));
+
+describe('stuckJobDetector', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mocks.deleteExpiredEphemeralRegistrationTokensActivity.mockResolvedValue({deleted: 0});
+    mocks.deleteExpiredReservationsActivity.mockResolvedValue({deleted: 0});
+    mocks.deleteExpiredRunnerSessionsActivity.mockResolvedValue({deleted: 0});
+    mocks.detectAndExpireStuckJobsActivity.mockResolvedValue({expired: 0});
+    mocks.reapStaleProvisionedRunnersActivity.mockResolvedValue({
+      reaped: 0,
+      reservationsReleased: 0,
+    });
+  });
+
+  it('runs expired session GC before stuck job expiry and logs deletions', async () => {
+    const {stuckJobDetector} = await import('./stuck-job-detector.js');
+    mocks.deleteExpiredRunnerSessionsActivity.mockResolvedValueOnce({deleted: 2});
+
+    await stuckJobDetector();
+
+    expect(mocks.deleteExpiredRunnerSessionsActivity).toHaveBeenCalledWith();
+    expect(mocks.detectAndExpireStuckJobsActivity).toHaveBeenCalledWith({thresholdSeconds: 180});
+    expect(mocks.info).toHaveBeenCalledWith('Stuck-job detector deleted expired runner sessions', {
+      deleted: 2,
+    });
+  });
+
+  it('continues stuck job expiry when expired session GC fails', async () => {
+    const {stuckJobDetector} = await import('./stuck-job-detector.js');
+    mocks.deleteExpiredRunnerSessionsActivity.mockRejectedValueOnce(new Error('database down'));
+
+    await stuckJobDetector();
+
+    expect(mocks.warn).toHaveBeenCalledWith(
+      'Stuck-job detector failed to delete expired runner sessions',
+      {error: 'database down'},
+    );
+    expect(mocks.detectAndExpireStuckJobsActivity).toHaveBeenCalledWith({thresholdSeconds: 180});
+  });
+
+  it('runs expired ephemeral token GC before stuck job expiry and logs deletions', async () => {
+    const {stuckJobDetector} = await import('./stuck-job-detector.js');
+    mocks.deleteExpiredEphemeralRegistrationTokensActivity.mockResolvedValueOnce({deleted: 3});
+
+    await stuckJobDetector();
+
+    expect(mocks.deleteExpiredEphemeralRegistrationTokensActivity).toHaveBeenCalledWith();
+    expect(mocks.detectAndExpireStuckJobsActivity).toHaveBeenCalledWith({thresholdSeconds: 180});
+    expect(mocks.info).toHaveBeenCalledWith(
+      'Stuck-job detector deleted expired ephemeral registration tokens',
+      {deleted: 3},
+    );
+  });
+
+  it('continues stuck job expiry when expired ephemeral token GC fails', async () => {
+    const {stuckJobDetector} = await import('./stuck-job-detector.js');
+    mocks.deleteExpiredEphemeralRegistrationTokensActivity.mockRejectedValueOnce(
+      new Error('database down'),
+    );
+
+    await stuckJobDetector();
+
+    expect(mocks.warn).toHaveBeenCalledWith(
+      'Stuck-job detector failed to delete expired ephemeral registration tokens',
+      {error: 'database down'},
+    );
+    expect(mocks.detectAndExpireStuckJobsActivity).toHaveBeenCalledWith({thresholdSeconds: 180});
+  });
+});

package/src/temporal/workflows/stuck-job-detector.ts

@@ -0,0 +1,67 @@
+import {log, proxyActivities} from '@temporalio/workflow';
+import {STUCK_JOB_THRESHOLD_SECONDS} from '#core/maintenance-policy.js';
+
+import type {createRunnersMaintenanceActivities} from '../activities/index.js';
+
+const {
+  deleteExpiredEphemeralRegistrationTokensActivity,
+  deleteExpiredReservationsActivity,
+  deleteExpiredRunnerSessionsActivity,
+  detectAndExpireStuckJobsActivity,
+  reapStaleProvisionedRunnersActivity,
+} = proxyActivities<ReturnType<typeof createRunnersMaintenanceActivities>>({
+  startToCloseTimeout: '60s',
+});
+
+export async function stuckJobDetector(): Promise<void> {
+  try {
+    const {deleted} = await deleteExpiredReservationsActivity();
+    if (deleted > 0) {
+      log.info('Stuck-job detector deleted expired runner reservations', {deleted});
+    }
+  } catch (error) {
+    log.warn('Stuck-job detector failed to delete expired runner reservations', {
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+
+  try {
+    const {deleted} = await deleteExpiredRunnerSessionsActivity();
+    if (deleted > 0) {
+      log.info('Stuck-job detector deleted expired runner sessions', {deleted});
+    }
+  } catch (error) {
+    log.warn('Stuck-job detector failed to delete expired runner sessions', {
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+
+  try {
+    const {deleted} = await deleteExpiredEphemeralRegistrationTokensActivity();
+    if (deleted > 0) {
+      log.info('Stuck-job detector deleted expired ephemeral registration tokens', {deleted});
+    }
+  } catch (error) {
+    log.warn('Stuck-job detector failed to delete expired ephemeral registration tokens', {
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+
+  const {expired} = await detectAndExpireStuckJobsActivity({
+    thresholdSeconds: STUCK_JOB_THRESHOLD_SECONDS,
+  });
+  if (expired > 0) {
+    log.info('Stuck-job detector expired job leases', {
+      expired,
+      thresholdSeconds: STUCK_JOB_THRESHOLD_SECONDS,
+    });
+  }
+
+  const {reaped, reservationsReleased} = await reapStaleProvisionedRunnersActivity();
+  if (reaped > 0) {
+    log.info('Stuck-job detector reaped stale provisioned runners', {
+      reaped,
+      reservationsReleased,
+    });
+  }
+}

package/test/env.ts

@@ -0,0 +1,11 @@
+process.env.POSTGRES_HOST ??= 'localhost';
+process.env.POSTGRES_PORT ??= '5432';
+process.env.POSTGRES_USERNAME ??= 'shipfox';
+process.env.POSTGRES_PASSWORD ??= 'password';
+process.env.POSTGRES_DATABASE = 'api_test';
+process.env.POSTGRES_MAX_CONNECTIONS ??= '5';
+process.env.WORKSPACE_JWT_SECRET = 'test-secret';
+process.env.AUTH_JWT_SECRET = 'test-secret';
+process.env.AUTH_JOB_LEASE_TOKEN_SECRET = 'test-lease-secret';
+process.env.AUTH_RUNNER_SESSION_TOKEN_SECRET = 'test-runner-session-secret';
+process.env.TZ = 'UTC';

package/test/factories/ephemeral-registration-token.ts

@@ -0,0 +1,41 @@
+import {extractDisplayPrefix, generateOpaqueToken, hashOpaqueToken} from '@shipfox/node-tokens';
+import {Factory} from 'fishery';
+import type {EphemeralRegistrationToken} from '#core/entities/ephemeral-registration-token.js';
+import {createEphemeralRegistrationToken} from '#db/ephemeral-registration-tokens.js';
+
+export interface EphemeralRegistrationTokenFactoryTransientParams {
+  rawToken?: string;
+}
+
+export const ephemeralRegistrationTokenFactory = Factory.define<
+  EphemeralRegistrationToken,
+  EphemeralRegistrationTokenFactoryTransientParams
+>(({onCreate, transientParams}) => {
+  const rawToken = transientParams.rawToken ?? generateOpaqueToken('ephemeralRegistrationToken');
+
+  onCreate((token) => {
+    return createEphemeralRegistrationToken({
+      workspaceId: token.workspaceId,
+      provisionerId: token.provisionerId,
+      reservationId: token.reservationId,
+      provisionedRunnerId: token.provisionedRunnerId,
+      hashedToken: token.hashedToken,
+      prefix: token.prefix,
+      expiresAt: token.expiresAt,
+    });
+  });
+
+  return {
+    id: crypto.randomUUID(),
+    workspaceId: crypto.randomUUID(),
+    provisionerId: crypto.randomUUID(),
+    reservationId: null,
+    provisionedRunnerId: `provisioned-runner-${crypto.randomUUID()}`,
+    hashedToken: hashOpaqueToken(rawToken),
+    prefix: extractDisplayPrefix(rawToken),
+    expiresAt: new Date(Date.now() + 60_000),
+    consumedAt: null,
+    consumedSessionId: null,
+    createdAt: new Date(),
+  };
+});

package/test/factories/manual-registration-token.ts

@@ -0,0 +1,37 @@
+import {extractDisplayPrefix, generateOpaqueToken, hashOpaqueToken} from '@shipfox/node-tokens';
+import {Factory} from 'fishery';
+import type {ManualRegistrationToken} from '#core/entities/manual-registration-token.js';
+import {createManualRegistrationToken} from '#db/manual-registration-tokens.js';
+
+export interface ManualRegistrationTokenFactoryTransientParams {
+  rawToken?: string;
+}
+
+export const manualRegistrationTokenFactory = Factory.define<
+  ManualRegistrationToken,
+  ManualRegistrationTokenFactoryTransientParams
+>(({onCreate, transientParams}) => {
+  const rawToken = transientParams.rawToken ?? generateOpaqueToken('manualRegistrationToken');
+
+  onCreate((token) => {
+    return createManualRegistrationToken({
+      workspaceId: token.workspaceId,
+      hashedToken: token.hashedToken,
+      prefix: token.prefix,
+      name: token.name ?? undefined,
+      expiresAt: token.expiresAt ?? undefined,
+    });
+  });
+
+  return {
+    id: crypto.randomUUID(),
+    workspaceId: crypto.randomUUID(),
+    hashedToken: hashOpaqueToken(rawToken),
+    prefix: extractDisplayPrefix(rawToken),
+    name: 'test runner',
+    expiresAt: null,
+    revokedAt: null,
+    createdAt: new Date(),
+    updatedAt: new Date(),
+  };
+});

package/test/factories/pending-job.ts

@@ -0,0 +1,44 @@
+import {Factory} from 'fishery';
+import {enqueueJobExecution} from '#db/job-executions.js';
+
+interface PendingJobAttrs {
+  workspaceId: string;
+  workflowRunId: string;
+  workflowRunAttemptId: string;
+  jobId: string;
+  jobExecutionId: string;
+  projectId: string;
+  requiredLabels: string[];
+}
+
+export const pendingJobFactory = Factory.define<PendingJobAttrs>(({onCreate}) => {
+  const jobId = crypto.randomUUID();
+  const jobExecutionId = crypto.randomUUID();
+  const workflowRunId = crypto.randomUUID();
+  const workflowRunAttemptId = crypto.randomUUID();
+  const workspaceId = crypto.randomUUID();
+  const projectId = crypto.randomUUID();
+
+  onCreate(async (attrs) => {
+    await enqueueJobExecution({
+      workspaceId: attrs.workspaceId,
+      workflowRunId: attrs.workflowRunId,
+      workflowRunAttemptId: attrs.workflowRunAttemptId,
+      jobId: attrs.jobId,
+      jobExecutionId: attrs.jobExecutionId,
+      projectId: attrs.projectId,
+      requiredLabels: attrs.requiredLabels,
+    });
+    return attrs;
+  });
+
+  return {
+    workspaceId,
+    workflowRunId,
+    workflowRunAttemptId,
+    jobId,
+    jobExecutionId,
+    projectId,
+    requiredLabels: ['linux'],
+  };
+});