@shipfox/api-workflows 2.0.0

package/src/db/schema/workflow-run-attempts.ts

@@ -0,0 +1,59 @@
+import type {WorkflowModel} from '@shipfox/api-definitions';
+import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';
+import {sql} from 'drizzle-orm';
+import {check, integer, jsonb, timestamp, uniqueIndex, uuid} from 'drizzle-orm/pg-core';
+import type {AgentToolMaterializationSnapshot} from '#core/agent-tools.js';
+import type {WorkflowRunAttempt} from '#core/entities/workflow-run-attempt.js';
+import {pgTable} from './common.js';
+import {workflowRunRerunModeEnum, workflowRunStatusEnum, workflowRuns} from './workflow-runs.js';
+
+export const workflowRunAttempts = pgTable(
+  'workflow_run_attempts',
+  {
+    id: uuidv7PrimaryKey(),
+    workflowRunId: uuid('workflow_run_id')
+      .notNull()
+      .references(() => workflowRuns.id, {onDelete: 'cascade'}),
+    attempt: integer('attempt').notNull(),
+    status: workflowRunStatusEnum('status').notNull().default('pending'),
+    rerunMode: workflowRunRerunModeEnum('rerun_mode'),
+    rerunByUserId: uuid('rerun_by_user_id'),
+    model: jsonb('model').$type<WorkflowModel>(),
+    agentToolMaterialization: jsonb(
+      'agent_tool_materialization',
+    ).$type<AgentToolMaterializationSnapshot>(),
+    version: integer('version').notNull().default(1),
+    createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
+    updatedAt: timestamp('updated_at', {withTimezone: true}).notNull().defaultNow(),
+    startedAt: timestamp('started_at', {withTimezone: true}),
+    finishedAt: timestamp('finished_at', {withTimezone: true}),
+  },
+  (table) => [
+    uniqueIndex('workflows_wra_workflow_run_attempt_unique').on(table.workflowRunId, table.attempt),
+    check('workflows_wra_attempt_positive_ck', sql`${table.attempt} > 0`),
+    uniqueIndex('workflows_wra_one_active_attempt_unique')
+      .on(table.workflowRunId)
+      .where(sql`${table.status} in ('pending', 'running')`),
+  ],
+);
+
+export type WorkflowRunAttemptDb = typeof workflowRunAttempts.$inferSelect;
+export type WorkflowRunAttemptCreateDb = typeof workflowRunAttempts.$inferInsert;
+
+export function toWorkflowRunAttempt(row: WorkflowRunAttemptDb): WorkflowRunAttempt {
+  return {
+    id: row.id,
+    workflowRunId: row.workflowRunId,
+    attempt: row.attempt,
+    status: row.status,
+    rerunMode: row.rerunMode ?? null,
+    rerunByUserId: row.rerunByUserId,
+    model: row.model ?? null,
+    agentToolMaterialization: row.agentToolMaterialization ?? null,
+    version: row.version,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+    startedAt: row.startedAt,
+    finishedAt: row.finishedAt,
+  };
+}

package/src/db/schema/workflow-runs.ts

@@ -0,0 +1,112 @@
+import {DEFAULT_RUN_TIMEOUT_MS} from '@shipfox/api-definitions';
+import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';
+import {sql} from 'drizzle-orm';
+import {
+  bigint,
+  check,
+  index,
+  integer,
+  jsonb,
+  pgEnum,
+  text,
+  timestamp,
+  uniqueIndex,
+  uuid,
+} from 'drizzle-orm/pg-core';
+import type {
+  TriggerPayload,
+  WorkflowRun,
+  WorkflowSourceSnapshot,
+} from '#core/entities/workflow-run.js';
+import {pgTable} from './common.js';
+
+export const workflowRunStatusEnum = pgEnum('workflows_run_status', [
+  'pending',
+  'running',
+  'succeeded',
+  'failed',
+  'cancelled',
+]);
+
+export const workflowRunRerunModeEnum = pgEnum('workflows_rerun_mode', ['all', 'failed']);
+
+export const workflowRuns = pgTable(
+  'workflow_runs',
+  {
+    id: uuidv7PrimaryKey(),
+    workspaceId: uuid('workspace_id').notNull(),
+    projectId: uuid('project_id').notNull(),
+    definitionId: uuid('definition_id').notNull(),
+    name: text('name').notNull(),
+    status: workflowRunStatusEnum('status').notNull().default('pending'),
+    currentAttempt: integer('current_attempt').notNull().default(1),
+    triggerProvider: text('trigger_provider'),
+    triggerSource: text('trigger_source').notNull(),
+    triggerEvent: text('trigger_event').notNull(),
+    triggerPayload: jsonb('trigger_payload').notNull().$type<TriggerPayload>(),
+    inputs: jsonb('inputs').$type<Record<string, unknown>>(),
+    sourceSnapshot: jsonb('source_snapshot').$type<WorkflowSourceSnapshot>(),
+    triggerIdempotencyKey: text('trigger_idempotency_key'),
+    timeoutMs: bigint('timeout_ms', {mode: 'number'}).notNull().default(DEFAULT_RUN_TIMEOUT_MS),
+    version: integer('version').notNull().default(1),
+    createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
+    updatedAt: timestamp('updated_at', {withTimezone: true}).notNull().defaultNow(),
+    startedAt: timestamp('started_at', {withTimezone: true}),
+    finishedAt: timestamp('finished_at', {withTimezone: true}),
+  },
+  (table) => [
+    uniqueIndex('workflows_wr_trigger_idempotency_key_unique').on(table.triggerIdempotencyKey),
+    index('workflows_wr_project_created_id_idx').on(table.projectId, table.createdAt, table.id),
+    index('workflows_wr_project_status_created_id_idx').on(
+      table.projectId,
+      table.status,
+      table.createdAt,
+      table.id,
+    ),
+    index('workflows_wr_project_definition_created_id_idx').on(
+      table.projectId,
+      table.definitionId,
+      table.createdAt,
+      table.id,
+    ),
+    index('workflows_wr_project_trigger_created_id_idx').on(
+      table.projectId,
+      table.triggerSource,
+      table.createdAt,
+      table.id,
+    ),
+    // Partial index backing the running-runs depth gauge, which counts on every
+    // Prometheus scrape. Indexes only active rows so the count stays cheap as the
+    // historical table grows.
+    index('workflows_wr_running_idx').on(table.status).where(sql`${table.status} = 'running'`),
+    check('workflows_wr_current_attempt_positive_ck', sql`${table.currentAttempt} > 0`),
+  ],
+);
+
+export type WorkflowRunDb = typeof workflowRuns.$inferSelect;
+export type WorkflowRunCreateDb = typeof workflowRuns.$inferInsert;
+
+export function toWorkflowRun(row: WorkflowRunDb): WorkflowRun {
+  return {
+    id: row.id,
+    workspaceId: row.workspaceId,
+    projectId: row.projectId,
+    definitionId: row.definitionId,
+    name: row.name,
+    status: row.status,
+    currentAttempt: row.currentAttempt,
+    triggerProvider: row.triggerProvider,
+    triggerSource: row.triggerSource,
+    triggerEvent: row.triggerEvent,
+    triggerPayload: row.triggerPayload as TriggerPayload,
+    inputs: row.inputs ?? null,
+    sourceSnapshot: row.sourceSnapshot ?? null,
+    triggerIdempotencyKey: row.triggerIdempotencyKey,
+    timeoutMs: row.timeoutMs,
+    version: row.version,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+    startedAt: row.startedAt,
+    finishedAt: row.finishedAt,
+  };
+}

package/src/db/workflow-runs/job-executions.test.ts

@@ -0,0 +1,99 @@
+import {WORKFLOWS_JOB_EXECUTION_TIMED_OUT} from '@shipfox/api-workflows-dto';
+import {and, eq, sql} from 'drizzle-orm';
+import {buildModel, workflowRunAttemptId} from '#test/helpers/workflow-runs.js';
+import {db} from '../db.js';
+import {workflowsOutbox} from '../schema/outbox.js';
+import {
+  createWorkflowRun,
+  failJobExecutionAsTimedOut,
+  getFirstJobExecutionByJobId,
+  getJobsByWorkflowRunId,
+  getStepsByJobId,
+  resolveJobExecutionAfterLeaseExpiry,
+  updateJobExecutionStatus,
+} from '../workflow-runs.js';
+
+describe('workflow run job executions', () => {
+  let workspaceId: string;
+  let projectId: string;
+  let definitionId: string;
+
+  beforeEach(() => {
+    workspaceId = crypto.randomUUID();
+    projectId = crypto.randomUUID();
+    definitionId = crypto.randomUUID();
+  });
+
+  test('derives timeout outbox attempt identity from the job execution', async () => {
+    const run = await createWorkflowRun({
+      workspaceId,
+      projectId,
+      definitionId,
+      model: buildModel({jobs: {build: {steps: [{run: 'echo build'}]}}}),
+      triggerPayload: {
+        source: 'manual',
+        event: 'fire',
+        subscriptionId: crypto.randomUUID(),
+        userId: crypto.randomUUID(),
+      },
+    });
+    const [job] = await getJobsByWorkflowRunId(run.id);
+    if (!job) throw new Error('Expected workflow job');
+    const execution = await getFirstJobExecutionByJobId(job.id);
+    if (!execution) throw new Error('Expected job execution');
+    const actualAttemptId = await workflowRunAttemptId(run.id);
+
+    await failJobExecutionAsTimedOut({
+      jobExecutionId: execution.id,
+      workflowRunAttemptId: crypto.randomUUID(),
+      expectedVersion: execution.version,
+    });
+
+    const [event] = await db()
+      .select({payload: workflowsOutbox.payload})
+      .from(workflowsOutbox)
+      .where(
+        and(
+          eq(workflowsOutbox.eventType, WORKFLOWS_JOB_EXECUTION_TIMED_OUT),
+          sql`${workflowsOutbox.payload}->>'jobExecutionId' = ${execution.id}`,
+        ),
+      );
+    expect(event?.payload).toMatchObject({
+      jobId: job.id,
+      jobExecutionId: execution.id,
+      workflowRunAttemptId: actualAttemptId,
+    });
+  });
+
+  test('does not cancel steps when lease expiry loses the execution version race', async () => {
+    const run = await createWorkflowRun({
+      workspaceId,
+      projectId,
+      definitionId,
+      model: buildModel({jobs: {build: {steps: [{run: 'echo build'}]}}}),
+      triggerPayload: {
+        source: 'manual',
+        event: 'fire',
+        subscriptionId: crypto.randomUUID(),
+        userId: crypto.randomUUID(),
+      },
+    });
+    const [job] = await getJobsByWorkflowRunId(run.id);
+    if (!job) throw new Error('Expected workflow job');
+    const execution = await getFirstJobExecutionByJobId(job.id);
+    if (!execution) throw new Error('Expected job execution');
+    await updateJobExecutionStatus({
+      jobExecutionId: execution.id,
+      status: 'running',
+      expectedVersion: execution.version,
+    });
+
+    await resolveJobExecutionAfterLeaseExpiry({
+      jobExecutionId: execution.id,
+      expectedVersion: execution.version,
+    });
+
+    const jobSteps = await getStepsByJobId(job.id);
+    expect(jobSteps.every((step) => step.status === 'pending')).toBe(true);
+  });
+});

package/src/db/workflow-runs/job-executions.ts

@@ -0,0 +1,421 @@
+import {WORKFLOWS_JOB_EXECUTION_TIMED_OUT} from '@shipfox/api-workflows-dto';
+import {and, asc, desc, eq, isNull, notInArray, sql} from 'drizzle-orm';
+import type {JobStatusReason} from '#core/entities/job.js';
+import type {JobExecution, JobExecutionStatus} from '#core/entities/job-execution.js';
+import {InterpolationUnresolvableError, JobNotFoundError} from '#core/errors.js';
+import {deriveJobExecutionOutputs} from '#core/job-transition/index.js';
+import {deriveCompletion, isTerminal} from '#core/step-transition/decide-step-transition.js';
+import type {RuntimeCompletionStatus} from '#core/workflow-scheduling/runtime-dag.js';
+import {
+  recordWorkflowJobExecutionLeaseExpiryResolved,
+  recordWorkflowJobExecutionQueued,
+  recordWorkflowJobExecutionStarted,
+  recordWorkflowJobExecutionStatusChanged,
+  recordWorkflowJobExecutionTimedOut,
+} from '#metrics/instance.js';
+import {db, type Tx} from '../db.js';
+import {writeWorkflowsOutboxEvent} from '../outbox-writes.js';
+import {runningJobExecutions} from '../runner-lease-table.js';
+import {jobExecutions, toJobExecution} from '../schema/job-executions.js';
+import {jobs, toJob} from '../schema/jobs.js';
+import {stepAttempts, toStepAttempt} from '../schema/step-attempts.js';
+import {steps, toStep} from '../schema/steps.js';
+import {workflowRunAttempts} from '../schema/workflow-run-attempts.js';
+import {toWorkflowRun, workflowRuns} from '../schema/workflow-runs.js';
+import {getDirectDependencyJobContexts} from './jobs.js';
+import {loadReferencedVariables} from './runs.js';
+import {
+  getWorkflowContextForJob,
+  optimisticLockRetry,
+  TERMINAL_EXECUTION_STATUSES,
+} from './shared.js';
+import {bulkUpdateStepStatuses, getStepsByJobExecutionIdForUpdate} from './steps.js';
+
+export async function lockActiveJobExecutionLeaseForUpdate(
+  params: {jobId: string; jobExecutionId: string; runnerSessionId: string},
+  tx: Tx,
+): Promise<boolean> {
+  const rows = await tx
+    .select({id: runningJobExecutions.id})
+    .from(runningJobExecutions)
+    .where(
+      and(
+        eq(runningJobExecutions.jobId, params.jobId),
+        eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),
+        eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),
+      ),
+    )
+    .limit(1)
+    .for('update');
+
+  return rows.length > 0;
+}
+
+export async function getJobExecutionById(id: string, tx?: Tx): Promise<JobExecution | undefined> {
+  const rows = await (tx ?? db())
+    .select()
+    .from(jobExecutions)
+    .where(eq(jobExecutions.id, id))
+    .limit(1);
+  const row = rows[0];
+  return row ? toJobExecution(row) : undefined;
+}
+
+export async function getJobExecutionsByWorkflowRunAttemptId(
+  workflowRunAttemptId: string,
+): Promise<JobExecution[]> {
+  const rows = await db()
+    .select({jobExecution: jobExecutions})
+    .from(jobExecutions)
+    .innerJoin(jobs, eq(jobExecutions.jobId, jobs.id))
+    .where(eq(jobs.workflowRunAttemptId, workflowRunAttemptId))
+    .orderBy(asc(jobExecutions.sequence), asc(jobExecutions.id));
+  return rows.map((row) => toJobExecution(row.jobExecution));
+}
+
+export async function getJobExecutionsByJobId(jobId: string): Promise<JobExecution[]> {
+  const rows = await db()
+    .select()
+    .from(jobExecutions)
+    .where(eq(jobExecutions.jobId, jobId))
+    .orderBy(asc(jobExecutions.sequence), asc(jobExecutions.id));
+  return rows.map(toJobExecution);
+}
+
+export async function getFirstJobExecutionByJobId(
+  jobId: string,
+  tx?: Tx,
+): Promise<JobExecution | undefined> {
+  const rows = await (tx ?? db())
+    .select()
+    .from(jobExecutions)
+    .where(eq(jobExecutions.jobId, jobId))
+    .orderBy(asc(jobExecutions.sequence), asc(jobExecutions.id))
+    .limit(1);
+  const row = rows[0];
+  return row ? toJobExecution(row) : undefined;
+}
+
+export async function getLatestJobExecutionByJobId(
+  jobId: string,
+  tx?: Tx,
+): Promise<JobExecution | undefined> {
+  const rows = await (tx ?? db())
+    .select()
+    .from(jobExecutions)
+    .where(eq(jobExecutions.jobId, jobId))
+    .orderBy(desc(jobExecutions.sequence), desc(jobExecutions.id))
+    .limit(1);
+  const row = rows[0];
+  return row ? toJobExecution(row) : undefined;
+}
+
+export interface UpdateJobExecutionStatusAtVersionParams {
+  jobExecutionId: string;
+  status: JobExecutionStatus;
+  expectedVersion: number;
+  statusReason?: JobStatusReason | null | undefined;
+  markTimedOut?: boolean;
+}
+
+async function resolveJobExecutionOutputs(
+  tx: Tx,
+  params: {
+    jobExecutionId: string;
+    status: JobExecutionStatus;
+    statusReason: JobStatusReason | null;
+  },
+): Promise<Record<string, unknown> | null> {
+  const [target] = await tx
+    .select({execution: jobExecutions, job: jobs, attempt: workflowRunAttempts, run: workflowRuns})
+    .from(jobExecutions)
+    .innerJoin(jobs, eq(jobExecutions.jobId, jobs.id))
+    .innerJoin(workflowRunAttempts, eq(jobs.workflowRunAttemptId, workflowRunAttempts.id))
+    .innerJoin(workflowRuns, eq(workflowRunAttempts.workflowRunId, workflowRuns.id))
+    .where(eq(jobExecutions.id, params.jobExecutionId))
+    .limit(1);
+  if (!target) throw new JobNotFoundError(params.jobExecutionId);
+  const model = target.attempt.model;
+  if (!model) return null;
+  const modelJob = model.jobs.find((job) => job.key === target.job.key);
+  if (!modelJob || modelJob.outputs === undefined) return null;
+
+  const executionRows = await tx
+    .select()
+    .from(jobExecutions)
+    .where(eq(jobExecutions.jobId, target.job.id))
+    .orderBy(asc(jobExecutions.sequence), asc(jobExecutions.id));
+  const executions = executionRows.map((row) =>
+    row.id === target.execution.id
+      ? toJobExecution({...row, status: params.status, statusReason: params.statusReason})
+      : toJobExecution(row),
+  );
+  const jobExecution = executions.find((execution) => execution.id === target.execution.id);
+  if (!jobExecution) throw new JobNotFoundError(params.jobExecutionId);
+
+  const stepRows = await tx
+    .select()
+    .from(steps)
+    .where(eq(steps.jobExecutionId, params.jobExecutionId))
+    .orderBy(asc(steps.position), asc(steps.id));
+  const attemptRows = await tx
+    .select()
+    .from(stepAttempts)
+    .where(eq(stepAttempts.jobExecutionId, params.jobExecutionId))
+    .orderBy(asc(stepAttempts.executionOrder), asc(stepAttempts.id));
+  const dependencyJobs = await getDirectDependencyJobContexts(target.job.id, tx);
+
+  return deriveJobExecutionOutputs({
+    run: toWorkflowRun(target.run),
+    modelJob,
+    job: toJob(target.job),
+    jobExecution,
+    executions,
+    steps: stepRows.map(toStep),
+    attempts: attemptRows.map(toStepAttempt),
+    jobs: dependencyJobs,
+    vars: await loadReferencedVariables({
+      model,
+      jobs: [modelJob],
+      workspaceId: target.run.workspaceId,
+      projectId: target.run.projectId,
+      definitionId: target.run.definitionId,
+    }),
+  });
+}
+
+async function updateJobExecutionStatusAtVersion(
+  tx: Tx,
+  params: UpdateJobExecutionStatusAtVersionParams,
+): Promise<{execution: JobExecution; changed: boolean} | null> {
+  let status = params.status;
+  let statusReason = params.statusReason ?? null;
+  let outputs: Record<string, unknown> | null | undefined;
+  if (TERMINAL_EXECUTION_STATUSES.includes(status)) {
+    outputs = null;
+  }
+  if (status === 'succeeded') {
+    try {
+      outputs = await resolveJobExecutionOutputs(tx, {
+        jobExecutionId: params.jobExecutionId,
+        status,
+        statusReason,
+      });
+    } catch (error) {
+      if (!(error instanceof InterpolationUnresolvableError)) throw error;
+      status = 'failed';
+      statusReason = 'unknown';
+      outputs = null;
+    }
+  }
+
+  const rows = await tx
+    .update(jobExecutions)
+    .set({
+      status,
+      statusReason,
+      ...(outputs === undefined ? {} : {outputs}),
+      version: sql`${jobExecutions.version} + 1`,
+      updatedAt: new Date(),
+      ...(params.markTimedOut ? {timedOutAt: new Date()} : {}),
+      ...(TERMINAL_EXECUTION_STATUSES.includes(status) ? {finishedAt: sql`now()`} : {}),
+    })
+    .where(
+      and(
+        eq(jobExecutions.id, params.jobExecutionId),
+        eq(jobExecutions.version, params.expectedVersion),
+        notInArray(jobExecutions.status, TERMINAL_EXECUTION_STATUSES),
+      ),
+    )
+    .returning();
+
+  const row = rows[0];
+  if (!row) return null;
+  return {execution: toJobExecution(row), changed: true};
+}
+
+export interface UpdateJobExecutionStatusParams {
+  jobExecutionId: string;
+  status: JobExecutionStatus;
+  expectedVersion: number;
+  statusReason?: JobStatusReason | null | undefined;
+}
+
+export async function updateJobExecutionStatus(
+  params: UpdateJobExecutionStatusParams,
+): Promise<JobExecution> {
+  const statusReason = params.statusReason ?? null;
+  const result = await db().transaction(async (tx) => {
+    return await optimisticLockRetry({
+      updateFn: () =>
+        updateJobExecutionStatusAtVersion(tx, {
+          jobExecutionId: params.jobExecutionId,
+          status: params.status,
+          expectedVersion: params.expectedVersion,
+          statusReason,
+        }),
+      fetchFn: async () => {
+        const row = (
+          await tx
+            .select()
+            .from(jobExecutions)
+            .where(eq(jobExecutions.id, params.jobExecutionId))
+            .limit(1)
+        )[0];
+        return row ? toJobExecution(row) : undefined;
+      },
+      matchFn: (execution) =>
+        (execution.status === params.status && execution.statusReason === statusReason) ||
+        TERMINAL_EXECUTION_STATUSES.includes(execution.status)
+          ? {execution, changed: false}
+          : null,
+      failureMessage: `Optimistic lock failure: job execution ${params.jobExecutionId} version ${params.expectedVersion}`,
+    });
+  });
+
+  if (result.changed) recordWorkflowJobExecutionStatusChanged(result.execution.status);
+
+  return result.execution;
+}
+
+export async function recordJobExecutionQueuedAt(params: {
+  jobExecutionId: string;
+  queuedAt: Date;
+}): Promise<void> {
+  const updated = await db()
+    .update(jobExecutions)
+    .set({queuedAt: params.queuedAt})
+    .where(and(eq(jobExecutions.id, params.jobExecutionId), isNull(jobExecutions.queuedAt)))
+    .returning({id: jobExecutions.id});
+
+  if (updated.length > 0) recordWorkflowJobExecutionQueued();
+}
+
+export async function recordJobExecutionStartedAt(params: {
+  jobExecutionId: string;
+  startedAt: Date;
+}): Promise<void> {
+  const updated = await db()
+    .update(jobExecutions)
+    .set({startedAt: params.startedAt})
+    .where(and(eq(jobExecutions.id, params.jobExecutionId), isNull(jobExecutions.startedAt)))
+    .returning({id: jobExecutions.id});
+
+  if (updated.length > 0) recordWorkflowJobExecutionStarted();
+}
+
+export async function failJobExecutionAsTimedOut(params: {
+  jobExecutionId: string;
+  workflowRunAttemptId: string;
+  expectedVersion: number;
+}): Promise<JobExecution> {
+  const result = await db().transaction(async (tx) => {
+    const updated = await optimisticLockRetry({
+      updateFn: () =>
+        updateJobExecutionStatusAtVersion(tx, {
+          jobExecutionId: params.jobExecutionId,
+          status: 'failed',
+          expectedVersion: params.expectedVersion,
+          statusReason: 'timed_out',
+          markTimedOut: true,
+        }),
+      fetchFn: async () => {
+        const row = (
+          await tx
+            .select()
+            .from(jobExecutions)
+            .where(eq(jobExecutions.id, params.jobExecutionId))
+            .limit(1)
+        )[0];
+        return row ? toJobExecution(row) : undefined;
+      },
+      matchFn: (execution) => (execution.timedOutAt !== null ? {execution, changed: false} : null),
+      failureMessage: `Optimistic lock failure: job execution ${params.jobExecutionId} version ${params.expectedVersion}`,
+    });
+
+    if (updated.changed) {
+      const identity = await getWorkflowContextForJob(updated.execution.jobId, tx);
+      await writeWorkflowsOutboxEvent(tx, {
+        type: WORKFLOWS_JOB_EXECUTION_TIMED_OUT,
+        payload: {
+          jobId: identity.jobId,
+          jobExecutionId: params.jobExecutionId,
+          workflowRunAttemptId: identity.workflowRunAttemptId,
+        },
+      });
+    }
+
+    return updated;
+  });
+
+  if (result.changed) {
+    recordWorkflowJobExecutionStatusChanged(result.execution.status);
+    recordWorkflowJobExecutionTimedOut();
+  }
+
+  return result.execution;
+}
+
+export async function resolveJobExecutionAfterLeaseExpiry(params: {
+  jobExecutionId: string;
+  expectedVersion: number;
+}): Promise<{status: RuntimeCompletionStatus; executionVersion: number}> {
+  const result = await db().transaction(async (tx) => {
+    const jobExecutionSteps = await getStepsByJobExecutionIdForUpdate(params.jobExecutionId, tx);
+    let changedJobExecution: JobExecution | null = null;
+
+    if (jobExecutionSteps.length === 0) {
+      throw new JobNotFoundError(params.jobExecutionId);
+    }
+
+    if (jobExecutionSteps.every((step) => isTerminal(step.status))) {
+      const status = deriveCompletion(jobExecutionSteps);
+      const updated = await updateJobExecutionStatusAtVersion(tx, {
+        jobExecutionId: params.jobExecutionId,
+        status,
+        expectedVersion: params.expectedVersion,
+        statusReason: statusReasonForStepCompletion(status),
+      });
+      changedJobExecution = updated?.changed ? updated.execution : null;
+    } else {
+      const updated = await updateJobExecutionStatusAtVersion(tx, {
+        jobExecutionId: params.jobExecutionId,
+        status: 'failed',
+        expectedVersion: params.expectedVersion,
+        statusReason: 'runner_lost',
+      });
+      if (updated?.changed) {
+        changedJobExecution = updated.execution;
+        await bulkUpdateStepStatuses(
+          {jobExecutionId: params.jobExecutionId, status: 'cancelled'},
+          tx,
+        );
+      }
+    }
+
+    const jobExecutionRow = (
+      await tx
+        .select()
+        .from(jobExecutions)
+        .where(eq(jobExecutions.id, params.jobExecutionId))
+        .limit(1)
+    )[0];
+    if (!jobExecutionRow) {
+      throw new Error(`Job execution not found resolving lease expiry: ${params.jobExecutionId}`);
+    }
+    const status: RuntimeCompletionStatus =
+      jobExecutionRow.status === 'succeeded' ? 'succeeded' : 'failed';
+    return {status, executionVersion: jobExecutionRow.version, changedJobExecution};
+  });
+
+  recordWorkflowJobExecutionLeaseExpiryResolved(result.status);
+  if (result.changedJobExecution) {
+    recordWorkflowJobExecutionStatusChanged(result.changedJobExecution.status);
+  }
+
+  return {status: result.status, executionVersion: result.executionVersion};
+}
+
+function statusReasonForStepCompletion(status: RuntimeCompletionStatus): JobStatusReason | null {
+  return status === 'failed' ? 'step_failed' : null;
+}