@shipfox/api-workflows 2.0.0

package/src/presentation/subscribers/on-workflow-run-cancelled.test.ts

@@ -0,0 +1,59 @@
+import type {WorkflowsWorkflowRunCancelledEventDto} from '@shipfox/api-workflows-dto';
+import {onWorkflowRunCancelled} from './on-workflow-run-cancelled.js';
+
+const signalMock = vi.fn();
+const getHandleMock = vi.fn(() => ({signal: signalMock}));
+
+vi.mock('@shipfox/node-temporal', () => ({
+  temporalClient: () => ({workflow: {getHandle: getHandleMock}}),
+}));
+
+function buildPayload(
+  overrides: Partial<WorkflowsWorkflowRunCancelledEventDto> = {},
+): WorkflowsWorkflowRunCancelledEventDto {
+  return {
+    workflowRunId: crypto.randomUUID(),
+    workflowRunAttemptId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    ...overrides,
+  };
+}
+
+describe('onWorkflowRunCancelled', () => {
+  beforeEach(() => {
+    getHandleMock.mockClear();
+    signalMock.mockReset();
+    signalMock.mockResolvedValue(undefined);
+  });
+
+  it('signals the run workflow that cancellation was requested', async () => {
+    const payload = buildPayload();
+
+    await onWorkflowRunCancelled(payload);
+
+    expect(getHandleMock).toHaveBeenCalledWith(
+      `workflow-run-attempt:${payload.workflowRunAttemptId}`,
+    );
+    expect(signalMock).toHaveBeenCalledWith('run-cancel');
+  });
+
+  it('discards a late event when the run workflow already terminated', async () => {
+    const notFound = new Error('gone');
+    notFound.name = 'WorkflowNotFoundError';
+    signalMock.mockRejectedValueOnce(notFound);
+
+    const result = onWorkflowRunCancelled(buildPayload());
+
+    await expect(result).resolves.toBeUndefined();
+    expect(signalMock).toHaveBeenCalledTimes(1);
+  });
+
+  it('rethrows a non-terminal signal failure', async () => {
+    const failure = new Error('temporal unavailable');
+    signalMock.mockRejectedValueOnce(failure);
+
+    const result = onWorkflowRunCancelled(buildPayload());
+
+    await expect(result).rejects.toBe(failure);
+  });
+});

package/src/presentation/subscribers/on-workflow-run-cancelled.ts

@@ -0,0 +1,29 @@
+import type {WorkflowsWorkflowRunCancelledEventDto} from '@shipfox/api-workflows-dto';
+import {logger} from '@shipfox/node-opentelemetry';
+import {temporalClient} from '@shipfox/node-temporal';
+import {RUN_CANCEL_SIGNAL} from '#temporal/constants.js';
+import {isWorkflowNotFound} from '#temporal/workflow-not-found.js';
+
+export async function onWorkflowRunCancelled(
+  payload: WorkflowsWorkflowRunCancelledEventDto,
+): Promise<void> {
+  logger().info(
+    {workflowRunId: payload.workflowRunId, workflowRunAttemptId: payload.workflowRunAttemptId},
+    'Signaling run attempt orchestration cancellation',
+  );
+  const handle = temporalClient().workflow.getHandle(
+    `workflow-run-attempt:${payload.workflowRunAttemptId}`,
+  );
+  try {
+    await handle.signal(RUN_CANCEL_SIGNAL);
+  } catch (err) {
+    if (isWorkflowNotFound(err)) {
+      logger().debug(
+        {workflowRunId: payload.workflowRunId, workflowRunAttemptId: payload.workflowRunAttemptId},
+        'Run attempt workflow already terminated; cancel event discarded',
+      );
+      return;
+    }
+    throw err;
+  }
+}

package/src/temporal/activities/index.ts

@@ -0,0 +1,45 @@
+import {
+  activateJobListenerActivity,
+  bulkSetStepStatuses,
+  cancelRunnerJobsActivity,
+  drainListenerEventsActivity,
+  enqueueJobExecutionForRunner,
+  evaluateJobActivationsActivity,
+  failJobExecutionAsTimedOutActivity,
+  failRunAsTimedOutActivity,
+  loadRunAttemptDag,
+  peekListenerBufferActivity,
+  recordListenerFiringOutcomeActivity,
+  releaseLeaseActivity,
+  resolveJobListenerActivity,
+  resolveJobStatusFromJobExecutionsActivity,
+  resolveLeaseExpiredJobExecutionActivity,
+  setJobExecutionStatus,
+  setJobStatus,
+  setRunAttemptStatus,
+  settleListenerJobExecutionActivity,
+} from './orchestration-activities.js';
+
+export function createOrchestrationActivities() {
+  return {
+    loadRunAttemptDag,
+    setRunAttemptStatus,
+    setJobStatus,
+    setJobExecutionStatus,
+    bulkSetStepStatuses,
+    cancelRunnerJobsActivity,
+    enqueueJobExecutionForRunner,
+    evaluateJobActivationsActivity,
+    failJobExecutionAsTimedOutActivity,
+    failRunAsTimedOutActivity,
+    activateJobListenerActivity,
+    drainListenerEventsActivity,
+    peekListenerBufferActivity,
+    resolveJobListenerActivity,
+    settleListenerJobExecutionActivity,
+    recordListenerFiringOutcomeActivity,
+    resolveLeaseExpiredJobExecutionActivity,
+    resolveJobStatusFromJobExecutionsActivity,
+    releaseLeaseActivity,
+  };
+}

package/src/temporal/activities/orchestration-activities.test.ts

@@ -0,0 +1,115 @@
+import {ApplicationFailure} from '@temporalio/common';
+import {
+  createWorkflowRun,
+  getJobExecutionsByJobId,
+  getJobsByWorkflowRunId,
+  updateJobExecutionStatus,
+} from '#db/index.js';
+import {stripSetupStep} from '#test/fixtures/strip-setup-step.js';
+import {workflowModel} from '#test/index.js';
+import {resolveLeaseExpiredJobExecutionActivity, setJobStatus} from './orchestration-activities.js';
+
+let workspaceId: string;
+let projectId: string;
+let definitionId: string;
+
+beforeEach(() => {
+  workspaceId = crypto.randomUUID();
+  projectId = crypto.randomUUID();
+  definitionId = crypto.randomUUID();
+});
+
+describe('resolveLeaseExpiredJobExecutionActivity', () => {
+  async function seedRunningJob(stepCount: number) {
+    const run = await createWorkflowRun({
+      workspaceId,
+      projectId,
+      definitionId,
+      model: workflowModel({
+        jobs: {build: {steps: Array.from({length: stepCount}, (_, i) => ({run: `step${i + 1}`}))}},
+      }),
+      triggerPayload: {
+        source: 'manual',
+        event: 'fire',
+        subscriptionId: crypto.randomUUID(),
+        userId: crypto.randomUUID(),
+      },
+    });
+    const jobId = (await getJobsByWorkflowRunId(run.id))[0]?.id as string;
+    const jobExecutionId = (await getJobExecutionsByJobId(jobId))[0]?.id as string;
+    const running = await updateJobExecutionStatus({
+      jobExecutionId,
+      status: 'running',
+      expectedVersion: 1,
+    });
+    return {jobId, jobExecutionId, runningVersion: running.version};
+  }
+
+  test('a malformed job (no steps) fails non-retryably so it never loops to the backstop', async () => {
+    const {jobId, jobExecutionId, runningVersion} = await seedRunningJob(0);
+    // createWorkflowRun always prepends the synthetic setup step, so strip it to
+    // reproduce a genuinely stepless (malformed) job.
+    await stripSetupStep(jobId);
+
+    const error = await resolveLeaseExpiredJobExecutionActivity({
+      jobExecutionId,
+      expectedVersion: runningVersion,
+    }).catch((err: unknown) => err);
+
+    expect(error).toBeInstanceOf(ApplicationFailure);
+    expect((error as ApplicationFailure).nonRetryable).toBe(true);
+  });
+
+  test('a well-formed job resolves without raising', async () => {
+    const {jobExecutionId, runningVersion} = await seedRunningJob(2);
+
+    const result = await resolveLeaseExpiredJobExecutionActivity({
+      jobExecutionId,
+      expectedVersion: runningVersion,
+    });
+
+    expect(result.status).toBe('failed');
+  });
+});
+
+describe('setJobStatus', () => {
+  test('default-gate skipped jobs record an evaluation trace', async () => {
+    const run = await createWorkflowRun({
+      workspaceId,
+      projectId,
+      definitionId,
+      model: workflowModel({
+        jobs: {
+          build: {steps: [{run: 'npm run build'}]},
+        },
+      }),
+      triggerPayload: {
+        source: 'manual',
+        event: 'fire',
+        subscriptionId: crypto.randomUUID(),
+        userId: crypto.randomUUID(),
+      },
+    });
+    const [job] = await getJobsByWorkflowRunId(run.id);
+    if (!job) throw new Error('Expected job');
+
+    await setJobStatus({
+      jobId: job.id,
+      status: 'skipped',
+      version: job.version,
+      statusReason: 'default_gate_rejected',
+    });
+
+    const [skipped] = await getJobsByWorkflowRunId(run.id);
+    expect(skipped?.evaluationTrace).toEqual([
+      {
+        expression: 'needs.all(n, n.status == "succeeded")',
+        roots: ['needs'],
+        fillTarget: 'job-activation',
+        evaluatedAt: 'job-activation',
+        value: 'false',
+        field: 'job.default_gate',
+      },
+    ]);
+  });
+});

package/src/temporal/activities/orchestration-activities.ts

@@ -0,0 +1,305 @@
+import {cancelRunnerJobs, enqueueJobExecution, releaseJobExecution} from '@shipfox/api-runners';
+import {ApplicationFailure} from '@temporalio/common';
+import {defaultJobConditionTrace} from '#core/condition-trace.js';
+import type {JobStatus, JobStatusReason, ResolutionReason} from '#core/entities/job.js';
+import type {PersistedEvaluationTraceEntry, StepStatus} from '#core/entities/step.js';
+import type {WorkflowRunStatus} from '#core/entities/workflow-run.js';
+import {JobNotFoundError} from '#core/errors.js';
+import type {
+  RuntimeCompletionStatus,
+  RuntimeDagNode,
+} from '#core/workflow-scheduling/runtime-dag.js';
+import {
+  activateJobListener,
+  bulkUpdateStepStatuses,
+  drainListenerEventsIntoExecution,
+  type EvaluateJobActivationsParams,
+  evaluateJobActivations,
+  failJobExecutionAsTimedOut,
+  failWorkflowRunAsTimedOut,
+  getJobExecutionsByWorkflowRunAttemptId,
+  getJobsByWorkflowRunAttemptId,
+  getWorkflowRunAttemptById,
+  getWorkflowRunByAttemptId,
+  type JobActivationDecision,
+  peekListenerBuffer,
+  resolveJobExecutionAfterLeaseExpiry,
+  resolveJobListener,
+  resolveJobStatusFromJobExecutions,
+  settleListenerJobExecution,
+  updateJobExecutionStatus,
+  updateJobStatus,
+  updateWorkflowRunStatus,
+} from '#db/index.js';
+import {recordWorkflowListenerExecution} from '#metrics/instance.js';
+
+export interface DagJob extends RuntimeDagNode {
+  id: string;
+  key: string;
+  mode: 'one_shot' | 'listening';
+  status: JobStatus;
+  jobExecutionId?: string | undefined;
+  executionVersion?: number;
+  executionTimeoutMs?: number | null | undefined;
+  listeningTimeoutMs?: number | null | undefined;
+  maxExecutions?: number | null | undefined;
+  onResolve?: 'finish' | 'cancel' | null | undefined;
+  batchDebounceMs?: number | null | undefined;
+  batchMaxSize?: number | null | undefined;
+  batchMaxWaitMs?: number | null | undefined;
+  dependencies: string[];
+  runner: string[];
+  version: number;
+}
+
+export interface RunDag {
+  workflowRunId: string;
+  runAttemptId: string;
+  workspaceId: string;
+  projectId: string;
+  runVersion: number;
+  runTimeoutMs: number;
+  jobs: DagJob[];
+}
+
+export async function loadRunAttemptDag(runAttemptId: string): Promise<RunDag> {
+  const run = await getWorkflowRunByAttemptId(runAttemptId);
+  if (!run) throw new Error(`Run not found for attempt: ${runAttemptId}`);
+  const attempt = await getWorkflowRunAttemptById(runAttemptId);
+  if (!attempt) throw new Error(`Run attempt not found: ${runAttemptId}`);
+
+  const jobs = await getJobsByWorkflowRunAttemptId(runAttemptId);
+  const jobExecutions = await getJobExecutionsByWorkflowRunAttemptId(runAttemptId);
+  const firstJobExecutions = new Map<string, (typeof jobExecutions)[number]>();
+  for (const jobExecution of jobExecutions) {
+    if (!firstJobExecutions.has(jobExecution.jobId)) {
+      firstJobExecutions.set(jobExecution.jobId, jobExecution);
+    }
+  }
+
+  return {
+    workflowRunId: run.id,
+    runAttemptId,
+    workspaceId: run.workspaceId,
+    projectId: run.projectId,
+    runVersion: attempt.version,
+    runTimeoutMs: run.timeoutMs,
+    jobs: jobs.flatMap((job) => {
+      const jobExecution = firstJobExecutions.get(job.id);
+      const modelJob = attempt.model?.jobs.find((item) => item.key === job.key);
+      if (!jobExecution && job.mode !== 'listening') return [];
+      return [
+        {
+          id: job.id,
+          key: job.key,
+          mode: job.mode,
+          status: job.status,
+          ...(jobExecution === undefined
+            ? {}
+            : {jobExecutionId: jobExecution.id, executionVersion: jobExecution.version}),
+          executionTimeoutMs: job.executionTimeoutMs,
+          listeningTimeoutMs: job.listeningTimeoutMs,
+          maxExecutions: job.maxExecutions,
+          onResolve: job.onResolve,
+          batchDebounceMs: job.batchDebounceMs,
+          batchMaxSize: job.batchMaxSize,
+          batchMaxWaitMs: job.batchMaxWaitMs,
+          dependencies: job.dependencies,
+          hasActivationCondition: modelJob?.if !== undefined,
+          runner: jobExecution?.runner ?? job.runner ?? [],
+          version: job.version,
+        },
+      ];
+    }),
+  };
+}
+
+export async function evaluateJobActivationsActivity(
+  params: EvaluateJobActivationsParams,
+): Promise<JobActivationDecision[]> {
+  return await evaluateJobActivations(params);
+}
+
+export async function setRunAttemptStatus(params: {
+  runAttemptId: string;
+  status: WorkflowRunStatus;
+  version: number;
+}): Promise<{newVersion: number; status: WorkflowRunStatus}> {
+  const updated = await updateWorkflowRunStatus({
+    workflowRunAttemptId: params.runAttemptId,
+    status: params.status,
+    expectedVersion: params.version,
+  });
+  return {newVersion: updated.version, status: updated.status};
+}
+
+export async function setJobStatus(params: {
+  jobId: string;
+  status: JobStatus;
+  version: number;
+  statusReason?: JobStatusReason | null | undefined;
+  evaluationTrace?: readonly PersistedEvaluationTraceEntry[] | null | undefined;
+}): Promise<{newVersion: number; status: JobStatus}> {
+  const updated = await updateJobStatus({
+    jobId: params.jobId,
+    status: params.status,
+    expectedVersion: params.version,
+    statusReason: params.statusReason,
+    evaluationTrace: jobStatusEvaluationTrace(params),
+  });
+  return {newVersion: updated.version, status: updated.status};
+}
+
+function jobStatusEvaluationTrace(params: {
+  status: JobStatus;
+  statusReason?: JobStatusReason | null | undefined;
+  evaluationTrace?: readonly PersistedEvaluationTraceEntry[] | null | undefined;
+}): readonly PersistedEvaluationTraceEntry[] | null | undefined {
+  if (params.evaluationTrace !== undefined) return params.evaluationTrace;
+  return params.status === 'skipped' && params.statusReason === 'default_gate_rejected'
+    ? defaultJobConditionTrace()
+    : undefined;
+}
+
+export async function setJobExecutionStatus(params: {
+  jobExecutionId: string;
+  status: Exclude<JobStatus, 'skipped'>;
+  version: number;
+  statusReason?: JobStatusReason | null | undefined;
+}): Promise<{newVersion: number; status: Exclude<JobStatus, 'skipped'>}> {
+  const updated = await updateJobExecutionStatus({
+    jobExecutionId: params.jobExecutionId,
+    status: params.status,
+    expectedVersion: params.version,
+    statusReason: params.statusReason,
+  });
+  return {newVersion: updated.version, status: updated.status};
+}
+
+export async function bulkSetStepStatuses(params: {
+  jobExecutionId: string;
+  status: Extract<StepStatus, 'failed' | 'cancelled'>;
+}): Promise<void> {
+  await bulkUpdateStepStatuses(params);
+}
+
+export async function resolveLeaseExpiredJobExecutionActivity(params: {
+  jobExecutionId: string;
+  expectedVersion: number;
+}): Promise<{status: RuntimeCompletionStatus; executionVersion: number}> {
+  try {
+    return await resolveJobExecutionAfterLeaseExpiry(params);
+  } catch (err) {
+    if (err instanceof JobNotFoundError) {
+      throw ApplicationFailure.nonRetryable(err.message, err.name);
+    }
+    throw err;
+  }
+}
+
+// Best-effort lease cleanup on job finalization. Idempotent: deleting an
+// already-released (or already-reaped) lease is a no-op. The workflow wraps the
+// call so a persistent failure never blocks the DAG result.
+export async function releaseLeaseActivity(params: {jobExecutionId: string}): Promise<void> {
+  await releaseJobExecution({jobExecutionId: params.jobExecutionId});
+}
+
+export async function enqueueJobExecutionForRunner(params: {
+  workspaceId: string;
+  workflowRunId: string;
+  jobId: string;
+  jobExecutionId: string;
+  runAttemptId: string;
+  projectId: string;
+  requiredLabels: string[];
+}): Promise<void> {
+  try {
+    await enqueueJobExecution({
+      workspaceId: params.workspaceId,
+      workflowRunId: params.workflowRunId,
+      workflowRunAttemptId: params.runAttemptId,
+      jobId: params.jobId,
+      jobExecutionId: params.jobExecutionId,
+      projectId: params.projectId,
+      requiredLabels: params.requiredLabels,
+    });
+  } catch (err) {
+    if (err instanceof Error && err.name === 'EmptyRequiredLabelsError') {
+      throw ApplicationFailure.nonRetryable(err.message, err.name);
+    }
+    throw err;
+  }
+}
+
+export async function cancelRunnerJobsActivity(params: {jobIds: string[]}): Promise<void> {
+  await cancelRunnerJobs({jobIds: params.jobIds});
+}
+
+export async function failJobExecutionAsTimedOutActivity(params: {
+  jobExecutionId: string;
+  runAttemptId: string;
+  expectedVersion: number;
+}): Promise<{newVersion: number}> {
+  const jobExecution = await failJobExecutionAsTimedOut({
+    jobExecutionId: params.jobExecutionId,
+    workflowRunAttemptId: params.runAttemptId,
+    expectedVersion: params.expectedVersion,
+  });
+  return {newVersion: jobExecution.version};
+}
+
+export async function failRunAsTimedOutActivity(params: {runAttemptId: string}): Promise<void> {
+  await failWorkflowRunAsTimedOut({runAttemptId: params.runAttemptId});
+}
+
+export async function resolveJobStatusFromJobExecutionsActivity(params: {
+  jobId: string;
+}): Promise<{status: RuntimeCompletionStatus; jobVersion: number}> {
+  return await resolveJobStatusFromJobExecutions(params);
+}
+
+export async function activateJobListenerActivity(params: {
+  jobId: string;
+  expectedVersion: number;
+}) {
+  return await activateJobListener(params);
+}
+
+export async function drainListenerEventsActivity(params: {
+  jobId: string;
+  expectedSequence: number;
+  maxSize?: number | undefined;
+}) {
+  return await drainListenerEventsIntoExecution(params);
+}
+
+export async function peekListenerBufferActivity(params: {jobId: string}) {
+  return await peekListenerBuffer(params);
+}
+
+export async function resolveJobListenerActivity(params: {
+  jobId: string;
+  reason: ResolutionReason;
+}) {
+  return await resolveJobListener(params);
+}
+
+export async function settleListenerJobExecutionActivity(params: {
+  jobExecutionId: string;
+  status: 'failed' | 'cancelled';
+}) {
+  await settleListenerJobExecution(params);
+}
+
+// The listener workflow owns every firing's terminal outcome, but a workflow
+// sandbox cannot emit metrics. This activity is the single recording point so
+// each firing is counted exactly once, including successes that settle through
+// the child's own resolution path rather than a listener DB write. The body is
+// synchronous; it returns a resolved promise because Temporal invokes activities
+// asynchronously.
+export function recordListenerFiringOutcomeActivity(params: {
+  outcome: 'succeeded' | 'failed' | 'cancelled';
+}): Promise<void> {
+  recordWorkflowListenerExecution(params.outcome);
+  return Promise.resolve();
+}

package/src/temporal/constants.ts

@@ -0,0 +1,6 @@
+export const WORKFLOWS_TASK_QUEUE = 'workflows-orchestrator';
+export const RUN_CANCEL_SIGNAL = 'run-cancel';
+export const JOB_FINISHED_SIGNAL = 'job-finished';
+export const JOB_LEASE_EXPIRED_SIGNAL = 'job-lease-expired';
+export const LISTENER_EVENTS_AVAILABLE_SIGNAL = 'events-available';
+export const LISTENER_RESOLVE_SIGNAL = 'resolve';

package/src/temporal/index.ts

@@ -0,0 +1,6 @@
+export {createOrchestrationActivities} from './activities/index.js';
+export {
+  JOB_FINISHED_SIGNAL,
+  JOB_LEASE_EXPIRED_SIGNAL,
+  WORKFLOWS_TASK_QUEUE,
+} from './constants.js';

package/src/temporal/workflow-not-found.ts

@@ -0,0 +1,9 @@
+// A signal to a workflow that has already terminated (it finished or hit the
+// timeout backstop first) throws WorkflowNotFoundError. Match on the error name
+// rather than `instanceof` so the check holds regardless of which @temporalio
+// package instance constructed the error.
+const WORKFLOW_NOT_FOUND = 'WorkflowNotFoundError';
+
+export function isWorkflowNotFound(err: unknown): boolean {
+  return err instanceof Error && err.name === WORKFLOW_NOT_FOUND;
+}

package/src/temporal/workflows/deadline.ts

@@ -0,0 +1,7 @@
+export function deadlineReached(deadline: number | undefined): boolean {
+  return deadline !== undefined && Date.now() >= deadline;
+}
+
+export function remainingMs(deadline: number | undefined): number | undefined {
+  return deadline === undefined ? undefined : Math.max(0, deadline - Date.now());
+}

package/src/temporal/workflows/index.ts

@@ -0,0 +1,11 @@
+export {
+  jobExecutionOrchestration,
+  jobFinishedSignal,
+  jobLeaseExpiredSignal,
+} from './job-execution-orchestration.js';
+export {
+  jobListenerOrchestration,
+  listenerEventsAvailableSignal,
+  listenerResolveSignal,
+} from './job-listener-orchestration.js';
+export {runOrchestration} from './run-orchestration.js';