@shipfox/api-workflows 12.2.0 → 12.3.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shipfox/api-workflows",
   "license": "MIT",
-  "version": "12.2.0",
+  "version": "12.3.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ShipfoxHQ/shipfox.git",
@@ -21,20 +21,22 @@
     "@temporalio/common": "1.18.1",
     "@temporalio/workflow": "1.18.1",
     "drizzle-orm": "^0.45.2",
+    "js-yaml": "^4.1.0",
     "zod": "^4.4.3",
     "@shipfox/api-auth-context": "12.2.0",
     "@shipfox/api-auth-dto": "12.0.0",
     "@shipfox/api-agent-dto": "12.2.0",
-    "@shipfox/annotations-dto": "12.0.0",
-    "@shipfox/api-definitions-dto": "12.2.0",
+    "@shipfox/annotations-dto": "12.3.0",
+    "@shipfox/api-definitions-dto": "12.3.0",
     "@shipfox/api-projects-dto": "12.2.0",
     "@shipfox/api-integration-core-dto": "12.2.0",
     "@shipfox/api-runners-dto": "12.2.0",
     "@shipfox/api-secrets-dto": "12.0.0",
-    "@shipfox/api-workflows-dto": "12.2.0",
-    "@shipfox/inter-module": "0.2.3",
+    "@shipfox/api-workflows-dto": "12.3.0",
     "@shipfox/api-workspaces-dto": "12.0.0",
-    "@shipfox/expression": "2.1.0",
+    "@shipfox/config": "1.2.4",
+    "@shipfox/inter-module": "0.2.3",
+    "@shipfox/expression": "2.2.0",
     "@shipfox/node-drizzle": "0.3.5",
     "@shipfox/node-error-monitoring": "0.3.0",
     "@shipfox/node-fastify": "0.4.2",
@@ -43,8 +45,8 @@
     "@shipfox/node-outbox": "0.2.6",
     "@shipfox/node-postgres": "0.5.0",
     "@shipfox/node-temporal": "0.4.5",
-    "@shipfox/workflow-document": "3.0.1",
-    "@shipfox/runner-labels": "0.2.0"
+    "@shipfox/runner-labels": "0.2.0",
+    "@shipfox/workflow-document": "3.0.1"
   },
   "imports": {
     "#*": "./dist/*"

package/src/config.test.ts

@@ -0,0 +1,85 @@
+import {randomUUID} from 'node:crypto';
+import {mkdtempSync, rmSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {loadRunnerCatalog} from './config.js';
+
+let dir: string;
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), 'api-workflows-'));
+});
+
+afterEach(() => {
+  rmSync(dir, {recursive: true, force: true});
+});
+
+function writeCatalog(contents: string): string {
+  const path = join(dir, `${randomUUID()}.yaml`);
+  writeFileSync(path, contents);
+  return path;
+}
+
+describe('loadRunnerCatalog', () => {
+  it('returns an empty catalog when no path is configured', () => {
+    expect(loadRunnerCatalog('')).toEqual({});
+  });
+
+  it.each(['', ' \n\t'])('treats an empty YAML document as empty', (contents) => {
+    expect(loadRunnerCatalog(writeCatalog(contents))).toEqual({});
+  });
+
+  it.each([
+    '# just a comment',
+    '---\n',
+    'null\n',
+    '~\n',
+  ])('rejects a null YAML document (%j)', (contents) => {
+    const path = writeCatalog(contents);
+
+    expect(() => loadRunnerCatalog(path)).toThrow(`Invalid runner catalog config at ${path}`);
+  });
+
+  it('loads and validates a YAML catalog', () => {
+    const path = writeCatalog(`
+ShipFox-4CPU:
+  - OS.Ubuntu-Latest
+  - CPU.4
+`);
+
+    expect(loadRunnerCatalog(path)).toEqual({
+      'shipfox-4cpu': ['cpu.4', 'os.ubuntu-latest'],
+    });
+  });
+
+  it('includes the file path when the catalog file is missing', () => {
+    const path = join(dir, 'missing.yaml');
+
+    expect(() => loadRunnerCatalog(path)).toThrow(`Cannot read runner catalog config at ${path}`);
+  });
+
+  it('includes the file path when YAML is malformed', () => {
+    const path = writeCatalog('runner: [unclosed');
+
+    expect(() => loadRunnerCatalog(path)).toThrow(`Cannot parse runner catalog config at ${path}`);
+  });
+
+  it.each([
+    ['bad name', 'label'],
+    ['name', 'not a label'],
+  ])('includes the file path when the catalog contains an invalid %s', (name, value) => {
+    const path = writeCatalog(`${name}:\n  - ${value}\n`);
+
+    expect(() => loadRunnerCatalog(path)).toThrow(`Invalid runner catalog config at ${path}`);
+  });
+
+  it('rejects oversized entries at startup', () => {
+    const path = writeCatalog(
+      `too-many:\n${Array.from({length: 21}, (_, index) => `  - label-${index}`).join('\n')}\n`,
+    );
+
+    expect(() => loadRunnerCatalog(path)).toThrow(
+      `Runner catalog entry "too-many" in ${path} has 21 labels`,
+    );
+  });
+});

package/src/config.ts

@@ -0,0 +1,71 @@
+import {readFileSync} from 'node:fs';
+import {createConfig, str} from '@shipfox/config';
+import {MAX_RUNNER_LABELS, parseRunnerCatalog, type RunnerCatalog} from '@shipfox/runner-labels';
+import yaml from 'js-yaml';
+
+export const config = createConfig({
+  RUNNER_CATALOG_PATH: str({
+    desc: 'Path to the YAML file that maps runner catalog names to complete label sets. Leave it empty to use every job runner value as a literal label. The file is loaded at startup; restart the API after changing it.',
+    default: '',
+  }),
+});
+
+/** Raised when the configured runner catalog cannot be read, parsed, or validated. */
+export class RunnerCatalogConfigError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = 'RunnerCatalogConfigError';
+  }
+}
+
+export function loadRunnerCatalog(filePath: string): RunnerCatalog {
+  if (filePath === '') return {};
+
+  let contents: string;
+  try {
+    contents = readFileSync(filePath, 'utf8');
+  } catch (error) {
+    throw new RunnerCatalogConfigError(
+      `Cannot read runner catalog config at ${filePath}: ${errorMessage(error)}`,
+      {cause: error},
+    );
+  }
+
+  if (contents.trim() === '') return {};
+
+  let raw: unknown;
+  try {
+    raw = yaml.load(contents);
+  } catch (error) {
+    throw new RunnerCatalogConfigError(
+      `Cannot parse runner catalog config at ${filePath}: ${errorMessage(error)}`,
+      {cause: error},
+    );
+  }
+
+  let catalog: RunnerCatalog;
+  try {
+    catalog = parseRunnerCatalog(raw);
+  } catch (error) {
+    throw new RunnerCatalogConfigError(
+      `Invalid runner catalog config at ${filePath}: ${errorMessage(error)}`,
+      {cause: error},
+    );
+  }
+
+  for (const [name, labels] of Object.entries(catalog)) {
+    if (labels.length > MAX_RUNNER_LABELS) {
+      throw new RunnerCatalogConfigError(
+        `Runner catalog entry "${name}" in ${filePath} has ${labels.length} labels; the maximum is ${MAX_RUNNER_LABELS}.`,
+      );
+    }
+  }
+
+  return catalog;
+}
+
+export const runnerCatalog = loadRunnerCatalog(config.RUNNER_CATALOG_PATH);
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/core/errors.ts

@@ -134,8 +134,15 @@ export class JobNotFoundError extends Error {
 }
 
 export class InvalidJobRunnerLabelsError extends Error {
-  constructor(readonly labels: readonly string[]) {
-    super(`Job runner labels are invalid: ${labels.join(', ')}`);
+  constructor(
+    readonly labels: readonly string[],
+    readonly requestedLabels: readonly string[] = labels,
+  ) {
+    const requestedSuffix =
+      requestedLabels.join(', ') === labels.join(', ')
+        ? ''
+        : ` (requested: ${requestedLabels.join(', ')})`;
+    super(`Job runner labels are invalid: ${labels.join(', ')}${requestedSuffix}`);
     this.name = 'InvalidJobRunnerLabelsError';
   }
 }

package/src/core/job-execution.test.ts

@@ -1139,6 +1139,7 @@ describe('step attempts', () => {
         projectId: expect.any(String),
         stepId,
         attempt: 1,
+        status: 'failed',
         logOutcome: 'abandoned',
       },
     ]);

package/src/core/step-config/materialize-workflow-model.catalog.test.ts

@@ -0,0 +1,117 @@
+import {mkdtempSync, rmSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {workflowModel} from '#test/index.js';
+
+let dir: string;
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), 'api-workflows-runner-catalog-'));
+});
+
+afterEach(() => {
+  vi.unstubAllEnvs();
+  vi.resetModules();
+  rmSync(dir, {recursive: true, force: true});
+});
+
+function template(source: string): string {
+  return `\${{ ${source} }}`;
+}
+
+function writeCatalog(contents: string): string {
+  const path = join(dir, 'runner-catalog.yaml');
+  writeFileSync(path, contents);
+  return path;
+}
+
+function executionContext(runner: string) {
+  return {
+    site: 'execution-creation' as const,
+    values: {
+      run: {
+        id: 'run-1',
+        name: 'Build',
+        definition_id: 'definition-1',
+        project_id: 'project-1',
+        workspace_id: 'workspace-1',
+        created_at: new Date('2026-06-30T12:00:00.000Z'),
+      },
+      trigger: {source: 'manual', event: 'fire'},
+      event: null,
+      inputs: null,
+      execution: {
+        index: 1,
+        name: 'build #1',
+        status: 'pending',
+        started_at: null,
+        finished_at: null,
+        events: [{data: {runner}, source: 'github', event: 'push'}],
+      },
+    },
+  };
+}
+
+describe('materializeJobRunner with a catalog', () => {
+  it('expands an interpolated catalog name and keeps free-form labels', async () => {
+    const path = writeCatalog(`
+shipfox-4cpu:
+  - arch.amd64
+  - cpu.4
+`);
+    vi.stubEnv('RUNNER_CATALOG_PATH', path);
+    vi.resetModules();
+
+    const {materializeJobRunner} = await import('./materialize-workflow-model.js');
+    const model = workflowModel({
+      jobs: {
+        build: {
+          runner: ['internal-network'],
+          runnerTemplates: [template('execution.events[0].data.runner')],
+          steps: [{run: 'npm test'}],
+        },
+      },
+    });
+    const [job] = model.jobs;
+    if (!job) throw new Error('Test model created no jobs');
+
+    const result = materializeJobRunner({
+      job,
+      context: executionContext('SHIPFOX-4CPU'),
+      definitionId: 'definition-1',
+    });
+
+    expect(result).toEqual(['arch.amd64', 'cpu.4', 'internal-network']);
+  });
+
+  it('validates the combined expanded label count', async () => {
+    const labels = (suffix: string) =>
+      Array.from({length: 8}, (_, index) => `  - label-${index}${suffix}`).join('\n');
+    const path = writeCatalog(
+      `first:\n${labels('')}\nsecond:\n${labels('-second')}\nthird:\n${labels('-third')}\n`,
+    );
+    vi.stubEnv('RUNNER_CATALOG_PATH', path);
+    vi.resetModules();
+
+    const {materializeJobRunner} = await import('./materialize-workflow-model.js');
+    const model = workflowModel({
+      jobs: {
+        build: {
+          runner: [],
+          runnerTemplates: [template('"first"'), template('"second"'), template('"third"')],
+          steps: [{run: 'npm test'}],
+        },
+      },
+    });
+    const [job] = model.jobs;
+    if (!job) throw new Error('Test model created no jobs');
+
+    expect(() =>
+      materializeJobRunner({
+        job,
+        context: executionContext('unused'),
+        definitionId: 'definition-1',
+      }),
+    ).toThrow('requested: first, second, third');
+  });
+});

package/src/core/step-config/materialize-workflow-model.ts

@@ -1,5 +1,6 @@
 import {DEFAULT_JOB_CHECKOUT, type WorkflowModel} from '@shipfox/api-definitions-dto';
-import {canonicalizeLabels, findInvalidLabels, MAX_RUNNER_LABELS} from '@shipfox/runner-labels';
+import {findInvalidLabels, MAX_RUNNER_LABELS, resolveRunnerLabels} from '@shipfox/runner-labels';
+import {runnerCatalog} from '#config.js';
 import type {AgentDefaultsResolver} from '#core/agent-defaults.js';
 import type {
   AgentToolMaterializationContext,
@@ -112,10 +113,11 @@ export function materializeJobRunner(params: {
       definitionId: params.definitionId,
     }),
   );
-  const labels = canonicalizeLabels([...params.job.runner, ...resolvedLabels]);
+  const requestedLabels = [...params.job.runner, ...resolvedLabels];
+  const labels = resolveRunnerLabels(requestedLabels, runnerCatalog);
   const invalidLabels = findInvalidLabels(labels);
   if (labels.length === 0 || labels.length > MAX_RUNNER_LABELS || invalidLabels.length > 0) {
-    throw new InvalidJobRunnerLabelsError(labels);
+    throw new InvalidJobRunnerLabelsError(labels, requestedLabels);
   }
   return labels;
 }

package/src/db/index.ts

@@ -34,6 +34,7 @@ export type {
   JobActivationDecision,
   ListWorkflowRunsParams,
   ListWorkflowRunsResult,
+  StepAttemptDetail,
   UpdateJobExecutionStatusParams,
   UpdateJobStatusParams,
   UpdateWorkflowRunStatusParams,
@@ -58,12 +59,15 @@ export {
   getJobById,
   getJobExecutionById,
   getJobExecutionDetail,
+  getJobExecutionFailureOrigin,
   getJobExecutionsByJobId,
   getJobExecutionsByWorkflowRunAttemptId,
   getJobScope,
   getJobsByWorkflowRunAttemptId,
   getJobsByWorkflowRunId,
   getLatestAttempt,
+  getLatestJobExecutionByJobId,
+  getStepAttemptDetail,
   getStepAttempts,
   getStepAttemptsByJobIds,
   getStepById,

package/src/db/workflow-runs/job-status.test.ts

@@ -473,7 +473,8 @@ describe('workflow run queries', () => {
     async function seedPendingJob() {
       const run = await createTestRun({workspaceId, projectId, definitionId});
       const jobId = (await getJobsByWorkflowRunId(run.id))[0]?.id as string;
-      return {run, jobId};
+      const execution = await getFirstJobExecutionByJobId(jobId);
+      return {run, jobId, jobExecutionId: execution?.id};
     }
 
     test.each([
@@ -482,13 +483,19 @@ describe('workflow run queries', () => {
       'cancelled',
       'skipped',
     ] as const)('writes one terminated event when a job becomes %s', async (status) => {
-      const {run, jobId} = await seedPendingJob();
+      const {run, jobId, jobExecutionId} = await seedPendingJob();
 
       await updateJobStatus({jobId, status, expectedVersion: 1});
 
       const events = await jobTerminatedEvents(jobId);
       expect(events).toHaveLength(1);
-      expect(events[0]).toMatchObject({jobId, workflowRunId: run.id, status, statusReason: null});
+      expect(events[0]).toMatchObject({
+        jobId,
+        jobExecutionId,
+        workflowRunId: run.id,
+        status,
+        statusReason: null,
+      });
     });
 
     test('writes status reason on the terminated event', async () => {

package/src/db/workflow-runs/jobs.ts

@@ -347,11 +347,18 @@ export async function updateJobStatusAtVersion(
   // version match lets a single caller win. Emitting here, in the same transaction,
   // makes the terminal fact fire exactly once across all paths.
   if (isJobTerminal(job.status)) {
+    const [currentExecution] = await tx
+      .select({id: jobExecutions.id})
+      .from(jobExecutions)
+      .where(eq(jobExecutions.jobId, job.id))
+      .orderBy(desc(jobExecutions.sequence), desc(jobExecutions.id))
+      .limit(1);
     const identity = await getWorkflowContextForJob(job.id, tx);
     await writeWorkflowsOutboxEvent(tx, {
       type: WORKFLOWS_JOB_TERMINATED,
       payload: {
         jobId: job.id,
+        jobExecutionId: currentExecution?.id ?? null,
         workflowRunId: identity.workflowRunId,
         workflowRunAttemptId: identity.workflowRunAttemptId,
         status: job.status,

package/src/db/workflow-runs/outbox.ts

@@ -35,7 +35,12 @@ export async function writeJobStepsSettledOutbox(
 
 export async function writeStepAttemptTerminatedOutbox(
   tx: Tx,
-  params: {stepId: string; attempt: number; logOutcome: LogOutcomeDto},
+  params: {
+    stepId: string;
+    attempt: number;
+    status: 'succeeded' | 'failed' | 'cancelled';
+    logOutcome: LogOutcomeDto;
+  },
 ): Promise<void> {
   const identity = await getStepAttemptTerminatedOutboxIdentity(tx, params.stepId);
 
@@ -49,6 +54,7 @@ export async function writeStepAttemptTerminatedOutbox(
       projectId: identity.projectId,
       stepId: params.stepId,
       attempt: params.attempt,
+      status: params.status,
       logOutcome: params.logOutcome,
     },
   });

package/src/db/workflow-runs/steps.test.ts

@@ -11,7 +11,10 @@ import {stepAttempts as stepAttemptsTable} from '../schema/step-attempts.js';
 import {steps as stepsTable} from '../schema/steps.js';
 import {
   createWorkflowRun,
+  getFirstJobExecutionByJobId,
+  getJobExecutionFailureOrigin,
   getJobsByWorkflowRunId,
+  getStepAttemptDetail,
   getStepAttempts,
   getStepsByJobId,
 } from '../workflow-runs.js';
@@ -60,6 +63,150 @@ describe('workflow run queries', () => {
     });
   });
 
+  describe('getStepAttemptDetail', () => {
+    test('joins a step attempt to its run for lazy troubleshooting reads', async () => {
+      const run = await createWorkflowRun({
+        workspaceId,
+        projectId,
+        definitionId,
+        model: buildModel({jobs: {build: {steps: [{run: 'echo hello'}]}}}),
+        triggerPayload: {
+          source: 'manual',
+          event: 'fire',
+          subscriptionId: crypto.randomUUID(),
+          userId: crypto.randomUUID(),
+        },
+      });
+      const [job] = await getJobsByWorkflowRunId(run.id);
+      if (!job) throw new Error('Expected a workflow job');
+      await stripSetupStep(job.id);
+      await nextStepForJob(job.id);
+
+      const [attempt] = await getStepAttempts(job.id);
+      if (!attempt) throw new Error('Expected a dispatched step attempt');
+
+      const detail = await getStepAttemptDetail({stepId: attempt.stepId, attempt: attempt.attempt});
+
+      expect(detail).toMatchObject({
+        workflowRunId: run.id,
+        workflowRunAttemptId: job.workflowRunAttemptId,
+        step: {id: attempt.stepId},
+        attempt: {id: attempt.id, attempt: attempt.attempt},
+      });
+    });
+  });
+
+  describe('getJobExecutionFailureOrigin', () => {
+    test('selects the current failed attempt instead of loading attempt history', async () => {
+      const run = await createWorkflowRun({
+        workspaceId,
+        projectId,
+        definitionId,
+        model: buildModel({
+          jobs: {build: {steps: [{run: 'echo hello'}, {run: 'echo goodbye'}]}},
+        }),
+        triggerPayload: {
+          source: 'manual',
+          event: 'fire',
+          subscriptionId: crypto.randomUUID(),
+          userId: crypto.randomUUID(),
+        },
+      });
+      const [job] = await getJobsByWorkflowRunId(run.id);
+      if (!job) throw new Error('Expected a workflow job');
+      await stripSetupStep(job.id);
+      const execution = await getFirstJobExecutionByJobId(job.id);
+      if (!execution) throw new Error('Expected a job execution');
+      await nextStepForJob(job.id);
+
+      const [step] = await getStepsByJobId(job.id);
+      const [attempt] = await getStepAttempts(job.id);
+      if (!step || !attempt) throw new Error('Expected a current step attempt');
+      await db()
+        .update(stepsTable)
+        .set({status: 'failed', error: {reason: 'command_failed', message: 'current failure'}})
+        .where(eq(stepsTable.id, step.id));
+      await db()
+        .update(stepAttemptsTable)
+        .set({
+          status: 'failed',
+          error: {reason: 'command_failed', message: 'current failure'},
+          finishedAt: new Date(),
+        })
+        .where(eq(stepAttemptsTable.id, attempt.id));
+
+      await expect(getJobExecutionFailureOrigin(execution.id)).resolves.toMatchObject({
+        jobExecutionId: execution.id,
+        stepId: step.id,
+        stepAttempt: attempt.attempt,
+        attemptStatus: 'failed',
+        attemptError: {message: 'current failure'},
+      });
+    });
+
+    test('returns the first step when the execution fails before any attempt starts', async () => {
+      const run = await createWorkflowRun({
+        workspaceId,
+        projectId,
+        definitionId,
+        model: buildModel({jobs: {build: {steps: [{run: 'echo hello'}]}}}),
+        triggerPayload: {
+          source: 'manual',
+          event: 'fire',
+          subscriptionId: crypto.randomUUID(),
+          userId: crypto.randomUUID(),
+        },
+      });
+      const [job] = await getJobsByWorkflowRunId(run.id);
+      if (!job) throw new Error('Expected a workflow job');
+      const execution = await getFirstJobExecutionByJobId(job.id);
+      if (!execution) throw new Error('Expected a job execution');
+
+      await expect(getJobExecutionFailureOrigin(execution.id)).resolves.toMatchObject({
+        jobExecutionId: execution.id,
+        stepAttempt: 1,
+        attemptStatus: null,
+      });
+    });
+
+    test('selects a condition-errored step before an earlier pending step', async () => {
+      const run = await createWorkflowRun({
+        workspaceId,
+        projectId,
+        definitionId,
+        model: buildModel({
+          jobs: {build: {steps: [{run: 'echo first'}, {run: 'echo second'}]}},
+        }),
+        triggerPayload: {
+          source: 'manual',
+          event: 'fire',
+          subscriptionId: crypto.randomUUID(),
+          userId: crypto.randomUUID(),
+        },
+      });
+      const [job] = await getJobsByWorkflowRunId(run.id);
+      if (!job) throw new Error('Expected a workflow job');
+      await stripSetupStep(job.id);
+      const execution = await getFirstJobExecutionByJobId(job.id);
+      if (!execution) throw new Error('Expected a job execution');
+
+      const steps = await getStepsByJobId(job.id);
+      const firstStep = steps[0];
+      const conditionErroredStep = steps[1];
+      if (!firstStep || !conditionErroredStep) throw new Error('Expected two workflow steps');
+      await db()
+        .update(stepsTable)
+        .set({status: 'skipped', statusReason: 'condition_errored'})
+        .where(eq(stepsTable.id, conditionErroredStep.id));
+
+      await expect(getJobExecutionFailureOrigin(execution.id)).resolves.toMatchObject({
+        stepId: conditionErroredStep.id,
+        stepStatus: 'skipped',
+      });
+      expect(firstStep.position).toBeLessThan(conditionErroredStep.position);
+    });
+  });
+
   describe('bulkUpdateStepStatuses', () => {
     test('updates all steps for a job to the given sweep status', async () => {
       const run = await createWorkflowRun({
@@ -183,6 +330,7 @@ describe('workflow run queries', () => {
           projectId,
           stepId: attempt?.stepId,
           attempt: 1,
+          status: 'cancelled',
           logOutcome: 'abandoned',
         },
       ]);