@shipfox/api-workflows 2.0.0

package/test/factories/workflow-model.ts

@@ -0,0 +1,268 @@
+import {DEFAULT_JOB_CHECKOUT, type WorkflowModel} from '@shipfox/api-definitions';
+import {
+  createWorkflowExpression,
+  parseWorkflowTemplate,
+  planInterpolationField,
+  type ResolvedFieldSegment,
+  type WorkflowInterpolationField,
+} from '@shipfox/expression';
+
+type ModelStep = WorkflowModel['jobs'][number]['steps'][number];
+type AgentThinking = Extract<ModelStep, {kind: 'agent'}>['thinking'];
+type Harness = Extract<ModelStep, {kind: 'agent'}>['harness'];
+type WorkflowEnvTemplates = NonNullable<NonNullable<WorkflowModel['templates']>['env']>;
+
+interface TestWorkflowStepBase {
+  readonly key?: string | undefined;
+  readonly name?: string | undefined;
+  readonly sourceLocation?: WorkflowModel['jobs'][number]['steps'][number]['sourceLocation'];
+  readonly if?: ModelStep['if'] | undefined;
+  readonly gate?: WorkflowModel['jobs'][number]['steps'][number]['gate'] | undefined;
+}
+
+interface TestRunStep extends TestWorkflowStepBase {
+  readonly run: string;
+  readonly env?: WorkflowModel['env'] | undefined;
+}
+
+interface TestAgentStep extends TestWorkflowStepBase {
+  readonly harness?: Harness | undefined;
+  readonly model?: string | undefined;
+  readonly provider?: string | undefined;
+  readonly prompt: string;
+  readonly thinking?: AgentThinking | undefined;
+  readonly tools?: readonly string[] | undefined;
+  readonly integrations?: Extract<ModelStep, {kind: 'agent'}>['integrations'] | undefined;
+}
+
+type TestWorkflowStep = TestRunStep | TestAgentStep;
+
+const DEFAULT_RUNNER_LABELS = ['ubuntu-latest'] as const;
+
+interface TestWorkflowJob {
+  readonly needs?: string | readonly string[] | undefined;
+  readonly name?: string | undefined;
+  readonly runner?: string | readonly string[] | undefined;
+  readonly runnerTemplates?: readonly string[] | undefined;
+  readonly checkout?: WorkflowModel['jobs'][number]['checkout'] | undefined;
+  readonly if?: string | undefined;
+  readonly success?: string | undefined;
+  readonly outputs?: Readonly<Record<string, string>> | undefined;
+  readonly env?: WorkflowModel['env'] | undefined;
+  readonly steps: readonly TestWorkflowStep[];
+}
+
+interface TestWorkflowModelInput {
+  readonly name?: string | undefined;
+  readonly runner?: string | readonly string[] | undefined;
+  readonly env?: WorkflowModel['env'] | undefined;
+  readonly jobs?: Readonly<Record<string, TestWorkflowJob>> | undefined;
+}
+
+export function workflowModel(input: TestWorkflowModelInput = {}): WorkflowModel {
+  const jobs = input.jobs ?? {
+    build: {
+      steps: [{run: 'echo hello'}],
+    },
+  };
+  const modelJobs = Object.entries(jobs).map(([key, job]) => {
+    const jobId = stableId(key);
+    return {
+      id: jobId,
+      key,
+      mode: 'one_shot' as const,
+      runner: normalizeStringArray(job.runner ?? input.runner ?? DEFAULT_RUNNER_LABELS),
+      ...(job.runnerTemplates === undefined
+        ? {}
+        : {
+            runnerTemplates: job.runnerTemplates.map((template) =>
+              requiredFieldTemplate('job.runner', template),
+            ),
+          }),
+      checkout: job.checkout ?? DEFAULT_JOB_CHECKOUT,
+      ...(job.if === undefined ? {} : {if: workflowExpression(job.if)}),
+      ...(job.success === undefined ? {} : {success: job.success}),
+      ...(job.outputs === undefined ? {} : {outputs: outputTemplates(job.outputs)}),
+      ...(job.name === undefined
+        ? {}
+        : {
+            name: fieldTemplate('job.name', job.name) ?? [
+              {kind: 'literal' as const, value: job.name},
+            ],
+          }),
+      ...optionalScopedEnv(job.env),
+      dependencies: normalizeStringArray(job.needs).map(stableId),
+      steps: job.steps.map((step, stepIndex) => normalizeStep(step, jobId, stepIndex)),
+    };
+  });
+
+  return {
+    kind: 'workflow',
+    name: input.name ?? 'Test Workflow',
+    ...optionalScopedEnv(input.env),
+    triggers: [],
+    jobs: modelJobs,
+    dependencies: modelJobs.flatMap((job) =>
+      job.dependencies.map((dependency) => ({from: dependency, to: job.id})),
+    ),
+  };
+}
+
+function workflowExpression(source: string) {
+  return createWorkflowExpression({
+    source,
+    check: {mode: 'syntax'},
+  });
+}
+
+function outputTemplates(outputs: Readonly<Record<string, string>>) {
+  return Object.fromEntries(
+    Object.entries(outputs).map(([key, source]) => [
+      key,
+      fieldTemplate('job.outputs', source) ?? [{kind: 'literal' as const, value: source}],
+    ]),
+  );
+}
+
+function normalizeStep(step: TestWorkflowStep, jobId: string, stepIndex: number): ModelStep {
+  const base = stepBase(step, jobId, stepIndex);
+  return 'run' in step
+    ? {
+        ...base,
+        kind: 'run',
+        command: {kind: 'shell', value: step.run},
+        ...optionalRunTemplates(step),
+        ...optionalStepEnv(step.env),
+      }
+    : {
+        ...base,
+        kind: 'agent',
+        ...(step.harness === undefined ? {} : {harness: step.harness}),
+        ...(step.model === undefined ? {} : {model: step.model}),
+        ...(step.provider === undefined ? {} : {provider: step.provider}),
+        ...(step.thinking === undefined ? {} : {thinking: step.thinking}),
+        ...(step.tools === undefined ? {} : {tools: step.tools}),
+        ...(step.integrations === undefined ? {} : {integrations: step.integrations}),
+        prompt: step.prompt,
+        ...optionalAgentTemplates(step),
+      };
+}
+
+function stepBase(step: TestWorkflowStep, jobId: string, stepIndex: number) {
+  return {
+    id:
+      step.key === undefined ? `${jobId}-step-${stepIndex + 1}` : `${jobId}-${stableId(step.key)}`,
+    ...(step.key === undefined ? {} : {key: step.key}),
+    ...(step.name === undefined ? {} : {name: step.name}),
+    ...(step.sourceLocation === undefined ? {} : {sourceLocation: step.sourceLocation}),
+    ...(step.if === undefined ? {} : {if: step.if}),
+    ...(step.gate === undefined ? {} : {gate: step.gate}),
+  };
+}
+
+function normalizeStringArray(value: string | readonly string[] | undefined): readonly string[] {
+  if (value === undefined) return [];
+  return typeof value === 'string' ? [value] : value;
+}
+
+function optionalScopedEnv(
+  env: WorkflowModel['env'] | undefined,
+):
+  | {env: NonNullable<WorkflowModel['env']>; templates: {env: WorkflowEnvTemplates}}
+  | {env: NonNullable<WorkflowModel['env']>}
+  | Record<string, never> {
+  if (env === undefined || Object.keys(env).length === 0) return {};
+  const templates = envTemplates(env);
+  return templates === undefined ? {env} : {env, templates: {env: templates}};
+}
+
+function optionalStepEnv(
+  env: WorkflowModel['env'] | undefined,
+): {env: NonNullable<WorkflowModel['env']>} | Record<string, never> {
+  if (env === undefined || Object.keys(env).length === 0) return {};
+  return {env};
+}
+
+function optionalRunTemplates(step: TestRunStep) {
+  const command = fieldTemplate('run', step.run);
+  const name = step.name === undefined ? undefined : fieldTemplate('step.name', step.name);
+  const env = envTemplates(step.env);
+  if (command === undefined && name === undefined && env === undefined) return {};
+  return {
+    templates: {
+      ...(command === undefined ? {} : {command}),
+      ...(name === undefined ? {} : {name}),
+      ...(env === undefined ? {} : {env}),
+    },
+  };
+}
+
+function optionalAgentTemplates(step: TestAgentStep) {
+  const prompt = fieldTemplate('agent.prompt', step.prompt);
+  const model = step.model === undefined ? undefined : fieldTemplate('agent.model', step.model);
+  const provider =
+    step.provider === undefined ? undefined : fieldTemplate('agent.provider', step.provider);
+  const name = step.name === undefined ? undefined : fieldTemplate('step.name', step.name);
+  if (prompt === undefined && model === undefined && provider === undefined && name === undefined) {
+    return {};
+  }
+  return {
+    templates: {
+      ...(prompt === undefined ? {} : {prompt}),
+      ...(model === undefined ? {} : {model}),
+      ...(provider === undefined ? {} : {provider}),
+      ...(name === undefined ? {} : {name}),
+    },
+  };
+}
+
+function envTemplates(env: WorkflowModel['env'] | undefined): WorkflowEnvTemplates | undefined {
+  if (env === undefined) return undefined;
+
+  const templates = Object.fromEntries(
+    Object.entries(env).flatMap(([key, value]) => {
+      const template = fieldTemplate('env.value', value);
+      return template === undefined ? [] : [[key, template]];
+    }),
+  );
+
+  return Object.keys(templates).length === 0 ? undefined : templates;
+}
+
+function fieldTemplate(
+  field: WorkflowInterpolationField,
+  source: string,
+): readonly ResolvedFieldSegment[] | undefined {
+  const segments = parseWorkflowTemplate(source);
+  if (!segments.some((segment) => segment.kind === 'expr')) return undefined;
+  const plan = planInterpolationField({field, segments});
+  if (!plan.ok) {
+    throw new Error(
+      `Invalid test workflow template for ${field}: ${plan.violations
+        .map((violation) => violation.source)
+        .join(', ')}`,
+    );
+  }
+  return plan.plan.field.segments;
+}
+
+function requiredFieldTemplate(
+  field: WorkflowInterpolationField,
+  source: string,
+): readonly ResolvedFieldSegment[] {
+  const template = fieldTemplate(field, source);
+  if (template === undefined) {
+    throw new Error(`Expected test workflow template for ${field}: ${source}`);
+  }
+  return template;
+}
+
+function stableId(sourceName: string): string {
+  const id = sourceName
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+
+  return id.length === 0 ? 'unnamed' : id;
+}

package/test/factories/workflow-run.ts

@@ -0,0 +1,58 @@
+import type {WorkflowModel} from '@shipfox/api-definitions';
+import {Factory} from 'fishery';
+import type {WorkflowRun} from '#core/entities/workflow-run.js';
+import {createWorkflowRun} from '#db/workflow-runs.js';
+import {workflowModel} from './workflow-model.js';
+
+interface WorkflowRunTransientParams {
+  model?: WorkflowModel | undefined;
+}
+
+export const workflowRunFactory = Factory.define<WorkflowRun, WorkflowRunTransientParams>(
+  ({transientParams, onCreate}) => {
+    const workspaceId = crypto.randomUUID();
+    const projectId = crypto.randomUUID();
+    const definitionId = crypto.randomUUID();
+
+    onCreate((run) => {
+      return createWorkflowRun({
+        workspaceId: run.workspaceId,
+        projectId: run.projectId,
+        definitionId: run.definitionId,
+        name: run.name,
+        model: transientParams.model ?? workflowModel({name: run.name}),
+        triggerPayload: run.triggerPayload,
+        inputs: run.inputs ?? undefined,
+        sourceSnapshot: run.sourceSnapshot,
+      });
+    });
+
+    return {
+      id: crypto.randomUUID(),
+      workspaceId,
+      projectId,
+      definitionId,
+      name: 'Test Workflow',
+      status: 'pending',
+      currentAttempt: 1,
+      triggerProvider: null,
+      triggerSource: 'manual',
+      triggerEvent: 'fire',
+      triggerPayload: {
+        source: 'manual',
+        event: 'fire',
+        subscriptionId: crypto.randomUUID(),
+        userId: crypto.randomUUID(),
+      },
+      inputs: null,
+      sourceSnapshot: null,
+      triggerIdempotencyKey: null,
+      timeoutMs: 30 * 24 * 60 * 60 * 1000,
+      version: 1,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+      startedAt: null,
+      finishedAt: null,
+    };
+  },
+);

package/test/fixtures/active-lease-token.ts

@@ -0,0 +1,85 @@
+import {db} from '#db/db.js';
+import {runnerSessions, runningJobExecutions} from '#db/runner-lease-table.js';
+import {
+  getFirstJobExecutionByJobId,
+  getJobById,
+  getWorkflowRunByAttemptId,
+} from '#db/workflow-runs.js';
+import {mintLeaseToken} from './lease-token.js';
+
+export interface MintActiveLeaseTokenParams {
+  jobId: string;
+  token?: {
+    workflowRunId?: string;
+    workflowRunAttemptId?: string;
+    projectId?: string;
+    workspaceId?: string;
+  };
+}
+
+export async function mintActiveLeaseToken(params: MintActiveLeaseTokenParams): Promise<string> {
+  const jobExecution = await getFirstJobExecutionByJobId(params.jobId);
+  if (!jobExecution) throw new Error('Expected job execution to exist');
+  const job = await getJobById(params.jobId);
+  if (!job) throw new Error('Expected job to exist');
+  const run = await getWorkflowRunByAttemptId(job.workflowRunAttemptId);
+  if (!run) throw new Error('Expected workflow run to exist');
+  const runnerSessionId = crypto.randomUUID();
+
+  await insertRunningJobLease({
+    workspaceId: run.workspaceId,
+    workflowRunId: run.id,
+    workflowRunAttemptId: job.workflowRunAttemptId,
+    jobId: params.jobId,
+    jobExecutionId: jobExecution.id,
+    projectId: run.projectId,
+    runnerSessionId,
+  });
+
+  return await mintLeaseToken({
+    jobId: params.jobId,
+    jobExecutionId: jobExecution.id,
+    workflowRunId: params.token?.workflowRunId ?? run.id,
+    workflowRunAttemptId: params.token?.workflowRunAttemptId ?? job.workflowRunAttemptId,
+    projectId: params.token?.projectId ?? run.projectId,
+    workspaceId: params.token?.workspaceId ?? run.workspaceId,
+    runnerSessionId,
+  });
+}
+
+export interface InsertRunningJobLeaseParams {
+  workspaceId: string;
+  workflowRunId: string;
+  workflowRunAttemptId: string;
+  jobId: string;
+  jobExecutionId: string;
+  projectId: string;
+  runnerSessionId: string;
+}
+
+export async function insertRunningJobLease(params: InsertRunningJobLeaseParams): Promise<void> {
+  await db()
+    .insert(runnerSessions)
+    .values({
+      id: params.runnerSessionId,
+      workspaceId: params.workspaceId,
+      scope: 'workspace',
+      registrationTokenId: crypto.randomUUID(),
+      registrationTokenKind: 'manual',
+      labels: ['linux'],
+    });
+
+  await db()
+    .insert(runningJobExecutions)
+    .values({
+      workspaceId: params.workspaceId,
+      workflowRunId: params.workflowRunId,
+      workflowRunAttemptId: params.workflowRunAttemptId,
+      jobId: params.jobId,
+      jobExecutionId: params.jobExecutionId,
+      projectId: params.projectId,
+      runnerSessionId: params.runnerSessionId,
+      requiredLabels: ['linux'],
+      runnerLabels: ['linux'],
+    });
+}

package/test/fixtures/job-with-steps.ts

@@ -0,0 +1,42 @@
+import type {Step} from '#core/entities/step.js';
+import {createWorkflowRun, getJobsByWorkflowRunId, getStepsByJobId} from '#db/workflow-runs.js';
+import {stripSetupStep} from '#test/fixtures/strip-setup-step.js';
+import {workflowModel} from '#test/index.js';
+
+// Jobs and steps have no standalone factory — they only exist as children
+// materialized from a WorkflowModel by createWorkflowRun, which also prepends a
+// synthetic "Set up job" step to every job. This fixture exercises step-execution
+// mechanics in isolation, so it strips that step (see stripSetupStep) to keep the
+// arrangement focused. The setup step's own behavior is covered by materialize /
+// createWorkflowRun / runner tests.
+export async function arrangeJobWithSteps(
+  stepCount: number,
+): Promise<{jobId: string; steps: Step[]}> {
+  const model = workflowModel({
+    name: 'Test Workflow',
+    jobs: {
+      build: {steps: Array.from({length: stepCount}, (_, i) => ({run: `echo step-${i}`}))},
+    },
+  });
+
+  const run = await createWorkflowRun({
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    definitionId: crypto.randomUUID(),
+    model,
+    triggerPayload: {
+      source: 'manual',
+      event: 'fire',
+      subscriptionId: crypto.randomUUID(),
+      userId: crypto.randomUUID(),
+    },
+  });
+
+  const jobs = await getJobsByWorkflowRunId(run.id);
+  const jobId = jobs[0]?.id as string;
+
+  await stripSetupStep(jobId);
+
+  const steps = await getStepsByJobId(jobId);
+  return {jobId, steps};
+}

package/test/fixtures/lease-token.ts

@@ -0,0 +1,44 @@
+import {JOB_LEASE_TOKEN_AUDIENCE} from '@shipfox/api-auth-dto';
+import {signHs256} from '@shipfox/node-jwt';
+import {getFirstJobExecutionByJobId} from '#db/workflow-runs.js';
+
+// Matches test/env.ts; the lease-token auth method reads this same value from config.
+const SECRET = process.env.AUTH_JOB_LEASE_TOKEN_SECRET ?? 'test-lease-secret';
+
+export interface MintLeaseTokenParams {
+  jobId: string;
+  jobExecutionId?: string;
+  // Override informational claims to pair a job with chosen surrounding context.
+  workflowRunId?: string;
+  workflowRunAttemptId?: string;
+  projectId?: string;
+  workspaceId?: string;
+  runnerSessionId?: string;
+  secret?: string;
+  expiresIn?: string;
+  audience?: string;
+}
+
+export async function mintLeaseToken(params: MintLeaseTokenParams): Promise<string> {
+  let jobExecutionId = params.jobExecutionId;
+  if (jobExecutionId === undefined) {
+    const jobExecution = await getFirstJobExecutionByJobId(params.jobId);
+    if (!jobExecution) throw new Error('Expected job execution to exist');
+    jobExecutionId = jobExecution.id;
+  }
+
+  return signHs256({
+    payload: {
+      jobId: params.jobId,
+      jobExecutionId,
+      workflowRunId: params.workflowRunId ?? crypto.randomUUID(),
+      workflowRunAttemptId: params.workflowRunAttemptId ?? crypto.randomUUID(),
+      projectId: params.projectId ?? crypto.randomUUID(),
+      workspaceId: params.workspaceId ?? crypto.randomUUID(),
+      runnerSessionId: params.runnerSessionId ?? crypto.randomUUID(),
+    },
+    secret: params.secret ?? SECRET,
+    expiresIn: params.expiresIn ?? '90m',
+    audience: params.audience ?? JOB_LEASE_TOKEN_AUDIENCE,
+  });
+}

package/test/fixtures/strip-setup-step.ts

@@ -0,0 +1,23 @@
+import {and, eq, inArray, sql} from 'drizzle-orm';
+import {db} from '#db/db.js';
+import {steps as stepsTable} from '#db/schema/steps.js';
+import {getStepsByJobId} from '#db/workflow-runs.js';
+
+// createWorkflowRun prepends a synthetic "Set up job" step to every job. Tests that
+// exercise step-execution mechanics in isolation strip it and renumber the user
+// steps back to 0-based, so their arrangement matches the steps they declared.
+export async function stripSetupStep(jobId: string): Promise<void> {
+  const jobSteps = await getStepsByJobId(jobId);
+  const stepIds = jobSteps.map((step) => step.id);
+  if (stepIds.length === 0) return;
+
+  await db().transaction(async (tx) => {
+    await tx
+      .delete(stepsTable)
+      .where(and(inArray(stepsTable.id, stepIds), eq(stepsTable.type, 'setup')));
+    await tx
+      .update(stepsTable)
+      .set({position: sql`${stepsTable.position} - 1`})
+      .where(inArray(stepsTable.id, stepIds));
+  });
+}

package/test/globalSetup.ts

@@ -0,0 +1,54 @@
+import './env.js';
+import {annotationsModule} from '@shipfox/annotations';
+import {agentModule} from '@shipfox/api-agent';
+import {runnersModule} from '@shipfox/api-runners';
+import {secretsModule} from '@shipfox/api-secrets';
+import {runMigrations} from '@shipfox/node-drizzle';
+import {closePostgresClient, createPostgresClient} from '@shipfox/node-postgres';
+import {sql} from 'drizzle-orm';
+import {closeDb, db, migrationsPath} from '#db/index.js';
+
+export async function setup() {
+  createPostgresClient();
+
+  if (!agentModule.database || Array.isArray(agentModule.database)) {
+    throw new Error('Agent module database is not configured');
+  }
+  await runMigrations(
+    agentModule.database.db(),
+    agentModule.database.migrationsPath,
+    '__drizzle_migrations_agent',
+  );
+  if (!runnersModule.database || Array.isArray(runnersModule.database)) {
+    throw new Error('Runners module database is not configured');
+  }
+  await runMigrations(
+    runnersModule.database.db(),
+    runnersModule.database.migrationsPath,
+    '__drizzle_migrations_runners',
+  );
+  if (!secretsModule.database || Array.isArray(secretsModule.database)) {
+    throw new Error('Secrets module database is not configured');
+  }
+  await runMigrations(
+    secretsModule.database.db(),
+    secretsModule.database.migrationsPath,
+    '__drizzle_migrations_secrets',
+  );
+  if (!annotationsModule.database || Array.isArray(annotationsModule.database)) {
+    throw new Error('Annotations module database is not configured');
+  }
+  await runMigrations(
+    annotationsModule.database.db(),
+    annotationsModule.database.migrationsPath,
+    '__drizzle_migrations_annotations',
+  );
+  await runMigrations(db(), migrationsPath, '__drizzle_migrations_workflows');
+  await db().execute(sql`TRUNCATE workflows_workflow_runs CASCADE`);
+  await db().execute(sql`TRUNCATE workflows_job_listener_events CASCADE`);
+  await db().execute(sql`TRUNCATE workflows_outbox CASCADE`);
+  await db().execute(sql`TRUNCATE annotations_annotations CASCADE`);
+
+  closeDb();
+  await closePostgresClient();
+}