@shipfox/api-workflows 2.0.0

package/src/core/workflow-run-creation.ts

@@ -0,0 +1,131 @@
+import {
+  type AgentDefaultsResolver,
+  catalogDefaultAgentResolver,
+} from '@shipfox/api-agent/core/resolve-agent-config';
+import type {WorkflowModel} from '@shipfox/api-definitions';
+import type {
+  AgentToolMaterializationContext,
+  AgentToolMaterializationSnapshot,
+} from './agent-tools.js';
+import type {JobExecution} from './entities/job-execution.js';
+import type {TriggerPayload, WorkflowRun} from './entities/workflow-run.js';
+import {
+  assembleCreationContext,
+  assembleExecutionCreationContext,
+} from './step-config/assemble-run-context.js';
+import {
+  type MaterializedWorkflowJob,
+  materializeJobRunner,
+  materializeWorkflowModel,
+} from './step-config/materialize-workflow-model.js';
+import {resolveJobExecutionName} from './step-config/resolve-job-execution-name.js';
+
+export function materializeWorkflowRunJobs(params: {
+  run: WorkflowRun;
+  model: WorkflowModel;
+  triggerPayload: TriggerPayload;
+  inputs?: Record<string, unknown> | null | undefined;
+  vars?: Record<string, string> | undefined;
+  resolveAgentDefaults?: AgentDefaultsResolver | undefined;
+  definitionId: string;
+  agentToolContext?: AgentToolMaterializationContext | undefined;
+  agentToolSnapshot?: AgentToolMaterializationSnapshot | null | undefined;
+}): readonly MaterializedWorkflowJob[] {
+  const context = assembleCreationContext({
+    run: params.run,
+    triggerPayload: params.triggerPayload,
+    inputs: params.inputs ?? null,
+    vars: params.vars,
+  });
+  return materializeWorkflowModel({
+    model: params.model,
+    context,
+    resolveAgentDefaults: params.resolveAgentDefaults ?? catalogDefaultAgentResolver,
+    definitionId: params.definitionId,
+    agentToolContext: params.agentToolContext,
+    agentToolSnapshot: params.agentToolSnapshot,
+  });
+}
+
+export function deriveInitialJobExecutionPlan(params: {
+  run: WorkflowRun;
+  modelJob: WorkflowModel['jobs'][number];
+  job: MaterializedWorkflowJob;
+  jobId: string;
+  sequence: number;
+  fallbackName: string;
+  triggerPayload: TriggerPayload;
+  inputs?: Record<string, unknown> | null | undefined;
+  vars?: Record<string, string> | undefined;
+}): {
+  name: string;
+  runner: readonly string[];
+  evaluationTrace: JobExecution['evaluationTrace'];
+} {
+  const nameContext = assembleExecutionCreationContext({
+    run: params.run,
+    triggerPayload: params.triggerPayload,
+    inputs: params.inputs ?? null,
+    vars: params.vars,
+    jobId: params.jobId,
+    sequence: params.sequence,
+    executionName: params.fallbackName,
+    status: 'pending',
+    triggerEvents: [],
+    priorExecutions: [],
+  });
+  const resolvedName = resolveJobExecutionName({
+    definitionId: params.run.definitionId,
+    job: params.job,
+    fallbackName: params.fallbackName,
+    context: nameContext.values,
+  });
+  const runnerContext = assembleExecutionCreationContext({
+    run: params.run,
+    triggerPayload: params.triggerPayload,
+    inputs: params.inputs ?? null,
+    vars: params.vars,
+    jobId: params.jobId,
+    sequence: params.sequence,
+    executionName: resolvedName.value,
+    status: 'pending',
+    triggerEvents: [],
+    priorExecutions: [],
+  });
+  return {
+    name: resolvedName.value,
+    runner: materializeJobRunner({
+      job: params.modelJob,
+      context: runnerContext,
+      definitionId: params.run.definitionId,
+    }),
+    evaluationTrace: resolvedName.trace,
+  };
+}
+
+export function deriveJobExecutionRunner(params: {
+  run: WorkflowRun;
+  modelJob: WorkflowModel['jobs'][number];
+  jobId: string;
+  sequence: number;
+  executionName: string;
+  status: JobExecution['status'];
+  triggerEvents?: readonly JobExecution['triggerEvents'][number][] | undefined;
+  priorExecutions?: readonly JobExecution[] | undefined;
+}): readonly string[] {
+  return materializeJobRunner({
+    job: params.modelJob,
+    context: assembleExecutionCreationContext({
+      run: params.run,
+      triggerPayload: params.run.triggerPayload,
+      inputs: params.run.inputs,
+      jobId: params.jobId,
+      sequence: params.sequence,
+      executionName: params.executionName,
+      status: params.status,
+      triggerEvents: params.triggerEvents ?? [],
+      priorExecutions: params.priorExecutions ?? [],
+    }),
+    definitionId: params.run.definitionId,
+  });
+}

package/src/core/workflow-scheduling/index.ts

@@ -0,0 +1,15 @@
+export {
+  createRuntimeRunProgress,
+  nonCompletedRuntimeJobIds,
+  type RuntimeRunProgress,
+  recordRuntimeJobResult,
+  recordSkippedRuntimeJob,
+  runtimeJobVersion,
+  shouldContinueStartedRun,
+} from './run-progress.js';
+export type {RuntimeCompletionStatus, RuntimeDagNode} from './runtime-dag.js';
+export type {RuntimeSchedulingCommand} from './runtime-scheduling-command.js';
+export {
+  type ScheduleRuntimeDagInput,
+  scheduleRuntimeDag,
+} from './schedule-runtime-dag.js';

package/src/core/workflow-scheduling/run-progress.test.ts

@@ -0,0 +1,46 @@
+import {createRuntimeRunProgress, recordSkippedRuntimeJob} from './run-progress.js';
+import type {RuntimeDagNode} from './runtime-dag.js';
+
+function job(
+  key: string,
+  mode: RuntimeDagNode['mode'],
+  status: string | undefined,
+): RuntimeDagNode & {status?: string | undefined} {
+  return {
+    id: `job-${key}`,
+    key,
+    mode,
+    dependencies: [],
+    version: 1,
+    ...(status === undefined ? {} : {status}),
+  };
+}
+
+describe('createRuntimeRunProgress', () => {
+  it('keeps resolved listening jobs completed so dependents can run', () => {
+    const progress = createRuntimeRunProgress([job('listen', 'listening', 'succeeded')]);
+
+    expect(progress.completed).toEqual(new Map([['listen', 'succeeded']]));
+  });
+
+  it.each([
+    'succeeded',
+    'failed',
+    'cancelled',
+    'skipped',
+  ] as const)('records carried terminal job status "%s" honestly', (status) => {
+    const progress = createRuntimeRunProgress([job('build', 'one_shot', status)]);
+
+    expect(progress.completed).toEqual(new Map([['build', status]]));
+  });
+
+  it('records skipped jobs as skipped', () => {
+    const skipped = job('deploy', 'one_shot', 'pending');
+    const progress = createRuntimeRunProgress([skipped]);
+
+    recordSkippedRuntimeJob(skipped, progress, 2);
+
+    expect(progress.completed).toEqual(new Map([['deploy', 'skipped']]));
+    expect(progress.jobVersions.get(skipped.id)).toBe(2);
+  });
+});

package/src/core/workflow-scheduling/run-progress.ts

@@ -0,0 +1,68 @@
+import type {RuntimeCompletionStatus, RuntimeDagNode} from './runtime-dag.js';
+
+type RuntimeProgressJob = RuntimeDagNode & {status?: string | undefined};
+const RUNTIME_COMPLETION_STATUSES = new Set<string>([
+  'succeeded',
+  'failed',
+  'cancelled',
+  'skipped',
+]);
+
+export interface RuntimeRunProgress {
+  completed: Map<string, RuntimeCompletionStatus>;
+  jobVersions: Map<string, number>;
+}
+
+export interface RuntimeJobResult {
+  status: RuntimeCompletionStatus;
+  jobVersion: number;
+}
+
+export function createRuntimeRunProgress(jobs: readonly RuntimeProgressJob[]): RuntimeRunProgress {
+  const completed = new Map<string, RuntimeCompletionStatus>();
+  const jobVersions = new Map<string, number>();
+
+  for (const job of jobs) {
+    jobVersions.set(job.id, job.version);
+    if (isRuntimeCompletionStatus(job.status)) completed.set(job.key, job.status);
+  }
+
+  return {completed, jobVersions};
+}
+
+export function runtimeJobVersion(job: RuntimeProgressJob, progress: RuntimeRunProgress): number {
+  return progress.jobVersions.get(job.id) ?? job.version;
+}
+
+export function recordSkippedRuntimeJob(
+  job: RuntimeProgressJob,
+  progress: RuntimeRunProgress,
+  newVersion: number,
+): void {
+  progress.jobVersions.set(job.id, newVersion);
+  progress.completed.set(job.key, 'skipped');
+}
+
+export function recordRuntimeJobResult(
+  job: RuntimeProgressJob,
+  progress: RuntimeRunProgress,
+  result: RuntimeJobResult,
+): void {
+  progress.completed.set(job.key, result.status);
+  progress.jobVersions.set(job.id, result.jobVersion);
+}
+
+export function nonCompletedRuntimeJobIds(
+  jobs: readonly RuntimeProgressJob[],
+  progress: RuntimeRunProgress,
+): string[] {
+  return jobs.filter((job) => !progress.completed.has(job.key)).map((job) => job.id);
+}
+
+export function shouldContinueStartedRun(status: string | undefined): boolean {
+  return status === undefined || status === 'pending' || status === 'running';
+}
+
+function isRuntimeCompletionStatus(status: string | undefined): status is RuntimeCompletionStatus {
+  return status !== undefined && RUNTIME_COMPLETION_STATUSES.has(status);
+}

package/src/core/workflow-scheduling/runtime-dag.ts

@@ -0,0 +1,10 @@
+export type RuntimeCompletionStatus = 'succeeded' | 'failed' | 'cancelled' | 'skipped';
+
+export interface RuntimeDagNode {
+  readonly id: string;
+  readonly key: string;
+  readonly mode: 'one_shot' | 'listening';
+  readonly dependencies: readonly string[];
+  readonly hasActivationCondition?: boolean | undefined;
+  readonly version: number;
+}

package/src/core/workflow-scheduling/runtime-scheduling-command.ts

@@ -0,0 +1,20 @@
+import type {RuntimeDagNode} from './runtime-dag.js';
+
+export type RuntimeSchedulingCommand<Node extends RuntimeDagNode = RuntimeDagNode> =
+  | {
+      readonly kind: 'start-job';
+      readonly job: Node;
+    }
+  | {
+      readonly kind: 'evaluate-job-activation';
+      readonly jobs: readonly Node[];
+    }
+  | {
+      readonly kind: 'skip-job';
+      readonly job: Node;
+      readonly statusReason: 'default_gate_rejected';
+    }
+  | {
+      readonly kind: 'complete-run';
+      readonly status: 'succeeded' | 'failed';
+    };

package/src/core/workflow-scheduling/schedule-runtime-dag.test.ts

@@ -0,0 +1,197 @@
+import type {RuntimeCompletionStatus, RuntimeDagNode} from './runtime-dag.js';
+import {scheduleRuntimeDag} from './schedule-runtime-dag.js';
+
+function job(
+  key: string,
+  dependencies: readonly string[] = [],
+  mode: RuntimeDagNode['mode'] = 'one_shot',
+  hasActivationCondition = false,
+): RuntimeDagNode {
+  return {
+    id: `job-${key}`,
+    key,
+    mode,
+    dependencies,
+    hasActivationCondition,
+    version: 1,
+  };
+}
+
+function completed(
+  entries: Readonly<Record<string, RuntimeCompletionStatus>>,
+): Map<string, RuntimeCompletionStatus> {
+  return new Map(Object.entries(entries));
+}
+
+function commandKinds(jobs: readonly RuntimeDagNode[], entries = completed({})): readonly string[] {
+  return scheduleRuntimeDag({jobs, completed: entries}).map((command) => command.kind);
+}
+
+describe('scheduleRuntimeDag', () => {
+  it('completes an empty run as succeeded', () => {
+    const commands = scheduleRuntimeDag({jobs: [], completed: completed({})});
+
+    expect(commands).toEqual([{kind: 'complete-run', status: 'succeeded'}]);
+  });
+
+  it('starts root jobs when nothing has completed', () => {
+    const commands = scheduleRuntimeDag({
+      jobs: [job('build'), job('test', ['build'])],
+      completed: completed({}),
+    });
+
+    expect(commands).toEqual([{kind: 'start-job', job: job('build')}]);
+  });
+
+  it('starts all ready jobs in the same transition', () => {
+    const jobs = [job('lint'), job('test'), job('deploy', ['lint', 'test'])];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({})});
+
+    expect(commands).toEqual([
+      {kind: 'start-job', job: jobs[0]},
+      {kind: 'start-job', job: jobs[1]},
+    ]);
+  });
+
+  it('starts a dependent job after all dependencies succeeded', () => {
+    const jobs = [job('build'), job('test', ['build'])];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({build: 'succeeded'})});
+
+    expect(commands).toEqual([{kind: 'start-job', job: jobs[1]}]);
+  });
+
+  it('skips jobs blocked by a failed dependency', () => {
+    const jobs = [job('build'), job('test', ['build'])];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({build: 'failed'})});
+
+    expect(commands).toEqual([
+      {kind: 'skip-job', job: jobs[1], statusReason: 'default_gate_rejected'},
+      {kind: 'complete-run', status: 'failed'},
+    ]);
+  });
+
+  it.each([
+    'failed',
+    'cancelled',
+    'skipped',
+  ] as const)('skips a no-if dependent when a dependency completed as %s', (status) => {
+    const jobs = [job('build'), job('test', ['build'])];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({build: status})});
+
+    expect(commands).toEqual([
+      {kind: 'skip-job', job: jobs[1], statusReason: 'default_gate_rejected'},
+      {kind: 'complete-run', status: status === 'failed' ? 'failed' : 'succeeded'},
+    ]);
+  });
+
+  it('skips remaining jobs when no ready node exists', () => {
+    const jobs = [job('a', ['missing'])];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({})});
+
+    expect(commands).toEqual([
+      {kind: 'skip-job', job: jobs[0], statusReason: 'default_gate_rejected'},
+      {kind: 'complete-run', status: 'succeeded'},
+    ]);
+  });
+
+  it('completes a run as failed when a completed job failed', () => {
+    const commands = scheduleRuntimeDag({
+      jobs: [job('build')],
+      completed: completed({build: 'failed'}),
+    });
+
+    expect(commands).toEqual([{kind: 'complete-run', status: 'failed'}]);
+  });
+
+  it('completes a run as succeeded when all jobs succeeded', () => {
+    const commands = scheduleRuntimeDag({
+      jobs: [job('build'), job('test', ['build'])],
+      completed: completed({build: 'succeeded', test: 'succeeded'}),
+    });
+
+    expect(commands).toEqual([{kind: 'complete-run', status: 'succeeded'}]);
+  });
+
+  it('completes a run as succeeded when all jobs skipped', () => {
+    const commands = scheduleRuntimeDag({
+      jobs: [job('build'), job('test', ['build'])],
+      completed: completed({build: 'skipped', test: 'skipped'}),
+    });
+
+    expect(commands).toEqual([{kind: 'complete-run', status: 'succeeded'}]);
+  });
+
+  it('does not start already-completed jobs', () => {
+    const kinds = commandKinds([job('build')], completed({build: 'succeeded'}));
+
+    expect(kinds).toEqual(['complete-run']);
+  });
+
+  it('does not restart jobs that are already running', () => {
+    const jobs = [job('build'), job('test', ['build'])];
+
+    const commands = scheduleRuntimeDag({
+      jobs,
+      completed: completed({}),
+      running: new Set(['build']),
+    });
+
+    expect(commands).toEqual([]);
+  });
+
+  it('starts listening jobs when they are ready', () => {
+    const jobs = [job('listen', [], 'listening')];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({})});
+
+    expect(commands).toEqual([{kind: 'start-job', job: jobs[0]}]);
+  });
+
+  it('starts one-shot siblings alongside listening jobs', () => {
+    const jobs = [job('listen', [], 'listening'), job('build')];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({})});
+
+    expect(commands).toEqual([
+      {kind: 'start-job', job: jobs[0]},
+      {kind: 'start-job', job: jobs[1]},
+    ]);
+  });
+
+  it('starts an unresolved listening dependency before its dependent jobs', () => {
+    const jobs = [job('listen', [], 'listening'), job('deploy', ['listen'])];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({})});
+
+    expect(commands).toEqual([{kind: 'start-job', job: jobs[0]}]);
+  });
+
+  it('starts jobs that need a resolved listening dependency', () => {
+    const jobs = [job('listen', [], 'listening'), job('deploy', ['listen'])];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({listen: 'succeeded'})});
+
+    expect(commands).toEqual([{kind: 'start-job', job: jobs[1]}]);
+  });
+
+  it('evaluates an explicit-if job after all dependencies are terminal', () => {
+    const jobs = [job('build'), job('notify', ['build'], 'one_shot', true)];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({build: 'failed'})});
+
+    expect(commands).toEqual([{kind: 'evaluate-job-activation', jobs: [jobs[1]]}]);
+  });
+
+  it('evaluates an explicit-if root job before starting it', () => {
+    const jobs = [job('notify', [], 'one_shot', true)];
+
+    const commands = scheduleRuntimeDag({jobs, completed: completed({})});
+
+    expect(commands).toEqual([{kind: 'evaluate-job-activation', jobs}]);
+  });
+});

package/src/core/workflow-scheduling/schedule-runtime-dag.ts

@@ -0,0 +1,115 @@
+import type {RuntimeCompletionStatus, RuntimeDagNode} from './runtime-dag.js';
+import type {RuntimeSchedulingCommand} from './runtime-scheduling-command.js';
+
+export interface ScheduleRuntimeDagInput<Job extends RuntimeDagNode = RuntimeDagNode> {
+  readonly jobs: readonly Job[];
+  readonly completed: ReadonlyMap<string, RuntimeCompletionStatus>;
+  readonly running?: ReadonlySet<string> | undefined;
+}
+
+export function scheduleRuntimeDag<Job extends RuntimeDagNode>(
+  input: ScheduleRuntimeDagInput<Job>,
+): readonly RuntimeSchedulingCommand<Job>[] {
+  const completed = new Map(input.completed);
+  const commands: RuntimeSchedulingCommand<Job>[] = [];
+  const running = input.running ?? new Set();
+
+  let skipped: readonly Job[];
+  do {
+    skipped = findDefaultSkippedJobs(input.jobs, completed, running);
+    for (const job of skipped) {
+      commands.push({kind: 'skip-job', job, statusReason: 'default_gate_rejected'});
+      completed.set(job.key, 'skipped');
+    }
+  } while (skipped.length > 0);
+
+  const activationCandidates = findActivationCandidates(input.jobs, completed, running);
+  if (activationCandidates.length > 0) {
+    commands.push({kind: 'evaluate-job-activation', jobs: activationCandidates});
+  }
+
+  const ready = findReadyJobs(input.jobs, completed, running);
+  if (ready.length > 0) {
+    commands.push(...ready.map((job): RuntimeSchedulingCommand<Job> => ({kind: 'start-job', job})));
+    return commands;
+  }
+  if (activationCandidates.length > 0) return commands;
+
+  if (input.jobs.some((job) => running.has(job.key) && !completed.has(job.key))) {
+    return commands;
+  }
+
+  const remaining = input.jobs.filter((job) => !completed.has(job.key));
+
+  if (remaining.length > 0) {
+    commands.push(
+      ...remaining.map(
+        (job): RuntimeSchedulingCommand<Job> => ({
+          kind: 'skip-job',
+          job,
+          statusReason: 'default_gate_rejected',
+        }),
+      ),
+    );
+    for (const job of remaining) completed.set(job.key, 'skipped');
+    commands.push({kind: 'complete-run', status: hasFailure(completed) ? 'failed' : 'succeeded'});
+    return commands;
+  }
+
+  commands.push({
+    kind: 'complete-run',
+    status: hasFailure(completed) ? 'failed' : 'succeeded',
+  });
+  return commands;
+}
+
+function findReadyJobs<Job extends RuntimeDagNode>(
+  jobs: readonly Job[],
+  completed: ReadonlyMap<string, RuntimeCompletionStatus>,
+  running: ReadonlySet<string>,
+): readonly Job[] {
+  return jobs.filter(
+    (job) =>
+      !completed.has(job.key) &&
+      !running.has(job.key) &&
+      !hasActivationCondition(job) &&
+      job.dependencies.every((dependency) => completed.get(dependency) === 'succeeded'),
+  );
+}
+
+function findDefaultSkippedJobs<Job extends RuntimeDagNode>(
+  jobs: readonly Job[],
+  completed: ReadonlyMap<string, RuntimeCompletionStatus>,
+  running: ReadonlySet<string>,
+): readonly Job[] {
+  return jobs.filter(
+    (job) =>
+      !completed.has(job.key) &&
+      !running.has(job.key) &&
+      !hasActivationCondition(job) &&
+      job.dependencies.every((dependency) => completed.has(dependency)) &&
+      job.dependencies.some((dependency) => completed.get(dependency) !== 'succeeded'),
+  );
+}
+
+function findActivationCandidates<Job extends RuntimeDagNode>(
+  jobs: readonly Job[],
+  completed: ReadonlyMap<string, RuntimeCompletionStatus>,
+  running: ReadonlySet<string>,
+): readonly Job[] {
+  return jobs.filter(
+    (job) =>
+      !completed.has(job.key) &&
+      !running.has(job.key) &&
+      hasActivationCondition(job) &&
+      job.dependencies.every((dependency) => completed.has(dependency)),
+  );
+}
+
+function hasFailure(completed: ReadonlyMap<string, RuntimeCompletionStatus>): boolean {
+  return Array.from(completed.values()).some((status) => status === 'failed');
+}
+
+function hasActivationCondition(job: RuntimeDagNode): boolean {
+  return job.hasActivationCondition === true;
+}

package/src/db/db.ts

@@ -0,0 +1,40 @@
+import {drizzle, type NodePgDatabase} from '@shipfox/node-drizzle';
+import {pgClient} from '@shipfox/node-postgres';
+import {jobExecutions} from './schema/job-executions.js';
+import {jobListenerEvents} from './schema/job-listener-events.js';
+import {jobs} from './schema/jobs.js';
+import {workflowsOutbox} from './schema/outbox.js';
+import {stepAttempts} from './schema/step-attempts.js';
+import {steps} from './schema/steps.js';
+import {workflowRunAttempts} from './schema/workflow-run-attempts.js';
+import {workflowRuns} from './schema/workflow-runs.js';
+
+export const schema = {
+  workflowRuns,
+  workflowRunAttempts,
+  jobs,
+  jobExecutions,
+  jobListenerEvents,
+  steps,
+  stepAttempts,
+  workflowsOutbox,
+};
+
+let _db: NodePgDatabase<typeof schema> | undefined;
+
+export function db() {
+  if (!_db) _db = drizzle(pgClient(), {schema});
+  return _db;
+}
+
+export function closeDb(): void {
+  _db = undefined;
+}
+
+export type Tx = Parameters<Parameters<NodePgDatabase<typeof schema>['transaction']>[0]>[0];
+
+// Lets callers express a unit of work to run atomically without depending on the
+// db() singleton or drizzle's transaction API directly.
+export function withTransaction<T>(fn: (tx: Tx) => Promise<T>): Promise<T> {
+  return db().transaction(fn);
+}