@shipfox/api-workflows 2.0.0

package/src/db/workflow-runs/run-create.ts

@@ -0,0 +1,373 @@
+import {
+  type AgentDefaultsResolver,
+  catalogDefaultAgentResolver,
+} from '@shipfox/api-agent/core/resolve-agent-config';
+import type {WorkflowModel} from '@shipfox/api-definitions';
+import {getVariablesByNamespace} from '@shipfox/api-secrets';
+import {analyzeContextKeyAccess, type ResolvedFieldSegment} from '@shipfox/expression';
+import {logger} from '@shipfox/node-opentelemetry';
+import {eq} from 'drizzle-orm';
+import {
+  createAgentToolMaterializationSnapshot,
+  loadAgentToolMaterializationContext,
+} from '#core/agent-tools.js';
+import type {
+  TriggerPayload,
+  WorkflowRun,
+  WorkflowSourceSnapshot,
+} from '#core/entities/workflow-run.js';
+import {InterpolationUnresolvableError} from '#core/errors.js';
+import type {MaterializedWorkflowJob} from '#core/step-config/materialize-workflow-model.js';
+import type {WorkflowStepTemplateDiagnostic} from '#core/step-config/resolve-step-config.js';
+import {
+  deriveInitialJobExecutionPlan,
+  materializeWorkflowRunJobs,
+} from '#core/workflow-run-creation.js';
+import {recordWorkflowRunCreated} from '#metrics/instance.js';
+import {db} from '../db.js';
+import {workflowRunAttempts} from '../schema/workflow-run-attempts.js';
+import {toWorkflowRun, workflowRuns} from '../schema/workflow-runs.js';
+import {type MaterializedRunGraphJob, persistMaterializedRunGraph} from './run-graph.js';
+
+export type WorkflowModelJob = WorkflowModel['jobs'][number];
+
+export interface ReferencedVariable {
+  readonly key: string;
+  readonly field: InterpolationUnresolvableError['field'];
+  readonly source: string;
+  readonly envKey?: string | undefined;
+}
+
+export interface CreateWorkflowRunParams {
+  workspaceId: string;
+  projectId: string;
+  definitionId: string;
+  name?: string | undefined;
+  model: WorkflowModel;
+  triggerPayload: TriggerPayload;
+  inputs?: Record<string, unknown> | undefined;
+  sourceSnapshot?: WorkflowSourceSnapshot | null | undefined;
+  triggerIdempotencyKey?: string | undefined;
+  resolveAgentDefaults?: AgentDefaultsResolver | undefined;
+}
+
+export async function createWorkflowRun(params: CreateWorkflowRunParams): Promise<WorkflowRun> {
+  const agentToolContext = await loadAgentToolMaterializationContext({
+    model: params.model,
+    workspaceId: params.workspaceId,
+    projectId: params.projectId,
+  });
+  const agentToolMaterialization = createAgentToolMaterializationSnapshot({
+    model: params.model,
+    context: agentToolContext,
+  });
+  const result = await db().transaction(async (tx) => {
+    const insertResult = await tx
+      .insert(workflowRuns)
+      .values({
+        workspaceId: params.workspaceId,
+        projectId: params.projectId,
+        definitionId: params.definitionId,
+        name: params.name ?? params.model.name,
+        status: 'pending',
+        currentAttempt: 1,
+        triggerProvider: params.triggerPayload.provider ?? null,
+        triggerSource: params.triggerPayload.source,
+        triggerEvent: params.triggerPayload.event,
+        triggerPayload: params.triggerPayload,
+        inputs: params.inputs ?? null,
+        sourceSnapshot: params.sourceSnapshot ?? null,
+        triggerIdempotencyKey: params.triggerIdempotencyKey ?? null,
+      })
+      .onConflictDoNothing({target: workflowRuns.triggerIdempotencyKey})
+      .returning();
+
+    const runRow = insertResult[0];
+    if (!runRow) {
+      // Conflict path: skip jobs/steps/outbox so the first insert keeps ownership of side effects.
+      if (!params.triggerIdempotencyKey) {
+        throw new Error('Insert returned no rows');
+      }
+      const existing = await tx
+        .select()
+        .from(workflowRuns)
+        .where(eq(workflowRuns.triggerIdempotencyKey, params.triggerIdempotencyKey))
+        .limit(1);
+      const existingRow = existing[0];
+      if (!existingRow) {
+        throw new Error(
+          `Idempotency conflict but existing run missing for key ${params.triggerIdempotencyKey}`,
+        );
+      }
+      return {run: toWorkflowRun(existingRow), created: false};
+    }
+
+    const run = toWorkflowRun(runRow);
+    const [attemptRow] = await tx
+      .insert(workflowRunAttempts)
+      .values({
+        workflowRunId: runRow.id,
+        attempt: 1,
+        status: 'pending',
+        model: params.model,
+        agentToolMaterialization,
+      })
+      .returning();
+    if (!attemptRow) throw new Error('Insert returned no rows');
+
+    // Resolving one-shot templates here gives interpolation access to the inserted run id.
+    // If resolution fails, the transaction rolls back the run, jobs, steps, and outbox event together.
+    // Listening steps are resolved later when a job execution is created.
+    const oneShotJobs = params.model.jobs.filter((job) => job.mode !== 'listening');
+    const vars = await loadReferencedVariables({
+      model: params.model,
+      jobs: oneShotJobs,
+      workspaceId: params.workspaceId,
+      projectId: params.projectId,
+      definitionId: params.definitionId,
+    });
+    const materializedJobs = materializeWorkflowRunJobs({
+      run,
+      model: params.model,
+      triggerPayload: params.triggerPayload,
+      inputs: params.inputs ?? null,
+      vars,
+      resolveAgentDefaults: params.resolveAgentDefaults ?? catalogDefaultAgentResolver,
+      definitionId: params.definitionId,
+      agentToolContext,
+      agentToolSnapshot: agentToolMaterialization,
+    });
+
+    const graphJobs = materializeRunGraphJobs({
+      params,
+      run,
+      vars,
+      materializedJobs,
+    });
+    await persistMaterializedRunGraph(tx, {
+      run,
+      workflowRunAttempt: attemptRow,
+      materializedJobs: graphJobs,
+    });
+
+    logTemplateDiagnostics({
+      workflowRunId: runRow.id,
+      diagnostics: materializedJobs.flatMap((job) =>
+        job.steps.flatMap((step) =>
+          (step.diagnostics ?? []).map((diagnostic) => ({
+            jobKey: job.key,
+            stepName: step.name,
+            ...diagnostic,
+          })),
+        ),
+      ),
+    });
+
+    return {run, created: true};
+  });
+
+  if (result.created)
+    recordWorkflowRunCreated(result.run.triggerPayload.provider ?? result.run.triggerSource);
+
+  return result.run;
+}
+
+export async function loadReferencedVariables(params: {
+  readonly model: WorkflowModel;
+  readonly jobs?: readonly WorkflowModelJob[] | undefined;
+  readonly workspaceId: string;
+  readonly projectId: string;
+  readonly definitionId: string;
+}): Promise<Record<string, string> | undefined> {
+  const references = referencedVariables(params.model, params.jobs ?? params.model.jobs);
+  const keys = [...new Set(references.map((reference) => reference.key))].sort();
+  if (keys.length === 0) return undefined;
+
+  const vars = await getVariablesByNamespace({
+    workspaceId: params.workspaceId,
+    projectId: params.projectId,
+    namespace: '',
+  });
+  const missingKey = keys.find((key) => !(key in vars));
+  if (missingKey !== undefined) {
+    const reference = references.find((candidate) => candidate.key === missingKey);
+    throw new InterpolationUnresolvableError(params.definitionId, {
+      field: reference?.field ?? 'env',
+      source: reference?.source ?? `vars.${missingKey}`,
+      ...(reference?.envKey === undefined ? {} : {envKey: reference.envKey}),
+    });
+  }
+
+  return vars;
+}
+
+function materializeRunGraphJobs(params: {
+  readonly params: CreateWorkflowRunParams;
+  readonly run: WorkflowRun;
+  readonly vars: Record<string, string> | undefined;
+  readonly materializedJobs: readonly MaterializedWorkflowJob[];
+}): readonly MaterializedRunGraphJob[] {
+  return params.materializedJobs.map((job, jobIndex) => ({
+    job: {
+      key: job.key,
+      mode: job.mode,
+      name: workflowTemplateSource(job.name),
+      status: 'pending' as const,
+      checkoutPersistCredentials: job.checkout.persistCredentials,
+      checkoutPermissionsContents: job.checkout.permissions.contents,
+      success: job.success ?? null,
+      executionTimeoutMs: job.executionTimeoutMs ?? null,
+      listeningTimeoutMs: job.listening?.timeoutMs ?? null,
+      maxExecutions: job.listening?.maxExecutions ?? null,
+      onResolve: job.listening?.onResolve ?? null,
+      batchDebounceMs: job.listening?.batch?.debounceMs ?? null,
+      batchMaxSize: job.listening?.batch?.maxSize ?? null,
+      batchMaxWaitMs: job.listening?.batch?.maxWaitMs ?? null,
+      listeningOn: job.listening?.on ? [...job.listening.on] : null,
+      listeningUntil: job.listening?.until ? [...job.listening.until] : null,
+      dependencies: [...job.dependencies],
+      runner: job.runner.length === 0 ? null : [...job.runner],
+      position: job.position,
+    },
+    createExecution: (jobRow) => {
+      if (jobRow.mode === 'listening') return undefined;
+
+      const fallbackName = `${jobRow.key} #1`;
+      const modelJob = params.params.model.jobs[jobIndex];
+      if (!modelJob) return undefined;
+      const executionPlan = deriveInitialJobExecutionPlan({
+        run: params.run,
+        modelJob,
+        job,
+        jobId: jobRow.id,
+        sequence: 1,
+        fallbackName,
+        triggerPayload: params.params.triggerPayload,
+        inputs: params.params.inputs ?? null,
+        vars: params.vars,
+      });
+
+      return {
+        sequence: 1,
+        name: executionPlan.name,
+        runner: [...executionPlan.runner],
+        status: 'pending' as const,
+        evaluationTrace:
+          executionPlan.evaluationTrace?.length === 0 ? null : executionPlan.evaluationTrace,
+      };
+    },
+    createSteps: () =>
+      job.steps.map((step) => ({
+        key: step.key,
+        name: step.name,
+        sourceLocation: step.sourceLocation,
+        status: step.status,
+        type: step.type,
+        config: step.config,
+        condition: step.condition ?? null,
+        configPlan: step.configPlan ?? null,
+        authoredConfig: step.authoredConfig,
+        position: step.position,
+      })),
+  }));
+}
+
+function referencedVariables(
+  model: WorkflowModel,
+  jobs: readonly WorkflowModelJob[],
+): readonly ReferencedVariable[] {
+  const references: ReferencedVariable[] = [];
+
+  if (jobs.length > 0) {
+    collectTemplateVariableReferences(model.templates?.env, references);
+  }
+
+  for (const job of jobs) {
+    collectFieldVariableReferences(job.name, references, {field: 'job.name'});
+    for (const template of job.runnerTemplates ?? []) {
+      collectFieldVariableReferences(template, references, {field: 'job.runner'});
+    }
+    collectTemplateVariableReferences(job.outputs, references, {field: 'job.outputs'});
+    collectTemplateVariableReferences(job.templates?.env, references);
+
+    for (const step of job.steps) {
+      collectFieldVariableReferences(step.templates?.name, references, {field: 'step.name'});
+      if (step.kind === 'run') {
+        collectFieldVariableReferences(step.templates?.command, references, {field: 'run'});
+        collectTemplateVariableReferences(step.templates?.env, references);
+      } else {
+        collectFieldVariableReferences(step.templates?.prompt, references, {field: 'agent.prompt'});
+        collectFieldVariableReferences(step.templates?.model, references, {field: 'agent.model'});
+        collectFieldVariableReferences(step.templates?.provider, references, {
+          field: 'agent.provider',
+        });
+      }
+    }
+  }
+
+  return references;
+}
+
+function collectTemplateVariableReferences(
+  templates: Readonly<Record<string, readonly ResolvedFieldSegment[]>> | undefined,
+  references: ReferencedVariable[],
+  source?: {
+    readonly field: InterpolationUnresolvableError['field'];
+  },
+): void {
+  for (const [envKey, template] of Object.entries(templates ?? {})) {
+    collectFieldVariableReferences(
+      template,
+      references,
+      source === undefined ? {field: 'env', envKey} : source,
+    );
+  }
+}
+
+function collectFieldVariableReferences(
+  template: readonly ResolvedFieldSegment[] | undefined,
+  references: ReferencedVariable[],
+  source: {
+    readonly field: InterpolationUnresolvableError['field'];
+    readonly envKey?: string | undefined;
+  },
+): void {
+  for (const segment of template ?? []) {
+    if (segment.kind === 'literal') continue;
+    const keyAccess = analyzeContextKeyAccess(segment.expression);
+    for (const reference of keyAccess.references) {
+      if (reference.root !== 'vars') continue;
+      references.push({
+        key: reference.key,
+        field: source.field,
+        source: segment.expression.source,
+        envKey: source.envKey,
+      });
+    }
+  }
+}
+
+function workflowTemplateSource(template: MaterializedWorkflowJob['name']): string | null {
+  if (template === undefined) return null;
+
+  return template
+    .map((segment) =>
+      segment.kind === 'literal' ? segment.value : `$${'{{'} ${segment.expression.source} ${'}}'}`,
+    )
+    .join('');
+}
+
+function logTemplateDiagnostics(params: {
+  readonly workflowRunId: string;
+  readonly diagnostics: readonly (WorkflowStepTemplateDiagnostic & {
+    readonly jobKey: string;
+    readonly stepName: string;
+  })[];
+}): void {
+  if (params.diagnostics.length === 0) return;
+
+  logger().warn(
+    {workflowRunId: params.workflowRunId, diagnostics: params.diagnostics},
+    'Workflow interpolation resolved with diagnostics',
+  );
+}

package/src/db/workflow-runs/run-graph.ts

@@ -0,0 +1,95 @@
+import {WORKFLOWS_WORKFLOW_RUN_ATTEMPT_CREATED} from '@shipfox/api-workflows-dto';
+import type {SQL} from 'drizzle-orm';
+import type {WorkflowRun} from '#core/entities/workflow-run.js';
+import type {Tx} from '../db.js';
+import {writeWorkflowsOutboxEvent} from '../outbox-writes.js';
+import {
+  type JobExecutionCreateDb,
+  type JobExecutionDb,
+  jobExecutions,
+} from '../schema/job-executions.js';
+import {type JobCreateDb, type JobDb, jobs} from '../schema/jobs.js';
+import {type StepCreateDb, steps} from '../schema/steps.js';
+
+type MaterializedRunGraphJobExecution = Omit<JobExecutionCreateDb, 'finishedAt' | 'jobId'> & {
+  readonly finishedAt?: JobExecutionCreateDb['finishedAt'] | SQL | undefined;
+};
+
+export interface MaterializedRunGraphJob {
+  readonly job: Omit<JobCreateDb, 'workflowRunAttemptId'>;
+  readonly createExecution?:
+    | ((job: JobDb) => MaterializedRunGraphJobExecution | undefined)
+    | undefined;
+  readonly createSteps?:
+    | ((params: {
+        readonly job: JobDb;
+        readonly jobExecution: JobExecutionDb;
+      }) => readonly Omit<StepCreateDb, 'jobExecutionId'>[])
+    | undefined;
+}
+
+export async function persistMaterializedRunGraph(
+  tx: Tx,
+  params: {
+    readonly run: Pick<WorkflowRun, 'id' | 'workspaceId' | 'projectId' | 'definitionId'>;
+    readonly workflowRunAttempt: {readonly id: string; readonly attempt: number};
+    readonly materializedJobs: readonly MaterializedRunGraphJob[];
+  },
+): Promise<void> {
+  const jobRows =
+    params.materializedJobs.length === 0
+      ? []
+      : await tx
+          .insert(jobs)
+          .values(
+            params.materializedJobs.map((materializedJob) => ({
+              ...materializedJob.job,
+              workflowRunAttemptId: params.workflowRunAttempt.id,
+            })),
+          )
+          .returning();
+
+  const jobExecutionValues = jobRows.flatMap((jobRow, jobIndex) => {
+    const materializedJob = params.materializedJobs[jobIndex];
+    const jobExecution = materializedJob?.createExecution?.(jobRow);
+    return jobExecution === undefined ? [] : [{...jobExecution, jobId: jobRow.id}];
+  });
+  const jobExecutionRows =
+    jobExecutionValues.length === 0
+      ? []
+      : await tx.insert(jobExecutions).values(jobExecutionValues).returning();
+
+  const jobById = new Map(jobRows.map((job) => [job.id, job]));
+  const materializedJobByJobId = new Map(
+    jobRows.flatMap((jobRow, index) => {
+      const materializedJob = params.materializedJobs[index];
+      return materializedJob === undefined ? [] : [[jobRow.id, materializedJob] as const];
+    }),
+  );
+  const stepValues = jobExecutionRows.flatMap((jobExecution) => {
+    const job = jobById.get(jobExecution.jobId);
+    const materializedJob = materializedJobByJobId.get(jobExecution.jobId);
+    if (!job || !materializedJob) return [];
+
+    return (materializedJob.createSteps?.({job, jobExecution}) ?? []).map((step) => ({
+      ...step,
+      jobExecutionId: jobExecution.id,
+    }));
+  });
+
+  if (stepValues.length > 0) {
+    await tx.insert(steps).values(stepValues);
+  }
+
+  await writeWorkflowsOutboxEvent(tx, {
+    type: WORKFLOWS_WORKFLOW_RUN_ATTEMPT_CREATED,
+    payload: {
+      workflowRunId: params.run.id,
+      workflowRunAttemptId: params.workflowRunAttempt.id,
+      attempt: params.workflowRunAttempt.attempt,
+      workspaceId: params.run.workspaceId,
+      projectId: params.run.projectId,
+      definitionId: params.run.definitionId,
+    },
+  });
+}