@shipfox/api-workflows 2.0.0

package/src/core/step-config/run.ts

@@ -0,0 +1,467 @@
+import type {WorkflowModel} from '@shipfox/api-definitions';
+import {
+  type MaterializedSecretBindingDto,
+  materializedSecretBindingSchema,
+  type SecretBindingSegmentDto,
+  secretStoreSchema,
+} from '@shipfox/api-secrets-dto';
+import {
+  analyzeContextKeyAccess,
+  type EvaluationTraceEntry,
+  evaluationTraceEntry,
+  hoistPlannedRunCommand,
+  type ResolvedField,
+  type ResolvedFieldSegment,
+  runnerFillTarget,
+  UnsafeRunInterpolationError,
+} from '@shipfox/expression';
+import type {PersistedEvaluationTraceEntry, StepConfigDispatchPlan} from '#core/entities/step.js';
+import {InterpolationUnresolvableError} from '#core/errors.js';
+import {
+  resolveStepField,
+  type StepConfigField,
+  stepConfigInterpolationError,
+  type WorkflowStepEvaluationTraceEntry,
+  type WorkflowStepTemplateDiagnostic,
+} from './fields.js';
+import type {WorkflowEvaluationContext} from './workflow-evaluation-context.js';
+
+type WorkflowModelJob = WorkflowModel['jobs'][number];
+type WorkflowModelStep = WorkflowModelJob['steps'][number];
+type WorkflowModelRunStep = Extract<WorkflowModelStep, {kind: 'run'}>;
+
+export type StepConfigMode = 'effective' | 'authored';
+
+interface WinningEnvValue {
+  readonly value: string;
+  readonly template?: ResolvedField['segments'];
+}
+
+export interface ResolveRunStepConfigParams {
+  readonly step: WorkflowModelRunStep;
+  readonly workflowEnv: WorkflowModel['env'];
+  readonly workflowEnvTemplates: WorkflowEnvTemplates | undefined;
+  readonly jobEnv: WorkflowModelJob['env'];
+  readonly jobEnvTemplates: WorkflowEnvTemplates | undefined;
+  readonly context: WorkflowEvaluationContext;
+  readonly mode: StepConfigMode;
+  readonly definitionId: string;
+}
+
+type WorkflowEnvTemplates = Readonly<Record<string, ResolvedField['segments']>>;
+
+export interface RunStepConfig {
+  readonly config: Record<string, unknown>;
+  readonly configPlan: StepConfigDispatchPlan | null;
+  readonly diagnostics: readonly WorkflowStepTemplateDiagnostic[];
+  readonly trace: readonly WorkflowStepEvaluationTraceEntry[];
+  readonly hasTemplates: boolean;
+}
+
+export function resolveRunStepConfig(params: ResolveRunStepConfigParams): RunStepConfig {
+  const env = winningEnv(params);
+  const envResolution = resolveEnv(env, params.context, params.mode, params.definitionId);
+  const commandResolution = resolveCommand({
+    step: params.step,
+    envKeys: [...Object.keys(envResolution.env), ...Object.keys(envResolution.configPlan)],
+    context: params.context,
+    mode: params.mode,
+    definitionId: params.definitionId,
+  });
+  const mergedEnv = {...envResolution.env, ...commandResolution.env};
+  const hasEnvConfig = Object.keys(mergedEnv).length > 0;
+  const envConfig = hasEnvConfig ? {env: mergedEnv} : {};
+  const planEnv = {...envResolution.configPlan, ...commandResolution.configPlan};
+  const hasConfigPlan = Object.keys(planEnv).length > 0;
+  const configPlan = hasConfigPlan ? ({env: planEnv} satisfies StepConfigDispatchPlan) : null;
+  const hasTemplates = envResolution.hasTemplates || commandResolution.hasTemplate;
+
+  return {
+    config: {run: commandResolution.command, ...envConfig},
+    configPlan,
+    diagnostics: [...envResolution.diagnostics, ...commandResolution.diagnostics],
+    trace: [...envResolution.trace, ...commandResolution.trace],
+    hasTemplates,
+  };
+}
+
+export function completeRunDispatchConfig(params: {
+  readonly config: Record<string, unknown>;
+  readonly plan: StepConfigDispatchPlan;
+  readonly context: WorkflowEvaluationContext;
+  readonly definitionId: string;
+  readonly trace: PersistedEvaluationTraceEntry[];
+}): void {
+  const env = {...readConfigEnv(params.config)};
+  const secretBindings: MaterializedSecretBindingDto[] = [];
+
+  if (params.plan.run !== undefined) {
+    const resolved = completeRunCommand({
+      field: params.plan.run,
+      context: params.context,
+      definitionId: params.definitionId,
+      reservedNames: Object.keys(env),
+      trace: params.trace,
+    });
+    params.config.run = resolved.command;
+    Object.assign(env, resolved.env);
+    secretBindings.push(...resolved.secretBindings);
+  }
+
+  if (params.plan.env !== undefined) {
+    for (const [key, field] of Object.entries(params.plan.env)) {
+      const completed = completeDispatchField({
+        field: 'env.value',
+        traceField: 'env',
+        errorField: 'env',
+        template: field,
+        context: params.context,
+        definitionId: params.definitionId,
+        envKey: key,
+        trace: params.trace,
+      });
+      if (completed.kind === 'binding') secretBindings.push(completed.binding);
+      else env[key] = completed.value;
+    }
+  }
+
+  if (Object.keys(env).length > 0) params.config.env = env;
+  if (secretBindings.length > 0) params.config.secret_bindings = secretBindings;
+}
+
+function completeRunCommand(params: {
+  readonly field: ResolvedField;
+  readonly context: WorkflowEvaluationContext;
+  readonly definitionId: string;
+  readonly reservedNames: Iterable<string>;
+  readonly trace: PersistedEvaluationTraceEntry[];
+}): {
+  readonly command: string;
+  readonly env: Readonly<Record<string, string>>;
+  readonly secretBindings: readonly MaterializedSecretBindingDto[];
+} {
+  const hoisted = hoistPlannedRunCommand({
+    field: params.field,
+    reservedNames: params.reservedNames,
+  });
+  const env: Record<string, string> = {};
+  const secretBindings: MaterializedSecretBindingDto[] = [];
+
+  for (const binding of hoisted.bindings) {
+    const completed = completeDispatchField({
+      field: 'run',
+      traceField: 'run',
+      errorField: 'run',
+      template: {segments: [binding.segment]},
+      context: params.context,
+      definitionId: params.definitionId,
+      envKey: binding.name,
+      trace: params.trace,
+    });
+    if (completed.kind === 'binding') secretBindings.push(completed.binding);
+    else env[binding.name] = completed.value;
+  }
+
+  return {command: hoisted.command, env, secretBindings};
+}
+
+type CompletedDispatchField =
+  | {readonly kind: 'value'; readonly value: string}
+  | {readonly kind: 'binding'; readonly binding: MaterializedSecretBindingDto};
+
+function completeDispatchField(params: {
+  readonly field: 'run' | 'env.value';
+  readonly traceField: StepConfigField;
+  readonly errorField: StepConfigField;
+  readonly template: ResolvedField;
+  readonly context: WorkflowEvaluationContext;
+  readonly definitionId: string;
+  readonly envKey?: string;
+  readonly trace: PersistedEvaluationTraceEntry[];
+}): CompletedDispatchField {
+  const resolved = resolveStepField(params);
+  params.trace.push(...tagTrace(resolved.trace, params.traceField, params.envKey));
+  if (resolved.kind === 'frozen') return {kind: 'value', value: resolved.value};
+  if (params.envKey !== undefined && containsOnlyRunnerSecretSegments(resolved.field)) {
+    params.trace.push(
+      ...runnerSecretReferenceTrace(resolved.field, params.traceField, params.envKey),
+    );
+    return {
+      kind: 'binding',
+      binding: secretBindingFromField(params.envKey, resolved.field),
+    };
+  }
+
+  const source = resolved.field.segments.find((segment) => segment.kind === 'deferred')?.expression
+    .source;
+  throw new InterpolationUnresolvableError(params.definitionId, {
+    field: params.errorField,
+    source: source ?? params.field,
+    ...(params.envKey === undefined ? {} : {envKey: params.envKey}),
+  });
+}
+
+function containsOnlyRunnerSecretSegments(field: ResolvedField): boolean {
+  return field.segments.every((segment) => {
+    if (segment.kind === 'literal') return true;
+    return (
+      segment.fillTarget === runnerFillTarget &&
+      segment.roots.length === 1 &&
+      segment.roots[0] === 'secrets'
+    );
+  });
+}
+
+function secretBindingFromField(
+  target: string,
+  field: ResolvedField,
+): MaterializedSecretBindingDto {
+  const binding = {
+    target,
+    segments: field.segments.map(secretBindingSegment),
+  };
+  return materializedSecretBindingSchema.parse(binding);
+}
+
+function secretBindingSegment(segment: ResolvedFieldSegment): SecretBindingSegmentDto {
+  if (segment.kind === 'literal') return {kind: 'literal', value: segment.value};
+
+  const keyAccess = analyzeContextKeyAccess(segment.expression);
+  const reference = keyAccess.references.find((candidate) => candidate.root === 'secrets');
+  if (reference === undefined) {
+    throw new Error(
+      `Runner secret segment did not contain a secret reference: ${segment.expression.source}`,
+    );
+  }
+
+  return {
+    kind: 'secret',
+    store: secretStoreSchema.parse(reference.store ?? 'local'),
+    key: reference.key,
+  };
+}
+
+function resolveCommand(params: {
+  readonly step: WorkflowModelRunStep;
+  readonly envKeys: readonly string[];
+  readonly context: WorkflowEvaluationContext;
+  readonly mode: StepConfigMode;
+  readonly definitionId: string;
+}): {
+  readonly command: string;
+  readonly env: Readonly<Record<string, string>>;
+  readonly configPlan: Readonly<Record<string, ResolvedField>>;
+  readonly diagnostics: readonly WorkflowStepTemplateDiagnostic[];
+  readonly trace: readonly WorkflowStepEvaluationTraceEntry[];
+  readonly hasTemplate: boolean;
+} {
+  const template = params.step.templates?.command;
+  const hasTemplate = template !== undefined;
+  if (!hasTemplate) {
+    return {
+      command: params.step.command.value,
+      env: {},
+      configPlan: {},
+      diagnostics: [],
+      trace: [],
+      hasTemplate: false,
+    };
+  }
+
+  const usesAuthoredMode = params.mode === 'authored';
+  if (usesAuthoredMode) {
+    return {
+      command: params.step.command.value,
+      env: {},
+      configPlan: {},
+      diagnostics: [],
+      trace: [],
+      hasTemplate: true,
+    };
+  }
+
+  const env: Record<string, string> = {};
+  const configPlan: Record<string, ResolvedField> = {};
+  const diagnostics: WorkflowStepTemplateDiagnostic[] = [];
+  const trace: WorkflowStepEvaluationTraceEntry[] = [];
+  const hoisted = hoistCommand(template, params.envKeys, params.definitionId);
+
+  for (const binding of hoisted.bindings) {
+    const resolved = resolveStepField({
+      field: 'run',
+      template: {segments: [binding.segment]},
+      context: params.context,
+      definitionId: params.definitionId,
+      errorField: 'run',
+    });
+    const resolvedToDispatchPlan = resolved.kind === 'residual';
+    if (resolvedToDispatchPlan) configPlan[binding.name] = resolved.field;
+    else env[binding.name] = resolved.value;
+    diagnostics.push(
+      ...resolved.diagnostics.map((diagnostic) => ({...diagnostic, field: 'run' as const})),
+    );
+    trace.push(...tagTrace(resolved.trace, 'run'));
+  }
+
+  return {
+    command: hoisted.command,
+    env,
+    configPlan,
+    diagnostics,
+    trace,
+    hasTemplate: true,
+  };
+}
+
+function hoistCommand(
+  template: ResolvedField['segments'],
+  reservedNames: readonly string[],
+  definitionId: string,
+): ReturnType<typeof hoistPlannedRunCommand> {
+  try {
+    return hoistPlannedRunCommand({
+      field: {segments: template},
+      reservedNames,
+    });
+  } catch (error) {
+    if (error instanceof UnsafeRunInterpolationError) {
+      throw stepConfigInterpolationError({definitionId, errorField: 'run'}, error);
+    }
+    throw error;
+  }
+}
+
+function resolveEnv(
+  env: Readonly<Record<string, WinningEnvValue>>,
+  context: WorkflowEvaluationContext,
+  mode: StepConfigMode,
+  definitionId: string,
+): {
+  readonly env: Readonly<Record<string, string>>;
+  readonly configPlan: Readonly<Record<string, ResolvedField>>;
+  readonly diagnostics: readonly WorkflowStepTemplateDiagnostic[];
+  readonly trace: readonly WorkflowStepEvaluationTraceEntry[];
+  readonly hasTemplates: boolean;
+} {
+  const resolvedEnv: Record<string, string> = {};
+  const configPlan: Record<string, ResolvedField> = {};
+  const diagnostics: WorkflowStepTemplateDiagnostic[] = [];
+  const trace: WorkflowStepEvaluationTraceEntry[] = [];
+  let hasTemplates = false;
+
+  for (const [key, entry] of Object.entries(env)) {
+    const template = entry.template;
+    const hasTemplate = template !== undefined;
+    const usesAuthoredMode = mode === 'authored';
+    const shouldUseAuthoredValue = !hasTemplate || usesAuthoredMode;
+
+    if (shouldUseAuthoredValue) {
+      resolvedEnv[key] = entry.value;
+      hasTemplates ||= hasTemplate;
+      continue;
+    }
+
+    hasTemplates = true;
+    const resolved = resolveStepField({
+      field: 'env.value',
+      template: {segments: template},
+      context,
+      definitionId,
+      errorField: 'env',
+      envKey: key,
+    });
+    const resolvedToDispatchPlan = resolved.kind === 'residual';
+    if (resolvedToDispatchPlan) configPlan[key] = resolved.field;
+    else resolvedEnv[key] = resolved.value;
+    diagnostics.push(
+      ...resolved.diagnostics.map((diagnostic) => ({
+        ...diagnostic,
+        field: 'env' as const,
+        envKey: key,
+      })),
+    );
+    trace.push(...tagTrace(resolved.trace, 'env', key));
+  }
+
+  return {env: resolvedEnv, configPlan, diagnostics, trace, hasTemplates};
+}
+
+function tagTrace(
+  trace: readonly EvaluationTraceEntry[],
+  field: StepConfigField,
+  envKey?: string,
+): WorkflowStepEvaluationTraceEntry[] {
+  return trace.map((entry) => ({
+    ...entry,
+    field,
+    ...(envKey === undefined ? {} : {envKey}),
+  }));
+}
+
+function runnerSecretReferenceTrace(
+  field: ResolvedField,
+  stepField: StepConfigField,
+  envKey: string,
+): WorkflowStepEvaluationTraceEntry[] {
+  return field.segments.flatMap((segment) => {
+    if (segment.kind === 'literal') return [];
+    return [
+      {
+        ...evaluationTraceEntry({
+          expression: segment.expression.source,
+          roots: segment.roots,
+          fillTarget: runnerFillTarget,
+          evaluatedAt: 'step-dispatch',
+          reference: true,
+        }),
+        field: stepField,
+        envKey,
+      },
+    ];
+  });
+}
+
+function winningEnv(params: {
+  readonly workflowEnv: WorkflowModel['env'];
+  readonly workflowEnvTemplates: WorkflowEnvTemplates | undefined;
+  readonly jobEnv: WorkflowModelJob['env'];
+  readonly jobEnvTemplates: WorkflowEnvTemplates | undefined;
+  readonly step: WorkflowModelStep;
+}): Readonly<Record<string, WinningEnvValue>> {
+  if (params.step.kind !== 'run') return {};
+
+  return mergeEnvLayers(
+    {env: params.workflowEnv, templates: params.workflowEnvTemplates},
+    {env: params.jobEnv, templates: params.jobEnvTemplates},
+    {env: params.step.env, templates: params.step.templates?.env},
+  );
+}
+
+function mergeEnvLayers(
+  ...layers: readonly {
+    readonly env: Readonly<Record<string, string>> | undefined;
+    readonly templates: WorkflowEnvTemplates | undefined;
+  }[]
+): Readonly<Record<string, WinningEnvValue>> {
+  const merged: Record<string, WinningEnvValue> = {};
+
+  for (const layer of layers) {
+    for (const [key, value] of Object.entries(layer.env ?? {})) {
+      const template = layer.templates?.[key];
+      const hasTemplate = template !== undefined;
+      merged[key] = hasTemplate ? {value, template} : {value};
+    }
+  }
+
+  return merged;
+}
+
+function readConfigEnv(config: Record<string, unknown>): Record<string, string> {
+  const env = config.env;
+  if (env === null || typeof env !== 'object' || Array.isArray(env)) return {};
+  return Object.fromEntries(
+    Object.entries(env).flatMap(([key, value]) =>
+      typeof value === 'string' ? [[key, value]] : [],
+    ),
+  );
+}

package/src/core/step-config/workflow-evaluation-context.ts

@@ -0,0 +1,6 @@
+import type {AvailabilitySite, WorkflowExpressionEvaluationContext} from '@shipfox/expression';
+
+export interface WorkflowEvaluationContext {
+  readonly site: AvailabilitySite;
+  readonly values: WorkflowExpressionEvaluationContext;
+}

package/src/core/step-transition/apply-step-transition.ts

@@ -0,0 +1,163 @@
+import type {LogOutcomeDto} from '@shipfox/api-workflows-dto';
+import type {Tx} from '#db/db.js';
+import {
+  applyStepResult,
+  finishStepAttempt,
+  getStepsByJobExecutionIdForUpdate,
+  rewindStepsToPending,
+  settleJobFailed,
+  writeJobStepsSettledOutbox,
+  writeStepRestartEnqueuedOutbox,
+} from '#db/workflow-runs.js';
+import {
+  deriveCompletion,
+  isTerminal,
+  type StepReport,
+  type StepTransitionDecision,
+} from './decide-step-transition.js';
+
+export type StepProgressionOutcome =
+  | {jobFinished: false}
+  | {jobFinished: true; status: 'succeeded' | 'failed'};
+
+export interface StepProgressionMetrics {
+  jobStepsSettledStatus?: 'succeeded' | 'failed';
+  stepRestartEnqueued?: boolean;
+}
+
+export interface StepProgressionResult {
+  outcome: StepProgressionOutcome;
+  metrics: StepProgressionMetrics;
+}
+
+export interface ApplyStepTransitionContext {
+  jobId: string;
+  jobExecutionId: string;
+  result: StepReport;
+  logOutcome: LogOutcomeDto;
+  // Gate evaluation audit payload to record on the attempt, when a gate ran.
+  gateResult?: Record<string, unknown> | null;
+}
+
+// Durable side of the decision: writes the attempt + projection for a transition,
+// then derives job completion from the post-apply projection and enqueues the
+// completion event in the same transaction. Only called for a running target, so
+// every path here is an actual state change (idempotent no-ops are handled by the
+// caller before deciding).
+export async function applyStepTransition(
+  decision: StepTransitionDecision,
+  ctx: ApplyStepTransitionContext,
+  tx: Tx,
+): Promise<StepProgressionResult> {
+  switch (decision.kind) {
+    case 'complete-step':
+    case 'complete-job': {
+      // A passing gate makes a step succeed even if the raw command status was
+      // 'failed', so the projection error is cleared regardless of the report.
+      await finishStepAttempt(
+        {
+          stepId: decision.stepId,
+          attempt: decision.attempt,
+          status: 'succeeded',
+          output: ctx.result.output ?? null,
+          response: ctx.result.response ?? null,
+          exitCode: ctx.result.exitCode ?? null,
+          logOutcome: ctx.logOutcome,
+          gateResult: ctx.gateResult ?? null,
+        },
+        tx,
+      );
+      await applyStepResult(
+        {
+          jobExecutionId: ctx.jobExecutionId,
+          stepId: decision.stepId,
+          status: 'succeeded',
+          error: null,
+        },
+        tx,
+      );
+      break;
+    }
+    case 'fail-job':
+    case 'fail-job-restart-exhausted': {
+      await finishStepAttempt(
+        {
+          stepId: decision.failedStepId,
+          attempt: decision.attempt,
+          status: 'failed',
+          output: ctx.result.output ?? null,
+          response: ctx.result.response ?? null,
+          error: decision.failureError,
+          exitCode: ctx.result.exitCode ?? null,
+          logOutcome: ctx.logOutcome,
+          gateResult: ctx.gateResult ?? null,
+        },
+        tx,
+      );
+      const status = await settleJobFailed(tx, {
+        jobId: ctx.jobId,
+        jobExecutionId: ctx.jobExecutionId,
+        failedStepId: decision.failedStepId,
+        error: decision.failureError ?? {message: 'Step failed'},
+      });
+      if (status !== null) {
+        return {
+          outcome: {jobFinished: true, status},
+          metrics: {jobStepsSettledStatus: status},
+        };
+      }
+      return {outcome: {jobFinished: false}, metrics: {}};
+    }
+    case 'restart-job-from-step': {
+      // Record the failed attempt FIRST (audit, with the restart feedback), then
+      // rewind the projection from restart_from so the prior result is preserved
+      // only in the attempt history. All in one transaction with the report.
+      await finishStepAttempt(
+        {
+          stepId: decision.failedStepId,
+          attempt: decision.attempt,
+          status: 'failed',
+          output: ctx.result.output ?? null,
+          response: ctx.result.response ?? null,
+          error: decision.failureError,
+          exitCode: ctx.result.exitCode ?? null,
+          logOutcome: ctx.logOutcome,
+          gateResult: ctx.gateResult ?? null,
+          restartFeedback: decision.feedback,
+        },
+        tx,
+      );
+      await rewindStepsToPending(
+        {jobExecutionId: ctx.jobExecutionId, fromPosition: decision.restartFromPosition},
+        tx,
+      );
+      await writeStepRestartEnqueuedOutbox(tx, {
+        jobId: ctx.jobId,
+        failedStepId: decision.failedStepId,
+        failedStepAttempt: decision.attempt,
+        restartFromStepId: decision.restartFromStepId,
+        feedback: decision.feedback,
+      });
+      // The job stays running; the next pull re-dispatches restart_from. Do not
+      // derive completion or emit a completion event.
+      return {outcome: {jobFinished: false}, metrics: {stepRestartEnqueued: true}};
+    }
+  }
+
+  // Re-derive completion from the post-apply projection and emit the
+  // steps-settled signal exactly once, here on the applied path.
+  const after = await getStepsByJobExecutionIdForUpdate(ctx.jobExecutionId, tx);
+  if (after.every((step) => isTerminal(step.status))) {
+    const status = deriveCompletion(after);
+    await writeJobStepsSettledOutbox(tx, {
+      jobId: ctx.jobId,
+      jobExecutionId: ctx.jobExecutionId,
+      status,
+    });
+    return {
+      outcome: {jobFinished: true, status},
+      metrics: {jobStepsSettledStatus: status},
+    };
+  }
+  return {outcome: {jobFinished: false}, metrics: {}};
+}