@shipfox/api-definitions 2.0.0

package/src/core/workflow-model/normalize-workflow-document.ts

@@ -0,0 +1,63 @@
+import {
+  DEFAULT_HARNESS_TOOL_DEPLOYMENT_CONFIG,
+  type HarnessToolDeploymentConfig,
+} from '@shipfox/api-agent-dto';
+import {canonicalizeLabels} from '@shipfox/runner-labels';
+import type {WorkflowDocument} from '@shipfox/workflow-document';
+import type {IntegrationValidationContext} from '../entities/integration-context.js';
+import type {WorkflowModel, WorkflowStepSourceLocationMap} from '../entities/workflow-model.js';
+import {
+  InvalidWorkflowModelError,
+  type WorkflowModelValidationIssue,
+} from './invalid-workflow-model-error.js';
+import {mapJobIds} from './map-job-ids.js';
+import {normalizeDependencies, validateCycles} from './normalize-dependencies.js';
+import {normalizeEnv} from './normalize-env.js';
+import {normalizeJobs} from './normalize-jobs.js';
+import {normalizeTriggers} from './normalize-triggers.js';
+
+export function normalizeWorkflowDocument(
+  document: WorkflowDocument,
+  options: {
+    defaultRunnerLabels?: readonly string[] | undefined;
+    harnessToolDeploymentConfig?: HarnessToolDeploymentConfig | undefined;
+    integrationValidationContext?: IntegrationValidationContext | undefined;
+    stepSourceLocations?: WorkflowStepSourceLocationMap | undefined;
+  } = {},
+): WorkflowModel {
+  const issues: WorkflowModelValidationIssue[] = [];
+  const defaultRunnerLabels = canonicalizeLabels(options.defaultRunnerLabels);
+  const harnessToolDeploymentConfig =
+    options.harnessToolDeploymentConfig ?? DEFAULT_HARNESS_TOOL_DEPLOYMENT_CONFIG;
+  const context = {
+    defaultRunnerLabels,
+    harnessToolDeploymentConfig,
+    integrationValidationContext: options.integrationValidationContext,
+  };
+  const jobIdBySourceName = mapJobIds(document, issues);
+  const triggers = normalizeTriggers(document, issues);
+  const jobs = normalizeJobs(
+    document,
+    jobIdBySourceName,
+    issues,
+    options.stepSourceLocations,
+    context,
+  );
+  const dependencies = normalizeDependencies(document.jobs, jobIdBySourceName, issues);
+  const workflowEnv = normalizeEnv({env: document.env, path: ['env'], issues});
+
+  validateCycles(document.jobs, jobIdBySourceName, issues);
+
+  if (issues.length > 0) {
+    throw new InvalidWorkflowModelError(issues);
+  }
+
+  return {
+    kind: 'workflow',
+    name: document.name,
+    ...workflowEnv,
+    triggers,
+    jobs,
+    dependencies,
+  };
+}

package/src/core/workflow-model/parse-duration-ms.ts

@@ -0,0 +1,58 @@
+import type {WorkflowModelValidationIssue} from './invalid-workflow-model-error.js';
+import {issue} from './validation-issue.js';
+
+const UNIT_MS = {
+  ms: 1,
+  s: 1000,
+  m: 60 * 1000,
+  h: 60 * 60 * 1000,
+  d: 24 * 60 * 60 * 1000,
+} as const;
+
+const MIN_DURATION_MS = 1000;
+const MAX_DURATION_MS = 24 * 60 * 60 * 1000;
+const DURATION_PATTERN = /^(\d+)(ms|s|m|h|d)$/;
+
+export function parseDurationMs(params: {
+  source: string | undefined;
+  path: Array<string | number>;
+  issues: WorkflowModelValidationIssue[];
+  minMs?: number | undefined;
+  maxMs?: number | undefined;
+  outOfRangeCode?: 'invalid-duration' | 'listening-timeout-exceeds-run-timeout' | undefined;
+  outOfRangeMessage?: string | undefined;
+}): number | undefined {
+  if (params.source === undefined) return undefined;
+
+  const match = DURATION_PATTERN.exec(params.source.trim());
+  if (!match) {
+    params.issues.push(
+      issue({
+        code: 'invalid-duration',
+        message: 'Duration must be an integer followed by ms, s, m, h, or d.',
+        path: params.path,
+        details: {source: params.source},
+      }),
+    );
+    return undefined;
+  }
+
+  const value = Number(match[1]);
+  const unit = match[2] as keyof typeof UNIT_MS;
+  const ms = value * UNIT_MS[unit];
+  const minMs = params.minMs ?? MIN_DURATION_MS;
+  const maxMs = params.maxMs ?? MAX_DURATION_MS;
+  if (!Number.isSafeInteger(ms) || ms < minMs || ms > maxMs) {
+    params.issues.push(
+      issue({
+        code: params.outOfRangeCode ?? 'invalid-duration',
+        message: params.outOfRangeMessage ?? 'Duration must be between 1s and 24h.',
+        path: params.path,
+        details: {source: params.source, min_ms: minMs, max_ms: maxMs},
+      }),
+    );
+    return undefined;
+  }
+
+  return ms;
+}

package/src/core/workflow-model/parse-interpolation-field.ts

@@ -0,0 +1,566 @@
+import {secretKeySchema, secretStoreSchema} from '@shipfox/api-secrets-dto';
+import {
+  type AvailabilitySite,
+  analyzeContextKeyAccess,
+  createWorkflowExpression,
+  type ExpressionTypeEnvironment,
+  extractCelUntrustedPathAccesses,
+  getWorkflowContextAvailability,
+  getWorkflowContextDefinition,
+  getWorkflowContextHost,
+  getWorkflowContextTypeEnvironment,
+  getWorkflowContextUntrustedPaths,
+  InvalidWorkflowExpressionError,
+  InvalidWorkflowTemplateError,
+  type PlanViolation,
+  parseWorkflowTemplate,
+  planInterpolationField,
+  unavailableRootsAt,
+  type WorkflowContextName,
+  type WorkflowInterpolationField,
+  type WorkflowTemplateExprSegment,
+  type WorkflowTemplateSegment,
+  workflowContextNames,
+  workflowInterpolationFieldAcceptsContext,
+  workflowInterpolationFieldAcceptsHost,
+  workflowInterpolationFieldAcceptsTrustTier,
+} from '@shipfox/expression';
+import type {WorkflowFieldTemplate} from '../entities/workflow-model.js';
+import type {
+  WorkflowModelValidationIssue,
+  WorkflowModelValidationIssueCode,
+  WorkflowModelValidationIssuePathSegment,
+} from './invalid-workflow-model-error.js';
+import {validateDirectJobReferences} from './validate-job-references.js';
+
+export type StoredInterpolationField =
+  | 'run'
+  | 'env.value'
+  | 'agent.prompt'
+  | 'agent.model'
+  | 'agent.provider'
+  | 'job.outputs'
+  | 'job.name'
+  | 'job.runner'
+  | 'step.name'
+  | 'step.feedback';
+
+export function parseInterpolationField(params: {
+  field: StoredInterpolationField;
+  source: string;
+  path: readonly WorkflowModelValidationIssuePathSegment[];
+  issues: WorkflowModelValidationIssue[];
+  fillSite?: AvailabilitySite;
+  allowedJobReferences?: ReadonlySet<string>;
+  typeOverlay?: ExpressionTypeEnvironment | undefined;
+}): WorkflowFieldTemplate | undefined {
+  const segments = parseTemplate(params);
+  if (segments === undefined) return undefined;
+
+  const expressionSegments = segments.filter(isExpressionSegment);
+  if (expressionSegments.length === 0) return undefined;
+
+  const checkedSegments = segments.map((segment) => {
+    if (segment.kind === 'literal') return segment;
+
+    const validatedSegment = validateExpressionSegment({...params, segment});
+    return validatedSegment ?? segment;
+  });
+
+  const plan = planInterpolationField({field: params.field, segments: checkedSegments});
+  if (!plan.ok) {
+    params.issues.push(
+      ...plan.violations.map((violation) => planViolationIssue(params, violation)),
+    );
+    return undefined;
+  }
+
+  return plan.plan.field.segments;
+}
+
+function parseTemplate(params: {
+  field: WorkflowInterpolationField;
+  source: string;
+  path: readonly WorkflowModelValidationIssuePathSegment[];
+  issues: WorkflowModelValidationIssue[];
+}): WorkflowTemplateSegment[] | undefined {
+  try {
+    return parseWorkflowTemplate(params.source);
+  } catch (error) {
+    params.issues.push(
+      issue({
+        code: 'invalid-interpolation-template',
+        message: `${fieldLabel(params.field)} must use valid $${'{{ }}'} interpolation syntax.`,
+        path: params.path,
+        details: {
+          field: params.field,
+          source: params.source,
+          reason:
+            error instanceof InvalidWorkflowTemplateError
+              ? error.reason
+              : 'Template source did not parse.',
+        },
+      }),
+    );
+    return undefined;
+  }
+}
+
+function validateExpressionSegment(params: {
+  field: StoredInterpolationField;
+  source: string;
+  path: readonly WorkflowModelValidationIssuePathSegment[];
+  issues: WorkflowModelValidationIssue[];
+  segment: WorkflowTemplateExprSegment;
+  fillSite?: AvailabilitySite;
+  allowedJobReferences?: ReadonlySet<string>;
+  typeOverlay?: ExpressionTypeEnvironment | undefined;
+}): WorkflowTemplateExprSegment | undefined {
+  const contextRoots = uniqueStrings(params.segment.contextRoots);
+  const knownRoots = contextRoots.filter(isWorkflowContextName);
+  const unknownRoots = contextRoots.filter((root) => !isWorkflowContextName(root));
+
+  const rejectedRoots = knownRoots.filter(
+    (root) => !workflowInterpolationFieldAcceptsContext(params.field, root),
+  );
+  if (rejectedRoots.length > 0) {
+    params.issues.push(untrustedContextIssue({...params, contextRoots, rejectedRoots}));
+    return undefined;
+  }
+
+  const rejectedHostRoots = knownRoots.filter(
+    (root) => !workflowInterpolationFieldAcceptsHost(params.field, getWorkflowContextHost(root)),
+  );
+  if (rejectedHostRoots.length > 0) {
+    params.issues.push(runnerContextInFieldIssue({...params, contextRoots, rejectedHostRoots}));
+    return undefined;
+  }
+
+  const keyAccess = analyzeContextKeyAccess(params.segment.expression);
+  if (keyAccess.violations.length > 0) {
+    params.issues.push(
+      ...keyAccess.violations.map((violation) =>
+        computedContextKeyIssue({
+          ...params,
+          contextRoots,
+          root: violation.root,
+          expression: violation.source,
+        }),
+      ),
+    );
+    return undefined;
+  }
+
+  const invalidReferenceIssue = validateContextKeyReferences({
+    ...params,
+    contextRoots,
+    references: keyAccess.references,
+  });
+  if (invalidReferenceIssue !== undefined) {
+    params.issues.push(invalidReferenceIssue);
+    return undefined;
+  }
+
+  const invalidJobReferenceIssue =
+    params.allowedJobReferences === undefined
+      ? undefined
+      : validateDirectJobReferences({
+          source: params.source,
+          expression: params.segment.expression,
+          field: params.field,
+          path: params.path,
+          allowedJobReferences: params.allowedJobReferences,
+        });
+  if (invalidJobReferenceIssue !== undefined) {
+    params.issues.push(invalidJobReferenceIssue);
+    return undefined;
+  }
+
+  const rejectedPathRoots = findUntrustedPathRoots(params.segment, params.field, knownRoots);
+  if (rejectedPathRoots.length > 0) {
+    params.issues.push(
+      untrustedContextIssue({...params, contextRoots, rejectedRoots: rejectedPathRoots}),
+    );
+    return undefined;
+  }
+
+  if (unknownRoots.length > 0) {
+    params.issues.push(
+      issue({
+        code: 'unknown-interpolation-context',
+        message: `${fieldLabel(params.field)} interpolation references unknown context ${formatList(
+          unknownRoots,
+        )}.`,
+        path: params.path,
+        details: {
+          field: params.field,
+          source: params.source,
+          expression: params.segment.expression.source,
+          contextRoots,
+          unknownRoots,
+        },
+      }),
+    );
+    return undefined;
+  }
+
+  const fillSite = params.fillSite;
+  if (fillSite !== undefined) {
+    const serverRoots = knownRoots.filter((root) => getWorkflowContextHost(root) === 'server');
+    const unavailableRoots = unavailableRootsAt(serverRoots, fillSite);
+    if (unavailableRoots.length > 0) {
+      params.issues.push(
+        unavailableContextIssue({...params, contextRoots, unavailableRoots, fillSite}),
+      );
+      return undefined;
+    }
+  }
+
+  if (
+    params.typeOverlay === undefined &&
+    (knownRoots.length === 0 || knownRoots.some((root) => hasSyntaxOnlyCheckMode(root)))
+  ) {
+    return params.segment;
+  }
+
+  try {
+    return {
+      ...params.segment,
+      expression: createWorkflowExpression({
+        source: params.segment.expression.source,
+        check: {
+          mode: 'typed',
+          // `undefined` preserves the legacy syntax-only path above. `{}` means
+          // callers intentionally requested typed checking with the standard roots.
+          typeEnvironment: mergeTypeEnvironments(knownRoots, params.typeOverlay),
+        },
+      }),
+    };
+  } catch (error) {
+    params.issues.push(
+      issue({
+        code: 'invalid-interpolation-expression',
+        message: `${fieldLabel(params.field)} interpolation expression did not type-check.`,
+        path: params.path,
+        details: {
+          field: params.field,
+          source: params.source,
+          expression: params.segment.expression.source,
+          contextRoots,
+          reason:
+            error instanceof InvalidWorkflowExpressionError
+              ? error.reason
+              : 'Expression source did not type-check.',
+        },
+      }),
+    );
+    return undefined;
+  }
+}
+
+function findUntrustedPathRoots(
+  segment: WorkflowTemplateExprSegment,
+  field: StoredInterpolationField,
+  knownRoots: readonly WorkflowContextName[],
+): readonly WorkflowContextName[] {
+  if (!isTrustedOnlyField(field)) return [];
+
+  const untrustedPathsByRoot = new Map<string, readonly string[]>();
+  for (const root of knownRoots) {
+    const paths = getWorkflowContextUntrustedPaths(root);
+    if (paths === undefined || paths.length === 0) continue;
+    untrustedPathsByRoot.set(root, paths);
+  }
+  if (knownRoots.includes('steps')) {
+    untrustedPathsByRoot.set('steps', [
+      ...new Set([...(untrustedPathsByRoot.get('steps') ?? []), 'outputs']),
+    ]);
+  }
+  if (untrustedPathsByRoot.size === 0) return [];
+
+  return extractCelUntrustedPathAccesses({
+    source: segment.expression.source,
+    untrustedPathsByRoot,
+  }).filter(isWorkflowContextName);
+}
+
+function isTrustedOnlyField(field: StoredInterpolationField): boolean {
+  return !workflowInterpolationFieldAcceptsTrustTier(field, 'untrusted');
+}
+
+function untrustedContextIssue(params: {
+  field: WorkflowInterpolationField;
+  source: string;
+  path: readonly WorkflowModelValidationIssuePathSegment[];
+  contextRoots: readonly string[];
+  rejectedRoots: readonly WorkflowContextName[];
+}): WorkflowModelValidationIssue {
+  return issue({
+    code: 'untrusted-context-in-field',
+    message:
+      params.field === 'run'
+        ? `Run command interpolation cannot use untrusted context ${formatList(
+            params.rejectedRoots,
+          )}. Bind untrusted values to env and reference the shell variable from run instead.`
+        : `${fieldLabel(params.field)} interpolation cannot use untrusted context ${formatList(
+            params.rejectedRoots,
+          )}.`,
+    path: params.path,
+    details: {
+      field: params.field,
+      source: params.source,
+      contextRoots: params.contextRoots,
+      rejectedRoots: params.rejectedRoots,
+    },
+  });
+}
+
+function runnerContextInFieldIssue(params: {
+  field: WorkflowInterpolationField;
+  source: string;
+  path: readonly WorkflowModelValidationIssuePathSegment[];
+  contextRoots: readonly string[];
+  rejectedHostRoots: readonly WorkflowContextName[];
+}): WorkflowModelValidationIssue {
+  return issue({
+    code: 'runner-context-in-field',
+    message: `${fieldLabel(params.field)} interpolation cannot use runner context ${formatList(
+      params.rejectedHostRoots,
+    )}. Bind secrets to a run-step env value instead.`,
+    path: params.path,
+    details: {
+      field: params.field,
+      source: params.source,
+      contextRoots: params.contextRoots,
+      rejectedRoots: params.rejectedHostRoots,
+    },
+  });
+}
+
+function computedContextKeyIssue(params: {
+  field: WorkflowInterpolationField;
+  source: string;
+  path: readonly WorkflowModelValidationIssuePathSegment[];
+  contextRoots: readonly string[];
+  root: string;
+  expression: string;
+}): WorkflowModelValidationIssue {
+  return issue({
+    code: 'computed-context-key',
+    message: `${fieldLabel(params.field)} interpolation must reference ${params.root} with a literal dot key.`,
+    path: params.path,
+    details: {
+      field: params.field,
+      source: params.source,
+      expression: params.expression,
+      contextRoots: params.contextRoots,
+      root: params.root,
+    },
+  });
+}
+
+function validateContextKeyReferences(params: {
+  field: WorkflowInterpolationField;
+  source: string;
+  path: readonly WorkflowModelValidationIssuePathSegment[];
+  contextRoots: readonly string[];
+  segment: WorkflowTemplateExprSegment;
+  references: ReturnType<typeof analyzeContextKeyAccess>['references'];
+}): WorkflowModelValidationIssue | undefined {
+  for (const reference of params.references) {
+    if (!secretKeySchema.safeParse(reference.key).success) {
+      return computedContextKeyIssue({
+        ...params,
+        root: reference.root,
+        expression: params.segment.expression.source,
+      });
+    }
+
+    if (reference.root !== 'secrets' || reference.store === undefined) continue;
+    if (secretStoreSchema.safeParse(reference.store).success) continue;
+
+    return issue({
+      code: 'unknown-secret-store',
+      message: `${fieldLabel(params.field)} interpolation references unknown secret store "${reference.store}".`,
+      path: params.path,
+      details: {
+        field: params.field,
+        source: params.source,
+        expression: params.segment.expression.source,
+        contextRoots: params.contextRoots,
+        store: reference.store,
+      },
+    });
+  }
+
+  return undefined;
+}
+
+const availabilitySiteLabels = {
+  ingest: 'ingest',
+  'run-creation': 'run creation',
+  'execution-creation': 'execution creation',
+  'job-activation': 'job activation',
+  'step-dispatch': 'step dispatch',
+  'step-report': 'step reporting',
+  'execution-resolution': 'execution resolution',
+  'job-resolution': 'job resolution',
+} as const satisfies Record<AvailabilitySite, string>;
+
+function unavailableContextIssue(params: {
+  field: WorkflowInterpolationField;
+  source: string;
+  path: readonly WorkflowModelValidationIssuePathSegment[];
+  segment: WorkflowTemplateExprSegment;
+  contextRoots: readonly string[];
+  unavailableRoots: readonly WorkflowContextName[];
+  fillSite: AvailabilitySite;
+}): WorkflowModelValidationIssue {
+  return issue({
+    code: 'context-unavailable-at-fill-site',
+    message: `${fieldLabel(params.field)} interpolation references ${contextNoun(
+      params.unavailableRoots,
+    )} ${formatList(params.unavailableRoots)} that ${availabilityVerb(
+      params.unavailableRoots,
+    )} not available at ${describeAvailabilitySite(params.fillSite)}. ${params.unavailableRoots
+      .map(unavailableRootAvailabilityMessage)
+      .join(' ')}`,
+    path: params.path,
+    details: {
+      field: params.field,
+      source: params.source,
+      expression: params.segment.expression.source,
+      contextRoots: params.contextRoots,
+      unavailableRoots: params.unavailableRoots,
+      fillSite: params.fillSite,
+    },
+  });
+}
+
+function planViolationIssue(
+  params: {
+    field: WorkflowInterpolationField;
+    source: string;
+    path: readonly WorkflowModelValidationIssuePathSegment[];
+  },
+  violation: PlanViolation,
+): WorkflowModelValidationIssue {
+  if (violation.reason === 'computed-context-key') {
+    return issue({
+      code: 'computed-context-key',
+      message: `${fieldLabel(params.field)} interpolation must reference ${formatList(
+        violation.contextRoots ?? [],
+      )} with a literal dot key.`,
+      path: params.path,
+      details: {
+        field: params.field,
+        source: params.source,
+        expression: violation.source,
+        contextRoots: violation.contextRoots,
+      },
+    });
+  }
+
+  return issue({
+    code: 'runner-context-not-bare',
+    message: `${fieldLabel(params.field)} interpolation references runner context in a larger expression; ${violation.hint}.`,
+    path: params.path,
+    details: {
+      field: params.field,
+      source: params.source,
+      expression: violation.source,
+      runnerRoots: violation.runnerRoots ?? [],
+    },
+  });
+}
+
+function contextNoun(roots: readonly WorkflowContextName[]): 'context' | 'contexts' {
+  return roots.length === 1 ? 'context' : 'contexts';
+}
+
+function availabilityVerb(roots: readonly WorkflowContextName[]): 'is' | 'are' {
+  return roots.length === 1 ? 'is' : 'are';
+}
+
+function unavailableRootAvailabilityMessage(root: WorkflowContextName): string {
+  const availability = getWorkflowContextAvailability(root);
+  if (availability === undefined) return `"${root}" is not available at any server site.`;
+  return `"${root}" becomes available at ${describeAvailabilitySite(availability)}.`;
+}
+
+function describeAvailabilitySite(site: AvailabilitySite): string {
+  return availabilitySiteLabels[site];
+}
+
+function hasSyntaxOnlyCheckMode(root: WorkflowContextName): boolean {
+  return getWorkflowContextDefinition(root).checkMode === 'syntax';
+}
+
+function mergeTypeEnvironments(
+  roots: readonly WorkflowContextName[],
+  typeOverlay?: ExpressionTypeEnvironment,
+): ExpressionTypeEnvironment {
+  const typeEnvironment: Record<string, ExpressionTypeEnvironment[string]> = {};
+
+  for (const root of roots) {
+    const overlayType = typeOverlay?.[root];
+    if (overlayType !== undefined) {
+      typeEnvironment[root] = overlayType;
+      continue;
+    }
+
+    const contextTypeEnvironment = getWorkflowContextTypeEnvironment(root);
+    if (contextTypeEnvironment === undefined) {
+      if (typeOverlay !== undefined) typeEnvironment[root] = {kind: 'map'};
+      continue;
+    }
+
+    Object.assign(typeEnvironment, contextTypeEnvironment);
+  }
+
+  return typeEnvironment;
+}
+
+function isExpressionSegment(
+  segment: WorkflowTemplateSegment,
+): segment is WorkflowTemplateExprSegment {
+  return segment.kind === 'expr';
+}
+
+function isWorkflowContextName(root: string): root is WorkflowContextName {
+  return (workflowContextNames as readonly string[]).includes(root);
+}
+
+function uniqueStrings(values: readonly string[]): readonly string[] {
+  return [...new Set(values)];
+}
+
+function formatList(values: readonly string[]): string {
+  return values.map((value) => `"${value}"`).join(', ');
+}
+
+function fieldLabel(field: WorkflowInterpolationField): string {
+  return `Workflow ${field}`;
+}
+
+function issue(params: {
+  code: WorkflowModelValidationIssueCode;
+  message: string;
+  path: readonly WorkflowModelValidationIssuePathSegment[];
+  details?: Readonly<Record<string, unknown>>;
+}): WorkflowModelValidationIssue {
+  if (params.details === undefined) {
+    return {
+      code: params.code,
+      message: params.message,
+      path: params.path,
+    };
+  }
+
+  return {
+    code: params.code,
+    message: params.message,
+    path: params.path,
+    details: params.details,
+  };
+}

package/src/core/workflow-model/stable-id.ts

@@ -0,0 +1,12 @@
+const nonStableIdPattern = /[^a-z0-9]+/g;
+const edgeDashPattern = /^-+|-+$/g;
+
+export function stableId(sourceName: string): string {
+  const id = sourceName
+    .trim()
+    .toLowerCase()
+    .replace(nonStableIdPattern, '-')
+    .replace(edgeDashPattern, '');
+
+  return id.length === 0 ? 'unnamed' : id;
+}