@shipfox/api-workflows 12.1.0 → 12.2.0

package/src/core/step-config/assemble-run-context.ts

@@ -1,12 +1,14 @@
 import {
   analyzeContextRootKeyAccess,
+  type ExpressionType,
   extractExactContextRoots,
   getWorkflowPredicateContextRoots,
+  rehydrateJsonExpressionRecord,
   type WorkflowExpressionEvaluationContext,
   type WorkflowPredicateContextRoot,
 } from '@shipfox/expression';
 import type {Job, JobListeningTrigger} from '#core/entities/job.js';
-import type {JobExecution} from '#core/entities/job-execution.js';
+import type {JobExecution, WorkflowExecutionEvent} from '#core/entities/job-execution.js';
 import type {Step, StepAttempt, StepStatus} from '#core/entities/step.js';
 import type {
   TriggerPayload,
@@ -17,9 +19,15 @@ import type {WorkflowEvaluationContext} from './workflow-evaluation-context.js';
 
 export interface JobContextInput {
   readonly job: Pick<Job, 'key' | 'status' | 'outputs'>;
+  readonly outputTypes?: Readonly<Record<string, ExpressionType>>;
   readonly executions: readonly JobExecution[];
 }
 
+export interface AssembleJobsContextOptions {
+  readonly skipTypedOutputRehydration?: boolean;
+  readonly skipCelNativeRehydration?: boolean;
+}
+
 export interface AssembleWorkflowRunContextParams {
   readonly run: Pick<
     WorkflowRun,
@@ -37,8 +45,11 @@ export interface AssembleWorkflowRunContextParams {
   readonly vars?: Record<string, string> | undefined;
 }
 
+interface AssembleContextOptions extends AssembleJobsContextOptions {}
+
 export function assembleWorkflowRunContext(
   params: AssembleWorkflowRunContextParams,
+  options: AssembleContextOptions = {},
 ): WorkflowExpressionEvaluationContext {
   const triggerReference = params.run.triggerReference;
   return {
@@ -48,7 +59,8 @@ export function assembleWorkflowRunContext(
     },
     run: {
       id: params.run.id,
-      number: params.run.number,
+      number:
+        options.skipCelNativeRehydration === true ? params.run.number : BigInt(params.run.number),
       name: params.run.name,
       project_id: params.run.projectId,
       workspace_id: params.run.workspaceId,
@@ -128,29 +140,62 @@ export function assembleExecutionCreationContext(
  */
 export function assembleExecutionsContext(
   executions: readonly JobExecution[],
+  outputTypes?: Readonly<Record<string, ExpressionType>>,
+  options: AssembleContextOptions = {},
 ): WorkflowExpressionEvaluationContext {
   return {
-    executions: executions.map((execution, index) => assembleExecutionContext(execution, index)),
+    executions: executions.map((execution, index) =>
+      assembleExecutionContext(execution, index, outputTypes, options),
+    ),
   };
 }
 
-function assembleExecutionContext(execution: JobExecution, index: number): Record<string, unknown> {
+function assembleExecutionContext(
+  execution: JobExecution,
+  index: number,
+  outputTypes?: Readonly<Record<string, ExpressionType>>,
+  options: AssembleContextOptions = {},
+): Record<string, unknown> {
+  const skipCelNativeRehydration = options.skipCelNativeRehydration === true;
   return {
-    index,
+    index: skipCelNativeRehydration ? index : BigInt(index),
     name: execution.name,
     status: execution.status,
     started_at: execution.startedAt,
     finished_at: execution.finishedAt,
-    events: execution.triggerEvents,
-    outputs: execution.outputs ?? {},
+    events: skipCelNativeRehydration
+      ? execution.triggerEvents
+      : execution.triggerEvents.map(assembleExecutionEventContext),
+    outputs:
+      options.skipTypedOutputRehydration === true
+        ? (execution.outputs ?? {})
+        : rehydrateJsonExpressionRecord(execution.outputs, outputTypes),
+  };
+}
+
+function assembleExecutionEventContext(event: WorkflowExecutionEvent): Record<string, unknown> {
+  const receivedAt = new Date(event.received_at);
+  /**
+   * An unparseable stored timestamp would become an `Invalid Date`, which compares
+   * false against every CEL `timestamp` and throws on `toISOString` during template
+   * resolution. Keeping the raw value fails loudly as a type mismatch instead.
+   */
+  if (Number.isNaN(receivedAt.getTime())) return {...event};
+
+  return {
+    ...event,
+    received_at: receivedAt,
   };
 }
 
 export function assembleJobsContext(
   jobs: readonly JobContextInput[],
+  options: AssembleJobsContextOptions = {},
 ): WorkflowExpressionEvaluationContext & {readonly jobs: Record<string, unknown>} {
   return {
-    jobs: Object.fromEntries(jobs.map((input) => [input.job.key, assembleJobContext(input)])),
+    jobs: Object.fromEntries(
+      jobs.map((input) => [input.job.key, assembleJobContext(input, options)]),
+    ),
   };
 }
 
@@ -166,7 +211,7 @@ export function assembleJobActivationContext(
     values: {
       ...assembleWorkflowRunContext(params),
       ...assembleJobsContext(params.jobs),
-      needs: params.jobs.map(assembleJobContext),
+      needs: params.jobs.map((input) => assembleJobContext(input, {})),
       vars: params.vars ?? {},
     },
   };
@@ -188,6 +233,8 @@ export interface ListenerSnapshotPlan {
   readonly jobKeys: ReadonlySet<string>;
 }
 
+export type ListenerFilterOutputTypes = Record<string, Record<string, ExpressionType>>;
+
 export function planListenerFilterSnapshots(params: {
   readonly on: readonly JobListeningTrigger[];
   readonly until: readonly JobListeningTrigger[] | null;
@@ -262,12 +309,15 @@ export function assembleListenerSnapshotContext(params: {
     params.plan.roots.has('run') ||
     params.plan.roots.has('trigger')
   ) {
-    const runContext = assembleWorkflowRunContext({
-      run: params.run,
-      triggerPayload: params.triggerPayload,
-      inputs: params.inputs,
-      vars: params.vars,
-    });
+    const runContext = assembleWorkflowRunContext(
+      {
+        run: params.run,
+        triggerPayload: params.triggerPayload,
+        inputs: params.inputs,
+        vars: params.vars,
+      },
+      {skipCelNativeRehydration: true},
+    );
     if (params.plan.roots.has('workflow')) context.workflow = runContext.workflow;
     if (params.plan.roots.has('run')) context.run = runContext.run;
     if (params.plan.roots.has('trigger')) context.trigger = runContext.trigger;
@@ -296,27 +346,69 @@ function requestedJobsContext(
   dependencyJobs: readonly JobContextInput[],
   jobKeys: ReadonlySet<string>,
 ): Record<string, unknown> {
-  if (jobKeys.size === 0) return assembleJobsContext(dependencyJobs).jobs;
+  // Listener snapshots cross a JSON outbox boundary; their type metadata is persisted separately.
+  const options: AssembleJobsContextOptions = {
+    skipCelNativeRehydration: true,
+    skipTypedOutputRehydration: true,
+  };
+  if (jobKeys.size === 0) {
+    return assembleJobsContext(dependencyJobs, options).jobs;
+  }
 
   const filtered = dependencyJobs.filter(({job}) => jobKeys.has(job.key));
 
-  return assembleJobsContext(filtered).jobs;
+  return assembleJobsContext(filtered, options).jobs;
 }
 
 export type ListenerTriggerWithSnapshot = JobListeningTrigger & {
   readonly filter_snapshot?: Record<string, unknown>;
+  readonly filter_output_types?: ListenerFilterOutputTypes;
 };
 
 export function applyListenerFilterSnapshots(
   plans: readonly MatcherSnapshotPlan[],
   context: WorkflowExpressionEvaluationContext,
+  outputTypes?: ListenerFilterOutputTypes,
 ): ListenerTriggerWithSnapshot[] {
   return plans.map((plan) => {
     const filterSnapshot = filterSnapshotForPlan(plan, context);
     if (filterSnapshot === undefined) return plan.matcher;
 
-    return {...plan.matcher, filter_snapshot: filterSnapshot};
+    const filterOutputTypes = filterOutputTypesForPlan(plan, context, outputTypes);
+
+    return {
+      ...plan.matcher,
+      filter_snapshot: filterSnapshot,
+      ...(filterOutputTypes === undefined ? {} : {filter_output_types: filterOutputTypes}),
+    };
+  });
+}
+
+export function listenerFilterOutputTypesForJobs(
+  jobs: readonly JobContextInput[],
+): ListenerFilterOutputTypes {
+  return Object.fromEntries(
+    jobs.flatMap(({job, outputTypes}) =>
+      outputTypes === undefined ? [] : [[job.key, {...outputTypes}] as const],
+    ),
+  );
+}
+
+function filterOutputTypesForPlan(
+  plan: MatcherSnapshotPlan,
+  context: WorkflowExpressionEvaluationContext,
+  outputTypes: ListenerFilterOutputTypes | undefined,
+): ListenerFilterOutputTypes | undefined {
+  if (outputTypes === undefined || !plan.roots.has('jobs')) return undefined;
+
+  const jobsSnapshot = jobsSnapshotForPlan(plan, context);
+  if (jobsSnapshot === undefined) return undefined;
+
+  const entries = Object.keys(jobsSnapshot).flatMap((jobKey) => {
+    const types = outputTypes[jobKey];
+    return types === undefined ? [] : [[jobKey, {...types}] as const];
   });
+  return entries.length === 0 ? undefined : Object.fromEntries(entries);
 }
 
 function filterSnapshotForPlan(
@@ -356,12 +448,24 @@ function jobsSnapshotForPlan(
   return snapshot;
 }
 
-function assembleJobContext({job, executions}: JobContextInput): Record<string, unknown> {
+function assembleJobContext(
+  {job, outputTypes, executions}: JobContextInput,
+  options: AssembleJobsContextOptions,
+): Record<string, unknown> {
+  const outputs =
+    options.skipTypedOutputRehydration === true
+      ? (job.outputs ?? {})
+      : rehydrateJsonExpressionRecord(job.outputs, outputTypes);
+
   return {
     key: job.key,
     status: job.status,
-    outputs: job.outputs ?? {},
-    executions: assembleExecutionsContext(executions).executions,
+    outputs,
+    executions: assembleExecutionsContext(
+      executions,
+      options.skipTypedOutputRehydration === true ? undefined : outputTypes,
+      options,
+    ).executions,
   };
 }
 
@@ -448,14 +552,8 @@ export function assembleStepDispatchContext(params: {
         ? {}
         : {
             execution: {
-              index: params.jobExecution.sequence,
-              name: params.jobExecution.name,
-              status: params.jobExecution.status,
+              ...assembleExecutionContext(params.jobExecution, params.jobExecution.sequence - 1),
               failed: stepAttemptContext.stepsFailed,
-              started_at: params.jobExecution.startedAt,
-              finished_at: params.jobExecution.finishedAt,
-              events: params.jobExecution.triggerEvents,
-              outputs: params.jobExecution.outputs ?? {},
             },
           }),
       ...(targetStep === undefined

package/src/core/step-config/fields.ts

@@ -35,13 +35,35 @@ export interface ResolveStepFieldParams {
 }
 
 export function resolveStepField(params: ResolveStepFieldParams): SiteResolvedField {
+  return resolveStepFieldAtSite(params, false);
+}
+
+function resolveStepFieldWithType(params: ResolveStepFieldParams): SiteResolvedField<unknown> {
+  return resolveStepFieldAtSite(params, true);
+}
+
+function resolveStepFieldAtSite(
+  params: ResolveStepFieldParams,
+  preserveSingleExpressionType: true,
+): SiteResolvedField<unknown>;
+function resolveStepFieldAtSite(
+  params: ResolveStepFieldParams,
+  preserveSingleExpressionType: false,
+): SiteResolvedField;
+function resolveStepFieldAtSite(
+  params: ResolveStepFieldParams,
+  preserveSingleExpressionType: boolean,
+): SiteResolvedField<unknown> {
   try {
-    return resolveFieldAtSite({
+    const resolveParams = {
       field: params.template,
       context: params.context.values,
       site: params.context.site,
       failurePolicy: getWorkflowInterpolationFieldFailurePolicy(params.field),
-    });
+    };
+    return preserveSingleExpressionType
+      ? resolveFieldAtSite({...resolveParams, preserveSingleExpressionType: true})
+      : resolveFieldAtSite(resolveParams);
   } catch (error) {
     if (error instanceof WorkflowTemplateResolutionError) {
       throw stepConfigInterpolationError(params, error);
@@ -74,11 +96,28 @@ export function completeStepField(params: ResolveStepFieldParams): string {
   return completeStepFieldWithTrace(params).value;
 }
 
+export function completeStepFieldWithType(params: ResolveStepFieldParams): unknown {
+  return completeStepFieldWithTypeAndTrace(params).value;
+}
+
 export function completeStepFieldWithTrace(params: ResolveStepFieldParams): {
   readonly value: string;
   readonly trace: SiteResolvedField['trace'];
 } {
-  const resolved = resolveStepField(params);
+  return completeResolvedStepField(params, resolveStepField(params));
+}
+
+export function completeStepFieldWithTypeAndTrace(params: ResolveStepFieldParams): {
+  readonly value: unknown;
+  readonly trace: SiteResolvedField<unknown>['trace'];
+} {
+  return completeResolvedStepField(params, resolveStepFieldWithType(params));
+}
+
+function completeResolvedStepField<Value>(
+  params: ResolveStepFieldParams,
+  resolved: SiteResolvedField<Value>,
+): {readonly value: Value; readonly trace: SiteResolvedField<Value>['trace']} {
   if (resolved.kind === 'frozen') return {value: resolved.value, trace: resolved.trace};
 
   const source = resolved.field.segments.find((segment) => segment.kind === 'deferred')?.expression

package/src/core/step-config/index.ts

@@ -14,8 +14,10 @@ export {
   assembleStepDispatchContext,
   assembleWorkflowRunContext,
   type JobContextInput,
+  type ListenerFilterOutputTypes,
   type ListenerSnapshotPlan,
   type ListenerTriggerWithSnapshot,
+  listenerFilterOutputTypesForJobs,
   type MatcherSnapshotPlan,
   planListenerFilterSnapshots,
 } from './assemble-run-context.js';

package/src/core/step-config/job-output-limits.ts

@@ -0,0 +1,130 @@
+import {
+  WORKFLOW_DOCUMENT_JOB_OUTPUTS_MAX_ENTRIES,
+  WORKFLOW_DOCUMENT_STEP_OUTPUT_SCHEMA_MAX_DEPTH,
+} from '@shipfox/workflow-document';
+import {JobOutputNotJsonSafeError} from '#core/errors.js';
+
+export const MAX_JOB_OUTPUT_ENTRIES = WORKFLOW_DOCUMENT_JOB_OUTPUTS_MAX_ENTRIES;
+export const MAX_JOB_OUTPUT_NESTING_DEPTH = WORKFLOW_DOCUMENT_STEP_OUTPUT_SCHEMA_MAX_DEPTH;
+export const MAX_JOB_OUTPUTS_TOTAL_BYTES = 64 * 1024;
+export const MAX_JOB_OUTPUT_VALUE_BYTES = 16 * 1024;
+
+const textEncoder = new TextEncoder();
+
+export type JsonSafeJobOutputValue =
+  | null
+  | string
+  | number
+  | boolean
+  | readonly JsonSafeJobOutputValue[]
+  | {[key: string]: JsonSafeJobOutputValue};
+
+export function normalizeJobOutputValue(value: unknown, outputKey: string): JsonSafeJobOutputValue {
+  return normalize(value, outputKey, new WeakSet<object>(), 0);
+}
+
+export function jobOutputValueByteLength(value: unknown): number {
+  if (typeof value === 'string') return textEncoder.encode(value).byteLength;
+
+  return jsonByteLength(value);
+}
+
+export function jobOutputRecordEntryByteLength(key: string, value: JsonSafeJobOutputValue): number {
+  return textEncoder.encode(JSON.stringify(key)).byteLength + 1 + jsonByteLength(value);
+}
+
+function jsonByteLength(value: unknown): number {
+  const serialized = JSON.stringify(value);
+  return textEncoder.encode(serialized ?? '').byteLength;
+}
+
+function normalize(
+  value: unknown,
+  outputKey: string,
+  ancestors: WeakSet<object>,
+  depth: number,
+): JsonSafeJobOutputValue {
+  if (value === null) return null;
+
+  switch (typeof value) {
+    case 'string':
+    case 'boolean':
+      return value;
+    case 'number':
+      if (!Number.isFinite(value)) {
+        throw new JobOutputNotJsonSafeError(outputKey, 'numbers must be finite');
+      }
+      return value;
+    case 'bigint': {
+      const numberValue = Number(value);
+      return Number.isSafeInteger(numberValue) ? numberValue : value.toString();
+    }
+    case 'undefined':
+      throw new JobOutputNotJsonSafeError(outputKey, 'undefined is not a JSON value');
+    case 'function':
+    case 'symbol':
+      throw new JobOutputNotJsonSafeError(
+        outputKey,
+        `values of type ${typeof value} are not JSON values`,
+      );
+    case 'object':
+      return normalizeObject(value, outputKey, ancestors, depth);
+  }
+
+  throw new JobOutputNotJsonSafeError(outputKey, 'the value has an unsupported type');
+}
+
+function normalizeObject(
+  value: object,
+  outputKey: string,
+  ancestors: WeakSet<object>,
+  depth: number,
+): JsonSafeJobOutputValue {
+  if (value instanceof Date) {
+    if (Number.isNaN(value.getTime())) {
+      throw new JobOutputNotJsonSafeError(outputKey, 'dates must be valid');
+    }
+    return value.toISOString();
+  }
+
+  if (ancestors.has(value)) {
+    throw new JobOutputNotJsonSafeError(outputKey, 'the value contains a circular reference');
+  }
+
+  if (depth >= MAX_JOB_OUTPUT_NESTING_DEPTH) {
+    throw new JobOutputNotJsonSafeError(
+      outputKey,
+      `values cannot be nested deeper than ${MAX_JOB_OUTPUT_NESTING_DEPTH} levels`,
+    );
+  }
+
+  if (Array.isArray(value)) {
+    ancestors.add(value);
+    try {
+      const normalized: JsonSafeJobOutputValue[] = [];
+      for (const item of value) {
+        normalized.push(normalize(item, outputKey, ancestors, depth + 1));
+      }
+      return normalized;
+    } finally {
+      ancestors.delete(value);
+    }
+  }
+
+  const prototype = Object.getPrototypeOf(value);
+  if (prototype !== Object.prototype && prototype !== null) {
+    throw new JobOutputNotJsonSafeError(outputKey, 'the value has an unsupported object type');
+  }
+
+  ancestors.add(value);
+  try {
+    return Object.fromEntries(
+      Object.keys(value).map((key) => [
+        key,
+        normalize((value as Record<string, unknown>)[key], outputKey, ancestors, depth + 1),
+      ]),
+    ) as {[key: string]: JsonSafeJobOutputValue};
+  } finally {
+    ancestors.delete(value);
+  }
+}