@olivervea/pipelines 2.0.0-beta.3 → 2.0.0-beta.30

package/src/describe/schemas.ts

@@ -0,0 +1,71 @@
+import { z } from 'zod';
+import { canonicalize } from '../util/canonical.ts';
+import type { ModelSchemasT } from './manifest-schema.ts';
+
+export type JsonSchema = Record<string, unknown>;
+
+function stripStrict(v: unknown): unknown {
+  if (Array.isArray(v)) return v.map(stripStrict);
+  if (v !== null && typeof v === 'object') {
+    const out: Record<string, unknown> = {};
+    for (const [k, x] of Object.entries(v))
+      if (!(k === 'additionalProperties' && x === false)) out[k] = stripStrict(x);
+    return out;
+  }
+  return v;
+}
+
+/** JSON Schema views of a zod schema. Throws if it is not representable (dates, transforms, …). */
+export function schemaViews(schema: z.ZodType): ModelSchemasT {
+  const view = (io: 'input' | 'output') => {
+    const js = z.toJSONSchema(schema, { io, target: 'draft-2020-12' }) as JsonSchema;
+    delete js.$schema; // constant noise; apiVersion pins the dialect
+    return canonicalize(js) as JsonSchema;
+  };
+  const output = view('output');
+  const input = view('input');
+  const same = JSON.stringify(stripStrict(output)) === JSON.stringify(input);
+  return same ? { output } : { output, input };
+}
+
+/** Whether `segments` lead to a field of a JSON Schema. */
+export function resolveField(
+  schema: JsonSchema | undefined,
+  segments: readonly string[],
+): boolean {
+  if (!schema) return false;
+  if (segments.length === 0) return true;
+  const [seg, ...rest] = segments as [string, ...string[]];
+  if (Array.isArray(schema.anyOf))
+    return (schema.anyOf as JsonSchema[]).some((b) => resolveField(b, segments));
+  if (Array.isArray(schema.oneOf))
+    return (schema.oneOf as JsonSchema[]).some((b) => resolveField(b, segments));
+  if (Array.isArray(schema.allOf))
+    return (schema.allOf as JsonSchema[]).some((b) => resolveField(b, segments));
+  const props = schema.properties as Record<string, JsonSchema> | undefined;
+  if (props && Object.hasOwn(props, seg)) return resolveField(props[seg], rest);
+  const extra = schema.additionalProperties;
+  if (extra && typeof extra === 'object') return resolveField(extra as JsonSchema, rest);
+  // A schema without type/properties/anyOf accepts anything (z.unknown / z.any).
+  return Object.keys(schema).every(
+    (k) => k === 'description' || k === 'title' || k === 'default',
+  );
+}
+
+export function literalValues(field: JsonSchema | undefined): unknown[] | undefined {
+  if (!field) return undefined;
+  if (Array.isArray(field.enum)) return field.enum as unknown[];
+  if (field.const !== undefined) return [field.const];
+  if (field.type === 'boolean') return [true, false];
+  const alts = (field.anyOf ?? field.oneOf) as JsonSchema[] | undefined;
+  if (Array.isArray(alts)) {
+    const all: unknown[] = [];
+    for (const a of alts) {
+      const l = literalValues(a);
+      if (!l) return undefined;
+      all.push(...l);
+    }
+    return all;
+  }
+  return undefined;
+}

package/src/describe/stages.ts

@@ -0,0 +1,166 @@
+import { configRefPath, configRefTarget, isConfigRef, unionAt } from '../core/config.ts';
+import { isStep } from '../core/steps.ts';
+import type { StageNode } from '../builder/pipeline.ts';
+import type { Stage, StepDescriptorT } from './manifest-schema.ts';
+import { literalValues, type JsonSchema } from './schemas.ts';
+import type { Avail, CaseOf, DescribeContext } from './context.ts';
+import { describeStep } from './step.ts';
+
+type StageGroup = Extract<StageNode, { kind: 'stage' }>;
+type WhenNode = Extract<StageNode, { kind: 'when' }>;
+type SelectNode = Extract<StageNode, { kind: 'select' }>;
+
+/** Describes stages, `when` blocks and selects in order, adding what each makes available to `avail`. */
+export function describeStages(
+  ctx: DescribeContext,
+  input: { name: string },
+  nodes: readonly StageNode[],
+  avail: Map<string, Avail>,
+  cases: readonly CaseOf[] = [],
+): Stage[] {
+  const out: Stage[] = [];
+  for (const n of nodes) {
+    if (n.kind === 'select') {
+      out.push(describeSelect(ctx, input, n, avail, cases, ctx.nextSelectId()));
+      continue;
+    }
+    if (n.kind === 'stage') {
+      const stage = describeStage(ctx, n, avail, cases);
+      if (stage) out.push(stage);
+    } else out.push(describeWhen(ctx, input, n, avail, cases));
+  }
+  return out;
+}
+
+function describeStage(
+  ctx: DescribeContext,
+  n: StageGroup,
+  avail: Map<string, Avail>,
+  cases: readonly CaseOf[],
+): Stage | undefined {
+  const descs = n.steps
+    .map((ss) => {
+      if (!isStep(ss.step)) {
+        ctx.problems.push('a stage entry is not a step');
+        return undefined;
+      }
+      ctx.registerStep(ss.step, 'stage', cases);
+      return describeStep(ctx, ss, avail, false, cases);
+    })
+    .filter((d): d is StepDescriptorT => d !== undefined);
+  // Outputs become available only after the barrier (same-stage siblings can't see each other).
+  for (const ss of n.steps)
+    if (isStep(ss.step))
+      avail.set(ss.step.name, {
+        model: ss.step.output?.name ?? null,
+        maybe: false,
+        ...(ss.step.matrix ? { matrix: true } : {}),
+      });
+  return descs.length ? { steps: descs } : undefined;
+}
+
+function describeWhen(
+  ctx: DescribeContext,
+  input: { name: string },
+  n: WhenNode,
+  avail: Map<string, Avail>,
+  cases: readonly CaseOf[],
+): Stage {
+  const problems = ctx.problems;
+  const when: Record<string, (string | number | boolean)[]> = {};
+  const props = (ctx.models.get(input.name)?.schemas.output.properties ?? {}) as Record<
+    string,
+    JsonSchema
+  >;
+  for (const [field, raw] of Object.entries(n.cond)) {
+    const values = (Array.isArray(raw) ? raw : [raw]) as (string | number | boolean)[];
+    if (!Object.hasOwn(props, field))
+      problems.push(
+        `when: '${field}' is not a field of the pipeline input ${input.name}`,
+      );
+    else {
+      const literals = literalValues(props[field]);
+      if (!literals)
+        problems.push(
+          `when: '${field}' is not a literal-like field (enum, boolean or literal)`,
+        );
+      else
+        for (const v of values)
+          if (!literals.includes(v))
+            problems.push(
+              `when: ${JSON.stringify(v)} is not a valid value of '${field}' (${literals.map((l) => JSON.stringify(l)).join(', ')})`,
+            );
+    }
+    if (values.length === 0) problems.push(`when: '${field}' has no values`);
+    when[field] = values;
+  }
+  const inner = new Map(avail);
+  const stages = describeStages(ctx, input, n.nodes, inner, cases);
+  for (const [name, a] of inner)
+    if (!avail.has(name)) avail.set(name, { ...a, maybe: true });
+  return { when, stages };
+}
+
+function describeSelect(
+  ctx: DescribeContext,
+  input: { name: string },
+  n: SelectNode,
+  avail: Map<string, Avail>,
+  cases: readonly CaseOf[],
+  id: number,
+): Stage {
+  const problems = ctx.problems;
+  if (!isConfigRef(n.on)) {
+    problems.push('select: the first argument must be a config reference');
+    return { select: '', discriminator: '', cases: {} };
+  }
+  const ref = configRefPath(n.on);
+  const { entry, case: kase } = configRefTarget(n.on);
+  ctx.useConfig(entry, `select '${ref}'`);
+  if (kase !== undefined && !cases.some((c) => c.cond === kase))
+    problems.push(
+      `select '${ref}': the reference is narrowed to the case '${kase}' and can only be used inside it`,
+    );
+  const union = unionAt(n.on);
+  if (!union)
+    problems.push(
+      `select '${ref}': it must reference a z.discriminatedUnion in config '${entry.name}'`,
+    );
+  else {
+    for (const v of union.values)
+      if (!Object.hasOwn(n.cases, v))
+        problems.push(`select '${ref}': no case for ${union.discriminator} '${v}'`);
+    for (const v of Object.keys(n.cases))
+      if (!union.values.includes(v))
+        problems.push(
+          `select '${ref}': '${v}' is not a value of ${union.discriminator} (${union.values.join(', ')})`,
+        );
+  }
+  const discriminator = union?.discriminator ?? '';
+  const branches = Object.entries(n.cases).map(([value, nodes]) => {
+    const inner = new Map(avail);
+    const stages = describeStages(ctx, input, nodes, inner, [
+      ...cases,
+      { select: id, value, cond: `${ref}.${discriminator}=${value}` },
+    ]);
+    return { value, inner, stages };
+  });
+  // A step every case has is as available as in its cases; one only some cases have is maybe-available.
+  const added = new Set(
+    branches.flatMap((b) => [...b.inner.keys()].filter((k) => !avail.has(k))),
+  );
+  for (const name of added) {
+    const present = branches
+      .map((b) => b.inner.get(name))
+      .filter((a): a is Avail => a !== undefined);
+    avail.set(name, {
+      ...present[0]!,
+      maybe: present.length < branches.length || present.some((a) => a.maybe),
+    });
+  }
+  return {
+    select: ref,
+    discriminator,
+    cases: Object.fromEntries(branches.map((b) => [b.value, b.stages])),
+  };
+}