@olivervea/pipelines 2.0.0-beta.1 → 2.0.0-beta.11

package/src/core/webhooks.ts

@@ -0,0 +1,30 @@
+import type { Secret } from "./secrets.ts";
+
+/** How a webhook trigger authenticates deliveries. Build one with `hmacSha256`, `githubSignature` or `bearer`. */
+export type WebhookVerify =
+  | { readonly scheme: "hmac-sha256"; readonly secret: Secret; readonly header: string; readonly prefix: string }
+  | { readonly scheme: "bearer"; readonly secret: Secret };
+
+/** HMAC-SHA256 of the raw body keyed by `secret`, hex, in `header` after `prefix`. */
+export function hmacSha256(o: { secret: Secret; header: string; prefix?: string }): WebhookVerify {
+  return Object.freeze({ scheme: "hmac-sha256", secret: o.secret, header: o.header, prefix: o.prefix ?? "" });
+}
+
+/** GitHub's delivery signature: `X-Hub-Signature-256: sha256=<hex>`. */
+export function githubSignature(secret: Secret): WebhookVerify {
+  return hmacSha256({ secret, header: "X-Hub-Signature-256", prefix: "sha256=" });
+}
+
+/** `Authorization: Bearer <secret>`. */
+export function bearer(secret: Secret): WebhookVerify {
+  return Object.freeze({ scheme: "bearer", secret });
+}
+
+/**
+ * What a webhook trigger mapper receives: the request headers (lower-case names, without `authorization`
+ * and the signature header) and the body (parsed JSON, or the raw text when it isn't JSON).
+ */
+export interface WebhookEvent {
+  readonly headers: Readonly<Record<string, string>>;
+  readonly body: unknown;
+}

package/src/ctx/index.ts

@@ -12,6 +12,8 @@ export interface CtxOptions {
   secrets: readonly Secret[];
   /** Secret values that take precedence over the environment (tests). */
   secretValues?: Readonly<Record<string, string>>;
+  /** Parsed config entries of the step (`ctx.config`). */
+  config?: Readonly<Record<string, unknown>>;
   paths: { root: string; workFiles: string; tmp: string };
   signal: AbortSignal;
   /** Where log lines go (default stderr). */
@@ -102,6 +104,7 @@ export function createCtx(o: CtxOptions): Ctx {
   const filesDeps = { run: o.run, workFiles: o.paths.workFiles, tmpDir: o.paths.tmp, root: o.paths.root, signal: o.signal, log };
   return {
     secrets: createSecrets(o.secrets, o.secretValues) as Ctx["secrets"],
+    config: Object.freeze({ ...o.config }) as Ctx["config"],
     sh,
     exec,
     readText: async (p) => readFileSync(resolve(p), "utf8"),

package/src/ctx/sh.ts

@@ -85,7 +85,8 @@ export function createSh(deps: ShDeps): { sh: Sh; exec: (argv: readonly string[]
     const scriptPath = typeof cmd === "object" ? cmd.script : isEmbeddedScript(cmd) ? cmd : undefined;
     if (scriptPath === undefined) {
       const shell = detectShell(deps.shellCandidates);
-      return { argv: [shell, "-c", cmd as string], what: `command \`${truncate(cmd as string)}\`` };
+      // `sh -c cmd sh a b`: the args are $1, $2… of the command.
+      return { argv: [shell, "-c", cmd as string, "sh", ...args], what: `command \`${truncate(cmd as string)}\`` };
     }
     const file = materialize(scriptPath);
     const first = readFileSync(file, "utf8").split("\n", 1)[0] ?? "";

package/src/ctx/types.ts

@@ -1,5 +1,6 @@
 import type { Skipped } from "../core/handlers.ts";
 import type { FileRef } from "../core/models.ts";
+import type { AnyConfigEntry, ConfigValues } from "../core/config.ts";
 import type { DeclaredSecrets } from "../core/secrets.ts";
 
 /** Facts about the current execution, from the service (see README "Environment"). */
@@ -79,9 +80,11 @@ export interface OutputFileOptions {
   name?: string;
 }
 
-export interface Ctx<Sec extends string = string> {
+export interface Ctx<Sec extends string = string, Cfg extends AnyConfigEntry = never> {
   /** Exactly the secrets the step declares, read from the environment. */
   readonly secrets: DeclaredSecrets<Sec>;
+  /** Exactly the config entries the step declares, from the run's config snapshot (defaults applied). */
+  readonly config: ConfigValues<Cfg>;
   readonly sh: Sh;
   /** Run a binary directly, without a shell (works in shell-less images). Rejects on non-zero exit. */
   exec(argv: readonly string[], opts?: ExecOptions): Promise<void>;

package/src/describe/describe.ts

@@ -1,9 +1,14 @@
 // `describe`: evaluates the contracts (models + pipeline, never handlers) into the manifest and runs
 // every describe-time check. All problems are collected and reported together.
 import { z } from "zod";
+import { CONFIG_NAME, isConfigEntry, SECRET_REF_KIND } from "../core/config.ts";
+import { CONFIG_CASE_KEY, CONFIG_REF_KEY } from "../core/config-refs.ts";
+import { configRefPath, configRefTarget, isConfigRef, unionAt } from "../core/config.ts";
 import { isModel, isOptional, MODEL_NAME, type AnyModel } from "../core/models.ts";
 import { SECRET_NAME, type Secret } from "../core/secrets.ts";
 import { RESERVED_STEP_NAMES, STEP_NAME, isStep, type AnyStep } from "../core/steps.ts";
+import { isImplementation } from "../core/handlers.ts";
+import { recordConfigRefs } from "../core/config-refs.ts";
 import { pipelineData, type MapperFunction, type StageNode, type StageStepNode } from "../builder/pipeline.ts";
 import { FailureContext } from "../library/models.ts";
 import { canonicalize, canonicalJson } from "../util/canonical.ts";
@@ -20,10 +25,12 @@ export class DescribeError extends Error {
 }
 
 export interface DescribeOptions {
-  /** Step names that have a handler file (`handlers/<name>.ts`). */
+  /** Step names that have a handler (a `handlers/*.ts` default-exporting `implement(step, …)`). */
   handlerNames: Iterable<string>;
   /** Step name → handler hash (from the build; includes the harness identity). */
   handlerHashes: Readonly<Record<string, string>>;
+  /** Kind → hash of the code that implements the steps of that kind which carry their implementation (from the build). */
+  kindHashes?: Readonly<Record<string, string>>;
   harness: HarnessInfo;
 }
 
@@ -127,14 +134,46 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
     return s.name;
   };
 
+  // ── config entries ──
+  const configs = new Map<string, { description: string; required: boolean; schema: JsonSchema; key: string; secretRefs: string[] }>();
+  const useConfig = (e: unknown, at: string): string | undefined => {
+    if (!isConfigEntry(e)) return void problems.push(`${at}: config entries must be config("name", "description", schema)`);
+    if (!CONFIG_NAME.test(e.name)) problems.push(`${at}: config name '${e.name}' must match ${CONFIG_NAME}`);
+    let schema: JsonSchema;
+    try {
+      schema = z.toJSONSchema(e.schema, { io: "input", target: "draft-2020-12" }) as JsonSchema;
+      delete schema.$schema;
+      schema = canonicalize(schema) as JsonSchema;
+    } catch (err) {
+      problems.push(`${at}: config '${e.name}' is not representable as JSON Schema (${(err as Error).message})`);
+      return e.name;
+    }
+    const key = JSON.stringify([e.description, schema]);
+    const prev = configs.get(e.name);
+    if (!prev) configs.set(e.name, { description: e.description, required: !e.schema.safeParse(undefined).success, schema, key, secretRefs: secretRefsOf(schema) });
+    else if (prev.key !== key) problems.push(`two different config entries are named '${e.name}' (one description and schema per config name)`);
+    return e.name;
+  };
+
   // ── step identity ──
   const steps = new Map<string, AnyStep>();
-  const registerStep = (s: AnyStep, where: string) => {
+  const placements = new Map<string, { step: AnyStep; cases: readonly CaseOf[] }[]>();
+  // Two places are mutually exclusive when they sit in different cases of the same select.
+  const exclusive = (a: readonly CaseOf[], b: readonly CaseOf[]) => a.some((x) => b.some((y) => y.select === x.select && y.value !== x.value));
+  const registerStep = (s: AnyStep, where: string, cases: readonly CaseOf[] = []) => {
     if (typeof s.name !== "string" || !STEP_NAME.test(s.name))
       problems.push(`${where}: step name '${String(s.name)}' must be lower-case kebab (${STEP_NAME})`);
     if (RESERVED.has(s.name)) problems.push(`${where}: '${s.name}' is reserved and cannot be a step name`);
-    if (steps.has(s.name)) problems.push(`duplicate step name '${s.name}'`);
-    steps.set(s.name, s);
+    const prev = placements.get(s.name) ?? [];
+    if (prev.some((p) => !exclusive(p.cases, cases))) problems.push(`duplicate step name '${s.name}'`);
+    else if (prev.length) {
+      if (prev.some((p) => p.step !== s && !isImplementation(p.step) && !isImplementation(s)))
+        problems.push(`step '${s.name}': two different steps with handler files share the name across select cases; handlers pair by name, so use one step or different names`);
+      if (prev.some((p) => (p.step.output?.name ?? null) !== (s.output?.name ?? null)))
+        problems.push(`step '${s.name}': its select cases must give it the same output model`);
+    }
+    placements.set(s.name, [...prev, { step: s, cases }]);
+    if (!steps.has(s.name)) steps.set(s.name, s);
   };
 
   const inputModel = data.input;
@@ -144,7 +183,7 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
   const outputViewOf = (modelName: string | null): JsonSchema | undefined => (modelName ? models.get(modelName)?.schemas.output : undefined);
 
   // ── one step ──
-  const describeStep = (node: StageStepNode, avail: Map<string, Avail>, failureHandler: boolean): StepDescriptorT => {
+  const describeStep = (node: StageStepNode, avail: Map<string, Avail>, failureHandler: boolean, cases: readonly CaseOf[] = []): StepDescriptorT => {
     const s = node.step;
     const at = `step '${s.name}'`;
     if (typeof s.image !== "string" || s.image.trim() === "") problems.push(`${at}: image must be a non-empty string`);
@@ -194,24 +233,51 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
       }
     }
 
+    for (const key of Object.keys(s.input as Record<string, unknown>))
+      if (key.startsWith("$")) problems.push(`${at}: input '${key}': keys starting with '$' are reserved`);
     const secretNames = s.secrets.map((x) => useSecret(x, at)).sort();
-    let handlerHash: string;
-    let config: Record<string, unknown> | undefined;
-    if (s.kind === "custom") {
-      if (!handlerNames.has(s.name)) problems.push(`${at} has no handler (expected handlers/${s.name}.ts default-exporting implement(${camel(s.name)}, …))`);
-      handlerHash = opts.handlerHashes[s.name] ?? "";
-      if (handlerNames.has(s.name) && !handlerHash) problems.push(`${at}: no handler hash was computed`);
-    } else if (s.builtin) {
+    const configNames = [...new Set((s.config ?? []).map((c) => useConfig(c, at)).filter((n): n is string => n !== undefined))].sort();
+    const carried = isImplementation(s);
+    let settings: Record<string, unknown> | undefined;
+    let settingsProblem: string | undefined;
+    if (s.settings !== undefined || carried) {
       try {
-        config = canonicalize(s.builtin.manifestConfig()) as Record<string, unknown>;
+        settings = canonicalize(recordConfigRefs(s.settings ?? {})) as Record<string, unknown>;
       } catch (e) {
-        problems.push(`${at}: invalid ${s.kind} configuration (${(e as Error).message})`);
-        config = {};
+        settingsProblem = (e as Error).message;
+        settings = {};
       }
-      handlerHash = libraryHandlerHash(s.kind, config, opts.harness);
+    }
+    const refs = settings ? recordedConfigRefs(settings) : [];
+    // A step whose settings reference parts of an entry needs the secrets those parts name; any other
+    // step receiving an entry needs every secret the entry names.
+    for (const name of configNames) {
+      const into = refs.filter((r) => r.ref.split(".")[0] === name);
+      const needed = into.length === 0
+        ? configs.get(name)?.secretRefs ?? []
+        : [...new Set(into.flatMap((r) => secretRefsOf(subSchemas(configs.get(name)?.schema, r.ref.split(".").slice(1), r.case, name))))].sort();
+      for (const ref of needed)
+        if (!secretNames.includes(ref)) problems.push(`${at}: config '${name}' references secret '${ref}' (secretRef), so the step must declare it in secrets`);
+    }
+    let handlerHash: string;
+    if (settingsProblem !== undefined) problems.push(`${at}: invalid ${s.kind} settings (${settingsProblem})`);
+    if (typeof s.kind !== "string" || s.kind === "") problems.push(`${at}: kind must be a non-empty string`);
+    if (carried && s.kind === "custom") problems.push(`${at} carries its implementation, so it needs a kind (step({ kind: "…", … })); a step with kind custom has its handler in handlers/`);
+    for (const { ref, case: kase } of refs) {
+      const [entry, ...path] = ref.split(".") as [string, ...string[]];
+      if (kase !== undefined && !cases.some((c) => c.cond === kase))
+        problems.push(`${at}: config reference '${ref}' is narrowed to the select case '${kase}' and can only be used inside it`);
+      if (!configNames.includes(entry)) problems.push(`${at}: config reference '${ref}': the step does not receive config entry '${entry}'`);
+      else if (!resolveField(configs.get(entry)?.schema, path)) problems.push(`${at}: config reference '${ref}': config '${entry}' has no field '${path.join(".")}'`);
+    }
+    if (carried) {
+      const code = opts.kindHashes?.[s.kind];
+      if (opts.kindHashes && !code) problems.push(`${at}: no hash was computed for the code of kind '${s.kind}'`);
+      handlerHash = libraryHandlerHash(s.kind, settings, opts.harness, code);
     } else {
-      problems.push(`${at}: library step of kind '${s.kind}' has no implementation`);
-      handlerHash = "";
+      if (!handlerNames.has(s.name)) problems.push(`${at} has no handler (add a handlers/*.ts default-exporting implement(${camel(s.name)}, …))`);
+      handlerHash = opts.handlerHashes[s.name] ?? "";
+      if (handlerNames.has(s.name) && !handlerHash) problems.push(`${at}: no handler hash was computed`);
     }
     if (typeof s.description !== "string") problems.push(`${at}: description must be a string (it may be empty)`);
     if (s.timeoutSeconds !== undefined && !(Number.isInteger(s.timeoutSeconds) && s.timeoutSeconds > 0)) problems.push(`${at}: timeoutSeconds must be a positive integer`);
@@ -229,8 +295,11 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
       handlerHash: handlerHash || "sha256:" + "0".repeat(64),
       ...(s.timeoutSeconds !== undefined ? { timeoutSeconds: s.timeoutSeconds } : {}),
       ...(s.retries !== undefined ? { retries: s.retries } : {}),
-      ...(config !== undefined ? { config: config as Record<string, z.core.util.JSONType> } : {}),
+      ...(settings !== undefined ? { config: settings as Record<string, z.core.util.JSONType> } : {}),
+      ...(configNames.length ? { configEntries: configNames } : {}),
+      ...(s.clusterAccess === true ? { clusterAccess: true as const } : {}),
     };
+    if (s.clusterAccess !== undefined && typeof s.clusterAccess !== "boolean") problems.push(`${at}: clusterAccess must be a boolean`);
     if (s.output && !isModel(s.output)) problems.push(`${at}: output must be a model`);
     return out;
   };
@@ -275,17 +344,22 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
   };
 
   // ── stages ──
-  const walk = (nodes: readonly StageNode[], avail: Map<string, Avail>): Stage[] => {
+  let selects = 0;
+  const walk = (nodes: readonly StageNode[], avail: Map<string, Avail>, cases: readonly CaseOf[] = []): Stage[] => {
     const out: Stage[] = [];
     for (const n of nodes) {
+      if (n.kind === "select") {
+        out.push(describeSelect(n, avail, cases, ++selects));
+        continue;
+      }
       if (n.kind === "stage") {
         const descs = n.steps.map((ss) => {
           if (!isStep(ss.step)) {
             problems.push("a stage entry is not a step");
             return undefined;
           }
-          registerStep(ss.step, "stage");
-          return describeStep(ss, avail, false);
+          registerStep(ss.step, "stage", cases);
+          return describeStep(ss, avail, false, cases);
         }).filter((d): d is StepDescriptorT => d !== undefined);
         // Outputs become available only after the barrier (same-stage siblings can't see each other).
         for (const ss of n.steps) if (isStep(ss.step)) avail.set(ss.step.name, { model: ss.step.output?.name ?? null, maybe: false });
@@ -305,13 +379,43 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
           when[field] = values;
         }
         const inner = new Map(avail);
-        const stages = walk(n.nodes, inner);
+        const stages = walk(n.nodes, inner, cases);
         for (const [name, a] of inner) if (!avail.has(name)) avail.set(name, { model: a.model, maybe: true });
         out.push({ when, stages });
       }
     }
     return out;
   };
+  const describeSelect = (n: Extract<StageNode, { kind: "select" }>, avail: Map<string, Avail>, cases: readonly CaseOf[], id: number): Stage => {
+    if (!isConfigRef(n.on)) {
+      problems.push("select: the first argument must be a config reference");
+      return { select: "", discriminator: "", cases: {} };
+    }
+    const ref = configRefPath(n.on);
+    const { entry, case: kase } = configRefTarget(n.on);
+    useConfig(entry, `select '${ref}'`);
+    if (kase !== undefined && !cases.some((c) => c.cond === kase))
+      problems.push(`select '${ref}': the reference is narrowed to the case '${kase}' and can only be used inside it`);
+    const union = unionAt(n.on);
+    if (!union) problems.push(`select '${ref}': it must reference a z.discriminatedUnion in config '${entry.name}'`);
+    else {
+      for (const v of union.values) if (!Object.hasOwn(n.cases, v)) problems.push(`select '${ref}': no case for ${union.discriminator} '${v}'`);
+      for (const v of Object.keys(n.cases)) if (!union.values.includes(v)) problems.push(`select '${ref}': '${v}' is not a value of ${union.discriminator} (${union.values.join(", ")})`);
+    }
+    const discriminator = union?.discriminator ?? "";
+    const branches = Object.entries(n.cases).map(([value, nodes]) => {
+      const inner = new Map(avail);
+      const stages = walk(nodes, inner, [...cases, { select: id, value, cond: `${ref}.${discriminator}=${value}` }]);
+      return { value, inner, stages };
+    });
+    // A step every case has is as available as in its cases; one only some cases have is maybe-available.
+    const added = new Set(branches.flatMap((b) => [...b.inner.keys()].filter((k) => !avail.has(k))));
+    for (const name of added) {
+      const present = branches.map((b) => b.inner.get(name)).filter((a): a is Avail => a !== undefined);
+      avail.set(name, { model: present[0]!.model, maybe: present.length < branches.length || present.some((a) => a.maybe) });
+    }
+    return { select: ref, discriminator, cases: Object.fromEntries(branches.map((b) => [b.value, b.stages])) };
+  };
   const stages = walk(data.nodes, new Map([["input", { model: inputModel.name, maybe: false }]]));
 
   // ── failure handlers ──
@@ -329,19 +433,26 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
     return { step: describeStep({ step: fh.step }, new Map(), true), steps: scope };
   }).filter((x) => x !== undefined);
 
-  // ── handler files without a step ──
+  // ── handlers without a step ──
   for (const h of handlerNames) {
     const s = steps.get(h);
-    if (!s) problems.push(`handlers/${h}.ts does not match any step of the pipeline`);
-    else if (s.kind !== "custom") problems.push(`handlers/${h}.ts: '${h}' is a library step (${s.kind}) and must not have a handler file`);
+    if (!s) problems.push(`a handler implements step '${h}', which is not in the pipeline`);
+    else if (isImplementation(s)) problems.push(`a handler implements '${h}', a ${s.kind} step that carries its own implementation`);
   }
 
   // ── triggers ──
   const triggerNames = new Set<string>();
   const triggers: TriggerT[] = data.triggers.map((t) => {
-    if (triggerNames.has(t.name)) problems.push(`duplicate trigger '${t.name}'${t.kind === "push" ? " (onPush may be used once)" : " (give each onPoll a distinct name)"}`);
+    if (triggerNames.has(t.name)) problems.push(`duplicate trigger '${t.name}'${t.kind === "push" ? " (onPush may be used once)" : " (give each onPoll and onWebhook a distinct name)"}`);
     triggerNames.add(t.name);
     if (t.kind === "push") return { kind: "push", name: "push" };
+    if (t.kind === "webhook") {
+      if (!STEP_NAME.test(t.name)) problems.push(`webhook trigger name '${t.name}' must be lower-case kebab`);
+      const secret = useSecret(t.verify.secret, `webhook '${t.name}'`);
+      if (t.verify.scheme === "bearer") return { kind: "webhook", name: t.name, verify: { scheme: "bearer", secret } };
+      if (!t.verify.header) problems.push(`webhook '${t.name}': the signature header must not be empty`);
+      return { kind: "webhook", name: t.name, verify: { scheme: "hmac-sha256", secret, header: t.verify.header, prefix: t.verify.prefix } };
+    }
     if (!STEP_NAME.test(t.name)) problems.push(`poll trigger name '${t.name}' must be lower-case kebab`);
     if (!(Number.isInteger(t.intervalSeconds) && t.intervalSeconds > 0)) problems.push(`poll '${t.name}': intervalSeconds must be a positive integer`);
     const declared = new Set(t.secrets.map((s) => useSecret(s, `poll '${t.name}'`)));
@@ -362,6 +473,13 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
     harness: opts.harness,
     models: Object.fromEntries([...models].map(([k, v]) => [k, v.schemas])),
     secrets: [...secrets].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([name, description]) => ({ name, description })),
+    ...(configs.size
+      ? {
+          config: [...configs]
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([name, c]) => ({ name, description: c.description, required: c.required, schema: c.schema })),
+        }
+      : {}),
     triggers,
     stages,
     failureHandlers,
@@ -374,6 +492,74 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
 /** Canonical manifest text (what `describe` prints and the service hashes). */
 export const manifestText = (m: ManifestT): string => canonicalJson(m);
 
+/**
+ * The schemas at `path` inside an entry's schema: every union branch that has the field, except where
+ * `kase` (`<entry>.<path>.<discriminator>=<value>`) narrows a union to one branch.
+ */
+export function subSchemas(schema: unknown, path: readonly string[], kase: string | undefined, entry: string): unknown[] {
+  let narrow: { at: string; discriminator: string; value: string } | undefined;
+  if (kase) {
+    const eq = kase.lastIndexOf("=");
+    const left = kase.slice(0, eq);
+    const dot = left.lastIndexOf(".");
+    narrow = { at: left.slice(0, dot), discriminator: left.slice(dot + 1), value: kase.slice(eq + 1) };
+  }
+  const walk = (node: unknown, rest: readonly string[], here: string): unknown[] => {
+    if (node === null || typeof node !== "object") return [];
+    const o = node as JsonSchema;
+    const branches = (o.oneOf ?? o.anyOf) as JsonSchema[] | undefined;
+    if (Array.isArray(branches)) {
+      const chosen = narrow && narrow.at === here
+        ? branches.filter((b) => ((b.properties as Record<string, JsonSchema> | undefined)?.[narrow!.discriminator] as JsonSchema | undefined)?.const === narrow!.value)
+        : branches;
+      return chosen.flatMap((b) => walk(b, rest, here));
+    }
+    if (rest.length === 0) return [node];
+    const props = o.properties as Record<string, JsonSchema> | undefined;
+    if (props && Object.hasOwn(props, rest[0]!)) return walk(props[rest[0]!], rest.slice(1), `${here}.${rest[0]}`);
+    return [];
+  };
+  return walk(schema, path, entry);
+}
+
+/** Secret names of every secretRef inside a config entry's JSON Schema. */
+export function secretRefsOf(schema: unknown): string[] {
+  const out = new Set<string>();
+  const walk = (v: unknown) => {
+    if (Array.isArray(v)) return v.forEach(walk);
+    if (v === null || typeof v !== "object") return;
+    const o = v as JsonSchema;
+    if (o["x-olve-kind"] === SECRET_REF_KIND) {
+      const name = (o.properties as { secret?: { const?: unknown } } | undefined)?.secret?.const;
+      if (typeof name === "string") out.add(name);
+    }
+    Object.values(o).forEach(walk);
+  };
+  walk(schema);
+  return [...out].sort();
+}
+
+/** The `{"$config": "<entry>.<path>"[, "$case": …]}` references in a step's recorded settings. */
+export function recordedConfigRefs(v: unknown, out: { ref: string; case?: string }[] = []): { ref: string; case?: string }[] {
+  if (Array.isArray(v)) v.forEach((x) => recordedConfigRefs(x, out));
+  else if (v !== null && typeof v === "object") {
+    const o = v as Record<string, unknown>;
+    const keys = Object.keys(o).sort();
+    const isRef = typeof o[CONFIG_REF_KEY] === "string"
+      && (keys.length === 1 ? keys[0] === CONFIG_REF_KEY : keys.length === 2 && keys[0] === CONFIG_CASE_KEY && keys[1] === CONFIG_REF_KEY && typeof o[CONFIG_CASE_KEY] === "string");
+    if (isRef) out.push({ ref: o[CONFIG_REF_KEY] as string, ...(keys.length === 2 ? { case: o[CONFIG_CASE_KEY] as string } : {}) });
+    else Object.values(o).forEach((x) => recordedConfigRefs(x, out));
+  }
+  return out;
+}
+
+/** A select case a step sits in: which select (numbered in describe order), which value, and its condition text. */
+interface CaseOf {
+  select: number;
+  value: string;
+  cond: string;
+}
+
 function camel(name: string): string {
   return name.replace(/-([a-z0-9])/g, (_, c: string) => c.toUpperCase());
 }

package/src/describe/manifest-schema.ts

@@ -7,6 +7,7 @@ export const API_VERSION = "2.0";
 const stepName = z.string().regex(/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/).describe("Step name: lower-case kebab, 1-63 chars. The step's identity.");
 const modelName = z.string().regex(/^[A-Za-z][A-Za-z0-9_]*$/).describe("Model name (key of `models`).");
 const secretName = z.string().regex(/^[A-Z_][A-Z0-9_]*$/).describe("Secret name; also the env var name in the step container.");
+const configName = z.string().regex(/^[a-z][a-zA-Z0-9]*$/).describe("Config entry name; a key of the pipeline's config document.");
 const hash = z.string().regex(/^sha256:[0-9a-f]{64}$/);
 const Json = z.json().meta({ id: "Json", description: "Any JSON value." });
 const jsonSchema = z.record(z.string(), z.unknown()).describe("A JSON Schema (draft 2020-12) object, canonical (sorted keys, no $schema).");
@@ -50,20 +51,34 @@ export const StepDescriptor = z
   .strictObject({
     name: stepName,
     description: z.string().describe("What the step does, shown in the UI (may be empty)."),
-    kind: z.enum(["custom", "script", "gitHubSource", "imageBuild", "helmDeployOverSsh"]).describe("'custom' = handlers/<name>.ts; others are SDK library steps."),
+    kind: z.string().min(1).describe("'custom' = the handler is a handlers/*.ts file; any other kind is a step that carries its implementation (an SDK or package step). A label: the service accepts any kind."),
     image: z.string().min(1).describe("Container image the step runs in."),
     inputTypes: z.record(z.string(), InputType).describe("Declared input entries, keyed like input.json."),
     input: z.record(z.string(), Mapping).describe("How the service assembles input.json: one mapping per input key (same keys as inputTypes)."),
     output: modelName.nullable().describe("Output model, or null if the step produces no output."),
     secrets: z.array(secretName).describe("Declared secrets (sorted). Mount exactly these, as env vars of the same name."),
-    handlerHash: hash.describe("Implementation identity: handler bundle (contract files excluded) + harness (bun + sdkHash); for library steps kind + config + harness."),
+    handlerHash: hash.describe("Implementation identity: handler bundle (contract files excluded) + harness (bun + sdkHash); for a step that carries its implementation kind + config + harness + the hash of the code of its kind."),
     timeoutSeconds: z.number().int().positive().optional(),
     retries: z.number().int().nonnegative().optional(),
-    config: z.record(z.string(), Json).optional().describe("Library-step configuration (display only; already covered by handlerHash)."),
+    config: z
+      .record(z.string(), Json)
+      .optional()
+      .describe("The step's settings (display only; already covered by handlerHash). A config reference is recorded as {\"$config\": \"<entry>.<field>…\"}; the entry is in configEntries."),
+    configEntries: z
+      .array(configName)
+      .optional()
+      .describe("Config entries the step receives (sorted; omitted when none). The service delivers them in input.json under '$config'."),
+    clusterAccess: z
+      .literal(true)
+      .optional()
+      .describe("The step pod gets the instance's step ServiceAccount token (Kubernetes:StepServiceAccount), if the instance grants one. Omitted = no token."),
   })
   .meta({ id: "Step" });
 
-export type Stage = { steps: z.infer<typeof StepDescriptor>[] } | { when: Record<string, (string | number | boolean)[]>; stages: Stage[] };
+export type Stage =
+  | { steps: z.infer<typeof StepDescriptor>[] }
+  | { when: Record<string, (string | number | boolean)[]>; stages: Stage[] }
+  | { select: string; discriminator: string; cases: Record<string, Stage[]> };
 export const Stage: z.ZodType<Stage> = z
   .union([
     z.strictObject({ steps: z.array(StepDescriptor).min(1).describe("Steps run in parallel; the stage is a barrier.") }),
@@ -73,6 +88,13 @@ export const Stage: z.ZodType<Stage> = z
       ),
       stages: z.array(z.lazy(() => Stage)),
     }),
+    z.strictObject({
+      select: z.string().min(1).describe("Config reference (`<entry>.<field>…`) to a discriminated union; the run's config snapshot picks the case."),
+      discriminator: z.string().min(1).describe("The union's literal field whose value names the case."),
+      cases: z
+        .record(z.string(), z.array(z.lazy(() => Stage)))
+        .describe("Discriminator value → stages. Exactly one case runs; a step name may repeat across cases, nowhere else."),
+    }),
   ])
   .meta({ id: "Stage" });
 
@@ -90,6 +112,21 @@ export const Trigger = z
         intervalSeconds: z.number().int().positive(),
       })
       .describe("Background poller."),
+    z
+      .strictObject({
+        kind: z.literal("webhook"),
+        name: z.string().regex(/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/),
+        verify: z.discriminatedUnion("scheme", [
+          z.strictObject({
+            scheme: z.literal("hmac-sha256"),
+            secret: secretName,
+            header: z.string().min(1),
+            prefix: z.string().describe("Text before the hex digest in the header, e.g. 'sha256='."),
+          }),
+          z.strictObject({ scheme: z.literal("bearer"), secret: secretName }),
+        ]),
+      })
+      .describe("Inbound HTTP delivery to /api/hooks/<pipeline id>/<name>. Event = {headers, body}; a null mapping ignores it."),
   ])
   .meta({ id: "Trigger" });
 
@@ -102,12 +139,23 @@ export const Manifest = z
     harness: z
       .strictObject({
         bun: z.string().describe("Bun runtime version compiled into the binary."),
-        sdk: z.string().describe("@olve/pipelines package version."),
+        sdk: z.string().describe("@olivervea/pipelines package version."),
         sdkHash: hash.describe("Hash of the SDK runtime sources + zod version."),
       })
       .describe("Harness identity. Already folded into every handlerHash."),
     models: z.record(modelName, ModelSchemas).describe("Every model referenced by the pipeline (input, step inputs/outputs)."),
     secrets: z.array(z.strictObject({ name: secretName, description: z.string() })).describe("All declared secrets (steps + poll triggers), sorted by name."),
+    config: z
+      .array(
+        z.strictObject({
+          name: configName,
+          description: z.string(),
+          required: z.boolean().describe("False when the schema accepts a missing value (optional, or a default)."),
+          schema: jsonSchema.describe("z.input view: validates the config document's value for this entry (defaults listed)."),
+        }),
+      )
+      .optional()
+      .describe("Config entries used by steps, sorted by name (omitted when none). Fields of kind secretRef ({secret: NAME}) name a secret."),
     triggers: z.array(Trigger),
     stages: z.array(Stage),
     failureHandlers: z.array(

package/src/dev/cli.ts

@@ -1,31 +1,31 @@
 #!/usr/bin/env bun
 // olve-dev — run a pipeline from source through the same harness path as the compiled binary.
 //   olve-dev describe <dir> [--sentinels]
-//   olve-dev run <dir> <step> [--input-file f] [--olve-dir d] [--secret NAME=value]...
+//   olve-dev run <dir> <step> [--input-file f] [--config-file f] [--olve-dir d] [--secret NAME=value]...
 //   olve-dev map-trigger <dir> <trigger> --event-file f [--olve-dir d]
 // $OLVE_DIR defaults to <dir>/.olve; outputs land in <dir>/.olve/work/.
-import { copyFileSync, mkdirSync } from "node:fs";
+import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { parseArgs } from "node:util";
 import { runHarness, type HarnessConfig } from "../harness/main.ts";
 import { EXIT } from "../harness/exit.ts";
-import { handlerHashes } from "../build/hashes.ts";
-import { harnessInfo, loadProject, ProjectError } from "../build/project.ts";
+import { handlerHashes, kindHashes } from "../build/hashes.ts";
+import { harnessInfo, loadPipeline, loadProject, ProjectError } from "../build/project.ts";
 
 const USAGE = `usage: olve-dev describe <dir> [--sentinels]
-       olve-dev run <dir> <step> [--input-file f] [--olve-dir d] [--secret NAME=value]...
+       olve-dev run <dir> <step> [--input-file f] [--config-file f] [--olve-dir d] [--secret NAME=value]...
        olve-dev map-trigger <dir> <trigger> --event-file f [--olve-dir d]`;
 
 export async function devConfig(dir: string, withHashes: boolean): Promise<HarnessConfig> {
-  const project = loadProject(dir);
-  const mod = (await import(project.pipelineFile)) as { default?: unknown };
-  if (!mod.default) throw new ProjectError(`${project.pipelineFile} has no default export (export default pipeline({...})...)`);
+  const project = await loadProject(dir);
+  const pipeline = await loadPipeline(project);
   const harness = harnessInfo();
   const hashes = withHashes ? await handlerHashes(project, harness) : {};
   return {
-    pipeline: mod.default,
+    pipeline,
     handlers: Object.fromEntries(project.handlers.map(([name, file]) => [name, () => import(file) as Promise<{ default: unknown }>])),
     handlerHashes: Object.fromEntries(Object.entries(hashes).map(([k, v]) => [k, v.handlerHash])),
+    ...(withHashes ? { kindHashes: await kindHashes(project, pipeline) } : {}),
     harness,
   };
 }
@@ -38,6 +38,7 @@ export async function devMain(argv: string[]): Promise<number> {
       allowPositionals: true,
       options: {
         "input-file": { type: "string" },
+        "config-file": { type: "string" },
         "event-file": { type: "string" },
         "olve-dir": { type: "string" },
         secret: { type: "string", multiple: true },
@@ -68,6 +69,12 @@ export async function devMain(argv: string[]): Promise<number> {
     }
     const file = mode === "run" ? values["input-file"] : values["event-file"];
     if (file) copyFileSync(resolve(file), join(olveDir, "input.json"));
+    if (mode === "run" && values["config-file"]) {
+      const inputFile = join(olveDir, "input.json");
+      const input = (existsSync(inputFile) ? JSON.parse(readFileSync(inputFile, "utf8")) : {}) as Record<string, unknown>;
+      input.$config = JSON.parse(readFileSync(resolve(values["config-file"]), "utf8"));
+      writeFileSync(inputFile, JSON.stringify(input));
+    }
     else if (mode === "map-trigger") return console.error("--event-file is required"), EXIT.usage;
     return await runHarness(await devConfig(dir, false), [mode, name]);
   } catch (e) {

package/src/harness/execute.ts

@@ -8,7 +8,7 @@ import type { Ctx, RunInfo } from "../ctx/types.ts";
 import { objectPath } from "../ctx/files.ts";
 import { sha256File } from "../util/hash.ts";
 import { EXIT } from "./exit.ts";
-import { validateInput, validateOutput } from "./validate.ts";
+import { validateConfig, validateInput, validateOutput } from "./validate.ts";
 
 export type FailureKind = "handler" | "input" | "output";
 export type Envelope =
@@ -58,13 +58,16 @@ export async function executeStep(args: {
   workFiles: string;
   log: (line: string) => void;
 }): Promise<Execution> {
-  const { step, ctx, log } = args;
+  const { step, log } = args;
   const input = validateInput(step, args.rawInput);
-  if (!input.ok) {
-    const message = `step '${step.name}': input does not match the step's inputs:\n  ${input.issues.join("\n  ")}`;
+  const config = validateConfig(step, args.rawInput);
+  if (!input.ok || !config.ok) {
+    const issues = [...(input.ok ? [] : input.issues), ...(config.ok ? [] : config.issues)];
+    const message = `step '${step.name}': input does not match the step's inputs:\n  ${issues.join("\n  ")}`;
     log(message);
     return { envelope: { status: "failed", error: { kind: "input", message } }, exitCode: EXIT.inputInvalid };
   }
+  const ctx: Ctx = { ...args.ctx, config: Object.freeze(config.value) as Ctx["config"] };
 
   let value: unknown;
   try {

package/src/harness/index.ts

@@ -1,4 +1,4 @@
-// @olve/pipelines/harness — the runtime the build tool compiles into the step binary.
+// @olivervea/pipelines/harness — the runtime the build tool compiles into the step binary.
 export { runHarness, defaultIO, allSteps, resolveStepFunction, type HarnessConfig, type HandlerRegistry, type HarnessIO } from "./main.ts";
 export { EXIT } from "./exit.ts";
 export { ENV, paths, runInfoFromEnv } from "./env.ts";