@olivervea/pipelines 2.0.0-beta.2 → 2.0.0-beta.20

package/src/core/steps.ts

@@ -1,29 +1,46 @@
-import type { z } from "zod";
-import { STEP, type StepBrand } from "./brands.ts";
-import type { AnyModel, Infer, Optional } from "./models.ts";
-import type { Secret } from "./secrets.ts";
-import type { Simplify } from "./types.ts";
+import type { z } from 'zod';
+import { ORIGIN, STEP, type StepBrand } from './brands.ts';
+import { callerFile } from './origin.ts';
+import type { AnyModel, Infer, Optional } from './models.ts';
+import type { External } from './external.ts';
+import type { AnyConfigEntry } from './config.ts';
+import { configRefsIn } from './config-refs.ts';
+import type { Secret } from './secrets.ts';
+import type { Simplify } from './types.ts';
 
-/** A step input entry: a model (matchable by model), `optional(model)`, or a plain zod schema (mapper required). */
-export type InputEntry = AnyModel | Optional | z.ZodType;
+/**
+ * A step input entry: a model (matchable by model), `optional(model)`, a plain zod schema (mapper required),
+ * or `external(model, …)`, answered from outside the run.
+ */
+export type InputEntry = AnyModel | Optional | External | z.ZodType;
 export type InputSpec = { readonly [key: string]: InputEntry };
 
 export type InferEntry<E> =
-  E extends Optional<infer M> ? Infer<M> | undefined
-  : E extends AnyModel ? Infer<E>
-  : E extends z.ZodType ? z.output<E>
-  : never;
+  E extends Optional<infer M>
+    ? Infer<M> | undefined
+    : E extends External<infer M>
+      ? Infer<M>
+      : E extends AnyModel
+        ? Infer<E>
+        : E extends z.ZodType
+          ? z.output<E>
+          : never;
 /** What a handler receives: the output view of every input entry. */
 export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
 
-/** `custom` steps have a handler file; the others are library steps implemented by the SDK. */
-export type StepKind = "custom" | "script" | "gitHubSource" | "imageBuild" | "helmDeployOverSsh";
+/**
+ * A label for what a step is, shown in the UI. `custom` (the default) is a step whose handler is in
+ * `handlers/`; a step that carries its implementation (`implement(step, …)` staged directly) names its
+ * kind, e.g. `gitHubSource`. Packages prefix theirs (`acme/deploy`).
+ */
+export type StepKind = string;
 
 export interface Step<
   N extends string = string,
   I extends InputSpec = InputSpec,
   O extends AnyModel | null = AnyModel | null,
   Sec extends string = string,
+  Cfg extends AnyConfigEntry = never,
 > {
   readonly [StepBrand]: true;
   readonly kind: StepKind;
@@ -33,35 +50,41 @@ export interface Step<
   readonly input: I;
   readonly output: O;
   readonly secrets: readonly Secret<Sec>[];
+  /** Config entries the step receives as `ctx.config`. */
+  readonly config: readonly Cfg[];
   readonly timeoutSeconds?: number;
   readonly retries?: number;
-  /** Library steps carry their own implementation and config. */
-  readonly builtin?: LibraryImplementation;
+  /** The step pod gets the instance's step ServiceAccount token (`Kubernetes:StepServiceAccount`). */
+  readonly clusterAccess?: boolean;
+  /** How the step is set up, as written in the definition (may hold config references). */
+  readonly settings?: Readonly<Record<string, unknown>>;
 }
-/** Runtime part of a library step (not part of the public contract). */
-export interface LibraryImplementation {
-  /** Config as emitted in the manifest (JSON only; must not contain paths). */
-  readonly manifestConfig: () => Record<string, unknown>;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  readonly run: (input: any, ctx: any) => Promise<unknown>;
-}
-export type LibraryStep = { readonly builtin: LibraryImplementation };
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export type AnyStep = Step<string, any, AnyModel | null, any>;
+export type AnyStep = Step<string, any, AnyModel | null, any, any>;
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export type SecretsOf<S> = S extends Step<any, any, any, infer Sec> ? Sec : never;
-export type OutputValue<S extends AnyStep> = S["output"] extends AnyModel ? Infer<S["output"]> : void;
+export type SecretsOf<S> = S extends Step<any, any, any, infer Sec, any> ? Sec : never;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type ConfigOf<S> = S extends Step<any, any, any, any, infer C> ? C : never;
+export type OutputValue<S extends AnyStep> = S['output'] extends AnyModel
+  ? Infer<S['output']>
+  : void;
 
 /** Step names must be literals (identity, duplicate detection and mapper keys depend on it). */
-export type LiteralName<N extends string> = N & (string extends N ? "❌ step name must be a string literal" : unknown);
+export type LiteralName<N extends string> = N &
+  (string extends N ? '❌ step name must be a string literal' : unknown);
 
-/** Step names: lower-case kebab, ≤ 63 chars (usable as K8s label values and URL segments). */
-export const STEP_NAME = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
+export { STEP_NAME } from './models.ts';
 /** Names that cannot be steps: `input` is the pipeline input's root in mappings. */
-export const RESERVED_STEP_NAMES: readonly string[] = ["input"];
+export const RESERVED_STEP_NAMES: readonly string[] = ['input'];
 
-export interface StepDef<N extends string, I extends InputSpec, O extends AnyModel | null, Sec extends readonly Secret[]> {
+export interface StepDef<
+  N extends string,
+  I extends InputSpec,
+  O extends AnyModel | null,
+  Sec extends readonly Secret[],
+  Cfg extends readonly AnyConfigEntry[] = [],
+> {
   name: LiteralName<N>;
   /** What the step does, shown in the UI. Required; may be empty. */
   description: string;
@@ -70,38 +93,66 @@ export interface StepDef<N extends string, I extends InputSpec, O extends AnyMod
   input?: I;
   output?: O;
   secrets?: Sec;
+  /** Config entries delivered as `ctx.config` (typed to exactly these). */
+  config?: Cfg;
   timeoutSeconds?: number;
   /** Automatic retries (an idempotency claim). Default 0. */
   retries?: number;
+  /**
+   * Ask for a Kubernetes API token in the step pod: the instance's step ServiceAccount
+   * (`Kubernetes:StepServiceAccount`), if it grants one. Default false (no token).
+   */
+  clusterAccess?: boolean;
+  /** A label for the step's kind (default `custom`). */
+  kind?: StepKind;
+  /**
+   * How the step is set up, shown in the manifest and part of the step's identity: the parameters of
+   * a reusable step. JSON values and config references (`target.prod.host`); every config entry it
+   * references is delivered to the step, and `resolveConfigRefs(value, ctx.config)` resolves them.
+   */
+  settings?: Readonly<Record<string, unknown>>;
 }
 
-export function makeStep(kind: StepKind, def: Record<string, unknown>, builtin?: LibraryImplementation): AnyStep {
+function makeStep(def: Record<string, unknown>, origin: string | undefined): AnyStep {
+  const config = new Map<string, AnyConfigEntry>();
+  for (const e of (def.config as readonly AnyConfigEntry[] | undefined) ?? [])
+    if (!config.has(e.name)) config.set(e.name, e);
+  configRefsIn(def.settings, config);
   const s = {
     [STEP]: true,
-    kind,
+    kind: def.kind ?? 'custom',
     name: def.name,
     description: def.description,
     image: def.image,
     input: Object.freeze({ ...((def.input as object | undefined) ?? {}) }),
     output: def.output ?? null,
     secrets: Object.freeze([...((def.secrets as readonly Secret[] | undefined) ?? [])]),
+    config: Object.freeze([...config.values()]),
     ...(def.timeoutSeconds !== undefined ? { timeoutSeconds: def.timeoutSeconds } : {}),
     ...(def.retries !== undefined ? { retries: def.retries } : {}),
-    ...(builtin ? { builtin } : {}),
+    ...(def.clusterAccess === true ? { clusterAccess: true } : {}),
+    ...(def.settings !== undefined ? { settings: def.settings } : {}),
+    [ORIGIN]: origin === undefined ? [] : [origin],
   };
   return Object.freeze(s) as unknown as AnyStep;
 }
 
 export function isStep(v: unknown): v is AnyStep {
-  return typeof v === "object" && v !== null && (v as Record<symbol, unknown>)[STEP] === true;
+  return (
+    typeof v === 'object' && v !== null && (v as Record<symbol, unknown>)[STEP] === true
+  );
 }
 
-/** Define a custom step: a contract (name, image, inputs, output, secrets). Its handler lives in `handlers/<name>.ts`. */
+/**
+ * Define a step: a contract (name, image, inputs, output, secrets). Its handler is a `handlers/*.ts`
+ * default-exporting `implement(step, …)`, or, for a reusable step, `implement(step, …)` staged directly.
+ */
 export function step<
   const N extends string,
   const I extends InputSpec = {},
   O extends AnyModel | null = null,
   const Sec extends readonly Secret[] = [],
->(def: StepDef<N, I, O, Sec>): Step<N, I, O, Sec[number]["name"]> {
-  return makeStep("custom", def as never) as never;
+  const Cfg extends readonly AnyConfigEntry[] = [],
+>(def: StepDef<N, I, O, Sec, Cfg>): Step<N, I, O, Sec[number]['name'], Cfg[number]> {
+  return makeStep(def as never, callerFile()) as never;
 }

package/src/core/types.ts

@@ -1,6 +1,10 @@
 export type Primitive = string | number | boolean | bigint | null | undefined;
 export type Simplify<T> = { [K in keyof T]: T[K] } & {};
 export type Awaitable<T> = T | Promise<T>;
-export type IsUnion<T, U = T> = T extends unknown ? ([U] extends [T] ? false : true) : never;
+export type IsUnion<T, U = T> = T extends unknown
+  ? [U] extends [T]
+    ? false
+    : true
+  : never;
 /** `Ok` when there is no error, else the error (a readable string-literal type). */
 export type OrError<Err, Ok> = [Err] extends [never] ? Ok : Err;

package/src/core/webhooks.ts

@@ -1,23 +1,37 @@
-import type { Secret } from "./secrets.ts";
+import type { Secret } from './secrets.ts';
 
 /** How a webhook trigger authenticates deliveries. Build one with `hmacSha256`, `githubSignature` or `bearer`. */
 export type WebhookVerify =
-  | { readonly scheme: "hmac-sha256"; readonly secret: Secret; readonly header: string; readonly prefix: string }
-  | { readonly scheme: "bearer"; readonly secret: Secret };
+  | {
+      readonly scheme: 'hmac-sha256';
+      readonly secret: Secret;
+      readonly header: string;
+      readonly prefix: string;
+    }
+  | { readonly scheme: 'bearer'; readonly secret: Secret };
 
 /** HMAC-SHA256 of the raw body keyed by `secret`, hex, in `header` after `prefix`. */
-export function hmacSha256(o: { secret: Secret; header: string; prefix?: string }): WebhookVerify {
-  return Object.freeze({ scheme: "hmac-sha256", secret: o.secret, header: o.header, prefix: o.prefix ?? "" });
+export function hmacSha256(o: {
+  secret: Secret;
+  header: string;
+  prefix?: string;
+}): WebhookVerify {
+  return Object.freeze({
+    scheme: 'hmac-sha256',
+    secret: o.secret,
+    header: o.header,
+    prefix: o.prefix ?? '',
+  });
 }
 
 /** GitHub's delivery signature: `X-Hub-Signature-256: sha256=<hex>`. */
 export function githubSignature(secret: Secret): WebhookVerify {
-  return hmacSha256({ secret, header: "X-Hub-Signature-256", prefix: "sha256=" });
+  return hmacSha256({ secret, header: 'X-Hub-Signature-256', prefix: 'sha256=' });
 }
 
 /** `Authorization: Bearer <secret>`. */
 export function bearer(secret: Secret): WebhookVerify {
-  return Object.freeze({ scheme: "bearer", secret });
+  return Object.freeze({ scheme: 'bearer', secret });
 }
 
 /**

package/src/ctx/files.ts

@@ -4,19 +4,31 @@
 // `path` minus the trailing "/" plus ".tar" for directories (a deterministic tar). The step writes
 // objects to `$OLVE_DIR/work/files/<object path>`; the service stores them under
 // `runs/<p>/<run>/<step>/<attempt>/files/<object path>`.
-import { copyFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
-import { basename, dirname, join, resolve } from "node:path";
-import { fileRefSchema, FILE_PATH, type FileRef } from "../core/models.ts";
-import { ENV } from "../harness/env.ts";
-import { sha256File } from "../util/hash.ts";
-import { dirEntries, extractTar, writeTar } from "../util/tar.ts";
-import type { OutputFileOptions, RunInfo } from "./types.ts";
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import { basename, dirname, join, resolve } from 'node:path';
+import { fileRefSchema, FILE_PATH, type FileRef } from '../core/models.ts';
+import { ENV } from '../harness/env.ts';
+import { sha256File } from '../util/hash.ts';
+import { dirEntries, extractTar, writeTar } from '../util/tar.ts';
+import type { OutputFileOptions, RunInfo } from './types.ts';
 
-export const objectPath = (refPath: string): string => (refPath.endsWith("/") ? refPath.slice(0, -1) + ".tar" : refPath);
-export const fileKey = (ref: Pick<FileRef, "run" | "step" | "attempt" | "path">): string => `${ref.run}/${ref.step}/${ref.attempt}/${ref.path}`;
+export const objectPath = (refPath: string): string =>
+  refPath.endsWith('/') ? refPath.slice(0, -1) + '.tar' : refPath;
+export const fileKey = (
+  ref: Pick<FileRef, 'run' | 'step' | 'attempt' | 'path'>,
+): string => `${ref.run}/${ref.step}/${ref.attempt}/${ref.path}`;
 
 /** Strips everything from `?` on (pre-signed credentials). */
-export const redactUrl = (url: string): string => url.replace(/[?#].*$/, "?…");
+export const redactUrl = (url: string): string => url.replace(/[?#].*$/, '?…');
 
 export interface FilesDeps {
   run: RunInfo;
@@ -28,42 +40,54 @@ export interface FilesDeps {
 }
 
 function checkName(name: string, what: string): void {
-  if (!FILE_PATH.test(name) || name.endsWith("/")) throw new Error(`${what}: '${name}' is not a valid relative file name (no leading '/', no '..', no trailing '/')`);
+  if (!FILE_PATH.test(name) || name.endsWith('/'))
+    throw new Error(
+      `${what}: '${name}' is not a valid relative file name (no leading '/', no '..', no trailing '/')`,
+    );
 }
 
 export function createOutput(deps: FilesDeps) {
   const used = new Set<string>();
   const claim = (obj: string) => {
-    if (used.has(obj) || existsSync(join(deps.workFiles, obj))) throw new Error(`output file '${obj}' already exists`);
+    if (used.has(obj) || existsSync(join(deps.workFiles, obj)))
+      throw new Error(`output file '${obj}' already exists`);
     used.add(obj);
     const dest = join(deps.workFiles, obj);
     mkdirSync(dirname(dest), { recursive: true });
     return dest;
   };
   const ref = (path: string, dest: string): FileRef =>
-    ({ run: deps.run.number, step: deps.run.step, attempt: deps.run.attempt, path, sha256: sha256File(dest) }) as FileRef;
+    ({
+      run: deps.run.number,
+      step: deps.run.step,
+      attempt: deps.run.attempt,
+      path,
+      sha256: sha256File(dest),
+    }) as FileRef;
 
   return {
     async file(path: string, opts: OutputFileOptions = {}): Promise<FileRef> {
       const src = resolve(path);
-      if (!statSync(src).isFile()) throw new Error(`ctx.output.file: '${path}' is not a file`);
+      if (!statSync(src).isFile())
+        throw new Error(`ctx.output.file: '${path}' is not a file`);
       const name = opts.name ?? basename(src);
-      checkName(name, "ctx.output.file");
+      checkName(name, 'ctx.output.file');
       const dest = claim(name);
       copyFileSync(src, dest);
       return ref(name, dest);
     },
     async dir(path: string, opts: OutputFileOptions = {}): Promise<FileRef> {
       const src = resolve(path);
-      if (!statSync(src).isDirectory()) throw new Error(`ctx.output.dir: '${path}' is not a directory`);
+      if (!statSync(src).isDirectory())
+        throw new Error(`ctx.output.dir: '${path}' is not a directory`);
       const name = opts.name ?? basename(src);
-      checkName(name, "ctx.output.dir");
-      const dest = claim(objectPath(name + "/"));
+      checkName(name, 'ctx.output.dir');
+      const dest = claim(objectPath(name + '/'));
       writeTar(dest, dirEntries(src));
-      return ref(name + "/", dest);
+      return ref(name + '/', dest);
     },
     async write(name: string, content: string | Uint8Array): Promise<FileRef> {
-      checkName(name, "ctx.output.write");
+      checkName(name, 'ctx.output.write');
       const dest = claim(name);
       writeFileSync(dest, content);
       return ref(name, dest);
@@ -72,20 +96,33 @@ export function createOutput(deps: FilesDeps) {
 }
 
 function urlMap(root: string): Record<string, string> {
-  const raw = process.env[ENV.fileUrls] ?? (existsSync(join(root, "files.json")) ? readFileSync(join(root, "files.json"), "utf8") : undefined);
+  const raw =
+    process.env[ENV.fileUrls] ??
+    (existsSync(join(root, 'files.json'))
+      ? readFileSync(join(root, 'files.json'), 'utf8')
+      : undefined);
   if (!raw) return {};
   try {
     const m = JSON.parse(raw) as unknown;
-    if (typeof m !== "object" || m === null || Array.isArray(m)) throw new Error("not an object");
+    if (typeof m !== 'object' || m === null || Array.isArray(m))
+      throw new Error('not an object');
     return m as Record<string, string>;
   } catch (e) {
-    throw new Error(`${ENV.fileUrls} / files.json is not a JSON object of file key → URL (${(e as Error).message})`);
+    throw new Error(
+      `${ENV.fileUrls} / files.json is not a JSON object of file key → URL (${(e as Error).message})`,
+    );
   }
 }
 
 const TRANSIENT = (status: number) => status >= 500 || status === 408 || status === 429;
 
-async function download(url: string, dest: string, headers: Record<string, string>, signal: AbortSignal, log: FilesDeps["log"]): Promise<void> {
+async function download(
+  url: string,
+  dest: string,
+  headers: Record<string, string>,
+  signal: AbortSignal,
+  log: FilesDeps['log'],
+): Promise<void> {
   const attempts = 4;
   for (let i = 1; ; i++) {
     let status = 0;
@@ -93,14 +130,19 @@ async function download(url: string, dest: string, headers: Record<string, strin
       const res = await fetch(url, { headers, signal });
       status = res.status;
       if (!res.ok) throw new Error(`HTTP ${res.status}`);
-      await Bun.write(dest + ".partial", res);
-      renameSync(dest + ".partial", dest);
+      await Bun.write(dest + '.partial', res);
+      renameSync(dest + '.partial', dest);
       return;
     } catch (e) {
-      rmSync(dest + ".partial", { force: true });
+      rmSync(dest + '.partial', { force: true });
       const transient = status === 0 || TRANSIENT(status);
-      if (signal.aborted || !transient || i >= attempts) throw new Error(`download of ${redactUrl(url)} failed: ${(e as Error).message}`);
-      log("download failed, retrying", { url: redactUrl(url), attempt: i, error: (e as Error).message });
+      if (signal.aborted || !transient || i >= attempts)
+        throw new Error(`download of ${redactUrl(url)} failed: ${(e as Error).message}`);
+      log('download failed, retrying', {
+        url: redactUrl(url),
+        attempt: i,
+        error: (e as Error).message,
+      });
       await Bun.sleep(1000 * 2 ** (i - 1));
     }
   }
@@ -110,29 +152,56 @@ export function createFetch(deps: FilesDeps) {
   const cache = new Map<string, string>();
   return async (input: FileRef): Promise<string> => {
     const parsed = fileRefSchema.safeParse(input);
-    if (!parsed.success) throw new Error(`ctx.files.fetch: not a valid file reference: ${JSON.stringify(input)}`);
+    if (!parsed.success)
+      throw new Error(
+        `ctx.files.fetch: not a valid file reference: ${JSON.stringify(input)}`,
+      );
     const ref = parsed.data;
     const key = fileKey(ref);
     const hit = cache.get(key);
     if (hit) return hit;
 
     const obj = objectPath(ref.path);
-    const base = join(deps.tmpDir, "files", String(ref.run), ref.step, String(ref.attempt));
-    const objFile = join(base, ".objects", obj);
+    const base = join(
+      deps.tmpDir,
+      'files',
+      String(ref.run),
+      ref.step,
+      String(ref.attempt),
+    );
+    const objFile = join(base, '.objects', obj);
     mkdirSync(dirname(objFile), { recursive: true });
 
-    const own = ref.run === deps.run.number && ref.step === deps.run.step && ref.attempt === deps.run.attempt;
+    const own =
+      ref.run === deps.run.number &&
+      ref.step === deps.run.step &&
+      ref.attempt === deps.run.attempt;
     const urls = urlMap(deps.root);
     const localDir = process.env[ENV.filesDir];
     const baseUrl = process.env[ENV.filesBaseUrl];
-    if (own && existsSync(join(deps.workFiles, obj))) copyFileSync(join(deps.workFiles, obj), objFile);
+    if (own && existsSync(join(deps.workFiles, obj)))
+      copyFileSync(join(deps.workFiles, obj), objFile);
     else if (urls[key]) await download(urls[key], objFile, {}, deps.signal, deps.log);
-    else if (localDir && existsSync(join(localDir, String(ref.run), ref.step, String(ref.attempt), "files", obj)))
-      copyFileSync(join(localDir, String(ref.run), ref.step, String(ref.attempt), "files", obj), objFile);
+    else if (
+      localDir &&
+      existsSync(
+        join(localDir, String(ref.run), ref.step, String(ref.attempt), 'files', obj),
+      )
+    )
+      copyFileSync(
+        join(localDir, String(ref.run), ref.step, String(ref.attempt), 'files', obj),
+        objFile,
+      );
     else if (baseUrl) {
       const token = process.env[ENV.filesToken];
-      const url = `${baseUrl.replace(/\/+$/, "")}/${ref.run}/${encodeURIComponent(ref.step)}/${ref.attempt}/${obj.split("/").map(encodeURIComponent).join("/")}`;
-      await download(url, objFile, token ? { authorization: `Bearer ${token}` } : {}, deps.signal, deps.log);
+      const url = `${baseUrl.replace(/\/+$/, '')}/${ref.run}/${encodeURIComponent(ref.step)}/${ref.attempt}/${obj.split('/').map(encodeURIComponent).join('/')}`;
+      await download(
+        url,
+        objFile,
+        token ? { authorization: `Bearer ${token}` } : {},
+        deps.signal,
+        deps.log,
+      );
     } else {
       throw new Error(
         `ctx.files.fetch: no source for ${key} (set ${ENV.fileUrls} / $OLVE_DIR/files.json, ${ENV.filesBaseUrl} or ${ENV.filesDir})`,
@@ -142,10 +211,12 @@ export function createFetch(deps: FilesDeps) {
     const actual = sha256File(objFile);
     if (actual !== ref.sha256) {
       rmSync(objFile, { force: true });
-      throw new Error(`ctx.files.fetch: ${key} has sha256 ${actual}, expected ${ref.sha256}`);
+      throw new Error(
+        `ctx.files.fetch: ${key} has sha256 ${actual}, expected ${ref.sha256}`,
+      );
     }
     let result = objFile;
-    if (ref.path.endsWith("/")) {
+    if (ref.path.endsWith('/')) {
       result = join(base, ref.path.slice(0, -1));
       rmSync(result, { recursive: true, force: true });
       mkdirSync(result, { recursive: true });