@olivervea/pipelines 0.0.0-stage → 2.0.0-beta.10

package/src/harness/env.ts

@@ -0,0 +1,58 @@
+// The environment contract between the service and the harness. Documented in README.md.
+import { join } from "node:path";
+
+export const ENV = {
+  /** Root of the harness IO directory (default /olve). */
+  dir: "OLVE_DIR",
+  pipeline: "OLVE_PIPELINE",
+  runId: "OLVE_RUN_ID",
+  run: "OLVE_RUN",
+  attempt: "OLVE_ATTEMPT",
+  versionStamp: "OLVE_VERSION_STAMP",
+  /** Pre-signed PUT for result.tar.gz (run / map-trigger). Optional. */
+  resultUrl: "OLVE_RESULT_URL",
+  /** JSON object: file key (`<run>/<step>/<attempt>/<path>`) → pre-signed GET URL. Optional. */
+  fileUrls: "OLVE_FILE_URLS",
+  /** Base URL for file downloads: GET `<base>/<run>/<step>/<attempt>/<object path>`. Optional. */
+  filesBaseUrl: "OLVE_FILES_BASE_URL",
+  /** Bearer token sent with OLVE_FILES_BASE_URL requests. Optional. */
+  filesToken: "OLVE_FILES_TOKEN",
+  /** Local directory laid out as `<run>/<step>/<attempt>/files/<object path>` (dev). Optional. */
+  filesDir: "OLVE_FILES_DIR",
+} as const;
+
+/** Env vars never passed to child processes (they carry credentials for the harness only). */
+export const PRIVATE_ENV: readonly string[] = [ENV.resultUrl, ENV.fileUrls, ENV.filesToken];
+
+export const olveDir = () => process.env[ENV.dir] || "/olve";
+export const paths = () => {
+  const root = olveDir();
+  return {
+    root,
+    input: join(root, "input.json"),
+    work: join(root, "work"),
+    output: join(root, "work", "output.json"),
+    files: join(root, "work", "files"),
+    tmp: join(root, "tmp"),
+    fileUrls: join(root, "files.json"),
+  };
+};
+
+function int(name: string, fallback: number): number {
+  const v = process.env[name];
+  if (v === undefined || v === "") return fallback;
+  const n = Number(v);
+  if (!Number.isInteger(n) || n < 0) throw new Error(`${name} must be a non-negative integer, got '${v}'`);
+  return n;
+}
+
+export function runInfoFromEnv(step: string) {
+  return {
+    pipeline: process.env[ENV.pipeline] ?? "local",
+    id: process.env[ENV.runId] ?? "local",
+    number: int(ENV.run, 0),
+    attempt: Math.max(1, int(ENV.attempt, 1)),
+    step,
+    version: process.env[ENV.versionStamp] ?? "0.0.0-local",
+  };
+}

package/src/harness/execute.ts

@@ -0,0 +1,97 @@
+// The shared execution path: validate input → handler → validate output → envelope.
+// Used by the compiled harness (`run`), the dev runner and testHandler.
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { isSkipped } from "../core/handlers.ts";
+import type { AnyStep } from "../core/steps.ts";
+import type { Ctx, RunInfo } from "../ctx/types.ts";
+import { objectPath } from "../ctx/files.ts";
+import { sha256File } from "../util/hash.ts";
+import { EXIT } from "./exit.ts";
+import { validateConfig, validateInput, validateOutput } from "./validate.ts";
+
+export type FailureKind = "handler" | "input" | "output";
+export type Envelope =
+  | { status: "succeeded"; output: unknown }
+  | { status: "skipped"; reason: string }
+  | { status: "failed"; error: { kind: FailureKind; message: string } };
+
+export interface Execution {
+  envelope: Envelope;
+  exitCode: number;
+}
+
+export type StepFunction = (input: Record<string, unknown>, ctx: Ctx) => unknown;
+
+/** Signal → exit code, when the handler failed after a cancellation. */
+export function cancelExitCode(signal: AbortSignal): number | undefined {
+  if (!signal.aborted) return undefined;
+  return signal.reason === "SIGINT" ? EXIT.sigint : EXIT.sigterm;
+}
+
+const errorMessage = (e: unknown): string => (e instanceof Error ? e.message : String(e));
+
+/** File refs in `value` that claim to be this attempt's own outputs must exist with the right hash. */
+function checkOwnFileRefs(value: unknown, run: RunInfo, workFiles: string): string[] {
+  const problems: string[] = [];
+  const walk = (v: unknown, at: string) => {
+    if (Array.isArray(v)) return v.forEach((x, i) => walk(x, `${at}[${i}]`));
+    if (typeof v !== "object" || v === null) return;
+    const o = v as Record<string, unknown>;
+    if (typeof o.path === "string" && typeof o.sha256 === "string" && o.run === run.number && o.step === run.step && o.attempt === run.attempt) {
+      const file = join(workFiles, objectPath(o.path));
+      if (!existsSync(file)) problems.push(`${at}: file reference '${o.path}' points to this attempt but no such output file was produced`);
+      else if (sha256File(file) !== o.sha256) problems.push(`${at}: file reference '${o.path}' has the wrong sha256 (the file changed after it was referenced)`);
+      return;
+    }
+    for (const [k, x] of Object.entries(o)) walk(x, `${at}.${k}`);
+  };
+  walk(value, "output");
+  return problems;
+}
+
+export async function executeStep(args: {
+  step: AnyStep;
+  fn: StepFunction;
+  rawInput: unknown;
+  ctx: Ctx;
+  workFiles: string;
+  log: (line: string) => void;
+}): Promise<Execution> {
+  const { step, log } = args;
+  const input = validateInput(step, args.rawInput);
+  const config = validateConfig(step, args.rawInput);
+  if (!input.ok || !config.ok) {
+    const issues = [...(input.ok ? [] : input.issues), ...(config.ok ? [] : config.issues)];
+    const message = `step '${step.name}': input does not match the step's inputs:\n  ${issues.join("\n  ")}`;
+    log(message);
+    return { envelope: { status: "failed", error: { kind: "input", message } }, exitCode: EXIT.inputInvalid };
+  }
+  const ctx: Ctx = { ...args.ctx, config: Object.freeze(config.value) as Ctx["config"] };
+
+  let value: unknown;
+  try {
+    value = await args.fn(input.value, ctx);
+  } catch (e) {
+    const cancelled = cancelExitCode(ctx.signal);
+    const message = cancelled
+      ? `step '${step.name}' was cancelled (${String(ctx.signal.reason)}): ${errorMessage(e)}`
+      : `step '${step.name}' failed: ${errorMessage(e)}`;
+    log(message);
+    if (e instanceof Error && e.stack && !cancelled) log(e.stack);
+    return { envelope: { status: "failed", error: { kind: "handler", message } }, exitCode: cancelled ?? EXIT.handlerFailed };
+  }
+
+  if (isSkipped(value)) {
+    log(`step '${step.name}' skipped: ${value.reason}`);
+    return { envelope: { status: "skipped", reason: value.reason }, exitCode: EXIT.ok };
+  }
+  const out = validateOutput(step, value);
+  const problems = out.ok ? checkOwnFileRefs(out.value, ctx.run, args.workFiles) : out.issues;
+  if (!out.ok || problems.length) {
+    const message = `step '${step.name}': returned value does not match ${step.output?.name ?? "its (absent) output model"}:\n  ${problems.join("\n  ")}`;
+    log(message);
+    return { envelope: { status: "failed", error: { kind: "output", message } }, exitCode: EXIT.outputInvalid };
+  }
+  return { envelope: { status: "succeeded", output: out.value }, exitCode: EXIT.ok };
+}

package/src/harness/exit.ts

@@ -0,0 +1,19 @@
+/** Harness exit codes (sysexits-style). Part of the service contract. */
+export const EXIT = {
+  ok: 0,
+  /** The handler (or trigger mapper) threw. */
+  handlerFailed: 1,
+  /** Bad command line: unknown mode, step or trigger. */
+  usage: 64,
+  /** input.json missing, not JSON, or not valid for the step's inputs (or the trigger event). */
+  inputInvalid: 65,
+  /** The handler's return value (or the mapped trigger input) failed strict validation. */
+  outputInvalid: 66,
+  /** Uploading result.tar.gz to OLVE_RESULT_URL failed (after retries). The envelope was still printed. */
+  uploadFailed: 74,
+  /** The pipeline is invalid: describe problems, missing/invalid handler module. */
+  configInvalid: 78,
+  /** Cancelled by SIGINT / SIGTERM (128 + signal). */
+  sigint: 130,
+  sigterm: 143,
+} as const;

package/src/harness/index.ts

@@ -0,0 +1,9 @@
+// @olivervea/pipelines/harness — the runtime the build tool compiles into the step binary.
+export { runHarness, defaultIO, allSteps, resolveStepFunction, type HarnessConfig, type HandlerRegistry, type HarnessIO } from "./main.ts";
+export { EXIT } from "./exit.ts";
+export { ENV, paths, runInfoFromEnv } from "./env.ts";
+export { executeStep, type Envelope, type Execution, type FailureKind } from "./execute.ts";
+export { RESULT_BEGIN, RESULT_END, MANIFEST_BEGIN, MANIFEST_END } from "./result.ts";
+export { describePipeline, DescribeError, manifestText } from "../describe/describe.ts";
+export type { ManifestT as Manifest } from "../describe/manifest-schema.ts";
+export type { HarnessInfo } from "../util/handler-hash.ts";

package/src/harness/main.ts

@@ -0,0 +1,270 @@
+// The harness: one executable, three modes.
+//   describe [--sentinels]   → manifest (canonical JSON) on stdout
+//   run <step>               → $OLVE_DIR/input.json → handler → $OLVE_DIR/work/output.json (+ result upload)
+//   map-trigger <trigger>    → $OLVE_DIR/input.json (event) → pipeline input → $OLVE_DIR/work/output.json
+import { mkdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { isImplementation } from "../core/handlers.ts";
+import { configRefPath, isConfigRef, unionAt } from "../core/config.ts";
+import type { AnyStep } from "../core/steps.ts";
+import type { WebhookEvent } from "../core/webhooks.ts";
+import { pipelineData, type PipelineData, type StageNode } from "../builder/pipeline.ts";
+import { createCtx, formatLog } from "../ctx/index.ts";
+import { DescribeError, describePipeline, manifestText } from "../describe/describe.ts";
+import type { HarnessInfo } from "../util/handler-hash.ts";
+import { ENV, paths, runInfoFromEnv } from "./env.ts";
+import { EXIT } from "./exit.ts";
+import { executeStep, type Envelope, type Execution, type StepFunction } from "./execute.ts";
+import { MANIFEST_BEGIN, MANIFEST_END, packResult, removeStale, sentinelBlock, uploadFile, writeEnvelope } from "./result.ts";
+import { formatIssues, parseStrict } from "./validate.ts";
+
+/** step name → lazy import of `handlers/<name>.ts`. Literal import() paths so Bun bundles them. */
+export type HandlerRegistry = Readonly<Record<string, () => Promise<{ default: unknown }>>>;
+
+export interface HarnessConfig {
+  /** Default export of pipeline.ts. */
+  pipeline: unknown;
+  handlers: HandlerRegistry;
+  handlerHashes: Readonly<Record<string, string>>;
+  /** Kind → hash of the code of the steps of that kind that carry their implementation. */
+  kindHashes?: Readonly<Record<string, string>>;
+  harness: HarnessInfo;
+}
+
+export interface HarnessIO {
+  stdout(text: string): Promise<void>;
+  stderr(line: string): void;
+}
+
+export const defaultIO: HarnessIO = {
+  stdout: async (text) => {
+    await Bun.write(Bun.stdout, text);
+  },
+  stderr: (line) => {
+    process.stderr.write(line.endsWith("\n") ? line : line + "\n");
+  },
+};
+
+const USAGE = "usage: step describe [--sentinels] | step run <step> | step map-trigger <trigger>";
+
+export function allSteps(data: PipelineData): AnyStep[] {
+  const out: AnyStep[] = [];
+  for (const { step } of stepsWithCases(data)) if (!out.includes(step)) out.push(step);
+  return out;
+}
+
+/** Every step with the `select` case conditions (`<ref>.<discriminator>=<value>`) it sits in. */
+export function stepsWithCases(data: PipelineData): { step: AnyStep; cases: string[] }[] {
+  const out: { step: AnyStep; cases: string[] }[] = [];
+  const walk = (nodes: readonly StageNode[], cases: string[]) => {
+    for (const n of nodes) {
+      if (n.kind === "stage") out.push(...n.steps.map((s) => ({ step: s.step, cases })));
+      else if (n.kind === "when") walk(n.nodes, cases);
+      else {
+        const union = isConfigRef(n.on) ? unionAt(n.on) : undefined;
+        for (const [value, inner] of Object.entries(n.cases))
+          walk(inner, union ? [...cases, `${configRefPath(n.on)}.${union.discriminator}=${value}`] : cases);
+      }
+    }
+  };
+  walk(data.nodes, []);
+  for (const fh of data.failureHandlers) out.push({ step: fh.step, cases: [] });
+  return out;
+}
+
+/** The step `run <name>` executes: the only one so named, or the one in the cases the service selected (`OLVE_CASES`). */
+export function findStep(data: PipelineData, name: string, selected: readonly string[]): AnyStep | string {
+  const candidates = stepsWithCases(data).filter((c) => c.step.name === name);
+  const distinct = candidates.filter((c, i) => candidates.findIndex((d) => d.step === c.step) === i);
+  if (distinct.length === 0) return `unknown step '${name}'`;
+  if (distinct.length === 1) return distinct[0]!.step;
+  const chosen = distinct.filter((c) => c.cases.every((k) => selected.includes(k)));
+  if (chosen.length === 1) return chosen[0]!.step;
+  return `step '${name}' is in several select cases; ${ENV_CASES} must name the selected one (got ${JSON.stringify(selected)})`;
+}
+
+/** The select case conditions of the step being run (JSON array), set by the service for steps inside a select. */
+export const ENV_CASES = "OLVE_CASES";
+
+/** Installs SIGTERM/SIGINT handlers that abort the returned signal (PID 1 ignores signals without them). */
+export function cancellation(): { signal: AbortSignal; dispose(): void } {
+  const ac = new AbortController();
+  const onTerm = () => ac.abort("SIGTERM");
+  const onInt = () => (ac.signal.aborted ? process.exit(EXIT.sigint) : ac.abort("SIGINT"));
+  process.on("SIGTERM", onTerm);
+  process.on("SIGINT", onInt);
+  return {
+    signal: ac.signal,
+    dispose() {
+      process.off("SIGTERM", onTerm);
+      process.off("SIGINT", onInt);
+    },
+  };
+}
+
+/** Resolves the function that implements `step` (carried by the step, or a handler module). */
+export async function resolveStepFunction(step: AnyStep, handlers: HandlerRegistry): Promise<StepFunction | string> {
+  if (isImplementation(step)) return step.handler as unknown as StepFunction;
+  const load = handlers[step.name];
+  if (!load) return `step '${step.name}' has no handler`;
+  let mod: { default: unknown };
+  try {
+    mod = await load();
+  } catch (e) {
+    return `the handler of step '${step.name}' failed to load: ${(e as Error)?.stack ?? String(e)}`;
+  }
+  const impl = mod.default;
+  if (!isImplementation(impl)) return `the handler of step '${step.name}' must default-export implement(<step '${step.name}'>, …)`;
+  if (impl.step !== step)
+    return `the handler of step '${step.name}' implements a different step object (it must be implement(<the '${step.name}' step from pipeline.ts>, …))`;
+  return impl.handler as unknown as StepFunction;
+}
+
+/** Writes output.json, prints the sentinel block and uploads result.tar.gz when OLVE_RESULT_URL is set. */
+export async function finish(exec: Execution, io: HarnessIO): Promise<number> {
+  const p = paths();
+  writeEnvelope(p.work, exec.envelope);
+  await io.stdout(sentinelBlock(exec.envelope));
+  const url = process.env[ENV.resultUrl];
+  if (!url) return exec.exitCode;
+  try {
+    mkdirSync(p.tmp, { recursive: true });
+    const archive = join(p.tmp, "result.tar.gz");
+    await packResult(p.work, archive);
+    await uploadFile(url, archive, io.stderr);
+  } catch (e) {
+    io.stderr(`result upload failed: ${(e as Error).message}`);
+    return exec.exitCode === EXIT.ok ? EXIT.uploadFailed : exec.exitCode;
+  }
+  return exec.exitCode;
+}
+
+function readJson(path: string): { ok: true; value: unknown } | { ok: false; message: string } {
+  try {
+    return { ok: true, value: JSON.parse(readFileSync(path, "utf8")) };
+  } catch (e) {
+    return { ok: false, message: `cannot read ${path}: ${(e as Error).message}` };
+  }
+}
+
+async function describeMode(cfg: HarnessConfig, args: string[], io: HarnessIO): Promise<number> {
+  const sentinels = args.includes("--sentinels");
+  try {
+    const text = manifestText(describePipeline(cfg.pipeline, { handlerNames: Object.keys(cfg.handlers), handlerHashes: cfg.handlerHashes, kindHashes: cfg.kindHashes, harness: cfg.harness }));
+    await io.stdout(sentinels ? `${MANIFEST_BEGIN}\n${text}${MANIFEST_END}\n` : text);
+    return EXIT.ok;
+  } catch (e) {
+    if (e instanceof DescribeError) {
+      io.stderr(e.message);
+      return EXIT.configInvalid;
+    }
+    throw e;
+  }
+}
+
+async function runMode(cfg: HarnessConfig, name: string | undefined, io: HarnessIO): Promise<number> {
+  if (!name) return io.stderr(USAGE), EXIT.usage;
+  const data = pipelineData(cfg.pipeline);
+  const step = findStep(data, name, selectedCases());
+  if (typeof step === "string") return io.stderr(step), EXIT.usage;
+  const fn = await resolveStepFunction(step, cfg.handlers);
+  if (typeof fn === "string") return io.stderr(fn), EXIT.configInvalid;
+
+  const p = paths();
+  removeStale(p.work);
+  mkdirSync(p.files, { recursive: true });
+  mkdirSync(p.tmp, { recursive: true });
+  const raw = readJson(p.input);
+  const cancel = cancellation();
+  try {
+    let exec: Execution;
+    if (!raw.ok) {
+      io.stderr(raw.message);
+      exec = { envelope: { status: "failed", error: { kind: "input", message: raw.message } }, exitCode: EXIT.inputInvalid };
+    } else {
+      const run = runInfoFromEnv(step.name);
+      const ctx = createCtx({ run, secrets: step.secrets, paths: { root: p.root, workFiles: p.files, tmp: p.tmp }, signal: cancel.signal, logSink: io.stderr });
+      exec = await executeStep({ step, fn, rawInput: raw.value, ctx, workFiles: p.files, log: io.stderr });
+    }
+    return await finish(exec, io);
+  } finally {
+    cancel.dispose();
+  }
+}
+
+function isWebhookEvent(v: unknown): boolean {
+  if (typeof v !== "object" || v === null || Array.isArray(v) || !("body" in v)) return false;
+  const headers = (v as { headers?: unknown }).headers;
+  return typeof headers === "object" && headers !== null && !Array.isArray(headers);
+}
+
+async function mapTriggerMode(cfg: HarnessConfig, name: string | undefined, io: HarnessIO): Promise<number> {
+  if (!name) return io.stderr(USAGE), EXIT.usage;
+  const data = pipelineData(cfg.pipeline);
+  const trigger = data.triggers.find((t) => t.name === name);
+  if (!trigger) return io.stderr(`unknown trigger '${name}' (known: ${data.triggers.map((t) => t.name).join(", ") || "none"})`), EXIT.usage;
+  const p = paths();
+  removeStale(p.work);
+  mkdirSync(p.work, { recursive: true });
+  const raw = readJson(p.input);
+  const fail = (kind: "input" | "output" | "handler", message: string, exitCode: number): Execution => {
+    io.stderr(message);
+    return { envelope: { status: "failed", error: { kind, message } }, exitCode };
+  };
+  let exec: Execution;
+  if (!raw.ok) exec = fail("input", raw.message, EXIT.inputInvalid);
+  else if (trigger.kind === "poll" && (typeof raw.value !== "object" || raw.value === null || typeof (raw.value as { value?: unknown }).value !== "string"))
+    exec = fail("input", `poll trigger '${name}': the event must be {"value": "<string>"}`, EXIT.inputInvalid);
+  else if (trigger.kind === "push" && (typeof raw.value !== "object" || raw.value === null || Array.isArray(raw.value)))
+    exec = fail("input", `push trigger: the event must be the webhook payload object`, EXIT.inputInvalid);
+  else if (trigger.kind === "webhook" && !isWebhookEvent(raw.value))
+    exec = fail("input", `webhook trigger '${name}': the event must be {"headers": {...}, "body": ...}`, EXIT.inputInvalid);
+  else {
+    let mapped: unknown;
+    let threw: unknown = undefined;
+    try {
+      mapped =
+        trigger.kind === "poll" ? await trigger.map((raw.value as { value: string }).value)
+        : trigger.kind === "webhook" ? await trigger.map(raw.value as WebhookEvent)
+        : await trigger.map(raw.value);
+    } catch (e) {
+      threw = e ?? new Error("undefined");
+    }
+    if (threw !== undefined) exec = fail("handler", `trigger '${name}' mapper threw: ${(threw as Error)?.message ?? String(threw)}`, EXIT.handlerFailed);
+    else if (mapped === null && trigger.kind === "webhook") exec = { envelope: { status: "skipped", reason: `trigger '${name}' ignored the delivery` }, exitCode: EXIT.ok };
+    else {
+      const r = parseStrict(data.input.schema, mapped, "input");
+      exec = r.ok
+        ? { envelope: { status: "succeeded", output: r.value } satisfies Envelope, exitCode: EXIT.ok }
+        : fail("output", `trigger '${name}': mapped input does not match ${data.input.name}:\n  ${r.issues.join("\n  ")}`, EXIT.outputInvalid);
+    }
+  }
+  return finish(exec, io);
+}
+
+/** Entry point of the compiled binary (and the dev runner). Returns the exit code. */
+export async function runHarness(cfg: HarnessConfig, argv: readonly string[], io: HarnessIO = defaultIO): Promise<number> {
+  const [mode, ...rest] = argv;
+  switch (mode) {
+    case "describe":
+      return describeMode(cfg, rest, io);
+    case "run":
+      return runMode(cfg, rest[0], io);
+    case "map-trigger":
+      return mapTriggerMode(cfg, rest[0], io);
+    default:
+      io.stderr(USAGE);
+      return EXIT.usage;
+  }
+}
+
+export { formatIssues, formatLog };
+
+function selectedCases(): string[] {
+  try {
+    const v = JSON.parse(process.env[ENV_CASES] ?? "[]") as unknown;
+    return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
+  } catch {
+    return [];
+  }
+}

package/src/harness/result.ts

@@ -0,0 +1,75 @@
+// Writing, printing and uploading the result of `run` / `map-trigger`.
+import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { canonicalJson } from "../util/canonical.ts";
+import { dirEntries, writeTarGz, type TarEntry } from "../util/tar.ts";
+import { redactUrl } from "../ctx/files.ts";
+import type { Envelope } from "./execute.ts";
+
+export const RESULT_BEGIN = "---OLVE-RESULT-BEGIN---";
+export const RESULT_END = "---OLVE-RESULT-END---";
+export const MANIFEST_BEGIN = "---OLVE-MANIFEST-BEGIN---";
+export const MANIFEST_END = "---OLVE-MANIFEST-END---";
+
+/** Atomically writes the envelope to `<work>/output.json`. */
+export function writeEnvelope(work: string, envelope: Envelope): string {
+  mkdirSync(work, { recursive: true });
+  const path = join(work, "output.json");
+  writeFileSync(path + ".partial", canonicalJson(envelope));
+  renameSync(path + ".partial", path);
+  return path;
+}
+
+/** The envelope as one compact line between sentinels (fallback channel via pod logs). */
+export function sentinelBlock(envelope: Envelope): string {
+  return `${RESULT_BEGIN}\n${JSON.stringify(JSON.parse(canonicalJson(envelope)))}\n${RESULT_END}\n`;
+}
+
+/** Packs output.json + files/ into result.tar.gz (deterministic). */
+export async function packResult(work: string, out: string): Promise<void> {
+  const entries: TarEntry[] = [{ path: "output.json", type: "file", mode: 0o644, source: join(work, "output.json") }];
+  const files = join(work, "files");
+  if (existsSync(files)) entries.push({ path: "files", type: "dir", mode: 0o755 }, ...dirEntries(files, "files"));
+  await writeTarGz(out, entries);
+}
+
+const TRANSIENT = (status: number) => status >= 500 || status === 408 || status === 429;
+
+/** PUTs `file` to a pre-signed URL; retries network errors / 5xx / 408 / 429 with backoff, fails fast on other 4xx. */
+export async function uploadFile(
+  url: string,
+  file: string,
+  log: (line: string) => void,
+  opts: { attempts?: number; baseDelayMs?: number } = {},
+): Promise<void> {
+  const attempts = opts.attempts ?? 5;
+  const base = opts.baseDelayMs ?? 1000;
+  for (let i = 1; ; i++) {
+    let status = 0;
+    let detail: string;
+    try {
+      const res = await fetch(url, { method: "PUT", body: Bun.file(file) });
+      status = res.status;
+      if (res.ok) return;
+      detail = `HTTP ${res.status} ${res.statusText}`;
+      await res.body?.cancel();
+    } catch (e) {
+      detail = (e as Error).message;
+    }
+    const transient = status === 0 || TRANSIENT(status);
+    if (!transient || i >= attempts) throw new Error(`upload to ${redactUrl(url)} failed: ${detail}`);
+    log(`upload to ${redactUrl(url)} failed (${detail}); retry ${i}/${attempts - 1}`);
+    await Bun.sleep(base * 2 ** (i - 1));
+  }
+}
+
+/**
+ * Clears results of a previous run in the same $OLVE_DIR (dev loops reuse it): output.json and the
+ * *contents* of files/. The directories themselves are kept (olve-init creates them 1777).
+ */
+export function removeStale(work: string): void {
+  rmSync(join(work, "output.json"), { force: true });
+  rmSync(join(work, "output.json.partial"), { force: true });
+  const files = join(work, "files");
+  if (existsSync(files)) for (const e of readdirSync(files)) rmSync(join(files, e), { recursive: true, force: true });
+}

package/src/harness/validate.ts

@@ -0,0 +1,92 @@
+import { z } from "zod";
+import { isModel, isOptional } from "../core/models.ts";
+import type { AnyStep } from "../core/steps.ts";
+
+export type Validation<T> = { ok: true; value: T } | { ok: false; issues: string[] };
+
+export function formatIssues(err: z.ZodError, prefix: string): string[] {
+  return err.issues.map((i) => `${prefix}${i.path.length ? "." + i.path.map(String).join(".") : ""}: ${i.message}`);
+}
+
+const isPlainObject = (v: unknown): v is Record<string, unknown> =>
+  typeof v === "object" && v !== null && !Array.isArray(v) && (Object.getPrototypeOf(v) === Object.prototype || Object.getPrototypeOf(v) === null);
+
+/**
+ * zod strips unknown keys silently (at every depth). Compares the raw value with the parsed one:
+ * any key present in a raw object but absent from the parsed object at the same place was stripped.
+ */
+export function strippedKeys(raw: unknown, parsed: unknown, path = ""): string[] {
+  if (Array.isArray(raw) && Array.isArray(parsed)) return raw.flatMap((x, i) => strippedKeys(x, parsed[i], `${path}[${i}]`));
+  if (isPlainObject(raw) && isPlainObject(parsed)) {
+    const out: string[] = [];
+    for (const k of Object.keys(raw)) {
+      const p = path ? `${path}.${k}` : k;
+      if (!Object.hasOwn(parsed, k)) out.push(p);
+      else out.push(...strippedKeys(raw[k], parsed[k], p));
+    }
+    return out;
+  }
+  return [];
+}
+
+/** Strict parse: zod validation plus rejection of unknown keys at any depth. */
+export function parseStrict(schema: z.ZodType, value: unknown, prefix: string): Validation<unknown> {
+  const r = schema.safeParse(value);
+  if (!r.success) return { ok: false, issues: formatIssues(r.error, prefix) };
+  const extra = strippedKeys(value, r.data);
+  if (extra.length) return { ok: false, issues: extra.map((k) => `${prefix}.${k}: unknown field (outputs are validated strictly)`) };
+  return { ok: true, value: r.data };
+}
+
+function entrySchema(e: unknown): z.ZodType {
+  if (isOptional(e)) return e.model.schema;
+  if (isModel(e)) return e.schema;
+  return e as z.ZodType;
+}
+
+/** Reserved input.json key holding the step's config entries (written by the service). */
+export const CONFIG_KEY = "$config";
+
+/** Parses `$config` with each declared entry's schema (defaults applied). Undeclared entries are ignored. */
+export function validateConfig(step: AnyStep, raw: unknown): Validation<Record<string, unknown>> {
+  const given = isPlainObject(raw) ? raw[CONFIG_KEY] : undefined;
+  if (given !== undefined && !isPlainObject(given)) return { ok: false, issues: [`input.${CONFIG_KEY} must be an object keyed by config entry names`] };
+  const issues: string[] = [];
+  const out: Record<string, unknown> = {};
+  for (const e of step.config ?? []) {
+    const value = given?.[e.name];
+    const r = e.schema.safeParse(value);
+    if (!r.success) issues.push(...formatIssues(r.error, `config.${e.name}`).map((i) => (value === undefined ? `${i} (config entry '${e.name}' is not set)` : i)));
+    else if (r.data !== undefined) out[e.name] = r.data;
+  }
+  return issues.length ? { ok: false, issues } : { ok: true, value: out };
+}
+
+/** Validates input.json against the step's input entries. Unknown top-level keys are rejected; nested values are parsed (unknown nested keys stripped). */
+export function validateInput(step: AnyStep, raw: unknown): Validation<Record<string, unknown>> {
+  if (!isPlainObject(raw)) return { ok: false, issues: ["input.json must be a JSON object keyed by the step's input names"] };
+  const issues: string[] = [];
+  const out: Record<string, unknown> = {};
+  const declared = step.input as Record<string, unknown>;
+  for (const k of Object.keys(raw)) if (k !== CONFIG_KEY && !Object.hasOwn(declared, k)) issues.push(`input.${k}: not an input of step '${step.name}'`);
+  for (const [k, e] of Object.entries(declared)) {
+    if (!Object.hasOwn(raw, k)) {
+      if (isOptional(e)) continue;
+      issues.push(`input.${k}: missing`);
+      continue;
+    }
+    const r = entrySchema(e).safeParse(raw[k]);
+    if (r.success) out[k] = r.data;
+    else issues.push(...formatIssues(r.error, `input.${k}`));
+  }
+  return issues.length ? { ok: false, issues } : { ok: true, value: out };
+}
+
+/** Validates a handler's return value strictly against the step's output model. */
+export function validateOutput(step: AnyStep, value: unknown): Validation<unknown> {
+  if (!step.output) {
+    if (value === undefined || value === null) return { ok: true, value: null };
+    return { ok: false, issues: [`step '${step.name}' declares no output model but the handler returned a value`] };
+  }
+  return parseStrict(step.output.schema, value, "output");
+}

package/src/index.ts

@@ -0,0 +1,15 @@
+/// <reference path="./assets.d.ts" />
+// @olivervea/pipelines — authoring API (contracts, handlers, pipeline builder).
+export { z } from "zod";
+export { model, fileRef, optional, type Model, type AnyModel, type Infer, type InferInput, type Plain, type FileRef, type Optional } from "./core/models.ts";
+export { secret, type Secret, type DeclaredSecrets } from "./core/secrets.ts";
+export { config, secretRef, isConfigRef, type ConfigEntry, type AnyConfigEntry, type ConfigValues, type ConfigRef, type ConfigNode, type Configurable, type DiscriminatorOf, type VariantOf } from "./core/config.ts";
+export { hmacSha256, githubSignature, bearer, type WebhookVerify, type WebhookEvent } from "./core/webhooks.ts";
+export { resolveConfigRefs, isLiteral, type Resolved } from "./core/config-refs.ts";
+export { step, type Step, type AnyStep, type StepDef, type StepInput, type StepKind, type ConfigOf, type InputEntry, type InputSpec, type LiteralName, type OutputValue } from "./core/steps.ts";
+export type { Simplify } from "./core/types.ts";
+export { implement, type Implementation, type Handler, type HandlerResult, type Skipped } from "./core/handlers.ts";
+export type { Ctx, RunInfo, Sh, ShOptions, ExecOptions, Http, HttpInit, RetryOptions, OutputFileOptions } from "./ctx/types.ts";
+export type { Ref, Mappable, MapperResult } from "./builder/refs.ts";
+export { pipeline, Pipeline, type PipelineDef, type InRefs, type InitialAvail, type WhenCond, type PushEvent, type StageEntry, type CheckStage, type MapperFn } from "./builder/pipeline.ts";
+export { steps, Source, BuiltImage, PushedImage, HelmRelease, FailureContext, type GitHubSourceDef, type ImageBuildDef, type ImageBuildTarballDef, type ImageBuildPushDef, type RegistryAuth, type RegistryAuthConfig, type HelmDeployOverSshDef, helmTarget, deployTarget, type DeployTargetOptions, type ComposeDeployDef, type ComposeTargetValue, type HelmDeployInClusterDef, type HelmDeploySshTargetDef, type HelmTargetDeployDef, type HelmTargetValue, type InClusterTargetValue, type SshTargetValue, httpHealthCheck, type HttpHealthCheckOptions, sshSession, type SshSession, serviceUrl, checkSpa } from "./library/index.ts";

package/src/library/checks.ts

@@ -0,0 +1,23 @@
+// Checks for deployed apps, used from custom test handlers.
+import type { Ctx } from "../ctx/types.ts";
+
+/** The in-cluster URL of a release's Service, assuming the chart names it after the release. */
+export function serviceUrl(release: { release: string; namespace: string }): string {
+  return `http://${release.release}.${release.namespace}.svc.cluster.local`;
+}
+
+/**
+ * Checks `url` serves a single-page app: `GET <url>/` references a `/assets/*.js` bundle (Vite's
+ * layout), and the bundle is served as JavaScript. Returns the page, for app-specific checks.
+ */
+export async function checkSpa(ctx: Pick<Ctx, "http" | "log">, url: string): Promise<{ html: string; bundle: string }> {
+  const base = url.replace(/\/$/, "");
+  const html = await (await ctx.http.expectOk(`${base}/`)).text();
+  const bundle = html.match(/src="(\/assets\/[^"]+\.js)"/)?.[1];
+  if (bundle === undefined) throw new Error(`${base}/ does not reference an /assets/*.js bundle`);
+  const js = await ctx.http.expectOk(`${base}${bundle}`);
+  await js.body?.cancel().catch(() => {});
+  if (!(js.headers.get("content-type")?.includes("javascript") ?? false)) throw new Error(`${base}${bundle} is not served as JavaScript`);
+  ctx.log("SPA served", { url: base, bundle });
+  return { html, bundle };
+}