@olivervea/pipelines 0.0.0-stage → 2.0.0-beta.10

package/src/core/handlers.ts

@@ -0,0 +1,50 @@
+import { IMPLEMENTATION, ORIGIN, SKIPPED, type ImplementationBrand, type SkippedBrand } from "./brands.ts";
+import type { Ctx } from "../ctx/types.ts";
+import { callerFile, originsOf } from "./origin.ts";
+import type { AnyStep, ConfigOf, OutputValue, SecretsOf, StepInput } from "./steps.ts";
+import type { Awaitable } from "./types.ts";
+
+/** Returned by `ctx.skip(reason)`: the step produced no output on purpose. */
+export interface Skipped {
+  readonly [SkippedBrand]: true;
+  readonly reason: string;
+}
+export function skipped(reason: string): Skipped {
+  return Object.freeze({ [SKIPPED]: true, reason }) as unknown as Skipped;
+}
+export function isSkipped(v: unknown): v is Skipped {
+  return typeof v === "object" && v !== null && (v as Record<symbol, unknown>)[SKIPPED] === true;
+}
+
+export type HandlerResult<S extends AnyStep> = OutputValue<S> | Skipped;
+export type Handler<S extends AnyStep> = (input: StepInput<S["input"]>, ctx: Ctx<SecretsOf<S>, ConfigOf<S>>) => Awaitable<HandlerResult<S>>;
+
+/** A step with its handler. It is itself a step: stage it directly, or default-export it from `handlers/*.ts`. */
+export type Implementation<S extends AnyStep = AnyStep> = S & {
+  readonly [ImplementationBrand]: true;
+  /** The step as defined. */
+  readonly step: S;
+  readonly handler: Handler<S>;
+};
+
+/** Block-bodied/async returns get no excess-property check; this restores it for handler outputs. */
+type NoExtraFields<R, T> = R extends Skipped | void
+  ? unknown
+  : { [K in Exclude<keyof R, keyof T>]: `❌ '${K & string}' is not a field of the output model` };
+
+/**
+ * Implement a step. Input, ctx and return types come from the step definition alone. Default-export
+ * the result from a `handlers/*.ts` file, or stage it directly (a reusable step: give it a `kind`).
+ */
+export function implement<S extends AnyStep, R extends HandlerResult<S>>(
+  step: S & (S extends { readonly [ImplementationBrand]: true } ? "❌ this step already carries its implementation" : unknown),
+  handler: (input: StepInput<S["input"]>, ctx: Ctx<SecretsOf<S>, ConfigOf<S>>) => Awaitable<R & NoExtraFields<R, OutputValue<S>>>,
+): Implementation<S> {
+  if (isImplementation(step)) throw new TypeError(`step '${(step as AnyStep).name}' already carries its implementation`);
+  const origins = [...new Set([...originsOf(step), callerFile()].filter((f): f is string => f !== undefined))];
+  return Object.freeze({ ...step, [IMPLEMENTATION]: true, [ORIGIN]: origins, step, handler: handler as unknown as Handler<S> }) as unknown as Implementation<S>;
+}
+
+export function isImplementation(v: unknown): v is Implementation {
+  return typeof v === "object" && v !== null && (v as Record<symbol, unknown>)[IMPLEMENTATION] === true;
+}

package/src/core/models.ts

@@ -0,0 +1,69 @@
+import { z } from "zod";
+import type { ModelBrand } from "./brands.ts";
+
+/** A named, nominal zod schema. The name is its identity in the manifest (one schema per name). */
+export interface Model<N extends string = string, S extends z.ZodType = z.ZodType> {
+  readonly kind: "model";
+  readonly name: N;
+  readonly schema: S;
+}
+export type AnyModel = Model<string, z.ZodType>;
+
+/** Output view of a model (`z.output`), nominally branded with the model name. The brand is optional, so plain literals still assign. */
+export type Infer<M extends AnyModel> = z.output<M["schema"]> & { readonly [ModelBrand]?: M["name"] };
+/** Input view of a model (`z.input`: fields with defaults are optional). What trigger mappers produce. */
+export type InferInput<M extends AnyModel> = z.input<M["schema"]>;
+/** Output view without the nominal brand. */
+export type Plain<M extends AnyModel> = z.output<M["schema"]>;
+
+/** Model names: identifier-like, so they are safe as JSON keys, C# names and URL segments. */
+export const MODEL_NAME = /^[A-Za-z][A-Za-z0-9_]*$/;
+
+export function model<const N extends string, S extends z.ZodType>(name: N, schema: S): Model<N, S> {
+  return Object.freeze({ kind: "model" as const, name, schema });
+}
+
+export function isModel(v: unknown): v is AnyModel {
+  return typeof v === "object" && v !== null && (v as { kind?: unknown }).kind === "model" && (v as { schema?: unknown }).schema instanceof z.ZodType;
+}
+
+// ─── File references ─────────────────────────────────────────────────────────────────────────
+
+/**
+ * One path segment: no `/`, `\` or NUL, and not `.` / `..`. A path is segments joined by `/`;
+ * a trailing `/` marks a directory reference (stored as a tar archive, see README).
+ */
+const SEGMENT = String.raw`(?!\.\.?(?:/|$))[^/\\\x00]+`;
+export const FILE_PATH_PATTERN = `^(?:${SEGMENT}/)*${SEGMENT}/?$`;
+export const FILE_PATH = new RegExp(FILE_PATH_PATTERN);
+
+const FileRefSchema = z
+  .object({
+    run: z.number().int().nonnegative(),
+    step: z.string().min(1),
+    attempt: z.number().int().positive(),
+    path: z.string().regex(FILE_PATH),
+    sha256: z.string().regex(/^[0-9a-f]{64}$/),
+  })
+  .brand<"FileRef">()
+  .meta({ "x-olve-kind": "fileRef" });
+
+/** A file or directory in some step attempt's output (`runs/<p>/<run>/<step>/<attempt>/files/<path>`). Branded: only `ctx.output.*` or a mapper ref produce one. */
+export type FileRef = z.output<typeof FileRefSchema>;
+/** Schema for a file reference field inside a model. Marked `"x-olve-kind": "fileRef"` in JSON Schema. */
+export const fileRef = () => FileRefSchema;
+export const fileRefSchema = FileRefSchema;
+
+// ─── optional() ──────────────────────────────────────────────────────────────────────────────
+
+/** A step input that may be missing because its producer was *maybe skipped* (inside a `when`). There must still be a producer. */
+export interface Optional<M extends AnyModel = AnyModel> {
+  readonly kind: "optional";
+  readonly model: M;
+}
+export function optional<M extends AnyModel>(model: M): Optional<M> {
+  return Object.freeze({ kind: "optional" as const, model });
+}
+export function isOptional(v: unknown): v is Optional {
+  return typeof v === "object" && v !== null && (v as { kind?: unknown }).kind === "optional" && isModel((v as { model?: unknown }).model);
+}

package/src/core/origin.ts

@@ -0,0 +1,20 @@
+import { ORIGIN } from "./brands.ts";
+
+/**
+ * The file that called the function calling this one, from the stack. A tail call drops its caller's
+ * frame, so this can name a module further up; the build only uses it to choose what to hash.
+ */
+export function callerFile(): string | undefined {
+  const files = (new Error().stack ?? "").split("\n").slice(1).map((f) => f.match(FRAME)?.[1]);
+  const own = files[0];
+  const callee = files[1];
+  const file = files.slice(2).find((f) => f && f !== own && f !== callee) ?? undefined;
+  return file?.startsWith("file://") ? new URL(file).pathname : file;
+}
+const FRAME = /\(?((?:file:\/\/)?\/[^()]*?):\d+(?::\d+)?\)?\s*$/;
+
+/** The modules that defined and implemented a step (where `step` and `implement` were called). */
+export function originsOf(v: unknown): string[] {
+  const o = (v as Record<symbol, unknown>)[ORIGIN];
+  return Array.isArray(o) ? o.filter((x): x is string => typeof x === "string") : [];
+}

package/src/core/secrets.ts

@@ -0,0 +1,13 @@
+/** A named pipeline secret. Steps (and poll triggers) declare the secrets they use; only those are mounted. */
+export interface Secret<N extends string = string> {
+  readonly kind: "secret";
+  readonly name: N;
+  readonly description: string;
+}
+/** Secret names double as environment variable names in the step container. */
+export const SECRET_NAME = /^[A-Z_][A-Z0-9_]*$/;
+
+export function secret<const N extends string>(name: N, description: string): Secret<N> {
+  return Object.freeze({ kind: "secret" as const, name, description });
+}
+export type DeclaredSecrets<N extends string> = { readonly [K in N]: string };

package/src/core/steps.ts

@@ -0,0 +1,138 @@
+import type { z } from "zod";
+import { ORIGIN, STEP, type StepBrand } from "./brands.ts";
+import { callerFile } from "./origin.ts";
+import type { AnyModel, Infer, Optional } from "./models.ts";
+import type { AnyConfigEntry } from "./config.ts";
+import { configRefsIn } from "./config-refs.ts";
+import type { Secret } from "./secrets.ts";
+import type { Simplify } from "./types.ts";
+
+/** A step input entry: a model (matchable by model), `optional(model)`, or a plain zod schema (mapper required). */
+export type InputEntry = AnyModel | Optional | z.ZodType;
+export type InputSpec = { readonly [key: string]: InputEntry };
+
+export type InferEntry<E> =
+  E extends Optional<infer M> ? Infer<M> | undefined
+  : E extends AnyModel ? Infer<E>
+  : E extends z.ZodType ? z.output<E>
+  : never;
+/** What a handler receives: the output view of every input entry. */
+export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
+
+/**
+ * A label for what a step is, shown in the UI. `custom` (the default) is a step whose handler is in
+ * `handlers/`; a step that carries its implementation (`implement(step, …)` staged directly) names its
+ * kind, e.g. `gitHubSource`. Packages prefix theirs (`acme/deploy`).
+ */
+export type StepKind = string;
+
+export interface Step<
+  N extends string = string,
+  I extends InputSpec = InputSpec,
+  O extends AnyModel | null = AnyModel | null,
+  Sec extends string = string,
+  Cfg extends AnyConfigEntry = never,
+> {
+  readonly [StepBrand]: true;
+  readonly kind: StepKind;
+  readonly name: N;
+  readonly description: string;
+  readonly image: string;
+  readonly input: I;
+  readonly output: O;
+  readonly secrets: readonly Secret<Sec>[];
+  /** Config entries the step receives as `ctx.config`. */
+  readonly config: readonly Cfg[];
+  readonly timeoutSeconds?: number;
+  readonly retries?: number;
+  /** The step pod gets the instance's step ServiceAccount token (`Kubernetes:StepServiceAccount`). */
+  readonly clusterAccess?: boolean;
+  /** How the step is set up, as written in the definition (may hold config references). */
+  readonly settings?: Readonly<Record<string, unknown>>;
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type AnyStep = Step<string, any, AnyModel | null, any, any>;
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type SecretsOf<S> = S extends Step<any, any, any, infer Sec, any> ? Sec : never;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type ConfigOf<S> = S extends Step<any, any, any, any, infer C> ? C : never;
+export type OutputValue<S extends AnyStep> = S["output"] extends AnyModel ? Infer<S["output"]> : void;
+
+/** Step names must be literals (identity, duplicate detection and mapper keys depend on it). */
+export type LiteralName<N extends string> = N & (string extends N ? "❌ step name must be a string literal" : unknown);
+
+/** Step names: lower-case kebab, ≤ 63 chars (usable as K8s label values and URL segments). */
+export const STEP_NAME = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
+/** Names that cannot be steps: `input` is the pipeline input's root in mappings. */
+export const RESERVED_STEP_NAMES: readonly string[] = ["input"];
+
+export interface StepDef<N extends string, I extends InputSpec, O extends AnyModel | null, Sec extends readonly Secret[], Cfg extends readonly AnyConfigEntry[] = []> {
+  name: LiteralName<N>;
+  /** What the step does, shown in the UI. Required; may be empty. */
+  description: string;
+  /** Any image: the harness brings its own runtime. */
+  image: string;
+  input?: I;
+  output?: O;
+  secrets?: Sec;
+  /** Config entries delivered as `ctx.config` (typed to exactly these). */
+  config?: Cfg;
+  timeoutSeconds?: number;
+  /** Automatic retries (an idempotency claim). Default 0. */
+  retries?: number;
+  /**
+   * Ask for a Kubernetes API token in the step pod: the instance's step ServiceAccount
+   * (`Kubernetes:StepServiceAccount`), if it grants one. Default false (no token).
+   */
+  clusterAccess?: boolean;
+  /** A label for the step's kind (default `custom`). */
+  kind?: StepKind;
+  /**
+   * How the step is set up, shown in the manifest and part of the step's identity: the parameters of
+   * a reusable step. JSON values and config references (`target.prod.host`); every config entry it
+   * references is delivered to the step, and `resolveConfigRefs(value, ctx.config)` resolves them.
+   */
+  settings?: Readonly<Record<string, unknown>>;
+}
+
+function makeStep(def: Record<string, unknown>, origin: string | undefined): AnyStep {
+  const config = new Map<string, AnyConfigEntry>();
+  for (const e of (def.config as readonly AnyConfigEntry[] | undefined) ?? []) if (!config.has(e.name)) config.set(e.name, e);
+  configRefsIn(def.settings, config);
+  const s = {
+    [STEP]: true,
+    kind: def.kind ?? "custom",
+    name: def.name,
+    description: def.description,
+    image: def.image,
+    input: Object.freeze({ ...((def.input as object | undefined) ?? {}) }),
+    output: def.output ?? null,
+    secrets: Object.freeze([...((def.secrets as readonly Secret[] | undefined) ?? [])]),
+    config: Object.freeze([...config.values()]),
+    ...(def.timeoutSeconds !== undefined ? { timeoutSeconds: def.timeoutSeconds } : {}),
+    ...(def.retries !== undefined ? { retries: def.retries } : {}),
+    ...(def.clusterAccess === true ? { clusterAccess: true } : {}),
+    ...(def.settings !== undefined ? { settings: def.settings } : {}),
+    [ORIGIN]: origin === undefined ? [] : [origin],
+  };
+  return Object.freeze(s) as unknown as AnyStep;
+}
+
+export function isStep(v: unknown): v is AnyStep {
+  return typeof v === "object" && v !== null && (v as Record<symbol, unknown>)[STEP] === true;
+}
+
+/**
+ * Define a step: a contract (name, image, inputs, output, secrets). Its handler is a `handlers/*.ts`
+ * default-exporting `implement(step, …)`, or, for a reusable step, `implement(step, …)` staged directly.
+ */
+export function step<
+  const N extends string,
+  const I extends InputSpec = {},
+  O extends AnyModel | null = null,
+  const Sec extends readonly Secret[] = [],
+  const Cfg extends readonly AnyConfigEntry[] = [],
+>(def: StepDef<N, I, O, Sec, Cfg>): Step<N, I, O, Sec[number]["name"], Cfg[number]> {
+  return makeStep(def as never, callerFile()) as never;
+}

package/src/core/types.ts

@@ -0,0 +1,6 @@
+export type Primitive = string | number | boolean | bigint | null | undefined;
+export type Simplify<T> = { [K in keyof T]: T[K] } & {};
+export type Awaitable<T> = T | Promise<T>;
+export type IsUnion<T, U = T> = T extends unknown ? ([U] extends [T] ? false : true) : never;
+/** `Ok` when there is no error, else the error (a readable string-literal type). */
+export type OrError<Err, Ok> = [Err] extends [never] ? Ok : Err;

package/src/core/webhooks.ts

@@ -0,0 +1,30 @@
+import type { Secret } from "./secrets.ts";
+
+/** How a webhook trigger authenticates deliveries. Build one with `hmacSha256`, `githubSignature` or `bearer`. */
+export type WebhookVerify =
+  | { readonly scheme: "hmac-sha256"; readonly secret: Secret; readonly header: string; readonly prefix: string }
+  | { readonly scheme: "bearer"; readonly secret: Secret };
+
+/** HMAC-SHA256 of the raw body keyed by `secret`, hex, in `header` after `prefix`. */
+export function hmacSha256(o: { secret: Secret; header: string; prefix?: string }): WebhookVerify {
+  return Object.freeze({ scheme: "hmac-sha256", secret: o.secret, header: o.header, prefix: o.prefix ?? "" });
+}
+
+/** GitHub's delivery signature: `X-Hub-Signature-256: sha256=<hex>`. */
+export function githubSignature(secret: Secret): WebhookVerify {
+  return hmacSha256({ secret, header: "X-Hub-Signature-256", prefix: "sha256=" });
+}
+
+/** `Authorization: Bearer <secret>`. */
+export function bearer(secret: Secret): WebhookVerify {
+  return Object.freeze({ scheme: "bearer", secret });
+}
+
+/**
+ * What a webhook trigger mapper receives: the request headers (lower-case names, without `authorization`
+ * and the signature header) and the body (parsed JSON, or the raw text when it isn't JSON).
+ */
+export interface WebhookEvent {
+  readonly headers: Readonly<Record<string, string>>;
+  readonly body: unknown;
+}

package/src/ctx/files.ts

@@ -0,0 +1,161 @@
+// File references: producing them (`ctx.output.*`) and fetching them (`ctx.files.fetch`).
+//
+// Storage rule (shared with the service): a ref's *object path* is `path` for files and
+// `path` minus the trailing "/" plus ".tar" for directories (a deterministic tar). The step writes
+// objects to `$OLVE_DIR/work/files/<object path>`; the service stores them under
+// `runs/<p>/<run>/<step>/<attempt>/files/<object path>`.
+import { copyFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { basename, dirname, join, resolve } from "node:path";
+import { fileRefSchema, FILE_PATH, type FileRef } from "../core/models.ts";
+import { ENV } from "../harness/env.ts";
+import { sha256File } from "../util/hash.ts";
+import { dirEntries, extractTar, writeTar } from "../util/tar.ts";
+import type { OutputFileOptions, RunInfo } from "./types.ts";
+
+export const objectPath = (refPath: string): string => (refPath.endsWith("/") ? refPath.slice(0, -1) + ".tar" : refPath);
+export const fileKey = (ref: Pick<FileRef, "run" | "step" | "attempt" | "path">): string => `${ref.run}/${ref.step}/${ref.attempt}/${ref.path}`;
+
+/** Strips everything from `?` on (pre-signed credentials). */
+export const redactUrl = (url: string): string => url.replace(/[?#].*$/, "?…");
+
+export interface FilesDeps {
+  run: RunInfo;
+  workFiles: string;
+  tmpDir: string;
+  root: string;
+  signal: AbortSignal;
+  log: (message: string, fields?: Record<string, unknown>) => void;
+}
+
+function checkName(name: string, what: string): void {
+  if (!FILE_PATH.test(name) || name.endsWith("/")) throw new Error(`${what}: '${name}' is not a valid relative file name (no leading '/', no '..', no trailing '/')`);
+}
+
+export function createOutput(deps: FilesDeps) {
+  const used = new Set<string>();
+  const claim = (obj: string) => {
+    if (used.has(obj) || existsSync(join(deps.workFiles, obj))) throw new Error(`output file '${obj}' already exists`);
+    used.add(obj);
+    const dest = join(deps.workFiles, obj);
+    mkdirSync(dirname(dest), { recursive: true });
+    return dest;
+  };
+  const ref = (path: string, dest: string): FileRef =>
+    ({ run: deps.run.number, step: deps.run.step, attempt: deps.run.attempt, path, sha256: sha256File(dest) }) as FileRef;
+
+  return {
+    async file(path: string, opts: OutputFileOptions = {}): Promise<FileRef> {
+      const src = resolve(path);
+      if (!statSync(src).isFile()) throw new Error(`ctx.output.file: '${path}' is not a file`);
+      const name = opts.name ?? basename(src);
+      checkName(name, "ctx.output.file");
+      const dest = claim(name);
+      copyFileSync(src, dest);
+      return ref(name, dest);
+    },
+    async dir(path: string, opts: OutputFileOptions = {}): Promise<FileRef> {
+      const src = resolve(path);
+      if (!statSync(src).isDirectory()) throw new Error(`ctx.output.dir: '${path}' is not a directory`);
+      const name = opts.name ?? basename(src);
+      checkName(name, "ctx.output.dir");
+      const dest = claim(objectPath(name + "/"));
+      writeTar(dest, dirEntries(src));
+      return ref(name + "/", dest);
+    },
+    async write(name: string, content: string | Uint8Array): Promise<FileRef> {
+      checkName(name, "ctx.output.write");
+      const dest = claim(name);
+      writeFileSync(dest, content);
+      return ref(name, dest);
+    },
+  };
+}
+
+function urlMap(root: string): Record<string, string> {
+  const raw = process.env[ENV.fileUrls] ?? (existsSync(join(root, "files.json")) ? readFileSync(join(root, "files.json"), "utf8") : undefined);
+  if (!raw) return {};
+  try {
+    const m = JSON.parse(raw) as unknown;
+    if (typeof m !== "object" || m === null || Array.isArray(m)) throw new Error("not an object");
+    return m as Record<string, string>;
+  } catch (e) {
+    throw new Error(`${ENV.fileUrls} / files.json is not a JSON object of file key → URL (${(e as Error).message})`);
+  }
+}
+
+const TRANSIENT = (status: number) => status >= 500 || status === 408 || status === 429;
+
+async function download(url: string, dest: string, headers: Record<string, string>, signal: AbortSignal, log: FilesDeps["log"]): Promise<void> {
+  const attempts = 4;
+  for (let i = 1; ; i++) {
+    let status = 0;
+    try {
+      const res = await fetch(url, { headers, signal });
+      status = res.status;
+      if (!res.ok) throw new Error(`HTTP ${res.status}`);
+      await Bun.write(dest + ".partial", res);
+      renameSync(dest + ".partial", dest);
+      return;
+    } catch (e) {
+      rmSync(dest + ".partial", { force: true });
+      const transient = status === 0 || TRANSIENT(status);
+      if (signal.aborted || !transient || i >= attempts) throw new Error(`download of ${redactUrl(url)} failed: ${(e as Error).message}`);
+      log("download failed, retrying", { url: redactUrl(url), attempt: i, error: (e as Error).message });
+      await Bun.sleep(1000 * 2 ** (i - 1));
+    }
+  }
+}
+
+export function createFetch(deps: FilesDeps) {
+  const cache = new Map<string, string>();
+  return async (input: FileRef): Promise<string> => {
+    const parsed = fileRefSchema.safeParse(input);
+    if (!parsed.success) throw new Error(`ctx.files.fetch: not a valid file reference: ${JSON.stringify(input)}`);
+    const ref = parsed.data;
+    const key = fileKey(ref);
+    const hit = cache.get(key);
+    if (hit) return hit;
+
+    const obj = objectPath(ref.path);
+    const base = join(deps.tmpDir, "files", String(ref.run), ref.step, String(ref.attempt));
+    const objFile = join(base, ".objects", obj);
+    mkdirSync(dirname(objFile), { recursive: true });
+
+    const own = ref.run === deps.run.number && ref.step === deps.run.step && ref.attempt === deps.run.attempt;
+    const urls = urlMap(deps.root);
+    const localDir = process.env[ENV.filesDir];
+    const baseUrl = process.env[ENV.filesBaseUrl];
+    if (own && existsSync(join(deps.workFiles, obj))) copyFileSync(join(deps.workFiles, obj), objFile);
+    else if (urls[key]) await download(urls[key], objFile, {}, deps.signal, deps.log);
+    else if (localDir && existsSync(join(localDir, String(ref.run), ref.step, String(ref.attempt), "files", obj)))
+      copyFileSync(join(localDir, String(ref.run), ref.step, String(ref.attempt), "files", obj), objFile);
+    else if (baseUrl) {
+      const token = process.env[ENV.filesToken];
+      const url = `${baseUrl.replace(/\/+$/, "")}/${ref.run}/${encodeURIComponent(ref.step)}/${ref.attempt}/${obj.split("/").map(encodeURIComponent).join("/")}`;
+      await download(url, objFile, token ? { authorization: `Bearer ${token}` } : {}, deps.signal, deps.log);
+    } else {
+      throw new Error(
+        `ctx.files.fetch: no source for ${key} (set ${ENV.fileUrls} / $OLVE_DIR/files.json, ${ENV.filesBaseUrl} or ${ENV.filesDir})`,
+      );
+    }
+
+    const actual = sha256File(objFile);
+    if (actual !== ref.sha256) {
+      rmSync(objFile, { force: true });
+      throw new Error(`ctx.files.fetch: ${key} has sha256 ${actual}, expected ${ref.sha256}`);
+    }
+    let result = objFile;
+    if (ref.path.endsWith("/")) {
+      result = join(base, ref.path.slice(0, -1));
+      rmSync(result, { recursive: true, force: true });
+      mkdirSync(result, { recursive: true });
+      await extractTar(objFile, result);
+    } else {
+      result = join(base, ref.path);
+      mkdirSync(dirname(result), { recursive: true });
+      renameSync(objFile, result);
+    }
+    cache.set(key, result);
+    return result;
+  };
+}

package/src/ctx/index.ts

@@ -0,0 +1,122 @@
+import { mkdirSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { skipped } from "../core/handlers.ts";
+import type { Secret } from "../core/secrets.ts";
+import { createFetch, createOutput, redactUrl } from "./files.ts";
+import { createSh } from "./sh.ts";
+import type { Ctx, Http, RetryOptions, RunInfo } from "./types.ts";
+
+export interface CtxOptions {
+  run: RunInfo;
+  /** Declared secrets of the step. */
+  secrets: readonly Secret[];
+  /** Secret values that take precedence over the environment (tests). */
+  secretValues?: Readonly<Record<string, string>>;
+  /** Parsed config entries of the step (`ctx.config`). */
+  config?: Readonly<Record<string, unknown>>;
+  paths: { root: string; workFiles: string; tmp: string };
+  signal: AbortSignal;
+  /** Where log lines go (default stderr). */
+  logSink?: (line: string) => void;
+}
+
+export function formatLog(message: string, fields?: Record<string, unknown>): string {
+  const parts = Object.entries(fields ?? {}).map(([k, v]) => `${k}=${typeof v === "string" && !/\s|"/.test(v) ? v : JSON.stringify(v)}`);
+  return parts.length ? `${message} ${parts.join(" ")}` : message;
+}
+
+export function createSecrets(declared: readonly Secret[], values?: Readonly<Record<string, string>>): Record<string, string> {
+  const obj: Record<string, string> = {};
+  for (const s of declared) {
+    Object.defineProperty(obj, s.name, {
+      enumerable: true,
+      get() {
+        const v = values?.[s.name] ?? process.env[s.name];
+        if (v === undefined) throw new Error(`secret ${s.name} is not set (the harness reads it from the environment variable ${s.name})`);
+        return v;
+      },
+    });
+  }
+  return Object.freeze(obj);
+}
+
+async function httpError(res: Response, url: string): Promise<Error> {
+  let body = "";
+  try {
+    body = (await res.text()).slice(0, 500);
+  } catch {
+    /* ignore */
+  }
+  return new Error(`HTTP ${res.status} ${res.statusText} from ${redactUrl(url)}${body ? `: ${body}` : ""}`);
+}
+
+export function createHttp(signal: AbortSignal): Http {
+  const f = (url: string, init?: RequestInit) => fetch(url, { ...init, signal: init?.signal ?? signal });
+  return {
+    fetch: f,
+    async getJson<T>(url: string, init?: { headers?: Record<string, string> }): Promise<T> {
+      const res = await f(url, { headers: { accept: "application/json", ...init?.headers } });
+      if (!res.ok) throw await httpError(res, url);
+      return (await res.json()) as T;
+    },
+    async expectOk(url: string, init?: { headers?: Record<string, string>; method?: string }): Promise<Response> {
+      const res = await f(url, { method: init?.method ?? "GET", headers: init?.headers });
+      if (!res.ok) throw await httpError(res, url);
+      return res;
+    },
+  };
+}
+
+export function createRetry(signal: AbortSignal, log: (m: string, f?: Record<string, unknown>) => void) {
+  return async <T>(opts: RetryOptions, fn: (attempt: number) => Promise<T>): Promise<T> => {
+    if (!(Number.isInteger(opts.attempts) && opts.attempts >= 1)) throw new Error("retry: attempts must be a positive integer");
+    let delay = Math.max(0, opts.delaySeconds) * 1000;
+    for (let attempt = 1; ; attempt++) {
+      try {
+        return await fn(attempt);
+      } catch (e) {
+        if (attempt >= opts.attempts || signal.aborted) throw e;
+        log("retrying", { attempt, of: opts.attempts, error: (e as Error)?.message ?? String(e) });
+        await new Promise<void>((res, rej) => {
+          const t = setTimeout(() => {
+            signal.removeEventListener("abort", onAbort);
+            res();
+          }, delay);
+          const onAbort = () => {
+            clearTimeout(t);
+            rej(new Error("retry cancelled: the step was cancelled"));
+          };
+          signal.addEventListener("abort", onAbort, { once: true });
+        });
+        delay *= opts.backoff ?? 1;
+      }
+    }
+  };
+}
+
+/** The real ctx handed to handlers by the harness (and by testHandler). */
+export function createCtx(o: CtxOptions): Ctx {
+  mkdirSync(o.paths.tmp, { recursive: true });
+  mkdirSync(o.paths.workFiles, { recursive: true });
+  const sink = o.logSink ?? ((line: string) => process.stderr.write(line + "\n"));
+  const log = (message: string, fields?: Record<string, unknown>) => sink(formatLog(message, fields));
+  const { sh, exec } = createSh({ tmpDir: o.paths.tmp, signal: o.signal });
+  const filesDeps = { run: o.run, workFiles: o.paths.workFiles, tmpDir: o.paths.tmp, root: o.paths.root, signal: o.signal, log };
+  return {
+    secrets: createSecrets(o.secrets, o.secretValues) as Ctx["secrets"],
+    config: Object.freeze({ ...o.config }) as Ctx["config"],
+    sh,
+    exec,
+    readText: async (p) => readFileSync(resolve(p), "utf8"),
+    readJson: async <T>(p: string) => JSON.parse(readFileSync(resolve(p), "utf8")) as T,
+    files: { fetch: createFetch(filesDeps) },
+    output: createOutput(filesDeps),
+    http: createHttp(o.signal),
+    retry: createRetry(o.signal, log),
+    log,
+    signal: o.signal,
+    run: Object.freeze({ ...o.run }),
+    tmpDir: o.paths.tmp,
+    skip: skipped,
+  };
+}