@olivervea/pipelines 0.0.0-stage → 2.0.0-beta.2

package/src/library/image-build.ts

@@ -0,0 +1,194 @@
+// steps.imageBuild — build a container image from a `Source` with kaniko, to an image tarball.
+//
+// No registry push: the tarball travels as a file ref and a deploy step imports it on the host.
+// Ported from v1's olve-lib.sh (olve_kaniko_build + build.sh's layer cache); footguns kept below.
+import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import type { Ctx, RunInfo } from "../ctx/types.ts";
+import type { FileRef } from "../core/models.ts";
+import { makeStep, type LibraryStep, type LiteralName, type Step } from "../core/steps.ts";
+import { BuiltImage, Source } from "./models.ts";
+
+/** kaniko's debug image (it has a busybox shell for debugging; the executor itself needs none). */
+export const KANIKO_IMAGE = "gcr.io/kaniko-project/executor:debug";
+
+/**
+ * Footgun: the build context must live under /kaniko, NOT /workspace (or another plain directory in
+ * the container): kaniko wipes the filesystem root between the stages of a multi-stage build, which
+ * breaks a later stage's COPY from the context. /kaniko survives stage transitions.
+ */
+export const KANIKO_CONTEXT = "/kaniko/build-context";
+
+const input = { source: Source };
+export type ImageBuildInput = typeof input;
+
+export interface ImageBuildDef<N extends string> {
+  name: LiteralName<N>;
+  /** What the step does, shown in the UI. Required; may be empty. */
+  description: string;
+  /** Image name, e.g. `olve-pipelines-v2`. The tag is the run's version (see `imageTag`). */
+  repository: string;
+  /**
+   * Registry repository for kaniko's layer cache (`--cache=true --cache-repo=…`), so unchanged
+   * layers (restore, apt, npm, …) are reused. kaniko pushes cache layers even with `--no-push`.
+   * A registry without GC grows with every changed layer — prune it when needed.
+   */
+  cacheRepo?: string;
+  /**
+   * kaniko `--registry-map` (registry → mirror prefix); kaniko falls back to the original registry if
+   * the mirror lacks the image. Defaults to the homelab pull-through caches (`DEFAULT_REGISTRY_MAP`);
+   * a map replaces the default entirely, `{}` turns mapping off.
+   */
+  registryMap?: Record<string, string>;
+  timeoutSeconds?: number;
+  /**
+   * Kill kaniko when it writes no output for this long (default 300 s; 0 disables), failing the
+   * attempt so `retries` can start a fresh one. Catches base-image pulls that stall on an idle
+   * connection (seen with mcr.microsoft.com), which kaniko itself never times out. Keep it well above
+   * the longest silent build phase (an AOT "Generating native code" runs ~2 min with no output).
+   */
+  stallTimeoutSeconds?: number;
+  /**
+   * Automatic retries of a failed attempt (a fresh pod: kaniko unpacks the base image over the
+   * container root and does not clean up when killed, so retrying inside the same pod is unsafe).
+   * A `--no-push` build is idempotent, so the default is 2 (three tries).
+   */
+  retries?: number;
+}
+
+/**
+ * Default `registryMap`: pull-through caches for mcr.microsoft.com and Docker Hub. kaniko under gVisor
+ * can stall forever pulling from mcr.microsoft.com directly.
+ */
+export const DEFAULT_REGISTRY_MAP: Readonly<Record<string, string>> = {
+  "mcr.microsoft.com": "mcr-cache.ovea.pro",
+  "index.docker.io": "hub-cache.ovea.pro",
+};
+
+/** How long a cache registry gets to answer `GET /v2/` before the build goes without it. */
+export const CACHE_PROBE_TIMEOUT_MS = 5000;
+
+/** The registry host of an image reference or mirror prefix (`registry.ovea.pro/a/b` → `registry.ovea.pro`). */
+export function registryHost(ref: string): string {
+  return ref.split("/")[0]!;
+}
+
+/** Whether a registry answers its API root: 200, or 401 (it wants a token). Never throws. */
+export async function registryUp(http: Ctx["http"], host: string, signal?: AbortSignal, timeoutMs = CACHE_PROBE_TIMEOUT_MS): Promise<boolean> {
+  try {
+    const timeout = AbortSignal.timeout(timeoutMs);
+    const res = await http.fetch(`https://${host}/v2/`, { signal: signal ? AbortSignal.any([signal, timeout]) : timeout });
+    await res.body?.cancel().catch(() => {});
+    return res.status === 200 || res.status === 401;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Drops the caches that don't answer, so a cache outage (GC, maintenance) never blocks a build: an
+ * unreachable mirror is left out of the registry map, an unreachable `cacheRepo` means no layer cache.
+ */
+export async function reachableCaches(
+  ctx: Pick<Ctx, "http" | "log" | "signal">,
+  caches: { registryMap: Record<string, string>; cacheRepo?: string },
+): Promise<{ registryMap: Record<string, string>; cacheRepo?: string }> {
+  const hosts = new Set([...Object.values(caches.registryMap), ...(caches.cacheRepo ? [caches.cacheRepo] : [])].map(registryHost));
+  const up = new Map(await Promise.all([...hosts].map(async (h) => [h, await registryUp(ctx.http, h, ctx.signal)] as const)));
+  const registryMap: Record<string, string> = {};
+  for (const [from, to] of Object.entries(caches.registryMap)) {
+    if (up.get(registryHost(to))) registryMap[from] = to;
+    else ctx.log(`${registryHost(to)} unreachable; pulling from ${from} directly`);
+  }
+  if (caches.cacheRepo && !up.get(registryHost(caches.cacheRepo))) {
+    ctx.log(`${registryHost(caches.cacheRepo)} unreachable; building without the layer cache`);
+    return { registryMap };
+  }
+  return { registryMap, ...(caches.cacheRepo ? { cacheRepo: caches.cacheRepo } : {}) };
+}
+
+/** Default `retries`: a stalled or flaky build gets two fresh attempts before the run fails. */
+export const DEFAULT_RETRIES = 2;
+
+/** Default `stallTimeoutSeconds`: well above the longest silent phase of a normal build. */
+export const DEFAULT_STALL_TIMEOUT_SECONDS = 300;
+
+/**
+ * The image tag for this run: the run's version stamp, plus `-attempt<n>` on reruns. A rebuilt image
+ * must not reuse the tag: deploys import it with `pullPolicy=Never`, so an unchanged tag would leave
+ * the old pods running.
+ */
+export function imageTag(run: Pick<RunInfo, "version" | "attempt">): string {
+  const tag = run.attempt > 1 ? `${run.version}-attempt${run.attempt}` : run.version;
+  if (!/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(tag)) throw new Error(`'${tag}' (from the run's version stamp) is not a valid image tag`);
+  return tag;
+}
+
+/** The name a container runtime gives an image after loading it (`x` → `docker.io/library/x`). */
+export function qualifyRepository(repository: string): string {
+  const first = repository.split("/")[0]!;
+  if (!repository.includes("/")) return `docker.io/library/${repository}`;
+  if (first.includes(".") || first.includes(":") || first === "localhost") return repository;
+  return `docker.io/${repository}`;
+}
+
+/**
+ * Builds `<source>/Dockerfile` with the source root as context, using kaniko, into an image tarball.
+ * Output: `BuiltImage { repository, tag, digest, tar }`.
+ */
+export function imageBuild<const N extends string>(def: ImageBuildDef<N>): Step<N, ImageBuildInput, typeof BuiltImage, never> & LibraryStep {
+  const registryMap = def.registryMap ?? DEFAULT_REGISTRY_MAP;
+  const mapped = Object.keys(registryMap).length > 0;
+  return makeStep(
+    "imageBuild",
+    { name: def.name, description: def.description, image: KANIKO_IMAGE, input, output: BuiltImage, secrets: [], timeoutSeconds: def.timeoutSeconds, retries: def.retries ?? DEFAULT_RETRIES },
+    {
+      manifestConfig: () => ({
+        repository: def.repository,
+        ...(def.cacheRepo ? { cacheRepo: def.cacheRepo } : {}),
+        ...(mapped ? { registryMap: { ...registryMap } } : {}),
+      }),
+      async run({ source }: { source: { tree: FileRef } }, ctx: Ctx) {
+        const tag = imageTag(ctx.run);
+        const caches = await reachableCaches(ctx, { registryMap, cacheRepo: def.cacheRepo });
+        const tree = await ctx.files.fetch(source.tree);
+        await ctx.exec(["mkdir", "-p", KANIKO_CONTEXT]);
+        await ctx.exec(["cp", "-r", `${tree}/.`, KANIKO_CONTEXT]);
+
+        // Outputs go to the harness's scratch dir (/olve/tmp, an emptyDir mount kaniko leaves alone;
+        // kaniko images have no /tmp), then into the step's files.
+        mkdirSync(ctx.tmpDir, { recursive: true });
+        const out = mkdtempSync(join(ctx.tmpDir, "image-build-"));
+        const tarPath = join(out, "image.tar");
+        const digestPath = join(out, "digest");
+        // No --single-snapshot: it is incompatible with multi-stage Dockerfiles whose later stages need
+        // the build context intact. --skip-push-permission-check: with --no-push kaniko otherwise checks
+        // the cache repo at startup and exits if it is down; later cache lookup and push errors are
+        // only warnings.
+        await ctx.exec([
+          "/kaniko/executor",
+          `--context=${KANIKO_CONTEXT}`,
+          `--dockerfile=${KANIKO_CONTEXT}/Dockerfile`,
+          "--no-push",
+          `--tar-path=${tarPath}`,
+          `--digest-file=${digestPath}`,
+          `--destination=${def.repository}:${tag}`,
+          ...(caches.cacheRepo ? ["--cache=true", `--cache-repo=${caches.cacheRepo}`, "--skip-push-permission-check"] : []),
+          ...Object.entries(caches.registryMap).map(([from, to]) => `--registry-map=${from}=${to}`),
+        ], { idleTimeoutSeconds: def.stallTimeoutSeconds ?? DEFAULT_STALL_TIMEOUT_SECONDS });
+
+        let digest: string;
+        try {
+          digest = readFileSync(digestPath, "utf8").trim();
+        } catch {
+          throw new Error(`kaniko did not write the image digest to ${digestPath}`);
+        }
+        if (!/^sha256:[0-9a-f]{64}$/.test(digest)) throw new Error(`kaniko wrote an unexpected digest: '${digest}'`);
+        const tar = await ctx.output.file(tarPath, { name: "image.tar" });
+        rmSync(tarPath); // the copy in the step's files is the one that counts; don't hold the image twice in /olve
+        ctx.log("image built", { image: `${def.repository}:${tag}`, digest });
+        return { repository: qualifyRepository(def.repository), tag, digest, tar };
+      },
+    },
+  ) as never;
+}

package/src/library/index.ts

@@ -0,0 +1,14 @@
+import { gitHubSource } from "./github-source.ts";
+import { helmDeployOverSsh } from "./helm-deploy-ssh.ts";
+import { imageBuild } from "./image-build.ts";
+import { script } from "./script.ts";
+
+export { Source, BuiltImage, HelmRelease, FailureContext } from "./models.ts";
+export { inputEnvName, type RunValue, type ScriptDef } from "./script.ts";
+export type { GitHubSourceDef } from "./github-source.ts";
+export type { ImageBuildDef } from "./image-build.ts";
+export type { HelmDeployOverSshDef } from "./helm-deploy-ssh.ts";
+export { httpHealthCheck, HTTP_HEALTH_DEFAULTS, type HttpHealthCheckOptions } from "./http-health.ts";
+
+/** SDK-provided steps (step + handler pairs with typed config). */
+export const steps = { script, gitHubSource, imageBuild, helmDeployOverSsh };

package/src/library/models.ts

@@ -0,0 +1,63 @@
+import { z } from "zod";
+import { fileRef, model } from "../core/models.ts";
+
+/** A source tree at a commit. `tree` references a directory (`path` ends with `/`). */
+export const Source = model("Source", z.object({ repo: z.string(), sha: z.string(), tree: fileRef() }));
+
+/**
+ * A container image built by a step, as an image tarball (`docker save` format) in the step's files.
+ * `repository` is fully qualified the way a container runtime names it after loading the tar
+ * (`olve-pipelines-v2` → `docker.io/library/olve-pipelines-v2`).
+ */
+export const BuiltImage = model(
+  "BuiltImage",
+  z.object({ repository: z.string(), tag: z.string(), digest: z.string(), tar: fileRef() }),
+);
+
+/** A Helm release that was installed or upgraded. */
+export const HelmRelease = model(
+  "HelmRelease",
+  z.object({ release: z.string(), namespace: z.string(), image: z.string() }),
+);
+
+/** One failed step of a run, as seen by a failure handler. */
+export const FailedStep = z.object({
+  step: z.string(),
+  attempt: z.number().int(),
+  /** `<pipeline id>#<run>/<step>@<attempt>` (spec §5.5). */
+  address: z.string(),
+  /** The attempt's failure reason (handler error, exit code, timeout, …). */
+  reason: z.string(),
+  /** The attempt's log (`GET …/runs/<n>/steps/<step>/logs?attempt=<a>`). */
+  logUrl: z.string(),
+});
+
+/**
+ * What a failure handler receives (`.onFailure(handler, …)`). The service builds it when a run fails and
+ * runs the handler once per failure, untracked and best-effort (not a run attempt). `failedSteps` lists
+ * the run's failed steps the handler is scoped to (all of them for an unscoped handler); never empty.
+ */
+export const FailureContext = model(
+  "FailureContext",
+  z.object({
+    pipeline: z.object({ id: z.string(), name: z.string() }),
+    run: z.object({
+      id: z.string(),
+      number: z.number().int(),
+      /** `<pipeline id>#<run>` (spec §5.5). */
+      address: z.string(),
+      /** The run's derived version, `yyyy.MM.dd.<run>`. */
+      versionStamp: z.string(),
+      trigger: z.enum(["push", "poll", "manual"]),
+      /** The run (`GET …/api/pipelines/<id>/runs/<n>`). */
+      url: z.string(),
+    }),
+    /** The pipeline version the run is pinned to, and the commit it was built from. */
+    version: z.object({ number: z.number().int(), commit: z.string().nullable() }),
+    /** The run's pipeline input (the pipeline input model's output view). */
+    input: z.unknown(),
+    /** One line summarising why the run failed. */
+    reason: z.string(),
+    failedSteps: z.array(FailedStep),
+  }),
+);

package/src/library/script.ts

@@ -0,0 +1,85 @@
+import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { Ctx, RunInfo } from "../ctx/types.ts";
+import type { AnyModel } from "../core/models.ts";
+import type { Secret } from "../core/secrets.ts";
+import { makeStep, type InputSpec, type LibraryStep, type Step, type StepDef, type StepInput } from "../core/steps.ts";
+import { sha256Hex } from "../util/hash.ts";
+
+/** A config value that may depend on the run: a constant or a function of the step's inputs and run info. */
+export type RunValue<I, T = string> = T | ((input: StepInput<I>, run: RunInfo) => T);
+
+export interface ScriptDef<N extends string, I extends InputSpec, O extends AnyModel | null, Sec extends readonly Secret[]>
+  extends StepDef<N, I, O, Sec> {
+  /** An embedded script: `import s from "./scripts/x.sh" with { type: "file" }`. */
+  script: string;
+  /** Extra environment; values may be functions of the inputs (evaluated in the step, not at describe). */
+  env?: Record<string, RunValue<I>>;
+}
+
+/** `deployTo` → `INPUT_DEPLOY_TO`. */
+export function inputEnvName(key: string): string {
+  return "INPUT_" + key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").replace(/[^A-Za-z0-9]+/g, "_").toUpperCase();
+}
+
+/**
+ * Runs a shell script (the zero-TypeScript path). Contract for the script:
+ *  - `$OLVE_INPUT`: path of the validated input as JSON; each input key is also `INPUT_<KEY>`
+ *    (strings raw, everything else JSON).
+ *  - `$OLVE_OUTPUT`: if the step has an output model, write the output JSON there. Objects of the
+ *    form `{"$file": "<path>"}` / `{"$dir": "<path>"}` are turned into file references.
+ *  - Declared secrets are in the environment under their names.
+ */
+export function script<
+  const N extends string,
+  const I extends InputSpec = {},
+  O extends AnyModel | null = null,
+  const Sec extends readonly Secret[] = [],
+>(def: ScriptDef<N, I, O, Sec>): Step<N, I, O, Sec[number]["name"]> & LibraryStep {
+  const env = def.env ?? {};
+  return makeStep("script", def as never, {
+    manifestConfig: () => ({
+      scriptSha256: "sha256:" + sha256Hex(readFileSync(def.script)),
+      env: Object.fromEntries(Object.entries(env).filter(([, v]) => typeof v === "string")),
+      computedEnv: Object.keys(env).filter((k) => typeof env[k] === "function").sort(),
+    }),
+    async run(input: Record<string, unknown>, ctx: Ctx) {
+      mkdirSync(ctx.tmpDir, { recursive: true });
+      const dir = mkdtempSync(join(ctx.tmpDir, "script-"));
+      const inputPath = join(dir, "input.json");
+      const outputPath = join(dir, "output.json");
+      writeFileSync(inputPath, JSON.stringify(input));
+      const vars: Record<string, string> = { OLVE_INPUT: inputPath };
+      for (const [k, v] of Object.entries(input)) {
+        if (v !== undefined) vars[inputEnvName(k)] = typeof v === "string" ? v : JSON.stringify(v);
+      }
+      for (const [k, v] of Object.entries(env)) {
+        vars[k] = typeof v === "function" ? (v as (i: unknown, r: RunInfo) => string)(input, ctx.run) : v;
+      }
+      if (def.output) vars.OLVE_OUTPUT = outputPath;
+      await ctx.sh({ script: def.script }, { env: vars });
+      if (!def.output) return undefined;
+      let raw: unknown;
+      try {
+        raw = JSON.parse(readFileSync(outputPath, "utf8"));
+      } catch (e) {
+        throw new Error(`script did not write valid JSON to $OLVE_OUTPUT: ${(e as Error).message}`);
+      }
+      return resolveFilePlaceholders(raw, ctx);
+    },
+  }) as never;
+}
+
+async function resolveFilePlaceholders(v: unknown, ctx: Ctx): Promise<unknown> {
+  if (Array.isArray(v)) return Promise.all(v.map((x) => resolveFilePlaceholders(x, ctx)));
+  if (typeof v === "object" && v !== null) {
+    const o = v as Record<string, unknown>;
+    const keys = Object.keys(o);
+    if (keys.length === 1 && typeof o.$file === "string") return ctx.output.file(o.$file);
+    if (keys.length === 1 && typeof o.$dir === "string") return ctx.output.dir(o.$dir);
+    const out: Record<string, unknown> = {};
+    for (const k of keys) out[k] = await resolveFilePlaceholders(o[k], ctx);
+    return out;
+  }
+  return v;
+}

package/src/testing/index.ts

@@ -0,0 +1,86 @@
+// @olve/pipelines/testing — run a handler under `bun test` through the real harness path
+// (validate input → handler → strict output validation), with a real ctx you can override.
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { isImplementation, type Implementation } from "../core/handlers.ts";
+import type { FileRef } from "../core/models.ts";
+import { isStep, type AnyStep, type LibraryStep, type OutputValue, type SecretsOf, type StepInput } from "../core/steps.ts";
+import { createCtx } from "../ctx/index.ts";
+import type { Ctx, RunInfo } from "../ctx/types.ts";
+import { executeStep, type FailureKind, type StepFunction } from "../harness/execute.ts";
+import { sha256Hex } from "../util/hash.ts";
+
+export interface TestHandlerOptions<S extends AnyStep> {
+  /** Replace parts of ctx (e.g. `sh`, `http`, `files`). The rest is the real implementation. */
+  ctx?: Partial<Ctx<SecretsOf<S>>>;
+  /** Secret values (instead of reading the environment). */
+  secrets?: Partial<Record<SecretsOf<S>, string>>;
+  run?: Partial<Omit<RunInfo, "step">>;
+  /** Directory used as $OLVE_DIR (default: a fresh temp dir). */
+  dir?: string;
+  /** Abort signal handed to ctx.signal. */
+  signal?: AbortSignal;
+}
+
+export type TestResult<S extends AnyStep> = {
+  /** $OLVE_DIR used for the run; output files are under `<dir>/work/files`. */
+  dir: string;
+  filesDir: string;
+  logs: string[];
+  exitCode: number;
+} & (
+  | { status: "succeeded"; output: OutputValue<S> }
+  | { status: "skipped"; reason: string }
+  | { status: "failed"; error: { kind: FailureKind; message: string } }
+);
+
+/**
+ * Runs a handler (the default export of `handlers/<step>.ts`) or a library step with `input`.
+ * Input is validated like `input.json`, the result like a real run: an invalid output gives
+ * `{ status: "failed", error: { kind: "output" } }`, a throwing handler `kind: "handler"`.
+ */
+export async function testHandler<S extends AnyStep>(
+  target: Implementation<S> | (S & LibraryStep),
+  input: StepInput<S["input"]>,
+  opts: TestHandlerOptions<S> = {},
+): Promise<TestResult<S>> {
+  let step: AnyStep;
+  let fn: StepFunction;
+  if (isImplementation(target)) {
+    step = target.step;
+    fn = target.handler as unknown as StepFunction;
+  } else if (isStep(target) && target.builtin) {
+    step = target;
+    fn = target.builtin.run as StepFunction;
+  } else throw new TypeError("testHandler expects implement(...) (a handler module's default export) or a library step");
+
+  const dir = opts.dir ?? mkdtempSync(join(tmpdir(), `olve-test-${step.name}-`));
+  const filesDir = join(dir, "work", "files");
+  const logs: string[] = [];
+  const run: RunInfo = { pipeline: "test", id: "test", number: 1, attempt: 1, version: "0.0.0-test", ...opts.run, step: step.name };
+  const real = createCtx({
+    run,
+    secrets: step.secrets,
+    secretValues: opts.secrets as Record<string, string> | undefined,
+    paths: { root: dir, workFiles: filesDir, tmp: join(dir, "tmp") },
+    signal: opts.signal ?? new AbortController().signal,
+    logSink: (l) => logs.push(l),
+  });
+  const ctx = { ...real, ...opts.ctx } as Ctx;
+  // Round-trip through JSON like input.json does (drops undefined, rejects non-JSON values early).
+  const raw = JSON.parse(JSON.stringify(input ?? {}));
+  const exec = await executeStep({ step, fn, rawInput: raw, ctx, workFiles: filesDir, log: (l) => logs.push(l) });
+  return { dir, filesDir, logs, exitCode: exec.exitCode, ...exec.envelope } as TestResult<S>;
+}
+
+/** A file reference for test inputs (`sha256` defaults to the hash of `content`). */
+export function testFileRef(ref: { step: string; path: string; run?: number; attempt?: number; sha256?: string; content?: string | Uint8Array }): FileRef {
+  return {
+    run: ref.run ?? 1,
+    step: ref.step,
+    attempt: ref.attempt ?? 1,
+    path: ref.path,
+    sha256: ref.sha256 ?? sha256Hex(ref.content ?? ""),
+  } as FileRef;
+}

package/src/util/canonical.ts

@@ -0,0 +1,19 @@
+// Canonical JSON: object keys sorted recursively (by UTF-16 code units), arrays kept in order
+// (order is meaningful: stages, steps, enum values), `undefined` members dropped. Stable for hashing.
+export function canonicalize(v: unknown): unknown {
+  if (Array.isArray(v)) return v.map(canonicalize);
+  if (v !== null && typeof v === "object") {
+    const out: Record<string, unknown> = {};
+    for (const k of Object.keys(v).sort()) {
+      const x = (v as Record<string, unknown>)[k];
+      if (x !== undefined) out[k] = canonicalize(x);
+    }
+    return out;
+  }
+  if (typeof v === "number" && !Number.isFinite(v)) throw new Error(`non-finite number in canonical JSON: ${v}`);
+  if (typeof v === "bigint" || typeof v === "function" || typeof v === "symbol") throw new Error(`${typeof v} is not JSON`);
+  return v;
+}
+
+/** Canonical JSON text, 2-space indented, with a trailing newline. */
+export const canonicalJson = (v: unknown): string => JSON.stringify(canonicalize(v), null, 2) + "\n";

package/src/util/handler-hash.ts

@@ -0,0 +1,22 @@
+import { canonicalJson } from "./canonical.ts";
+import { sha256Hex } from "./hash.ts";
+
+/** Identity of the harness compiled into a binary. A Bun or SDK bump legitimately bumps every handler. */
+export interface HarnessInfo {
+  /** Bun version (runtime compiled into the binary). */
+  bun: string;
+  /** @olve/pipelines package version. */
+  sdk: string;
+  /** `sha256:` hash over the SDK runtime sources and the zod version. */
+  sdkHash: string;
+}
+
+/** Custom step: hash of the handler bundle (contract files + SDK external) combined with the harness. */
+export function composeHandlerHash(bundleHash: string, harness: HarnessInfo): string {
+  return "sha256:" + sha256Hex(canonicalJson({ bundle: bundleHash, harness }));
+}
+
+/** Library step: kind + canonical config + harness. */
+export function libraryHandlerHash(kind: string, config: unknown, harness: HarnessInfo): string {
+  return "sha256:" + sha256Hex(canonicalJson({ library: kind, config, harness }));
+}

package/src/util/hash.ts

@@ -0,0 +1,22 @@
+import { openSync, readSync, closeSync } from "node:fs";
+
+export function sha256Hex(data: string | Uint8Array | ArrayBuffer): string {
+  return new Bun.CryptoHasher("sha256").update(data).digest("hex");
+}
+
+/** Streaming SHA-256 of a file (constant memory). */
+export function sha256File(path: string): string {
+  const h = new Bun.CryptoHasher("sha256");
+  const fd = openSync(path, "r");
+  try {
+    const buf = new Uint8Array(1 << 20);
+    for (;;) {
+      const n = readSync(fd, buf, 0, buf.length, null);
+      if (n <= 0) break;
+      h.update(buf.subarray(0, n));
+    }
+  } finally {
+    closeSync(fd);
+  }
+  return h.digest("hex");
+}

package/src/util/tar.ts

@@ -0,0 +1,154 @@
+// Deterministic tar writer (ustar + PAX long names), streaming from disk. Entries are sorted,
+// with mtime 0, uid/gid 0, no user names — so equal content gives byte-identical archives (and
+// sha256s). Extraction uses Bun.Archive.
+import { closeSync, createReadStream, createWriteStream, lstatSync, openSync, readdirSync, readlinkSync, readSync, renameSync, rmSync, writeSync } from "node:fs";
+import { join } from "node:path";
+import { pipeline } from "node:stream/promises";
+import { createGzip } from "node:zlib";
+
+export type TarEntry =
+  | { path: string; type: "file"; mode: number; source: string }
+  | { path: string; type: "bytes"; mode: number; data: Uint8Array }
+  | { path: string; type: "dir"; mode: number }
+  | { path: string; type: "symlink"; target: string };
+
+const enc = new TextEncoder();
+
+function octal(n: number, len: number): string {
+  return n.toString(8).padStart(len - 1, "0") + "\0";
+}
+
+function header(name: string, size: number, mode: number, typeflag: string, linkname = ""): Uint8Array {
+  const h = new Uint8Array(512);
+  const put = (s: string | Uint8Array, off: number, len: number) => {
+    const b = typeof s === "string" ? enc.encode(s) : s;
+    h.set(b.subarray(0, len), off);
+  };
+  put(name, 0, 100);
+  put(octal(mode & 0o7777, 8), 100, 8);
+  put(octal(0, 8), 108, 8);
+  put(octal(0, 8), 116, 8);
+  put(octal(size, 12), 124, 12);
+  put(octal(0, 12), 136, 12);
+  put("        ", 148, 8); // checksum placeholder
+  put(typeflag, 156, 1);
+  put(linkname, 157, 100);
+  put("ustar\0", 257, 6);
+  put("00", 263, 2);
+  let sum = 0;
+  for (const b of h) sum += b;
+  put(octal(sum, 7) + " ", 148, 8);
+  return h;
+}
+
+function paxRecord(key: string, value: string): string {
+  const body = ` ${key}=${value}\n`;
+  let len = enc.encode(body).length;
+  let total = len + String(len).length;
+  if (String(total).length !== String(len).length) total = len + String(total).length;
+  return `${total}${body}`;
+}
+
+/** Writes `entries` (sorted by path here) as an uncompressed tar to `out`. */
+export function writeTar(out: string, entries: readonly TarEntry[]): void {
+  const sorted = [...entries].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const tmp = `${out}.partial`;
+  const fd = openSync(tmp, "w");
+  const write = (b: Uint8Array) => {
+    let off = 0;
+    while (off < b.length) off += writeSync(fd, b, off, b.length - off);
+  };
+  const pad = (size: number) => {
+    const r = size % 512;
+    if (r) write(new Uint8Array(512 - r));
+  };
+  try {
+    for (const e of sorted) {
+      const name = e.type === "dir" && !e.path.endsWith("/") ? e.path + "/" : e.path;
+      const link = e.type === "symlink" ? e.target : "";
+      let size = 0;
+      if (e.type === "file") size = lstatSync(e.source).size;
+      else if (e.type === "bytes") size = e.data.length;
+      const pax: string[] = [];
+      if (enc.encode(name).length > 100) pax.push(paxRecord("path", name));
+      if (enc.encode(link).length > 100) pax.push(paxRecord("linkpath", link));
+      if (pax.length) {
+        const body = enc.encode(pax.join(""));
+        write(header("././@PaxHeader", body.length, 0o644, "x"));
+        write(body);
+        pad(body.length);
+      }
+      const shortName = enc.encode(name).length > 100 ? name.slice(0, 99) : name;
+      const mode = e.type === "symlink" ? 0o777 : e.mode;
+      const flag = e.type === "dir" ? "5" : e.type === "symlink" ? "2" : "0";
+      write(header(shortName, size, mode, flag, link.length > 100 ? link.slice(0, 99) : link));
+      if (e.type === "bytes") {
+        write(e.data);
+        pad(size);
+      } else if (e.type === "file") {
+        const src = openSync(e.source, "r");
+        try {
+          const buf = new Uint8Array(1 << 20);
+          let left = size;
+          while (left > 0) {
+            const n = readSync(src, buf, 0, Math.min(buf.length, left), null);
+            if (n <= 0) throw new Error(`${e.source} shrank while archiving`);
+            write(buf.subarray(0, n));
+            left -= n;
+          }
+        } finally {
+          closeSync(src);
+        }
+        pad(size);
+      }
+    }
+    write(new Uint8Array(1024));
+  } finally {
+    closeSync(fd);
+  }
+  renameSync(tmp, out);
+}
+
+/** Entries for the contents of `dir` (recursively), with paths prefixed by `prefix` ("" = none). */
+export function dirEntries(dir: string, prefix = ""): TarEntry[] {
+  const out: TarEntry[] = [];
+  const walk = (abs: string, rel: string) => {
+    for (const name of readdirSync(abs).sort()) {
+      const a = join(abs, name);
+      const r = rel ? `${rel}/${name}` : name;
+      const st = lstatSync(a);
+      if (st.isSymbolicLink()) out.push({ path: r, type: "symlink", target: readlinkSync(a) });
+      else if (st.isDirectory()) {
+        out.push({ path: r, type: "dir", mode: 0o755 });
+        walk(a, r);
+      } else if (st.isFile()) out.push({ path: r, type: "file", mode: st.mode & 0o111 ? 0o755 : 0o644, source: a });
+      // sockets, fifos, devices are skipped
+    }
+  };
+  walk(dir, prefix);
+  return out;
+}
+
+/** gzip `src` into `out` (streaming; zlib writes mtime 0, so output is deterministic). */
+export async function gzipFile(src: string, out: string, level = 6): Promise<void> {
+  const tmp = `${out}.partial`;
+  await pipeline(createReadStream(src), createGzip({ level }), createWriteStream(tmp));
+  renameSync(tmp, out);
+}
+
+/** Writes a deterministic `.tar.gz`. */
+export async function writeTarGz(out: string, entries: readonly TarEntry[], level = 6): Promise<void> {
+  const tar = `${out}.tar-tmp`;
+  try {
+    writeTar(tar, entries);
+    await gzipFile(tar, out, level);
+  } finally {
+    rmSync(tar, { force: true });
+  }
+}
+
+/** Extracts a tar or tar.gz file into `dir`. */
+export async function extractTar(archive: string, dir: string): Promise<void> {
+  const a = new Bun.Archive(await Bun.file(archive).bytes());
+  await a.extract(dir);
+}