@olivervea/pipelines 2.0.0-beta.2 → 2.0.0-beta.20

package/src/harness/validate.ts

@@ -1,22 +1,30 @@
-import { z } from "zod";
-import { isModel, isOptional } from "../core/models.ts";
-import type { AnyStep } from "../core/steps.ts";
+import { z } from 'zod';
+import { isModel, isOptional } from '../core/models.ts';
+import { isExternal } from '../core/external.ts';
+import type { AnyStep } from '../core/steps.ts';
 
 export type Validation<T> = { ok: true; value: T } | { ok: false; issues: string[] };
 
 export function formatIssues(err: z.ZodError, prefix: string): string[] {
-  return err.issues.map((i) => `${prefix}${i.path.length ? "." + i.path.map(String).join(".") : ""}: ${i.message}`);
+  return err.issues.map(
+    (i) =>
+      `${prefix}${i.path.length ? '.' + i.path.map(String).join('.') : ''}: ${i.message}`,
+  );
 }
 
 const isPlainObject = (v: unknown): v is Record<string, unknown> =>
-  typeof v === "object" && v !== null && !Array.isArray(v) && (Object.getPrototypeOf(v) === Object.prototype || Object.getPrototypeOf(v) === null);
+  typeof v === 'object' &&
+  v !== null &&
+  !Array.isArray(v) &&
+  (Object.getPrototypeOf(v) === Object.prototype || Object.getPrototypeOf(v) === null);
 
 /**
  * zod strips unknown keys silently (at every depth). Compares the raw value with the parsed one:
  * any key present in a raw object but absent from the parsed object at the same place was stripped.
  */
-export function strippedKeys(raw: unknown, parsed: unknown, path = ""): string[] {
-  if (Array.isArray(raw) && Array.isArray(parsed)) return raw.flatMap((x, i) => strippedKeys(x, parsed[i], `${path}[${i}]`));
+export function strippedKeys(raw: unknown, parsed: unknown, path = ''): string[] {
+  if (Array.isArray(raw) && Array.isArray(parsed))
+    return raw.flatMap((x, i) => strippedKeys(x, parsed[i], `${path}[${i}]`));
   if (isPlainObject(raw) && isPlainObject(parsed)) {
     const out: string[] = [];
     for (const k of Object.keys(raw)) {
@@ -30,27 +38,76 @@ export function strippedKeys(raw: unknown, parsed: unknown, path = ""): string[]
 }
 
 /** Strict parse: zod validation plus rejection of unknown keys at any depth. */
-export function parseStrict(schema: z.ZodType, value: unknown, prefix: string): Validation<unknown> {
+export function parseStrict(
+  schema: z.ZodType,
+  value: unknown,
+  prefix: string,
+): Validation<unknown> {
   const r = schema.safeParse(value);
   if (!r.success) return { ok: false, issues: formatIssues(r.error, prefix) };
   const extra = strippedKeys(value, r.data);
-  if (extra.length) return { ok: false, issues: extra.map((k) => `${prefix}.${k}: unknown field (outputs are validated strictly)`) };
+  if (extra.length)
+    return {
+      ok: false,
+      issues: extra.map(
+        (k) => `${prefix}.${k}: unknown field (outputs are validated strictly)`,
+      ),
+    };
   return { ok: true, value: r.data };
 }
 
 function entrySchema(e: unknown): z.ZodType {
-  if (isOptional(e)) return e.model.schema;
+  if (isOptional(e) || isExternal(e)) return e.model.schema;
   if (isModel(e)) return e.schema;
   return e as z.ZodType;
 }
 
+/** Reserved input.json key holding the step's config entries (written by the service). */
+export const CONFIG_KEY = '$config';
+
+/** Parses `$config` with each declared entry's schema (defaults applied). Undeclared entries are ignored. */
+export function validateConfig(
+  step: AnyStep,
+  raw: unknown,
+): Validation<Record<string, unknown>> {
+  const given = isPlainObject(raw) ? raw[CONFIG_KEY] : undefined;
+  if (given !== undefined && !isPlainObject(given))
+    return {
+      ok: false,
+      issues: [`input.${CONFIG_KEY} must be an object keyed by config entry names`],
+    };
+  const issues: string[] = [];
+  const out: Record<string, unknown> = {};
+  for (const e of step.config ?? []) {
+    const value = given?.[e.name];
+    const r = e.schema.safeParse(value);
+    if (!r.success)
+      issues.push(
+        ...formatIssues(r.error, `config.${e.name}`).map((i) =>
+          value === undefined ? `${i} (config entry '${e.name}' is not set)` : i,
+        ),
+      );
+    else if (r.data !== undefined) out[e.name] = r.data;
+  }
+  return issues.length ? { ok: false, issues } : { ok: true, value: out };
+}
+
 /** Validates input.json against the step's input entries. Unknown top-level keys are rejected; nested values are parsed (unknown nested keys stripped). */
-export function validateInput(step: AnyStep, raw: unknown): Validation<Record<string, unknown>> {
-  if (!isPlainObject(raw)) return { ok: false, issues: ["input.json must be a JSON object keyed by the step's input names"] };
+export function validateInput(
+  step: AnyStep,
+  raw: unknown,
+): Validation<Record<string, unknown>> {
+  if (!isPlainObject(raw))
+    return {
+      ok: false,
+      issues: ["input.json must be a JSON object keyed by the step's input names"],
+    };
   const issues: string[] = [];
   const out: Record<string, unknown> = {};
   const declared = step.input as Record<string, unknown>;
-  for (const k of Object.keys(raw)) if (!Object.hasOwn(declared, k)) issues.push(`input.${k}: not an input of step '${step.name}'`);
+  for (const k of Object.keys(raw))
+    if (k !== CONFIG_KEY && !Object.hasOwn(declared, k))
+      issues.push(`input.${k}: not an input of step '${step.name}'`);
   for (const [k, e] of Object.entries(declared)) {
     if (!Object.hasOwn(raw, k)) {
       if (isOptional(e)) continue;
@@ -68,7 +125,12 @@ export function validateInput(step: AnyStep, raw: unknown): Validation<Record<st
 export function validateOutput(step: AnyStep, value: unknown): Validation<unknown> {
   if (!step.output) {
     if (value === undefined || value === null) return { ok: true, value: null };
-    return { ok: false, issues: [`step '${step.name}' declares no output model but the handler returned a value`] };
+    return {
+      ok: false,
+      issues: [
+        `step '${step.name}' declares no output model but the handler returned a value`,
+      ],
+    };
   }
-  return parseStrict(step.output.schema, value, "output");
+  return parseStrict(step.output.schema, value, 'output');
 }

package/src/index.ts

@@ -1,12 +1,125 @@
 /// <reference path="./assets.d.ts" />
-// @olve/pipelines — authoring API (contracts, handlers, pipeline builder).
-export { z } from "zod";
-export { model, fileRef, optional, type Model, type AnyModel, type Infer, type InferInput, type Plain, type FileRef, type Optional } from "./core/models.ts";
-export { secret, type Secret, type DeclaredSecrets } from "./core/secrets.ts";
-export { hmacSha256, githubSignature, bearer, type WebhookVerify, type WebhookEvent } from "./core/webhooks.ts";
-export { step, type Step, type AnyStep, type StepDef, type StepInput, type StepKind, type InputEntry, type InputSpec, type LibraryStep, type OutputValue } from "./core/steps.ts";
-export { implement, type Implementation, type Handler, type HandlerResult, type Skipped } from "./core/handlers.ts";
-export type { Ctx, RunInfo, Sh, ShOptions, ExecOptions, Http, HttpInit, RetryOptions, OutputFileOptions } from "./ctx/types.ts";
-export type { Ref, Mappable, MapperResult } from "./builder/refs.ts";
-export { pipeline, Pipeline, type PipelineDef, type InRefs, type InitialAvail, type WhenCond, type PushEvent, type StageEntry, type CheckStage, type MapperFn } from "./builder/pipeline.ts";
-export { steps, Source, BuiltImage, HelmRelease, FailureContext, type RunValue, type ScriptDef, type GitHubSourceDef, type ImageBuildDef, type HelmDeployOverSshDef, httpHealthCheck, type HttpHealthCheckOptions } from "./library/index.ts";
+// @olivervea/pipelines — authoring API (contracts, handlers, pipeline builder).
+export { z } from 'zod';
+export {
+  model,
+  fileRef,
+  optional,
+  type Model,
+  type AnyModel,
+  type Infer,
+  type InferInput,
+  type Plain,
+  type FileRef,
+  type Optional,
+} from './core/models.ts';
+export { secret, type Secret, type DeclaredSecrets } from './core/secrets.ts';
+export {
+  config,
+  secretRef,
+  isConfigRef,
+  type ConfigEntry,
+  type AnyConfigEntry,
+  type ConfigValues,
+  type ConfigRef,
+  type ConfigNode,
+  type Configurable,
+  type DiscriminatorOf,
+  type VariantOf,
+} from './core/config.ts';
+export {
+  hmacSha256,
+  githubSignature,
+  bearer,
+  type WebhookVerify,
+  type WebhookEvent,
+} from './core/webhooks.ts';
+export { resolveConfigRefs, isLiteral, type Resolved } from './core/config-refs.ts';
+export {
+  callback,
+  external,
+  person,
+  poll,
+  type External,
+  type ExternalOptions,
+  type ExternalSource,
+  type PollOptions,
+} from './core/external.ts';
+export {
+  step,
+  type Step,
+  type AnyStep,
+  type StepDef,
+  type StepInput,
+  type StepKind,
+  type ConfigOf,
+  type InputEntry,
+  type InputSpec,
+  type LiteralName,
+  type OutputValue,
+} from './core/steps.ts';
+export type { Simplify } from './core/types.ts';
+export {
+  implement,
+  type Implementation,
+  type Handler,
+  type HandlerResult,
+  type Skipped,
+} from './core/handlers.ts';
+export type {
+  Ctx,
+  RunInfo,
+  Sh,
+  ShOptions,
+  ExecOptions,
+  Http,
+  HttpInit,
+  RetryOptions,
+  OutputFileOptions,
+} from './ctx/types.ts';
+export type { Ref, Mappable, MapperResult } from './builder/refs.ts';
+export {
+  pipeline,
+  Pipeline,
+  type PipelineDef,
+  type RetentionPolicy,
+  type InRefs,
+  type InitialAvail,
+  type WhenCond,
+  type PushEvent,
+  type StageEntry,
+  type CheckStage,
+  type MapperFn,
+} from './builder/pipeline.ts';
+export {
+  steps,
+  Source,
+  BuiltImage,
+  PushedImage,
+  HelmRelease,
+  FailureContext,
+  type GitHubSourceDef,
+  type ImageBuildDef,
+  type ImageBuildTarballDef,
+  type ImageBuildPushDef,
+  type RegistryAuth,
+  type RegistryAuthConfig,
+  type HelmDeployOverSshDef,
+  helmTarget,
+  deployTarget,
+  type DeployTargetOptions,
+  type ComposeDeployDef,
+  type ComposeTargetValue,
+  type HelmDeployInClusterDef,
+  type HelmDeploySshTargetDef,
+  type HelmTargetDeployDef,
+  type HelmTargetValue,
+  type InClusterTargetValue,
+  type SshTargetValue,
+  httpHealthCheck,
+  type HttpHealthCheckOptions,
+  sshSession,
+  type SshSession,
+  serviceUrl,
+  checkSpa,
+} from './library/index.ts';

package/src/library/checks.ts

@@ -0,0 +1,28 @@
+// Checks for deployed apps, used from custom test handlers.
+import type { Ctx } from '../ctx/types.ts';
+
+/** The in-cluster URL of a release's Service, assuming the chart names it after the release. */
+export function serviceUrl(release: { release: string; namespace: string }): string {
+  return `http://${release.release}.${release.namespace}.svc.cluster.local`;
+}
+
+/**
+ * Checks `url` serves a single-page app: `GET <url>/` references a `/assets/*.js` bundle (Vite's
+ * layout), and the bundle is served as JavaScript. Returns the page, for app-specific checks.
+ */
+export async function checkSpa(
+  ctx: Pick<Ctx, 'http' | 'log'>,
+  url: string,
+): Promise<{ html: string; bundle: string }> {
+  const base = url.replace(/\/$/, '');
+  const html = await (await ctx.http.expectOk(`${base}/`)).text();
+  const bundle = html.match(/src="(\/assets\/[^"]+\.js)"/)?.[1];
+  if (bundle === undefined)
+    throw new Error(`${base}/ does not reference an /assets/*.js bundle`);
+  const js = await ctx.http.expectOk(`${base}${bundle}`);
+  await js.body?.cancel().catch(() => {});
+  if (!(js.headers.get('content-type')?.includes('javascript') ?? false))
+    throw new Error(`${base}${bundle} is not served as JavaScript`);
+  ctx.log('SPA served', { url: base, bundle });
+  return { html, bundle };
+}

package/src/library/compose-deploy.ts

@@ -0,0 +1,113 @@
+// steps.composeDeploy — ask a Docker Compose deployer to recreate a service with an image. The step is
+// thin on purpose: the deployer (pull, recreate, health, rollback) lives beside the compose project
+// and is fixed there, behind a versioned HTTP API, without editing pipelines.
+import type { Ctx } from '../ctx/types.ts';
+import type { Configurable } from '../core/config.ts';
+import { resolveConfigRefs } from '../core/config-refs.ts';
+import { implement, type Implementation } from '../core/handlers.ts';
+import type { Secret } from '../core/secrets.ts';
+import { step, type LiteralName, type Step } from '../core/steps.ts';
+import { httpHealthCheck } from './http-health.ts';
+import { HelmRelease, PushedImage } from './models.ts';
+
+/** The step's own image: the work is one HTTP call from the harness. */
+export const COMPOSE_DEPLOY_IMAGE = 'alpine:3.22';
+
+/** A resolved `kind: compose` target. */
+export interface ComposeTargetValue {
+  kind: 'compose';
+  deployerUrl: string;
+  service: string;
+  token: { secret: string };
+  healthUrl?: string | undefined;
+}
+
+/** What the deployer answers to `POST /v1/deploy`. */
+export interface ComposeDeployResult {
+  status: 'deployed' | 'rolled-back' | 'failed';
+  service: string;
+  image: string;
+  previousImage?: string;
+  message?: string;
+  log?: string[];
+}
+
+export interface ComposeDeployDef<N extends string, K extends string> {
+  name: LiteralName<N>;
+  /** What the step does, shown in the UI. Required; may be empty. */
+  description: string;
+  /** A compose target, usually a `select` case's reference. */
+  target: Configurable<ComposeTargetValue>;
+  /** The secret the target's `token` names (declared by the step). */
+  token: Secret<K>;
+  timeoutSeconds?: number;
+}
+
+const input = { image: PushedImage };
+
+/**
+ * Deploys a registry image (by digest) to a compose service: `POST <deployerUrl>/v1/deploy
+ * { service, image: <repository>@<digest> }` with the target's bearer token. The deployer pulls the
+ * image, recreates the service, waits for its health and rolls back on failure; the step reports its
+ * answer, then (with the target's `healthUrl`) checks health itself. Output: a `HelmRelease` with the
+ * service as `release` and `compose` as `namespace`, so it can share a step name with the helm deploys.
+ */
+export function composeDeploy<const N extends string, const K extends string>(
+  def: ComposeDeployDef<N, K>,
+): Implementation<Step<N, typeof input, typeof HelmRelease, K>> {
+  const contract = step({
+    name: def.name,
+    description: def.description,
+    kind: 'composeDeploy',
+    image: COMPOSE_DEPLOY_IMAGE,
+    input,
+    output: HelmRelease,
+    secrets: [def.token],
+    timeoutSeconds: def.timeoutSeconds,
+    settings: { target: def.target, token: def.token.name },
+  });
+  return implement(contract, async ({ image }, ctx: Ctx) => {
+    const target = resolveConfigRefs(def.target, ctx.config ?? {}) as ComposeTargetValue;
+    if (target?.kind !== 'compose')
+      throw new Error(
+        `composeDeploy '${def.name}': the target is kind '${String(target?.kind)}', not 'compose'`,
+      );
+    if (target.token.secret !== def.token.name)
+      throw new Error(
+        `composeDeploy '${def.name}': the target's token names secret ${target.token.secret}, but the step declares ${def.token.name}`,
+      );
+    const token = (ctx.secrets as Record<string, string>)[def.token.name];
+    const ref = `${image.repository}@${image.digest}`;
+    const url = `${target.deployerUrl.replace(/\/+$/, '')}/v1/deploy`;
+    ctx.log('deploying', {
+      service: target.service,
+      image: ref,
+      deployer: target.deployerUrl,
+    });
+    const res = await ctx.http.fetch(url, {
+      method: 'POST',
+      headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+      body: JSON.stringify({ service: target.service, image: ref }),
+    });
+    const text = await res.text();
+    let result: ComposeDeployResult | undefined;
+    try {
+      result = JSON.parse(text) as ComposeDeployResult;
+    } catch {
+      // reported below
+    }
+    for (const line of result?.log ?? []) ctx.log(`deployer: ${line}`);
+    if (!res.ok || result?.status !== 'deployed')
+      throw new Error(
+        `the deployer answered HTTP ${res.status}: ${result ? `${result.status}${result.message ? `: ${result.message}` : ''}` : text.slice(0, 300)}`,
+      );
+    if (target.healthUrl)
+      await httpHealthCheck(ctx, target.healthUrl, { attempts: 12, delaySeconds: 5 });
+    ctx.log('deployed', {
+      service: target.service,
+      image: ref,
+      previous: result.previousImage ?? '-',
+    });
+    return { release: target.service, namespace: 'compose', image: ref };
+  }) as never;
+}

package/src/library/github-source.ts

@@ -2,19 +2,25 @@
 //
 // Runs entirely inside the harness (HTTP + tar extraction in Bun): no git, wget or tar is needed
 // in the image, so v1's busybox-wget TLS footgun (`--no-check-certificate`) is gone.
-import { mkdirSync, mkdtempSync, readdirSync, statSync } from "node:fs";
-import { join } from "node:path";
-import { z } from "zod";
-import type { Ctx } from "../ctx/types.ts";
-import type { Secret } from "../core/secrets.ts";
-import { makeStep, type LibraryStep, type LiteralName, type Step } from "../core/steps.ts";
-import { extractTar } from "../util/tar.ts";
-import { Source } from "./models.ts";
+import { mkdirSync, mkdtempSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { z } from 'zod';
+import type { Ctx } from '../ctx/types.ts';
+import type { Secret } from '../core/secrets.ts';
+import type { Configurable } from '../core/config.ts';
+import { resolveConfigRefs } from '../core/config-refs.ts';
+import { implement, type Implementation } from '../core/handlers.ts';
+import { step, type LiteralName, type Step } from '../core/steps.ts';
+import { extractTar } from '../util/tar.ts';
+import { Source } from './models.ts';
 
 /** Image the step runs in. The work happens in the harness, so any image would do. */
-export const GITHUB_SOURCE_IMAGE = "alpine:3.22";
+export const GITHUB_SOURCE_IMAGE = 'alpine:3.22';
 
-const input = { repo: z.string().regex(/^[\w.-]+\/[\w.-]+$/, "expected owner/name"), sha: z.string().min(1) };
+const input = {
+  repo: z.string().regex(/^[\w.-]+\/[\w.-]+$/, 'expected owner/name'),
+  sha: z.string().min(1),
+};
 export type GitHubSourceInput = typeof input;
 
 export interface GitHubSourceDef<N extends string, T extends string> {
@@ -23,11 +29,29 @@ export interface GitHubSourceDef<N extends string, T extends string> {
   description: string;
   /** Token sent to the GitHub API (needed for private repositories; raises the rate limit). */
   token?: Secret<T>;
+  /** GitHub API base (default `https://api.github.com`; GitHub Enterprise: `https://<host>/api/v3`). */
+  apiUrl?: Configurable<string | undefined>;
   timeoutSeconds?: number;
 }
 
+export const GITHUB_API_URL = 'https://api.github.com';
+
+/** A declared secret's value, or "" when it is not set. */
+function optionalSecret(ctx: Pick<Ctx, 'secrets'>, name: string): string {
+  try {
+    return ((ctx.secrets as Record<string, string>)[name] ?? '').trim();
+  } catch {
+    return '';
+  }
+}
+
 /** Tarball URL of `repo` at `sha` (the API redirects to codeload.github.com). */
-export const gitHubTarballUrl = (repo: string, sha: string): string => `https://api.github.com/repos/${repo}/tarball/${encodeURIComponent(sha)}`;
+export const gitHubTarballUrl = (
+  repo: string,
+  sha: string,
+  apiUrl = GITHUB_API_URL,
+): string =>
+  `${apiUrl.replace(/\/+$/, '')}/repos/${repo}/tarball/${encodeURIComponent(sha)}`;
 
 /**
  * Downloads `repo` at `sha` from GitHub as a tarball and outputs it as a `Source` whose `tree` is the
@@ -36,42 +60,71 @@ export const gitHubTarballUrl = (repo: string, sha: string): string => `https://
  */
 export function gitHubSource<const N extends string, const T extends string = never>(
   def: GitHubSourceDef<N, T>,
-): Step<N, GitHubSourceInput, typeof Source, T> & LibraryStep {
-  return makeStep(
-    "gitHubSource",
-    { name: def.name, description: def.description, image: GITHUB_SOURCE_IMAGE, input, output: Source, secrets: def.token ? [def.token] : [], timeoutSeconds: def.timeoutSeconds },
-    {
-      manifestConfig: () => ({ ...(def.token ? { token: def.token.name } : {}) }),
-      async run({ repo, sha }: { repo: string; sha: string }, ctx: Ctx) {
-        const headers: Record<string, string> = { accept: "application/vnd.github+json", "user-agent": "olve-pipelines" };
-        if (def.token) headers.authorization = `Bearer ${ctx.secrets[def.token.name]}`;
-        const url = gitHubTarballUrl(repo, sha);
-        mkdirSync(ctx.tmpDir, { recursive: true });
-        const dir = mkdtempSync(join(ctx.tmpDir, "github-source-"));
-        const archive = join(dir, "repo.tar.gz");
-
-        ctx.log("fetching source", { repo, sha });
-        // Retries network errors and 5xx; a 4xx (bad sha, no access) fails at once.
-        const status = await ctx.retry({ attempts: 4, delaySeconds: 2, backoff: 2 }, async () => {
-          const res = await ctx.http.fetch(url, { headers });
-          if (res.status >= 500) throw new Error(`HTTP ${res.status} from ${url}`);
-          if (res.ok) await Bun.write(archive, res);
-          return res.status;
-        });
-        if (status < 200 || status >= 300) {
-          throw new Error(`cannot fetch ${repo}@${sha}: HTTP ${status} from ${url}${def.token ? "" : " (no token configured: a private repository needs one)"}`);
-        }
+): Implementation<Step<N, GitHubSourceInput, typeof Source, T>> {
+  const contract = step({
+    name: def.name,
+    description: def.description,
+    kind: 'gitHubSource',
+    image: GITHUB_SOURCE_IMAGE,
+    input,
+    output: Source,
+    secrets: def.token ? [def.token] : [],
+    timeoutSeconds: def.timeoutSeconds,
+    settings: {
+      ...(def.token ? { token: def.token.name } : {}),
+      ...(def.apiUrl !== undefined ? { apiUrl: def.apiUrl } : {}),
+    },
+  });
+  return implement(contract, async ({ repo, sha }, ctx: Ctx) => {
+    const headers: Record<string, string> = {
+      accept: 'application/vnd.github+json',
+      'user-agent': 'olve-pipelines',
+    };
+    const token = def.token ? optionalSecret(ctx, def.token.name) : '';
+    if (token) headers.authorization = `Bearer ${token}`;
+    else if (def.token)
+      ctx.log(
+        `${def.token.name} is not set: fetching anonymously (public repositories only)`,
+      );
+    const url = gitHubTarballUrl(
+      repo,
+      sha,
+      resolveConfigRefs(def.apiUrl, ctx.config ?? {}) ?? GITHUB_API_URL,
+    );
+    mkdirSync(ctx.tmpDir, { recursive: true });
+    const dir = mkdtempSync(join(ctx.tmpDir, 'github-source-'));
+    const archive = join(dir, 'repo.tar.gz');
 
-        const extracted = join(dir, "x");
-        mkdirSync(extracted);
-        await extractTar(archive, extracted);
-        // GitHub tarballs hold one top-level directory, plus a pax_global_header entry (the commit id)
-        // that some extractors materialise as a file.
-        const roots = readdirSync(extracted).filter((e) => e !== "pax_global_header" && statSync(join(extracted, e)).isDirectory());
-        if (roots.length !== 1) throw new Error(`unexpected tarball layout for ${repo}@${sha}: top-level entries ${JSON.stringify(readdirSync(extracted))}`);
-        const tree = await ctx.output.dir(join(extracted, roots[0]!), { name: "tree" });
-        return { repo, sha, tree };
+    ctx.log('fetching source', { repo, sha });
+    // Retries network errors and 5xx; a 4xx (bad sha, no access) fails at once.
+    const status = await ctx.retry(
+      { attempts: 4, delaySeconds: 2, backoff: 2 },
+      async () => {
+        const res = await ctx.http.fetch(url, { headers });
+        if (res.status >= 500) throw new Error(`HTTP ${res.status} from ${url}`);
+        if (res.ok) await Bun.write(archive, res);
+        return res.status;
       },
-    },
-  ) as never;
+    );
+    if (status < 200 || status >= 300) {
+      throw new Error(
+        `cannot fetch ${repo}@${sha}: HTTP ${status} from ${url}${def.token ? '' : ' (no token configured: a private repository needs one)'}`,
+      );
+    }
+
+    const extracted = join(dir, 'x');
+    mkdirSync(extracted);
+    await extractTar(archive, extracted);
+    // GitHub tarballs hold one top-level directory, plus a pax_global_header entry (the commit id)
+    // that some extractors materialise as a file.
+    const roots = readdirSync(extracted).filter(
+      (e) => e !== 'pax_global_header' && statSync(join(extracted, e)).isDirectory(),
+    );
+    if (roots.length !== 1)
+      throw new Error(
+        `unexpected tarball layout for ${repo}@${sha}: top-level entries ${JSON.stringify(readdirSync(extracted))}`,
+      );
+    const tree = await ctx.output.dir(join(extracted, roots[0]!), { name: 'tree' });
+    return { repo, sha, tree };
+  }) as never;
 }