@olivervea/pipelines 2.0.0-beta.16 → 2.0.0-beta.18

package/README.md

@@ -146,14 +146,14 @@ Library steps carry their own implementation and image; they take typed settings
 | Step | Image | Input → output | Config |
 |---|---|---|---|
 | `steps.gitHubSource` | `alpine:3.22` (work happens in the harness) | `{ repo: "owner/name", sha }` (plain schemas: needs a mapper) → `Source` | `token?: Secret` (sent as `Authorization: Bearer` when set; unset = anonymous; becomes the step's secret), `apiUrl?` (default `https://api.github.com`) |
-| `steps.imageBuild` | `gcr.io/kaniko-project/executor:debug` | `{ source: Source }` → `BuiltImage { repository, tag, digest, tar }`; with `push`: `PushedImage { repository, tag, digest }` | `repository` (image name; tag = run version, `-attempt<n>` on reruns), `context?` (dir in the source tree with the `Dockerfile`; default the root), `push?` (push to `repository` instead of a tarball), `contentTag?` (with `push`: tag `src-<context hash>`, skip when the registry has it), `registryMap?` (kaniko `--registry-map`: registry → mirror prefix; default: the homelab pull-through caches, `{}` = off), `stallTimeoutSeconds?` (default 300, 0 = off), `retries?` (default 2), `registryAuth?` (`{ username, password: Secret }`: log in to the `repository` registry; or a config reference to `{ username, password: { secret: NAME } }`, with `NAME` in `secrets`), `insecureRegistry?` (that registry is plain HTTP), `secrets?` (declared secrets a config `registryAuth` may name), `enabled?` (`false` skips the step) |
+| `steps.imageBuild` | `gcr.io/kaniko-project/executor:debug` | `{ source: Source }` → `BuiltImage { repository, tag, digest, tar }`; with `push`: `PushedImage { repository, tag, digest }` | `repository` (image name; tag = run version, `-attempt<n>` on reruns), `context?` (dir in the source tree with the `Dockerfile`; default the root), `push?` (push to `repository` instead of a tarball), `contentTag?` (with `push`: tag `src-<context hash>`, skip when the registry has it), `registryMap?` (kaniko `--registry-map`: registry → mirror prefix; default none), `stallTimeoutSeconds?` (default 300, 0 = off), `retries?` (default 2), `registryAuth?` (`{ username, password: Secret }`: log in to the `repository` registry; or a config reference to `{ username, password: { secret: NAME } }`, with `NAME` in `secrets`), `insecureRegistry?` (that registry is plain HTTP), `secrets?` (declared secrets a config `registryAuth` may name), `enabled?` (`false` skips the step) |
 | `steps.helmDeployOverSsh` | `alpine:3.22` (+ `apk add openssh-client` at run time) | `{ source: Source, <key>: BuiltImage }` per `images` (`PushedImage` with `fromRegistry`) → `HelmRelease { release, namespace, image? }` (`image`: the first image; absent without images) | `host` (`user@host`), `sshKey: Secret`, `release`, `namespace`, `chart` (dir in the source), `values?` (chart-relative `-f`), `set?`, `images?` (input key → values prefix; default `{ image: "image" }`; `{}` = the chart alone), `fromRegistry?`, `imageRefs?` (`PushedImage` input key → values prefix: set as `<prefix>.repository`/`.tag`/`.digest`, not pulled; works next to tarball `images`), `deployments?` (default `[release]`, none without images), `rolloutTimeoutSeconds?` (default 120), `dependencies?` (pull `Chart.yaml`'s dependencies on the host), `helmWait?` / `atomic?` (`helm --wait` / `--atomic`), `healthUrl?` (URL or `{ url, attempts?, delaySeconds? }`) |
 | `steps.helmDeployInCluster` (and the `target` form of `helmDeployOverSsh`) | `alpine/helm:3.16.2` (ssh: `alpine:3.22`) | `{ source: Source, <key>: PushedImage }` per `images`, `<key>: optional(PushedImage)` per `imageRefs` → `HelmRelease` (`image`: `repository:tag@digest` of the first image) | `target` (a `select` case's reference to a `helmTarget()` variant: `{ kind: "inCluster", namespace, release, values, set, url? }` for helmDeployInCluster, the same plus `host` with `kind: "ssh"` for helmDeployOverSsh; `url` is where step pods reach the release, default its in-cluster Service URL, and is passed on in `HelmRelease.url`), `sshKey: Secret` (ssh only), `chart`, `inlineValues?` (objects written as values files after the target's), `images?`, `imageRefs?`, `deployments?` (ssh), `rolloutTimeoutSeconds?` (default 120), `healthPath?`; helmDeployInCluster declares `clusterAccess` |
 | `steps.composeDeploy` | `alpine:3.22` (work happens in the harness) | `{ image: PushedImage }` → `HelmRelease { release: <service>, namespace: "compose", image: <repository>@<digest> }` | `target` (a `select` case's reference to the compose variant of `deployTarget()`: `{ kind: "compose", deployerUrl, service, token: { secret }, healthUrl? }`), `token: Secret` (must be the secret the target names; the step's only secret), `timeoutSeconds?` |
 
 - `gitHubSource` downloads `<apiUrl>/repos/<repo>/tarball/<sha>` (retrying 5xx), strips the top-level directory and outputs it as `tree`.
 - `imageBuild` copies the tree to `/kaniko/build-context` (kaniko wipes other directories between multi-stage builds), builds `--context=/kaniko/build-context[/<context>] --dockerfile=<that>/Dockerfile` with `--no-push --tar-path --digest-file`, and outputs the tarball. A missing context or Dockerfile fails the step before kaniko starts. `repository` in the output is fully qualified the way a runtime names a loaded image (`x` → `docker.io/library/x`). A **stall watchdog** kills kaniko when it prints nothing for `stallTimeoutSeconds` (a base-image pull can stall on an idle connection at "Unpacking rootfs", and kaniko has no read timeout), failing the attempt; `retries` then starts a fresh attempt. The retry is deliberately a new pod, never a second kaniko in the same container: kaniko unpacks the base image over the container root and does not clean up when killed.
-- **Caches never block a build.** `registryMap` defaults to the homelab pull-through caches (`DEFAULT_REGISTRY_MAP`: `mcr.microsoft.com` → `mcr-cache.ovea.pro`, `index.docker.io` → `hub-cache.ovea.pro`; kaniko under gVisor stalls pulling from mcr.microsoft.com directly). A map you pass replaces the default; `{}` turns mapping off. Before kaniko starts, every mirror host gets a 5 s `GET https://<host>/v2/` (200 or 401 = up); one that doesn't answer is dropped with a log line (`mcr-cache.ovea.pro unreachable; pulling from mcr.microsoft.com directly`). After that, kaniko itself copes: it falls back to the original registry when a mirror can't serve an image's manifest (`RetrieveRemoteImage`; `--skip-default-registry-fallback` is never set). A cache that dies mid-download (after the manifest came from it) fails or stalls the attempt; `retries` starts a fresh one, which probes again.
+- **Caches never block a build.** `registryMap` maps registries to pull-through caches (e.g. `{ "mcr.microsoft.com": "mcr-cache.example.com" }`; kaniko under gVisor can stall pulling from mcr.microsoft.com directly, a nearby cache doesn't). There is none by default. Before kaniko starts, every mirror host gets a 5 s `GET https://<host>/v2/` (200 or 401 = up); one that doesn't answer is dropped with a log line (`<cache> unreachable; pulling from <registry> directly`). After that, kaniko itself copes: it falls back to the original registry when a mirror can't serve an image's manifest (`RetrieveRemoteImage`; `--skip-default-registry-fallback` is never set). A cache that dies mid-download (after the manifest came from it) fails or stalls the attempt; `retries` starts a fresh one, which probes again.
 - **Registry mode** (`push: true`, for images too big for the step's ~1 GiB scratch space and Garage): kaniko runs without `--no-push`/`--tar-path`, pushes to `repository` (it must start with a registry host; anonymous unless `registryAuth`) and the output is a `PushedImage` with the digest from `--digest-file`. With `contentTag`, the tag is `src-` + the first 12 hex of the sha256 of the context directory's deterministic tar (the bytes `ctx.output.dir` would write), computed from the fetched tree; a `HEAD https://<host>/v2/<path>/manifests/<tag>` (answering a Basic or Bearer challenge: anonymous token, or the `registryAuth` login; an anonymous lookup that is denied counts as "not there") that answers a `Docker-Content-Digest` skips kaniko and outputs that digest. With `registryAuth`, the step declares the password secret and writes `$DOCKER_CONFIG/config.json` (`/kaniko/.docker` in the kaniko image) for that registry before kaniko runs, so pushes and base-image pulls from it are logged in. `insecureRegistry` makes the lookups plain HTTP and adds kaniko `--insecure --insecure-registry=<host>`.
 - `helmDeployOverSsh` gets each image into the host's **k3s** containerd: a `BuiltImage` tar is streamed in (`nerdctl --address /run/k3s/containerd/containerd.sock --namespace k8s.io load`), a `PushedImage` (`fromRegistry`) is pulled there (`… pull -q`, so the rollout doesn't wait on the pull). It copies the chart to `/tmp/<release>-helm-<namespace>`, runs `helm upgrade --install` from inside it (values, then `set`, then per image `<prefix>.repository`/`.tag`/`.pullPolicy` — `Never` for tarballs, `IfNotPresent` for registry images — which always win), waits for `kubectl rollout status` of each of `deployments` in order (`deploy/<name>` for a bare name), and with `healthUrl` polls it (5 × 5 s unless given) from the harness. `images: {}` deploys the chart alone: the step's only input is `source`, no image is imported and no image flag set, and nothing is waited on unless `deployments` names it. `dependencies: true` runs `helm pull <name> --repo <repository> --version <version>` (or the `oci://` reference) into the copied chart's `charts/` for each dependency in `Chart.yaml`; `helmWait` / `atomic` add `--wait` / `--atomic` with `--timeout=<rolloutTimeoutSeconds>s` and make `deployments` default to none. Several images (one chart, two Deployments):
 
@@ -253,6 +253,7 @@ export default pipeline({ name: "Olve.Pipelines", description: "…", input: M.D
 - **Describe checks the mapper twice.** It records the mapper with proxies, and it statically analyses the mapper's source. The source must be an object literal of member chains, literals, constant arrays and nested object literals. Ternaries, `&&`, `||`, `??`, `!`, comparisons, arithmetic, template strings, calls, spread and optional chaining are all rejected. Free identifiers such as module constants are allowed; they are recorded as `{const}`.
 - **Triggers.** A pipeline may call `onPush` at most once. Each `onPoll` and `onWebhook` needs a distinct `name` (`onPoll` defaults to `"poll"`). `onWebhook({ name, verify }, ({ headers, body }) => input | null)` maps a delivery to `/api/hooks/<pipeline id>/<name>`; `verify` is `hmacSha256({ secret, header, prefix? })`, `githubSignature(secret)` or `bearer(secret)`, and `null` ignores the delivery.
 - **`select` (branch on config).** `.select(TARGET.prod, { ssh: (p, prod) => p.stage(…), inCluster: (p, prod) => p.stage(…) })`: the first argument references a `z.discriminatedUnion` in a config entry (`DiscriminatorOf<U>` is its one literal field; a union with no or several literal fields is a type error). Cases are exhaustive and checked at compile time; each gets the reference narrowed to its variant (`ConfigNode<VariantOf<U, D, V>>`), recorded as `{"$config": "target.prod", "$case": "target.prod.kind=ssh"}` and usable only inside that case. The manifest records `{select, discriminator, cases}`; the service resolves it per run from the pinned config snapshot (`when` is per run input). The same step name may appear in several cases (same output model; not two different custom steps, since handlers pair by name); a step every case has is definitely available afterwards, one only some have is maybe-available. The harness picks the variant of a same-named step from `OLVE_CASES` (a JSON array of the case conditions the step sits in, set by the service).
+- **Retention.** `pipeline({ …, retention: { keepLatestRuns?, minRunAgeDays?, buildGraceDays? } })` overrides the instance's retention thresholds for this pipeline's runs and binaries, field by field (non-negative integers; unset = the instance's value). Recorded on the manifest root as `retention`; the service applies the active version's.
 - **Failure handlers.** `.onFailure(handler, { steps? })` takes a step whose inputs are all `FailureContext`. When a run fails, the service runs each handler whose scope (default: any step) contains a failed step, once per failure: `run <handler>` in the handler's image with its declared secrets, from the run's pinned binary. It is untracked and best-effort — not an attempt of the run, never retried, and its own failure triggers nothing; its log is kept at `runs/<p>/<run>/_failure/<handler>/<id>/log`. `FailureContext` carries `pipeline {id, name}`, `run {id, number, address, versionStamp, trigger, url}`, `version {number, commit}`, the run's `input`, a one-line `reason`, and `failedSteps[]` (`step, attempt, address, reason, logUrl`) — only the steps in the handler's scope, never empty.
 
 ### Testing handlers
@@ -397,6 +398,7 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
   - `{kind: "poll", name, url, headers, secrets, valuePath, intervalSeconds}`, where header values may contain `$SECRET:NAME`;
   - `{kind: "webhook", name, verify: {scheme: "hmac-sha256", secret, header, prefix} | {scheme: "bearer", secret}}`. The `map-trigger` event is `{headers, body}`; a `null` mapping yields a `skipped` envelope (the delivery is ignored).
 - **`failureHandlers`** is a list of `{step: Step, steps: [names] | null}`. Each handler's inputs are all FailureContext (`{"ref": "failure"}`).
+- **`retention`** (when present) is `{keepLatestRuns?, minRunAgeDays?, buildGraceDays?}`, non-negative integers that replace the instance's thresholds for this pipeline. Omitted when the definition sets none.
 - **`secrets`** lists every secret used by steps, polls and webhooks, sorted, as `{name, description}`.
 - **`config`** lists every config entry steps declare, sorted by name, as `{name, description, required, schema}`. `schema` is the `z.input` view (defaults listed); it validates the entry's value in the config document. `required` is false when the schema accepts a missing value. A `secretRef` field is marked `"x-olve-kind": "secretRef"`, with the secret's name as `properties.secret.const`. Omitted when there are no entries.
 - **Step `configEntries`** names the entries the step receives (sorted; omitted when none). It is part of the step's content, like `secrets`. The step key `config` is the step's `settings`.

package/manifest.schema.json

@@ -165,6 +165,28 @@
           "minLength": 1,
           "type": "string"
         },
+        "retention": {
+          "additionalProperties": false,
+          "description": "Retention overrides for this pipeline (omitted when none). Each field replaces the instance setting of the same meaning.",
+          "properties": {
+            "buildGraceDays": {
+              "maximum": 9007199254740991,
+              "minimum": 0,
+              "type": "integer"
+            },
+            "keepLatestRuns": {
+              "maximum": 9007199254740991,
+              "minimum": 0,
+              "type": "integer"
+            },
+            "minRunAgeDays": {
+              "maximum": 9007199254740991,
+              "minimum": 0,
+              "type": "integer"
+            }
+          },
+          "type": "object"
+        },
         "secrets": {
           "description": "All declared secrets (steps + poll triggers), sorted by name.",
           "items": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.16",
+  "version": "2.0.0-beta.18",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/builder/pipeline.ts

@@ -183,10 +183,21 @@ export interface FailureHandlerNode {
   readonly steps: readonly AnyStep[] | null;
 }
 
+/** Per-pipeline retention; each field overrides the instance's `Retention:*` setting of the same meaning. */
+export interface RetentionPolicy {
+  /** A run is never deleted while among its pipeline's newest N runs. */
+  keepLatestRuns?: number;
+  /** A run is never deleted while younger than this. */
+  minRunAgeDays?: number;
+  /** An unused compiled binary is deleted this long after it was last modified. */
+  buildGraceDays?: number;
+}
+
 export interface PipelineData {
   readonly name: string;
   readonly description: string | undefined;
   readonly input: AnyModel;
+  readonly retention: RetentionPolicy | undefined;
   readonly nodes: StageNode[];
   readonly triggers: TriggerNode[];
   readonly failureHandlers: FailureHandlerNode[];
@@ -216,6 +227,8 @@ export interface PipelineDef<In extends AnyModel> {
   description?: string;
   /** The pipeline input model (what triggers produce, what `when` matches on). */
   input: In;
+  /** Retention overrides for this pipeline's runs and binaries (unset fields keep the instance's values). */
+  retention?: RetentionPolicy;
 }
 
 export class Pipeline<In extends AnyModel, A extends AvailMap> {
@@ -228,6 +241,7 @@ export class Pipeline<In extends AnyModel, A extends AvailMap> {
       name: def.name,
       description: def.description,
       input: def.input,
+      retention: def.retention,
       nodes: [],
       triggers: shared?.triggers ?? [],
       failureHandlers: shared?.failureHandlers ?? [],

package/src/describe/describe.ts

@@ -488,6 +488,15 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
     return { kind: "poll", name: t.name, url: t.url, headers: { ...t.headers }, secrets: [...declared].sort(), valuePath: t.valuePath, intervalSeconds: t.intervalSeconds };
   });
 
+  // ── retention ──
+  const retention: Record<string, number> = {};
+  for (const key of ["keepLatestRuns", "minRunAgeDays", "buildGraceDays"] as const) {
+    const v = data.retention?.[key];
+    if (v === undefined) continue;
+    if (!(Number.isInteger(v) && v >= 0)) problems.push(`retention.${key} must be a non-negative integer`);
+    else retention[key] = v;
+  }
+
   if (problems.length) throw new DescribeError(problems);
 
   const manifest = canonicalize({
@@ -495,6 +504,7 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
     name: data.name,
     description: data.description,
     input: inputModel.name,
+    ...(Object.keys(retention).length ? { retention } : {}),
     harness: opts.harness,
     models: Object.fromEntries([...models].map(([k, v]) => [k, v.schemas])),
     secrets: [...secrets].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([name, description]) => ({ name, description })),

package/src/describe/manifest-schema.ts

@@ -182,6 +182,14 @@ export const Manifest = z
       )
       .optional()
       .describe("Config entries used by steps, sorted by name (omitted when none). Fields of kind secretRef ({secret: NAME}) name a secret."),
+    retention: z
+      .strictObject({
+        keepLatestRuns: z.int().min(0).optional(),
+        minRunAgeDays: z.int().min(0).optional(),
+        buildGraceDays: z.int().min(0).optional(),
+      })
+      .optional()
+      .describe("Retention overrides for this pipeline (omitted when none). Each field replaces the instance setting of the same meaning."),
     triggers: z.array(Trigger),
     stages: z.array(Stage),
     failureHandlers: z.array(

package/src/index.ts

@@ -12,5 +12,5 @@ export type { Simplify } from "./core/types.ts";
 export { implement, type Implementation, type Handler, type HandlerResult, type Skipped } from "./core/handlers.ts";
 export type { Ctx, RunInfo, Sh, ShOptions, ExecOptions, Http, HttpInit, RetryOptions, OutputFileOptions } from "./ctx/types.ts";
 export type { Ref, Mappable, MapperResult } from "./builder/refs.ts";
-export { pipeline, Pipeline, type PipelineDef, type InRefs, type InitialAvail, type WhenCond, type PushEvent, type StageEntry, type CheckStage, type MapperFn } from "./builder/pipeline.ts";
+export { pipeline, Pipeline, type PipelineDef, type RetentionPolicy, type InRefs, type InitialAvail, type WhenCond, type PushEvent, type StageEntry, type CheckStage, type MapperFn } from "./builder/pipeline.ts";
 export { steps, Source, BuiltImage, PushedImage, HelmRelease, FailureContext, type GitHubSourceDef, type ImageBuildDef, type ImageBuildTarballDef, type ImageBuildPushDef, type RegistryAuth, type RegistryAuthConfig, type HelmDeployOverSshDef, helmTarget, deployTarget, type DeployTargetOptions, type ComposeDeployDef, type ComposeTargetValue, type HelmDeployInClusterDef, type HelmDeploySshTargetDef, type HelmTargetDeployDef, type HelmTargetValue, type InClusterTargetValue, type SshTargetValue, httpHealthCheck, type HttpHealthCheckOptions, sshSession, type SshSession, serviceUrl, checkSpa } from "./library/index.ts";

package/src/library/image-build.ts

@@ -102,14 +102,8 @@ export interface ImageBuildPushDef<N extends string, K extends string = never> e
   contentTag?: boolean;
 }
 
-/**
- * Default `registryMap`: pull-through caches for mcr.microsoft.com and Docker Hub. kaniko under gVisor
- * can stall forever pulling from mcr.microsoft.com directly.
- */
-export const DEFAULT_REGISTRY_MAP: Readonly<Record<string, string>> = {
-  "mcr.microsoft.com": "mcr-cache.ovea.pro",
-  "index.docker.io": "hub-cache.ovea.pro",
-};
+/** Default `registryMap`: none, kaniko pulls from the registries themselves. */
+export const DEFAULT_REGISTRY_MAP: Readonly<Record<string, string>> = {};
 
 /** How long a cache registry gets to answer `GET /v2/` before the build goes without it. */
 export const CACHE_PROBE_TIMEOUT_MS = 5000;