@olivervea/pipelines 2.0.0-beta.28 → 2.0.0-beta.29

package/README.md

@@ -139,14 +139,14 @@ Library steps carry their own implementation and image; they take typed settings
 | Step | Image | Input → output | Config |
 |---|---|---|---|
 | `steps.gitHubSource` | `alpine:3.22` (work happens in the harness) | `{ repo: "owner/name", sha }` (plain schemas: needs a mapper) → `Source` | `token?: Secret` (sent as `Authorization: Bearer` when set; unset = anonymous; becomes the step's secret), `apiUrl?` (default `https://api.github.com`) |
-| `steps.imageBuild` | `gcr.io/kaniko-project/executor:v1.24.0-debug`, pinned by digest | `{ source: Source }` → `BuiltImage { repository, tag, digest, tar }`; with `push`: `PushedImage { repository, tag, digest, cached }` | `repository` (image name; tag = run version, `-attempt<n>` on reruns), `context?` (dir in the source tree with the `Dockerfile`; default the root), `push?` (push to `repository` instead of a tarball), `contentTag?` (with `push`: tag `src-<hash>` of the context and the build recipe; skip when the registry has it, with `cached: true`), `registryMap?` (kaniko `--registry-map`: registry → mirror prefix; default none), `stallTimeoutSeconds?` (default 300, 0 = off), `retries?` (default 2), `registryAuth?` (`{ username, password: Secret }`: log in to the `repository` registry; or a config reference to `{ username, password: { secret: NAME } }`, with `NAME` in `secrets`), `insecureRegistry?` (that registry is plain HTTP), `secrets?` (declared secrets a config `registryAuth` may name), `enabled?` (`false` skips the step) |
+| `steps.imageBuild` | `gcr.io/kaniko-project/executor:v1.24.0-debug`, pinned by digest | `{ source: Source }` → `BuiltImage { repository, tag, digest, tar }`; with `push`: `PushedImage { repository, tag, digest, cached }` | `repository` (image name; tag = run version, `-attempt<n>` on reruns), `context?` (dir in the source tree with the `Dockerfile`; default the root), `push?` (push to `repository` instead of a tarball), `contentTag?` (with `push`: tag `src-<hash>` of the context and the build recipe; skip when the registry has it, with `cached: true`), `registryMap?` (kaniko `--registry-map`: registry → mirror prefix; default none), `cacheRepo?` (kaniko's layer cache repository; default `<repository>/cache` with `push`, none for a tarball; `false` = off), `stallTimeoutSeconds?` (default 300, 0 = off), `retries?` (default 2), `registryAuth?` (`{ username, password: Secret }`: log in to the `repository` registry; or a config reference to `{ username, password: { secret: NAME } }`, with `NAME` in `secrets`), `insecureRegistry?` (that registry is plain HTTP), `secrets?` (declared secrets a config `registryAuth` may name), `enabled?` (`false` skips the step) |
 | `steps.helmDeployOverSsh` | `alpine:3.22` (+ `apk add openssh-client` at run time) | `{ source: Source, <key>: BuiltImage }` per `images` (`PushedImage` with `fromRegistry`) → `HelmRelease { release, namespace, image? }` (`image`: the first image; absent without images) | `host` (`user@host`), `sshKey: Secret`, `release`, `namespace`, `chart` (dir in the source), `values?` (chart-relative `-f`), `set?`, `images?` (input key → values prefix; default `{ image: "image" }`; `{}` = the chart alone), `fromRegistry?`, `imageRefs?` (`PushedImage` input key → values prefix: set as `<prefix>.repository`/`.tag`/`.digest`, not pulled; works next to tarball `images`), `deployments?` (default `[release]`, none without images), `rolloutTimeoutSeconds?` (default 120), `dependencies?` (pull `Chart.yaml`'s dependencies on the host), `helmWait?` / `atomic?` (`helm --wait` / `--atomic`), `healthUrl?` (URL or `{ url, attempts?, delaySeconds? }`) |
 | `steps.helmDeployInCluster` (and the `target` form of `helmDeployOverSsh`) | `alpine/helm:3.16.2` (ssh: `alpine:3.22`) | `{ source: Source, <key>: PushedImage }` per `images`, `<key>: optional(PushedImage)` per `imageRefs` → `HelmRelease` (`image`: `repository:tag@digest` of the first image) | `target` (a `select` case's reference to a `helmTarget()` variant: `{ kind: "inCluster", namespace, release, values, set, url? }` for helmDeployInCluster, the same plus `host` with `kind: "ssh"` for helmDeployOverSsh; `url` is where step pods reach the release, default its in-cluster Service URL, and is passed on in `HelmRelease.url`), `sshKey: Secret` (ssh only), `chart`, `inlineValues?` (objects written as values files after the target's; a resolved `secretRef`, `{ secret: NAME }`, becomes that secret's value, and the file holding it is deleted after the upgrade), `secrets?` (the secrets those `secretRef`s name; the step declares them), `images?`, `imageRefs?`, `deployments?` (ssh), `rolloutTimeoutSeconds?` (default 120), `healthPath?`; helmDeployInCluster declares `clusterAccess` |
 | `steps.composeDeploy` | `alpine:3.22` (work happens in the harness) | `{ image: PushedImage }` → `HelmRelease { release: <service>, namespace: "compose", image: <repository>@<digest> }` | `target` (a `select` case's reference to the compose variant of `deployTarget()`: `{ kind: "compose", deployerUrl, service, token: { secret }, healthUrl? }`), `token: Secret` (must be the secret the target names; the step's only secret), `timeoutSeconds?` |
 
 - `gitHubSource` downloads `<apiUrl>/repos/<repo>/tarball/<sha>` (retrying 5xx), strips the top-level directory and outputs it as `tree`.
 - `imageBuild` copies the tree to `/kaniko/build-context` (kaniko wipes other directories between multi-stage builds), builds `--context=/kaniko/build-context[/<context>] --dockerfile=<that>/Dockerfile` with `--no-push --tar-path --digest-file`, and outputs the tarball. A missing context or Dockerfile fails the step before kaniko starts. `repository` in the output is fully qualified the way a runtime names a loaded image (`x` → `docker.io/library/x`). A **stall watchdog** kills kaniko when it prints nothing for `stallTimeoutSeconds` (a base-image pull can stall on an idle connection at "Unpacking rootfs", and kaniko has no read timeout), failing the attempt; `retries` then starts a fresh attempt. The retry is deliberately a new pod, never a second kaniko in the same container: kaniko unpacks the base image over the container root and does not clean up when killed.
-- **Caches never block a build.** `registryMap` maps registries to pull-through caches (e.g. `{ "mcr.microsoft.com": "mcr-cache.example.com" }`; kaniko under gVisor can stall pulling from mcr.microsoft.com directly, a nearby cache doesn't). There is none by default. Before kaniko starts, every mirror host gets a 5 s `GET https://<host>/v2/` (200 or 401 = up); one that doesn't answer is dropped with a log line (`<cache> unreachable; pulling from <registry> directly`). After that, kaniko itself copes: it falls back to the original registry when a mirror can't serve an image's manifest (`RetrieveRemoteImage`; `--skip-default-registry-fallback` is never set). A cache that dies mid-download (after the manifest came from it) fails or stalls the attempt; `retries` starts a fresh one, which probes again.
+- **Caches never block a build.** `registryMap` maps registries to pull-through caches (e.g. `{ "mcr.microsoft.com": "mcr-cache.example.com" }`; kaniko under gVisor can stall pulling from mcr.microsoft.com directly, a nearby cache doesn't). There is none by default. Before kaniko starts, every mirror host gets a 5 s `GET https://<host>/v2/` (200 or 401 = up); one that doesn't answer is dropped with a log line (`<cache> unreachable; pulling from <registry> directly`). After that, kaniko itself copes: it falls back to the original registry when a mirror can't serve an image's manifest (`RetrieveRemoteImage`; `--skip-default-registry-fallback` is never set). A cache that dies mid-download (after the manifest came from it) fails or stalls the attempt; `retries` starts a fresh one, which probes again. The layer cache (`cacheRepo`) is probed the same way and left out when its registry doesn't answer (`<host> unreachable; building without the layer cache`); kaniko treats a cache lookup error as a miss and a failed cache push as a warning.
 - **Registry mode** (`push: true`, for images too big for the step's ~1 GiB scratch space and Garage): kaniko runs without `--no-push`/`--tar-path`, pushes to `repository` (it must start with a registry host; anonymous unless `registryAuth`) and the output is a `PushedImage` with the digest from `--digest-file`. With `contentTag`, the tag is `src-` + the first 12 hex of the sha256 of the context directory's deterministic tar (the bytes `ctx.output.dir` would write), computed from the fetched tree; a `HEAD https://<host>/v2/<path>/manifests/<tag>` (answering a Basic or Bearer challenge: anonymous token, or the `registryAuth` login; an anonymous lookup that is denied counts as "not there") that answers a `Docker-Content-Digest` skips kaniko and outputs that digest. With `registryAuth`, the step declares the password secret and writes `$DOCKER_CONFIG/config.json` (`/kaniko/.docker` in the kaniko image) for that registry before kaniko runs, so pushes and base-image pulls from it are logged in. `insecureRegistry` makes the lookups plain HTTP and adds kaniko `--insecure --insecure-registry=<host>`.
 - `helmDeployOverSsh` gets each image into the host's **k3s** containerd: a `BuiltImage` tar is streamed in (`nerdctl --address /run/k3s/containerd/containerd.sock --namespace k8s.io load`), a `PushedImage` (`fromRegistry`) is pulled there (`… pull -q`, so the rollout doesn't wait on the pull). It copies the chart to `/tmp/<release>-helm-<namespace>`, runs `helm upgrade --install` from inside it (values, then `set`, then per image `<prefix>.repository`/`.tag`/`.pullPolicy` — `Never` for tarballs, `IfNotPresent` for registry images — which always win), waits for `kubectl rollout status` of each of `deployments` in order (`deploy/<name>` for a bare name), and with `healthUrl` polls it (5 × 5 s unless given) from the harness. `images: {}` deploys the chart alone: the step's only input is `source`, no image is imported and no image flag set, and nothing is waited on unless `deployments` names it. `dependencies: true` runs `helm pull <name> --repo <repository> --version <version>` (or the `oci://` reference) into the copied chart's `charts/` for each dependency in `Chart.yaml`; `helmWait` / `atomic` add `--wait` / `--atomic` with `--timeout=<rolloutTimeoutSeconds>s` and make `deployments` default to none. Several images (one chart, two Deployments):
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.28",
+  "version": "2.0.0-beta.29",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/library/image-build.ts

@@ -101,6 +101,12 @@ export interface ImageBuildDef<N extends string, K extends string = never> {
    * Used for kaniko's push, base image and cache pulls from that registry, and the `contentTag` lookup.
    */
   registryAuth?: RegistryAuth<K> | ConfigRef<RegistryAuthConfig | undefined>;
+  /**
+   * kaniko's layer cache: a repository it reuses unchanged layers from and pushes new ones to. Default
+   * `<repository>/cache` when pushing, none for a tarball build; `false` turns it off. Login and plain
+   * HTTP apply when it is on `repository`'s registry. An unreachable cache is skipped, never a failure.
+   */
+  cacheRepo?: Configurable<string | false | undefined>;
   /** The registry of `repository` speaks plain HTTP (a local `registry:2`). */
   insecureRegistry?: Configurable<boolean | undefined>;
   /** Secrets a config-referenced `registryAuth` may name (the step declares them). */
@@ -197,6 +203,22 @@ export async function reachableMirrors(
   return reachable;
 }
 
+/** The layer cache repository when its registry answers, else nothing (logged). */
+export async function reachableCache(
+  ctx: Pick<Ctx, 'http' | 'log' | 'signal'>,
+  cacheRepo: string | false,
+  insecureHosts: readonly string[] = [],
+): Promise<string | undefined> {
+  if (!cacheRepo) return undefined;
+  const host = registryHost(cacheRepo);
+  if (
+    await registryUp(ctx.http, host, ctx.signal, undefined, insecureHosts.includes(host))
+  )
+    return cacheRepo;
+  ctx.log(`${host} unreachable; building without the layer cache`);
+  return undefined;
+}
+
 /** Default `retries`: a stalled or flaky build gets two fresh attempts before the run fails. */
 export const DEFAULT_RETRIES = 2;
 
@@ -446,6 +468,7 @@ export function imageBuild<const N extends string, const K extends string = neve
       ...(push ? { push: true } : {}),
       ...(contentTag ? { contentTag: true } : {}),
       ...(mapped ? { registryMap } : {}),
+      ...(def.cacheRepo !== undefined ? { cacheRepo: def.cacheRepo } : {}),
       ...(authSecret
         ? {
             registryAuth: {
@@ -468,6 +491,7 @@ export function imageBuild<const N extends string, const K extends string = neve
       {
         repository: def.repository,
         registryMap: def.registryMap,
+        cacheRepo: def.cacheRepo,
         insecureRegistry: def.insecureRegistry,
         registryAuth: def.registryAuth,
         enabled: def.enabled,
@@ -520,6 +544,12 @@ export function imageBuild<const N extends string, const K extends string = neve
       (d.registryMap as Record<string, string> | undefined) ?? DEFAULT_REGISTRY_MAP,
       insecure ? [host] : [],
     );
+    const cache = await reachableCache(
+      ctx,
+      (d.cacheRepo as string | false | undefined) ??
+        (push ? `${repository}/cache` : false),
+      insecure ? [host] : [],
+    );
     await ctx.exec(['mkdir', '-p', KANIKO_CONTEXT]);
     await ctx.exec(['cp', '-r', `${tree}/.`, KANIKO_CONTEXT]);
     const kanikoContext = context ? `${KANIKO_CONTEXT}/${context}` : KANIKO_CONTEXT;
@@ -543,6 +573,7 @@ export function imageBuild<const N extends string, const K extends string = neve
         `--destination=${repository}:${tag}`,
         ...(insecure ? ['--insecure', `--insecure-registry=${host}`] : []),
         ...KANIKO_BUILD_FLAGS,
+        ...(cache ? ['--cache=true', `--cache-repo=${cache}`] : []),
         ...Object.entries(mirrors).map(([from, to]) => `--registry-map=${from}=${to}`),
       ],
       { idleTimeoutSeconds: def.stallTimeoutSeconds ?? DEFAULT_STALL_TIMEOUT_SECONDS },