@olivervea/pipelines 2.0.0-beta.23 → 2.0.0-beta.25

package/README.md

@@ -139,9 +139,9 @@ Library steps carry their own implementation and image; they take typed settings
 | Step | Image | Input → output | Config |
 |---|---|---|---|
 | `steps.gitHubSource` | `alpine:3.22` (work happens in the harness) | `{ repo: "owner/name", sha }` (plain schemas: needs a mapper) → `Source` | `token?: Secret` (sent as `Authorization: Bearer` when set; unset = anonymous; becomes the step's secret), `apiUrl?` (default `https://api.github.com`) |
-| `steps.imageBuild` | `gcr.io/kaniko-project/executor:debug` | `{ source: Source }` → `BuiltImage { repository, tag, digest, tar }`; with `push`: `PushedImage { repository, tag, digest }` | `repository` (image name; tag = run version, `-attempt<n>` on reruns), `context?` (dir in the source tree with the `Dockerfile`; default the root), `push?` (push to `repository` instead of a tarball), `contentTag?` (with `push`: tag `src-<context hash>`, skip when the registry has it), `registryMap?` (kaniko `--registry-map`: registry → mirror prefix; default none), `stallTimeoutSeconds?` (default 300, 0 = off), `retries?` (default 2), `registryAuth?` (`{ username, password: Secret }`: log in to the `repository` registry; or a config reference to `{ username, password: { secret: NAME } }`, with `NAME` in `secrets`), `insecureRegistry?` (that registry is plain HTTP), `secrets?` (declared secrets a config `registryAuth` may name), `enabled?` (`false` skips the step) |
+| `steps.imageBuild` | `gcr.io/kaniko-project/executor:v1.24.0-debug`, pinned by digest | `{ source: Source }` → `BuiltImage { repository, tag, digest, tar }`; with `push`: `PushedImage { repository, tag, digest, cached }` | `repository` (image name; tag = run version, `-attempt<n>` on reruns), `context?` (dir in the source tree with the `Dockerfile`; default the root), `push?` (push to `repository` instead of a tarball), `contentTag?` (with `push`: tag `src-<hash>` of the context and the build recipe; skip when the registry has it, with `cached: true`), `registryMap?` (kaniko `--registry-map`: registry → mirror prefix; default none), `stallTimeoutSeconds?` (default 300, 0 = off), `retries?` (default 2), `registryAuth?` (`{ username, password: Secret }`: log in to the `repository` registry; or a config reference to `{ username, password: { secret: NAME } }`, with `NAME` in `secrets`), `insecureRegistry?` (that registry is plain HTTP), `secrets?` (declared secrets a config `registryAuth` may name), `enabled?` (`false` skips the step) |
 | `steps.helmDeployOverSsh` | `alpine:3.22` (+ `apk add openssh-client` at run time) | `{ source: Source, <key>: BuiltImage }` per `images` (`PushedImage` with `fromRegistry`) → `HelmRelease { release, namespace, image? }` (`image`: the first image; absent without images) | `host` (`user@host`), `sshKey: Secret`, `release`, `namespace`, `chart` (dir in the source), `values?` (chart-relative `-f`), `set?`, `images?` (input key → values prefix; default `{ image: "image" }`; `{}` = the chart alone), `fromRegistry?`, `imageRefs?` (`PushedImage` input key → values prefix: set as `<prefix>.repository`/`.tag`/`.digest`, not pulled; works next to tarball `images`), `deployments?` (default `[release]`, none without images), `rolloutTimeoutSeconds?` (default 120), `dependencies?` (pull `Chart.yaml`'s dependencies on the host), `helmWait?` / `atomic?` (`helm --wait` / `--atomic`), `healthUrl?` (URL or `{ url, attempts?, delaySeconds? }`) |
-| `steps.helmDeployInCluster` (and the `target` form of `helmDeployOverSsh`) | `alpine/helm:3.16.2` (ssh: `alpine:3.22`) | `{ source: Source, <key>: PushedImage }` per `images`, `<key>: optional(PushedImage)` per `imageRefs` → `HelmRelease` (`image`: `repository:tag@digest` of the first image) | `target` (a `select` case's reference to a `helmTarget()` variant: `{ kind: "inCluster", namespace, release, values, set, url? }` for helmDeployInCluster, the same plus `host` with `kind: "ssh"` for helmDeployOverSsh; `url` is where step pods reach the release, default its in-cluster Service URL, and is passed on in `HelmRelease.url`), `sshKey: Secret` (ssh only), `chart`, `inlineValues?` (objects written as values files after the target's), `images?`, `imageRefs?`, `deployments?` (ssh), `rolloutTimeoutSeconds?` (default 120), `healthPath?`; helmDeployInCluster declares `clusterAccess` |
+| `steps.helmDeployInCluster` (and the `target` form of `helmDeployOverSsh`) | `alpine/helm:3.16.2` (ssh: `alpine:3.22`) | `{ source: Source, <key>: PushedImage }` per `images`, `<key>: optional(PushedImage)` per `imageRefs` → `HelmRelease` (`image`: `repository:tag@digest` of the first image) | `target` (a `select` case's reference to a `helmTarget()` variant: `{ kind: "inCluster", namespace, release, values, set, url? }` for helmDeployInCluster, the same plus `host` with `kind: "ssh"` for helmDeployOverSsh; `url` is where step pods reach the release, default its in-cluster Service URL, and is passed on in `HelmRelease.url`), `sshKey: Secret` (ssh only), `chart`, `inlineValues?` (objects written as values files after the target's; a resolved `secretRef`, `{ secret: NAME }`, becomes that secret's value, and the file holding it is deleted after the upgrade), `secrets?` (the secrets those `secretRef`s name; the step declares them), `images?`, `imageRefs?`, `deployments?` (ssh), `rolloutTimeoutSeconds?` (default 120), `healthPath?`; helmDeployInCluster declares `clusterAccess` |
 | `steps.composeDeploy` | `alpine:3.22` (work happens in the harness) | `{ image: PushedImage }` → `HelmRelease { release: <service>, namespace: "compose", image: <repository>@<digest> }` | `target` (a `select` case's reference to the compose variant of `deployTarget()`: `{ kind: "compose", deployerUrl, service, token: { secret }, healthUrl? }`), `token: Secret` (must be the secret the target names; the step's only secret), `timeoutSeconds?` |
 
 - `gitHubSource` downloads `<apiUrl>/repos/<repo>/tarball/<sha>` (retrying 5xx), strips the top-level directory and outputs it as `tree`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.23",
+  "version": "2.0.0-beta.25",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/library/helm-deploy-ssh.ts

@@ -275,6 +275,8 @@ export async function deployOverSsh(
     rolloutTimeoutSeconds: number;
     dependencies?: readonly ChartDependency[];
     wait?: HelmWait;
+    /** Files in the chart holding secret values: deleted from the host after the upgrade, also when it fails (a successful upgrade removes the whole chart). */
+    secretFiles?: readonly string[];
   },
 ): Promise<void> {
   const ssh = await sshSession(ctx, { host: o.host, key: o.key });
@@ -306,15 +308,22 @@ export async function deployOverSsh(
     });
     await ssh.run(helmPullCommand(dep, remoteChart));
   }
-  await ssh.run(
-    helmUpgradeCommand({
-      ...o.flags,
-      remoteChart,
-      release: o.release,
-      namespace: o.namespace,
-      ...(o.wait ? { wait: o.wait } : {}),
-    }),
-  );
+  try {
+    await ssh.run(
+      helmUpgradeCommand({
+        ...o.flags,
+        remoteChart,
+        release: o.release,
+        namespace: o.namespace,
+        ...(o.wait ? { wait: o.wait } : {}),
+      }),
+    );
+  } finally {
+    if (o.secretFiles?.length)
+      await ssh.run(
+        `rm -f ${o.secretFiles.map((f) => shq(`${remoteChart}/${f}`)).join(' ')}`,
+      );
+  }
   for (const d of o.deployments) {
     const target = d.includes('/') ? d : `deploy/${d}`;
     ctx.log('waiting for rollout', { deployment: target });

package/src/library/helm-deploy.ts

@@ -1,7 +1,7 @@
 // Helm deploys to a target from config (`helmTarget()`): steps.helmDeployInCluster runs helm in the step
 // pod against its own cluster; steps.helmDeployOverSsh({ target, … }) runs it on a k3s host. Pick one
 // per target kind with `select`. Images come from a registry, pinned by digest.
-import { existsSync, writeFileSync } from 'node:fs';
+import { existsSync, rmSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { z } from 'zod';
 import type { Ctx } from '../ctx/types.ts';
@@ -191,6 +191,13 @@ export interface HelmTargetDeployDef<
    * overrides only when set.
    */
   inlineValues?: readonly Readonly<Record<string, unknown>>[];
+  /**
+   * Secrets the `secretRef`s in `inlineValues` may name; the step declares them. Each `{ secret: NAME }`
+   * in the resolved values becomes the secret's value, so a chart can render its own Secret from a
+   * pipeline secret. The values files holding one are deleted after the upgrade (helm's release record
+   * keeps the values, as for any Helm value).
+   */
+  secrets?: readonly Secret[];
   /**
    * The images to deploy: step input key → the chart's values prefix. Each input is a `PushedImage`;
    * the step sets `<prefix>.repository`, `.tag`, `.digest` and `.pullPolicy=IfNotPresent`. `{}` deploys
@@ -274,6 +281,7 @@ interface AnyTargetDef {
   chart: string;
   target: unknown;
   inlineValues?: readonly Readonly<Record<string, unknown>>[];
+  secrets?: readonly Secret[];
   images?: Readonly<Record<string, string>>;
   imageRefs?: Readonly<Record<string, string>>;
   rolloutTimeoutSeconds?: number;
@@ -308,6 +316,11 @@ function targetDeploy(
     ...Object.fromEntries(refEntries.map(([key]) => [key, optional(PushedImage)])),
   };
   const configurable = { target: def.target, inlineValues: def.inlineValues ?? [] };
+  const declaredSecrets = [
+    ...(sshKey ? [sshKey] : []),
+    ...(def.secrets ?? []).filter((s) => s.name !== sshKey?.name),
+  ];
+  const valueSecrets = new Set((def.secrets ?? []).map((s) => s.name));
 
   const contract = step({
     name: def.name,
@@ -316,7 +329,7 @@ function targetDeploy(
     image: inCluster ? HELM_IMAGE : HELM_DEPLOY_IMAGE,
     input,
     output: HelmRelease,
-    secrets: sshKey ? [sshKey] : [],
+    secrets: declaredSecrets,
     timeoutSeconds: def.timeoutSeconds,
     ...(inCluster ? { clusterAccess: true } : {}),
     settings: {
@@ -349,11 +362,27 @@ function targetDeploy(
       );
     const chartDir = chartDirectory(await ctx.files.fetch(input.source.tree), def.chart);
 
+    const secretFiles: string[] = [];
     const generated = inlineValues.map((values, i) => {
       const file = `olve-values-${i}.json`;
-      writeFileSync(join(chartDir, file), JSON.stringify(values));
+      const resolved = resolveSecretRefs(values, (name) => {
+        if (!valueSecrets.has(name))
+          throw new Error(
+            `${kind} '${def.name}': inlineValues names secret ${name}, which the step does not declare (add it to secrets)`,
+          );
+        const value = (ctx.secrets as Record<string, string | undefined>)[name];
+        if (!value) throw new Error(`secret ${name} (inlineValues) is not set`);
+        return value;
+      });
+      if (resolved.secret) secretFiles.push(file);
+      writeFileSync(join(chartDir, file), JSON.stringify(resolved.value), {
+        mode: resolved.secret ? 0o600 : 0o644,
+      });
       return file;
     });
+    const removeSecretFiles = () => {
+      for (const file of secretFiles) rmSync(join(chartDir, file), { force: true });
+    };
     const flags: HelmImageFlags[] = imageEntries.map(([key, prefix]) => {
       const image = input[key]!;
       return {
@@ -395,7 +424,8 @@ function targetDeploy(
         flags: helm,
         deployments: deployments ?? (imageEntries.length ? [release] : []),
         rolloutTimeoutSeconds: rolloutTimeout,
-      });
+        secretFiles,
+      }).finally(removeSecretFiles);
     } else {
       if (!existsSync(SERVICE_ACCOUNT_TOKEN))
         throw new Error(
@@ -407,29 +437,33 @@ function targetDeploy(
         namespace,
         images: flags.map((f) => `${f.repository}@${f.digest}`).join(' '),
       });
-      await ctx.exec(
-        [
-          'helm',
-          'upgrade',
-          '--install',
-          release,
-          '.',
-          '-n',
-          namespace,
-          ...helmFlags(helm),
-          '--wait',
-          `--timeout=${rolloutTimeout}s`,
-        ],
-        {
-          cwd: chartDir,
-          env: {
-            HOME: home,
-            HELM_CACHE_HOME: join(home, 'cache'),
-            HELM_CONFIG_HOME: join(home, 'config'),
-            HELM_DATA_HOME: join(home, 'data'),
+      try {
+        await ctx.exec(
+          [
+            'helm',
+            'upgrade',
+            '--install',
+            release,
+            '.',
+            '-n',
+            namespace,
+            ...helmFlags(helm),
+            '--wait',
+            `--timeout=${rolloutTimeout}s`,
+          ],
+          {
+            cwd: chartDir,
+            env: {
+              HOME: home,
+              HELM_CACHE_HOME: join(home, 'cache'),
+              HELM_CONFIG_HOME: join(home, 'config'),
+              HELM_DATA_HOME: join(home, 'data'),
+            },
           },
-        },
-      );
+        );
+      } finally {
+        removeSecretFiles();
+      }
     }
 
     const url = (target.url ?? serviceUrl({ release, namespace })).replace(/\/+$/, '');
@@ -447,3 +481,35 @@ function targetDeploy(
     } as never;
   }) as never;
 }
+
+/**
+ * `v` with every `{ secret: NAME }` (a resolved `secretRef`) replaced by `lookup(NAME)`; `secret` says
+ * whether there was one.
+ */
+export function resolveSecretRefs(
+  v: unknown,
+  lookup: (name: string) => string,
+): { value: unknown; secret: boolean } {
+  let secret = false;
+  const walk = (x: unknown): unknown => {
+    if (Array.isArray(x)) return x.map(walk);
+    if (
+      x !== null &&
+      typeof x === 'object' &&
+      Object.getPrototypeOf(x) === Object.prototype
+    ) {
+      const entries = Object.entries(x as Record<string, unknown>);
+      if (
+        entries.length === 1 &&
+        entries[0]![0] === 'secret' &&
+        typeof entries[0]![1] === 'string'
+      ) {
+        secret = true;
+        return lookup(entries[0]![1]);
+      }
+      return Object.fromEntries(entries.map(([k, y]) => [k, walk(y)]));
+    }
+    return x;
+  };
+  return { value: walk(v), secret };
+}

package/src/library/image-build.ts

@@ -18,12 +18,30 @@ import { isConfigRef, type ConfigRef, type Configurable } from '../core/config.t
 import { isLiteral, resolveConfigRefs } from '../core/config-refs.ts';
 import { implement, type Implementation } from '../core/handlers.ts';
 import { step, type LiteralName, type Step } from '../core/steps.ts';
-import { sha256File } from '../util/hash.ts';
+import { sha256File, sha256Hex } from '../util/hash.ts';
 import { dirEntries, writeTar } from '../util/tar.ts';
 import { BuiltImage, PushedImage, Source } from './models.ts';
 
-/** kaniko's debug image (it has a busybox shell for debugging; the executor itself needs none). */
-export const KANIKO_IMAGE = 'gcr.io/kaniko-project/executor:debug';
+/** kaniko's debug image, pinned by digest. The handler needs its busybox for `mkdir` and `cp`. */
+export const KANIKO_IMAGE =
+  'gcr.io/kaniko-project/executor:v1.24.0-debug@sha256:2562c4fe551399514277ffff7dcca9a3b1628c4ea38cb017d7286dc6ea52f4cd';
+
+/** The executor flags every build uses; part of `BUILD_RECIPE`. */
+export const KANIKO_BUILD_FLAGS: readonly string[] = [
+  '--snapshot-mode=redo',
+  '--use-new-run',
+  '--compressed-caching=false',
+];
+
+/**
+ * What decides the built image besides its context: the executor and its flags. A content tag hashes
+ * it with the context, so a change here rebuilds every `contentTag` image once instead of reusing an
+ * image another build produced.
+ */
+export const BUILD_RECIPE = JSON.stringify({
+  kaniko: KANIKO_IMAGE,
+  flags: KANIKO_BUILD_FLAGS,
+});
 
 /**
  * Footgun: the build context must live under /kaniko, NOT /workspace (or another plain directory in
@@ -216,14 +234,17 @@ function checkRelative(what: string, path: string): string {
   return clean === '.' ? '' : clean;
 }
 
-/** The `src-<12 hex>` tag of a directory: the sha256 of its deterministic tar (as `ctx.output.dir` writes it). */
-export function contentTagOf(dir: string, tmpDir: string): string {
+/**
+ * The `src-<12 hex>` tag of a build context: the sha256 of the build recipe and of the directory's
+ * deterministic tar (as `ctx.output.dir` writes it).
+ */
+export function contentTagOf(dir: string, tmpDir: string, recipe = BUILD_RECIPE): string {
   mkdirSync(tmpDir, { recursive: true });
   const work = mkdtempSync(join(tmpDir, 'content-tag-'));
   try {
     const tar = join(work, 'context.tar');
     writeTar(tar, dirEntries(dir));
-    return `src-${sha256File(tar).slice(0, 12)}`;
+    return `src-${sha256Hex(`${recipe}\n${sha256File(tar)}`).slice(0, 12)}`;
   } finally {
     rmSync(work, { recursive: true, force: true });
   }
@@ -486,11 +507,11 @@ export function imageBuild<const N extends string, const K extends string = neve
     if (contentTag) {
       const digest = await registryDigest(ctx.http, repository, tag, access);
       if (digest) {
-        ctx.log('image already in the registry', {
+        ctx.log('image already in the registry; build skipped', {
           image: `${repository}:${tag}`,
           digest,
         });
-        return { repository, tag, digest };
+        return { repository, tag, digest, cached: true };
       }
     }
 
@@ -521,9 +542,7 @@ export function imageBuild<const N extends string, const K extends string = neve
         `--digest-file=${digestPath}`,
         `--destination=${repository}:${tag}`,
         ...(insecure ? ['--insecure', `--insecure-registry=${host}`] : []),
-        '--snapshot-mode=redo',
-        '--use-new-run',
-        '--compressed-caching=false',
+        ...KANIKO_BUILD_FLAGS,
         ...Object.entries(mirrors).map(([from, to]) => `--registry-map=${from}=${to}`),
       ],
       { idleTimeoutSeconds: def.stallTimeoutSeconds ?? DEFAULT_STALL_TIMEOUT_SECONDS },
@@ -539,7 +558,7 @@ export function imageBuild<const N extends string, const K extends string = neve
       throw new Error(`kaniko wrote an unexpected digest: '${digest}'`);
     if (push) {
       ctx.log('image pushed', { image: `${repository}:${tag}`, digest });
-      return { repository, tag, digest };
+      return { repository, tag, digest, cached: false };
     }
     const tar = await ctx.output.file(tarPath, { name: 'image.tar' });
     rmSync(tarPath); // the copy in the step's files is the one that counts; don't hold the image twice in /olve

package/src/library/models.ts

@@ -24,11 +24,17 @@ export const BuiltImage = model(
 
 /**
  * A container image pushed to a registry by a step (`imageBuild` with `push: true`), pulled from there
- * by the deploy. `repository` includes the registry host.
+ * by the deploy. `repository` includes the registry host. `cached` is true when `contentTag` found the
+ * image already in the registry and the build was skipped.
  */
 export const PushedImage = model(
   'PushedImage',
-  z.object({ repository: z.string(), tag: z.string(), digest: z.string() }),
+  z.object({
+    repository: z.string(),
+    tag: z.string(),
+    digest: z.string(),
+    cached: z.boolean().optional(),
+  }),
 );
 
 /** A Helm release that was installed or upgraded. */