@olivervea/pipelines 2.0.0-beta.3 → 2.0.0-beta.30

package/src/library/index.ts

@@ -1,16 +1,50 @@
-import { gitHubSource } from "./github-source.ts";
-import { helmDeployOverSsh } from "./helm-deploy-ssh.ts";
-import { imageBuild } from "./image-build.ts";
-import { script } from "./script.ts";
+import { gitHubSource } from './github-source.ts';
+import { composeDeploy } from './compose-deploy.ts';
+import { helmDeployInCluster } from './helm-deploy.ts';
+import { helmDeployOverSsh } from './helm-deploy-ssh.ts';
+import { imageBuild } from './image-build.ts';
 
-export { Source, BuiltImage, PushedImage, HelmRelease, FailureContext } from "./models.ts";
-export { inputEnvName, type RunValue, type ScriptDef } from "./script.ts";
-export type { GitHubSourceDef } from "./github-source.ts";
-export type { ImageBuildDef, ImageBuildTarballDef, ImageBuildPushDef } from "./image-build.ts";
-export type { HelmDeployOverSshDef } from "./helm-deploy-ssh.ts";
-export { sshSession, shq, type SshSession } from "./ssh.ts";
-export { serviceUrl, checkSpa } from "./checks.ts";
-export { httpHealthCheck, HTTP_HEALTH_DEFAULTS, type HttpHealthCheckOptions } from "./http-health.ts";
+export {
+  Source,
+  BuiltImage,
+  PushedImage,
+  HelmRelease,
+  FailureContext,
+} from './models.ts';
+export type { GitHubSourceDef } from './github-source.ts';
+export type {
+  ImageBuildDef,
+  ImageBuildTarballDef,
+  ImageBuildPushDef,
+  RegistryAuth,
+  RegistryAuthConfig,
+} from './image-build.ts';
+export type { HelmDeployOverSshDef } from './helm-deploy-ssh.ts';
+export {
+  type ComposeDeployDef,
+  type ComposeTargetValue,
+  type ComposeDeployResult,
+} from './compose-deploy.ts';
+export {
+  helmTarget,
+  deployTarget,
+  type DeployTargetOptions,
+  type HelmDeployInClusterDef,
+  type HelmDeploySshTargetDef,
+  type HelmTargetDeployDef,
+  type HelmTargetValue,
+  type InClusterTargetValue,
+  type SshTargetValue,
+} from './helm-deploy.ts';
+export { sshSession, type SshSession } from './ssh.ts';
+export { serviceUrl, checkSpa } from './checks.ts';
+export { httpHealthCheck, type HttpHealthCheckOptions } from './http-health.ts';
 
-/** SDK-provided steps (step + handler pairs with typed config). */
-export const steps = { script, gitHubSource, imageBuild, helmDeployOverSsh };
+/** SDK-provided steps: each is `implement(step(…), …)` with typed settings, staged directly. */
+export const steps = {
+  gitHubSource,
+  imageBuild,
+  helmDeployOverSsh,
+  helmDeployInCluster,
+  composeDeploy,
+};

package/src/library/models.ts

@@ -1,8 +1,11 @@
-import { z } from "zod";
-import { fileRef, model } from "../core/models.ts";
+import { z } from 'zod';
+import { fileRef, model } from '../core/models.ts';
 
 /** A source tree at a commit. `tree` references a directory (`path` ends with `/`). */
-export const Source = model("Source", z.object({ repo: z.string(), sha: z.string(), tree: fileRef() }));
+export const Source = model(
+  'Source',
+  z.object({ repo: z.string(), sha: z.string(), tree: fileRef() }),
+);
 
 /**
  * A container image built by a step, as an image tarball (`docker save` format) in the step's files.
@@ -10,30 +13,48 @@ export const Source = model("Source", z.object({ repo: z.string(), sha: z.string
  * (`olve-pipelines-v2` → `docker.io/library/olve-pipelines-v2`).
  */
 export const BuiltImage = model(
-  "BuiltImage",
-  z.object({ repository: z.string(), tag: z.string(), digest: z.string(), tar: fileRef() }),
+  'BuiltImage',
+  z.object({
+    repository: z.string(),
+    tag: z.string(),
+    digest: z.string(),
+    tar: fileRef(),
+  }),
 );
 
 /**
  * A container image pushed to a registry by a step (`imageBuild` with `push: true`), pulled from there
- * by the deploy. `repository` includes the registry host.
+ * by the deploy. `repository` includes the registry host. `cached` is true when `contentTag` found the
+ * image already in the registry and the build was skipped.
  */
 export const PushedImage = model(
-  "PushedImage",
-  z.object({ repository: z.string(), tag: z.string(), digest: z.string() }),
+  'PushedImage',
+  z.object({
+    repository: z.string(),
+    tag: z.string(),
+    digest: z.string(),
+    cached: z.boolean().optional(),
+  }),
 );
 
 /** A Helm release that was installed or upgraded. */
 export const HelmRelease = model(
-  "HelmRelease",
-  z.object({ release: z.string(), namespace: z.string(), image: z.string() }),
+  'HelmRelease',
+  z.object({
+    release: z.string(),
+    namespace: z.string(),
+    /** The release's main image; absent for a chart deployed without images. */
+    image: z.string().optional(),
+    /** Where step pods reach the release (the target's `url`); absent = its in-cluster Service URL (`serviceUrl`). */
+    url: z.string().optional(),
+  }),
 );
 
 /** One failed step of a run, as seen by a failure handler. */
 export const FailedStep = z.object({
   step: z.string(),
   attempt: z.number().int(),
-  /** `<pipeline id>#<run>/<step>@<attempt>` (spec §5.5). */
+  /** `<pipeline id>#<run>/<step>@<attempt>`. */
   address: z.string(),
   /** The attempt's failure reason (handler error, exit code, timeout, …). */
   reason: z.string(),
@@ -47,17 +68,17 @@ export const FailedStep = z.object({
  * the run's failed steps the handler is scoped to (all of them for an unscoped handler); never empty.
  */
 export const FailureContext = model(
-  "FailureContext",
+  'FailureContext',
   z.object({
     pipeline: z.object({ id: z.string(), name: z.string() }),
     run: z.object({
       id: z.string(),
       number: z.number().int(),
-      /** `<pipeline id>#<run>` (spec §5.5). */
+      /** `<pipeline id>#<run>`. */
       address: z.string(),
       /** The run's derived version, `yyyy.MM.dd.<run>`. */
       versionStamp: z.string(),
-      trigger: z.enum(["push", "poll", "manual"]),
+      trigger: z.enum(['push', 'poll', 'manual', 'webhook']),
       /** The run (`GET …/api/pipelines/<id>/runs/<n>`). */
       url: z.string(),
     }),

package/src/library/ssh.ts

@@ -1,7 +1,7 @@
-// sshSession — run commands on a host over SSH from a handler (alpine image: installs openssh-client).
-import { chmodSync, mkdirSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import type { Ctx } from "../ctx/types.ts";
+// sshSession — run commands on a host over SSH from a handler (installs openssh-client when the image lacks ssh).
+import { chmodSync, mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { Ctx } from '../ctx/types.ts';
 
 const SAFE = /^[A-Za-z0-9_\/.:=@%+,-]+$/;
 /** POSIX shell quoting (safe words are left bare, so logs stay readable). */
@@ -9,10 +9,18 @@ export function shq(s: string): string {
   return SAFE.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`;
 }
 
+/** Installs the ssh client unless the image has one: apk (Alpine) or apt-get (Debian, Ubuntu). */
+export const SSH_CLIENT_INSTALL =
+  'command -v ssh >/dev/null 2>&1 || if command -v apk >/dev/null 2>&1; then apk add --no-cache openssh-client; ' +
+  'else apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openssh-client; fi';
+
 export interface SshSession {
   readonly host: string;
-  /** Runs a remote command; a non-zero exit throws. */
-  run(remote: string): Promise<void>;
+  /**
+   * Runs a remote command; a non-zero exit throws. With `tty`, the command gets a terminal, so a client that
+   * dies (a cancelled step) hangs it up: without one, the command gets no signal and carries on.
+   */
+  run(remote: string, options?: { tty?: boolean }): Promise<void>;
   /** Runs a remote command and returns its stdout. */
   text(remote: string): Promise<string>;
   /** `scp -r` a local file or directory to a remote path. */
@@ -22,24 +30,42 @@ export interface SshSession {
 }
 
 /**
- * Opens an SSH "session" to `host` (`user@host`) with a private key: installs openssh-client (apk),
+ * Opens an SSH "session" to `host` (`user@host`) with a private key: installs openssh-client if needed,
  * writes the key to `<tmpDir>/ssh/id`. The host key is not verified: hosts are reached over the tailnet.
  */
-export async function sshSession(ctx: Pick<Ctx, "exec" | "sh" | "tmpDir">, opts: { host: string; key: string }): Promise<SshSession> {
-  await ctx.exec(["apk", "add", "--no-cache", "openssh-client"]);
-  const dir = join(ctx.tmpDir, "ssh");
+export async function sshSession(
+  ctx: Pick<Ctx, 'exec' | 'sh' | 'tmpDir'>,
+  opts: { host: string; key: string },
+): Promise<SshSession> {
+  await ctx.sh(SSH_CLIENT_INSTALL);
+  const dir = join(ctx.tmpDir, 'ssh');
   mkdirSync(dir, { recursive: true, mode: 0o700 });
-  const keyFile = join(dir, "id");
+  const keyFile = join(dir, 'id');
   // OpenSSH rejects a key without a trailing newline.
-  writeFileSync(keyFile, opts.key.endsWith("\n") ? opts.key : opts.key + "\n", { mode: 0o600 });
+  writeFileSync(keyFile, opts.key.endsWith('\n') ? opts.key : opts.key + '\n', {
+    mode: 0o600,
+  });
   chmodSync(keyFile, 0o600);
-  const sshOpts = ["-i", keyFile, "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", "-o", "BatchMode=yes"];
-  const argv = (remote: string) => ["ssh", ...sshOpts, opts.host, remote];
+  const sshOpts = [
+    '-i',
+    keyFile,
+    '-o',
+    'StrictHostKeyChecking=no',
+    '-o',
+    'UserKnownHostsFile=/dev/null',
+    '-o',
+    'LogLevel=ERROR',
+    '-o',
+    'BatchMode=yes',
+  ];
+  const argv = (remote: string) => ['ssh', ...sshOpts, opts.host, remote];
   return {
     host: opts.host,
-    run: (remote) => ctx.exec(argv(remote)),
-    text: (remote) => ctx.sh.text(argv(remote).map(shq).join(" ")),
-    copy: (local, remote) => ctx.exec(["scp", ...sshOpts, "-r", local, `${opts.host}:${remote}`]),
-    shell: (remote) => argv(remote).map(shq).join(" "),
+    run: (remote, options) =>
+      ctx.exec(options?.tty ? ['ssh', '-tt', ...argv(remote).slice(1)] : argv(remote)),
+    text: (remote) => ctx.sh.text(argv(remote).map(shq).join(' ')),
+    copy: (local, remote) =>
+      ctx.exec(['scp', ...sshOpts, '-r', local, `${opts.host}:${remote}`]),
+    shell: (remote) => argv(remote).map(shq).join(' '),
   };
 }

package/src/testing/index.ts

@@ -1,26 +1,42 @@
 // @olivervea/pipelines/testing — run a handler under `bun test` through the real harness path
 // (validate input → handler → strict output validation), with a real ctx you can override.
-import { mkdtempSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { isImplementation, type Implementation } from "../core/handlers.ts";
-import type { FileRef } from "../core/models.ts";
-import { isStep, type AnyStep, type LibraryStep, type OutputValue, type SecretsOf, type StepInput } from "../core/steps.ts";
-import { createCtx } from "../ctx/index.ts";
-import type { Ctx, RunInfo } from "../ctx/types.ts";
-import { executeStep, type FailureKind, type StepFunction } from "../harness/execute.ts";
-import { sha256Hex } from "../util/hash.ts";
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { isImplementation, type Implementation } from '../core/handlers.ts';
+import { cellName, parseCell } from '../core/matrix.ts';
+import type { FileRef } from '../core/models.ts';
+import type {
+  AnyStep,
+  CellOf,
+  ConfigOf,
+  OutputValue,
+  SecretsOf,
+  StepInput,
+} from '../core/steps.ts';
+import { createCtx } from '../ctx/index.ts';
+import type { Ctx, RunInfo } from '../ctx/types.ts';
+import type { StepLink } from '../ctx/links.ts';
+import { EXIT } from '../harness/exit.ts';
+import { executeStep, type FailureKind, type StepFunction } from '../harness/execute.ts';
+import { sha256Hex } from '../util/hash.ts';
 
 export interface TestHandlerOptions<S extends AnyStep> {
   /** Replace parts of ctx (e.g. `sh`, `http`, `files`). The rest is the real implementation. */
-  ctx?: Partial<Ctx<SecretsOf<S>>>;
+  ctx?: Partial<Ctx<SecretsOf<S>, ConfigOf<S>>>;
+  /** The step's config entries (z.input view: defaults are applied), like `$config` in input.json. */
+  config?: Record<string, unknown>;
   /** Secret values (instead of reading the environment). */
   secrets?: Partial<Record<SecretsOf<S>, string>>;
-  run?: Partial<Omit<RunInfo, "step">>;
+  run?: Partial<Omit<RunInfo, 'step'>>;
   /** Directory used as $OLVE_DIR (default: a fresh temp dir). */
   dir?: string;
   /** Abort signal handed to ctx.signal. */
   signal?: AbortSignal;
+  /** `<step>/<input>` → URL returned by `ctx.inputUrl`. */
+  callbackUrls?: Record<string, string>;
+  /** The cell to run a matrix step in (`ctx.cell`), like `OLVE_CELL`. */
+  cell?: CellOf<S>;
 }
 
 export type TestResult<S extends AnyStep> = {
@@ -29,58 +45,107 @@ export type TestResult<S extends AnyStep> = {
   filesDir: string;
   logs: string[];
   exitCode: number;
+  /** What the handler passed to `ctx.link`; absent when nothing. */
+  links?: StepLink[];
 } & (
-  | { status: "succeeded"; output: OutputValue<S> }
-  | { status: "skipped"; reason: string }
-  | { status: "failed"; error: { kind: FailureKind; message: string } }
+  | { status: 'succeeded'; output: OutputValue<S> }
+  | { status: 'skipped'; reason: string }
+  | { status: 'failed'; error: { kind: FailureKind; message: string } }
 );
 
 /**
- * Runs a handler (the default export of `handlers/<step>.ts`) or a library step with `input`.
+ * Runs a step that carries its implementation with `input`.
  * Input is validated like `input.json`, the result like a real run: an invalid output gives
  * `{ status: "failed", error: { kind: "output" } }`, a throwing handler `kind: "handler"`.
  */
 export async function testHandler<S extends AnyStep>(
-  target: Implementation<S> | (S & LibraryStep),
-  input: StepInput<S["input"]>,
+  target: Implementation<S>,
+  input: StepInput<S['input']>,
   opts: TestHandlerOptions<S> = {},
 ): Promise<TestResult<S>> {
-  let step: AnyStep;
-  let fn: StepFunction;
-  if (isImplementation(target)) {
-    step = target.step;
-    fn = target.handler as unknown as StepFunction;
-  } else if (isStep(target) && target.builtin) {
-    step = target;
-    fn = target.builtin.run as StepFunction;
-  } else throw new TypeError("testHandler expects implement(...) (a handler module's default export) or a library step");
+  if (!isImplementation(target))
+    throw new TypeError(
+      'testHandler expects implement(…): a step that carries its implementation',
+    );
+  const step: AnyStep = target.step;
+  const fn = target.handler as unknown as StepFunction;
 
+  const cell = step.matrix
+    ? parseCell(
+        step.name,
+        step.matrix,
+        opts.cell === undefined ? undefined : JSON.stringify(opts.cell),
+      )
+    : undefined;
   const dir = opts.dir ?? mkdtempSync(join(tmpdir(), `olve-test-${step.name}-`));
-  const filesDir = join(dir, "work", "files");
+  const filesDir = join(dir, 'work', 'files');
   const logs: string[] = [];
-  const run: RunInfo = { pipeline: "test", id: "test", number: 1, attempt: 1, version: "0.0.0-test", ...opts.run, step: step.name };
+  if (cell && !cell.ok) {
+    logs.push(cell.message);
+    return {
+      dir,
+      filesDir,
+      logs,
+      exitCode: EXIT.inputInvalid,
+      status: 'failed',
+      error: { kind: 'input', message: cell.message },
+    };
+  }
+  const run: RunInfo = {
+    pipeline: 'test',
+    id: 'test',
+    number: 1,
+    attempt: 1,
+    version: '0.0.0-test',
+    ...opts.run,
+    step: cell?.ok ? cellName(step.name, cell.cell) : step.name,
+  };
   const real = createCtx({
     run,
     secrets: step.secrets,
     secretValues: opts.secrets as Record<string, string> | undefined,
-    paths: { root: dir, workFiles: filesDir, tmp: join(dir, "tmp") },
+    paths: { root: dir, workFiles: filesDir, tmp: join(dir, 'tmp') },
     signal: opts.signal ?? new AbortController().signal,
     logSink: (l) => logs.push(l),
+    callbackUrls: opts.callbackUrls,
+    ...(cell?.ok ? { cell: cell.cell } : {}),
   });
   const ctx = { ...real, ...opts.ctx } as Ctx;
   // Round-trip through JSON like input.json does (drops undefined, rejects non-JSON values early).
-  const raw = JSON.parse(JSON.stringify(input ?? {}));
-  const exec = await executeStep({ step, fn, rawInput: raw, ctx, workFiles: filesDir, log: (l) => logs.push(l) });
-  return { dir, filesDir, logs, exitCode: exec.exitCode, ...exec.envelope } as TestResult<S>;
+  const raw = JSON.parse(
+    JSON.stringify({ ...input, ...(opts.config ? { $config: opts.config } : {}) }),
+  );
+  const exec = await executeStep({
+    step,
+    fn,
+    rawInput: raw,
+    ctx,
+    workFiles: filesDir,
+    log: (l) => logs.push(l),
+  });
+  return {
+    dir,
+    filesDir,
+    logs,
+    exitCode: exec.exitCode,
+    ...exec.envelope,
+  } as TestResult<S>;
 }
 
 /** A file reference for test inputs (`sha256` defaults to the hash of `content`). */
-export function testFileRef(ref: { step: string; path: string; run?: number; attempt?: number; sha256?: string; content?: string | Uint8Array }): FileRef {
+export function testFileRef(ref: {
+  step: string;
+  path: string;
+  run?: number;
+  attempt?: number;
+  sha256?: string;
+  content?: string | Uint8Array;
+}): FileRef {
   return {
     run: ref.run ?? 1,
     step: ref.step,
     attempt: ref.attempt ?? 1,
     path: ref.path,
-    sha256: ref.sha256 ?? sha256Hex(ref.content ?? ""),
+    sha256: ref.sha256 ?? sha256Hex(ref.content ?? ''),
   } as FileRef;
 }

package/src/util/canonical.ts

@@ -2,7 +2,7 @@
 // (order is meaningful: stages, steps, enum values), `undefined` members dropped. Stable for hashing.
 export function canonicalize(v: unknown): unknown {
   if (Array.isArray(v)) return v.map(canonicalize);
-  if (v !== null && typeof v === "object") {
+  if (v !== null && typeof v === 'object') {
     const out: Record<string, unknown> = {};
     for (const k of Object.keys(v).sort()) {
       const x = (v as Record<string, unknown>)[k];
@@ -10,10 +10,13 @@ export function canonicalize(v: unknown): unknown {
     }
     return out;
   }
-  if (typeof v === "number" && !Number.isFinite(v)) throw new Error(`non-finite number in canonical JSON: ${v}`);
-  if (typeof v === "bigint" || typeof v === "function" || typeof v === "symbol") throw new Error(`${typeof v} is not JSON`);
+  if (typeof v === 'number' && !Number.isFinite(v))
+    throw new Error(`non-finite number in canonical JSON: ${v}`);
+  if (typeof v === 'bigint' || typeof v === 'function' || typeof v === 'symbol')
+    throw new Error(`${typeof v} is not JSON`);
   return v;
 }
 
 /** Canonical JSON text, 2-space indented, with a trailing newline. */
-export const canonicalJson = (v: unknown): string => JSON.stringify(canonicalize(v), null, 2) + "\n";
+export const canonicalJson = (v: unknown): string =>
+  JSON.stringify(canonicalize(v), null, 2) + '\n';

package/src/util/code-hash.ts

@@ -0,0 +1,17 @@
+import { canonicalJson } from './canonical.ts';
+import { sha256Hex } from './hash.ts';
+
+/** Identity of the harness compiled into a binary. */
+export interface HarnessInfo {
+  /** Bun version (runtime compiled into the binary). */
+  bun: string;
+  /** @olivervea/pipelines package version. */
+  sdk: string;
+  /** `sha256:` hash over the SDK runtime sources and the zod version. */
+  sdkHash: string;
+}
+
+/** The hash of a whole build: its code bundled with the SDK external, combined with the harness. */
+export function composeCodeHash(bundleHash: string, harness: HarnessInfo): string {
+  return 'sha256:' + sha256Hex(canonicalJson({ bundle: bundleHash, harness }));
+}

package/src/util/hash.ts

@@ -1,13 +1,13 @@
-import { openSync, readSync, closeSync } from "node:fs";
+import { openSync, readSync, closeSync } from 'node:fs';
 
 export function sha256Hex(data: string | Uint8Array | ArrayBuffer): string {
-  return new Bun.CryptoHasher("sha256").update(data).digest("hex");
+  return new Bun.CryptoHasher('sha256').update(data).digest('hex');
 }
 
 /** Streaming SHA-256 of a file (constant memory). */
 export function sha256File(path: string): string {
-  const h = new Bun.CryptoHasher("sha256");
-  const fd = openSync(path, "r");
+  const h = new Bun.CryptoHasher('sha256');
+  const fd = openSync(path, 'r');
   try {
     const buf = new Uint8Array(1 << 20);
     for (;;) {
@@ -18,5 +18,5 @@ export function sha256File(path: string): string {
   } finally {
     closeSync(fd);
   }
-  return h.digest("hex");
+  return h.digest('hex');
 }