@olivervea/pipelines 2.0.0-beta.3 → 2.0.0-beta.31

package/src/harness/result.ts

@@ -1,22 +1,29 @@
 // Writing, printing and uploading the result of `run` / `map-trigger`.
-import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import { canonicalJson } from "../util/canonical.ts";
-import { dirEntries, writeTarGz, type TarEntry } from "../util/tar.ts";
-import { redactUrl } from "../ctx/files.ts";
-import type { Envelope } from "./execute.ts";
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import { canonicalJson } from '../util/canonical.ts';
+import { dirEntries, writeTarGz, type TarEntry } from '../util/tar.ts';
+import { redactUrl } from '../ctx/files.ts';
+import type { Envelope } from './execute.ts';
 
-export const RESULT_BEGIN = "---OLVE-RESULT-BEGIN---";
-export const RESULT_END = "---OLVE-RESULT-END---";
-export const MANIFEST_BEGIN = "---OLVE-MANIFEST-BEGIN---";
-export const MANIFEST_END = "---OLVE-MANIFEST-END---";
+export const RESULT_BEGIN = '---OLVE-RESULT-BEGIN---';
+export const RESULT_END = '---OLVE-RESULT-END---';
+export const MANIFEST_BEGIN = '---OLVE-MANIFEST-BEGIN---';
+export const MANIFEST_END = '---OLVE-MANIFEST-END---';
 
 /** Atomically writes the envelope to `<work>/output.json`. */
 export function writeEnvelope(work: string, envelope: Envelope): string {
   mkdirSync(work, { recursive: true });
-  const path = join(work, "output.json");
-  writeFileSync(path + ".partial", canonicalJson(envelope));
-  renameSync(path + ".partial", path);
+  const path = join(work, 'output.json');
+  writeFileSync(path + '.partial', canonicalJson(envelope));
+  renameSync(path + '.partial', path);
   return path;
 }
 
@@ -27,9 +34,15 @@ export function sentinelBlock(envelope: Envelope): string {
 
 /** Packs output.json + files/ into result.tar.gz (deterministic). */
 export async function packResult(work: string, out: string): Promise<void> {
-  const entries: TarEntry[] = [{ path: "output.json", type: "file", mode: 0o644, source: join(work, "output.json") }];
-  const files = join(work, "files");
-  if (existsSync(files)) entries.push({ path: "files", type: "dir", mode: 0o755 }, ...dirEntries(files, "files"));
+  const entries: TarEntry[] = [
+    { path: 'output.json', type: 'file', mode: 0o644, source: join(work, 'output.json') },
+  ];
+  const files = join(work, 'files');
+  if (existsSync(files))
+    entries.push(
+      { path: 'files', type: 'dir', mode: 0o755 },
+      ...dirEntries(files, 'files'),
+    );
   await writeTarGz(out, entries);
 }
 
@@ -40,15 +53,20 @@ export async function uploadFile(
   url: string,
   file: string,
   log: (line: string) => void,
-  opts: { attempts?: number; baseDelayMs?: number } = {},
+  opts: {
+    attempts?: number;
+    baseDelayMs?: number;
+    sleep?: (ms: number) => Promise<void>;
+  } = {},
 ): Promise<void> {
   const attempts = opts.attempts ?? 5;
   const base = opts.baseDelayMs ?? 1000;
+  const sleep = opts.sleep ?? ((ms: number) => Bun.sleep(ms));
   for (let i = 1; ; i++) {
     let status = 0;
     let detail: string;
     try {
-      const res = await fetch(url, { method: "PUT", body: Bun.file(file) });
+      const res = await fetch(url, { method: 'PUT', body: Bun.file(file) });
       status = res.status;
       if (res.ok) return;
       detail = `HTTP ${res.status} ${res.statusText}`;
@@ -57,9 +75,10 @@ export async function uploadFile(
       detail = (e as Error).message;
     }
     const transient = status === 0 || TRANSIENT(status);
-    if (!transient || i >= attempts) throw new Error(`upload to ${redactUrl(url)} failed: ${detail}`);
+    if (!transient || i >= attempts)
+      throw new Error(`upload to ${redactUrl(url)} failed: ${detail}`);
     log(`upload to ${redactUrl(url)} failed (${detail}); retry ${i}/${attempts - 1}`);
-    await Bun.sleep(base * 2 ** (i - 1));
+    await sleep(base * 2 ** (i - 1));
   }
 }
 
@@ -68,8 +87,10 @@ export async function uploadFile(
  * *contents* of files/. The directories themselves are kept (olve-init creates them 1777).
  */
 export function removeStale(work: string): void {
-  rmSync(join(work, "output.json"), { force: true });
-  rmSync(join(work, "output.json.partial"), { force: true });
-  const files = join(work, "files");
-  if (existsSync(files)) for (const e of readdirSync(files)) rmSync(join(files, e), { recursive: true, force: true });
+  rmSync(join(work, 'output.json'), { force: true });
+  rmSync(join(work, 'output.json.partial'), { force: true });
+  const files = join(work, 'files');
+  if (existsSync(files))
+    for (const e of readdirSync(files))
+      rmSync(join(files, e), { recursive: true, force: true });
 }

package/src/harness/validate.ts

@@ -1,22 +1,31 @@
-import { z } from "zod";
-import { isModel, isOptional } from "../core/models.ts";
-import type { AnyStep } from "../core/steps.ts";
+import { z } from 'zod';
+import { isModel, isOptional } from '../core/models.ts';
+import { isExternal } from '../core/external.ts';
+import { isCells } from '../core/matrix.ts';
+import type { AnyStep } from '../core/steps.ts';
 
 export type Validation<T> = { ok: true; value: T } | { ok: false; issues: string[] };
 
 export function formatIssues(err: z.ZodError, prefix: string): string[] {
-  return err.issues.map((i) => `${prefix}${i.path.length ? "." + i.path.map(String).join(".") : ""}: ${i.message}`);
+  return err.issues.map(
+    (i) =>
+      `${prefix}${i.path.length ? '.' + i.path.map(String).join('.') : ''}: ${i.message}`,
+  );
 }
 
 const isPlainObject = (v: unknown): v is Record<string, unknown> =>
-  typeof v === "object" && v !== null && !Array.isArray(v) && (Object.getPrototypeOf(v) === Object.prototype || Object.getPrototypeOf(v) === null);
+  typeof v === 'object' &&
+  v !== null &&
+  !Array.isArray(v) &&
+  (Object.getPrototypeOf(v) === Object.prototype || Object.getPrototypeOf(v) === null);
 
 /**
  * zod strips unknown keys silently (at every depth). Compares the raw value with the parsed one:
  * any key present in a raw object but absent from the parsed object at the same place was stripped.
  */
-export function strippedKeys(raw: unknown, parsed: unknown, path = ""): string[] {
-  if (Array.isArray(raw) && Array.isArray(parsed)) return raw.flatMap((x, i) => strippedKeys(x, parsed[i], `${path}[${i}]`));
+export function strippedKeys(raw: unknown, parsed: unknown, path = ''): string[] {
+  if (Array.isArray(raw) && Array.isArray(parsed))
+    return raw.flatMap((x, i) => strippedKeys(x, parsed[i], `${path}[${i}]`));
   if (isPlainObject(raw) && isPlainObject(parsed)) {
     const out: string[] = [];
     for (const k of Object.keys(raw)) {
@@ -30,27 +39,80 @@ export function strippedKeys(raw: unknown, parsed: unknown, path = ""): string[]
 }
 
 /** Strict parse: zod validation plus rejection of unknown keys at any depth. */
-export function parseStrict(schema: z.ZodType, value: unknown, prefix: string): Validation<unknown> {
+export function parseStrict(
+  schema: z.ZodType,
+  value: unknown,
+  prefix: string,
+): Validation<unknown> {
   const r = schema.safeParse(value);
   if (!r.success) return { ok: false, issues: formatIssues(r.error, prefix) };
   const extra = strippedKeys(value, r.data);
-  if (extra.length) return { ok: false, issues: extra.map((k) => `${prefix}.${k}: unknown field (outputs are validated strictly)`) };
+  if (extra.length)
+    return {
+      ok: false,
+      issues: extra.map(
+        (k) => `${prefix}.${k}: unknown field (outputs are validated strictly)`,
+      ),
+    };
   return { ok: true, value: r.data };
 }
 
 function entrySchema(e: unknown): z.ZodType {
-  if (isOptional(e)) return e.model.schema;
+  if (isCells(e))
+    return z.array(
+      z.object({ cell: z.record(z.string(), z.string()), output: e.model.schema }),
+    );
+  if (isOptional(e) || isExternal(e)) return e.model.schema;
   if (isModel(e)) return e.schema;
   return e as z.ZodType;
 }
 
+/** Reserved input.json key holding the step's config entries (written by the service). */
+export const CONFIG_KEY = '$config';
+
+/** Parses `$config` with each declared entry's schema (defaults applied). Undeclared entries are ignored. */
+export function validateConfig(
+  step: AnyStep,
+  raw: unknown,
+): Validation<Record<string, unknown>> {
+  const given = isPlainObject(raw) ? raw[CONFIG_KEY] : undefined;
+  if (given !== undefined && !isPlainObject(given))
+    return {
+      ok: false,
+      issues: [`input.${CONFIG_KEY} must be an object keyed by config entry names`],
+    };
+  const issues: string[] = [];
+  const out: Record<string, unknown> = {};
+  for (const e of step.config ?? []) {
+    const value = given?.[e.name];
+    const r = e.schema.safeParse(value);
+    if (!r.success)
+      issues.push(
+        ...formatIssues(r.error, `config.${e.name}`).map((i) =>
+          value === undefined ? `${i} (config entry '${e.name}' is not set)` : i,
+        ),
+      );
+    else if (r.data !== undefined) out[e.name] = r.data;
+  }
+  return issues.length ? { ok: false, issues } : { ok: true, value: out };
+}
+
 /** Validates input.json against the step's input entries. Unknown top-level keys are rejected; nested values are parsed (unknown nested keys stripped). */
-export function validateInput(step: AnyStep, raw: unknown): Validation<Record<string, unknown>> {
-  if (!isPlainObject(raw)) return { ok: false, issues: ["input.json must be a JSON object keyed by the step's input names"] };
+export function validateInput(
+  step: AnyStep,
+  raw: unknown,
+): Validation<Record<string, unknown>> {
+  if (!isPlainObject(raw))
+    return {
+      ok: false,
+      issues: ["input.json must be a JSON object keyed by the step's input names"],
+    };
   const issues: string[] = [];
   const out: Record<string, unknown> = {};
   const declared = step.input as Record<string, unknown>;
-  for (const k of Object.keys(raw)) if (!Object.hasOwn(declared, k)) issues.push(`input.${k}: not an input of step '${step.name}'`);
+  for (const k of Object.keys(raw))
+    if (k !== CONFIG_KEY && !Object.hasOwn(declared, k))
+      issues.push(`input.${k}: not an input of step '${step.name}'`);
   for (const [k, e] of Object.entries(declared)) {
     if (!Object.hasOwn(raw, k)) {
       if (isOptional(e)) continue;
@@ -68,7 +130,12 @@ export function validateInput(step: AnyStep, raw: unknown): Validation<Record<st
 export function validateOutput(step: AnyStep, value: unknown): Validation<unknown> {
   if (!step.output) {
     if (value === undefined || value === null) return { ok: true, value: null };
-    return { ok: false, issues: [`step '${step.name}' declares no output model but the handler returned a value`] };
+    return {
+      ok: false,
+      issues: [
+        `step '${step.name}' declares no output model but the handler returned a value`,
+      ],
+    };
   }
-  return parseStrict(step.output.schema, value, "output");
+  return parseStrict(step.output.schema, value, 'output');
 }

package/src/index.ts

@@ -1,12 +1,141 @@
 /// <reference path="./assets.d.ts" />
 // @olivervea/pipelines — authoring API (contracts, handlers, pipeline builder).
-export { z } from "zod";
-export { model, fileRef, optional, type Model, type AnyModel, type Infer, type InferInput, type Plain, type FileRef, type Optional } from "./core/models.ts";
-export { secret, type Secret, type DeclaredSecrets } from "./core/secrets.ts";
-export { hmacSha256, githubSignature, bearer, type WebhookVerify, type WebhookEvent } from "./core/webhooks.ts";
-export { step, type Step, type AnyStep, type StepDef, type StepInput, type StepKind, type InputEntry, type InputSpec, type LibraryStep, type OutputValue } from "./core/steps.ts";
-export { implement, type Implementation, type Handler, type HandlerResult, type Skipped } from "./core/handlers.ts";
-export type { Ctx, RunInfo, Sh, ShOptions, ExecOptions, Http, HttpInit, RetryOptions, OutputFileOptions } from "./ctx/types.ts";
-export type { Ref, Mappable, MapperResult } from "./builder/refs.ts";
-export { pipeline, Pipeline, type PipelineDef, type InRefs, type InitialAvail, type WhenCond, type PushEvent, type StageEntry, type CheckStage, type MapperFn } from "./builder/pipeline.ts";
-export { steps, Source, BuiltImage, PushedImage, HelmRelease, FailureContext, type RunValue, type ScriptDef, type GitHubSourceDef, type ImageBuildDef, type ImageBuildTarballDef, type ImageBuildPushDef, type HelmDeployOverSshDef, httpHealthCheck, type HttpHealthCheckOptions, sshSession, shq, type SshSession, serviceUrl, checkSpa } from "./library/index.ts";
+export { z } from 'zod';
+export {
+  model,
+  fileRef,
+  optional,
+  type Model,
+  type AnyModel,
+  type Infer,
+  type InferInput,
+  type Plain,
+  type FileRef,
+  type Optional,
+} from './core/models.ts';
+export { secret, type Secret, type DeclaredSecrets } from './core/secrets.ts';
+export {
+  cells,
+  keepRunning,
+  cascadeFailure,
+  type Axes,
+  type Cell,
+  type CellOutput,
+  type Cells,
+  type FailureStrategy,
+  type Matrix,
+  type MatrixDef,
+} from './core/matrix.ts';
+export {
+  config,
+  secretRef,
+  isConfigRef,
+  type ConfigEntry,
+  type AnyConfigEntry,
+  type ConfigValues,
+  type ConfigRef,
+  type ConfigNode,
+  type Configurable,
+  type DiscriminatorOf,
+  type VariantOf,
+} from './core/config.ts';
+export {
+  hmacSha256,
+  githubSignature,
+  bearer,
+  type WebhookVerify,
+  type WebhookEvent,
+} from './core/webhooks.ts';
+export { resolveConfigRefs, isLiteral, type Resolved } from './core/config-refs.ts';
+export {
+  callback,
+  external,
+  person,
+  poll,
+  type External,
+  type ExternalOptions,
+  type ExternalSource,
+  type PollOptions,
+} from './core/external.ts';
+export {
+  step,
+  type Step,
+  type AnyStep,
+  type StepDef,
+  type StepInput,
+  type StepKind,
+  type ConfigOf,
+  type CellOf,
+  type InputEntry,
+  type InputSpec,
+  type LiteralName,
+  type OutputValue,
+} from './core/steps.ts';
+export type { Simplify } from './core/types.ts';
+export {
+  implement,
+  type Implementation,
+  type CtxOf,
+  type Handler,
+  type HandlerResult,
+  type Skipped,
+} from './core/handlers.ts';
+export type {
+  Ctx,
+  StepCtx,
+  RunInfo,
+  Sh,
+  ShOptions,
+  ExecOptions,
+  Http,
+  HttpInit,
+  RetryOptions,
+  OutputFileOptions,
+} from './ctx/types.ts';
+export type { StepLink } from './ctx/links.ts';
+export type { Ref, Mappable, MapperResult } from './builder/refs.ts';
+export {
+  pipeline,
+  Pipeline,
+  type PipelineDef,
+  type RetentionPolicy,
+  type InRefs,
+  type InitialAvail,
+  type WhenCond,
+  type PushEvent,
+  type StageEntry,
+  type CheckStage,
+  type MapperFn,
+} from './builder/pipeline.ts';
+export {
+  steps,
+  Source,
+  BuiltImage,
+  PushedImage,
+  HelmRelease,
+  FailureContext,
+  type GitHubSourceDef,
+  type ImageBuildDef,
+  type ImageBuildTarballDef,
+  type ImageBuildPushDef,
+  type RegistryAuth,
+  type RegistryAuthConfig,
+  type HelmDeployOverSshDef,
+  helmTarget,
+  deployTarget,
+  type DeployTargetOptions,
+  type ComposeDeployDef,
+  type ComposeTargetValue,
+  type HelmDeployInClusterDef,
+  type HelmDeploySshTargetDef,
+  type HelmTargetDeployDef,
+  type HelmTargetValue,
+  type InClusterTargetValue,
+  type SshTargetValue,
+  httpHealthCheck,
+  type HttpHealthCheckOptions,
+  sshSession,
+  type SshSession,
+  serviceUrl,
+  checkSpa,
+} from './library/index.ts';

package/src/library/checks.ts

@@ -1,5 +1,5 @@
 // Checks for deployed apps, used from custom test handlers.
-import type { Ctx } from "../ctx/types.ts";
+import type { Ctx } from '../ctx/types.ts';
 
 /** The in-cluster URL of a release's Service, assuming the chart names it after the release. */
 export function serviceUrl(release: { release: string; namespace: string }): string {
@@ -10,14 +10,19 @@ export function serviceUrl(release: { release: string; namespace: string }): str
  * Checks `url` serves a single-page app: `GET <url>/` references a `/assets/*.js` bundle (Vite's
  * layout), and the bundle is served as JavaScript. Returns the page, for app-specific checks.
  */
-export async function checkSpa(ctx: Pick<Ctx, "http" | "log">, url: string): Promise<{ html: string; bundle: string }> {
-  const base = url.replace(/\/$/, "");
+export async function checkSpa(
+  ctx: Pick<Ctx, 'http' | 'log'>,
+  url: string,
+): Promise<{ html: string; bundle: string }> {
+  const base = url.replace(/\/$/, '');
   const html = await (await ctx.http.expectOk(`${base}/`)).text();
   const bundle = html.match(/src="(\/assets\/[^"]+\.js)"/)?.[1];
-  if (bundle === undefined) throw new Error(`${base}/ does not reference an /assets/*.js bundle`);
+  if (bundle === undefined)
+    throw new Error(`${base}/ does not reference an /assets/*.js bundle`);
   const js = await ctx.http.expectOk(`${base}${bundle}`);
   await js.body?.cancel().catch(() => {});
-  if (!(js.headers.get("content-type")?.includes("javascript") ?? false)) throw new Error(`${base}${bundle} is not served as JavaScript`);
-  ctx.log("SPA served", { url: base, bundle });
+  if (!(js.headers.get('content-type')?.includes('javascript') ?? false))
+    throw new Error(`${base}${bundle} is not served as JavaScript`);
+  ctx.log('SPA served', { url: base, bundle });
   return { html, bundle };
 }

package/src/library/compose-deploy.ts

@@ -0,0 +1,115 @@
+// steps.composeDeploy — ask a Docker Compose deployer to recreate a service with an image. The step is
+// thin on purpose: the deployer (pull, recreate, health, rollback) lives beside the compose project
+// and is fixed there, behind a versioned HTTP API, without editing pipelines.
+import type { Ctx } from '../ctx/types.ts';
+import type { Configurable } from '../core/config.ts';
+import { resolveConfigRefs } from '../core/config-refs.ts';
+import { implement, type Implementation } from '../core/handlers.ts';
+import type { Secret } from '../core/secrets.ts';
+import { step, type LiteralName, type Step } from '../core/steps.ts';
+import { httpHealthCheck } from './http-health.ts';
+import { HelmRelease, PushedImage } from './models.ts';
+
+/** The step's own image: the work is one HTTP call from the harness. */
+export const COMPOSE_DEPLOY_IMAGE = 'alpine:3.22';
+
+/** A resolved `kind: compose` target. */
+export interface ComposeTargetValue {
+  kind: 'compose';
+  deployerUrl: string;
+  service: string;
+  token: { secret: string };
+  healthUrl?: string | undefined;
+  link?: string | undefined;
+}
+
+/** What the deployer answers to `POST /v1/deploy`. */
+export interface ComposeDeployResult {
+  status: 'deployed' | 'rolled-back' | 'failed';
+  service: string;
+  image: string;
+  previousImage?: string;
+  message?: string;
+  log?: string[];
+}
+
+export interface ComposeDeployDef<N extends string, K extends string> {
+  name: LiteralName<N>;
+  /** What the step does, shown in the UI. Required; may be empty. */
+  description: string;
+  /** A compose target, usually a `select` case's reference. */
+  target: Configurable<ComposeTargetValue>;
+  /** The secret the target's `token` names (declared by the step). */
+  token: Secret<K>;
+  timeoutSeconds?: number;
+}
+
+const input = { image: PushedImage };
+
+/**
+ * Deploys a registry image (by digest) to a compose service: `POST <deployerUrl>/v1/deploy
+ * { service, image: <repository>@<digest> }` with the target's bearer token. The deployer pulls the
+ * image, recreates the service, waits for its health and rolls back on failure; the step reports its
+ * answer, then (with the target's `healthUrl`) checks health itself. Output: a `HelmRelease` with the
+ * service as `release` and `compose` as `namespace`, so it can share a step name with the helm deploys.
+ */
+export function composeDeploy<const N extends string, const K extends string>(
+  def: ComposeDeployDef<N, K>,
+): Implementation<Step<N, typeof input, typeof HelmRelease, K>> {
+  const contract = step({
+    name: def.name,
+    description: def.description,
+    kind: 'composeDeploy',
+    image: COMPOSE_DEPLOY_IMAGE,
+    input,
+    output: HelmRelease,
+    secrets: [def.token],
+    timeoutSeconds: def.timeoutSeconds,
+    settings: { target: def.target, token: def.token.name },
+  });
+  return implement(contract, async ({ image }, ctx: Ctx) => {
+    const target = resolveConfigRefs(def.target, ctx.config ?? {}) as ComposeTargetValue;
+    if (target?.kind !== 'compose')
+      throw new Error(
+        `composeDeploy '${def.name}': the target is kind '${String(target?.kind)}', not 'compose'`,
+      );
+    if (target.token.secret !== def.token.name)
+      throw new Error(
+        `composeDeploy '${def.name}': the target's token names secret ${target.token.secret}, but the step declares ${def.token.name}`,
+      );
+    const token = (ctx.secrets as Record<string, string>)[def.token.name];
+    const ref = `${image.repository}@${image.digest}`;
+    const url = `${target.deployerUrl.replace(/\/+$/, '')}/v1/deploy`;
+    ctx.log('deploying', {
+      service: target.service,
+      image: ref,
+      deployer: target.deployerUrl,
+    });
+    const res = await ctx.http.fetch(url, {
+      method: 'POST',
+      headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+      body: JSON.stringify({ service: target.service, image: ref }),
+    });
+    const text = await res.text();
+    let result: ComposeDeployResult | undefined;
+    try {
+      result = JSON.parse(text) as ComposeDeployResult;
+    } catch {
+      // reported below
+    }
+    for (const line of result?.log ?? []) ctx.log(`deployer: ${line}`);
+    if (!res.ok || result?.status !== 'deployed')
+      throw new Error(
+        `the deployer answered HTTP ${res.status}: ${result ? `${result.status}${result.message ? `: ${result.message}` : ''}` : text.slice(0, 300)}`,
+      );
+    if (target.healthUrl)
+      await httpHealthCheck(ctx, target.healthUrl, { attempts: 12, delaySeconds: 5 });
+    if (target.link) ctx.link(target.link);
+    ctx.log('deployed', {
+      service: target.service,
+      image: ref,
+      previous: result.previousImage ?? '-',
+    });
+    return { release: target.service, namespace: 'compose', image: ref };
+  }) as never;
+}