@olivervea/pipelines 2.0.0-beta.3 → 2.0.0-beta.31

package/src/ctx/files.ts

@@ -4,19 +4,31 @@
 // `path` minus the trailing "/" plus ".tar" for directories (a deterministic tar). The step writes
 // objects to `$OLVE_DIR/work/files/<object path>`; the service stores them under
 // `runs/<p>/<run>/<step>/<attempt>/files/<object path>`.
-import { copyFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
-import { basename, dirname, join, resolve } from "node:path";
-import { fileRefSchema, FILE_PATH, type FileRef } from "../core/models.ts";
-import { ENV } from "../harness/env.ts";
-import { sha256File } from "../util/hash.ts";
-import { dirEntries, extractTar, writeTar } from "../util/tar.ts";
-import type { OutputFileOptions, RunInfo } from "./types.ts";
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import { basename, dirname, join, resolve } from 'node:path';
+import { fileRefSchema, FILE_PATH, type FileRef } from '../core/models.ts';
+import { ENV } from '../harness/env.ts';
+import { sha256File } from '../util/hash.ts';
+import { dirEntries, extractTar, writeTar } from '../util/tar.ts';
+import type { OutputFileOptions, RunInfo } from './types.ts';
 
-export const objectPath = (refPath: string): string => (refPath.endsWith("/") ? refPath.slice(0, -1) + ".tar" : refPath);
-export const fileKey = (ref: Pick<FileRef, "run" | "step" | "attempt" | "path">): string => `${ref.run}/${ref.step}/${ref.attempt}/${ref.path}`;
+export const objectPath = (refPath: string): string =>
+  refPath.endsWith('/') ? refPath.slice(0, -1) + '.tar' : refPath;
+export const fileKey = (
+  ref: Pick<FileRef, 'run' | 'step' | 'attempt' | 'path'>,
+): string => `${ref.run}/${ref.step}/${ref.attempt}/${ref.path}`;
 
 /** Strips everything from `?` on (pre-signed credentials). */
-export const redactUrl = (url: string): string => url.replace(/[?#].*$/, "?…");
+export const redactUrl = (url: string): string => url.replace(/[?#].*$/, '?…');
 
 export interface FilesDeps {
   run: RunInfo;
@@ -25,45 +37,59 @@ export interface FilesDeps {
   root: string;
   signal: AbortSignal;
   log: (message: string, fields?: Record<string, unknown>) => void;
+  /** Backoff between download retries (default: real sleep). */
+  sleep?: (ms: number) => Promise<void>;
 }
 
 function checkName(name: string, what: string): void {
-  if (!FILE_PATH.test(name) || name.endsWith("/")) throw new Error(`${what}: '${name}' is not a valid relative file name (no leading '/', no '..', no trailing '/')`);
+  if (!FILE_PATH.test(name) || name.endsWith('/'))
+    throw new Error(
+      `${what}: '${name}' is not a valid relative file name (no leading '/', no '..', no trailing '/')`,
+    );
 }
 
 export function createOutput(deps: FilesDeps) {
   const used = new Set<string>();
   const claim = (obj: string) => {
-    if (used.has(obj) || existsSync(join(deps.workFiles, obj))) throw new Error(`output file '${obj}' already exists`);
+    if (used.has(obj) || existsSync(join(deps.workFiles, obj)))
+      throw new Error(`output file '${obj}' already exists`);
     used.add(obj);
     const dest = join(deps.workFiles, obj);
     mkdirSync(dirname(dest), { recursive: true });
     return dest;
   };
   const ref = (path: string, dest: string): FileRef =>
-    ({ run: deps.run.number, step: deps.run.step, attempt: deps.run.attempt, path, sha256: sha256File(dest) }) as FileRef;
+    ({
+      run: deps.run.number,
+      step: deps.run.step,
+      attempt: deps.run.attempt,
+      path,
+      sha256: sha256File(dest),
+    }) as FileRef;
 
   return {
     async file(path: string, opts: OutputFileOptions = {}): Promise<FileRef> {
       const src = resolve(path);
-      if (!statSync(src).isFile()) throw new Error(`ctx.output.file: '${path}' is not a file`);
+      if (!statSync(src).isFile())
+        throw new Error(`ctx.output.file: '${path}' is not a file`);
       const name = opts.name ?? basename(src);
-      checkName(name, "ctx.output.file");
+      checkName(name, 'ctx.output.file');
       const dest = claim(name);
       copyFileSync(src, dest);
       return ref(name, dest);
     },
     async dir(path: string, opts: OutputFileOptions = {}): Promise<FileRef> {
       const src = resolve(path);
-      if (!statSync(src).isDirectory()) throw new Error(`ctx.output.dir: '${path}' is not a directory`);
+      if (!statSync(src).isDirectory())
+        throw new Error(`ctx.output.dir: '${path}' is not a directory`);
       const name = opts.name ?? basename(src);
-      checkName(name, "ctx.output.dir");
-      const dest = claim(objectPath(name + "/"));
+      checkName(name, 'ctx.output.dir');
+      const dest = claim(objectPath(name + '/'));
       writeTar(dest, dirEntries(src));
-      return ref(name + "/", dest);
+      return ref(name + '/', dest);
     },
     async write(name: string, content: string | Uint8Array): Promise<FileRef> {
-      checkName(name, "ctx.output.write");
+      checkName(name, 'ctx.output.write');
       const dest = claim(name);
       writeFileSync(dest, content);
       return ref(name, dest);
@@ -72,20 +98,34 @@ export function createOutput(deps: FilesDeps) {
 }
 
 function urlMap(root: string): Record<string, string> {
-  const raw = process.env[ENV.fileUrls] ?? (existsSync(join(root, "files.json")) ? readFileSync(join(root, "files.json"), "utf8") : undefined);
+  const raw =
+    process.env[ENV.fileUrls] ??
+    (existsSync(join(root, 'files.json'))
+      ? readFileSync(join(root, 'files.json'), 'utf8')
+      : undefined);
   if (!raw) return {};
   try {
     const m = JSON.parse(raw) as unknown;
-    if (typeof m !== "object" || m === null || Array.isArray(m)) throw new Error("not an object");
+    if (typeof m !== 'object' || m === null || Array.isArray(m))
+      throw new Error('not an object');
     return m as Record<string, string>;
   } catch (e) {
-    throw new Error(`${ENV.fileUrls} / files.json is not a JSON object of file key → URL (${(e as Error).message})`);
+    throw new Error(
+      `${ENV.fileUrls} / files.json is not a JSON object of file key → URL (${(e as Error).message})`,
+    );
   }
 }
 
 const TRANSIENT = (status: number) => status >= 500 || status === 408 || status === 429;
 
-async function download(url: string, dest: string, headers: Record<string, string>, signal: AbortSignal, log: FilesDeps["log"]): Promise<void> {
+async function download(
+  url: string,
+  dest: string,
+  headers: Record<string, string>,
+  signal: AbortSignal,
+  log: FilesDeps['log'],
+  sleep: (ms: number) => Promise<void>,
+): Promise<void> {
   const attempts = 4;
   for (let i = 1; ; i++) {
     let status = 0;
@@ -93,46 +133,81 @@ async function download(url: string, dest: string, headers: Record<string, strin
       const res = await fetch(url, { headers, signal });
       status = res.status;
       if (!res.ok) throw new Error(`HTTP ${res.status}`);
-      await Bun.write(dest + ".partial", res);
-      renameSync(dest + ".partial", dest);
+      await Bun.write(dest + '.partial', res);
+      renameSync(dest + '.partial', dest);
       return;
     } catch (e) {
-      rmSync(dest + ".partial", { force: true });
+      rmSync(dest + '.partial', { force: true });
       const transient = status === 0 || TRANSIENT(status);
-      if (signal.aborted || !transient || i >= attempts) throw new Error(`download of ${redactUrl(url)} failed: ${(e as Error).message}`);
-      log("download failed, retrying", { url: redactUrl(url), attempt: i, error: (e as Error).message });
-      await Bun.sleep(1000 * 2 ** (i - 1));
+      if (signal.aborted || !transient || i >= attempts)
+        throw new Error(`download of ${redactUrl(url)} failed: ${(e as Error).message}`);
+      log('download failed, retrying', {
+        url: redactUrl(url),
+        attempt: i,
+        error: (e as Error).message,
+      });
+      await sleep(1000 * 2 ** (i - 1));
     }
   }
 }
 
 export function createFetch(deps: FilesDeps) {
   const cache = new Map<string, string>();
+  const sleep = deps.sleep ?? ((ms: number) => Bun.sleep(ms));
   return async (input: FileRef): Promise<string> => {
     const parsed = fileRefSchema.safeParse(input);
-    if (!parsed.success) throw new Error(`ctx.files.fetch: not a valid file reference: ${JSON.stringify(input)}`);
+    if (!parsed.success)
+      throw new Error(
+        `ctx.files.fetch: not a valid file reference: ${JSON.stringify(input)}`,
+      );
     const ref = parsed.data;
     const key = fileKey(ref);
     const hit = cache.get(key);
     if (hit) return hit;
 
     const obj = objectPath(ref.path);
-    const base = join(deps.tmpDir, "files", String(ref.run), ref.step, String(ref.attempt));
-    const objFile = join(base, ".objects", obj);
+    const base = join(
+      deps.tmpDir,
+      'files',
+      String(ref.run),
+      ref.step,
+      String(ref.attempt),
+    );
+    const objFile = join(base, '.objects', obj);
     mkdirSync(dirname(objFile), { recursive: true });
 
-    const own = ref.run === deps.run.number && ref.step === deps.run.step && ref.attempt === deps.run.attempt;
+    const own =
+      ref.run === deps.run.number &&
+      ref.step === deps.run.step &&
+      ref.attempt === deps.run.attempt;
     const urls = urlMap(deps.root);
     const localDir = process.env[ENV.filesDir];
     const baseUrl = process.env[ENV.filesBaseUrl];
-    if (own && existsSync(join(deps.workFiles, obj))) copyFileSync(join(deps.workFiles, obj), objFile);
-    else if (urls[key]) await download(urls[key], objFile, {}, deps.signal, deps.log);
-    else if (localDir && existsSync(join(localDir, String(ref.run), ref.step, String(ref.attempt), "files", obj)))
-      copyFileSync(join(localDir, String(ref.run), ref.step, String(ref.attempt), "files", obj), objFile);
+    if (own && existsSync(join(deps.workFiles, obj)))
+      copyFileSync(join(deps.workFiles, obj), objFile);
+    else if (urls[key])
+      await download(urls[key], objFile, {}, deps.signal, deps.log, sleep);
+    else if (
+      localDir &&
+      existsSync(
+        join(localDir, String(ref.run), ref.step, String(ref.attempt), 'files', obj),
+      )
+    )
+      copyFileSync(
+        join(localDir, String(ref.run), ref.step, String(ref.attempt), 'files', obj),
+        objFile,
+      );
     else if (baseUrl) {
       const token = process.env[ENV.filesToken];
-      const url = `${baseUrl.replace(/\/+$/, "")}/${ref.run}/${encodeURIComponent(ref.step)}/${ref.attempt}/${obj.split("/").map(encodeURIComponent).join("/")}`;
-      await download(url, objFile, token ? { authorization: `Bearer ${token}` } : {}, deps.signal, deps.log);
+      const url = `${baseUrl.replace(/\/+$/, '')}/${ref.run}/${encodeURIComponent(ref.step)}/${ref.attempt}/${obj.split('/').map(encodeURIComponent).join('/')}`;
+      await download(
+        url,
+        objFile,
+        token ? { authorization: `Bearer ${token}` } : {},
+        deps.signal,
+        deps.log,
+        sleep,
+      );
     } else {
       throw new Error(
         `ctx.files.fetch: no source for ${key} (set ${ENV.fileUrls} / $OLVE_DIR/files.json, ${ENV.filesBaseUrl} or ${ENV.filesDir})`,
@@ -142,10 +217,12 @@ export function createFetch(deps: FilesDeps) {
     const actual = sha256File(objFile);
     if (actual !== ref.sha256) {
       rmSync(objFile, { force: true });
-      throw new Error(`ctx.files.fetch: ${key} has sha256 ${actual}, expected ${ref.sha256}`);
+      throw new Error(
+        `ctx.files.fetch: ${key} has sha256 ${actual}, expected ${ref.sha256}`,
+      );
     }
     let result = objFile;
-    if (ref.path.endsWith("/")) {
+    if (ref.path.endsWith('/')) {
       result = join(base, ref.path.slice(0, -1));
       rmSync(result, { recursive: true, force: true });
       mkdirSync(result, { recursive: true });

package/src/ctx/index.ts

@@ -1,10 +1,12 @@
-import { mkdirSync, readFileSync } from "node:fs";
-import { resolve } from "node:path";
-import { skipped } from "../core/handlers.ts";
-import type { Secret } from "../core/secrets.ts";
-import { createFetch, createOutput, redactUrl } from "./files.ts";
-import { createSh } from "./sh.ts";
-import type { Ctx, Http, RetryOptions, RunInfo } from "./types.ts";
+import { mkdirSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { skipped } from '../core/handlers.ts';
+import { createLinks } from './links.ts';
+import type { Secret } from '../core/secrets.ts';
+import type { Cell } from '../core/matrix.ts';
+import { createFetch, createOutput, redactUrl } from './files.ts';
+import { createSh } from './sh.ts';
+import type { Ctx, Http, RetryOptions, RunInfo } from './types.ts';
 
 export interface CtxOptions {
   run: RunInfo;
@@ -12,25 +14,42 @@ export interface CtxOptions {
   secrets: readonly Secret[];
   /** Secret values that take precedence over the environment (tests). */
   secretValues?: Readonly<Record<string, string>>;
+  /** Parsed config entries of the step (`ctx.config`). */
+  config?: Readonly<Record<string, unknown>>;
   paths: { root: string; workFiles: string; tmp: string };
   signal: AbortSignal;
   /** Where log lines go (default stderr). */
   logSink?: (line: string) => void;
+  /** `<step>/<input>` → callback URL (`ctx.inputUrl`). */
+  callbackUrls?: Readonly<Record<string, string>>;
+  /** A matrix step's cell (`ctx.cell`). */
+  cell?: Cell;
+  /** Commands' output goes through this process (the harness masks secrets there). */
+  pipeOutput?: boolean;
 }
 
 export function formatLog(message: string, fields?: Record<string, unknown>): string {
-  const parts = Object.entries(fields ?? {}).map(([k, v]) => `${k}=${typeof v === "string" && !/\s|"/.test(v) ? v : JSON.stringify(v)}`);
-  return parts.length ? `${message} ${parts.join(" ")}` : message;
+  const parts = Object.entries(fields ?? {}).map(
+    ([k, v]) =>
+      `${k}=${typeof v === 'string' && !/\s|"/.test(v) ? v : JSON.stringify(v)}`,
+  );
+  return parts.length ? `${message} ${parts.join(' ')}` : message;
 }
 
-export function createSecrets(declared: readonly Secret[], values?: Readonly<Record<string, string>>): Record<string, string> {
+export function createSecrets(
+  declared: readonly Secret[],
+  values?: Readonly<Record<string, string>>,
+): Record<string, string> {
   const obj: Record<string, string> = {};
   for (const s of declared) {
     Object.defineProperty(obj, s.name, {
       enumerable: true,
       get() {
         const v = values?.[s.name] ?? process.env[s.name];
-        if (v === undefined) throw new Error(`secret ${s.name} is not set (the harness reads it from the environment variable ${s.name})`);
+        if (v === undefined)
+          throw new Error(
+            `secret ${s.name} is not set (the harness reads it from the environment variable ${s.name})`,
+          );
         return v;
       },
     });
@@ -39,53 +58,81 @@ export function createSecrets(declared: readonly Secret[], values?: Readonly<Rec
 }
 
 async function httpError(res: Response, url: string): Promise<Error> {
-  let body = "";
+  let body = '';
   try {
     body = (await res.text()).slice(0, 500);
   } catch {
     /* ignore */
   }
-  return new Error(`HTTP ${res.status} ${res.statusText} from ${redactUrl(url)}${body ? `: ${body}` : ""}`);
+  return new Error(
+    `HTTP ${res.status} ${res.statusText} from ${redactUrl(url)}${body ? `: ${body}` : ''}`,
+  );
 }
 
 export function createHttp(signal: AbortSignal): Http {
-  const f = (url: string, init?: RequestInit) => fetch(url, { ...init, signal: init?.signal ?? signal });
+  const f = (url: string, init?: RequestInit) =>
+    fetch(url, { ...init, signal: init?.signal ?? signal });
   return {
     fetch: f,
-    async getJson<T>(url: string, init?: { headers?: Record<string, string> }): Promise<T> {
-      const res = await f(url, { headers: { accept: "application/json", ...init?.headers } });
+    async getJson<T>(
+      url: string,
+      init?: { headers?: Record<string, string> },
+    ): Promise<T> {
+      const res = await f(url, {
+        headers: { accept: 'application/json', ...init?.headers },
+      });
       if (!res.ok) throw await httpError(res, url);
       return (await res.json()) as T;
     },
-    async expectOk(url: string, init?: { headers?: Record<string, string>; method?: string }): Promise<Response> {
-      const res = await f(url, { method: init?.method ?? "GET", headers: init?.headers });
+    async expectOk(
+      url: string,
+      init?: { headers?: Record<string, string>; method?: string },
+    ): Promise<Response> {
+      const res = await f(url, { method: init?.method ?? 'GET', headers: init?.headers });
       if (!res.ok) throw await httpError(res, url);
       return res;
     },
   };
 }
 
-export function createRetry(signal: AbortSignal, log: (m: string, f?: Record<string, unknown>) => void) {
-  return async <T>(opts: RetryOptions, fn: (attempt: number) => Promise<T>): Promise<T> => {
-    if (!(Number.isInteger(opts.attempts) && opts.attempts >= 1)) throw new Error("retry: attempts must be a positive integer");
+/** Waits `ms`, or rejects at once when the step is cancelled. */
+function cancellableSleep(ms: number, signal: AbortSignal): Promise<void> {
+  return new Promise<void>((res, rej) => {
+    const t = setTimeout(() => {
+      signal.removeEventListener('abort', onAbort);
+      res();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(t);
+      rej(new Error('retry cancelled: the step was cancelled'));
+    };
+    signal.addEventListener('abort', onAbort, { once: true });
+  });
+}
+
+export function createRetry(
+  signal: AbortSignal,
+  log: (m: string, f?: Record<string, unknown>) => void,
+  sleep: (ms: number, signal: AbortSignal) => Promise<void> = cancellableSleep,
+) {
+  return async <T>(
+    opts: RetryOptions,
+    fn: (attempt: number) => Promise<T>,
+  ): Promise<T> => {
+    if (!(Number.isInteger(opts.attempts) && opts.attempts >= 1))
+      throw new Error('retry: attempts must be a positive integer');
     let delay = Math.max(0, opts.delaySeconds) * 1000;
     for (let attempt = 1; ; attempt++) {
       try {
         return await fn(attempt);
       } catch (e) {
         if (attempt >= opts.attempts || signal.aborted) throw e;
-        log("retrying", { attempt, of: opts.attempts, error: (e as Error)?.message ?? String(e) });
-        await new Promise<void>((res, rej) => {
-          const t = setTimeout(() => {
-            signal.removeEventListener("abort", onAbort);
-            res();
-          }, delay);
-          const onAbort = () => {
-            clearTimeout(t);
-            rej(new Error("retry cancelled: the step was cancelled"));
-          };
-          signal.addEventListener("abort", onAbort, { once: true });
+        log('retrying', {
+          attempt,
+          of: opts.attempts,
+          error: (e as Error)?.message ?? String(e),
         });
+        await sleep(delay, signal);
         delay *= opts.backoff ?? 1;
       }
     }
@@ -96,16 +143,30 @@ export function createRetry(signal: AbortSignal, log: (m: string, f?: Record<str
 export function createCtx(o: CtxOptions): Ctx {
   mkdirSync(o.paths.tmp, { recursive: true });
   mkdirSync(o.paths.workFiles, { recursive: true });
-  const sink = o.logSink ?? ((line: string) => process.stderr.write(line + "\n"));
-  const log = (message: string, fields?: Record<string, unknown>) => sink(formatLog(message, fields));
-  const { sh, exec } = createSh({ tmpDir: o.paths.tmp, signal: o.signal });
-  const filesDeps = { run: o.run, workFiles: o.paths.workFiles, tmpDir: o.paths.tmp, root: o.paths.root, signal: o.signal, log };
+  const sink = o.logSink ?? ((line: string) => process.stderr.write(line + '\n'));
+  const log = (message: string, fields?: Record<string, unknown>) =>
+    sink(formatLog(message, fields));
+  const { sh, exec } = createSh({
+    tmpDir: o.paths.tmp,
+    signal: o.signal,
+    pipeOutput: o.pipeOutput,
+  });
+  const filesDeps = {
+    run: o.run,
+    workFiles: o.paths.workFiles,
+    tmpDir: o.paths.tmp,
+    root: o.paths.root,
+    signal: o.signal,
+    log,
+  };
   return {
-    secrets: createSecrets(o.secrets, o.secretValues) as Ctx["secrets"],
+    ...(o.cell ? { cell: o.cell } : {}),
+    secrets: createSecrets(o.secrets, o.secretValues) as Ctx['secrets'],
+    config: Object.freeze({ ...o.config }) as Ctx['config'],
     sh,
     exec,
-    readText: async (p) => readFileSync(resolve(p), "utf8"),
-    readJson: async <T>(p: string) => JSON.parse(readFileSync(resolve(p), "utf8")) as T,
+    readText: async (p) => readFileSync(resolve(p), 'utf8'),
+    readJson: async <T>(p: string) => JSON.parse(readFileSync(resolve(p), 'utf8')) as T,
     files: { fetch: createFetch(filesDeps) },
     output: createOutput(filesDeps),
     http: createHttp(o.signal),
@@ -115,5 +176,15 @@ export function createCtx(o: CtxOptions): Ctx {
     run: Object.freeze({ ...o.run }),
     tmpDir: o.paths.tmp,
     skip: skipped,
+    link: createLinks().link,
+    inputUrl: (step, input) => {
+      const name = typeof step === 'string' ? step : step.name;
+      const url = o.callbackUrls?.[`${name}/${input}`];
+      if (!url)
+        throw new Error(
+          `no callback URL for input '${input}' of step '${name}': it must list callback() in from, and the instance needs a public base URL`,
+        );
+      return url;
+    },
   };
 }

package/src/ctx/links.ts

@@ -0,0 +1,30 @@
+export interface StepLink {
+  url: string;
+  label?: string;
+}
+
+export const MAX_LINKS = 10;
+export const MAX_LINK_URL = 2048;
+export const MAX_LINK_LABEL = 100;
+
+/** `ctx.link`, collecting into `links`. */
+export function createLinks(): {
+  links: StepLink[];
+  link: (url: string, label?: string) => void;
+} {
+  const links: StepLink[] = [];
+  const link = (url: string, label?: string) => {
+    if (typeof url !== 'string' || !/^https?:\/\/[^/\s]/i.test(url) || !URL.canParse(url))
+      throw new Error(`ctx.link: '${String(url)}' is not an absolute http or https URL`);
+    if (url.length > MAX_LINK_URL)
+      throw new Error(`ctx.link: the URL is longer than ${MAX_LINK_URL} characters`);
+    if (label !== undefined && typeof label !== 'string')
+      throw new Error('ctx.link: the label must be a string');
+    if (label !== undefined && label.length > MAX_LINK_LABEL)
+      throw new Error(`ctx.link: the label is longer than ${MAX_LINK_LABEL} characters`);
+    if (links.length >= MAX_LINKS)
+      throw new Error(`ctx.link: a step can add at most ${MAX_LINKS} links`);
+    links.push(label ? { url, label } : { url });
+  };
+  return { links, link };
+}