@olivervea/pipelines 2.0.0-beta.2 → 2.0.0-beta.20

package/src/ctx/index.ts

@@ -1,10 +1,10 @@
-import { mkdirSync, readFileSync } from "node:fs";
-import { resolve } from "node:path";
-import { skipped } from "../core/handlers.ts";
-import type { Secret } from "../core/secrets.ts";
-import { createFetch, createOutput, redactUrl } from "./files.ts";
-import { createSh } from "./sh.ts";
-import type { Ctx, Http, RetryOptions, RunInfo } from "./types.ts";
+import { mkdirSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { skipped } from '../core/handlers.ts';
+import type { Secret } from '../core/secrets.ts';
+import { createFetch, createOutput, redactUrl } from './files.ts';
+import { createSh } from './sh.ts';
+import type { Ctx, Http, RetryOptions, RunInfo } from './types.ts';
 
 export interface CtxOptions {
   run: RunInfo;
@@ -12,25 +12,38 @@ export interface CtxOptions {
   secrets: readonly Secret[];
   /** Secret values that take precedence over the environment (tests). */
   secretValues?: Readonly<Record<string, string>>;
+  /** Parsed config entries of the step (`ctx.config`). */
+  config?: Readonly<Record<string, unknown>>;
   paths: { root: string; workFiles: string; tmp: string };
   signal: AbortSignal;
   /** Where log lines go (default stderr). */
   logSink?: (line: string) => void;
+  /** `<step>/<input>` → callback URL (`ctx.inputUrl`). */
+  callbackUrls?: Readonly<Record<string, string>>;
 }
 
 export function formatLog(message: string, fields?: Record<string, unknown>): string {
-  const parts = Object.entries(fields ?? {}).map(([k, v]) => `${k}=${typeof v === "string" && !/\s|"/.test(v) ? v : JSON.stringify(v)}`);
-  return parts.length ? `${message} ${parts.join(" ")}` : message;
+  const parts = Object.entries(fields ?? {}).map(
+    ([k, v]) =>
+      `${k}=${typeof v === 'string' && !/\s|"/.test(v) ? v : JSON.stringify(v)}`,
+  );
+  return parts.length ? `${message} ${parts.join(' ')}` : message;
 }
 
-export function createSecrets(declared: readonly Secret[], values?: Readonly<Record<string, string>>): Record<string, string> {
+export function createSecrets(
+  declared: readonly Secret[],
+  values?: Readonly<Record<string, string>>,
+): Record<string, string> {
   const obj: Record<string, string> = {};
   for (const s of declared) {
     Object.defineProperty(obj, s.name, {
       enumerable: true,
       get() {
         const v = values?.[s.name] ?? process.env[s.name];
-        if (v === undefined) throw new Error(`secret ${s.name} is not set (the harness reads it from the environment variable ${s.name})`);
+        if (v === undefined)
+          throw new Error(
+            `secret ${s.name} is not set (the harness reads it from the environment variable ${s.name})`,
+          );
         return v;
       },
     });
@@ -39,52 +52,74 @@ export function createSecrets(declared: readonly Secret[], values?: Readonly<Rec
 }
 
 async function httpError(res: Response, url: string): Promise<Error> {
-  let body = "";
+  let body = '';
   try {
     body = (await res.text()).slice(0, 500);
   } catch {
     /* ignore */
   }
-  return new Error(`HTTP ${res.status} ${res.statusText} from ${redactUrl(url)}${body ? `: ${body}` : ""}`);
+  return new Error(
+    `HTTP ${res.status} ${res.statusText} from ${redactUrl(url)}${body ? `: ${body}` : ''}`,
+  );
 }
 
 export function createHttp(signal: AbortSignal): Http {
-  const f = (url: string, init?: RequestInit) => fetch(url, { ...init, signal: init?.signal ?? signal });
+  const f = (url: string, init?: RequestInit) =>
+    fetch(url, { ...init, signal: init?.signal ?? signal });
   return {
     fetch: f,
-    async getJson<T>(url: string, init?: { headers?: Record<string, string> }): Promise<T> {
-      const res = await f(url, { headers: { accept: "application/json", ...init?.headers } });
+    async getJson<T>(
+      url: string,
+      init?: { headers?: Record<string, string> },
+    ): Promise<T> {
+      const res = await f(url, {
+        headers: { accept: 'application/json', ...init?.headers },
+      });
       if (!res.ok) throw await httpError(res, url);
       return (await res.json()) as T;
     },
-    async expectOk(url: string, init?: { headers?: Record<string, string>; method?: string }): Promise<Response> {
-      const res = await f(url, { method: init?.method ?? "GET", headers: init?.headers });
+    async expectOk(
+      url: string,
+      init?: { headers?: Record<string, string>; method?: string },
+    ): Promise<Response> {
+      const res = await f(url, { method: init?.method ?? 'GET', headers: init?.headers });
       if (!res.ok) throw await httpError(res, url);
       return res;
     },
   };
 }
 
-export function createRetry(signal: AbortSignal, log: (m: string, f?: Record<string, unknown>) => void) {
-  return async <T>(opts: RetryOptions, fn: (attempt: number) => Promise<T>): Promise<T> => {
-    if (!(Number.isInteger(opts.attempts) && opts.attempts >= 1)) throw new Error("retry: attempts must be a positive integer");
+export function createRetry(
+  signal: AbortSignal,
+  log: (m: string, f?: Record<string, unknown>) => void,
+) {
+  return async <T>(
+    opts: RetryOptions,
+    fn: (attempt: number) => Promise<T>,
+  ): Promise<T> => {
+    if (!(Number.isInteger(opts.attempts) && opts.attempts >= 1))
+      throw new Error('retry: attempts must be a positive integer');
     let delay = Math.max(0, opts.delaySeconds) * 1000;
     for (let attempt = 1; ; attempt++) {
       try {
         return await fn(attempt);
       } catch (e) {
         if (attempt >= opts.attempts || signal.aborted) throw e;
-        log("retrying", { attempt, of: opts.attempts, error: (e as Error)?.message ?? String(e) });
+        log('retrying', {
+          attempt,
+          of: opts.attempts,
+          error: (e as Error)?.message ?? String(e),
+        });
         await new Promise<void>((res, rej) => {
           const t = setTimeout(() => {
-            signal.removeEventListener("abort", onAbort);
+            signal.removeEventListener('abort', onAbort);
             res();
           }, delay);
           const onAbort = () => {
             clearTimeout(t);
-            rej(new Error("retry cancelled: the step was cancelled"));
+            rej(new Error('retry cancelled: the step was cancelled'));
           };
-          signal.addEventListener("abort", onAbort, { once: true });
+          signal.addEventListener('abort', onAbort, { once: true });
         });
         delay *= opts.backoff ?? 1;
       }
@@ -96,16 +131,25 @@ export function createRetry(signal: AbortSignal, log: (m: string, f?: Record<str
 export function createCtx(o: CtxOptions): Ctx {
   mkdirSync(o.paths.tmp, { recursive: true });
   mkdirSync(o.paths.workFiles, { recursive: true });
-  const sink = o.logSink ?? ((line: string) => process.stderr.write(line + "\n"));
-  const log = (message: string, fields?: Record<string, unknown>) => sink(formatLog(message, fields));
+  const sink = o.logSink ?? ((line: string) => process.stderr.write(line + '\n'));
+  const log = (message: string, fields?: Record<string, unknown>) =>
+    sink(formatLog(message, fields));
   const { sh, exec } = createSh({ tmpDir: o.paths.tmp, signal: o.signal });
-  const filesDeps = { run: o.run, workFiles: o.paths.workFiles, tmpDir: o.paths.tmp, root: o.paths.root, signal: o.signal, log };
+  const filesDeps = {
+    run: o.run,
+    workFiles: o.paths.workFiles,
+    tmpDir: o.paths.tmp,
+    root: o.paths.root,
+    signal: o.signal,
+    log,
+  };
   return {
-    secrets: createSecrets(o.secrets, o.secretValues) as Ctx["secrets"],
+    secrets: createSecrets(o.secrets, o.secretValues) as Ctx['secrets'],
+    config: Object.freeze({ ...o.config }) as Ctx['config'],
     sh,
     exec,
-    readText: async (p) => readFileSync(resolve(p), "utf8"),
-    readJson: async <T>(p: string) => JSON.parse(readFileSync(resolve(p), "utf8")) as T,
+    readText: async (p) => readFileSync(resolve(p), 'utf8'),
+    readJson: async <T>(p: string) => JSON.parse(readFileSync(resolve(p), 'utf8')) as T,
     files: { fetch: createFetch(filesDeps) },
     output: createOutput(filesDeps),
     http: createHttp(o.signal),
@@ -115,5 +159,14 @@ export function createCtx(o: CtxOptions): Ctx {
     run: Object.freeze({ ...o.run }),
     tmpDir: o.paths.tmp,
     skip: skipped,
+    inputUrl: (step, input) => {
+      const name = typeof step === 'string' ? step : step.name;
+      const url = o.callbackUrls?.[`${name}/${input}`];
+      if (!url)
+        throw new Error(
+          `no callback URL for input '${input}' of step '${name}': it must list callback() in from, and the instance needs a public base URL`,
+        );
+      return url;
+    },
   };
 }

package/src/ctx/sh.ts

@@ -1,32 +1,49 @@
-import { chmodSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
-import { basename, isAbsolute, join } from "node:path";
-import type { ExecOptions, Sh, ShOptions } from "./types.ts";
-import { PRIVATE_ENV } from "../harness/env.ts";
+import {
+  chmodSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import { basename, isAbsolute, join } from 'node:path';
+import type { ExecOptions, Sh, ShOptions } from './types.ts';
+import { PRIVATE_ENV } from '../harness/env.ts';
 
 /** Shells looked for, in order. kaniko only has /busybox/sh; distroless has none. */
-export const SHELL_CANDIDATES = ["/bin/sh", "/busybox/sh", "/bin/bash"] as const;
+export const SHELL_CANDIDATES = ['/bin/sh', '/busybox/sh', '/bin/bash'] as const;
 
 export class ShellNotFoundError extends Error {
-  override name = "ShellNotFoundError";
+  override name = 'ShellNotFoundError';
   constructor() {
     super(
-      `no shell found in this image (looked for ${SHELL_CANDIDATES.join(", ")}). ` +
+      `no shell found in this image (looked for ${SHELL_CANDIDATES.join(', ')}). ` +
         `ctx.sh needs a shell: use an image that has one, or run binaries directly with ctx.exec([...]).`,
     );
   }
 }
 
 export class CommandError extends Error {
-  override name = "CommandError";
-  constructor(readonly what: string, readonly exitCode: number | null, readonly signalCode: string | null) {
-    super(`${what} failed with ${signalCode ? `signal ${signalCode}` : `exit code ${exitCode}`}`);
+  override name = 'CommandError';
+  constructor(
+    readonly what: string,
+    readonly exitCode: number | null,
+    readonly signalCode: string | null,
+  ) {
+    super(
+      `${what} failed with ${signalCode ? `signal ${signalCode}` : `exit code ${exitCode}`}`,
+    );
   }
 }
 
 /** The command went quiet for longer than its `idleTimeoutSeconds` and was killed. */
 export class CommandStalledError extends CommandError {
-  override name = "CommandStalledError";
-  constructor(what: string, readonly idleTimeoutSeconds: number, signalCode: string | null) {
+  override name = 'CommandStalledError';
+  constructor(
+    what: string,
+    readonly idleTimeoutSeconds: number,
+    signalCode: string | null,
+  ) {
     super(what, null, signalCode);
     this.message = `${what} produced no output for ${idleTimeoutSeconds} s and was killed`;
   }
@@ -42,7 +59,7 @@ export function detectShell(candidates: readonly string[] = SHELL_CANDIDATES): s
 
 /** True for the value of `import x from "./x.sh" with { type: "file" }` (compiled or from source). */
 export function isEmbeddedScript(s: string): boolean {
-  if (s.startsWith("/$bunfs/") || s.startsWith("B:/~BUN/")) return true;
+  if (s.startsWith('/$bunfs/') || s.startsWith('B:/~BUN/')) return true;
   if (!isAbsolute(s) || !/\.(sh|bash)$/.test(s) || /\s/.test(s)) return false;
   try {
     return statSync(s).isFile();
@@ -54,7 +71,8 @@ export function isEmbeddedScript(s: string): boolean {
 /** Environment for child processes: the harness env minus harness-only credentials, plus extras. */
 export function childEnv(extra?: Record<string, string>): Record<string, string> {
   const env: Record<string, string> = {};
-  for (const [k, v] of Object.entries(process.env)) if (v !== undefined && !PRIVATE_ENV.includes(k)) env[k] = v;
+  for (const [k, v] of Object.entries(process.env))
+    if (v !== undefined && !PRIVATE_ENV.includes(k)) env[k] = v;
   return { ...env, ...extra };
 }
 
@@ -64,7 +82,10 @@ export interface ShDeps {
   shellCandidates?: readonly string[];
 }
 
-export function createSh(deps: ShDeps): { sh: Sh; exec: (argv: readonly string[], opts?: ExecOptions) => Promise<void> } {
+export function createSh(deps: ShDeps): {
+  sh: Sh;
+  exec: (argv: readonly string[], opts?: ExecOptions) => Promise<void>;
+} {
   const copied = new Map<string, string>();
   let counter = 0;
 
@@ -72,7 +93,7 @@ export function createSh(deps: ShDeps): { sh: Sh; exec: (argv: readonly string[]
   const materialize = (script: string): string => {
     const hit = copied.get(script);
     if (hit) return hit;
-    const dir = join(deps.tmpDir, "scripts");
+    const dir = join(deps.tmpDir, 'scripts');
     mkdirSync(dir, { recursive: true });
     const dest = join(dir, `${++counter}-${basename(script)}`);
     writeFileSync(dest, readFileSync(script));
@@ -81,20 +102,34 @@ export function createSh(deps: ShDeps): { sh: Sh; exec: (argv: readonly string[]
     return dest;
   };
 
-  const argvFor = (cmd: string | { script: string }, args: readonly string[]): { argv: string[]; what: string } => {
-    const scriptPath = typeof cmd === "object" ? cmd.script : isEmbeddedScript(cmd) ? cmd : undefined;
+  const argvFor = (
+    cmd: string | { script: string },
+    args: readonly string[],
+  ): { argv: string[]; what: string } => {
+    const scriptPath =
+      typeof cmd === 'object' ? cmd.script : isEmbeddedScript(cmd) ? cmd : undefined;
     if (scriptPath === undefined) {
       const shell = detectShell(deps.shellCandidates);
-      return { argv: [shell, "-c", cmd as string], what: `command \`${truncate(cmd as string)}\`` };
+      // `sh -c cmd sh a b`: the args are $1, $2… of the command.
+      return {
+        argv: [shell, '-c', cmd as string, 'sh', ...args],
+        what: `command \`${truncate(cmd as string)}\``,
+      };
     }
     const file = materialize(scriptPath);
-    const first = readFileSync(file, "utf8").split("\n", 1)[0] ?? "";
+    const first = readFileSync(file, 'utf8').split('\n', 1)[0] ?? '';
     const shebang = /^#!\s*(\S+)(?:\s+(.*))?$/.exec(first);
     if (shebang && existsSync(shebang[1]!)) {
       const interpArgs = shebang[2] ? [shebang[2].trim()] : [];
-      return { argv: [shebang[1]!, ...interpArgs, file, ...args], what: `script ${basename(scriptPath)}` };
+      return {
+        argv: [shebang[1]!, ...interpArgs, file, ...args],
+        what: `script ${basename(scriptPath)}`,
+      };
     }
-    return { argv: [detectShell(deps.shellCandidates), file, ...args], what: `script ${basename(scriptPath)}` };
+    return {
+      argv: [detectShell(deps.shellCandidates), file, ...args],
+      what: `script ${basename(scriptPath)}`,
+    };
   };
 
   const run = async (
@@ -103,22 +138,26 @@ export function createSh(deps: ShDeps): { sh: Sh; exec: (argv: readonly string[]
     opts: { env?: Record<string, string>; cwd?: string; idleTimeoutSeconds?: number },
     capture: boolean,
   ): Promise<string> => {
-    if (deps.signal.aborted) throw new Error(`${what} not started: the step was cancelled`);
-    const idleSeconds = opts.idleTimeoutSeconds !== undefined && opts.idleTimeoutSeconds > 0 ? opts.idleTimeoutSeconds : undefined;
-    let proc: Bun.Subprocess<"ignore", "pipe" | "inherit", "pipe" | "inherit">;
+    if (deps.signal.aborted)
+      throw new Error(`${what} not started: the step was cancelled`);
+    const idleSeconds =
+      opts.idleTimeoutSeconds !== undefined && opts.idleTimeoutSeconds > 0
+        ? opts.idleTimeoutSeconds
+        : undefined;
+    let proc: Bun.Subprocess<'ignore', 'pipe' | 'inherit', 'pipe' | 'inherit'>;
     try {
       proc = Bun.spawn(argv, {
         cwd: opts.cwd,
         env: childEnv(opts.env),
-        stdin: "ignore",
-        stdout: capture || idleSeconds ? "pipe" : "inherit",
-        stderr: idleSeconds ? "pipe" : "inherit",
+        stdin: 'ignore',
+        stdout: capture || idleSeconds ? 'pipe' : 'inherit',
+        stderr: idleSeconds ? 'pipe' : 'inherit',
       });
     } catch (e) {
       throw new Error(`cannot start ${what} (${argv[0]}): ${(e as Error).message}`);
     }
-    const onAbort = () => proc.kill("SIGTERM");
-    deps.signal.addEventListener("abort", onAbort, { once: true });
+    const onAbort = () => proc.kill('SIGTERM');
+    deps.signal.addEventListener('abort', onAbort, { once: true });
 
     // Watchdog: every chunk of output re-arms the timer; on expiry TERM, then KILL after a grace.
     let stalled = false;
@@ -129,13 +168,16 @@ export function createSh(deps: ShDeps): { sh: Sh; exec: (argv: readonly string[]
       clearTimeout(idleTimer);
       idleTimer = setTimeout(() => {
         stalled = true;
-        proc.kill("SIGTERM");
-        killTimer = setTimeout(() => proc.kill("SIGKILL"), STALL_KILL_GRACE_MS);
+        proc.kill('SIGTERM');
+        killTimer = setTimeout(() => proc.kill('SIGKILL'), STALL_KILL_GRACE_MS);
       }, idleSeconds * 1000);
     };
     const captured: Uint8Array[] = [];
     const readers: { cancel(): Promise<void> }[] = [];
-    const pump = async (stream: ReadableStream<Uint8Array>, sink: (chunk: Uint8Array) => void) => {
+    const pump = async (
+      stream: ReadableStream<Uint8Array>,
+      sink: (chunk: Uint8Array) => void,
+    ) => {
       const reader = stream.getReader();
       readers.push(reader);
       try {
@@ -150,9 +192,18 @@ export function createSh(deps: ShDeps): { sh: Sh; exec: (argv: readonly string[]
       }
     };
     const pumps: Promise<void>[] = [];
-    if (capture) pumps.push(pump(proc.stdout as ReadableStream<Uint8Array>, (c) => captured.push(c)));
-    else if (idleSeconds) pumps.push(pump(proc.stdout as ReadableStream<Uint8Array>, (c) => process.stdout.write(c)));
-    if (idleSeconds) pumps.push(pump(proc.stderr as ReadableStream<Uint8Array>, (c) => process.stderr.write(c)));
+    if (capture)
+      pumps.push(
+        pump(proc.stdout as ReadableStream<Uint8Array>, (c) => captured.push(c)),
+      );
+    else if (idleSeconds)
+      pumps.push(
+        pump(proc.stdout as ReadableStream<Uint8Array>, (c) => process.stdout.write(c)),
+      );
+    if (idleSeconds)
+      pumps.push(
+        pump(proc.stderr as ReadableStream<Uint8Array>, (c) => process.stderr.write(c)),
+      );
     arm();
 
     try {
@@ -160,13 +211,23 @@ export function createSh(deps: ShDeps): { sh: Sh; exec: (argv: readonly string[]
       // A killed command's children may still hold the pipes open: don't wait for them after a stall.
       if (stalled) await Promise.all(readers.map((r) => r.cancel().catch(() => {})));
       else await Promise.all(pumps);
-      if (stalled) throw new CommandStalledError(what, idleSeconds!, proc.signalCode ? String(proc.signalCode) : null);
-      if (code !== 0) throw new CommandError(what, proc.signalCode ? null : code, proc.signalCode ? String(proc.signalCode) : null);
-      return capture ? Buffer.concat(captured).toString("utf8") : "";
+      if (stalled)
+        throw new CommandStalledError(
+          what,
+          idleSeconds!,
+          proc.signalCode ? String(proc.signalCode) : null,
+        );
+      if (code !== 0)
+        throw new CommandError(
+          what,
+          proc.signalCode ? null : code,
+          proc.signalCode ? String(proc.signalCode) : null,
+        );
+      return capture ? Buffer.concat(captured).toString('utf8') : '';
     } finally {
       clearTimeout(idleTimer);
       clearTimeout(killTimer);
-      deps.signal.removeEventListener("abort", onAbort);
+      deps.signal.removeEventListener('abort', onAbort);
     }
   };
 
@@ -184,13 +245,13 @@ export function createSh(deps: ShDeps): { sh: Sh; exec: (argv: readonly string[]
   ) as Sh;
 
   const exec = async (argv: readonly string[], opts: ExecOptions = {}) => {
-    if (argv.length === 0) throw new Error("ctx.exec needs at least the program");
-    await run([...argv], `\`${truncate(argv.join(" "))}\``, opts, false);
+    if (argv.length === 0) throw new Error('ctx.exec needs at least the program');
+    await run([...argv], `\`${truncate(argv.join(' '))}\``, opts, false);
   };
   return { sh, exec };
 }
 
 function truncate(s: string, n = 120): string {
-  const one = s.replace(/\s+/g, " ").trim();
-  return one.length > n ? one.slice(0, n - 1) + "…" : one;
+  const one = s.replace(/\s+/g, ' ').trim();
+  return one.length > n ? one.slice(0, n - 1) + '…' : one;
 }

package/src/ctx/types.ts

@@ -1,6 +1,7 @@
-import type { Skipped } from "../core/handlers.ts";
-import type { FileRef } from "../core/models.ts";
-import type { DeclaredSecrets } from "../core/secrets.ts";
+import type { Skipped } from '../core/handlers.ts';
+import type { FileRef } from '../core/models.ts';
+import type { AnyConfigEntry, ConfigValues } from '../core/config.ts';
+import type { DeclaredSecrets } from '../core/secrets.ts';
 
 /** Facts about the current execution, from the service (see README "Environment"). */
 export interface RunInfo {
@@ -79,9 +80,11 @@ export interface OutputFileOptions {
   name?: string;
 }
 
-export interface Ctx<Sec extends string = string> {
+export interface Ctx<Sec extends string = string, Cfg extends AnyConfigEntry = never> {
   /** Exactly the secrets the step declares, read from the environment. */
   readonly secrets: DeclaredSecrets<Sec>;
+  /** Exactly the config entries the step declares, from the run's config snapshot (defaults applied). */
+  readonly config: ConfigValues<Cfg>;
   readonly sh: Sh;
   /** Run a binary directly, without a shell (works in shell-less images). Rejects on non-zero exit. */
   exec(argv: readonly string[], opts?: ExecOptions): Promise<void>;
@@ -110,4 +113,9 @@ export interface Ctx<Sec extends string = string> {
   readonly tmpDir: string;
   /** Runtime escape hatch: produce no output. Prefer `when` blocks. */
   skip(reason: string): Skipped;
+  /**
+   * This run's callback URL for an external input with `from: [callback()]` (of any step of the pipeline):
+   * hand it to the system that will answer. POST the value as JSON, or `{"$decline": "reason"}`.
+   */
+  inputUrl(step: string | { readonly name: string }, input: string): string;
 }