@olivervea/pipelines 2.0.0-beta.31 → 2.0.0-beta.32

package/README.md

@@ -333,6 +333,7 @@ This section is everything the C# side needs in order to run the binary.
 | `OLVE_RUN` | run | Run number (integer ≥ 0). Written into produced file refs. Default 0. |
 | `OLVE_ATTEMPT` | run | Attempt number (integer ≥ 1). Written into produced file refs. Default 1. |
 | `OLVE_VERSION_STAMP` | run | Derived version, e.g. `2026.10.21.412` → `ctx.run.version`. |
+| `OLVE_SECRETS` | run | The step's declared secret names, comma-separated: the values the binary masks in its output (above). Unset or empty: no masking. |
 | `OLVE_CELL` | run | Only for a matrix step: the cell, as compact JSON in axis order (`{"os":"linux","arch":"x64"}`) → `ctx.cell`. Missing, or not one of the step's cells → exit 65. `ctx.run.step` and produced file refs carry the cell's name, `build-cli[os=linux,arch=x64]`. |
 | `OLVE_CASES` | run | Only for a step inside a `select`: a JSON array of the case conditions it sits in (`["target.prod.kind=ssh"]`); `run <step>` uses it to pick the variant of a name several cases use. |
 | `<SECRET_NAME>` | per step | Each declared secret as an env var of the **same name**: `secretKeyRef` key = name. Mount only the step's `secrets` (for poll triggers, the poll's `secrets`). Child processes inherit them. The mount is optional: an unset secret leaves the variable unset and `ctx.secrets.<NAME>` throws on access, so a handler can check `process.env.<NAME>` first and skip. |
@@ -344,7 +345,9 @@ This section is everything the C# side needs in order to run the binary.
 
 `OLVE_RESULT_URL`, `OLVE_FILE_URLS`, `OLVE_FILES_TOKEN` and `OLVE_CALLBACK_URLS` are removed from the environment of child processes. URLs are never logged past `?`.
 
-**Secret masking** (`run`): every value of the step's declared secrets is replaced with `***` in what the step writes: `ctx.log`, `console`, anything written to stdout or stderr, the output of `ctx.exec` and `ctx.sh` commands (piped through the harness when the step has a secret; inherited otherwise), and the envelope's failure message or skip reason. Output is masked a line at a time; a multi-line value is masked line by line, and values (or lines) shorter than 4 characters are left alone. Outputs and files are data and are not masked. A process that writes to the pod's stdout around the harness (a file descriptor it opened itself) is not masked.
+**Secret masking** (`run`): with `OLVE_SECRETS` set, the binary runs the step as its own child and replaces every value of those secrets with `***` in the child's stdout and stderr, a line at a time, from the pipeline's first import to its exit. That covers every way of printing: logs, `console`, direct writes, commands and the processes they start. The envelope's failure message, skip reason and links are masked too; outputs and files are data and aren't. A multi-line value is masked line by line, and values (or lines) shorter than 4 characters are left alone. Not masked: a value the step transforms before printing (escaped, encoded, cut short) and a process writing to the pod's stdout directly (`/proc/1/fd/1`).
+
+An error nothing catches (a rejected promise, a throw in a timer) fails the step.
 
 `ctx.files.fetch` looks for a file in this order:
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.31",
+  "version": "2.0.0-beta.32",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/build/build.ts

@@ -61,8 +61,11 @@ export function generateEntry(
   writeFileSync(
     entry,
     `// GENERATED by olve-build — do not edit. Entry point of the step binary.\n` +
-      `import pipeline from ${JSON.stringify(relPosix(gen, project.pipelineFile))};\n` +
-      `import { runHarness } from "@olivervea/pipelines/harness";\n\n` +
+      `import { runHarness, supervise } from "@olivervea/pipelines/harness";\n\n` +
+      `// Secret masking starts before the pipeline's code runs: the masked child imports it.\n` +
+      `const supervised = await supervise(process.argv.slice(2));\n` +
+      `if (supervised !== undefined) process.exit(supervised);\n` +
+      `const { default: pipeline } = await import(${JSON.stringify(relPosix(gen, project.pipelineFile))});\n` +
       `process.exit(await runHarness(${config}, process.argv.slice(2)));\n`,
   );
   return entry;

package/src/ctx/index.ts

@@ -24,8 +24,6 @@ export interface CtxOptions {
   callbackUrls?: Readonly<Record<string, string>>;
   /** A matrix step's cell (`ctx.cell`). */
   cell?: Cell;
-  /** Commands' output goes through this process (the harness masks secrets there). */
-  pipeOutput?: boolean;
 }
 
 export function formatLog(message: string, fields?: Record<string, unknown>): string {
@@ -146,11 +144,7 @@ export function createCtx(o: CtxOptions): Ctx {
   const sink = o.logSink ?? ((line: string) => process.stderr.write(line + '\n'));
   const log = (message: string, fields?: Record<string, unknown>) =>
     sink(formatLog(message, fields));
-  const { sh, exec } = createSh({
-    tmpDir: o.paths.tmp,
-    signal: o.signal,
-    pipeOutput: o.pipeOutput,
-  });
+  const { sh, exec } = createSh({ tmpDir: o.paths.tmp, signal: o.signal });
   const filesDeps = {
     run: o.run,
     workFiles: o.paths.workFiles,

package/src/ctx/sh.ts

@@ -79,8 +79,6 @@ export function childEnv(extra?: Record<string, string>): Record<string, string>
 export interface ShDeps {
   tmpDir: string;
   signal: AbortSignal;
-  /** Pipe commands' output through this process's stdout/stderr (where secrets are masked). */
-  pipeOutput?: boolean;
   shellCandidates?: readonly string[];
 }
 
@@ -152,8 +150,8 @@ export function createSh(deps: ShDeps): {
         cwd: opts.cwd,
         env: childEnv(opts.env),
         stdin: 'ignore',
-        stdout: capture || idleSeconds || deps.pipeOutput ? 'pipe' : 'inherit',
-        stderr: idleSeconds || deps.pipeOutput ? 'pipe' : 'inherit',
+        stdout: capture || idleSeconds ? 'pipe' : 'inherit',
+        stderr: idleSeconds ? 'pipe' : 'inherit',
       });
     } catch (e) {
       throw new Error(`cannot start ${what} (${argv[0]}): ${(e as Error).message}`);
@@ -198,11 +196,11 @@ export function createSh(deps: ShDeps): {
       pumps.push(
         pump(proc.stdout as ReadableStream<Uint8Array>, (c) => captured.push(c)),
       );
-    else if (idleSeconds || deps.pipeOutput)
+    else if (idleSeconds)
       pumps.push(
         pump(proc.stdout as ReadableStream<Uint8Array>, (c) => process.stdout.write(c)),
       );
-    if (idleSeconds || deps.pipeOutput)
+    if (idleSeconds)
       pumps.push(
         pump(proc.stderr as ReadableStream<Uint8Array>, (c) => process.stderr.write(c)),
       );

package/src/harness/index.ts

@@ -7,6 +7,7 @@ export {
   type HarnessIO,
 } from './main.ts';
 export { EXIT } from './exit.ts';
+export { supervise } from './supervise.ts';
 export { ENV, paths, runInfoFromEnv } from './env.ts';
 export {
   executeStep,

package/src/harness/main.ts

@@ -8,7 +8,7 @@ import { isImplementation } from '../core/handlers.ts';
 import { configRefPath, isConfigRef, unionAt } from '../core/config.ts';
 import type { AnyStep } from '../core/steps.ts';
 import { cellName, parseCell } from '../core/matrix.ts';
-import { installMasking, maskEnvelope, maskPatterns, maskText } from './mask.ts';
+import { maskEnvelope, maskPatterns } from './mask.ts';
 import type { WebhookEvent } from '../core/webhooks.ts';
 import { pipelineData, type PipelineData, type StageNode } from '../builder/pipeline.ts';
 import { createCtx, formatLog } from '../ctx/index.ts';
@@ -200,11 +200,11 @@ async function runMode(
     : undefined;
   const cancel = cancellation();
   const masks = maskPatterns(step.secrets.map((s) => process.env[s.name]));
-  const unmask = masks.length > 0 ? installMasking(masks) : () => {};
-  if (masks.length > 0) {
-    const stderr = io.stderr;
-    io = { ...io, stderr: (line) => stderr(maskText(line, masks)) };
-  }
+  // An error nothing caught (a rejected promise, a throw in a timer) fails the step.
+  let uncaught: unknown;
+  const onUncaught = (e: unknown) => void (uncaught ??= e);
+  process.on('unhandledRejection', onUncaught);
+  process.on('uncaughtException', onUncaught);
   try {
     let exec: Execution;
     if (!raw.ok || (cell && !cell.ok)) {
@@ -224,7 +224,6 @@ async function runMode(
         logSink: io.stderr,
         callbackUrls: callbackUrlsFromEnv(),
         ...(cell?.ok ? { cell: cell.cell } : {}),
-        pipeOutput: masks.length > 0,
       });
       exec = await executeStep({
         step,
@@ -235,10 +234,18 @@ async function runMode(
         log: io.stderr,
       });
     }
-    unmask();
+    if (uncaught !== undefined && exec.envelope.status !== 'failed') {
+      const message = `step '${step.name}' failed: an error nothing caught: ${uncaught instanceof Error ? uncaught.message : String(uncaught)}`;
+      io.stderr(message);
+      exec = {
+        envelope: { status: 'failed', error: { kind: 'handler', message } },
+        exitCode: EXIT.handlerFailed,
+      };
+    }
     return await finish({ ...exec, envelope: maskEnvelope(exec.envelope, masks) }, io);
   } finally {
-    unmask();
+    process.off('unhandledRejection', onUncaught);
+    process.off('uncaughtException', onUncaught);
     cancel.dispose();
   }
 }

package/src/harness/mask.ts

@@ -1,10 +1,8 @@
-import { format } from 'node:util';
 import type { Envelope } from './execute.ts';
 
 /**
- * Masks the step's declared secret values in what it writes to stdout and stderr. Output is masked a
- * line at a time, so a value split across two writes is still found; a multi-line value (a key file)
- * is masked line by line.
+ * Masks secret values in text: a stream a line at a time, so a value split across two writes is
+ * still found; a multi-line value (a key file) line by line.
  */
 
 export const MASK = '***';
@@ -17,7 +15,7 @@ const MAX_PENDING = 64 * 1024;
 export function maskPatterns(values: readonly (string | undefined)[]): string[] {
   const patterns = new Set<string>();
   for (const value of values)
-    for (const line of (value ?? '').split(/\r?\n/))
+    for (const line of (value ?? '').split(/\r\n|\r|\n/))
       if (line.trim().length >= MIN_MASKED_LENGTH) patterns.add(line.trim());
   return [...patterns].sort((a, b) => b.length - a.length);
 }
@@ -41,7 +39,12 @@ export function lineMasker(patterns: readonly string[], write: (text: string) =>
         write(maskText(pending.slice(0, end + 1), patterns));
         pending = pending.slice(end + 1);
       }
-      if (pending.length > MAX_PENDING) this.flush();
+      if (pending.length > MAX_PENDING) {
+        // Keep enough back that a value cut here is whole in what follows.
+        const keep = Math.max(0, (patterns[0]?.length ?? 1) - 1);
+        write(maskText(pending.slice(0, pending.length - keep), patterns));
+        pending = pending.slice(pending.length - keep);
+      }
     },
     flush() {
       pending += decoder.decode();
@@ -53,6 +56,15 @@ export function lineMasker(patterns: readonly string[], write: (text: string) =>
 
 /** The envelope with its failure message or skip reason masked; outputs are data and stay as they are. */
 export function maskEnvelope(envelope: Envelope, patterns: readonly string[]): Envelope {
+  if (envelope.links)
+    envelope = {
+      ...envelope,
+      links: envelope.links.map((l) => ({
+        ...l,
+        url: maskText(l.url, patterns),
+        ...(l.label !== undefined ? { label: maskText(l.label, patterns) } : {}),
+      })),
+    };
   if (envelope.status === 'failed')
     return {
       ...envelope,
@@ -62,37 +74,3 @@ export function maskEnvelope(envelope: Envelope, patterns: readonly string[]): E
     return { ...envelope, reason: maskText(envelope.reason, patterns) };
   return envelope;
 }
-
-type Write = typeof process.stdout.write;
-
-/**
- * Routes process.stdout and process.stderr through maskers until the returned function is called,
- * which flushes them and restores the streams.
- */
-export function installMasking(patterns: readonly string[]): () => void {
-  const streams = [process.stdout, process.stderr];
-  const originals = streams.map((s) => s.write.bind(s) as Write);
-  const maskers = originals.map((original) =>
-    lineMasker(patterns, (text) => void original(text)),
-  );
-  streams.forEach((s, i) => {
-    s.write = ((chunk: string | Uint8Array, a?: unknown, b?: unknown) => {
-      maskers[i]!.write(chunk);
-      const callback = typeof a === 'function' ? a : typeof b === 'function' ? b : null;
-      if (callback) queueMicrotask(() => (callback as () => void)());
-      return true;
-    }) as Write;
-  });
-  // Bun's console writes natively, past process.stdout.write.
-  const methods = ['log', 'info', 'debug', 'warn', 'error'] as const;
-  const consoleOriginals = methods.map((m) => console[m]);
-  methods.forEach((m) => {
-    const stream = m === 'warn' || m === 'error' ? process.stderr : process.stdout;
-    console[m] = (...args: unknown[]) => void stream.write(format(...args) + '\n');
-  });
-  return () => {
-    maskers.forEach((m) => m.flush());
-    streams.forEach((s, i) => (s.write = originals[i]!));
-    methods.forEach((m, i) => (console[m] = consoleOriginals[i]!));
-  };
-}

package/src/harness/supervise.ts

@@ -0,0 +1,63 @@
+/**
+ * Secret masking for `run`: when the service names the step's secrets (`OLVE_SECRETS`), the binary
+ * runs itself as a child and masks the child's stdout and stderr bytes, so every way of printing is
+ * covered, from the pipeline's first import to its last timer. It runs before the pipeline module
+ * is imported.
+ */
+import { readFileSync } from 'node:fs';
+import { constants } from 'node:os';
+import { lineMasker, maskPatterns } from './mask.ts';
+
+const SUPERVISED = 'OLVE_SUPERVISED';
+const FORWARDED = ['SIGTERM', 'SIGINT', 'SIGHUP'] as const;
+
+/** The secret values this run must not print, from `OLVE_SECRETS` (names, comma-separated). */
+export function secretPatterns(env: NodeJS.ProcessEnv = process.env): string[] {
+  const names = (env.OLVE_SECRETS ?? '').split(',').filter(Boolean);
+  return maskPatterns(names.map((n) => env[n]));
+}
+
+/** This process's own command line, loader included (`ld-linux … /olve/step run x` in a pod). */
+function ownArgv(): string[] {
+  try {
+    const argv = readFileSync('/proc/self/cmdline', 'utf8').split('\0').filter(Boolean);
+    if (argv.length > 0) return argv;
+  } catch {
+    // not Linux
+  }
+  return [process.execPath, ...process.argv.slice(1)];
+}
+
+/**
+ * Runs the step as a masked child and answers its exit code, or undefined when this process should
+ * run the step itself: it is the child, the mode isn't `run`, or there is nothing to mask.
+ */
+export async function supervise(argv: readonly string[]): Promise<number | undefined> {
+  if (argv[0] !== 'run' || process.env[SUPERVISED]) return undefined;
+  const patterns = secretPatterns();
+  if (patterns.length === 0) return undefined;
+
+  const child = Bun.spawn(ownArgv(), {
+    env: { ...process.env, [SUPERVISED]: '1' },
+    stdin: 'inherit',
+    stdout: 'pipe',
+    stderr: 'pipe',
+  });
+  const forward = (signal: NodeJS.Signals) => () => child.kill(signal);
+  for (const s of FORWARDED) process.on(s, forward(s));
+
+  const pump = async (stream: ReadableStream<Uint8Array>, fd: 1 | 2) => {
+    const out = fd === 1 ? process.stdout : process.stderr;
+    const masker = lineMasker(patterns, (text) => void out.write(text));
+    for await (const chunk of stream) masker.write(chunk);
+    masker.flush();
+  };
+  const pumps = Promise.all([pump(child.stdout, 1), pump(child.stderr, 2)]);
+  await child.exited;
+  // A detached grandchild may hold the pipes open: don't wait for it long.
+  await Promise.race([pumps, Bun.sleep(5000)]);
+  const signal = child.signalCode
+    ? constants.signals[child.signalCode as NodeJS.Signals]
+    : undefined;
+  return child.exitCode ?? (signal ? 128 + signal : 1);
+}