@olivervea/pipelines 2.0.0-beta.30 → 2.0.0-beta.32

package/README.md

@@ -333,6 +333,7 @@ This section is everything the C# side needs in order to run the binary.
 | `OLVE_RUN` | run | Run number (integer ≥ 0). Written into produced file refs. Default 0. |
 | `OLVE_ATTEMPT` | run | Attempt number (integer ≥ 1). Written into produced file refs. Default 1. |
 | `OLVE_VERSION_STAMP` | run | Derived version, e.g. `2026.10.21.412` → `ctx.run.version`. |
+| `OLVE_SECRETS` | run | The step's declared secret names, comma-separated: the values the binary masks in its output (above). Unset or empty: no masking. |
 | `OLVE_CELL` | run | Only for a matrix step: the cell, as compact JSON in axis order (`{"os":"linux","arch":"x64"}`) → `ctx.cell`. Missing, or not one of the step's cells → exit 65. `ctx.run.step` and produced file refs carry the cell's name, `build-cli[os=linux,arch=x64]`. |
 | `OLVE_CASES` | run | Only for a step inside a `select`: a JSON array of the case conditions it sits in (`["target.prod.kind=ssh"]`); `run <step>` uses it to pick the variant of a name several cases use. |
 | `<SECRET_NAME>` | per step | Each declared secret as an env var of the **same name**: `secretKeyRef` key = name. Mount only the step's `secrets` (for poll triggers, the poll's `secrets`). Child processes inherit them. The mount is optional: an unset secret leaves the variable unset and `ctx.secrets.<NAME>` throws on access, so a handler can check `process.env.<NAME>` first and skip. |
@@ -344,6 +345,10 @@ This section is everything the C# side needs in order to run the binary.
 
 `OLVE_RESULT_URL`, `OLVE_FILE_URLS`, `OLVE_FILES_TOKEN` and `OLVE_CALLBACK_URLS` are removed from the environment of child processes. URLs are never logged past `?`.
 
+**Secret masking** (`run`): with `OLVE_SECRETS` set, the binary runs the step as its own child and replaces every value of those secrets with `***` in the child's stdout and stderr, a line at a time, from the pipeline's first import to its exit. That covers every way of printing: logs, `console`, direct writes, commands and the processes they start. The envelope's failure message, skip reason and links are masked too; outputs and files are data and aren't. A multi-line value is masked line by line, and values (or lines) shorter than 4 characters are left alone. Not masked: a value the step transforms before printing (escaped, encoded, cut short) and a process writing to the pod's stdout directly (`/proc/1/fd/1`).
+
+An error nothing catches (a rejected promise, a throw in a timer) fails the step.
+
 `ctx.files.fetch` looks for a file in this order:
 
 1. the attempt's own `work/files`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.30",
+  "version": "2.0.0-beta.32",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/build/build.ts

@@ -61,8 +61,11 @@ export function generateEntry(
   writeFileSync(
     entry,
     `// GENERATED by olve-build — do not edit. Entry point of the step binary.\n` +
-      `import pipeline from ${JSON.stringify(relPosix(gen, project.pipelineFile))};\n` +
-      `import { runHarness } from "@olivervea/pipelines/harness";\n\n` +
+      `import { runHarness, supervise } from "@olivervea/pipelines/harness";\n\n` +
+      `// Secret masking starts before the pipeline's code runs: the masked child imports it.\n` +
+      `const supervised = await supervise(process.argv.slice(2));\n` +
+      `if (supervised !== undefined) process.exit(supervised);\n` +
+      `const { default: pipeline } = await import(${JSON.stringify(relPosix(gen, project.pipelineFile))});\n` +
       `process.exit(await runHarness(${config}, process.argv.slice(2)));\n`,
   );
   return entry;

package/src/harness/index.ts

@@ -7,6 +7,7 @@ export {
   type HarnessIO,
 } from './main.ts';
 export { EXIT } from './exit.ts';
+export { supervise } from './supervise.ts';
 export { ENV, paths, runInfoFromEnv } from './env.ts';
 export {
   executeStep,

package/src/harness/main.ts

@@ -8,6 +8,7 @@ import { isImplementation } from '../core/handlers.ts';
 import { configRefPath, isConfigRef, unionAt } from '../core/config.ts';
 import type { AnyStep } from '../core/steps.ts';
 import { cellName, parseCell } from '../core/matrix.ts';
+import { maskEnvelope, maskPatterns } from './mask.ts';
 import type { WebhookEvent } from '../core/webhooks.ts';
 import { pipelineData, type PipelineData, type StageNode } from '../builder/pipeline.ts';
 import { createCtx, formatLog } from '../ctx/index.ts';
@@ -198,6 +199,12 @@ async function runMode(
     ? parseCell(step.name, step.matrix, process.env[ENV.cell])
     : undefined;
   const cancel = cancellation();
+  const masks = maskPatterns(step.secrets.map((s) => process.env[s.name]));
+  // An error nothing caught (a rejected promise, a throw in a timer) fails the step.
+  let uncaught: unknown;
+  const onUncaught = (e: unknown) => void (uncaught ??= e);
+  process.on('unhandledRejection', onUncaught);
+  process.on('uncaughtException', onUncaught);
   try {
     let exec: Execution;
     if (!raw.ok || (cell && !cell.ok)) {
@@ -227,8 +234,18 @@ async function runMode(
         log: io.stderr,
       });
     }
-    return await finish(exec, io);
+    if (uncaught !== undefined && exec.envelope.status !== 'failed') {
+      const message = `step '${step.name}' failed: an error nothing caught: ${uncaught instanceof Error ? uncaught.message : String(uncaught)}`;
+      io.stderr(message);
+      exec = {
+        envelope: { status: 'failed', error: { kind: 'handler', message } },
+        exitCode: EXIT.handlerFailed,
+      };
+    }
+    return await finish({ ...exec, envelope: maskEnvelope(exec.envelope, masks) }, io);
   } finally {
+    process.off('unhandledRejection', onUncaught);
+    process.off('uncaughtException', onUncaught);
     cancel.dispose();
   }
 }

package/src/harness/mask.ts

@@ -0,0 +1,76 @@
+import type { Envelope } from './execute.ts';
+
+/**
+ * Masks secret values in text: a stream a line at a time, so a value split across two writes is
+ * still found; a multi-line value (a key file) line by line.
+ */
+
+export const MASK = '***';
+/** Shorter values (and lines of multi-line values) are left alone: they would mask ordinary text. */
+export const MIN_MASKED_LENGTH = 4;
+/** A line longer than this is masked and written without waiting for its end. */
+const MAX_PENDING = 64 * 1024;
+
+/** The strings to mask for these secret values, longest first. */
+export function maskPatterns(values: readonly (string | undefined)[]): string[] {
+  const patterns = new Set<string>();
+  for (const value of values)
+    for (const line of (value ?? '').split(/\r\n|\r|\n/))
+      if (line.trim().length >= MIN_MASKED_LENGTH) patterns.add(line.trim());
+  return [...patterns].sort((a, b) => b.length - a.length);
+}
+
+export function maskText(text: string, patterns: readonly string[]): string {
+  let out = text;
+  for (const p of patterns) if (out.includes(p)) out = out.split(p).join(MASK);
+  return out;
+}
+
+/** A writer that masks complete lines and holds back the unfinished one until it ends or `flush`. */
+export function lineMasker(patterns: readonly string[], write: (text: string) => void) {
+  const decoder = new TextDecoder();
+  let pending = '';
+  return {
+    write(chunk: string | Uint8Array) {
+      pending +=
+        typeof chunk === 'string' ? chunk : decoder.decode(chunk, { stream: true });
+      const end = Math.max(pending.lastIndexOf('\n'), pending.lastIndexOf('\r'));
+      if (end >= 0) {
+        write(maskText(pending.slice(0, end + 1), patterns));
+        pending = pending.slice(end + 1);
+      }
+      if (pending.length > MAX_PENDING) {
+        // Keep enough back that a value cut here is whole in what follows.
+        const keep = Math.max(0, (patterns[0]?.length ?? 1) - 1);
+        write(maskText(pending.slice(0, pending.length - keep), patterns));
+        pending = pending.slice(pending.length - keep);
+      }
+    },
+    flush() {
+      pending += decoder.decode();
+      if (pending) write(maskText(pending, patterns));
+      pending = '';
+    },
+  };
+}
+
+/** The envelope with its failure message or skip reason masked; outputs are data and stay as they are. */
+export function maskEnvelope(envelope: Envelope, patterns: readonly string[]): Envelope {
+  if (envelope.links)
+    envelope = {
+      ...envelope,
+      links: envelope.links.map((l) => ({
+        ...l,
+        url: maskText(l.url, patterns),
+        ...(l.label !== undefined ? { label: maskText(l.label, patterns) } : {}),
+      })),
+    };
+  if (envelope.status === 'failed')
+    return {
+      ...envelope,
+      error: { ...envelope.error, message: maskText(envelope.error.message, patterns) },
+    };
+  if (envelope.status === 'skipped')
+    return { ...envelope, reason: maskText(envelope.reason, patterns) };
+  return envelope;
+}

package/src/harness/supervise.ts

@@ -0,0 +1,63 @@
+/**
+ * Secret masking for `run`: when the service names the step's secrets (`OLVE_SECRETS`), the binary
+ * runs itself as a child and masks the child's stdout and stderr bytes, so every way of printing is
+ * covered, from the pipeline's first import to its last timer. It runs before the pipeline module
+ * is imported.
+ */
+import { readFileSync } from 'node:fs';
+import { constants } from 'node:os';
+import { lineMasker, maskPatterns } from './mask.ts';
+
+const SUPERVISED = 'OLVE_SUPERVISED';
+const FORWARDED = ['SIGTERM', 'SIGINT', 'SIGHUP'] as const;
+
+/** The secret values this run must not print, from `OLVE_SECRETS` (names, comma-separated). */
+export function secretPatterns(env: NodeJS.ProcessEnv = process.env): string[] {
+  const names = (env.OLVE_SECRETS ?? '').split(',').filter(Boolean);
+  return maskPatterns(names.map((n) => env[n]));
+}
+
+/** This process's own command line, loader included (`ld-linux … /olve/step run x` in a pod). */
+function ownArgv(): string[] {
+  try {
+    const argv = readFileSync('/proc/self/cmdline', 'utf8').split('\0').filter(Boolean);
+    if (argv.length > 0) return argv;
+  } catch {
+    // not Linux
+  }
+  return [process.execPath, ...process.argv.slice(1)];
+}
+
+/**
+ * Runs the step as a masked child and answers its exit code, or undefined when this process should
+ * run the step itself: it is the child, the mode isn't `run`, or there is nothing to mask.
+ */
+export async function supervise(argv: readonly string[]): Promise<number | undefined> {
+  if (argv[0] !== 'run' || process.env[SUPERVISED]) return undefined;
+  const patterns = secretPatterns();
+  if (patterns.length === 0) return undefined;
+
+  const child = Bun.spawn(ownArgv(), {
+    env: { ...process.env, [SUPERVISED]: '1' },
+    stdin: 'inherit',
+    stdout: 'pipe',
+    stderr: 'pipe',
+  });
+  const forward = (signal: NodeJS.Signals) => () => child.kill(signal);
+  for (const s of FORWARDED) process.on(s, forward(s));
+
+  const pump = async (stream: ReadableStream<Uint8Array>, fd: 1 | 2) => {
+    const out = fd === 1 ? process.stdout : process.stderr;
+    const masker = lineMasker(patterns, (text) => void out.write(text));
+    for await (const chunk of stream) masker.write(chunk);
+    masker.flush();
+  };
+  const pumps = Promise.all([pump(child.stdout, 1), pump(child.stderr, 2)]);
+  await child.exited;
+  // A detached grandchild may hold the pipes open: don't wait for it long.
+  await Promise.race([pumps, Bun.sleep(5000)]);
+  const signal = child.signalCode
+    ? constants.signals[child.signalCode as NodeJS.Signals]
+    : undefined;
+  return child.exitCode ?? (signal ? 128 + signal : 1);
+}