@olivervea/pipelines 2.0.0-beta.30 → 2.0.0-beta.31

package/README.md

@@ -344,6 +344,8 @@ This section is everything the C# side needs in order to run the binary.
 
 `OLVE_RESULT_URL`, `OLVE_FILE_URLS`, `OLVE_FILES_TOKEN` and `OLVE_CALLBACK_URLS` are removed from the environment of child processes. URLs are never logged past `?`.
 
+**Secret masking** (`run`): every value of the step's declared secrets is replaced with `***` in what the step writes: `ctx.log`, `console`, anything written to stdout or stderr, the output of `ctx.exec` and `ctx.sh` commands (piped through the harness when the step has a secret; inherited otherwise), and the envelope's failure message or skip reason. Output is masked a line at a time; a multi-line value is masked line by line, and values (or lines) shorter than 4 characters are left alone. Outputs and files are data and are not masked. A process that writes to the pod's stdout around the harness (a file descriptor it opened itself) is not masked.
+
 `ctx.files.fetch` looks for a file in this order:
 
 1. the attempt's own `work/files`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.30",
+  "version": "2.0.0-beta.31",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/ctx/index.ts

@@ -24,6 +24,8 @@ export interface CtxOptions {
   callbackUrls?: Readonly<Record<string, string>>;
   /** A matrix step's cell (`ctx.cell`). */
   cell?: Cell;
+  /** Commands' output goes through this process (the harness masks secrets there). */
+  pipeOutput?: boolean;
 }
 
 export function formatLog(message: string, fields?: Record<string, unknown>): string {
@@ -144,7 +146,11 @@ export function createCtx(o: CtxOptions): Ctx {
   const sink = o.logSink ?? ((line: string) => process.stderr.write(line + '\n'));
   const log = (message: string, fields?: Record<string, unknown>) =>
     sink(formatLog(message, fields));
-  const { sh, exec } = createSh({ tmpDir: o.paths.tmp, signal: o.signal });
+  const { sh, exec } = createSh({
+    tmpDir: o.paths.tmp,
+    signal: o.signal,
+    pipeOutput: o.pipeOutput,
+  });
   const filesDeps = {
     run: o.run,
     workFiles: o.paths.workFiles,

package/src/ctx/sh.ts

@@ -79,6 +79,8 @@ export function childEnv(extra?: Record<string, string>): Record<string, string>
 export interface ShDeps {
   tmpDir: string;
   signal: AbortSignal;
+  /** Pipe commands' output through this process's stdout/stderr (where secrets are masked). */
+  pipeOutput?: boolean;
   shellCandidates?: readonly string[];
 }
 
@@ -150,8 +152,8 @@ export function createSh(deps: ShDeps): {
         cwd: opts.cwd,
         env: childEnv(opts.env),
         stdin: 'ignore',
-        stdout: capture || idleSeconds ? 'pipe' : 'inherit',
-        stderr: idleSeconds ? 'pipe' : 'inherit',
+        stdout: capture || idleSeconds || deps.pipeOutput ? 'pipe' : 'inherit',
+        stderr: idleSeconds || deps.pipeOutput ? 'pipe' : 'inherit',
       });
     } catch (e) {
       throw new Error(`cannot start ${what} (${argv[0]}): ${(e as Error).message}`);
@@ -196,11 +198,11 @@ export function createSh(deps: ShDeps): {
       pumps.push(
         pump(proc.stdout as ReadableStream<Uint8Array>, (c) => captured.push(c)),
       );
-    else if (idleSeconds)
+    else if (idleSeconds || deps.pipeOutput)
       pumps.push(
         pump(proc.stdout as ReadableStream<Uint8Array>, (c) => process.stdout.write(c)),
       );
-    if (idleSeconds)
+    if (idleSeconds || deps.pipeOutput)
       pumps.push(
         pump(proc.stderr as ReadableStream<Uint8Array>, (c) => process.stderr.write(c)),
       );

package/src/harness/main.ts

@@ -8,6 +8,7 @@ import { isImplementation } from '../core/handlers.ts';
 import { configRefPath, isConfigRef, unionAt } from '../core/config.ts';
 import type { AnyStep } from '../core/steps.ts';
 import { cellName, parseCell } from '../core/matrix.ts';
+import { installMasking, maskEnvelope, maskPatterns, maskText } from './mask.ts';
 import type { WebhookEvent } from '../core/webhooks.ts';
 import { pipelineData, type PipelineData, type StageNode } from '../builder/pipeline.ts';
 import { createCtx, formatLog } from '../ctx/index.ts';
@@ -198,6 +199,12 @@ async function runMode(
     ? parseCell(step.name, step.matrix, process.env[ENV.cell])
     : undefined;
   const cancel = cancellation();
+  const masks = maskPatterns(step.secrets.map((s) => process.env[s.name]));
+  const unmask = masks.length > 0 ? installMasking(masks) : () => {};
+  if (masks.length > 0) {
+    const stderr = io.stderr;
+    io = { ...io, stderr: (line) => stderr(maskText(line, masks)) };
+  }
   try {
     let exec: Execution;
     if (!raw.ok || (cell && !cell.ok)) {
@@ -217,6 +224,7 @@ async function runMode(
         logSink: io.stderr,
         callbackUrls: callbackUrlsFromEnv(),
         ...(cell?.ok ? { cell: cell.cell } : {}),
+        pipeOutput: masks.length > 0,
       });
       exec = await executeStep({
         step,
@@ -227,8 +235,10 @@ async function runMode(
         log: io.stderr,
       });
     }
-    return await finish(exec, io);
+    unmask();
+    return await finish({ ...exec, envelope: maskEnvelope(exec.envelope, masks) }, io);
   } finally {
+    unmask();
     cancel.dispose();
   }
 }

package/src/harness/mask.ts

@@ -0,0 +1,98 @@
+import { format } from 'node:util';
+import type { Envelope } from './execute.ts';
+
+/**
+ * Masks the step's declared secret values in what it writes to stdout and stderr. Output is masked a
+ * line at a time, so a value split across two writes is still found; a multi-line value (a key file)
+ * is masked line by line.
+ */
+
+export const MASK = '***';
+/** Shorter values (and lines of multi-line values) are left alone: they would mask ordinary text. */
+export const MIN_MASKED_LENGTH = 4;
+/** A line longer than this is masked and written without waiting for its end. */
+const MAX_PENDING = 64 * 1024;
+
+/** The strings to mask for these secret values, longest first. */
+export function maskPatterns(values: readonly (string | undefined)[]): string[] {
+  const patterns = new Set<string>();
+  for (const value of values)
+    for (const line of (value ?? '').split(/\r?\n/))
+      if (line.trim().length >= MIN_MASKED_LENGTH) patterns.add(line.trim());
+  return [...patterns].sort((a, b) => b.length - a.length);
+}
+
+export function maskText(text: string, patterns: readonly string[]): string {
+  let out = text;
+  for (const p of patterns) if (out.includes(p)) out = out.split(p).join(MASK);
+  return out;
+}
+
+/** A writer that masks complete lines and holds back the unfinished one until it ends or `flush`. */
+export function lineMasker(patterns: readonly string[], write: (text: string) => void) {
+  const decoder = new TextDecoder();
+  let pending = '';
+  return {
+    write(chunk: string | Uint8Array) {
+      pending +=
+        typeof chunk === 'string' ? chunk : decoder.decode(chunk, { stream: true });
+      const end = Math.max(pending.lastIndexOf('\n'), pending.lastIndexOf('\r'));
+      if (end >= 0) {
+        write(maskText(pending.slice(0, end + 1), patterns));
+        pending = pending.slice(end + 1);
+      }
+      if (pending.length > MAX_PENDING) this.flush();
+    },
+    flush() {
+      pending += decoder.decode();
+      if (pending) write(maskText(pending, patterns));
+      pending = '';
+    },
+  };
+}
+
+/** The envelope with its failure message or skip reason masked; outputs are data and stay as they are. */
+export function maskEnvelope(envelope: Envelope, patterns: readonly string[]): Envelope {
+  if (envelope.status === 'failed')
+    return {
+      ...envelope,
+      error: { ...envelope.error, message: maskText(envelope.error.message, patterns) },
+    };
+  if (envelope.status === 'skipped')
+    return { ...envelope, reason: maskText(envelope.reason, patterns) };
+  return envelope;
+}
+
+type Write = typeof process.stdout.write;
+
+/**
+ * Routes process.stdout and process.stderr through maskers until the returned function is called,
+ * which flushes them and restores the streams.
+ */
+export function installMasking(patterns: readonly string[]): () => void {
+  const streams = [process.stdout, process.stderr];
+  const originals = streams.map((s) => s.write.bind(s) as Write);
+  const maskers = originals.map((original) =>
+    lineMasker(patterns, (text) => void original(text)),
+  );
+  streams.forEach((s, i) => {
+    s.write = ((chunk: string | Uint8Array, a?: unknown, b?: unknown) => {
+      maskers[i]!.write(chunk);
+      const callback = typeof a === 'function' ? a : typeof b === 'function' ? b : null;
+      if (callback) queueMicrotask(() => (callback as () => void)());
+      return true;
+    }) as Write;
+  });
+  // Bun's console writes natively, past process.stdout.write.
+  const methods = ['log', 'info', 'debug', 'warn', 'error'] as const;
+  const consoleOriginals = methods.map((m) => console[m]);
+  methods.forEach((m) => {
+    const stream = m === 'warn' || m === 'error' ? process.stderr : process.stdout;
+    console[m] = (...args: unknown[]) => void stream.write(format(...args) + '\n');
+  });
+  return () => {
+    maskers.forEach((m) => m.flush());
+    streams.forEach((s, i) => (s.write = originals[i]!));
+    methods.forEach((m, i) => (console[m] = consoleOriginals[i]!));
+  };
+}