@olivervea/pipelines 2.0.0-beta.31 → 2.0.0-beta.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -97,7 +97,7 @@ Rules:
97
97
  - A **plain-schema input** has no model to match on, so the step needs a mapper.
98
98
  - **`optional(M)`** means "maybe skipped": `M` must still have an earlier producer.
99
99
  - **`cells(M)`** reads every cell of a matrix step whose output is `M` (see "Matrix steps").
100
- - **`external(M, { title, description?, from? })`** is answered from outside the run; `from` lists `callback()` (another system POSTs to the URL an earlier step gets from `ctx.inputUrl(step, input)`), `poll({ url, valuePath, intervalSeconds? })` and `person()` (the default). It needs no producer and no mapper entry; the step waits until it is answered, then the value (validated against `M`) is in its input like any other. Failure handlers can't have one.
100
+ - **`external(M, { title, description?, from? })`** is answered from outside the run; `from` lists `callback()` (another system POSTs to the URL an earlier step gets from `ctx.inputUrl(step, input)`), `poll({ url, valuePath, intervalSeconds? })` and `person()` (the default). It needs no producer and no mapper entry; the step waits until it is answered, then the value (validated against `M`) is in its input like any other. Failure handlers can't have one. `link: { input, path?, label? }` is the link shown on the step while it waits: `input` is one of the same step's regular (non-external) inputs (typed to their names), `path` a dot path inside its value (`url` → `input.review.url`; none = the value itself is the URL), `label` defaults to `title`. A value that is not an absolute http(s) URL sets no link; the step waits all the same.
101
101
 
102
102
  ### Config
103
103
 
@@ -333,6 +333,7 @@ This section is everything the C# side needs in order to run the binary.
333
333
  | `OLVE_RUN` | run | Run number (integer ≥ 0). Written into produced file refs. Default 0. |
334
334
  | `OLVE_ATTEMPT` | run | Attempt number (integer ≥ 1). Written into produced file refs. Default 1. |
335
335
  | `OLVE_VERSION_STAMP` | run | Derived version, e.g. `2026.10.21.412` → `ctx.run.version`. |
336
+ | `OLVE_SECRETS` | run | The step's declared secret names, comma-separated: the values the binary masks in its output (above). Unset or empty: no masking. |
336
337
  | `OLVE_CELL` | run | Only for a matrix step: the cell, as compact JSON in axis order (`{"os":"linux","arch":"x64"}`) → `ctx.cell`. Missing, or not one of the step's cells → exit 65. `ctx.run.step` and produced file refs carry the cell's name, `build-cli[os=linux,arch=x64]`. |
337
338
  | `OLVE_CASES` | run | Only for a step inside a `select`: a JSON array of the case conditions it sits in (`["target.prod.kind=ssh"]`); `run <step>` uses it to pick the variant of a name several cases use. |
338
339
  | `<SECRET_NAME>` | per step | Each declared secret as an env var of the **same name**: `secretKeyRef` key = name. Mount only the step's `secrets` (for poll triggers, the poll's `secrets`). Child processes inherit them. The mount is optional: an unset secret leaves the variable unset and `ctx.secrets.<NAME>` throws on access, so a handler can check `process.env.<NAME>` first and skip. |
@@ -344,7 +345,9 @@ This section is everything the C# side needs in order to run the binary.
344
345
 
345
346
  `OLVE_RESULT_URL`, `OLVE_FILE_URLS`, `OLVE_FILES_TOKEN` and `OLVE_CALLBACK_URLS` are removed from the environment of child processes. URLs are never logged past `?`.
346
347
 
347
- **Secret masking** (`run`): every value of the step's declared secrets is replaced with `***` in what the step writes: `ctx.log`, `console`, anything written to stdout or stderr, the output of `ctx.exec` and `ctx.sh` commands (piped through the harness when the step has a secret; inherited otherwise), and the envelope's failure message or skip reason. Output is masked a line at a time; a multi-line value is masked line by line, and values (or lines) shorter than 4 characters are left alone. Outputs and files are data and are not masked. A process that writes to the pod's stdout around the harness (a file descriptor it opened itself) is not masked.
348
+ **Secret masking** (`run`): with `OLVE_SECRETS` set, the binary runs the step as its own child and replaces every value of those secrets with `***` in the child's stdout and stderr, a line at a time, from the pipeline's first import to its exit. That covers every way of printing: logs, `console`, direct writes, commands and the processes they start. The envelope's failure message, skip reason and links are masked too; outputs and files are data and aren't. A multi-line value is masked line by line, and values (or lines) shorter than 4 characters are left alone. Not masked: a value the step transforms before printing (escaped, encoded, cut short) and a process writing to the pod's stdout directly (`/proc/1/fd/1`).
349
+
350
+ An error nothing catches (a rejected promise, a throw in a timer) fails the step.
348
351
 
349
352
  `ctx.files.fetch` looks for a file in this order:
350
353
 
@@ -363,7 +366,7 @@ It retries network errors, 5xx, 408 and 429. It always verifies `sha256`.
363
366
  { "status": "failed", "error": { "kind": "handler" | "input" | "output", "message": "…" } }
364
367
  ```
365
368
 
366
- Any of the three may carry `"links": [{ "url": "https://…", "label": "…" }]`, what the handler passed to `ctx.link` (absolute http(s) URLs of at most 2048 characters, labels of at most 100, at most 10). The key is left out when there are none. The server refuses an envelope whose links break these limits.
369
+ Any of the three may carry `"links": [{ "url": "https://…", "label": "…" }]`, what the handler passed to `ctx.link` (absolute http(s) URLs of at most 2048 characters, labels of at most 100, at most 10). The key is left out when there are none. The server refuses an envelope whose links break these limits. The attempt keeps the links its external inputs' `link` gave it while it waited first, then adds the envelope's, leaving out a URL already there.
367
370
 
368
371
  For `map-trigger`, `output` is the pipeline input after it has been parsed by the input model, so defaults are applied. Validate it again server-side.
369
372
 
@@ -419,7 +422,7 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
419
422
  - a nested object of mappings.
420
423
 
421
424
  An object whose only key is `ref` (with a string value) or `const` is a tag. Describe never emits a nested object with exactly that key set. Model-matched inputs are emitted already resolved (`{"ref": "build"}`).
422
- - **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from}`, `from` being `{kind: "person"}`, `{kind: "callback"}` or `{kind: "poll", url, valuePath, intervalSeconds}`. Those keys are not in `input` or `inputTypes`; the service merges the answers into `input.json`.
425
+ - **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from, link?}`, `from` being `{kind: "person"}`, `{kind: "callback"}` or `{kind: "poll", url, valuePath, intervalSeconds}`, and `link` `{input, path?, label?}` (omitted when not set, as are its `path` and `label`). Those keys are not in `input` or `inputTypes`; the service merges the answers into `input.json`.
423
426
  - **`inputTypes`** declares each input key: `{model, optional}`, `{model, optional: false, cells: true}` for a `cells` input (mapped with a whole-step ref to a matrix step whose output is that model), or `{schema: {output, input?}}` for a plain schema.
424
427
  - **Step `matrix`** (matrix steps only, never a failure handler) is `{axes: [{name, values}…], exclude?: [{axis: value…}…], failure: {kind: "keepRunning" | "cascadeFailure"}, maxParallel?}`. `axes` is a list because canonical JSON sorts keys; `exclude` is omitted when empty; `failure.kind` is always written. All of it is part of what the step declares.
425
428
  - **`codeHash`** is the build's identity: `sha256(canonical {bundle, harness})`. `bundle` hashes the entry point bundled with everything it imports, with `@olivervea/pipelines` external. It is tree-shaken and includes embedded scripts and dependencies, so any change to code in use changes it, and with it the manifest, which makes a new pipeline version. A step version covers what the step declares: image, `inputTypes`/model versions, `output` model version, `secrets`, `configEntries` (and those entries' schemas), `input` mappings and `config`.
@@ -442,6 +445,7 @@ Describe runs these checks; reconcile should re-run the manifest-level ones:
442
445
  - availability, ambiguity, maybe-available as required, `optional` without a producer, plain schema without a mapper;
443
446
  - mapper source analysis and recording, unknown fields in refs, mapper keys that are missing or extra;
444
447
  - `when` on an unknown or non-literal field, or with an invalid value;
448
+ - an external input's `link` naming anything but one of the step's regular inputs, an empty path segment, a label that is empty or over 100 characters;
445
449
  - a poll header referencing an undeclared secret, duplicate triggers;
446
450
  - conflicting secret descriptions;
447
451
  - config names, one description and schema per config name, a step receiving a `secretRef` without declaring its secret, and input keys starting with `$` (reserved);
@@ -584,6 +584,32 @@
584
584
  "minItems": 1,
585
585
  "type": "array"
586
586
  },
587
+ "link": {
588
+ "additionalProperties": false,
589
+ "description": "The link shown while the step waits: the absolute http(s) URL at 'path' inside the value of the step's input 'input'. Omitted when none.",
590
+ "properties": {
591
+ "input": {
592
+ "description": "One of the same step's inputs (not an external one).",
593
+ "minLength": 1,
594
+ "type": "string"
595
+ },
596
+ "label": {
597
+ "description": "Default: the input's title.",
598
+ "maxLength": 100,
599
+ "minLength": 1,
600
+ "type": "string"
601
+ },
602
+ "path": {
603
+ "description": "Dot path inside that input's value; omitted = the value itself is the URL.",
604
+ "minLength": 1,
605
+ "type": "string"
606
+ }
607
+ },
608
+ "required": [
609
+ "input"
610
+ ],
611
+ "type": "object"
612
+ },
587
613
  "model": {
588
614
  "description": "The value is validated against this model's input view.",
589
615
  "pattern": "^[A-Za-z][A-Za-z0-9_]*$",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@olivervea/pipelines",
3
- "version": "2.0.0-beta.31",
3
+ "version": "2.0.0-beta.33",
4
4
  "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -61,8 +61,11 @@ export function generateEntry(
61
61
  writeFileSync(
62
62
  entry,
63
63
  `// GENERATED by olve-build — do not edit. Entry point of the step binary.\n` +
64
- `import pipeline from ${JSON.stringify(relPosix(gen, project.pipelineFile))};\n` +
65
- `import { runHarness } from "@olivervea/pipelines/harness";\n\n` +
64
+ `import { runHarness, supervise } from "@olivervea/pipelines/harness";\n\n` +
65
+ `// Secret masking starts before the pipeline's code runs: the masked child imports it.\n` +
66
+ `const supervised = await supervise(process.argv.slice(2));\n` +
67
+ `if (supervised !== undefined) process.exit(supervised);\n` +
68
+ `const { default: pipeline } = await import(${JSON.stringify(relPosix(gen, project.pipelineFile))});\n` +
66
69
  `process.exit(await runHarness(${config}, process.argv.slice(2)));\n`,
67
70
  );
68
71
  return entry;
@@ -47,32 +47,49 @@ export function poll(options: PollOptions): ExternalSource {
47
47
  * A step input whose value comes from outside the run. The step waits until it is answered (the value is
48
48
  * validated against `model`), then runs with the value in its input like any other.
49
49
  */
50
- export interface External<M extends AnyModel = AnyModel> {
50
+ export interface External<M extends AnyModel = AnyModel, L extends string = string> {
51
51
  readonly kind: 'external';
52
52
  readonly model: M;
53
53
  readonly title: string;
54
54
  readonly description: string;
55
55
  readonly from: readonly ExternalSource[];
56
+ readonly link?: ExternalLink<L>;
56
57
  }
57
58
 
58
- export interface ExternalOptions {
59
+ /**
60
+ * The link shown on the step while it waits: the URL is the value of one of the step's other
61
+ * (non-external) inputs, or a field inside it.
62
+ */
63
+ export interface ExternalLink<L extends string = string> {
64
+ /** One of the same step's non-external inputs. */
65
+ readonly input: L;
66
+ /** Dot path inside that input's value (`url`, `review.url`); omitted = the value itself is the URL. */
67
+ readonly path?: string;
68
+ /** Default: the input's title. */
69
+ readonly label?: string;
70
+ }
71
+
72
+ export interface ExternalOptions<L extends string = string> {
59
73
  /** What is asked, shown to whoever answers. */
60
74
  title: string;
61
75
  description?: string;
62
76
  /** Who may answer (default: a person). */
63
77
  from?: readonly ExternalSource[];
78
+ /** The link shown while the step waits, e.g. `{ input: "review", path: "url" }`. */
79
+ link?: ExternalLink<L>;
64
80
  }
65
81
 
66
- export function external<M extends AnyModel>(
82
+ export function external<M extends AnyModel, const L extends string = never>(
67
83
  model: M,
68
- options: ExternalOptions,
69
- ): External<M> {
84
+ options: ExternalOptions<L>,
85
+ ): External<M, L> {
70
86
  return Object.freeze({
71
87
  kind: 'external' as const,
72
88
  model,
73
89
  title: options.title,
74
90
  description: options.description ?? '',
75
91
  from: Object.freeze([...(options.from ?? [person()])]),
92
+ ...(options.link !== undefined ? { link: Object.freeze({ ...options.link }) } : {}),
76
93
  });
77
94
  }
78
95
 
package/src/core/steps.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import type { z } from 'zod';
2
2
  import { STEP, type StepBrand } from './brands.ts';
3
3
  import type { AnyModel, Infer, Optional } from './models.ts';
4
- import type { External } from './external.ts';
4
+ import type { External, ExternalLink } from './external.ts';
5
5
  import type { AnyConfigEntry } from './config.ts';
6
6
  import { configRefsIn } from './config-refs.ts';
7
7
  import type { Secret } from './secrets.ts';
@@ -35,6 +35,23 @@ export type InferEntry<E> =
35
35
  : E extends z.ZodType
36
36
  ? z.output<E>
37
37
  : never;
38
+ /** The keys of a step's inputs that are not external. */
39
+ export type RegularInputKey<I> = {
40
+ [K in keyof I]: I[K] extends External ? never : K;
41
+ }[keyof I] &
42
+ string;
43
+
44
+ /** An external input's `link.input` must name one of the same step's non-external inputs. */
45
+ export type ExternalLinksCheck<I> = {
46
+ [K in keyof I]: I[K] extends External<AnyModel, infer L>
47
+ ? [L] extends [never]
48
+ ? unknown
49
+ : [L] extends [RegularInputKey<I>]
50
+ ? unknown
51
+ : { readonly link?: ExternalLink<RegularInputKey<I>> }
52
+ : unknown;
53
+ };
54
+
38
55
  /** What a handler receives: the output view of every input entry. */
39
56
  export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
40
57
 
@@ -106,7 +123,7 @@ export interface StepDef<
106
123
  description: string;
107
124
  /** Any image: the harness brings its own runtime. */
108
125
  image: string;
109
- input?: I;
126
+ input?: I & ExternalLinksCheck<I>;
110
127
  output?: O;
111
128
  secrets?: Sec;
112
129
  /** Config entries delivered as `ctx.config` (typed to exactly these). */
package/src/ctx/index.ts CHANGED
@@ -24,8 +24,6 @@ export interface CtxOptions {
24
24
  callbackUrls?: Readonly<Record<string, string>>;
25
25
  /** A matrix step's cell (`ctx.cell`). */
26
26
  cell?: Cell;
27
- /** Commands' output goes through this process (the harness masks secrets there). */
28
- pipeOutput?: boolean;
29
27
  }
30
28
 
31
29
  export function formatLog(message: string, fields?: Record<string, unknown>): string {
@@ -146,11 +144,7 @@ export function createCtx(o: CtxOptions): Ctx {
146
144
  const sink = o.logSink ?? ((line: string) => process.stderr.write(line + '\n'));
147
145
  const log = (message: string, fields?: Record<string, unknown>) =>
148
146
  sink(formatLog(message, fields));
149
- const { sh, exec } = createSh({
150
- tmpDir: o.paths.tmp,
151
- signal: o.signal,
152
- pipeOutput: o.pipeOutput,
153
- });
147
+ const { sh, exec } = createSh({ tmpDir: o.paths.tmp, signal: o.signal });
154
148
  const filesDeps = {
155
149
  run: o.run,
156
150
  workFiles: o.paths.workFiles,
package/src/ctx/sh.ts CHANGED
@@ -79,8 +79,6 @@ export function childEnv(extra?: Record<string, string>): Record<string, string>
79
79
  export interface ShDeps {
80
80
  tmpDir: string;
81
81
  signal: AbortSignal;
82
- /** Pipe commands' output through this process's stdout/stderr (where secrets are masked). */
83
- pipeOutput?: boolean;
84
82
  shellCandidates?: readonly string[];
85
83
  }
86
84
 
@@ -152,8 +150,8 @@ export function createSh(deps: ShDeps): {
152
150
  cwd: opts.cwd,
153
151
  env: childEnv(opts.env),
154
152
  stdin: 'ignore',
155
- stdout: capture || idleSeconds || deps.pipeOutput ? 'pipe' : 'inherit',
156
- stderr: idleSeconds || deps.pipeOutput ? 'pipe' : 'inherit',
153
+ stdout: capture || idleSeconds ? 'pipe' : 'inherit',
154
+ stderr: idleSeconds ? 'pipe' : 'inherit',
157
155
  });
158
156
  } catch (e) {
159
157
  throw new Error(`cannot start ${what} (${argv[0]}): ${(e as Error).message}`);
@@ -198,11 +196,11 @@ export function createSh(deps: ShDeps): {
198
196
  pumps.push(
199
197
  pump(proc.stdout as ReadableStream<Uint8Array>, (c) => captured.push(c)),
200
198
  );
201
- else if (idleSeconds || deps.pipeOutput)
199
+ else if (idleSeconds)
202
200
  pumps.push(
203
201
  pump(proc.stdout as ReadableStream<Uint8Array>, (c) => process.stdout.write(c)),
204
202
  );
205
- if (idleSeconds || deps.pipeOutput)
203
+ if (idleSeconds)
206
204
  pumps.push(
207
205
  pump(proc.stderr as ReadableStream<Uint8Array>, (c) => process.stderr.write(c)),
208
206
  );
@@ -211,6 +211,30 @@ export const StepDescriptor = z
211
211
  )
212
212
  .min(1)
213
213
  .describe('Who or what may answer.'),
214
+ link: z
215
+ .strictObject({
216
+ input: z
217
+ .string()
218
+ .min(1)
219
+ .describe("One of the same step's inputs (not an external one)."),
220
+ path: z
221
+ .string()
222
+ .min(1)
223
+ .optional()
224
+ .describe(
225
+ "Dot path inside that input's value; omitted = the value itself is the URL.",
226
+ ),
227
+ label: z
228
+ .string()
229
+ .min(1)
230
+ .max(100)
231
+ .optional()
232
+ .describe("Default: the input's title."),
233
+ })
234
+ .optional()
235
+ .describe(
236
+ "The link shown while the step waits: the absolute http(s) URL at 'path' inside the value of the step's input 'input'. Omitted when none.",
237
+ ),
214
238
  }),
215
239
  )
216
240
  .optional()
@@ -124,6 +124,7 @@ function describeExternalInputs(
124
124
  ): ExternalInputs {
125
125
  const problems = ctx.problems;
126
126
  const externalInputs: ExternalInputs = {};
127
+ const regular = new Set(allEntries.filter(([, e]) => !isExternal(e)).map(([k]) => k));
127
128
  for (const [key, e] of allEntries) {
128
129
  if (!isExternal(e)) continue;
129
130
  if (failureHandler)
@@ -153,16 +154,48 @@ function describeExternalInputs(
153
154
  if (f.valuePath.trim() === '')
154
155
  problems.push(`${at} external input '${key}': poll valuePath must not be empty`);
155
156
  }
157
+ const link = e.link;
158
+ if (link !== undefined) {
159
+ if (!regular.has(link.input))
160
+ problems.push(
161
+ `${at} external input '${key}': link input '${link.input}' must name one of the step's regular (non-external) inputs`,
162
+ );
163
+ if (link.path !== undefined && !isDotPath(link.path))
164
+ problems.push(
165
+ `${at} external input '${key}': link path must be a dot path of non-empty field names`,
166
+ );
167
+ if (
168
+ link.label !== undefined &&
169
+ (typeof link.label !== 'string' ||
170
+ link.label.length < 1 ||
171
+ link.label.length > 100)
172
+ )
173
+ problems.push(
174
+ `${at} external input '${key}': link label must be 1 to 100 characters`,
175
+ );
176
+ }
156
177
  externalInputs[key] = {
157
178
  model: ctx.useModel(e.model, `${at} external input '${key}'`),
158
179
  title: e.title,
159
180
  description: e.description,
160
181
  from: e.from.map((f) => ({ ...f })),
182
+ ...(link !== undefined
183
+ ? {
184
+ link: {
185
+ input: link.input,
186
+ ...(link.path !== undefined ? { path: link.path } : {}),
187
+ ...(link.label !== undefined ? { label: link.label } : {}),
188
+ },
189
+ }
190
+ : {}),
161
191
  };
162
192
  }
163
193
  return externalInputs;
164
194
  }
165
195
 
196
+ const isDotPath = (path: unknown): boolean =>
197
+ typeof path === 'string' && path.split('.').every((segment) => segment !== '');
198
+
166
199
  function describeInputTypes(
167
200
  ctx: DescribeContext,
168
201
  at: string,
@@ -7,6 +7,7 @@ export {
7
7
  type HarnessIO,
8
8
  } from './main.ts';
9
9
  export { EXIT } from './exit.ts';
10
+ export { supervise } from './supervise.ts';
10
11
  export { ENV, paths, runInfoFromEnv } from './env.ts';
11
12
  export {
12
13
  executeStep,
@@ -8,7 +8,7 @@ import { isImplementation } from '../core/handlers.ts';
8
8
  import { configRefPath, isConfigRef, unionAt } from '../core/config.ts';
9
9
  import type { AnyStep } from '../core/steps.ts';
10
10
  import { cellName, parseCell } from '../core/matrix.ts';
11
- import { installMasking, maskEnvelope, maskPatterns, maskText } from './mask.ts';
11
+ import { maskEnvelope, maskPatterns } from './mask.ts';
12
12
  import type { WebhookEvent } from '../core/webhooks.ts';
13
13
  import { pipelineData, type PipelineData, type StageNode } from '../builder/pipeline.ts';
14
14
  import { createCtx, formatLog } from '../ctx/index.ts';
@@ -200,11 +200,11 @@ async function runMode(
200
200
  : undefined;
201
201
  const cancel = cancellation();
202
202
  const masks = maskPatterns(step.secrets.map((s) => process.env[s.name]));
203
- const unmask = masks.length > 0 ? installMasking(masks) : () => {};
204
- if (masks.length > 0) {
205
- const stderr = io.stderr;
206
- io = { ...io, stderr: (line) => stderr(maskText(line, masks)) };
207
- }
203
+ // An error nothing caught (a rejected promise, a throw in a timer) fails the step.
204
+ let uncaught: unknown;
205
+ const onUncaught = (e: unknown) => void (uncaught ??= e);
206
+ process.on('unhandledRejection', onUncaught);
207
+ process.on('uncaughtException', onUncaught);
208
208
  try {
209
209
  let exec: Execution;
210
210
  if (!raw.ok || (cell && !cell.ok)) {
@@ -224,7 +224,6 @@ async function runMode(
224
224
  logSink: io.stderr,
225
225
  callbackUrls: callbackUrlsFromEnv(),
226
226
  ...(cell?.ok ? { cell: cell.cell } : {}),
227
- pipeOutput: masks.length > 0,
228
227
  });
229
228
  exec = await executeStep({
230
229
  step,
@@ -235,10 +234,18 @@ async function runMode(
235
234
  log: io.stderr,
236
235
  });
237
236
  }
238
- unmask();
237
+ if (uncaught !== undefined && exec.envelope.status !== 'failed') {
238
+ const message = `step '${step.name}' failed: an error nothing caught: ${uncaught instanceof Error ? uncaught.message : String(uncaught)}`;
239
+ io.stderr(message);
240
+ exec = {
241
+ envelope: { status: 'failed', error: { kind: 'handler', message } },
242
+ exitCode: EXIT.handlerFailed,
243
+ };
244
+ }
239
245
  return await finish({ ...exec, envelope: maskEnvelope(exec.envelope, masks) }, io);
240
246
  } finally {
241
- unmask();
247
+ process.off('unhandledRejection', onUncaught);
248
+ process.off('uncaughtException', onUncaught);
242
249
  cancel.dispose();
243
250
  }
244
251
  }
@@ -1,10 +1,8 @@
1
- import { format } from 'node:util';
2
1
  import type { Envelope } from './execute.ts';
3
2
 
4
3
  /**
5
- * Masks the step's declared secret values in what it writes to stdout and stderr. Output is masked a
6
- * line at a time, so a value split across two writes is still found; a multi-line value (a key file)
7
- * is masked line by line.
4
+ * Masks secret values in text: a stream a line at a time, so a value split across two writes is
5
+ * still found; a multi-line value (a key file) line by line.
8
6
  */
9
7
 
10
8
  export const MASK = '***';
@@ -17,7 +15,7 @@ const MAX_PENDING = 64 * 1024;
17
15
  export function maskPatterns(values: readonly (string | undefined)[]): string[] {
18
16
  const patterns = new Set<string>();
19
17
  for (const value of values)
20
- for (const line of (value ?? '').split(/\r?\n/))
18
+ for (const line of (value ?? '').split(/\r\n|\r|\n/))
21
19
  if (line.trim().length >= MIN_MASKED_LENGTH) patterns.add(line.trim());
22
20
  return [...patterns].sort((a, b) => b.length - a.length);
23
21
  }
@@ -41,7 +39,12 @@ export function lineMasker(patterns: readonly string[], write: (text: string) =>
41
39
  write(maskText(pending.slice(0, end + 1), patterns));
42
40
  pending = pending.slice(end + 1);
43
41
  }
44
- if (pending.length > MAX_PENDING) this.flush();
42
+ if (pending.length > MAX_PENDING) {
43
+ // Keep enough back that a value cut here is whole in what follows.
44
+ const keep = Math.max(0, (patterns[0]?.length ?? 1) - 1);
45
+ write(maskText(pending.slice(0, pending.length - keep), patterns));
46
+ pending = pending.slice(pending.length - keep);
47
+ }
45
48
  },
46
49
  flush() {
47
50
  pending += decoder.decode();
@@ -53,6 +56,15 @@ export function lineMasker(patterns: readonly string[], write: (text: string) =>
53
56
 
54
57
  /** The envelope with its failure message or skip reason masked; outputs are data and stay as they are. */
55
58
  export function maskEnvelope(envelope: Envelope, patterns: readonly string[]): Envelope {
59
+ if (envelope.links)
60
+ envelope = {
61
+ ...envelope,
62
+ links: envelope.links.map((l) => ({
63
+ ...l,
64
+ url: maskText(l.url, patterns),
65
+ ...(l.label !== undefined ? { label: maskText(l.label, patterns) } : {}),
66
+ })),
67
+ };
56
68
  if (envelope.status === 'failed')
57
69
  return {
58
70
  ...envelope,
@@ -62,37 +74,3 @@ export function maskEnvelope(envelope: Envelope, patterns: readonly string[]): E
62
74
  return { ...envelope, reason: maskText(envelope.reason, patterns) };
63
75
  return envelope;
64
76
  }
65
-
66
- type Write = typeof process.stdout.write;
67
-
68
- /**
69
- * Routes process.stdout and process.stderr through maskers until the returned function is called,
70
- * which flushes them and restores the streams.
71
- */
72
- export function installMasking(patterns: readonly string[]): () => void {
73
- const streams = [process.stdout, process.stderr];
74
- const originals = streams.map((s) => s.write.bind(s) as Write);
75
- const maskers = originals.map((original) =>
76
- lineMasker(patterns, (text) => void original(text)),
77
- );
78
- streams.forEach((s, i) => {
79
- s.write = ((chunk: string | Uint8Array, a?: unknown, b?: unknown) => {
80
- maskers[i]!.write(chunk);
81
- const callback = typeof a === 'function' ? a : typeof b === 'function' ? b : null;
82
- if (callback) queueMicrotask(() => (callback as () => void)());
83
- return true;
84
- }) as Write;
85
- });
86
- // Bun's console writes natively, past process.stdout.write.
87
- const methods = ['log', 'info', 'debug', 'warn', 'error'] as const;
88
- const consoleOriginals = methods.map((m) => console[m]);
89
- methods.forEach((m) => {
90
- const stream = m === 'warn' || m === 'error' ? process.stderr : process.stdout;
91
- console[m] = (...args: unknown[]) => void stream.write(format(...args) + '\n');
92
- });
93
- return () => {
94
- maskers.forEach((m) => m.flush());
95
- streams.forEach((s, i) => (s.write = originals[i]!));
96
- methods.forEach((m, i) => (console[m] = consoleOriginals[i]!));
97
- };
98
- }
@@ -0,0 +1,63 @@
1
+ /**
2
+ * Secret masking for `run`: when the service names the step's secrets (`OLVE_SECRETS`), the binary
3
+ * runs itself as a child and masks the child's stdout and stderr bytes, so every way of printing is
4
+ * covered, from the pipeline's first import to its last timer. It runs before the pipeline module
5
+ * is imported.
6
+ */
7
+ import { readFileSync } from 'node:fs';
8
+ import { constants } from 'node:os';
9
+ import { lineMasker, maskPatterns } from './mask.ts';
10
+
11
+ const SUPERVISED = 'OLVE_SUPERVISED';
12
+ const FORWARDED = ['SIGTERM', 'SIGINT', 'SIGHUP'] as const;
13
+
14
+ /** The secret values this run must not print, from `OLVE_SECRETS` (names, comma-separated). */
15
+ export function secretPatterns(env: NodeJS.ProcessEnv = process.env): string[] {
16
+ const names = (env.OLVE_SECRETS ?? '').split(',').filter(Boolean);
17
+ return maskPatterns(names.map((n) => env[n]));
18
+ }
19
+
20
+ /** This process's own command line, loader included (`ld-linux … /olve/step run x` in a pod). */
21
+ function ownArgv(): string[] {
22
+ try {
23
+ const argv = readFileSync('/proc/self/cmdline', 'utf8').split('\0').filter(Boolean);
24
+ if (argv.length > 0) return argv;
25
+ } catch {
26
+ // not Linux
27
+ }
28
+ return [process.execPath, ...process.argv.slice(1)];
29
+ }
30
+
31
+ /**
32
+ * Runs the step as a masked child and answers its exit code, or undefined when this process should
33
+ * run the step itself: it is the child, the mode isn't `run`, or there is nothing to mask.
34
+ */
35
+ export async function supervise(argv: readonly string[]): Promise<number | undefined> {
36
+ if (argv[0] !== 'run' || process.env[SUPERVISED]) return undefined;
37
+ const patterns = secretPatterns();
38
+ if (patterns.length === 0) return undefined;
39
+
40
+ const child = Bun.spawn(ownArgv(), {
41
+ env: { ...process.env, [SUPERVISED]: '1' },
42
+ stdin: 'inherit',
43
+ stdout: 'pipe',
44
+ stderr: 'pipe',
45
+ });
46
+ const forward = (signal: NodeJS.Signals) => () => child.kill(signal);
47
+ for (const s of FORWARDED) process.on(s, forward(s));
48
+
49
+ const pump = async (stream: ReadableStream<Uint8Array>, fd: 1 | 2) => {
50
+ const out = fd === 1 ? process.stdout : process.stderr;
51
+ const masker = lineMasker(patterns, (text) => void out.write(text));
52
+ for await (const chunk of stream) masker.write(chunk);
53
+ masker.flush();
54
+ };
55
+ const pumps = Promise.all([pump(child.stdout, 1), pump(child.stderr, 2)]);
56
+ await child.exited;
57
+ // A detached grandchild may hold the pipes open: don't wait for it long.
58
+ await Promise.race([pumps, Bun.sleep(5000)]);
59
+ const signal = child.signalCode
60
+ ? constants.signals[child.signalCode as NodeJS.Signals]
61
+ : undefined;
62
+ return child.exitCode ?? (signal ? 128 + signal : 1);
63
+ }
package/src/index.ts CHANGED
@@ -53,6 +53,7 @@ export {
53
53
  person,
54
54
  poll,
55
55
  type External,
56
+ type ExternalLink,
56
57
  type ExternalOptions,
57
58
  type ExternalSource,
58
59
  type PollOptions,