@olivervea/pipelines 2.0.0-beta.31 → 2.0.0-beta.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/manifest.schema.json +26 -0
- package/package.json +1 -1
- package/src/build/build.ts +5 -2
- package/src/core/external.ts +22 -5
- package/src/core/steps.ts +19 -2
- package/src/ctx/index.ts +1 -7
- package/src/ctx/sh.ts +4 -6
- package/src/describe/manifest-schema.ts +24 -0
- package/src/describe/step.ts +33 -0
- package/src/harness/index.ts +1 -0
- package/src/harness/main.ts +16 -9
- package/src/harness/mask.ts +18 -40
- package/src/harness/supervise.ts +63 -0
- package/src/index.ts +1 -0
package/README.md
CHANGED
|
@@ -97,7 +97,7 @@ Rules:
|
|
|
97
97
|
- A **plain-schema input** has no model to match on, so the step needs a mapper.
|
|
98
98
|
- **`optional(M)`** means "maybe skipped": `M` must still have an earlier producer.
|
|
99
99
|
- **`cells(M)`** reads every cell of a matrix step whose output is `M` (see "Matrix steps").
|
|
100
|
-
- **`external(M, { title, description?, from? })`** is answered from outside the run; `from` lists `callback()` (another system POSTs to the URL an earlier step gets from `ctx.inputUrl(step, input)`), `poll({ url, valuePath, intervalSeconds? })` and `person()` (the default). It needs no producer and no mapper entry; the step waits until it is answered, then the value (validated against `M`) is in its input like any other. Failure handlers can't have one.
|
|
100
|
+
- **`external(M, { title, description?, from? })`** is answered from outside the run; `from` lists `callback()` (another system POSTs to the URL an earlier step gets from `ctx.inputUrl(step, input)`), `poll({ url, valuePath, intervalSeconds? })` and `person()` (the default). It needs no producer and no mapper entry; the step waits until it is answered, then the value (validated against `M`) is in its input like any other. Failure handlers can't have one. `link: { input, path?, label? }` is the link shown on the step while it waits: `input` is one of the same step's regular (non-external) inputs (typed to their names), `path` a dot path inside its value (`url` → `input.review.url`; none = the value itself is the URL), `label` defaults to `title`. A value that is not an absolute http(s) URL sets no link; the step waits all the same.
|
|
101
101
|
|
|
102
102
|
### Config
|
|
103
103
|
|
|
@@ -333,6 +333,7 @@ This section is everything the C# side needs in order to run the binary.
|
|
|
333
333
|
| `OLVE_RUN` | run | Run number (integer ≥ 0). Written into produced file refs. Default 0. |
|
|
334
334
|
| `OLVE_ATTEMPT` | run | Attempt number (integer ≥ 1). Written into produced file refs. Default 1. |
|
|
335
335
|
| `OLVE_VERSION_STAMP` | run | Derived version, e.g. `2026.10.21.412` → `ctx.run.version`. |
|
|
336
|
+
| `OLVE_SECRETS` | run | The step's declared secret names, comma-separated: the values the binary masks in its output (above). Unset or empty: no masking. |
|
|
336
337
|
| `OLVE_CELL` | run | Only for a matrix step: the cell, as compact JSON in axis order (`{"os":"linux","arch":"x64"}`) → `ctx.cell`. Missing, or not one of the step's cells → exit 65. `ctx.run.step` and produced file refs carry the cell's name, `build-cli[os=linux,arch=x64]`. |
|
|
337
338
|
| `OLVE_CASES` | run | Only for a step inside a `select`: a JSON array of the case conditions it sits in (`["target.prod.kind=ssh"]`); `run <step>` uses it to pick the variant of a name several cases use. |
|
|
338
339
|
| `<SECRET_NAME>` | per step | Each declared secret as an env var of the **same name**: `secretKeyRef` key = name. Mount only the step's `secrets` (for poll triggers, the poll's `secrets`). Child processes inherit them. The mount is optional: an unset secret leaves the variable unset and `ctx.secrets.<NAME>` throws on access, so a handler can check `process.env.<NAME>` first and skip. |
|
|
@@ -344,7 +345,9 @@ This section is everything the C# side needs in order to run the binary.
|
|
|
344
345
|
|
|
345
346
|
`OLVE_RESULT_URL`, `OLVE_FILE_URLS`, `OLVE_FILES_TOKEN` and `OLVE_CALLBACK_URLS` are removed from the environment of child processes. URLs are never logged past `?`.
|
|
346
347
|
|
|
347
|
-
**Secret masking** (`run`):
|
|
348
|
+
**Secret masking** (`run`): with `OLVE_SECRETS` set, the binary runs the step as its own child and replaces every value of those secrets with `***` in the child's stdout and stderr, a line at a time, from the pipeline's first import to its exit. That covers every way of printing: logs, `console`, direct writes, commands and the processes they start. The envelope's failure message, skip reason and links are masked too; outputs and files are data and aren't. A multi-line value is masked line by line, and values (or lines) shorter than 4 characters are left alone. Not masked: a value the step transforms before printing (escaped, encoded, cut short) and a process writing to the pod's stdout directly (`/proc/1/fd/1`).
|
|
349
|
+
|
|
350
|
+
An error nothing catches (a rejected promise, a throw in a timer) fails the step.
|
|
348
351
|
|
|
349
352
|
`ctx.files.fetch` looks for a file in this order:
|
|
350
353
|
|
|
@@ -363,7 +366,7 @@ It retries network errors, 5xx, 408 and 429. It always verifies `sha256`.
|
|
|
363
366
|
{ "status": "failed", "error": { "kind": "handler" | "input" | "output", "message": "…" } }
|
|
364
367
|
```
|
|
365
368
|
|
|
366
|
-
Any of the three may carry `"links": [{ "url": "https://…", "label": "…" }]`, what the handler passed to `ctx.link` (absolute http(s) URLs of at most 2048 characters, labels of at most 100, at most 10). The key is left out when there are none. The server refuses an envelope whose links break these limits.
|
|
369
|
+
Any of the three may carry `"links": [{ "url": "https://…", "label": "…" }]`, what the handler passed to `ctx.link` (absolute http(s) URLs of at most 2048 characters, labels of at most 100, at most 10). The key is left out when there are none. The server refuses an envelope whose links break these limits. The attempt keeps the links its external inputs' `link` gave it while it waited first, then adds the envelope's, leaving out a URL already there.
|
|
367
370
|
|
|
368
371
|
For `map-trigger`, `output` is the pipeline input after it has been parsed by the input model, so defaults are applied. Validate it again server-side.
|
|
369
372
|
|
|
@@ -419,7 +422,7 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
|
|
|
419
422
|
- a nested object of mappings.
|
|
420
423
|
|
|
421
424
|
An object whose only key is `ref` (with a string value) or `const` is a tag. Describe never emits a nested object with exactly that key set. Model-matched inputs are emitted already resolved (`{"ref": "build"}`).
|
|
422
|
-
- **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from}`, `from` being `{kind: "person"}`, `{kind: "callback"}` or `{kind: "poll", url, valuePath, intervalSeconds}
|
|
425
|
+
- **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from, link?}`, `from` being `{kind: "person"}`, `{kind: "callback"}` or `{kind: "poll", url, valuePath, intervalSeconds}`, and `link` `{input, path?, label?}` (omitted when not set, as are its `path` and `label`). Those keys are not in `input` or `inputTypes`; the service merges the answers into `input.json`.
|
|
423
426
|
- **`inputTypes`** declares each input key: `{model, optional}`, `{model, optional: false, cells: true}` for a `cells` input (mapped with a whole-step ref to a matrix step whose output is that model), or `{schema: {output, input?}}` for a plain schema.
|
|
424
427
|
- **Step `matrix`** (matrix steps only, never a failure handler) is `{axes: [{name, values}…], exclude?: [{axis: value…}…], failure: {kind: "keepRunning" | "cascadeFailure"}, maxParallel?}`. `axes` is a list because canonical JSON sorts keys; `exclude` is omitted when empty; `failure.kind` is always written. All of it is part of what the step declares.
|
|
425
428
|
- **`codeHash`** is the build's identity: `sha256(canonical {bundle, harness})`. `bundle` hashes the entry point bundled with everything it imports, with `@olivervea/pipelines` external. It is tree-shaken and includes embedded scripts and dependencies, so any change to code in use changes it, and with it the manifest, which makes a new pipeline version. A step version covers what the step declares: image, `inputTypes`/model versions, `output` model version, `secrets`, `configEntries` (and those entries' schemas), `input` mappings and `config`.
|
|
@@ -442,6 +445,7 @@ Describe runs these checks; reconcile should re-run the manifest-level ones:
|
|
|
442
445
|
- availability, ambiguity, maybe-available as required, `optional` without a producer, plain schema without a mapper;
|
|
443
446
|
- mapper source analysis and recording, unknown fields in refs, mapper keys that are missing or extra;
|
|
444
447
|
- `when` on an unknown or non-literal field, or with an invalid value;
|
|
448
|
+
- an external input's `link` naming anything but one of the step's regular inputs, an empty path segment, a label that is empty or over 100 characters;
|
|
445
449
|
- a poll header referencing an undeclared secret, duplicate triggers;
|
|
446
450
|
- conflicting secret descriptions;
|
|
447
451
|
- config names, one description and schema per config name, a step receiving a `secretRef` without declaring its secret, and input keys starting with `$` (reserved);
|
package/manifest.schema.json
CHANGED
|
@@ -584,6 +584,32 @@
|
|
|
584
584
|
"minItems": 1,
|
|
585
585
|
"type": "array"
|
|
586
586
|
},
|
|
587
|
+
"link": {
|
|
588
|
+
"additionalProperties": false,
|
|
589
|
+
"description": "The link shown while the step waits: the absolute http(s) URL at 'path' inside the value of the step's input 'input'. Omitted when none.",
|
|
590
|
+
"properties": {
|
|
591
|
+
"input": {
|
|
592
|
+
"description": "One of the same step's inputs (not an external one).",
|
|
593
|
+
"minLength": 1,
|
|
594
|
+
"type": "string"
|
|
595
|
+
},
|
|
596
|
+
"label": {
|
|
597
|
+
"description": "Default: the input's title.",
|
|
598
|
+
"maxLength": 100,
|
|
599
|
+
"minLength": 1,
|
|
600
|
+
"type": "string"
|
|
601
|
+
},
|
|
602
|
+
"path": {
|
|
603
|
+
"description": "Dot path inside that input's value; omitted = the value itself is the URL.",
|
|
604
|
+
"minLength": 1,
|
|
605
|
+
"type": "string"
|
|
606
|
+
}
|
|
607
|
+
},
|
|
608
|
+
"required": [
|
|
609
|
+
"input"
|
|
610
|
+
],
|
|
611
|
+
"type": "object"
|
|
612
|
+
},
|
|
587
613
|
"model": {
|
|
588
614
|
"description": "The value is validated against this model's input view.",
|
|
589
615
|
"pattern": "^[A-Za-z][A-Za-z0-9_]*$",
|
package/package.json
CHANGED
package/src/build/build.ts
CHANGED
|
@@ -61,8 +61,11 @@ export function generateEntry(
|
|
|
61
61
|
writeFileSync(
|
|
62
62
|
entry,
|
|
63
63
|
`// GENERATED by olve-build — do not edit. Entry point of the step binary.\n` +
|
|
64
|
-
`import
|
|
65
|
-
|
|
64
|
+
`import { runHarness, supervise } from "@olivervea/pipelines/harness";\n\n` +
|
|
65
|
+
`// Secret masking starts before the pipeline's code runs: the masked child imports it.\n` +
|
|
66
|
+
`const supervised = await supervise(process.argv.slice(2));\n` +
|
|
67
|
+
`if (supervised !== undefined) process.exit(supervised);\n` +
|
|
68
|
+
`const { default: pipeline } = await import(${JSON.stringify(relPosix(gen, project.pipelineFile))});\n` +
|
|
66
69
|
`process.exit(await runHarness(${config}, process.argv.slice(2)));\n`,
|
|
67
70
|
);
|
|
68
71
|
return entry;
|
package/src/core/external.ts
CHANGED
|
@@ -47,32 +47,49 @@ export function poll(options: PollOptions): ExternalSource {
|
|
|
47
47
|
* A step input whose value comes from outside the run. The step waits until it is answered (the value is
|
|
48
48
|
* validated against `model`), then runs with the value in its input like any other.
|
|
49
49
|
*/
|
|
50
|
-
export interface External<M extends AnyModel = AnyModel> {
|
|
50
|
+
export interface External<M extends AnyModel = AnyModel, L extends string = string> {
|
|
51
51
|
readonly kind: 'external';
|
|
52
52
|
readonly model: M;
|
|
53
53
|
readonly title: string;
|
|
54
54
|
readonly description: string;
|
|
55
55
|
readonly from: readonly ExternalSource[];
|
|
56
|
+
readonly link?: ExternalLink<L>;
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
|
|
59
|
+
/**
|
|
60
|
+
* The link shown on the step while it waits: the URL is the value of one of the step's other
|
|
61
|
+
* (non-external) inputs, or a field inside it.
|
|
62
|
+
*/
|
|
63
|
+
export interface ExternalLink<L extends string = string> {
|
|
64
|
+
/** One of the same step's non-external inputs. */
|
|
65
|
+
readonly input: L;
|
|
66
|
+
/** Dot path inside that input's value (`url`, `review.url`); omitted = the value itself is the URL. */
|
|
67
|
+
readonly path?: string;
|
|
68
|
+
/** Default: the input's title. */
|
|
69
|
+
readonly label?: string;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface ExternalOptions<L extends string = string> {
|
|
59
73
|
/** What is asked, shown to whoever answers. */
|
|
60
74
|
title: string;
|
|
61
75
|
description?: string;
|
|
62
76
|
/** Who may answer (default: a person). */
|
|
63
77
|
from?: readonly ExternalSource[];
|
|
78
|
+
/** The link shown while the step waits, e.g. `{ input: "review", path: "url" }`. */
|
|
79
|
+
link?: ExternalLink<L>;
|
|
64
80
|
}
|
|
65
81
|
|
|
66
|
-
export function external<M extends AnyModel>(
|
|
82
|
+
export function external<M extends AnyModel, const L extends string = never>(
|
|
67
83
|
model: M,
|
|
68
|
-
options: ExternalOptions
|
|
69
|
-
): External<M> {
|
|
84
|
+
options: ExternalOptions<L>,
|
|
85
|
+
): External<M, L> {
|
|
70
86
|
return Object.freeze({
|
|
71
87
|
kind: 'external' as const,
|
|
72
88
|
model,
|
|
73
89
|
title: options.title,
|
|
74
90
|
description: options.description ?? '',
|
|
75
91
|
from: Object.freeze([...(options.from ?? [person()])]),
|
|
92
|
+
...(options.link !== undefined ? { link: Object.freeze({ ...options.link }) } : {}),
|
|
76
93
|
});
|
|
77
94
|
}
|
|
78
95
|
|
package/src/core/steps.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { z } from 'zod';
|
|
2
2
|
import { STEP, type StepBrand } from './brands.ts';
|
|
3
3
|
import type { AnyModel, Infer, Optional } from './models.ts';
|
|
4
|
-
import type { External } from './external.ts';
|
|
4
|
+
import type { External, ExternalLink } from './external.ts';
|
|
5
5
|
import type { AnyConfigEntry } from './config.ts';
|
|
6
6
|
import { configRefsIn } from './config-refs.ts';
|
|
7
7
|
import type { Secret } from './secrets.ts';
|
|
@@ -35,6 +35,23 @@ export type InferEntry<E> =
|
|
|
35
35
|
: E extends z.ZodType
|
|
36
36
|
? z.output<E>
|
|
37
37
|
: never;
|
|
38
|
+
/** The keys of a step's inputs that are not external. */
|
|
39
|
+
export type RegularInputKey<I> = {
|
|
40
|
+
[K in keyof I]: I[K] extends External ? never : K;
|
|
41
|
+
}[keyof I] &
|
|
42
|
+
string;
|
|
43
|
+
|
|
44
|
+
/** An external input's `link.input` must name one of the same step's non-external inputs. */
|
|
45
|
+
export type ExternalLinksCheck<I> = {
|
|
46
|
+
[K in keyof I]: I[K] extends External<AnyModel, infer L>
|
|
47
|
+
? [L] extends [never]
|
|
48
|
+
? unknown
|
|
49
|
+
: [L] extends [RegularInputKey<I>]
|
|
50
|
+
? unknown
|
|
51
|
+
: { readonly link?: ExternalLink<RegularInputKey<I>> }
|
|
52
|
+
: unknown;
|
|
53
|
+
};
|
|
54
|
+
|
|
38
55
|
/** What a handler receives: the output view of every input entry. */
|
|
39
56
|
export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
|
|
40
57
|
|
|
@@ -106,7 +123,7 @@ export interface StepDef<
|
|
|
106
123
|
description: string;
|
|
107
124
|
/** Any image: the harness brings its own runtime. */
|
|
108
125
|
image: string;
|
|
109
|
-
input?: I
|
|
126
|
+
input?: I & ExternalLinksCheck<I>;
|
|
110
127
|
output?: O;
|
|
111
128
|
secrets?: Sec;
|
|
112
129
|
/** Config entries delivered as `ctx.config` (typed to exactly these). */
|
package/src/ctx/index.ts
CHANGED
|
@@ -24,8 +24,6 @@ export interface CtxOptions {
|
|
|
24
24
|
callbackUrls?: Readonly<Record<string, string>>;
|
|
25
25
|
/** A matrix step's cell (`ctx.cell`). */
|
|
26
26
|
cell?: Cell;
|
|
27
|
-
/** Commands' output goes through this process (the harness masks secrets there). */
|
|
28
|
-
pipeOutput?: boolean;
|
|
29
27
|
}
|
|
30
28
|
|
|
31
29
|
export function formatLog(message: string, fields?: Record<string, unknown>): string {
|
|
@@ -146,11 +144,7 @@ export function createCtx(o: CtxOptions): Ctx {
|
|
|
146
144
|
const sink = o.logSink ?? ((line: string) => process.stderr.write(line + '\n'));
|
|
147
145
|
const log = (message: string, fields?: Record<string, unknown>) =>
|
|
148
146
|
sink(formatLog(message, fields));
|
|
149
|
-
const { sh, exec } = createSh({
|
|
150
|
-
tmpDir: o.paths.tmp,
|
|
151
|
-
signal: o.signal,
|
|
152
|
-
pipeOutput: o.pipeOutput,
|
|
153
|
-
});
|
|
147
|
+
const { sh, exec } = createSh({ tmpDir: o.paths.tmp, signal: o.signal });
|
|
154
148
|
const filesDeps = {
|
|
155
149
|
run: o.run,
|
|
156
150
|
workFiles: o.paths.workFiles,
|
package/src/ctx/sh.ts
CHANGED
|
@@ -79,8 +79,6 @@ export function childEnv(extra?: Record<string, string>): Record<string, string>
|
|
|
79
79
|
export interface ShDeps {
|
|
80
80
|
tmpDir: string;
|
|
81
81
|
signal: AbortSignal;
|
|
82
|
-
/** Pipe commands' output through this process's stdout/stderr (where secrets are masked). */
|
|
83
|
-
pipeOutput?: boolean;
|
|
84
82
|
shellCandidates?: readonly string[];
|
|
85
83
|
}
|
|
86
84
|
|
|
@@ -152,8 +150,8 @@ export function createSh(deps: ShDeps): {
|
|
|
152
150
|
cwd: opts.cwd,
|
|
153
151
|
env: childEnv(opts.env),
|
|
154
152
|
stdin: 'ignore',
|
|
155
|
-
stdout: capture || idleSeconds
|
|
156
|
-
stderr: idleSeconds
|
|
153
|
+
stdout: capture || idleSeconds ? 'pipe' : 'inherit',
|
|
154
|
+
stderr: idleSeconds ? 'pipe' : 'inherit',
|
|
157
155
|
});
|
|
158
156
|
} catch (e) {
|
|
159
157
|
throw new Error(`cannot start ${what} (${argv[0]}): ${(e as Error).message}`);
|
|
@@ -198,11 +196,11 @@ export function createSh(deps: ShDeps): {
|
|
|
198
196
|
pumps.push(
|
|
199
197
|
pump(proc.stdout as ReadableStream<Uint8Array>, (c) => captured.push(c)),
|
|
200
198
|
);
|
|
201
|
-
else if (idleSeconds
|
|
199
|
+
else if (idleSeconds)
|
|
202
200
|
pumps.push(
|
|
203
201
|
pump(proc.stdout as ReadableStream<Uint8Array>, (c) => process.stdout.write(c)),
|
|
204
202
|
);
|
|
205
|
-
if (idleSeconds
|
|
203
|
+
if (idleSeconds)
|
|
206
204
|
pumps.push(
|
|
207
205
|
pump(proc.stderr as ReadableStream<Uint8Array>, (c) => process.stderr.write(c)),
|
|
208
206
|
);
|
|
@@ -211,6 +211,30 @@ export const StepDescriptor = z
|
|
|
211
211
|
)
|
|
212
212
|
.min(1)
|
|
213
213
|
.describe('Who or what may answer.'),
|
|
214
|
+
link: z
|
|
215
|
+
.strictObject({
|
|
216
|
+
input: z
|
|
217
|
+
.string()
|
|
218
|
+
.min(1)
|
|
219
|
+
.describe("One of the same step's inputs (not an external one)."),
|
|
220
|
+
path: z
|
|
221
|
+
.string()
|
|
222
|
+
.min(1)
|
|
223
|
+
.optional()
|
|
224
|
+
.describe(
|
|
225
|
+
"Dot path inside that input's value; omitted = the value itself is the URL.",
|
|
226
|
+
),
|
|
227
|
+
label: z
|
|
228
|
+
.string()
|
|
229
|
+
.min(1)
|
|
230
|
+
.max(100)
|
|
231
|
+
.optional()
|
|
232
|
+
.describe("Default: the input's title."),
|
|
233
|
+
})
|
|
234
|
+
.optional()
|
|
235
|
+
.describe(
|
|
236
|
+
"The link shown while the step waits: the absolute http(s) URL at 'path' inside the value of the step's input 'input'. Omitted when none.",
|
|
237
|
+
),
|
|
214
238
|
}),
|
|
215
239
|
)
|
|
216
240
|
.optional()
|
package/src/describe/step.ts
CHANGED
|
@@ -124,6 +124,7 @@ function describeExternalInputs(
|
|
|
124
124
|
): ExternalInputs {
|
|
125
125
|
const problems = ctx.problems;
|
|
126
126
|
const externalInputs: ExternalInputs = {};
|
|
127
|
+
const regular = new Set(allEntries.filter(([, e]) => !isExternal(e)).map(([k]) => k));
|
|
127
128
|
for (const [key, e] of allEntries) {
|
|
128
129
|
if (!isExternal(e)) continue;
|
|
129
130
|
if (failureHandler)
|
|
@@ -153,16 +154,48 @@ function describeExternalInputs(
|
|
|
153
154
|
if (f.valuePath.trim() === '')
|
|
154
155
|
problems.push(`${at} external input '${key}': poll valuePath must not be empty`);
|
|
155
156
|
}
|
|
157
|
+
const link = e.link;
|
|
158
|
+
if (link !== undefined) {
|
|
159
|
+
if (!regular.has(link.input))
|
|
160
|
+
problems.push(
|
|
161
|
+
`${at} external input '${key}': link input '${link.input}' must name one of the step's regular (non-external) inputs`,
|
|
162
|
+
);
|
|
163
|
+
if (link.path !== undefined && !isDotPath(link.path))
|
|
164
|
+
problems.push(
|
|
165
|
+
`${at} external input '${key}': link path must be a dot path of non-empty field names`,
|
|
166
|
+
);
|
|
167
|
+
if (
|
|
168
|
+
link.label !== undefined &&
|
|
169
|
+
(typeof link.label !== 'string' ||
|
|
170
|
+
link.label.length < 1 ||
|
|
171
|
+
link.label.length > 100)
|
|
172
|
+
)
|
|
173
|
+
problems.push(
|
|
174
|
+
`${at} external input '${key}': link label must be 1 to 100 characters`,
|
|
175
|
+
);
|
|
176
|
+
}
|
|
156
177
|
externalInputs[key] = {
|
|
157
178
|
model: ctx.useModel(e.model, `${at} external input '${key}'`),
|
|
158
179
|
title: e.title,
|
|
159
180
|
description: e.description,
|
|
160
181
|
from: e.from.map((f) => ({ ...f })),
|
|
182
|
+
...(link !== undefined
|
|
183
|
+
? {
|
|
184
|
+
link: {
|
|
185
|
+
input: link.input,
|
|
186
|
+
...(link.path !== undefined ? { path: link.path } : {}),
|
|
187
|
+
...(link.label !== undefined ? { label: link.label } : {}),
|
|
188
|
+
},
|
|
189
|
+
}
|
|
190
|
+
: {}),
|
|
161
191
|
};
|
|
162
192
|
}
|
|
163
193
|
return externalInputs;
|
|
164
194
|
}
|
|
165
195
|
|
|
196
|
+
const isDotPath = (path: unknown): boolean =>
|
|
197
|
+
typeof path === 'string' && path.split('.').every((segment) => segment !== '');
|
|
198
|
+
|
|
166
199
|
function describeInputTypes(
|
|
167
200
|
ctx: DescribeContext,
|
|
168
201
|
at: string,
|
package/src/harness/index.ts
CHANGED
package/src/harness/main.ts
CHANGED
|
@@ -8,7 +8,7 @@ import { isImplementation } from '../core/handlers.ts';
|
|
|
8
8
|
import { configRefPath, isConfigRef, unionAt } from '../core/config.ts';
|
|
9
9
|
import type { AnyStep } from '../core/steps.ts';
|
|
10
10
|
import { cellName, parseCell } from '../core/matrix.ts';
|
|
11
|
-
import {
|
|
11
|
+
import { maskEnvelope, maskPatterns } from './mask.ts';
|
|
12
12
|
import type { WebhookEvent } from '../core/webhooks.ts';
|
|
13
13
|
import { pipelineData, type PipelineData, type StageNode } from '../builder/pipeline.ts';
|
|
14
14
|
import { createCtx, formatLog } from '../ctx/index.ts';
|
|
@@ -200,11 +200,11 @@ async function runMode(
|
|
|
200
200
|
: undefined;
|
|
201
201
|
const cancel = cancellation();
|
|
202
202
|
const masks = maskPatterns(step.secrets.map((s) => process.env[s.name]));
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
203
|
+
// An error nothing caught (a rejected promise, a throw in a timer) fails the step.
|
|
204
|
+
let uncaught: unknown;
|
|
205
|
+
const onUncaught = (e: unknown) => void (uncaught ??= e);
|
|
206
|
+
process.on('unhandledRejection', onUncaught);
|
|
207
|
+
process.on('uncaughtException', onUncaught);
|
|
208
208
|
try {
|
|
209
209
|
let exec: Execution;
|
|
210
210
|
if (!raw.ok || (cell && !cell.ok)) {
|
|
@@ -224,7 +224,6 @@ async function runMode(
|
|
|
224
224
|
logSink: io.stderr,
|
|
225
225
|
callbackUrls: callbackUrlsFromEnv(),
|
|
226
226
|
...(cell?.ok ? { cell: cell.cell } : {}),
|
|
227
|
-
pipeOutput: masks.length > 0,
|
|
228
227
|
});
|
|
229
228
|
exec = await executeStep({
|
|
230
229
|
step,
|
|
@@ -235,10 +234,18 @@ async function runMode(
|
|
|
235
234
|
log: io.stderr,
|
|
236
235
|
});
|
|
237
236
|
}
|
|
238
|
-
|
|
237
|
+
if (uncaught !== undefined && exec.envelope.status !== 'failed') {
|
|
238
|
+
const message = `step '${step.name}' failed: an error nothing caught: ${uncaught instanceof Error ? uncaught.message : String(uncaught)}`;
|
|
239
|
+
io.stderr(message);
|
|
240
|
+
exec = {
|
|
241
|
+
envelope: { status: 'failed', error: { kind: 'handler', message } },
|
|
242
|
+
exitCode: EXIT.handlerFailed,
|
|
243
|
+
};
|
|
244
|
+
}
|
|
239
245
|
return await finish({ ...exec, envelope: maskEnvelope(exec.envelope, masks) }, io);
|
|
240
246
|
} finally {
|
|
241
|
-
|
|
247
|
+
process.off('unhandledRejection', onUncaught);
|
|
248
|
+
process.off('uncaughtException', onUncaught);
|
|
242
249
|
cancel.dispose();
|
|
243
250
|
}
|
|
244
251
|
}
|
package/src/harness/mask.ts
CHANGED
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import { format } from 'node:util';
|
|
2
1
|
import type { Envelope } from './execute.ts';
|
|
3
2
|
|
|
4
3
|
/**
|
|
5
|
-
* Masks
|
|
6
|
-
*
|
|
7
|
-
* is masked line by line.
|
|
4
|
+
* Masks secret values in text: a stream a line at a time, so a value split across two writes is
|
|
5
|
+
* still found; a multi-line value (a key file) line by line.
|
|
8
6
|
*/
|
|
9
7
|
|
|
10
8
|
export const MASK = '***';
|
|
@@ -17,7 +15,7 @@ const MAX_PENDING = 64 * 1024;
|
|
|
17
15
|
export function maskPatterns(values: readonly (string | undefined)[]): string[] {
|
|
18
16
|
const patterns = new Set<string>();
|
|
19
17
|
for (const value of values)
|
|
20
|
-
for (const line of (value ?? '').split(/\r
|
|
18
|
+
for (const line of (value ?? '').split(/\r\n|\r|\n/))
|
|
21
19
|
if (line.trim().length >= MIN_MASKED_LENGTH) patterns.add(line.trim());
|
|
22
20
|
return [...patterns].sort((a, b) => b.length - a.length);
|
|
23
21
|
}
|
|
@@ -41,7 +39,12 @@ export function lineMasker(patterns: readonly string[], write: (text: string) =>
|
|
|
41
39
|
write(maskText(pending.slice(0, end + 1), patterns));
|
|
42
40
|
pending = pending.slice(end + 1);
|
|
43
41
|
}
|
|
44
|
-
if (pending.length > MAX_PENDING)
|
|
42
|
+
if (pending.length > MAX_PENDING) {
|
|
43
|
+
// Keep enough back that a value cut here is whole in what follows.
|
|
44
|
+
const keep = Math.max(0, (patterns[0]?.length ?? 1) - 1);
|
|
45
|
+
write(maskText(pending.slice(0, pending.length - keep), patterns));
|
|
46
|
+
pending = pending.slice(pending.length - keep);
|
|
47
|
+
}
|
|
45
48
|
},
|
|
46
49
|
flush() {
|
|
47
50
|
pending += decoder.decode();
|
|
@@ -53,6 +56,15 @@ export function lineMasker(patterns: readonly string[], write: (text: string) =>
|
|
|
53
56
|
|
|
54
57
|
/** The envelope with its failure message or skip reason masked; outputs are data and stay as they are. */
|
|
55
58
|
export function maskEnvelope(envelope: Envelope, patterns: readonly string[]): Envelope {
|
|
59
|
+
if (envelope.links)
|
|
60
|
+
envelope = {
|
|
61
|
+
...envelope,
|
|
62
|
+
links: envelope.links.map((l) => ({
|
|
63
|
+
...l,
|
|
64
|
+
url: maskText(l.url, patterns),
|
|
65
|
+
...(l.label !== undefined ? { label: maskText(l.label, patterns) } : {}),
|
|
66
|
+
})),
|
|
67
|
+
};
|
|
56
68
|
if (envelope.status === 'failed')
|
|
57
69
|
return {
|
|
58
70
|
...envelope,
|
|
@@ -62,37 +74,3 @@ export function maskEnvelope(envelope: Envelope, patterns: readonly string[]): E
|
|
|
62
74
|
return { ...envelope, reason: maskText(envelope.reason, patterns) };
|
|
63
75
|
return envelope;
|
|
64
76
|
}
|
|
65
|
-
|
|
66
|
-
type Write = typeof process.stdout.write;
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* Routes process.stdout and process.stderr through maskers until the returned function is called,
|
|
70
|
-
* which flushes them and restores the streams.
|
|
71
|
-
*/
|
|
72
|
-
export function installMasking(patterns: readonly string[]): () => void {
|
|
73
|
-
const streams = [process.stdout, process.stderr];
|
|
74
|
-
const originals = streams.map((s) => s.write.bind(s) as Write);
|
|
75
|
-
const maskers = originals.map((original) =>
|
|
76
|
-
lineMasker(patterns, (text) => void original(text)),
|
|
77
|
-
);
|
|
78
|
-
streams.forEach((s, i) => {
|
|
79
|
-
s.write = ((chunk: string | Uint8Array, a?: unknown, b?: unknown) => {
|
|
80
|
-
maskers[i]!.write(chunk);
|
|
81
|
-
const callback = typeof a === 'function' ? a : typeof b === 'function' ? b : null;
|
|
82
|
-
if (callback) queueMicrotask(() => (callback as () => void)());
|
|
83
|
-
return true;
|
|
84
|
-
}) as Write;
|
|
85
|
-
});
|
|
86
|
-
// Bun's console writes natively, past process.stdout.write.
|
|
87
|
-
const methods = ['log', 'info', 'debug', 'warn', 'error'] as const;
|
|
88
|
-
const consoleOriginals = methods.map((m) => console[m]);
|
|
89
|
-
methods.forEach((m) => {
|
|
90
|
-
const stream = m === 'warn' || m === 'error' ? process.stderr : process.stdout;
|
|
91
|
-
console[m] = (...args: unknown[]) => void stream.write(format(...args) + '\n');
|
|
92
|
-
});
|
|
93
|
-
return () => {
|
|
94
|
-
maskers.forEach((m) => m.flush());
|
|
95
|
-
streams.forEach((s, i) => (s.write = originals[i]!));
|
|
96
|
-
methods.forEach((m, i) => (console[m] = consoleOriginals[i]!));
|
|
97
|
-
};
|
|
98
|
-
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Secret masking for `run`: when the service names the step's secrets (`OLVE_SECRETS`), the binary
|
|
3
|
+
* runs itself as a child and masks the child's stdout and stderr bytes, so every way of printing is
|
|
4
|
+
* covered, from the pipeline's first import to its last timer. It runs before the pipeline module
|
|
5
|
+
* is imported.
|
|
6
|
+
*/
|
|
7
|
+
import { readFileSync } from 'node:fs';
|
|
8
|
+
import { constants } from 'node:os';
|
|
9
|
+
import { lineMasker, maskPatterns } from './mask.ts';
|
|
10
|
+
|
|
11
|
+
const SUPERVISED = 'OLVE_SUPERVISED';
|
|
12
|
+
const FORWARDED = ['SIGTERM', 'SIGINT', 'SIGHUP'] as const;
|
|
13
|
+
|
|
14
|
+
/** The secret values this run must not print, from `OLVE_SECRETS` (names, comma-separated). */
|
|
15
|
+
export function secretPatterns(env: NodeJS.ProcessEnv = process.env): string[] {
|
|
16
|
+
const names = (env.OLVE_SECRETS ?? '').split(',').filter(Boolean);
|
|
17
|
+
return maskPatterns(names.map((n) => env[n]));
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** This process's own command line, loader included (`ld-linux … /olve/step run x` in a pod). */
|
|
21
|
+
function ownArgv(): string[] {
|
|
22
|
+
try {
|
|
23
|
+
const argv = readFileSync('/proc/self/cmdline', 'utf8').split('\0').filter(Boolean);
|
|
24
|
+
if (argv.length > 0) return argv;
|
|
25
|
+
} catch {
|
|
26
|
+
// not Linux
|
|
27
|
+
}
|
|
28
|
+
return [process.execPath, ...process.argv.slice(1)];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Runs the step as a masked child and answers its exit code, or undefined when this process should
|
|
33
|
+
* run the step itself: it is the child, the mode isn't `run`, or there is nothing to mask.
|
|
34
|
+
*/
|
|
35
|
+
export async function supervise(argv: readonly string[]): Promise<number | undefined> {
|
|
36
|
+
if (argv[0] !== 'run' || process.env[SUPERVISED]) return undefined;
|
|
37
|
+
const patterns = secretPatterns();
|
|
38
|
+
if (patterns.length === 0) return undefined;
|
|
39
|
+
|
|
40
|
+
const child = Bun.spawn(ownArgv(), {
|
|
41
|
+
env: { ...process.env, [SUPERVISED]: '1' },
|
|
42
|
+
stdin: 'inherit',
|
|
43
|
+
stdout: 'pipe',
|
|
44
|
+
stderr: 'pipe',
|
|
45
|
+
});
|
|
46
|
+
const forward = (signal: NodeJS.Signals) => () => child.kill(signal);
|
|
47
|
+
for (const s of FORWARDED) process.on(s, forward(s));
|
|
48
|
+
|
|
49
|
+
const pump = async (stream: ReadableStream<Uint8Array>, fd: 1 | 2) => {
|
|
50
|
+
const out = fd === 1 ? process.stdout : process.stderr;
|
|
51
|
+
const masker = lineMasker(patterns, (text) => void out.write(text));
|
|
52
|
+
for await (const chunk of stream) masker.write(chunk);
|
|
53
|
+
masker.flush();
|
|
54
|
+
};
|
|
55
|
+
const pumps = Promise.all([pump(child.stdout, 1), pump(child.stderr, 2)]);
|
|
56
|
+
await child.exited;
|
|
57
|
+
// A detached grandchild may hold the pipes open: don't wait for it long.
|
|
58
|
+
await Promise.race([pumps, Bun.sleep(5000)]);
|
|
59
|
+
const signal = child.signalCode
|
|
60
|
+
? constants.signals[child.signalCode as NodeJS.Signals]
|
|
61
|
+
: undefined;
|
|
62
|
+
return child.exitCode ?? (signal ? 128 + signal : 1);
|
|
63
|
+
}
|