@olivervea/pipelines 2.0.0-beta.29 → 2.0.0-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -10
- package/manifest.schema.json +87 -0
- package/package.json +1 -1
- package/src/builder/pipeline.ts +32 -14
- package/src/core/handlers.ts +13 -4
- package/src/core/matrix.ts +150 -0
- package/src/core/models.ts +6 -1
- package/src/core/steps.ts +38 -11
- package/src/ctx/index.ts +4 -0
- package/src/ctx/types.ts +7 -0
- package/src/describe/context.ts +6 -1
- package/src/describe/manifest-schema.ts +48 -0
- package/src/describe/stages.ts +3 -2
- package/src/describe/step.ts +155 -6
- package/src/harness/env.ts +2 -0
- package/src/harness/main.ts +10 -4
- package/src/harness/validate.ts +5 -0
- package/src/index.ts +15 -0
- package/src/testing/index.ts +25 -1
package/README.md
CHANGED
|
@@ -79,7 +79,7 @@ export const deployBeta = step({
|
|
|
79
79
|
name: "deploy-beta", // identity: lower-case kebab, ≤ 63 chars
|
|
80
80
|
description: "Deploys to beta.", // required (may be ""); shown in the UI, not versioned
|
|
81
81
|
image: "alpine:latest", // any image; the harness brings its own runtime
|
|
82
|
-
input: { artifact: BuildArtifact }, // model | optional(model) | plain zod schema | external(model, …)
|
|
82
|
+
input: { artifact: BuildArtifact }, // model | optional(model) | cells(model) | plain zod schema | external(model, …)
|
|
83
83
|
output: BetaDeployment, // a model, or omitted (no output)
|
|
84
84
|
secrets: [SSH_PRIVATE_KEY], // only these are mounted; ctx.secrets is typed to them
|
|
85
85
|
timeoutSeconds: 900, // optional
|
|
@@ -96,6 +96,7 @@ Rules:
|
|
|
96
96
|
- **Trigger mappers** produce `z.input`, so fields with defaults may be left out. **Handlers, mapper refs and `when`** see `z.output`.
|
|
97
97
|
- A **plain-schema input** has no model to match on, so the step needs a mapper.
|
|
98
98
|
- **`optional(M)`** means "maybe skipped": `M` must still have an earlier producer.
|
|
99
|
+
- **`cells(M)`** reads every cell of a matrix step whose output is `M` (see "Matrix steps").
|
|
99
100
|
- **`external(M, { title, description?, from? })`** is answered from outside the run; `from` lists `callback()` (another system POSTs to the URL an earlier step gets from `ctx.inputUrl(step, input)`), `poll({ url, valuePath, intervalSeconds? })` and `person()` (the default). It needs no producer and no mapper entry; the step waits until it is answered, then the value (validated against `M`) is in its input like any other. Failure handlers can't have one.
|
|
100
101
|
|
|
101
102
|
### Config
|
|
@@ -192,6 +193,37 @@ export function smokeTest<const N extends string>(def: { name: LiteralName<N>; d
|
|
|
192
193
|
- **`settings`**: what the step was set up with, recorded as the step's manifest `config` (so part of its version). JSON values and config references; the step receives every entry its settings reference.
|
|
193
194
|
- `implement` refuses a step that already carries an implementation.
|
|
194
195
|
|
|
196
|
+
### Matrix steps
|
|
197
|
+
|
|
198
|
+
```ts
|
|
199
|
+
import { cascadeFailure, cells, implement, step } from "@olivervea/pipelines";
|
|
200
|
+
|
|
201
|
+
export const buildCli = implement(step({
|
|
202
|
+
name: "build-cli", description: "", image: "oven/bun:1.4.3-slim",
|
|
203
|
+
input: { source: Source }, output: BuiltBinary,
|
|
204
|
+
matrix: {
|
|
205
|
+
axes: { os: ["linux", "windows", "macos"], arch: ["x64", "arm64"] }, // key order is the axis order
|
|
206
|
+
exclude: [{ os: "windows", arch: "arm64" }], // names one or more axes
|
|
207
|
+
failure: cascadeFailure(), // default keepRunning()
|
|
208
|
+
maxParallel: 2, // optional
|
|
209
|
+
},
|
|
210
|
+
}), async ({ source }, ctx) => {
|
|
211
|
+
const { os, arch } = ctx.cell; // { os: "linux" | "windows" | "macos"; arch: "x64" | "arm64" }
|
|
212
|
+
// …
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
export const publish = implement(step({
|
|
216
|
+
name: "publish", description: "", image: "alpine", input: { binaries: cells(BuiltBinary) },
|
|
217
|
+
}), async ({ binaries }) => { for (const { cell, output } of binaries) { /* … */ } });
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
- **Cells.** The step runs once per cell: the cross product of the axes (first axis outermost) minus the excludes. 1 to 4 axes, 1 to 64 cells. An axis name matches `^[a-z][a-zA-Z0-9]{0,30}$`, a value `^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`, values unique per axis. Exclude keys and values are type-checked against the axes.
|
|
221
|
+
- **`ctx.cell`** is the cell, typed from the axes' literals. It exists only on a matrix step's ctx.
|
|
222
|
+
- **A cell's name** is `build-cli[os=linux,arch=x64]`: `ctx.run.step` and the `step` of the cell's file references hold it.
|
|
223
|
+
- **Failure.** `keepRunning()` lets every cell run to the end; `cascadeFailure()` cancels the cells still running or waiting once a cell has failed its last allowed attempt.
|
|
224
|
+
- **Reading a matrix step.** A later step reads it only through `cells(M)`, matched by model like other inputs or wired whole with a mapper (`[publish, from => ({ binaries: from["build-cli"] })]`; a field of a matrix step is a type error). The value is `{ cell, output }[]` in cell order, one entry per cell with an output; `cell` is `Readonly<Record<string, string>>`. A matrix step can't be read as one output, and a matrix step inside a `when` can't be read by `cells` (it may be skipped).
|
|
225
|
+
- **Failure handlers** can't be matrix steps.
|
|
226
|
+
|
|
195
227
|
### Handlers
|
|
196
228
|
|
|
197
229
|
```ts
|
|
@@ -219,7 +251,8 @@ export const deployBeta = implement(step({ name: "deploy-beta", /* … */ }), as
|
|
|
219
251
|
| `config.<entry>` | Typed to the step's declared config entries (output view, defaults applied). From input.json's `$config`. An optional entry that is not set is `undefined`. |
|
|
220
252
|
| `log(message, fields?)` | Writes a `message k=v …` line to stderr. |
|
|
221
253
|
| `signal` | Aborted on SIGTERM/SIGINT. Running commands are sent SIGTERM. |
|
|
222
|
-
| `run` | `{ pipeline, id, number, attempt, step, version }` (from the env, see below). |
|
|
254
|
+
| `run` | `{ pipeline, id, number, attempt, step, version }` (from the env, see below). For a matrix step `step` is the cell's name. |
|
|
255
|
+
| `cell` | Matrix steps only: the cell being run, in axis order (from `OLVE_CELL`). |
|
|
223
256
|
| `tmpDir` | A writable scratch directory (`$OLVE_DIR/tmp`). Kaniko has no `/tmp`. |
|
|
224
257
|
| `skip(reason)` | Produce no output (prefer `when`). |
|
|
225
258
|
|
|
@@ -257,6 +290,8 @@ const r = await testHandler(deployBeta, { artifact }, { secrets: { SSH_PRIVATE_K
|
|
|
257
290
|
expect(r).toMatchObject({ status: "succeeded", output: { release: "olve-pipelines" } }); // r.filesDir, r.logs, r.exitCode
|
|
258
291
|
```
|
|
259
292
|
|
|
293
|
+
A matrix step takes the cell to run: `testHandler(buildCli, { source }, { cell: { os: "linux", arch: "x64" } })`; without one it fails with `kind: "input"` and exit code 65, like a pod without `OLVE_CELL`.
|
|
294
|
+
|
|
260
295
|
`testHandler` uses the same validation path as a real run. Invalid input gives `status: "failed"` with `kind: "input"`; an invalid return value gives `kind: "output"`. Any part of `ctx` can be replaced; whatever you don't replace is the real implementation. `testHandler` takes any implementation: a custom step, a library step or a reusable step.
|
|
261
296
|
|
|
262
297
|
## Harness contract (service ⇄ step binary)
|
|
@@ -272,7 +307,7 @@ This section is everything the C# side needs in order to run the binary.
|
|
|
272
307
|
| Mode | Reads | Writes | Notes |
|
|
273
308
|
|---|---|---|---|
|
|
274
309
|
| `describe [--sentinels]` | the binary itself | stdout: the manifest (canonical JSON) | With `--sentinels`, stdout gets the manifest between `---OLVE-MANIFEST-BEGIN---` and `---OLVE-MANIFEST-END---`. Needs no env and no files. |
|
|
275
|
-
| `run <step>` | `$OLVE_DIR/input.json` | `$OLVE_DIR/work/output.json`, `$OLVE_DIR/work/files/…` | Steps in stages and failure-handler steps. |
|
|
310
|
+
| `run <step>` | `$OLVE_DIR/input.json` | `$OLVE_DIR/work/output.json`, `$OLVE_DIR/work/files/…` | Steps in stages and failure-handler steps. A matrix step's cell runs as `run <step>` (the step's own name) with `OLVE_CELL`. |
|
|
276
311
|
| `map-trigger <trigger>` | `$OLVE_DIR/input.json` (the event) | `$OLVE_DIR/work/output.json` | `<trigger>` is the trigger's `name` from the manifest. |
|
|
277
312
|
|
|
278
313
|
`run` and `map-trigger` also print the envelope on stdout, as one compact JSON line between `---OLVE-RESULT-BEGIN---` and `---OLVE-RESULT-END---`. The block always comes last. Use it as a fallback channel from the pod log. User output and stderr logs also go to the pod log, so never parse anything outside the sentinels.
|
|
@@ -281,7 +316,7 @@ This section is everything the C# side needs in order to run the binary.
|
|
|
281
316
|
|
|
282
317
|
| Path | Who | What |
|
|
283
318
|
|---|---|---|
|
|
284
|
-
| `input.json` | service (via olve-init) | **run**: a JSON object with exactly the step's input keys, built from the manifest's `input` mappings. Leave out a key whose ref root step was skipped; this is allowed only for `optional` inputs. Unknown keys → exit 65. For a step with `configEntries`, the reserved key `"$config"` holds `{ <entry>: <value> }` from the run's config (entries that are not set are left out); each entry is parsed with its schema, and a missing required entry or an invalid value → exit 65 (`config.<entry>…`). Other steps ignore `$config`. **map-trigger push**: `{"ref": "refs/heads/<branch>", "after": "<sha>", "repository": {"full_name": "<owner>/<repo>"}}` (`PushEvent`). **map-trigger poll**: `{"value": "<extracted value as string>"}`. **map-trigger webhook**: `{"headers": {…}, "body": …}` (`WebhookEvent`). |
|
|
319
|
+
| `input.json` | service (via olve-init) | **run**: a JSON object with exactly the step's input keys, built from the manifest's `input` mappings. Leave out a key whose ref root step was skipped; this is allowed only for `optional` inputs. Unknown keys → exit 65. For a step with `configEntries`, the reserved key `"$config"` holds `{ <entry>: <value> }` from the run's config (entries that are not set are left out); each entry is parsed with its schema, and a missing required entry or an invalid value → exit 65 (`config.<entry>…`). Other steps ignore `$config`. A `cells` input is a list in cell order, one entry per cell whose latest successful attempt has an output: `[{"cell": {"os": "linux", "arch": "x64"}, "output": {…}}, …]`. **map-trigger push**: `{"ref": "refs/heads/<branch>", "after": "<sha>", "repository": {"full_name": "<owner>/<repo>"}}` (`PushEvent`). **map-trigger poll**: `{"value": "<extracted value as string>"}`. **map-trigger webhook**: `{"headers": {…}, "body": …}` (`WebhookEvent`). |
|
|
285
320
|
| `work/output.json` | harness | The envelope. Any stale copy is deleted at start, and the new file is written atomically. |
|
|
286
321
|
| `work/files/<object path>` | harness | Output files and directory tars (see "File references"). |
|
|
287
322
|
| `tmp/` | harness | Scratch space. Point `TMPDIR` here. |
|
|
@@ -298,6 +333,7 @@ This section is everything the C# side needs in order to run the binary.
|
|
|
298
333
|
| `OLVE_RUN` | run | Run number (integer ≥ 0). Written into produced file refs. Default 0. |
|
|
299
334
|
| `OLVE_ATTEMPT` | run | Attempt number (integer ≥ 1). Written into produced file refs. Default 1. |
|
|
300
335
|
| `OLVE_VERSION_STAMP` | run | Derived version, e.g. `2026.10.21.412` → `ctx.run.version`. |
|
|
336
|
+
| `OLVE_CELL` | run | Only for a matrix step: the cell, as compact JSON in axis order (`{"os":"linux","arch":"x64"}`) → `ctx.cell`. Missing, or not one of the step's cells → exit 65. `ctx.run.step` and produced file refs carry the cell's name, `build-cli[os=linux,arch=x64]`. |
|
|
301
337
|
| `OLVE_CASES` | run | Only for a step inside a `select`: a JSON array of the case conditions it sits in (`["target.prod.kind=ssh"]`); `run <step>` uses it to pick the variant of a name several cases use. |
|
|
302
338
|
| `<SECRET_NAME>` | per step | Each declared secret as an env var of the **same name**: `secretKeyRef` key = name. Mount only the step's `secrets` (for poll triggers, the poll's `secrets`). Child processes inherit them. The mount is optional: an unset secret leaves the variable unset and `ctx.secrets.<NAME>` throws on access, so a handler can check `process.env.<NAME>` first and skip. |
|
|
303
339
|
| `OLVE_RESULT_URL` | optional | Pre-signed **PUT** for `result.tar.gz` (see below). Unset means no upload. |
|
|
@@ -336,7 +372,7 @@ For `map-trigger`, `output` is the pipeline input after it has been parsed by th
|
|
|
336
372
|
| 0 | ok (succeeded or skipped) | yes |
|
|
337
373
|
| 1 | the handler or trigger mapper threw | failed / `handler` |
|
|
338
374
|
| 64 | usage: unknown mode, step or trigger | none |
|
|
339
|
-
| 65 | `input.json` missing, not JSON, or invalid for the step (or event), including its `$config` | failed / `input` |
|
|
375
|
+
| 65 | `input.json` missing, not JSON, or invalid for the step (or event), including its `$config`; a matrix step's `OLVE_CELL` missing or not one of its cells | failed / `input` |
|
|
340
376
|
| 66 | the return value failed strict validation, or an own file ref is missing or changed | failed / `output` |
|
|
341
377
|
| 74 | the `result.tar.gz` upload failed after retries. Used only when the step itself succeeded; otherwise the step's own code wins. | yes (printed) |
|
|
342
378
|
| 78 | the pipeline is invalid (describe problems, or a step without an implementation) | none |
|
|
@@ -357,6 +393,8 @@ The service unpacks the archive into `runs/<p>/<run>/<step>/<attempt>/`.
|
|
|
357
393
|
{ "run": 412, "step": "build", "attempt": 1, "path": "chart/", "sha256": "<64 hex>" }
|
|
358
394
|
```
|
|
359
395
|
|
|
396
|
+
- `step` is the step's name, or a matrix cell's name (`build-cli[os=linux,arch=x64]`).
|
|
397
|
+
|
|
360
398
|
- In JSON Schema, a reference is marked `"x-olve-kind": "fileRef"`.
|
|
361
399
|
- `path` is relative, with no `.`/`..` segments, no `\`, no NUL and no leading `/`. The pattern is in the schema. A **trailing `/` means a directory**.
|
|
362
400
|
- **Storage rule:** the object path is `path` for files. For directories, it is `path` without the trailing `/`, plus `.tar`: `chart/` → `chart.tar`, an uncompressed deterministic tar.
|
|
@@ -365,7 +403,7 @@ The service unpacks the archive into `runs/<p>/<run>/<step>/<attempt>/`.
|
|
|
365
403
|
|
|
366
404
|
## Manifest
|
|
367
405
|
|
|
368
|
-
Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-12, `$defs` `Manifest`, `Step`, `Stage`, `Mapping`, `Trigger`, `InputType`, `ModelSchemas`). The text is canonical JSON: sorted keys, 2-space indent, a trailing newline, and no `$schema` in the model schemas. Hash the text as printed.
|
|
406
|
+
Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-12, `$defs` `Manifest`, `Step`, `Stage`, `Mapping`, `Trigger`, `InputType`, `ModelSchemas`, `Matrix`). The text is canonical JSON: sorted keys, 2-space indent, a trailing newline, and no `$schema` in the model schemas. Hash the text as printed.
|
|
369
407
|
|
|
370
408
|
- **`apiVersion`** is `"2.0"`.
|
|
371
409
|
- **`harness`** is `{ bun, sdk, sdkHash }`.
|
|
@@ -380,7 +418,8 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
|
|
|
380
418
|
|
|
381
419
|
An object whose only key is `ref` (with a string value) or `const` is a tag. Describe never emits a nested object with exactly that key set. Model-matched inputs are emitted already resolved (`{"ref": "build"}`).
|
|
382
420
|
- **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from}`, `from` being `{kind: "person"}`, `{kind: "callback"}` or `{kind: "poll", url, valuePath, intervalSeconds}`. Those keys are not in `input` or `inputTypes`; the service merges the answers into `input.json`.
|
|
383
|
-
- **`inputTypes`** declares each input key: `{model, optional}`, or `{schema: {output, input?}}` for a plain schema.
|
|
421
|
+
- **`inputTypes`** declares each input key: `{model, optional}`, `{model, optional: false, cells: true}` for a `cells` input (mapped with a whole-step ref to a matrix step whose output is that model), or `{schema: {output, input?}}` for a plain schema.
|
|
422
|
+
- **Step `matrix`** (matrix steps only, never a failure handler) is `{axes: [{name, values}…], exclude?: [{axis: value…}…], failure: {kind: "keepRunning" | "cascadeFailure"}, maxParallel?}`. `axes` is a list because canonical JSON sorts keys; `exclude` is omitted when empty; `failure.kind` is always written. All of it is part of what the step declares.
|
|
384
423
|
- **`codeHash`** is the build's identity: `sha256(canonical {bundle, harness})`. `bundle` hashes the entry point bundled with everything it imports, with `@olivervea/pipelines` external. It is tree-shaken and includes embedded scripts and dependencies, so any change to code in use changes it, and with it the manifest, which makes a new pipeline version. A step version covers what the step declares: image, `inputTypes`/model versions, `output` model version, `secrets`, `configEntries` (and those entries' schemas), `input` mappings and `config`.
|
|
385
424
|
- **`stages`** is an array whose items are either `{steps: [Step…]}` (parallel, with a barrier) or `{when: {field: [values]}, stages: [...]}`.
|
|
386
425
|
- **`triggers`**:
|
|
@@ -404,7 +443,8 @@ Describe runs these checks; reconcile should re-run the manifest-level ones:
|
|
|
404
443
|
- a poll header referencing an undeclared secret, duplicate triggers;
|
|
405
444
|
- conflicting secret descriptions;
|
|
406
445
|
- config names, one description and schema per config name, a step receiving a `secretRef` without declaring its secret, and input keys starting with `$` (reserved);
|
|
407
|
-
- an `onFailure` scope naming an unknown step, and failure-handler inputs
|
|
446
|
+
- an `onFailure` scope naming an unknown step, and failure-handler inputs;
|
|
447
|
+
- a matrix's axes, values, excludes, cell count, failure strategy and `maxParallel`, a matrix failure handler, and reading a matrix step other than whole through a `cells` input (or a `cells` input reading anything else).
|
|
408
448
|
|
|
409
449
|
## Build tool
|
|
410
450
|
|
|
@@ -447,7 +487,7 @@ olve-dev run <dir> <step> [--input-file f] [--config-file f] [--olve-dir d] [--s
|
|
|
447
487
|
olve-dev map-trigger <dir> <trigger> --event-file f [--olve-dir d]
|
|
448
488
|
```
|
|
449
489
|
|
|
450
|
-
`olve-dev` runs handlers from source, through the same validation path as the binary. `$OLVE_DIR` defaults to `<dir>/.olve`, and outputs land in `<dir>/.olve/work/`. Set `OLVE_FILES_DIR` to a local pool to fetch file refs. `--config-file` (a JSON object of config entries) becomes the input's `$config`.
|
|
490
|
+
`olve-dev` runs handlers from source, through the same validation path as the binary. `$OLVE_DIR` defaults to `<dir>/.olve`, and outputs land in `<dir>/.olve/work/`. Set `OLVE_FILES_DIR` to a local pool to fetch file refs. `--config-file` (a JSON object of config entries) becomes the input's `$config`. A matrix step's cell comes from `OLVE_CELL` in the environment, as in the pod.
|
|
451
491
|
|
|
452
492
|
## Development
|
|
453
493
|
|
|
@@ -455,7 +495,6 @@ olve-dev map-trigger <dir> <trigger> --event-file f [--olve-dir d]
|
|
|
455
495
|
bun install
|
|
456
496
|
bunx tsc --noEmit -p . # type-check (TS 7)
|
|
457
497
|
bun scripts/check-compile-fail.ts # compile-fail cases (also run by bun test)
|
|
458
|
-
TSC=/path/to/tsc-5.9/bin/tsc bun scripts/check-compile-fail.ts # the other TS line
|
|
459
498
|
bun test # runtime + build/e2e tests
|
|
460
499
|
bun scripts/gen-manifest-schema.ts # after changing src/describe/manifest-schema.ts
|
|
461
500
|
```
|
package/manifest.schema.json
CHANGED
|
@@ -5,6 +5,11 @@
|
|
|
5
5
|
{
|
|
6
6
|
"additionalProperties": false,
|
|
7
7
|
"properties": {
|
|
8
|
+
"cells": {
|
|
9
|
+
"const": true,
|
|
10
|
+
"description": "cells(Model): every cell of a matrix step whose output is the model, mapped with a whole-step ref; the value is [{cell, output}, …] in cell order. Omitted otherwise.",
|
|
11
|
+
"type": "boolean"
|
|
12
|
+
},
|
|
8
13
|
"model": {
|
|
9
14
|
"description": "Model name (key of `models`).",
|
|
10
15
|
"pattern": "^[A-Za-z][A-Za-z0-9_]*$",
|
|
@@ -284,6 +289,84 @@
|
|
|
284
289
|
}
|
|
285
290
|
]
|
|
286
291
|
},
|
|
292
|
+
"Matrix": {
|
|
293
|
+
"additionalProperties": false,
|
|
294
|
+
"description": "A matrix step runs once per cell, named '<step>[<axis>=<value>,…]' in axis order; OLVE_CELL holds the cell as compact JSON.",
|
|
295
|
+
"properties": {
|
|
296
|
+
"axes": {
|
|
297
|
+
"description": "In order: the cells are the cross product, first axis outermost, minus the excludes (1 to 64 cells).",
|
|
298
|
+
"items": {
|
|
299
|
+
"additionalProperties": false,
|
|
300
|
+
"properties": {
|
|
301
|
+
"name": {
|
|
302
|
+
"pattern": "^[a-z][a-zA-Z0-9]{0,30}$",
|
|
303
|
+
"type": "string"
|
|
304
|
+
},
|
|
305
|
+
"values": {
|
|
306
|
+
"description": "Unique per axis.",
|
|
307
|
+
"items": {
|
|
308
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$",
|
|
309
|
+
"type": "string"
|
|
310
|
+
},
|
|
311
|
+
"minItems": 1,
|
|
312
|
+
"type": "array"
|
|
313
|
+
}
|
|
314
|
+
},
|
|
315
|
+
"required": [
|
|
316
|
+
"name",
|
|
317
|
+
"values"
|
|
318
|
+
],
|
|
319
|
+
"type": "object"
|
|
320
|
+
},
|
|
321
|
+
"maxItems": 4,
|
|
322
|
+
"minItems": 1,
|
|
323
|
+
"type": "array"
|
|
324
|
+
},
|
|
325
|
+
"exclude": {
|
|
326
|
+
"description": "Each entry names one or more axes with one of their values; a cell matching every named axis is left out (omitted when none).",
|
|
327
|
+
"items": {
|
|
328
|
+
"additionalProperties": {
|
|
329
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$",
|
|
330
|
+
"type": "string"
|
|
331
|
+
},
|
|
332
|
+
"propertyNames": {
|
|
333
|
+
"pattern": "^[a-z][a-zA-Z0-9]{0,30}$",
|
|
334
|
+
"type": "string"
|
|
335
|
+
},
|
|
336
|
+
"type": "object"
|
|
337
|
+
},
|
|
338
|
+
"type": "array"
|
|
339
|
+
},
|
|
340
|
+
"failure": {
|
|
341
|
+
"additionalProperties": false,
|
|
342
|
+
"description": "keepRunning: every cell runs to the end. cascadeFailure: a cell whose last allowed attempt failed cancels the step's other scheduled or running cells.",
|
|
343
|
+
"properties": {
|
|
344
|
+
"kind": {
|
|
345
|
+
"enum": [
|
|
346
|
+
"keepRunning",
|
|
347
|
+
"cascadeFailure"
|
|
348
|
+
],
|
|
349
|
+
"type": "string"
|
|
350
|
+
}
|
|
351
|
+
},
|
|
352
|
+
"required": [
|
|
353
|
+
"kind"
|
|
354
|
+
],
|
|
355
|
+
"type": "object"
|
|
356
|
+
},
|
|
357
|
+
"maxParallel": {
|
|
358
|
+
"description": "At most this many of the step's cells running at once in a run.",
|
|
359
|
+
"maximum": 9007199254740991,
|
|
360
|
+
"minimum": 1,
|
|
361
|
+
"type": "integer"
|
|
362
|
+
}
|
|
363
|
+
},
|
|
364
|
+
"required": [
|
|
365
|
+
"axes",
|
|
366
|
+
"failure"
|
|
367
|
+
],
|
|
368
|
+
"type": "object"
|
|
369
|
+
},
|
|
287
370
|
"ModelSchemas": {
|
|
288
371
|
"additionalProperties": false,
|
|
289
372
|
"properties": {
|
|
@@ -556,6 +639,10 @@
|
|
|
556
639
|
"minLength": 1,
|
|
557
640
|
"type": "string"
|
|
558
641
|
},
|
|
642
|
+
"matrix": {
|
|
643
|
+
"$ref": "#/$defs/Matrix",
|
|
644
|
+
"description": "Set on a matrix step (never on a failure handler); omitted otherwise."
|
|
645
|
+
},
|
|
559
646
|
"name": {
|
|
560
647
|
"description": "Step name: lower-case kebab, 1-63 chars. The step's identity.",
|
|
561
648
|
"pattern": "^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$",
|
package/package.json
CHANGED
package/src/builder/pipeline.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { PIPELINE } from '../core/brands.ts';
|
|
2
2
|
import type { AnyModel, Infer, InferInput, Plain } from '../core/models.ts';
|
|
3
3
|
import type { Secret } from '../core/secrets.ts';
|
|
4
|
-
import type { AnyStep, OutputValue, Step } from '../core/steps.ts';
|
|
4
|
+
import type { AnyStep, CellOf, OutputValue, Step } from '../core/steps.ts';
|
|
5
|
+
import type { Cells, CellOutput } from '../core/matrix.ts';
|
|
5
6
|
import { isStep } from '../core/steps.ts';
|
|
6
7
|
import type { WebhookEvent, WebhookVerify } from '../core/webhooks.ts';
|
|
7
8
|
import type { IsUnion, OrError, Simplify } from '../core/types.ts';
|
|
@@ -66,15 +67,25 @@ type EntryError<
|
|
|
66
67
|
: IsUnion<ProducersOf<A, M['name']>> extends true
|
|
67
68
|
? `❌ ${M['name']} (input '${Key}' of step '${SN}') has several producers; use a mapper to pick one`
|
|
68
69
|
: never
|
|
69
|
-
: E extends
|
|
70
|
-
? [ProducersOf<A,
|
|
71
|
-
? `❌ ${
|
|
72
|
-
: [DefiniteProducersOf<A,
|
|
73
|
-
? `❌ ${
|
|
74
|
-
: IsUnion<ProducersOf<A,
|
|
75
|
-
? `❌ ${
|
|
70
|
+
: E extends Cells<infer M>
|
|
71
|
+
? [ProducersOf<A, `cells:${M['name']}`>] extends [never]
|
|
72
|
+
? `❌ ${M['name']} (input '${Key}' of step '${SN}') has no matrix step producing it at this stage; cells() reads a matrix step`
|
|
73
|
+
: [DefiniteProducersOf<A, `cells:${M['name']}`>] extends [never]
|
|
74
|
+
? `❌ ${M['name']} (input '${Key}' of step '${SN}') is only maybe-available (produced inside a when block); cells() cannot read a maybe-skipped matrix step`
|
|
75
|
+
: IsUnion<ProducersOf<A, `cells:${M['name']}`>> extends true
|
|
76
|
+
? `❌ ${M['name']} (input '${Key}' of step '${SN}') has several matrix steps producing it; use a mapper to pick one`
|
|
76
77
|
: never
|
|
77
|
-
:
|
|
78
|
+
: E extends AnyModel
|
|
79
|
+
? [ProducersOf<A, E['name']>] extends [never]
|
|
80
|
+
? [ProducersOf<A, `cells:${E['name']}`>] extends [never]
|
|
81
|
+
? `❌ ${E['name']} (input '${Key}' of step '${SN}') is not available at this stage`
|
|
82
|
+
: `❌ ${E['name']} (input '${Key}' of step '${SN}') is produced by a matrix step; read it with cells(${E['name']})`
|
|
83
|
+
: [DefiniteProducersOf<A, E['name']>] extends [never]
|
|
84
|
+
? `❌ ${E['name']} (input '${Key}' of step '${SN}') is only maybe-available (produced inside a when block); declare it optional(${E['name']})`
|
|
85
|
+
: IsUnion<ProducersOf<A, E['name']>> extends true
|
|
86
|
+
? `❌ ${E['name']} (input '${Key}' of step '${SN}') has several producers; use a mapper to pick one`
|
|
87
|
+
: never
|
|
88
|
+
: `❌ input '${Key}' of step '${SN}' is a plain schema, not a model, so it cannot be matched by model; add a mapper`;
|
|
78
89
|
|
|
79
90
|
type DefaultMatchError<S extends AnyStep, A extends AvailMap> = {
|
|
80
91
|
[K in keyof S['input'] & string]: EntryError<S['name'], K, S['input'][K], A>;
|
|
@@ -138,12 +149,19 @@ export type CheckStage<E extends readonly unknown[], A extends AvailMap> = {
|
|
|
138
149
|
[K in keyof E]: CheckEntry<NoInfer<E>, K, E[K], A>;
|
|
139
150
|
};
|
|
140
151
|
|
|
152
|
+
// A matrix step's slot holds its cells, under `cells:<model>`: only cells() inputs match it.
|
|
141
153
|
type OutputsOf<E extends readonly unknown[], Maybe extends boolean> = {
|
|
142
|
-
[S in StepOfEntry<E[number]> as S['name']]:
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
154
|
+
[S in StepOfEntry<E[number]> as S['name']]: [CellOf<S>] extends [never]
|
|
155
|
+
? Slot<
|
|
156
|
+
S['output'] extends AnyModel ? S['output']['name'] : null,
|
|
157
|
+
OutputValue<S>,
|
|
158
|
+
Maybe
|
|
159
|
+
>
|
|
160
|
+
: Slot<
|
|
161
|
+
S['output'] extends AnyModel ? `cells:${S['output']['name']}` : null,
|
|
162
|
+
S['output'] extends AnyModel ? CellOutput<S['output'], CellOf<S>>[] : void,
|
|
163
|
+
Maybe
|
|
164
|
+
>;
|
|
147
165
|
};
|
|
148
166
|
type MakeMaybe<X extends AvailMap> = {
|
|
149
167
|
[K in keyof X]: Slot<X[K]['model'], X[K]['value'], true>;
|
package/src/core/handlers.ts
CHANGED
|
@@ -4,8 +4,15 @@ import {
|
|
|
4
4
|
type ImplementationBrand,
|
|
5
5
|
type SkippedBrand,
|
|
6
6
|
} from './brands.ts';
|
|
7
|
-
import type {
|
|
8
|
-
import type {
|
|
7
|
+
import type { StepCtx } from '../ctx/types.ts';
|
|
8
|
+
import type {
|
|
9
|
+
AnyStep,
|
|
10
|
+
CellOf,
|
|
11
|
+
ConfigOf,
|
|
12
|
+
OutputValue,
|
|
13
|
+
SecretsOf,
|
|
14
|
+
StepInput,
|
|
15
|
+
} from './steps.ts';
|
|
9
16
|
import type { Awaitable } from './types.ts';
|
|
10
17
|
|
|
11
18
|
/** Returned by `ctx.skip(reason)`: the step produced no output on purpose. */
|
|
@@ -25,9 +32,11 @@ export function isSkipped(v: unknown): v is Skipped {
|
|
|
25
32
|
}
|
|
26
33
|
|
|
27
34
|
export type HandlerResult<S extends AnyStep> = OutputValue<S> | Skipped;
|
|
35
|
+
/** The ctx a handler of `S` gets. */
|
|
36
|
+
export type CtxOf<S extends AnyStep> = StepCtx<SecretsOf<S>, ConfigOf<S>, CellOf<S>>;
|
|
28
37
|
export type Handler<S extends AnyStep> = (
|
|
29
38
|
input: StepInput<S['input']>,
|
|
30
|
-
ctx:
|
|
39
|
+
ctx: CtxOf<S>,
|
|
31
40
|
) => Awaitable<HandlerResult<S>>;
|
|
32
41
|
|
|
33
42
|
/** A step with its handler. It is itself a step: stage it like any value. */
|
|
@@ -58,7 +67,7 @@ export function implement<S extends AnyStep, R extends HandlerResult<S>>(
|
|
|
58
67
|
: unknown),
|
|
59
68
|
handler: (
|
|
60
69
|
input: StepInput<S['input']>,
|
|
61
|
-
ctx:
|
|
70
|
+
ctx: CtxOf<S>,
|
|
62
71
|
) => Awaitable<R & NoExtraFields<R, OutputValue<S>>>,
|
|
63
72
|
): Implementation<S> {
|
|
64
73
|
if (isImplementation(step))
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { isModel, type AnyModel, type Infer } from './models.ts';
|
|
2
|
+
import type { Simplify } from './types.ts';
|
|
3
|
+
|
|
4
|
+
/** Axis name → its values, in order. Object key order is the axis order. */
|
|
5
|
+
export type Axes = { readonly [axis: string]: readonly string[] };
|
|
6
|
+
/** One combination of axis values. */
|
|
7
|
+
export type Cell = Readonly<Record<string, string>>;
|
|
8
|
+
/** The cell type of a step whose axes are `Ax`. */
|
|
9
|
+
export type CellOfAxes<Ax extends Axes> = Simplify<{
|
|
10
|
+
readonly [K in keyof Ax as string extends K ? never : K]: Ax[K][number];
|
|
11
|
+
}>;
|
|
12
|
+
|
|
13
|
+
/** What the step's other cells do when one fails. */
|
|
14
|
+
export type FailureStrategy =
|
|
15
|
+
{ readonly kind: 'keepRunning' } | { readonly kind: 'cascadeFailure' };
|
|
16
|
+
|
|
17
|
+
/** Every cell runs to the end (the default). */
|
|
18
|
+
export function keepRunning(): FailureStrategy {
|
|
19
|
+
return Object.freeze({ kind: 'keepRunning' as const });
|
|
20
|
+
}
|
|
21
|
+
/** The first failed cell cancels the cells still running or waiting. */
|
|
22
|
+
export function cascadeFailure(): FailureStrategy {
|
|
23
|
+
return Object.freeze({ kind: 'cascadeFailure' as const });
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface MatrixDef<Ax extends Axes> {
|
|
27
|
+
/** The cells are the cross product of the axes, first axis outermost. */
|
|
28
|
+
axes: Ax;
|
|
29
|
+
/** Each entry names axes with one of their values; a cell matching all of them is left out. */
|
|
30
|
+
exclude?: readonly NoInfer<Partial<CellOfAxes<Ax>>>[];
|
|
31
|
+
/** Default `keepRunning()`. */
|
|
32
|
+
failure?: FailureStrategy;
|
|
33
|
+
/** At most this many of the step's cells running at once in a run. */
|
|
34
|
+
maxParallel?: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** A matrix step's matrix, as the step carries it. */
|
|
38
|
+
export interface Matrix<Ax extends Axes = Axes> {
|
|
39
|
+
readonly axes: Ax;
|
|
40
|
+
readonly exclude: readonly Cell[];
|
|
41
|
+
readonly failure: FailureStrategy;
|
|
42
|
+
readonly maxParallel?: number;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export const AXIS_NAME = /^[a-z][a-zA-Z0-9]{0,30}$/;
|
|
46
|
+
export const AXIS_VALUE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$/;
|
|
47
|
+
export const MAX_AXES = 4;
|
|
48
|
+
export const MAX_CELLS = 64;
|
|
49
|
+
|
|
50
|
+
export function makeMatrix(def: MatrixDef<Axes>): Matrix {
|
|
51
|
+
return Object.freeze({
|
|
52
|
+
axes: Object.freeze(
|
|
53
|
+
Object.fromEntries(
|
|
54
|
+
Object.entries(def.axes ?? {}).map(([k, v]) => [
|
|
55
|
+
k,
|
|
56
|
+
Object.freeze(Array.isArray(v) ? [...v] : v),
|
|
57
|
+
]),
|
|
58
|
+
),
|
|
59
|
+
),
|
|
60
|
+
exclude: Object.freeze((def.exclude ?? []).map((e) => Object.freeze({ ...e }))),
|
|
61
|
+
failure: def.failure ?? keepRunning(),
|
|
62
|
+
...(def.maxParallel !== undefined ? { maxParallel: def.maxParallel } : {}),
|
|
63
|
+
}) as Matrix;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const matches = (cell: Cell, x: Cell) =>
|
|
67
|
+
Object.entries(x).every(([k, v]) => cell[k] === v);
|
|
68
|
+
|
|
69
|
+
/** The cells in order: the cross product in axis order (first axis outermost) minus the excludes. */
|
|
70
|
+
export function matrixCells(m: Matrix): Cell[] {
|
|
71
|
+
let cells: Record<string, string>[] = [{}];
|
|
72
|
+
for (const [axis, values] of Object.entries(m.axes))
|
|
73
|
+
cells = cells.flatMap((c) =>
|
|
74
|
+
(Array.isArray(values) ? values : []).map((v) => ({ ...c, [axis]: v })),
|
|
75
|
+
);
|
|
76
|
+
return cells.filter((c) => !m.exclude.some((x) => matches(c, x)));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** `build-cli[os=linux,arch=x64]`: the name a cell's attempts, files and address use. */
|
|
80
|
+
export function cellName(step: string, cell: Cell): string {
|
|
81
|
+
return `${step}[${Object.entries(cell)
|
|
82
|
+
.map(([k, v]) => `${k}=${v}`)
|
|
83
|
+
.join(',')}]`;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** The cell `raw` (JSON text, as in `OLVE_CELL`) names, in axis order, or what is wrong. */
|
|
87
|
+
export function parseCell(
|
|
88
|
+
step: string,
|
|
89
|
+
m: Matrix,
|
|
90
|
+
raw: string | undefined,
|
|
91
|
+
): { ok: true; cell: Cell } | { ok: false; message: string } {
|
|
92
|
+
const axes = Object.keys(m.axes);
|
|
93
|
+
const example = JSON.stringify(matrixCells(m)[0] ?? {});
|
|
94
|
+
if (raw === undefined || raw === '')
|
|
95
|
+
return {
|
|
96
|
+
ok: false,
|
|
97
|
+
message: `step '${step}' is a matrix step: OLVE_CELL must name the cell to run, e.g. ${example}`,
|
|
98
|
+
};
|
|
99
|
+
let v: unknown;
|
|
100
|
+
try {
|
|
101
|
+
v = JSON.parse(raw);
|
|
102
|
+
} catch (e) {
|
|
103
|
+
return { ok: false, message: `OLVE_CELL is not JSON: ${(e as Error).message}` };
|
|
104
|
+
}
|
|
105
|
+
if (typeof v !== 'object' || v === null || Array.isArray(v))
|
|
106
|
+
return {
|
|
107
|
+
ok: false,
|
|
108
|
+
message: `OLVE_CELL must be a JSON object of axis values, e.g. ${example}`,
|
|
109
|
+
};
|
|
110
|
+
const given = v as Record<string, unknown>;
|
|
111
|
+
const ordered: Record<string, string> = {};
|
|
112
|
+
for (const a of axes) {
|
|
113
|
+
if (typeof given[a] !== 'string') break;
|
|
114
|
+
ordered[a] = given[a] as string;
|
|
115
|
+
}
|
|
116
|
+
const exact =
|
|
117
|
+
Object.keys(given).length === axes.length &&
|
|
118
|
+
Object.keys(ordered).length === axes.length;
|
|
119
|
+
const all = matrixCells(m);
|
|
120
|
+
if (exact && all.some((c) => cellName(step, c) === cellName(step, ordered)))
|
|
121
|
+
return { ok: true, cell: Object.freeze(ordered) };
|
|
122
|
+
return {
|
|
123
|
+
ok: false,
|
|
124
|
+
message: `OLVE_CELL ${raw} is not a cell of step '${step}' (cells: ${all.map((c) => cellName(step, c)).join(', ')})`,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// ─── cells(): reading a matrix step ──────────────────────────────────────────────────────────
|
|
129
|
+
|
|
130
|
+
/** A step input that reads every cell of a matrix step whose output is `model`. */
|
|
131
|
+
export interface Cells<M extends AnyModel = AnyModel> {
|
|
132
|
+
readonly kind: 'cells';
|
|
133
|
+
readonly model: M;
|
|
134
|
+
}
|
|
135
|
+
export function cells<M extends AnyModel>(model: M): Cells<M> {
|
|
136
|
+
return Object.freeze({ kind: 'cells' as const, model });
|
|
137
|
+
}
|
|
138
|
+
export function isCells(v: unknown): v is Cells {
|
|
139
|
+
return (
|
|
140
|
+
typeof v === 'object' &&
|
|
141
|
+
v !== null &&
|
|
142
|
+
(v as { kind?: unknown }).kind === 'cells' &&
|
|
143
|
+
isModel((v as { model?: unknown }).model)
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
/** One entry of a `cells` input, in cell order. */
|
|
147
|
+
export interface CellOutput<M extends AnyModel, C extends Cell = Cell> {
|
|
148
|
+
readonly cell: C;
|
|
149
|
+
readonly output: Infer<M>;
|
|
150
|
+
}
|
package/src/core/models.ts
CHANGED
|
@@ -48,11 +48,16 @@ export const FILE_PATH_PATTERN = `^(?:${SEGMENT}/)*${SEGMENT}/?$`;
|
|
|
48
48
|
export const FILE_PATH = new RegExp(FILE_PATH_PATTERN);
|
|
49
49
|
/** Step names: lower-case kebab, ≤ 63 chars (usable as K8s label values, URL segments and storage paths). */
|
|
50
50
|
export const STEP_NAME = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
|
|
51
|
+
const CELL_PART = String.raw`[a-z][a-zA-Z0-9]{0,30}=[A-Za-z0-9][A-Za-z0-9._-]{0,62}`;
|
|
52
|
+
/** A step name, or a matrix cell's name (`build-cli[os=linux,arch=x64]`): what a file reference's `step` holds. */
|
|
53
|
+
export const ATTEMPT_STEP = new RegExp(
|
|
54
|
+
String.raw`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\[${CELL_PART}(?:,${CELL_PART})*\])?$`,
|
|
55
|
+
);
|
|
51
56
|
|
|
52
57
|
const FileRefSchema = z
|
|
53
58
|
.object({
|
|
54
59
|
run: z.number().int().nonnegative(),
|
|
55
|
-
step: z.string().regex(
|
|
60
|
+
step: z.string().regex(ATTEMPT_STEP),
|
|
56
61
|
attempt: z.number().int().positive(),
|
|
57
62
|
path: z.string().regex(FILE_PATH),
|
|
58
63
|
sha256: z.string().regex(/^[0-9a-f]{64}$/),
|
package/src/core/steps.ts
CHANGED
|
@@ -6,24 +6,35 @@ import type { AnyConfigEntry } from './config.ts';
|
|
|
6
6
|
import { configRefsIn } from './config-refs.ts';
|
|
7
7
|
import type { Secret } from './secrets.ts';
|
|
8
8
|
import type { Simplify } from './types.ts';
|
|
9
|
+
import {
|
|
10
|
+
makeMatrix,
|
|
11
|
+
type Axes,
|
|
12
|
+
type CellOfAxes,
|
|
13
|
+
type CellOutput,
|
|
14
|
+
type Cells,
|
|
15
|
+
type Matrix,
|
|
16
|
+
type MatrixDef,
|
|
17
|
+
} from './matrix.ts';
|
|
9
18
|
|
|
10
19
|
/**
|
|
11
|
-
* A step input entry: a model (matchable by model), `optional(model)`,
|
|
12
|
-
* or `external(model, …)`, answered from outside the run.
|
|
20
|
+
* A step input entry: a model (matchable by model), `optional(model)`, `cells(model)` (every cell of a
|
|
21
|
+
* matrix step), a plain zod schema (mapper required), or `external(model, …)`, answered from outside the run.
|
|
13
22
|
*/
|
|
14
|
-
export type InputEntry = AnyModel | Optional | External | z.ZodType;
|
|
23
|
+
export type InputEntry = AnyModel | Optional | Cells | External | z.ZodType;
|
|
15
24
|
export type InputSpec = { readonly [key: string]: InputEntry };
|
|
16
25
|
|
|
17
26
|
export type InferEntry<E> =
|
|
18
27
|
E extends Optional<infer M>
|
|
19
28
|
? Infer<M> | undefined
|
|
20
|
-
: E extends
|
|
21
|
-
?
|
|
22
|
-
: E extends
|
|
23
|
-
? Infer<
|
|
24
|
-
: E extends
|
|
25
|
-
?
|
|
26
|
-
:
|
|
29
|
+
: E extends Cells<infer M>
|
|
30
|
+
? CellOutput<M>[]
|
|
31
|
+
: E extends External<infer M>
|
|
32
|
+
? Infer<M>
|
|
33
|
+
: E extends AnyModel
|
|
34
|
+
? Infer<E>
|
|
35
|
+
: E extends z.ZodType
|
|
36
|
+
? z.output<E>
|
|
37
|
+
: never;
|
|
27
38
|
/** What a handler receives: the output view of every input entry. */
|
|
28
39
|
export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
|
|
29
40
|
|
|
@@ -56,6 +67,8 @@ export interface Step<
|
|
|
56
67
|
readonly clusterAccess?: boolean;
|
|
57
68
|
/** How the step is set up, as written in the definition (may hold config references). */
|
|
58
69
|
readonly settings?: Readonly<Record<string, unknown>>;
|
|
70
|
+
/** Set on a matrix step: it runs once per cell. */
|
|
71
|
+
readonly matrix?: Matrix;
|
|
59
72
|
}
|
|
60
73
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
61
74
|
export type AnyStep = Step<string, any, AnyModel | null, any, any>;
|
|
@@ -64,6 +77,10 @@ export type AnyStep = Step<string, any, AnyModel | null, any, any>;
|
|
|
64
77
|
export type SecretsOf<S> = S extends Step<any, any, any, infer Sec, any> ? Sec : never;
|
|
65
78
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
66
79
|
export type ConfigOf<S> = S extends Step<any, any, any, any, infer C> ? C : never;
|
|
80
|
+
/** The cell type of a matrix step (`never` for any other step). */
|
|
81
|
+
export type CellOf<S> = S extends { readonly matrix: Matrix<infer Ax> }
|
|
82
|
+
? CellOfAxes<Ax>
|
|
83
|
+
: never;
|
|
67
84
|
export type OutputValue<S extends AnyStep> = S['output'] extends AnyModel
|
|
68
85
|
? Infer<S['output']>
|
|
69
86
|
: void;
|
|
@@ -82,6 +99,7 @@ export interface StepDef<
|
|
|
82
99
|
O extends AnyModel | null,
|
|
83
100
|
Sec extends readonly Secret[],
|
|
84
101
|
Cfg extends readonly AnyConfigEntry[] = [],
|
|
102
|
+
Ax extends Axes = never,
|
|
85
103
|
> {
|
|
86
104
|
name: LiteralName<N>;
|
|
87
105
|
/** What the step does, shown in the UI. Required; may be empty. */
|
|
@@ -109,6 +127,8 @@ export interface StepDef<
|
|
|
109
127
|
* references is delivered to the step, and `resolveConfigRefs(value, ctx.config)` resolves them.
|
|
110
128
|
*/
|
|
111
129
|
settings?: Readonly<Record<string, unknown>>;
|
|
130
|
+
/** Run the step once per cell of these axes; the handler reads its cell from `ctx.cell`. */
|
|
131
|
+
matrix?: MatrixDef<Ax>;
|
|
112
132
|
}
|
|
113
133
|
|
|
114
134
|
function makeStep(def: Record<string, unknown>): AnyStep {
|
|
@@ -130,6 +150,9 @@ function makeStep(def: Record<string, unknown>): AnyStep {
|
|
|
130
150
|
...(def.retries !== undefined ? { retries: def.retries } : {}),
|
|
131
151
|
...(def.clusterAccess === true ? { clusterAccess: true } : {}),
|
|
132
152
|
...(def.settings !== undefined ? { settings: def.settings } : {}),
|
|
153
|
+
...(def.matrix !== undefined
|
|
154
|
+
? { matrix: makeMatrix(def.matrix as MatrixDef<Axes>) }
|
|
155
|
+
: {}),
|
|
133
156
|
};
|
|
134
157
|
return Object.freeze(s) as unknown as AnyStep;
|
|
135
158
|
}
|
|
@@ -150,6 +173,10 @@ export function step<
|
|
|
150
173
|
O extends AnyModel | null = null,
|
|
151
174
|
const Sec extends readonly Secret[] = [],
|
|
152
175
|
const Cfg extends readonly AnyConfigEntry[] = [],
|
|
153
|
-
|
|
176
|
+
const Ax extends Axes = never,
|
|
177
|
+
>(
|
|
178
|
+
def: StepDef<N, I, O, Sec, Cfg, Ax>,
|
|
179
|
+
): Step<N, I, O, Sec[number]['name'], Cfg[number]> &
|
|
180
|
+
([Ax] extends [never] ? unknown : { readonly matrix: Matrix<Ax> }) {
|
|
154
181
|
return makeStep(def as never) as never;
|
|
155
182
|
}
|
package/src/ctx/index.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { resolve } from 'node:path';
|
|
|
3
3
|
import { skipped } from '../core/handlers.ts';
|
|
4
4
|
import { createLinks } from './links.ts';
|
|
5
5
|
import type { Secret } from '../core/secrets.ts';
|
|
6
|
+
import type { Cell } from '../core/matrix.ts';
|
|
6
7
|
import { createFetch, createOutput, redactUrl } from './files.ts';
|
|
7
8
|
import { createSh } from './sh.ts';
|
|
8
9
|
import type { Ctx, Http, RetryOptions, RunInfo } from './types.ts';
|
|
@@ -21,6 +22,8 @@ export interface CtxOptions {
|
|
|
21
22
|
logSink?: (line: string) => void;
|
|
22
23
|
/** `<step>/<input>` → callback URL (`ctx.inputUrl`). */
|
|
23
24
|
callbackUrls?: Readonly<Record<string, string>>;
|
|
25
|
+
/** A matrix step's cell (`ctx.cell`). */
|
|
26
|
+
cell?: Cell;
|
|
24
27
|
}
|
|
25
28
|
|
|
26
29
|
export function formatLog(message: string, fields?: Record<string, unknown>): string {
|
|
@@ -151,6 +154,7 @@ export function createCtx(o: CtxOptions): Ctx {
|
|
|
151
154
|
log,
|
|
152
155
|
};
|
|
153
156
|
return {
|
|
157
|
+
...(o.cell ? { cell: o.cell } : {}),
|
|
154
158
|
secrets: createSecrets(o.secrets, o.secretValues) as Ctx['secrets'],
|
|
155
159
|
config: Object.freeze({ ...o.config }) as Ctx['config'],
|
|
156
160
|
sh,
|
package/src/ctx/types.ts
CHANGED
|
@@ -124,3 +124,10 @@ export interface Ctx<Sec extends string = string, Cfg extends AnyConfigEntry = n
|
|
|
124
124
|
*/
|
|
125
125
|
inputUrl(step: string | { readonly name: string }, input: string): string;
|
|
126
126
|
}
|
|
127
|
+
|
|
128
|
+
/** The ctx a step's handler gets: a matrix step's also carries its cell (`ctx.cell`), in axis order. */
|
|
129
|
+
export type StepCtx<Sec extends string, Cfg extends AnyConfigEntry, Cell> = Ctx<
|
|
130
|
+
Sec,
|
|
131
|
+
Cfg
|
|
132
|
+
> &
|
|
133
|
+
([Cell] extends [never] ? unknown : { readonly cell: Cell });
|
package/src/describe/context.ts
CHANGED
|
@@ -12,10 +12,11 @@ import { schemaViews, type JsonSchema } from './schemas.ts';
|
|
|
12
12
|
export const FAILURE_ROOT = 'failure';
|
|
13
13
|
const RESERVED = new Set([...RESERVED_STEP_NAMES, FAILURE_ROOT]);
|
|
14
14
|
|
|
15
|
-
/** What a step can read at a point of the pipeline: its output model,
|
|
15
|
+
/** What a step can read at a point of the pipeline: its output model, whether it may have been skipped, and whether it is a matrix step. */
|
|
16
16
|
export interface Avail {
|
|
17
17
|
model: string | null;
|
|
18
18
|
maybe: boolean;
|
|
19
|
+
matrix?: boolean;
|
|
19
20
|
}
|
|
20
21
|
|
|
21
22
|
/** A select case a step sits in: which select (numbered in describe order), which value, and its condition text. */
|
|
@@ -154,6 +155,10 @@ export class DescribeContext {
|
|
|
154
155
|
problems.push(
|
|
155
156
|
`step '${s.name}': its select cases must give it the same output model`,
|
|
156
157
|
);
|
|
158
|
+
else if (prev.some((p) => !p.step.matrix !== !s.matrix))
|
|
159
|
+
problems.push(
|
|
160
|
+
`step '${s.name}': its select cases must all make it a matrix step, or none`,
|
|
161
|
+
);
|
|
157
162
|
this.placements.set(s.name, [...prev, { step: s, cases }]);
|
|
158
163
|
if (!this.steps.has(s.name)) this.steps.set(s.name, s);
|
|
159
164
|
}
|
|
@@ -76,6 +76,12 @@ export const InputType = z
|
|
|
76
76
|
optional: z
|
|
77
77
|
.boolean()
|
|
78
78
|
.describe('optional(Model): may be absent because its producer was skipped.'),
|
|
79
|
+
cells: z
|
|
80
|
+
.literal(true)
|
|
81
|
+
.optional()
|
|
82
|
+
.describe(
|
|
83
|
+
'cells(Model): every cell of a matrix step whose output is the model, mapped with a whole-step ref; the value is [{cell, output}, …] in cell order. Omitted otherwise.',
|
|
84
|
+
),
|
|
79
85
|
}),
|
|
80
86
|
z.strictObject({
|
|
81
87
|
schema: ModelSchemas.describe(
|
|
@@ -85,6 +91,45 @@ export const InputType = z
|
|
|
85
91
|
])
|
|
86
92
|
.meta({ id: 'InputType' });
|
|
87
93
|
|
|
94
|
+
const axisName = z.string().regex(/^[a-z][a-zA-Z0-9]{0,30}$/);
|
|
95
|
+
const axisValue = z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$/);
|
|
96
|
+
|
|
97
|
+
export const Matrix = z
|
|
98
|
+
.strictObject({
|
|
99
|
+
axes: z
|
|
100
|
+
.array(
|
|
101
|
+
z.strictObject({
|
|
102
|
+
name: axisName,
|
|
103
|
+
values: z.array(axisValue).min(1).describe('Unique per axis.'),
|
|
104
|
+
}),
|
|
105
|
+
)
|
|
106
|
+
.min(1)
|
|
107
|
+
.max(4)
|
|
108
|
+
.describe(
|
|
109
|
+
'In order: the cells are the cross product, first axis outermost, minus the excludes (1 to 64 cells).',
|
|
110
|
+
),
|
|
111
|
+
exclude: z
|
|
112
|
+
.array(z.record(axisName, axisValue))
|
|
113
|
+
.optional()
|
|
114
|
+
.describe(
|
|
115
|
+
'Each entry names one or more axes with one of their values; a cell matching every named axis is left out (omitted when none).',
|
|
116
|
+
),
|
|
117
|
+
failure: z
|
|
118
|
+
.strictObject({ kind: z.enum(['keepRunning', 'cascadeFailure']) })
|
|
119
|
+
.describe(
|
|
120
|
+
"keepRunning: every cell runs to the end. cascadeFailure: a cell whose last allowed attempt failed cancels the step's other scheduled or running cells.",
|
|
121
|
+
),
|
|
122
|
+
maxParallel: z
|
|
123
|
+
.int()
|
|
124
|
+
.min(1)
|
|
125
|
+
.optional()
|
|
126
|
+
.describe("At most this many of the step's cells running at once in a run."),
|
|
127
|
+
})
|
|
128
|
+
.describe(
|
|
129
|
+
"A matrix step runs once per cell, named '<step>[<axis>=<value>,…]' in axis order; OLVE_CELL holds the cell as compact JSON.",
|
|
130
|
+
)
|
|
131
|
+
.meta({ id: 'Matrix' });
|
|
132
|
+
|
|
88
133
|
export const StepDescriptor = z
|
|
89
134
|
.strictObject({
|
|
90
135
|
name: stepName,
|
|
@@ -114,6 +159,9 @@ export const StepDescriptor = z
|
|
|
114
159
|
),
|
|
115
160
|
timeoutSeconds: z.number().int().positive().optional(),
|
|
116
161
|
retries: z.number().int().nonnegative().optional(),
|
|
162
|
+
matrix: Matrix.optional().describe(
|
|
163
|
+
'Set on a matrix step (never on a failure handler); omitted otherwise.',
|
|
164
|
+
),
|
|
117
165
|
config: z
|
|
118
166
|
.record(z.string(), Json)
|
|
119
167
|
.optional()
|
package/src/describe/stages.ts
CHANGED
|
@@ -54,6 +54,7 @@ function describeStage(
|
|
|
54
54
|
avail.set(ss.step.name, {
|
|
55
55
|
model: ss.step.output?.name ?? null,
|
|
56
56
|
maybe: false,
|
|
57
|
+
...(ss.step.matrix ? { matrix: true } : {}),
|
|
57
58
|
});
|
|
58
59
|
return descs.length ? { steps: descs } : undefined;
|
|
59
60
|
}
|
|
@@ -96,7 +97,7 @@ function describeWhen(
|
|
|
96
97
|
const inner = new Map(avail);
|
|
97
98
|
const stages = describeStages(ctx, input, n.nodes, inner, cases);
|
|
98
99
|
for (const [name, a] of inner)
|
|
99
|
-
if (!avail.has(name)) avail.set(name, {
|
|
100
|
+
if (!avail.has(name)) avail.set(name, { ...a, maybe: true });
|
|
100
101
|
return { when, stages };
|
|
101
102
|
}
|
|
102
103
|
|
|
@@ -153,7 +154,7 @@ function describeSelect(
|
|
|
153
154
|
.map((b) => b.inner.get(name))
|
|
154
155
|
.filter((a): a is Avail => a !== undefined);
|
|
155
156
|
avail.set(name, {
|
|
156
|
-
|
|
157
|
+
...present[0]!,
|
|
157
158
|
maybe: present.length < branches.length || present.some((a) => a.maybe),
|
|
158
159
|
});
|
|
159
160
|
}
|
package/src/describe/step.ts
CHANGED
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { isModel, isOptional } from '../core/models.ts';
|
|
3
3
|
import { isExternal } from '../core/external.ts';
|
|
4
|
+
import {
|
|
5
|
+
AXIS_NAME,
|
|
6
|
+
AXIS_VALUE,
|
|
7
|
+
MAX_AXES,
|
|
8
|
+
MAX_CELLS,
|
|
9
|
+
isCells,
|
|
10
|
+
matrixCells,
|
|
11
|
+
type Matrix,
|
|
12
|
+
} from '../core/matrix.ts';
|
|
4
13
|
import type { AnyStep } from '../core/steps.ts';
|
|
5
14
|
import { isImplementation } from '../core/handlers.ts';
|
|
6
15
|
import { recordConfigRefs } from '../core/config-refs.ts';
|
|
@@ -20,6 +29,7 @@ import {
|
|
|
20
29
|
} from './context.ts';
|
|
21
30
|
|
|
22
31
|
type InputTypes = StepDescriptorT['inputTypes'];
|
|
32
|
+
type MatrixT = NonNullable<StepDescriptorT['matrix']>;
|
|
23
33
|
type ExternalInputs = NonNullable<StepDescriptorT['externalInputs']>;
|
|
24
34
|
type InputEntries = [string, unknown][];
|
|
25
35
|
|
|
@@ -77,6 +87,9 @@ export function describeStep(
|
|
|
77
87
|
problems.push(`${at}: timeoutSeconds must be a positive integer`);
|
|
78
88
|
if (s.retries !== undefined && !(Number.isInteger(s.retries) && s.retries >= 0))
|
|
79
89
|
problems.push(`${at}: retries must be a non-negative integer`);
|
|
90
|
+
if (s.matrix !== undefined && failureHandler)
|
|
91
|
+
problems.push(`${at}: a failure handler cannot be a matrix step`);
|
|
92
|
+
const matrix = s.matrix !== undefined ? describeMatrix(ctx, at, s.matrix) : undefined;
|
|
80
93
|
|
|
81
94
|
const out: StepDescriptorT = {
|
|
82
95
|
name: s.name,
|
|
@@ -89,6 +102,7 @@ export function describeStep(
|
|
|
89
102
|
secrets: secretNames,
|
|
90
103
|
...(s.timeoutSeconds !== undefined ? { timeoutSeconds: s.timeoutSeconds } : {}),
|
|
91
104
|
...(s.retries !== undefined ? { retries: s.retries } : {}),
|
|
105
|
+
...(matrix !== undefined ? { matrix } : {}),
|
|
92
106
|
...(settings !== undefined
|
|
93
107
|
? { config: settings as Record<string, z.core.util.JSONType> }
|
|
94
108
|
: {}),
|
|
@@ -156,7 +170,13 @@ function describeInputTypes(
|
|
|
156
170
|
): InputTypes {
|
|
157
171
|
const inputTypes: InputTypes = {};
|
|
158
172
|
for (const [key, e] of entries) {
|
|
159
|
-
if (
|
|
173
|
+
if (isCells(e))
|
|
174
|
+
inputTypes[key] = {
|
|
175
|
+
model: ctx.useModel(e.model, `${at} input '${key}'`),
|
|
176
|
+
optional: false,
|
|
177
|
+
cells: true,
|
|
178
|
+
};
|
|
179
|
+
else if (isOptional(e))
|
|
160
180
|
inputTypes[key] = {
|
|
161
181
|
model: ctx.useModel(e.model, `${at} input '${key}'`),
|
|
162
182
|
optional: true,
|
|
@@ -176,7 +196,7 @@ function describeInputTypes(
|
|
|
176
196
|
}
|
|
177
197
|
} else
|
|
178
198
|
ctx.problems.push(
|
|
179
|
-
`${at} input '${key}': must be a model, optional(model) or a zod schema`,
|
|
199
|
+
`${at} input '${key}': must be a model, optional(model), cells(model) or a zod schema`,
|
|
180
200
|
);
|
|
181
201
|
}
|
|
182
202
|
return inputTypes;
|
|
@@ -209,6 +229,25 @@ function matchInputsByModel(
|
|
|
209
229
|
const problems = ctx.problems;
|
|
210
230
|
const input: Record<string, Mapping> = {};
|
|
211
231
|
for (const [key, e] of entries) {
|
|
232
|
+
if (isCells(e)) {
|
|
233
|
+
const producers = [...avail].filter(
|
|
234
|
+
([, a]) => a.matrix && a.model === e.model.name,
|
|
235
|
+
);
|
|
236
|
+
if (producers.length === 1) {
|
|
237
|
+
const [producer, a] = producers[0]!;
|
|
238
|
+
if (a.maybe) problems.push(maybeCells(at, e.model.name, key));
|
|
239
|
+
input[key] = { ref: producer };
|
|
240
|
+
} else
|
|
241
|
+
problems.push(
|
|
242
|
+
producers.length === 0
|
|
243
|
+
? `${at}: ${e.model.name} (input '${key}') has no matrix step producing it at this stage; cells() reads a matrix step`
|
|
244
|
+
: `${at}: ${e.model.name} (input '${key}') has several matrix steps producing it (${producers
|
|
245
|
+
.map(([n]) => n)
|
|
246
|
+
.sort()
|
|
247
|
+
.join(', ')}); use a mapper to pick one`,
|
|
248
|
+
);
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
212
251
|
const optional = isOptional(e);
|
|
213
252
|
const m = optional ? e.model : isModel(e) ? e : undefined;
|
|
214
253
|
if (!m) {
|
|
@@ -217,8 +256,13 @@ function matchInputsByModel(
|
|
|
217
256
|
);
|
|
218
257
|
continue;
|
|
219
258
|
}
|
|
220
|
-
const producers = [...avail].filter(([, a]) => a.model === m.name);
|
|
221
|
-
|
|
259
|
+
const producers = [...avail].filter(([, a]) => !a.matrix && a.model === m.name);
|
|
260
|
+
const matrixProducer = [...avail].find(([, a]) => a.matrix && a.model === m.name);
|
|
261
|
+
if (producers.length === 0 && matrixProducer) {
|
|
262
|
+
problems.push(
|
|
263
|
+
`${at}: ${m.name} (input '${key}') is produced by matrix step '${matrixProducer[0]}'; read it with cells(${m.name})`,
|
|
264
|
+
);
|
|
265
|
+
} else if (producers.length === 0) {
|
|
222
266
|
problems.push(
|
|
223
267
|
`${at}: ${m.name} (input '${key}') is not available at this stage${optional ? ' (optional() means maybe-skipped; there must still be an earlier producer)' : ''}`,
|
|
224
268
|
);
|
|
@@ -273,11 +317,47 @@ function mapStep(
|
|
|
273
317
|
for (const k of Object.keys(recorded))
|
|
274
318
|
if (!(k in declared))
|
|
275
319
|
problems.push(`${at}: mapper provides '${k}', which is not an input of the step`);
|
|
276
|
-
for (const [k, m] of Object.entries(recorded))
|
|
277
|
-
|
|
320
|
+
for (const [k, m] of Object.entries(recorded)) {
|
|
321
|
+
const e = declared[k];
|
|
322
|
+
if (isCells(e)) checkCellsMapping(ctx, `${at} input '${k}'`, m, avail, e.model.name);
|
|
323
|
+
else checkMapping(ctx, `${at} input '${k}'`, m, avail, isOptional(e), true);
|
|
324
|
+
}
|
|
278
325
|
return recorded;
|
|
279
326
|
}
|
|
280
327
|
|
|
328
|
+
const maybeCells = (at: string, model: string, key: string) =>
|
|
329
|
+
`${at}: ${model} (input '${key}') is only maybe-available (produced inside a when block); cells() cannot read a maybe-skipped matrix step`;
|
|
330
|
+
|
|
331
|
+
function checkCellsMapping(
|
|
332
|
+
ctx: DescribeContext,
|
|
333
|
+
at: string,
|
|
334
|
+
m: Mapping,
|
|
335
|
+
avail: Map<string, Avail>,
|
|
336
|
+
model: string,
|
|
337
|
+
): void {
|
|
338
|
+
const problems = ctx.problems;
|
|
339
|
+
const ref =
|
|
340
|
+
Object.keys(m).length === 1 && typeof (m as { ref?: unknown }).ref === 'string'
|
|
341
|
+
? (m as { ref: string }).ref
|
|
342
|
+
: undefined;
|
|
343
|
+
if (ref === undefined || ref.includes('.'))
|
|
344
|
+
return void problems.push(
|
|
345
|
+
`${at}: a cells() input takes a whole matrix step (from['<step>']), not a field or an object`,
|
|
346
|
+
);
|
|
347
|
+
const a = avail.get(ref);
|
|
348
|
+
if (!a) return void problems.push(`${at}: '${ref}' is not available at this stage`);
|
|
349
|
+
if (!a.matrix)
|
|
350
|
+
return void problems.push(
|
|
351
|
+
`${at}: '${ref}' is not a matrix step; cells() reads a matrix step`,
|
|
352
|
+
);
|
|
353
|
+
if (a.model !== model)
|
|
354
|
+
problems.push(`${at}: '${ref}' outputs ${a.model ?? 'nothing'}, not ${model}`);
|
|
355
|
+
if (a.maybe)
|
|
356
|
+
problems.push(
|
|
357
|
+
`${at}: '${ref}' is only maybe-available (produced inside a when block); cells() cannot read a maybe-skipped matrix step`,
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
|
|
281
361
|
function checkMapping(
|
|
282
362
|
ctx: DescribeContext,
|
|
283
363
|
at: string,
|
|
@@ -299,6 +379,10 @@ function checkMapping(
|
|
|
299
379
|
];
|
|
300
380
|
const a = avail.get(root);
|
|
301
381
|
if (!a) return void problems.push(`${at}: '${root}' is not available at this stage`);
|
|
382
|
+
if (a.matrix)
|
|
383
|
+
return void problems.push(
|
|
384
|
+
`${at}: '${(m as { ref: string }).ref}' — '${root}' is a matrix step; read it whole through a cells() input`,
|
|
385
|
+
);
|
|
302
386
|
if (!resolveField(ctx.outputViewOf(a.model), path))
|
|
303
387
|
problems.push(
|
|
304
388
|
`${at}: '${(m as { ref: string }).ref}' — model ${a.model} has no field '${path.join('.')}'`,
|
|
@@ -393,3 +477,68 @@ function checkConfigRefs(
|
|
|
393
477
|
);
|
|
394
478
|
}
|
|
395
479
|
}
|
|
480
|
+
|
|
481
|
+
function describeMatrix(
|
|
482
|
+
ctx: DescribeContext,
|
|
483
|
+
at: string,
|
|
484
|
+
m: Matrix,
|
|
485
|
+
): MatrixT | undefined {
|
|
486
|
+
const problems = ctx.problems;
|
|
487
|
+
const before = problems.length;
|
|
488
|
+
const axes = Object.entries(
|
|
489
|
+
typeof m.axes === 'object' && m.axes !== null ? m.axes : {},
|
|
490
|
+
) as [string, unknown][];
|
|
491
|
+
if (axes.length < 1 || axes.length > MAX_AXES)
|
|
492
|
+
problems.push(
|
|
493
|
+
`${at}: a matrix has 1 to ${MAX_AXES} axes (this one has ${axes.length})`,
|
|
494
|
+
);
|
|
495
|
+
for (const [name, values] of axes) {
|
|
496
|
+
if (!AXIS_NAME.test(name))
|
|
497
|
+
problems.push(`${at}: matrix axis name '${name}' must match ${AXIS_NAME}`);
|
|
498
|
+
if (!Array.isArray(values) || values.some((v) => typeof v !== 'string')) {
|
|
499
|
+
problems.push(`${at}: matrix axis '${name}' must be a list of strings`);
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
if (values.length === 0) problems.push(`${at}: matrix axis '${name}' has no values`);
|
|
503
|
+
for (const v of values as string[])
|
|
504
|
+
if (!AXIS_VALUE.test(v))
|
|
505
|
+
problems.push(
|
|
506
|
+
`${at}: matrix axis '${name}': value '${v}' must match ${AXIS_VALUE}`,
|
|
507
|
+
);
|
|
508
|
+
for (const v of new Set(
|
|
509
|
+
(values as string[]).filter((v, i) => values.indexOf(v) !== i),
|
|
510
|
+
))
|
|
511
|
+
problems.push(`${at}: matrix axis '${name}': value '${v}' is listed twice`);
|
|
512
|
+
}
|
|
513
|
+
const known = new Map(axes);
|
|
514
|
+
m.exclude.forEach((x, i) => {
|
|
515
|
+
const where = `${at}: matrix exclude ${i + 1}`;
|
|
516
|
+
if (Object.keys(x).length === 0)
|
|
517
|
+
problems.push(`${where} must name at least one axis`);
|
|
518
|
+
for (const [k, v] of Object.entries(x)) {
|
|
519
|
+
const values = known.get(k);
|
|
520
|
+
if (!values) problems.push(`${where}: '${k}' is not an axis`);
|
|
521
|
+
else if (!(Array.isArray(values) && values.includes(v)))
|
|
522
|
+
problems.push(`${where}: '${String(v)}' is not a value of axis '${k}'`);
|
|
523
|
+
}
|
|
524
|
+
});
|
|
525
|
+
const kind = (m.failure as { kind?: unknown } | undefined)?.kind;
|
|
526
|
+
if (kind !== 'keepRunning' && kind !== 'cascadeFailure')
|
|
527
|
+
problems.push(`${at}: matrix failure must be keepRunning() or cascadeFailure()`);
|
|
528
|
+
if (
|
|
529
|
+
m.maxParallel !== undefined &&
|
|
530
|
+
!(Number.isInteger(m.maxParallel) && m.maxParallel >= 1)
|
|
531
|
+
)
|
|
532
|
+
problems.push(`${at}: matrix maxParallel must be a positive integer`);
|
|
533
|
+
if (problems.length > before) return undefined;
|
|
534
|
+
const count = matrixCells(m).length;
|
|
535
|
+
if (count === 0) problems.push(`${at}: the matrix excludes every cell`);
|
|
536
|
+
else if (count > MAX_CELLS)
|
|
537
|
+
problems.push(`${at}: the matrix has ${count} cells (at most ${MAX_CELLS})`);
|
|
538
|
+
return {
|
|
539
|
+
axes: axes.map(([name, values]) => ({ name, values: [...(values as string[])] })),
|
|
540
|
+
...(m.exclude.length ? { exclude: m.exclude.map((x) => ({ ...x })) } : {}),
|
|
541
|
+
failure: { kind: kind as MatrixT['failure']['kind'] },
|
|
542
|
+
...(m.maxParallel !== undefined ? { maxParallel: m.maxParallel } : {}),
|
|
543
|
+
};
|
|
544
|
+
}
|
package/src/harness/env.ts
CHANGED
|
@@ -9,6 +9,8 @@ export const ENV = {
|
|
|
9
9
|
run: 'OLVE_RUN',
|
|
10
10
|
attempt: 'OLVE_ATTEMPT',
|
|
11
11
|
versionStamp: 'OLVE_VERSION_STAMP',
|
|
12
|
+
/** A matrix step's cell, as compact JSON in axis order (`{"os":"linux","arch":"x64"}`). */
|
|
13
|
+
cell: 'OLVE_CELL',
|
|
12
14
|
/** Pre-signed PUT for result.tar.gz (run / map-trigger). Optional. */
|
|
13
15
|
resultUrl: 'OLVE_RESULT_URL',
|
|
14
16
|
/** JSON object: file key (`<run>/<step>/<attempt>/<path>`) → pre-signed GET URL. Optional. */
|
package/src/harness/main.ts
CHANGED
|
@@ -7,6 +7,7 @@ import { join } from 'node:path';
|
|
|
7
7
|
import { isImplementation } from '../core/handlers.ts';
|
|
8
8
|
import { configRefPath, isConfigRef, unionAt } from '../core/config.ts';
|
|
9
9
|
import type { AnyStep } from '../core/steps.ts';
|
|
10
|
+
import { cellName, parseCell } from '../core/matrix.ts';
|
|
10
11
|
import type { WebhookEvent } from '../core/webhooks.ts';
|
|
11
12
|
import { pipelineData, type PipelineData, type StageNode } from '../builder/pipeline.ts';
|
|
12
13
|
import { createCtx, formatLog } from '../ctx/index.ts';
|
|
@@ -193,17 +194,21 @@ async function runMode(
|
|
|
193
194
|
mkdirSync(p.files, { recursive: true });
|
|
194
195
|
mkdirSync(p.tmp, { recursive: true });
|
|
195
196
|
const raw = readJson(p.input);
|
|
197
|
+
const cell = step.matrix
|
|
198
|
+
? parseCell(step.name, step.matrix, process.env[ENV.cell])
|
|
199
|
+
: undefined;
|
|
196
200
|
const cancel = cancellation();
|
|
197
201
|
try {
|
|
198
202
|
let exec: Execution;
|
|
199
|
-
if (!raw.ok) {
|
|
200
|
-
|
|
203
|
+
if (!raw.ok || (cell && !cell.ok)) {
|
|
204
|
+
const message = !raw.ok ? raw.message : cell && !cell.ok ? cell.message : '';
|
|
205
|
+
io.stderr(message);
|
|
201
206
|
exec = {
|
|
202
|
-
envelope: { status: 'failed', error: { kind: 'input', message
|
|
207
|
+
envelope: { status: 'failed', error: { kind: 'input', message } },
|
|
203
208
|
exitCode: EXIT.inputInvalid,
|
|
204
209
|
};
|
|
205
210
|
} else {
|
|
206
|
-
const run = runInfoFromEnv(step.name);
|
|
211
|
+
const run = runInfoFromEnv(cell?.ok ? cellName(step.name, cell.cell) : step.name);
|
|
207
212
|
const ctx = createCtx({
|
|
208
213
|
run,
|
|
209
214
|
secrets: step.secrets,
|
|
@@ -211,6 +216,7 @@ async function runMode(
|
|
|
211
216
|
signal: cancel.signal,
|
|
212
217
|
logSink: io.stderr,
|
|
213
218
|
callbackUrls: callbackUrlsFromEnv(),
|
|
219
|
+
...(cell?.ok ? { cell: cell.cell } : {}),
|
|
214
220
|
});
|
|
215
221
|
exec = await executeStep({
|
|
216
222
|
step,
|
package/src/harness/validate.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { isModel, isOptional } from '../core/models.ts';
|
|
3
3
|
import { isExternal } from '../core/external.ts';
|
|
4
|
+
import { isCells } from '../core/matrix.ts';
|
|
4
5
|
import type { AnyStep } from '../core/steps.ts';
|
|
5
6
|
|
|
6
7
|
export type Validation<T> = { ok: true; value: T } | { ok: false; issues: string[] };
|
|
@@ -57,6 +58,10 @@ export function parseStrict(
|
|
|
57
58
|
}
|
|
58
59
|
|
|
59
60
|
function entrySchema(e: unknown): z.ZodType {
|
|
61
|
+
if (isCells(e))
|
|
62
|
+
return z.array(
|
|
63
|
+
z.object({ cell: z.record(z.string(), z.string()), output: e.model.schema }),
|
|
64
|
+
);
|
|
60
65
|
if (isOptional(e) || isExternal(e)) return e.model.schema;
|
|
61
66
|
if (isModel(e)) return e.schema;
|
|
62
67
|
return e as z.ZodType;
|
package/src/index.ts
CHANGED
|
@@ -14,6 +14,18 @@ export {
|
|
|
14
14
|
type Optional,
|
|
15
15
|
} from './core/models.ts';
|
|
16
16
|
export { secret, type Secret, type DeclaredSecrets } from './core/secrets.ts';
|
|
17
|
+
export {
|
|
18
|
+
cells,
|
|
19
|
+
keepRunning,
|
|
20
|
+
cascadeFailure,
|
|
21
|
+
type Axes,
|
|
22
|
+
type Cell,
|
|
23
|
+
type CellOutput,
|
|
24
|
+
type Cells,
|
|
25
|
+
type FailureStrategy,
|
|
26
|
+
type Matrix,
|
|
27
|
+
type MatrixDef,
|
|
28
|
+
} from './core/matrix.ts';
|
|
17
29
|
export {
|
|
18
30
|
config,
|
|
19
31
|
secretRef,
|
|
@@ -53,6 +65,7 @@ export {
|
|
|
53
65
|
type StepInput,
|
|
54
66
|
type StepKind,
|
|
55
67
|
type ConfigOf,
|
|
68
|
+
type CellOf,
|
|
56
69
|
type InputEntry,
|
|
57
70
|
type InputSpec,
|
|
58
71
|
type LiteralName,
|
|
@@ -62,12 +75,14 @@ export type { Simplify } from './core/types.ts';
|
|
|
62
75
|
export {
|
|
63
76
|
implement,
|
|
64
77
|
type Implementation,
|
|
78
|
+
type CtxOf,
|
|
65
79
|
type Handler,
|
|
66
80
|
type HandlerResult,
|
|
67
81
|
type Skipped,
|
|
68
82
|
} from './core/handlers.ts';
|
|
69
83
|
export type {
|
|
70
84
|
Ctx,
|
|
85
|
+
StepCtx,
|
|
71
86
|
RunInfo,
|
|
72
87
|
Sh,
|
|
73
88
|
ShOptions,
|
package/src/testing/index.ts
CHANGED
|
@@ -4,9 +4,11 @@ import { mkdtempSync } from 'node:fs';
|
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
6
|
import { isImplementation, type Implementation } from '../core/handlers.ts';
|
|
7
|
+
import { cellName, parseCell } from '../core/matrix.ts';
|
|
7
8
|
import type { FileRef } from '../core/models.ts';
|
|
8
9
|
import type {
|
|
9
10
|
AnyStep,
|
|
11
|
+
CellOf,
|
|
10
12
|
ConfigOf,
|
|
11
13
|
OutputValue,
|
|
12
14
|
SecretsOf,
|
|
@@ -15,6 +17,7 @@ import type {
|
|
|
15
17
|
import { createCtx } from '../ctx/index.ts';
|
|
16
18
|
import type { Ctx, RunInfo } from '../ctx/types.ts';
|
|
17
19
|
import type { StepLink } from '../ctx/links.ts';
|
|
20
|
+
import { EXIT } from '../harness/exit.ts';
|
|
18
21
|
import { executeStep, type FailureKind, type StepFunction } from '../harness/execute.ts';
|
|
19
22
|
import { sha256Hex } from '../util/hash.ts';
|
|
20
23
|
|
|
@@ -32,6 +35,8 @@ export interface TestHandlerOptions<S extends AnyStep> {
|
|
|
32
35
|
signal?: AbortSignal;
|
|
33
36
|
/** `<step>/<input>` → URL returned by `ctx.inputUrl`. */
|
|
34
37
|
callbackUrls?: Record<string, string>;
|
|
38
|
+
/** The cell to run a matrix step in (`ctx.cell`), like `OLVE_CELL`. */
|
|
39
|
+
cell?: CellOf<S>;
|
|
35
40
|
}
|
|
36
41
|
|
|
37
42
|
export type TestResult<S extends AnyStep> = {
|
|
@@ -65,9 +70,27 @@ export async function testHandler<S extends AnyStep>(
|
|
|
65
70
|
const step: AnyStep = target.step;
|
|
66
71
|
const fn = target.handler as unknown as StepFunction;
|
|
67
72
|
|
|
73
|
+
const cell = step.matrix
|
|
74
|
+
? parseCell(
|
|
75
|
+
step.name,
|
|
76
|
+
step.matrix,
|
|
77
|
+
opts.cell === undefined ? undefined : JSON.stringify(opts.cell),
|
|
78
|
+
)
|
|
79
|
+
: undefined;
|
|
68
80
|
const dir = opts.dir ?? mkdtempSync(join(tmpdir(), `olve-test-${step.name}-`));
|
|
69
81
|
const filesDir = join(dir, 'work', 'files');
|
|
70
82
|
const logs: string[] = [];
|
|
83
|
+
if (cell && !cell.ok) {
|
|
84
|
+
logs.push(cell.message);
|
|
85
|
+
return {
|
|
86
|
+
dir,
|
|
87
|
+
filesDir,
|
|
88
|
+
logs,
|
|
89
|
+
exitCode: EXIT.inputInvalid,
|
|
90
|
+
status: 'failed',
|
|
91
|
+
error: { kind: 'input', message: cell.message },
|
|
92
|
+
};
|
|
93
|
+
}
|
|
71
94
|
const run: RunInfo = {
|
|
72
95
|
pipeline: 'test',
|
|
73
96
|
id: 'test',
|
|
@@ -75,7 +98,7 @@ export async function testHandler<S extends AnyStep>(
|
|
|
75
98
|
attempt: 1,
|
|
76
99
|
version: '0.0.0-test',
|
|
77
100
|
...opts.run,
|
|
78
|
-
step: step.name,
|
|
101
|
+
step: cell?.ok ? cellName(step.name, cell.cell) : step.name,
|
|
79
102
|
};
|
|
80
103
|
const real = createCtx({
|
|
81
104
|
run,
|
|
@@ -85,6 +108,7 @@ export async function testHandler<S extends AnyStep>(
|
|
|
85
108
|
signal: opts.signal ?? new AbortController().signal,
|
|
86
109
|
logSink: (l) => logs.push(l),
|
|
87
110
|
callbackUrls: opts.callbackUrls,
|
|
111
|
+
...(cell?.ok ? { cell: cell.cell } : {}),
|
|
88
112
|
});
|
|
89
113
|
const ctx = { ...real, ...opts.ctx } as Ctx;
|
|
90
114
|
// Round-trip through JSON like input.json does (drops undefined, rejects non-JSON values early).
|