@olivervea/pipelines 2.0.0-beta.29 → 2.0.0-beta.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -79,7 +79,7 @@ export const deployBeta = step({
79
79
  name: "deploy-beta", // identity: lower-case kebab, ≤ 63 chars
80
80
  description: "Deploys to beta.", // required (may be ""); shown in the UI, not versioned
81
81
  image: "alpine:latest", // any image; the harness brings its own runtime
82
- input: { artifact: BuildArtifact }, // model | optional(model) | plain zod schema | external(model, …)
82
+ input: { artifact: BuildArtifact }, // model | optional(model) | cells(model) | plain zod schema | external(model, …)
83
83
  output: BetaDeployment, // a model, or omitted (no output)
84
84
  secrets: [SSH_PRIVATE_KEY], // only these are mounted; ctx.secrets is typed to them
85
85
  timeoutSeconds: 900, // optional
@@ -96,6 +96,7 @@ Rules:
96
96
  - **Trigger mappers** produce `z.input`, so fields with defaults may be left out. **Handlers, mapper refs and `when`** see `z.output`.
97
97
  - A **plain-schema input** has no model to match on, so the step needs a mapper.
98
98
  - **`optional(M)`** means "maybe skipped": `M` must still have an earlier producer.
99
+ - **`cells(M)`** reads every cell of a matrix step whose output is `M` (see "Matrix steps").
99
100
  - **`external(M, { title, description?, from? })`** is answered from outside the run; `from` lists `callback()` (another system POSTs to the URL an earlier step gets from `ctx.inputUrl(step, input)`), `poll({ url, valuePath, intervalSeconds? })` and `person()` (the default). It needs no producer and no mapper entry; the step waits until it is answered, then the value (validated against `M`) is in its input like any other. Failure handlers can't have one.
100
101
 
101
102
  ### Config
@@ -192,6 +193,37 @@ export function smokeTest<const N extends string>(def: { name: LiteralName<N>; d
192
193
  - **`settings`**: what the step was set up with, recorded as the step's manifest `config` (so part of its version). JSON values and config references; the step receives every entry its settings reference.
193
194
  - `implement` refuses a step that already carries an implementation.
194
195
 
196
+ ### Matrix steps
197
+
198
+ ```ts
199
+ import { cascadeFailure, cells, implement, step } from "@olivervea/pipelines";
200
+
201
+ export const buildCli = implement(step({
202
+ name: "build-cli", description: "", image: "oven/bun:1.4.3-slim",
203
+ input: { source: Source }, output: BuiltBinary,
204
+ matrix: {
205
+ axes: { os: ["linux", "windows", "macos"], arch: ["x64", "arm64"] }, // key order is the axis order
206
+ exclude: [{ os: "windows", arch: "arm64" }], // names one or more axes
207
+ failure: cascadeFailure(), // default keepRunning()
208
+ maxParallel: 2, // optional
209
+ },
210
+ }), async ({ source }, ctx) => {
211
+ const { os, arch } = ctx.cell; // { os: "linux" | "windows" | "macos"; arch: "x64" | "arm64" }
212
+ // …
213
+ });
214
+
215
+ export const publish = implement(step({
216
+ name: "publish", description: "", image: "alpine", input: { binaries: cells(BuiltBinary) },
217
+ }), async ({ binaries }) => { for (const { cell, output } of binaries) { /* … */ } });
218
+ ```
219
+
220
+ - **Cells.** The step runs once per cell: the cross product of the axes (first axis outermost) minus the excludes. 1 to 4 axes, 1 to 64 cells. An axis name matches `^[a-z][a-zA-Z0-9]{0,30}$`, a value `^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`, values unique per axis. Exclude keys and values are type-checked against the axes.
221
+ - **`ctx.cell`** is the cell, typed from the axes' literals. It exists only on a matrix step's ctx.
222
+ - **A cell's name** is `build-cli[os=linux,arch=x64]`: `ctx.run.step` and the `step` of the cell's file references hold it.
223
+ - **Failure.** `keepRunning()` lets every cell run to the end; `cascadeFailure()` cancels the cells still running or waiting once a cell has failed its last allowed attempt.
224
+ - **Reading a matrix step.** A later step reads it only through `cells(M)`, matched by model like other inputs or wired whole with a mapper (`[publish, from => ({ binaries: from["build-cli"] })]`; a field of a matrix step is a type error). The value is `{ cell, output }[]` in cell order, one entry per cell with an output; `cell` is `Readonly<Record<string, string>>`. A matrix step can't be read as one output, and a matrix step inside a `when` can't be read by `cells` (it may be skipped).
225
+ - **Failure handlers** can't be matrix steps.
226
+
195
227
  ### Handlers
196
228
 
197
229
  ```ts
@@ -219,7 +251,8 @@ export const deployBeta = implement(step({ name: "deploy-beta", /* … */ }), as
219
251
  | `config.<entry>` | Typed to the step's declared config entries (output view, defaults applied). From input.json's `$config`. An optional entry that is not set is `undefined`. |
220
252
  | `log(message, fields?)` | Writes a `message k=v …` line to stderr. |
221
253
  | `signal` | Aborted on SIGTERM/SIGINT. Running commands are sent SIGTERM. |
222
- | `run` | `{ pipeline, id, number, attempt, step, version }` (from the env, see below). |
254
+ | `run` | `{ pipeline, id, number, attempt, step, version }` (from the env, see below). For a matrix step `step` is the cell's name. |
255
+ | `cell` | Matrix steps only: the cell being run, in axis order (from `OLVE_CELL`). |
223
256
  | `tmpDir` | A writable scratch directory (`$OLVE_DIR/tmp`). Kaniko has no `/tmp`. |
224
257
  | `skip(reason)` | Produce no output (prefer `when`). |
225
258
 
@@ -257,6 +290,8 @@ const r = await testHandler(deployBeta, { artifact }, { secrets: { SSH_PRIVATE_K
257
290
  expect(r).toMatchObject({ status: "succeeded", output: { release: "olve-pipelines" } }); // r.filesDir, r.logs, r.exitCode
258
291
  ```
259
292
 
293
+ A matrix step takes the cell to run: `testHandler(buildCli, { source }, { cell: { os: "linux", arch: "x64" } })`; without one it fails with `kind: "input"` and exit code 65, like a pod without `OLVE_CELL`.
294
+
260
295
  `testHandler` uses the same validation path as a real run. Invalid input gives `status: "failed"` with `kind: "input"`; an invalid return value gives `kind: "output"`. Any part of `ctx` can be replaced; whatever you don't replace is the real implementation. `testHandler` takes any implementation: a custom step, a library step or a reusable step.
261
296
 
262
297
  ## Harness contract (service ⇄ step binary)
@@ -272,7 +307,7 @@ This section is everything the C# side needs in order to run the binary.
272
307
  | Mode | Reads | Writes | Notes |
273
308
  |---|---|---|---|
274
309
  | `describe [--sentinels]` | the binary itself | stdout: the manifest (canonical JSON) | With `--sentinels`, stdout gets the manifest between `---OLVE-MANIFEST-BEGIN---` and `---OLVE-MANIFEST-END---`. Needs no env and no files. |
275
- | `run <step>` | `$OLVE_DIR/input.json` | `$OLVE_DIR/work/output.json`, `$OLVE_DIR/work/files/…` | Steps in stages and failure-handler steps. |
310
+ | `run <step>` | `$OLVE_DIR/input.json` | `$OLVE_DIR/work/output.json`, `$OLVE_DIR/work/files/…` | Steps in stages and failure-handler steps. A matrix step's cell runs as `run <step>` (the step's own name) with `OLVE_CELL`. |
276
311
  | `map-trigger <trigger>` | `$OLVE_DIR/input.json` (the event) | `$OLVE_DIR/work/output.json` | `<trigger>` is the trigger's `name` from the manifest. |
277
312
 
278
313
  `run` and `map-trigger` also print the envelope on stdout, as one compact JSON line between `---OLVE-RESULT-BEGIN---` and `---OLVE-RESULT-END---`. The block always comes last. Use it as a fallback channel from the pod log. User output and stderr logs also go to the pod log, so never parse anything outside the sentinels.
@@ -281,7 +316,7 @@ This section is everything the C# side needs in order to run the binary.
281
316
 
282
317
  | Path | Who | What |
283
318
  |---|---|---|
284
- | `input.json` | service (via olve-init) | **run**: a JSON object with exactly the step's input keys, built from the manifest's `input` mappings. Leave out a key whose ref root step was skipped; this is allowed only for `optional` inputs. Unknown keys → exit 65. For a step with `configEntries`, the reserved key `"$config"` holds `{ <entry>: <value> }` from the run's config (entries that are not set are left out); each entry is parsed with its schema, and a missing required entry or an invalid value → exit 65 (`config.<entry>…`). Other steps ignore `$config`. **map-trigger push**: `{"ref": "refs/heads/<branch>", "after": "<sha>", "repository": {"full_name": "<owner>/<repo>"}}` (`PushEvent`). **map-trigger poll**: `{"value": "<extracted value as string>"}`. **map-trigger webhook**: `{"headers": {…}, "body": …}` (`WebhookEvent`). |
319
+ | `input.json` | service (via olve-init) | **run**: a JSON object with exactly the step's input keys, built from the manifest's `input` mappings. Leave out a key whose ref root step was skipped; this is allowed only for `optional` inputs. Unknown keys → exit 65. For a step with `configEntries`, the reserved key `"$config"` holds `{ <entry>: <value> }` from the run's config (entries that are not set are left out); each entry is parsed with its schema, and a missing required entry or an invalid value → exit 65 (`config.<entry>…`). Other steps ignore `$config`. A `cells` input is a list in cell order, one entry per cell whose latest successful attempt has an output: `[{"cell": {"os": "linux", "arch": "x64"}, "output": {…}}, …]`. **map-trigger push**: `{"ref": "refs/heads/<branch>", "after": "<sha>", "repository": {"full_name": "<owner>/<repo>"}}` (`PushEvent`). **map-trigger poll**: `{"value": "<extracted value as string>"}`. **map-trigger webhook**: `{"headers": {…}, "body": …}` (`WebhookEvent`). |
285
320
  | `work/output.json` | harness | The envelope. Any stale copy is deleted at start, and the new file is written atomically. |
286
321
  | `work/files/<object path>` | harness | Output files and directory tars (see "File references"). |
287
322
  | `tmp/` | harness | Scratch space. Point `TMPDIR` here. |
@@ -298,6 +333,7 @@ This section is everything the C# side needs in order to run the binary.
298
333
  | `OLVE_RUN` | run | Run number (integer ≥ 0). Written into produced file refs. Default 0. |
299
334
  | `OLVE_ATTEMPT` | run | Attempt number (integer ≥ 1). Written into produced file refs. Default 1. |
300
335
  | `OLVE_VERSION_STAMP` | run | Derived version, e.g. `2026.10.21.412` → `ctx.run.version`. |
336
+ | `OLVE_CELL` | run | Only for a matrix step: the cell, as compact JSON in axis order (`{"os":"linux","arch":"x64"}`) → `ctx.cell`. Missing, or not one of the step's cells → exit 65. `ctx.run.step` and produced file refs carry the cell's name, `build-cli[os=linux,arch=x64]`. |
301
337
  | `OLVE_CASES` | run | Only for a step inside a `select`: a JSON array of the case conditions it sits in (`["target.prod.kind=ssh"]`); `run <step>` uses it to pick the variant of a name several cases use. |
302
338
  | `<SECRET_NAME>` | per step | Each declared secret as an env var of the **same name**: `secretKeyRef` key = name. Mount only the step's `secrets` (for poll triggers, the poll's `secrets`). Child processes inherit them. The mount is optional: an unset secret leaves the variable unset and `ctx.secrets.<NAME>` throws on access, so a handler can check `process.env.<NAME>` first and skip. |
303
339
  | `OLVE_RESULT_URL` | optional | Pre-signed **PUT** for `result.tar.gz` (see below). Unset means no upload. |
@@ -336,7 +372,7 @@ For `map-trigger`, `output` is the pipeline input after it has been parsed by th
336
372
  | 0 | ok (succeeded or skipped) | yes |
337
373
  | 1 | the handler or trigger mapper threw | failed / `handler` |
338
374
  | 64 | usage: unknown mode, step or trigger | none |
339
- | 65 | `input.json` missing, not JSON, or invalid for the step (or event), including its `$config` | failed / `input` |
375
+ | 65 | `input.json` missing, not JSON, or invalid for the step (or event), including its `$config`; a matrix step's `OLVE_CELL` missing or not one of its cells | failed / `input` |
340
376
  | 66 | the return value failed strict validation, or an own file ref is missing or changed | failed / `output` |
341
377
  | 74 | the `result.tar.gz` upload failed after retries. Used only when the step itself succeeded; otherwise the step's own code wins. | yes (printed) |
342
378
  | 78 | the pipeline is invalid (describe problems, or a step without an implementation) | none |
@@ -357,6 +393,8 @@ The service unpacks the archive into `runs/<p>/<run>/<step>/<attempt>/`.
357
393
  { "run": 412, "step": "build", "attempt": 1, "path": "chart/", "sha256": "<64 hex>" }
358
394
  ```
359
395
 
396
+ - `step` is the step's name, or a matrix cell's name (`build-cli[os=linux,arch=x64]`).
397
+
360
398
  - In JSON Schema, a reference is marked `"x-olve-kind": "fileRef"`.
361
399
  - `path` is relative, with no `.`/`..` segments, no `\`, no NUL and no leading `/`. The pattern is in the schema. A **trailing `/` means a directory**.
362
400
  - **Storage rule:** the object path is `path` for files. For directories, it is `path` without the trailing `/`, plus `.tar`: `chart/` → `chart.tar`, an uncompressed deterministic tar.
@@ -365,7 +403,7 @@ The service unpacks the archive into `runs/<p>/<run>/<step>/<attempt>/`.
365
403
 
366
404
  ## Manifest
367
405
 
368
- Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-12, `$defs` `Manifest`, `Step`, `Stage`, `Mapping`, `Trigger`, `InputType`, `ModelSchemas`). The text is canonical JSON: sorted keys, 2-space indent, a trailing newline, and no `$schema` in the model schemas. Hash the text as printed.
406
+ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-12, `$defs` `Manifest`, `Step`, `Stage`, `Mapping`, `Trigger`, `InputType`, `ModelSchemas`, `Matrix`). The text is canonical JSON: sorted keys, 2-space indent, a trailing newline, and no `$schema` in the model schemas. Hash the text as printed.
369
407
 
370
408
  - **`apiVersion`** is `"2.0"`.
371
409
  - **`harness`** is `{ bun, sdk, sdkHash }`.
@@ -380,7 +418,8 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
380
418
 
381
419
  An object whose only key is `ref` (with a string value) or `const` is a tag. Describe never emits a nested object with exactly that key set. Model-matched inputs are emitted already resolved (`{"ref": "build"}`).
382
420
  - **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from}`, `from` being `{kind: "person"}`, `{kind: "callback"}` or `{kind: "poll", url, valuePath, intervalSeconds}`. Those keys are not in `input` or `inputTypes`; the service merges the answers into `input.json`.
383
- - **`inputTypes`** declares each input key: `{model, optional}`, or `{schema: {output, input?}}` for a plain schema.
421
+ - **`inputTypes`** declares each input key: `{model, optional}`, `{model, optional: false, cells: true}` for a `cells` input (mapped with a whole-step ref to a matrix step whose output is that model), or `{schema: {output, input?}}` for a plain schema.
422
+ - **Step `matrix`** (matrix steps only, never a failure handler) is `{axes: [{name, values}…], exclude?: [{axis: value…}…], failure: {kind: "keepRunning" | "cascadeFailure"}, maxParallel?}`. `axes` is a list because canonical JSON sorts keys; `exclude` is omitted when empty; `failure.kind` is always written. All of it is part of what the step declares.
384
423
  - **`codeHash`** is the build's identity: `sha256(canonical {bundle, harness})`. `bundle` hashes the entry point bundled with everything it imports, with `@olivervea/pipelines` external. It is tree-shaken and includes embedded scripts and dependencies, so any change to code in use changes it, and with it the manifest, which makes a new pipeline version. A step version covers what the step declares: image, `inputTypes`/model versions, `output` model version, `secrets`, `configEntries` (and those entries' schemas), `input` mappings and `config`.
385
424
  - **`stages`** is an array whose items are either `{steps: [Step…]}` (parallel, with a barrier) or `{when: {field: [values]}, stages: [...]}`.
386
425
  - **`triggers`**:
@@ -404,7 +443,8 @@ Describe runs these checks; reconcile should re-run the manifest-level ones:
404
443
  - a poll header referencing an undeclared secret, duplicate triggers;
405
444
  - conflicting secret descriptions;
406
445
  - config names, one description and schema per config name, a step receiving a `secretRef` without declaring its secret, and input keys starting with `$` (reserved);
407
- - an `onFailure` scope naming an unknown step, and failure-handler inputs.
446
+ - an `onFailure` scope naming an unknown step, and failure-handler inputs;
447
+ - a matrix's axes, values, excludes, cell count, failure strategy and `maxParallel`, a matrix failure handler, and reading a matrix step other than whole through a `cells` input (or a `cells` input reading anything else).
408
448
 
409
449
  ## Build tool
410
450
 
@@ -447,7 +487,7 @@ olve-dev run <dir> <step> [--input-file f] [--config-file f] [--olve-dir d] [--s
447
487
  olve-dev map-trigger <dir> <trigger> --event-file f [--olve-dir d]
448
488
  ```
449
489
 
450
- `olve-dev` runs handlers from source, through the same validation path as the binary. `$OLVE_DIR` defaults to `<dir>/.olve`, and outputs land in `<dir>/.olve/work/`. Set `OLVE_FILES_DIR` to a local pool to fetch file refs. `--config-file` (a JSON object of config entries) becomes the input's `$config`.
490
+ `olve-dev` runs handlers from source, through the same validation path as the binary. `$OLVE_DIR` defaults to `<dir>/.olve`, and outputs land in `<dir>/.olve/work/`. Set `OLVE_FILES_DIR` to a local pool to fetch file refs. `--config-file` (a JSON object of config entries) becomes the input's `$config`. A matrix step's cell comes from `OLVE_CELL` in the environment, as in the pod.
451
491
 
452
492
  ## Development
453
493
 
@@ -455,7 +495,6 @@ olve-dev map-trigger <dir> <trigger> --event-file f [--olve-dir d]
455
495
  bun install
456
496
  bunx tsc --noEmit -p . # type-check (TS 7)
457
497
  bun scripts/check-compile-fail.ts # compile-fail cases (also run by bun test)
458
- TSC=/path/to/tsc-5.9/bin/tsc bun scripts/check-compile-fail.ts # the other TS line
459
498
  bun test # runtime + build/e2e tests
460
499
  bun scripts/gen-manifest-schema.ts # after changing src/describe/manifest-schema.ts
461
500
  ```
@@ -5,6 +5,11 @@
5
5
  {
6
6
  "additionalProperties": false,
7
7
  "properties": {
8
+ "cells": {
9
+ "const": true,
10
+ "description": "cells(Model): every cell of a matrix step whose output is the model, mapped with a whole-step ref; the value is [{cell, output}, …] in cell order. Omitted otherwise.",
11
+ "type": "boolean"
12
+ },
8
13
  "model": {
9
14
  "description": "Model name (key of `models`).",
10
15
  "pattern": "^[A-Za-z][A-Za-z0-9_]*$",
@@ -284,6 +289,84 @@
284
289
  }
285
290
  ]
286
291
  },
292
+ "Matrix": {
293
+ "additionalProperties": false,
294
+ "description": "A matrix step runs once per cell, named '<step>[<axis>=<value>,…]' in axis order; OLVE_CELL holds the cell as compact JSON.",
295
+ "properties": {
296
+ "axes": {
297
+ "description": "In order: the cells are the cross product, first axis outermost, minus the excludes (1 to 64 cells).",
298
+ "items": {
299
+ "additionalProperties": false,
300
+ "properties": {
301
+ "name": {
302
+ "pattern": "^[a-z][a-zA-Z0-9]{0,30}$",
303
+ "type": "string"
304
+ },
305
+ "values": {
306
+ "description": "Unique per axis.",
307
+ "items": {
308
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$",
309
+ "type": "string"
310
+ },
311
+ "minItems": 1,
312
+ "type": "array"
313
+ }
314
+ },
315
+ "required": [
316
+ "name",
317
+ "values"
318
+ ],
319
+ "type": "object"
320
+ },
321
+ "maxItems": 4,
322
+ "minItems": 1,
323
+ "type": "array"
324
+ },
325
+ "exclude": {
326
+ "description": "Each entry names one or more axes with one of their values; a cell matching every named axis is left out (omitted when none).",
327
+ "items": {
328
+ "additionalProperties": {
329
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$",
330
+ "type": "string"
331
+ },
332
+ "propertyNames": {
333
+ "pattern": "^[a-z][a-zA-Z0-9]{0,30}$",
334
+ "type": "string"
335
+ },
336
+ "type": "object"
337
+ },
338
+ "type": "array"
339
+ },
340
+ "failure": {
341
+ "additionalProperties": false,
342
+ "description": "keepRunning: every cell runs to the end. cascadeFailure: a cell whose last allowed attempt failed cancels the step's other scheduled or running cells.",
343
+ "properties": {
344
+ "kind": {
345
+ "enum": [
346
+ "keepRunning",
347
+ "cascadeFailure"
348
+ ],
349
+ "type": "string"
350
+ }
351
+ },
352
+ "required": [
353
+ "kind"
354
+ ],
355
+ "type": "object"
356
+ },
357
+ "maxParallel": {
358
+ "description": "At most this many of the step's cells running at once in a run.",
359
+ "maximum": 9007199254740991,
360
+ "minimum": 1,
361
+ "type": "integer"
362
+ }
363
+ },
364
+ "required": [
365
+ "axes",
366
+ "failure"
367
+ ],
368
+ "type": "object"
369
+ },
287
370
  "ModelSchemas": {
288
371
  "additionalProperties": false,
289
372
  "properties": {
@@ -556,6 +639,10 @@
556
639
  "minLength": 1,
557
640
  "type": "string"
558
641
  },
642
+ "matrix": {
643
+ "$ref": "#/$defs/Matrix",
644
+ "description": "Set on a matrix step (never on a failure handler); omitted otherwise."
645
+ },
559
646
  "name": {
560
647
  "description": "Step name: lower-case kebab, 1-63 chars. The step's identity.",
561
648
  "pattern": "^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@olivervea/pipelines",
3
- "version": "2.0.0-beta.29",
3
+ "version": "2.0.0-beta.30",
4
4
  "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -1,7 +1,8 @@
1
1
  import { PIPELINE } from '../core/brands.ts';
2
2
  import type { AnyModel, Infer, InferInput, Plain } from '../core/models.ts';
3
3
  import type { Secret } from '../core/secrets.ts';
4
- import type { AnyStep, OutputValue, Step } from '../core/steps.ts';
4
+ import type { AnyStep, CellOf, OutputValue, Step } from '../core/steps.ts';
5
+ import type { Cells, CellOutput } from '../core/matrix.ts';
5
6
  import { isStep } from '../core/steps.ts';
6
7
  import type { WebhookEvent, WebhookVerify } from '../core/webhooks.ts';
7
8
  import type { IsUnion, OrError, Simplify } from '../core/types.ts';
@@ -66,15 +67,25 @@ type EntryError<
66
67
  : IsUnion<ProducersOf<A, M['name']>> extends true
67
68
  ? `❌ ${M['name']} (input '${Key}' of step '${SN}') has several producers; use a mapper to pick one`
68
69
  : never
69
- : E extends AnyModel
70
- ? [ProducersOf<A, E['name']>] extends [never]
71
- ? `❌ ${E['name']} (input '${Key}' of step '${SN}') is not available at this stage`
72
- : [DefiniteProducersOf<A, E['name']>] extends [never]
73
- ? `❌ ${E['name']} (input '${Key}' of step '${SN}') is only maybe-available (produced inside a when block); declare it optional(${E['name']})`
74
- : IsUnion<ProducersOf<A, E['name']>> extends true
75
- ? `❌ ${E['name']} (input '${Key}' of step '${SN}') has several producers; use a mapper to pick one`
70
+ : E extends Cells<infer M>
71
+ ? [ProducersOf<A, `cells:${M['name']}`>] extends [never]
72
+ ? `❌ ${M['name']} (input '${Key}' of step '${SN}') has no matrix step producing it at this stage; cells() reads a matrix step`
73
+ : [DefiniteProducersOf<A, `cells:${M['name']}`>] extends [never]
74
+ ? `❌ ${M['name']} (input '${Key}' of step '${SN}') is only maybe-available (produced inside a when block); cells() cannot read a maybe-skipped matrix step`
75
+ : IsUnion<ProducersOf<A, `cells:${M['name']}`>> extends true
76
+ ? `❌ ${M['name']} (input '${Key}' of step '${SN}') has several matrix steps producing it; use a mapper to pick one`
76
77
  : never
77
- : `❌ input '${Key}' of step '${SN}' is a plain schema, not a model, so it cannot be matched by model; add a mapper`;
78
+ : E extends AnyModel
79
+ ? [ProducersOf<A, E['name']>] extends [never]
80
+ ? [ProducersOf<A, `cells:${E['name']}`>] extends [never]
81
+ ? `❌ ${E['name']} (input '${Key}' of step '${SN}') is not available at this stage`
82
+ : `❌ ${E['name']} (input '${Key}' of step '${SN}') is produced by a matrix step; read it with cells(${E['name']})`
83
+ : [DefiniteProducersOf<A, E['name']>] extends [never]
84
+ ? `❌ ${E['name']} (input '${Key}' of step '${SN}') is only maybe-available (produced inside a when block); declare it optional(${E['name']})`
85
+ : IsUnion<ProducersOf<A, E['name']>> extends true
86
+ ? `❌ ${E['name']} (input '${Key}' of step '${SN}') has several producers; use a mapper to pick one`
87
+ : never
88
+ : `❌ input '${Key}' of step '${SN}' is a plain schema, not a model, so it cannot be matched by model; add a mapper`;
78
89
 
79
90
  type DefaultMatchError<S extends AnyStep, A extends AvailMap> = {
80
91
  [K in keyof S['input'] & string]: EntryError<S['name'], K, S['input'][K], A>;
@@ -138,12 +149,19 @@ export type CheckStage<E extends readonly unknown[], A extends AvailMap> = {
138
149
  [K in keyof E]: CheckEntry<NoInfer<E>, K, E[K], A>;
139
150
  };
140
151
 
152
+ // A matrix step's slot holds its cells, under `cells:<model>`: only cells() inputs match it.
141
153
  type OutputsOf<E extends readonly unknown[], Maybe extends boolean> = {
142
- [S in StepOfEntry<E[number]> as S['name']]: Slot<
143
- S['output'] extends AnyModel ? S['output']['name'] : null,
144
- OutputValue<S>,
145
- Maybe
146
- >;
154
+ [S in StepOfEntry<E[number]> as S['name']]: [CellOf<S>] extends [never]
155
+ ? Slot<
156
+ S['output'] extends AnyModel ? S['output']['name'] : null,
157
+ OutputValue<S>,
158
+ Maybe
159
+ >
160
+ : Slot<
161
+ S['output'] extends AnyModel ? `cells:${S['output']['name']}` : null,
162
+ S['output'] extends AnyModel ? CellOutput<S['output'], CellOf<S>>[] : void,
163
+ Maybe
164
+ >;
147
165
  };
148
166
  type MakeMaybe<X extends AvailMap> = {
149
167
  [K in keyof X]: Slot<X[K]['model'], X[K]['value'], true>;
@@ -4,8 +4,15 @@ import {
4
4
  type ImplementationBrand,
5
5
  type SkippedBrand,
6
6
  } from './brands.ts';
7
- import type { Ctx } from '../ctx/types.ts';
8
- import type { AnyStep, ConfigOf, OutputValue, SecretsOf, StepInput } from './steps.ts';
7
+ import type { StepCtx } from '../ctx/types.ts';
8
+ import type {
9
+ AnyStep,
10
+ CellOf,
11
+ ConfigOf,
12
+ OutputValue,
13
+ SecretsOf,
14
+ StepInput,
15
+ } from './steps.ts';
9
16
  import type { Awaitable } from './types.ts';
10
17
 
11
18
  /** Returned by `ctx.skip(reason)`: the step produced no output on purpose. */
@@ -25,9 +32,11 @@ export function isSkipped(v: unknown): v is Skipped {
25
32
  }
26
33
 
27
34
  export type HandlerResult<S extends AnyStep> = OutputValue<S> | Skipped;
35
+ /** The ctx a handler of `S` gets. */
36
+ export type CtxOf<S extends AnyStep> = StepCtx<SecretsOf<S>, ConfigOf<S>, CellOf<S>>;
28
37
  export type Handler<S extends AnyStep> = (
29
38
  input: StepInput<S['input']>,
30
- ctx: Ctx<SecretsOf<S>, ConfigOf<S>>,
39
+ ctx: CtxOf<S>,
31
40
  ) => Awaitable<HandlerResult<S>>;
32
41
 
33
42
  /** A step with its handler. It is itself a step: stage it like any value. */
@@ -58,7 +67,7 @@ export function implement<S extends AnyStep, R extends HandlerResult<S>>(
58
67
  : unknown),
59
68
  handler: (
60
69
  input: StepInput<S['input']>,
61
- ctx: Ctx<SecretsOf<S>, ConfigOf<S>>,
70
+ ctx: CtxOf<S>,
62
71
  ) => Awaitable<R & NoExtraFields<R, OutputValue<S>>>,
63
72
  ): Implementation<S> {
64
73
  if (isImplementation(step))
@@ -0,0 +1,150 @@
1
+ import { isModel, type AnyModel, type Infer } from './models.ts';
2
+ import type { Simplify } from './types.ts';
3
+
4
+ /** Axis name → its values, in order. Object key order is the axis order. */
5
+ export type Axes = { readonly [axis: string]: readonly string[] };
6
+ /** One combination of axis values. */
7
+ export type Cell = Readonly<Record<string, string>>;
8
+ /** The cell type of a step whose axes are `Ax`. */
9
+ export type CellOfAxes<Ax extends Axes> = Simplify<{
10
+ readonly [K in keyof Ax as string extends K ? never : K]: Ax[K][number];
11
+ }>;
12
+
13
+ /** What the step's other cells do when one fails. */
14
+ export type FailureStrategy =
15
+ { readonly kind: 'keepRunning' } | { readonly kind: 'cascadeFailure' };
16
+
17
+ /** Every cell runs to the end (the default). */
18
+ export function keepRunning(): FailureStrategy {
19
+ return Object.freeze({ kind: 'keepRunning' as const });
20
+ }
21
+ /** The first failed cell cancels the cells still running or waiting. */
22
+ export function cascadeFailure(): FailureStrategy {
23
+ return Object.freeze({ kind: 'cascadeFailure' as const });
24
+ }
25
+
26
+ export interface MatrixDef<Ax extends Axes> {
27
+ /** The cells are the cross product of the axes, first axis outermost. */
28
+ axes: Ax;
29
+ /** Each entry names axes with one of their values; a cell matching all of them is left out. */
30
+ exclude?: readonly NoInfer<Partial<CellOfAxes<Ax>>>[];
31
+ /** Default `keepRunning()`. */
32
+ failure?: FailureStrategy;
33
+ /** At most this many of the step's cells running at once in a run. */
34
+ maxParallel?: number;
35
+ }
36
+
37
+ /** A matrix step's matrix, as the step carries it. */
38
+ export interface Matrix<Ax extends Axes = Axes> {
39
+ readonly axes: Ax;
40
+ readonly exclude: readonly Cell[];
41
+ readonly failure: FailureStrategy;
42
+ readonly maxParallel?: number;
43
+ }
44
+
45
+ export const AXIS_NAME = /^[a-z][a-zA-Z0-9]{0,30}$/;
46
+ export const AXIS_VALUE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$/;
47
+ export const MAX_AXES = 4;
48
+ export const MAX_CELLS = 64;
49
+
50
+ export function makeMatrix(def: MatrixDef<Axes>): Matrix {
51
+ return Object.freeze({
52
+ axes: Object.freeze(
53
+ Object.fromEntries(
54
+ Object.entries(def.axes ?? {}).map(([k, v]) => [
55
+ k,
56
+ Object.freeze(Array.isArray(v) ? [...v] : v),
57
+ ]),
58
+ ),
59
+ ),
60
+ exclude: Object.freeze((def.exclude ?? []).map((e) => Object.freeze({ ...e }))),
61
+ failure: def.failure ?? keepRunning(),
62
+ ...(def.maxParallel !== undefined ? { maxParallel: def.maxParallel } : {}),
63
+ }) as Matrix;
64
+ }
65
+
66
+ const matches = (cell: Cell, x: Cell) =>
67
+ Object.entries(x).every(([k, v]) => cell[k] === v);
68
+
69
+ /** The cells in order: the cross product in axis order (first axis outermost) minus the excludes. */
70
+ export function matrixCells(m: Matrix): Cell[] {
71
+ let cells: Record<string, string>[] = [{}];
72
+ for (const [axis, values] of Object.entries(m.axes))
73
+ cells = cells.flatMap((c) =>
74
+ (Array.isArray(values) ? values : []).map((v) => ({ ...c, [axis]: v })),
75
+ );
76
+ return cells.filter((c) => !m.exclude.some((x) => matches(c, x)));
77
+ }
78
+
79
+ /** `build-cli[os=linux,arch=x64]`: the name a cell's attempts, files and address use. */
80
+ export function cellName(step: string, cell: Cell): string {
81
+ return `${step}[${Object.entries(cell)
82
+ .map(([k, v]) => `${k}=${v}`)
83
+ .join(',')}]`;
84
+ }
85
+
86
+ /** The cell `raw` (JSON text, as in `OLVE_CELL`) names, in axis order, or what is wrong. */
87
+ export function parseCell(
88
+ step: string,
89
+ m: Matrix,
90
+ raw: string | undefined,
91
+ ): { ok: true; cell: Cell } | { ok: false; message: string } {
92
+ const axes = Object.keys(m.axes);
93
+ const example = JSON.stringify(matrixCells(m)[0] ?? {});
94
+ if (raw === undefined || raw === '')
95
+ return {
96
+ ok: false,
97
+ message: `step '${step}' is a matrix step: OLVE_CELL must name the cell to run, e.g. ${example}`,
98
+ };
99
+ let v: unknown;
100
+ try {
101
+ v = JSON.parse(raw);
102
+ } catch (e) {
103
+ return { ok: false, message: `OLVE_CELL is not JSON: ${(e as Error).message}` };
104
+ }
105
+ if (typeof v !== 'object' || v === null || Array.isArray(v))
106
+ return {
107
+ ok: false,
108
+ message: `OLVE_CELL must be a JSON object of axis values, e.g. ${example}`,
109
+ };
110
+ const given = v as Record<string, unknown>;
111
+ const ordered: Record<string, string> = {};
112
+ for (const a of axes) {
113
+ if (typeof given[a] !== 'string') break;
114
+ ordered[a] = given[a] as string;
115
+ }
116
+ const exact =
117
+ Object.keys(given).length === axes.length &&
118
+ Object.keys(ordered).length === axes.length;
119
+ const all = matrixCells(m);
120
+ if (exact && all.some((c) => cellName(step, c) === cellName(step, ordered)))
121
+ return { ok: true, cell: Object.freeze(ordered) };
122
+ return {
123
+ ok: false,
124
+ message: `OLVE_CELL ${raw} is not a cell of step '${step}' (cells: ${all.map((c) => cellName(step, c)).join(', ')})`,
125
+ };
126
+ }
127
+
128
+ // ─── cells(): reading a matrix step ──────────────────────────────────────────────────────────
129
+
130
+ /** A step input that reads every cell of a matrix step whose output is `model`. */
131
+ export interface Cells<M extends AnyModel = AnyModel> {
132
+ readonly kind: 'cells';
133
+ readonly model: M;
134
+ }
135
+ export function cells<M extends AnyModel>(model: M): Cells<M> {
136
+ return Object.freeze({ kind: 'cells' as const, model });
137
+ }
138
+ export function isCells(v: unknown): v is Cells {
139
+ return (
140
+ typeof v === 'object' &&
141
+ v !== null &&
142
+ (v as { kind?: unknown }).kind === 'cells' &&
143
+ isModel((v as { model?: unknown }).model)
144
+ );
145
+ }
146
+ /** One entry of a `cells` input, in cell order. */
147
+ export interface CellOutput<M extends AnyModel, C extends Cell = Cell> {
148
+ readonly cell: C;
149
+ readonly output: Infer<M>;
150
+ }
@@ -48,11 +48,16 @@ export const FILE_PATH_PATTERN = `^(?:${SEGMENT}/)*${SEGMENT}/?$`;
48
48
  export const FILE_PATH = new RegExp(FILE_PATH_PATTERN);
49
49
  /** Step names: lower-case kebab, ≤ 63 chars (usable as K8s label values, URL segments and storage paths). */
50
50
  export const STEP_NAME = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
51
+ const CELL_PART = String.raw`[a-z][a-zA-Z0-9]{0,30}=[A-Za-z0-9][A-Za-z0-9._-]{0,62}`;
52
+ /** A step name, or a matrix cell's name (`build-cli[os=linux,arch=x64]`): what a file reference's `step` holds. */
53
+ export const ATTEMPT_STEP = new RegExp(
54
+ String.raw`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\[${CELL_PART}(?:,${CELL_PART})*\])?$`,
55
+ );
51
56
 
52
57
  const FileRefSchema = z
53
58
  .object({
54
59
  run: z.number().int().nonnegative(),
55
- step: z.string().regex(STEP_NAME),
60
+ step: z.string().regex(ATTEMPT_STEP),
56
61
  attempt: z.number().int().positive(),
57
62
  path: z.string().regex(FILE_PATH),
58
63
  sha256: z.string().regex(/^[0-9a-f]{64}$/),
package/src/core/steps.ts CHANGED
@@ -6,24 +6,35 @@ import type { AnyConfigEntry } from './config.ts';
6
6
  import { configRefsIn } from './config-refs.ts';
7
7
  import type { Secret } from './secrets.ts';
8
8
  import type { Simplify } from './types.ts';
9
+ import {
10
+ makeMatrix,
11
+ type Axes,
12
+ type CellOfAxes,
13
+ type CellOutput,
14
+ type Cells,
15
+ type Matrix,
16
+ type MatrixDef,
17
+ } from './matrix.ts';
9
18
 
10
19
  /**
11
- * A step input entry: a model (matchable by model), `optional(model)`, a plain zod schema (mapper required),
12
- * or `external(model, …)`, answered from outside the run.
20
+ * A step input entry: a model (matchable by model), `optional(model)`, `cells(model)` (every cell of a
21
+ * matrix step), a plain zod schema (mapper required), or `external(model, …)`, answered from outside the run.
13
22
  */
14
- export type InputEntry = AnyModel | Optional | External | z.ZodType;
23
+ export type InputEntry = AnyModel | Optional | Cells | External | z.ZodType;
15
24
  export type InputSpec = { readonly [key: string]: InputEntry };
16
25
 
17
26
  export type InferEntry<E> =
18
27
  E extends Optional<infer M>
19
28
  ? Infer<M> | undefined
20
- : E extends External<infer M>
21
- ? Infer<M>
22
- : E extends AnyModel
23
- ? Infer<E>
24
- : E extends z.ZodType
25
- ? z.output<E>
26
- : never;
29
+ : E extends Cells<infer M>
30
+ ? CellOutput<M>[]
31
+ : E extends External<infer M>
32
+ ? Infer<M>
33
+ : E extends AnyModel
34
+ ? Infer<E>
35
+ : E extends z.ZodType
36
+ ? z.output<E>
37
+ : never;
27
38
  /** What a handler receives: the output view of every input entry. */
28
39
  export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
29
40
 
@@ -56,6 +67,8 @@ export interface Step<
56
67
  readonly clusterAccess?: boolean;
57
68
  /** How the step is set up, as written in the definition (may hold config references). */
58
69
  readonly settings?: Readonly<Record<string, unknown>>;
70
+ /** Set on a matrix step: it runs once per cell. */
71
+ readonly matrix?: Matrix;
59
72
  }
60
73
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
61
74
  export type AnyStep = Step<string, any, AnyModel | null, any, any>;
@@ -64,6 +77,10 @@ export type AnyStep = Step<string, any, AnyModel | null, any, any>;
64
77
  export type SecretsOf<S> = S extends Step<any, any, any, infer Sec, any> ? Sec : never;
65
78
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
66
79
  export type ConfigOf<S> = S extends Step<any, any, any, any, infer C> ? C : never;
80
+ /** The cell type of a matrix step (`never` for any other step). */
81
+ export type CellOf<S> = S extends { readonly matrix: Matrix<infer Ax> }
82
+ ? CellOfAxes<Ax>
83
+ : never;
67
84
  export type OutputValue<S extends AnyStep> = S['output'] extends AnyModel
68
85
  ? Infer<S['output']>
69
86
  : void;
@@ -82,6 +99,7 @@ export interface StepDef<
82
99
  O extends AnyModel | null,
83
100
  Sec extends readonly Secret[],
84
101
  Cfg extends readonly AnyConfigEntry[] = [],
102
+ Ax extends Axes = never,
85
103
  > {
86
104
  name: LiteralName<N>;
87
105
  /** What the step does, shown in the UI. Required; may be empty. */
@@ -109,6 +127,8 @@ export interface StepDef<
109
127
  * references is delivered to the step, and `resolveConfigRefs(value, ctx.config)` resolves them.
110
128
  */
111
129
  settings?: Readonly<Record<string, unknown>>;
130
+ /** Run the step once per cell of these axes; the handler reads its cell from `ctx.cell`. */
131
+ matrix?: MatrixDef<Ax>;
112
132
  }
113
133
 
114
134
  function makeStep(def: Record<string, unknown>): AnyStep {
@@ -130,6 +150,9 @@ function makeStep(def: Record<string, unknown>): AnyStep {
130
150
  ...(def.retries !== undefined ? { retries: def.retries } : {}),
131
151
  ...(def.clusterAccess === true ? { clusterAccess: true } : {}),
132
152
  ...(def.settings !== undefined ? { settings: def.settings } : {}),
153
+ ...(def.matrix !== undefined
154
+ ? { matrix: makeMatrix(def.matrix as MatrixDef<Axes>) }
155
+ : {}),
133
156
  };
134
157
  return Object.freeze(s) as unknown as AnyStep;
135
158
  }
@@ -150,6 +173,10 @@ export function step<
150
173
  O extends AnyModel | null = null,
151
174
  const Sec extends readonly Secret[] = [],
152
175
  const Cfg extends readonly AnyConfigEntry[] = [],
153
- >(def: StepDef<N, I, O, Sec, Cfg>): Step<N, I, O, Sec[number]['name'], Cfg[number]> {
176
+ const Ax extends Axes = never,
177
+ >(
178
+ def: StepDef<N, I, O, Sec, Cfg, Ax>,
179
+ ): Step<N, I, O, Sec[number]['name'], Cfg[number]> &
180
+ ([Ax] extends [never] ? unknown : { readonly matrix: Matrix<Ax> }) {
154
181
  return makeStep(def as never) as never;
155
182
  }
package/src/ctx/index.ts CHANGED
@@ -3,6 +3,7 @@ import { resolve } from 'node:path';
3
3
  import { skipped } from '../core/handlers.ts';
4
4
  import { createLinks } from './links.ts';
5
5
  import type { Secret } from '../core/secrets.ts';
6
+ import type { Cell } from '../core/matrix.ts';
6
7
  import { createFetch, createOutput, redactUrl } from './files.ts';
7
8
  import { createSh } from './sh.ts';
8
9
  import type { Ctx, Http, RetryOptions, RunInfo } from './types.ts';
@@ -21,6 +22,8 @@ export interface CtxOptions {
21
22
  logSink?: (line: string) => void;
22
23
  /** `<step>/<input>` → callback URL (`ctx.inputUrl`). */
23
24
  callbackUrls?: Readonly<Record<string, string>>;
25
+ /** A matrix step's cell (`ctx.cell`). */
26
+ cell?: Cell;
24
27
  }
25
28
 
26
29
  export function formatLog(message: string, fields?: Record<string, unknown>): string {
@@ -151,6 +154,7 @@ export function createCtx(o: CtxOptions): Ctx {
151
154
  log,
152
155
  };
153
156
  return {
157
+ ...(o.cell ? { cell: o.cell } : {}),
154
158
  secrets: createSecrets(o.secrets, o.secretValues) as Ctx['secrets'],
155
159
  config: Object.freeze({ ...o.config }) as Ctx['config'],
156
160
  sh,
package/src/ctx/types.ts CHANGED
@@ -124,3 +124,10 @@ export interface Ctx<Sec extends string = string, Cfg extends AnyConfigEntry = n
124
124
  */
125
125
  inputUrl(step: string | { readonly name: string }, input: string): string;
126
126
  }
127
+
128
+ /** The ctx a step's handler gets: a matrix step's also carries its cell (`ctx.cell`), in axis order. */
129
+ export type StepCtx<Sec extends string, Cfg extends AnyConfigEntry, Cell> = Ctx<
130
+ Sec,
131
+ Cfg
132
+ > &
133
+ ([Cell] extends [never] ? unknown : { readonly cell: Cell });
@@ -12,10 +12,11 @@ import { schemaViews, type JsonSchema } from './schemas.ts';
12
12
  export const FAILURE_ROOT = 'failure';
13
13
  const RESERVED = new Set([...RESERVED_STEP_NAMES, FAILURE_ROOT]);
14
14
 
15
- /** What a step can read at a point of the pipeline: its output model, and whether it may have been skipped. */
15
+ /** What a step can read at a point of the pipeline: its output model, whether it may have been skipped, and whether it is a matrix step. */
16
16
  export interface Avail {
17
17
  model: string | null;
18
18
  maybe: boolean;
19
+ matrix?: boolean;
19
20
  }
20
21
 
21
22
  /** A select case a step sits in: which select (numbered in describe order), which value, and its condition text. */
@@ -154,6 +155,10 @@ export class DescribeContext {
154
155
  problems.push(
155
156
  `step '${s.name}': its select cases must give it the same output model`,
156
157
  );
158
+ else if (prev.some((p) => !p.step.matrix !== !s.matrix))
159
+ problems.push(
160
+ `step '${s.name}': its select cases must all make it a matrix step, or none`,
161
+ );
157
162
  this.placements.set(s.name, [...prev, { step: s, cases }]);
158
163
  if (!this.steps.has(s.name)) this.steps.set(s.name, s);
159
164
  }
@@ -76,6 +76,12 @@ export const InputType = z
76
76
  optional: z
77
77
  .boolean()
78
78
  .describe('optional(Model): may be absent because its producer was skipped.'),
79
+ cells: z
80
+ .literal(true)
81
+ .optional()
82
+ .describe(
83
+ 'cells(Model): every cell of a matrix step whose output is the model, mapped with a whole-step ref; the value is [{cell, output}, …] in cell order. Omitted otherwise.',
84
+ ),
79
85
  }),
80
86
  z.strictObject({
81
87
  schema: ModelSchemas.describe(
@@ -85,6 +91,45 @@ export const InputType = z
85
91
  ])
86
92
  .meta({ id: 'InputType' });
87
93
 
94
+ const axisName = z.string().regex(/^[a-z][a-zA-Z0-9]{0,30}$/);
95
+ const axisValue = z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$/);
96
+
97
+ export const Matrix = z
98
+ .strictObject({
99
+ axes: z
100
+ .array(
101
+ z.strictObject({
102
+ name: axisName,
103
+ values: z.array(axisValue).min(1).describe('Unique per axis.'),
104
+ }),
105
+ )
106
+ .min(1)
107
+ .max(4)
108
+ .describe(
109
+ 'In order: the cells are the cross product, first axis outermost, minus the excludes (1 to 64 cells).',
110
+ ),
111
+ exclude: z
112
+ .array(z.record(axisName, axisValue))
113
+ .optional()
114
+ .describe(
115
+ 'Each entry names one or more axes with one of their values; a cell matching every named axis is left out (omitted when none).',
116
+ ),
117
+ failure: z
118
+ .strictObject({ kind: z.enum(['keepRunning', 'cascadeFailure']) })
119
+ .describe(
120
+ "keepRunning: every cell runs to the end. cascadeFailure: a cell whose last allowed attempt failed cancels the step's other scheduled or running cells.",
121
+ ),
122
+ maxParallel: z
123
+ .int()
124
+ .min(1)
125
+ .optional()
126
+ .describe("At most this many of the step's cells running at once in a run."),
127
+ })
128
+ .describe(
129
+ "A matrix step runs once per cell, named '<step>[<axis>=<value>,…]' in axis order; OLVE_CELL holds the cell as compact JSON.",
130
+ )
131
+ .meta({ id: 'Matrix' });
132
+
88
133
  export const StepDescriptor = z
89
134
  .strictObject({
90
135
  name: stepName,
@@ -114,6 +159,9 @@ export const StepDescriptor = z
114
159
  ),
115
160
  timeoutSeconds: z.number().int().positive().optional(),
116
161
  retries: z.number().int().nonnegative().optional(),
162
+ matrix: Matrix.optional().describe(
163
+ 'Set on a matrix step (never on a failure handler); omitted otherwise.',
164
+ ),
117
165
  config: z
118
166
  .record(z.string(), Json)
119
167
  .optional()
@@ -54,6 +54,7 @@ function describeStage(
54
54
  avail.set(ss.step.name, {
55
55
  model: ss.step.output?.name ?? null,
56
56
  maybe: false,
57
+ ...(ss.step.matrix ? { matrix: true } : {}),
57
58
  });
58
59
  return descs.length ? { steps: descs } : undefined;
59
60
  }
@@ -96,7 +97,7 @@ function describeWhen(
96
97
  const inner = new Map(avail);
97
98
  const stages = describeStages(ctx, input, n.nodes, inner, cases);
98
99
  for (const [name, a] of inner)
99
- if (!avail.has(name)) avail.set(name, { model: a.model, maybe: true });
100
+ if (!avail.has(name)) avail.set(name, { ...a, maybe: true });
100
101
  return { when, stages };
101
102
  }
102
103
 
@@ -153,7 +154,7 @@ function describeSelect(
153
154
  .map((b) => b.inner.get(name))
154
155
  .filter((a): a is Avail => a !== undefined);
155
156
  avail.set(name, {
156
- model: present[0]!.model,
157
+ ...present[0]!,
157
158
  maybe: present.length < branches.length || present.some((a) => a.maybe),
158
159
  });
159
160
  }
@@ -1,6 +1,15 @@
1
1
  import { z } from 'zod';
2
2
  import { isModel, isOptional } from '../core/models.ts';
3
3
  import { isExternal } from '../core/external.ts';
4
+ import {
5
+ AXIS_NAME,
6
+ AXIS_VALUE,
7
+ MAX_AXES,
8
+ MAX_CELLS,
9
+ isCells,
10
+ matrixCells,
11
+ type Matrix,
12
+ } from '../core/matrix.ts';
4
13
  import type { AnyStep } from '../core/steps.ts';
5
14
  import { isImplementation } from '../core/handlers.ts';
6
15
  import { recordConfigRefs } from '../core/config-refs.ts';
@@ -20,6 +29,7 @@ import {
20
29
  } from './context.ts';
21
30
 
22
31
  type InputTypes = StepDescriptorT['inputTypes'];
32
+ type MatrixT = NonNullable<StepDescriptorT['matrix']>;
23
33
  type ExternalInputs = NonNullable<StepDescriptorT['externalInputs']>;
24
34
  type InputEntries = [string, unknown][];
25
35
 
@@ -77,6 +87,9 @@ export function describeStep(
77
87
  problems.push(`${at}: timeoutSeconds must be a positive integer`);
78
88
  if (s.retries !== undefined && !(Number.isInteger(s.retries) && s.retries >= 0))
79
89
  problems.push(`${at}: retries must be a non-negative integer`);
90
+ if (s.matrix !== undefined && failureHandler)
91
+ problems.push(`${at}: a failure handler cannot be a matrix step`);
92
+ const matrix = s.matrix !== undefined ? describeMatrix(ctx, at, s.matrix) : undefined;
80
93
 
81
94
  const out: StepDescriptorT = {
82
95
  name: s.name,
@@ -89,6 +102,7 @@ export function describeStep(
89
102
  secrets: secretNames,
90
103
  ...(s.timeoutSeconds !== undefined ? { timeoutSeconds: s.timeoutSeconds } : {}),
91
104
  ...(s.retries !== undefined ? { retries: s.retries } : {}),
105
+ ...(matrix !== undefined ? { matrix } : {}),
92
106
  ...(settings !== undefined
93
107
  ? { config: settings as Record<string, z.core.util.JSONType> }
94
108
  : {}),
@@ -156,7 +170,13 @@ function describeInputTypes(
156
170
  ): InputTypes {
157
171
  const inputTypes: InputTypes = {};
158
172
  for (const [key, e] of entries) {
159
- if (isOptional(e))
173
+ if (isCells(e))
174
+ inputTypes[key] = {
175
+ model: ctx.useModel(e.model, `${at} input '${key}'`),
176
+ optional: false,
177
+ cells: true,
178
+ };
179
+ else if (isOptional(e))
160
180
  inputTypes[key] = {
161
181
  model: ctx.useModel(e.model, `${at} input '${key}'`),
162
182
  optional: true,
@@ -176,7 +196,7 @@ function describeInputTypes(
176
196
  }
177
197
  } else
178
198
  ctx.problems.push(
179
- `${at} input '${key}': must be a model, optional(model) or a zod schema`,
199
+ `${at} input '${key}': must be a model, optional(model), cells(model) or a zod schema`,
180
200
  );
181
201
  }
182
202
  return inputTypes;
@@ -209,6 +229,25 @@ function matchInputsByModel(
209
229
  const problems = ctx.problems;
210
230
  const input: Record<string, Mapping> = {};
211
231
  for (const [key, e] of entries) {
232
+ if (isCells(e)) {
233
+ const producers = [...avail].filter(
234
+ ([, a]) => a.matrix && a.model === e.model.name,
235
+ );
236
+ if (producers.length === 1) {
237
+ const [producer, a] = producers[0]!;
238
+ if (a.maybe) problems.push(maybeCells(at, e.model.name, key));
239
+ input[key] = { ref: producer };
240
+ } else
241
+ problems.push(
242
+ producers.length === 0
243
+ ? `${at}: ${e.model.name} (input '${key}') has no matrix step producing it at this stage; cells() reads a matrix step`
244
+ : `${at}: ${e.model.name} (input '${key}') has several matrix steps producing it (${producers
245
+ .map(([n]) => n)
246
+ .sort()
247
+ .join(', ')}); use a mapper to pick one`,
248
+ );
249
+ continue;
250
+ }
212
251
  const optional = isOptional(e);
213
252
  const m = optional ? e.model : isModel(e) ? e : undefined;
214
253
  if (!m) {
@@ -217,8 +256,13 @@ function matchInputsByModel(
217
256
  );
218
257
  continue;
219
258
  }
220
- const producers = [...avail].filter(([, a]) => a.model === m.name);
221
- if (producers.length === 0) {
259
+ const producers = [...avail].filter(([, a]) => !a.matrix && a.model === m.name);
260
+ const matrixProducer = [...avail].find(([, a]) => a.matrix && a.model === m.name);
261
+ if (producers.length === 0 && matrixProducer) {
262
+ problems.push(
263
+ `${at}: ${m.name} (input '${key}') is produced by matrix step '${matrixProducer[0]}'; read it with cells(${m.name})`,
264
+ );
265
+ } else if (producers.length === 0) {
222
266
  problems.push(
223
267
  `${at}: ${m.name} (input '${key}') is not available at this stage${optional ? ' (optional() means maybe-skipped; there must still be an earlier producer)' : ''}`,
224
268
  );
@@ -273,11 +317,47 @@ function mapStep(
273
317
  for (const k of Object.keys(recorded))
274
318
  if (!(k in declared))
275
319
  problems.push(`${at}: mapper provides '${k}', which is not an input of the step`);
276
- for (const [k, m] of Object.entries(recorded))
277
- checkMapping(ctx, `${at} input '${k}'`, m, avail, isOptional(declared[k]), true);
320
+ for (const [k, m] of Object.entries(recorded)) {
321
+ const e = declared[k];
322
+ if (isCells(e)) checkCellsMapping(ctx, `${at} input '${k}'`, m, avail, e.model.name);
323
+ else checkMapping(ctx, `${at} input '${k}'`, m, avail, isOptional(e), true);
324
+ }
278
325
  return recorded;
279
326
  }
280
327
 
328
+ const maybeCells = (at: string, model: string, key: string) =>
329
+ `${at}: ${model} (input '${key}') is only maybe-available (produced inside a when block); cells() cannot read a maybe-skipped matrix step`;
330
+
331
+ function checkCellsMapping(
332
+ ctx: DescribeContext,
333
+ at: string,
334
+ m: Mapping,
335
+ avail: Map<string, Avail>,
336
+ model: string,
337
+ ): void {
338
+ const problems = ctx.problems;
339
+ const ref =
340
+ Object.keys(m).length === 1 && typeof (m as { ref?: unknown }).ref === 'string'
341
+ ? (m as { ref: string }).ref
342
+ : undefined;
343
+ if (ref === undefined || ref.includes('.'))
344
+ return void problems.push(
345
+ `${at}: a cells() input takes a whole matrix step (from['<step>']), not a field or an object`,
346
+ );
347
+ const a = avail.get(ref);
348
+ if (!a) return void problems.push(`${at}: '${ref}' is not available at this stage`);
349
+ if (!a.matrix)
350
+ return void problems.push(
351
+ `${at}: '${ref}' is not a matrix step; cells() reads a matrix step`,
352
+ );
353
+ if (a.model !== model)
354
+ problems.push(`${at}: '${ref}' outputs ${a.model ?? 'nothing'}, not ${model}`);
355
+ if (a.maybe)
356
+ problems.push(
357
+ `${at}: '${ref}' is only maybe-available (produced inside a when block); cells() cannot read a maybe-skipped matrix step`,
358
+ );
359
+ }
360
+
281
361
  function checkMapping(
282
362
  ctx: DescribeContext,
283
363
  at: string,
@@ -299,6 +379,10 @@ function checkMapping(
299
379
  ];
300
380
  const a = avail.get(root);
301
381
  if (!a) return void problems.push(`${at}: '${root}' is not available at this stage`);
382
+ if (a.matrix)
383
+ return void problems.push(
384
+ `${at}: '${(m as { ref: string }).ref}' — '${root}' is a matrix step; read it whole through a cells() input`,
385
+ );
302
386
  if (!resolveField(ctx.outputViewOf(a.model), path))
303
387
  problems.push(
304
388
  `${at}: '${(m as { ref: string }).ref}' — model ${a.model} has no field '${path.join('.')}'`,
@@ -393,3 +477,68 @@ function checkConfigRefs(
393
477
  );
394
478
  }
395
479
  }
480
+
481
+ function describeMatrix(
482
+ ctx: DescribeContext,
483
+ at: string,
484
+ m: Matrix,
485
+ ): MatrixT | undefined {
486
+ const problems = ctx.problems;
487
+ const before = problems.length;
488
+ const axes = Object.entries(
489
+ typeof m.axes === 'object' && m.axes !== null ? m.axes : {},
490
+ ) as [string, unknown][];
491
+ if (axes.length < 1 || axes.length > MAX_AXES)
492
+ problems.push(
493
+ `${at}: a matrix has 1 to ${MAX_AXES} axes (this one has ${axes.length})`,
494
+ );
495
+ for (const [name, values] of axes) {
496
+ if (!AXIS_NAME.test(name))
497
+ problems.push(`${at}: matrix axis name '${name}' must match ${AXIS_NAME}`);
498
+ if (!Array.isArray(values) || values.some((v) => typeof v !== 'string')) {
499
+ problems.push(`${at}: matrix axis '${name}' must be a list of strings`);
500
+ continue;
501
+ }
502
+ if (values.length === 0) problems.push(`${at}: matrix axis '${name}' has no values`);
503
+ for (const v of values as string[])
504
+ if (!AXIS_VALUE.test(v))
505
+ problems.push(
506
+ `${at}: matrix axis '${name}': value '${v}' must match ${AXIS_VALUE}`,
507
+ );
508
+ for (const v of new Set(
509
+ (values as string[]).filter((v, i) => values.indexOf(v) !== i),
510
+ ))
511
+ problems.push(`${at}: matrix axis '${name}': value '${v}' is listed twice`);
512
+ }
513
+ const known = new Map(axes);
514
+ m.exclude.forEach((x, i) => {
515
+ const where = `${at}: matrix exclude ${i + 1}`;
516
+ if (Object.keys(x).length === 0)
517
+ problems.push(`${where} must name at least one axis`);
518
+ for (const [k, v] of Object.entries(x)) {
519
+ const values = known.get(k);
520
+ if (!values) problems.push(`${where}: '${k}' is not an axis`);
521
+ else if (!(Array.isArray(values) && values.includes(v)))
522
+ problems.push(`${where}: '${String(v)}' is not a value of axis '${k}'`);
523
+ }
524
+ });
525
+ const kind = (m.failure as { kind?: unknown } | undefined)?.kind;
526
+ if (kind !== 'keepRunning' && kind !== 'cascadeFailure')
527
+ problems.push(`${at}: matrix failure must be keepRunning() or cascadeFailure()`);
528
+ if (
529
+ m.maxParallel !== undefined &&
530
+ !(Number.isInteger(m.maxParallel) && m.maxParallel >= 1)
531
+ )
532
+ problems.push(`${at}: matrix maxParallel must be a positive integer`);
533
+ if (problems.length > before) return undefined;
534
+ const count = matrixCells(m).length;
535
+ if (count === 0) problems.push(`${at}: the matrix excludes every cell`);
536
+ else if (count > MAX_CELLS)
537
+ problems.push(`${at}: the matrix has ${count} cells (at most ${MAX_CELLS})`);
538
+ return {
539
+ axes: axes.map(([name, values]) => ({ name, values: [...(values as string[])] })),
540
+ ...(m.exclude.length ? { exclude: m.exclude.map((x) => ({ ...x })) } : {}),
541
+ failure: { kind: kind as MatrixT['failure']['kind'] },
542
+ ...(m.maxParallel !== undefined ? { maxParallel: m.maxParallel } : {}),
543
+ };
544
+ }
@@ -9,6 +9,8 @@ export const ENV = {
9
9
  run: 'OLVE_RUN',
10
10
  attempt: 'OLVE_ATTEMPT',
11
11
  versionStamp: 'OLVE_VERSION_STAMP',
12
+ /** A matrix step's cell, as compact JSON in axis order (`{"os":"linux","arch":"x64"}`). */
13
+ cell: 'OLVE_CELL',
12
14
  /** Pre-signed PUT for result.tar.gz (run / map-trigger). Optional. */
13
15
  resultUrl: 'OLVE_RESULT_URL',
14
16
  /** JSON object: file key (`<run>/<step>/<attempt>/<path>`) → pre-signed GET URL. Optional. */
@@ -7,6 +7,7 @@ import { join } from 'node:path';
7
7
  import { isImplementation } from '../core/handlers.ts';
8
8
  import { configRefPath, isConfigRef, unionAt } from '../core/config.ts';
9
9
  import type { AnyStep } from '../core/steps.ts';
10
+ import { cellName, parseCell } from '../core/matrix.ts';
10
11
  import type { WebhookEvent } from '../core/webhooks.ts';
11
12
  import { pipelineData, type PipelineData, type StageNode } from '../builder/pipeline.ts';
12
13
  import { createCtx, formatLog } from '../ctx/index.ts';
@@ -193,17 +194,21 @@ async function runMode(
193
194
  mkdirSync(p.files, { recursive: true });
194
195
  mkdirSync(p.tmp, { recursive: true });
195
196
  const raw = readJson(p.input);
197
+ const cell = step.matrix
198
+ ? parseCell(step.name, step.matrix, process.env[ENV.cell])
199
+ : undefined;
196
200
  const cancel = cancellation();
197
201
  try {
198
202
  let exec: Execution;
199
- if (!raw.ok) {
200
- io.stderr(raw.message);
203
+ if (!raw.ok || (cell && !cell.ok)) {
204
+ const message = !raw.ok ? raw.message : cell && !cell.ok ? cell.message : '';
205
+ io.stderr(message);
201
206
  exec = {
202
- envelope: { status: 'failed', error: { kind: 'input', message: raw.message } },
207
+ envelope: { status: 'failed', error: { kind: 'input', message } },
203
208
  exitCode: EXIT.inputInvalid,
204
209
  };
205
210
  } else {
206
- const run = runInfoFromEnv(step.name);
211
+ const run = runInfoFromEnv(cell?.ok ? cellName(step.name, cell.cell) : step.name);
207
212
  const ctx = createCtx({
208
213
  run,
209
214
  secrets: step.secrets,
@@ -211,6 +216,7 @@ async function runMode(
211
216
  signal: cancel.signal,
212
217
  logSink: io.stderr,
213
218
  callbackUrls: callbackUrlsFromEnv(),
219
+ ...(cell?.ok ? { cell: cell.cell } : {}),
214
220
  });
215
221
  exec = await executeStep({
216
222
  step,
@@ -1,6 +1,7 @@
1
1
  import { z } from 'zod';
2
2
  import { isModel, isOptional } from '../core/models.ts';
3
3
  import { isExternal } from '../core/external.ts';
4
+ import { isCells } from '../core/matrix.ts';
4
5
  import type { AnyStep } from '../core/steps.ts';
5
6
 
6
7
  export type Validation<T> = { ok: true; value: T } | { ok: false; issues: string[] };
@@ -57,6 +58,10 @@ export function parseStrict(
57
58
  }
58
59
 
59
60
  function entrySchema(e: unknown): z.ZodType {
61
+ if (isCells(e))
62
+ return z.array(
63
+ z.object({ cell: z.record(z.string(), z.string()), output: e.model.schema }),
64
+ );
60
65
  if (isOptional(e) || isExternal(e)) return e.model.schema;
61
66
  if (isModel(e)) return e.schema;
62
67
  return e as z.ZodType;
package/src/index.ts CHANGED
@@ -14,6 +14,18 @@ export {
14
14
  type Optional,
15
15
  } from './core/models.ts';
16
16
  export { secret, type Secret, type DeclaredSecrets } from './core/secrets.ts';
17
+ export {
18
+ cells,
19
+ keepRunning,
20
+ cascadeFailure,
21
+ type Axes,
22
+ type Cell,
23
+ type CellOutput,
24
+ type Cells,
25
+ type FailureStrategy,
26
+ type Matrix,
27
+ type MatrixDef,
28
+ } from './core/matrix.ts';
17
29
  export {
18
30
  config,
19
31
  secretRef,
@@ -53,6 +65,7 @@ export {
53
65
  type StepInput,
54
66
  type StepKind,
55
67
  type ConfigOf,
68
+ type CellOf,
56
69
  type InputEntry,
57
70
  type InputSpec,
58
71
  type LiteralName,
@@ -62,12 +75,14 @@ export type { Simplify } from './core/types.ts';
62
75
  export {
63
76
  implement,
64
77
  type Implementation,
78
+ type CtxOf,
65
79
  type Handler,
66
80
  type HandlerResult,
67
81
  type Skipped,
68
82
  } from './core/handlers.ts';
69
83
  export type {
70
84
  Ctx,
85
+ StepCtx,
71
86
  RunInfo,
72
87
  Sh,
73
88
  ShOptions,
@@ -4,9 +4,11 @@ import { mkdtempSync } from 'node:fs';
4
4
  import { tmpdir } from 'node:os';
5
5
  import { join } from 'node:path';
6
6
  import { isImplementation, type Implementation } from '../core/handlers.ts';
7
+ import { cellName, parseCell } from '../core/matrix.ts';
7
8
  import type { FileRef } from '../core/models.ts';
8
9
  import type {
9
10
  AnyStep,
11
+ CellOf,
10
12
  ConfigOf,
11
13
  OutputValue,
12
14
  SecretsOf,
@@ -15,6 +17,7 @@ import type {
15
17
  import { createCtx } from '../ctx/index.ts';
16
18
  import type { Ctx, RunInfo } from '../ctx/types.ts';
17
19
  import type { StepLink } from '../ctx/links.ts';
20
+ import { EXIT } from '../harness/exit.ts';
18
21
  import { executeStep, type FailureKind, type StepFunction } from '../harness/execute.ts';
19
22
  import { sha256Hex } from '../util/hash.ts';
20
23
 
@@ -32,6 +35,8 @@ export interface TestHandlerOptions<S extends AnyStep> {
32
35
  signal?: AbortSignal;
33
36
  /** `<step>/<input>` → URL returned by `ctx.inputUrl`. */
34
37
  callbackUrls?: Record<string, string>;
38
+ /** The cell to run a matrix step in (`ctx.cell`), like `OLVE_CELL`. */
39
+ cell?: CellOf<S>;
35
40
  }
36
41
 
37
42
  export type TestResult<S extends AnyStep> = {
@@ -65,9 +70,27 @@ export async function testHandler<S extends AnyStep>(
65
70
  const step: AnyStep = target.step;
66
71
  const fn = target.handler as unknown as StepFunction;
67
72
 
73
+ const cell = step.matrix
74
+ ? parseCell(
75
+ step.name,
76
+ step.matrix,
77
+ opts.cell === undefined ? undefined : JSON.stringify(opts.cell),
78
+ )
79
+ : undefined;
68
80
  const dir = opts.dir ?? mkdtempSync(join(tmpdir(), `olve-test-${step.name}-`));
69
81
  const filesDir = join(dir, 'work', 'files');
70
82
  const logs: string[] = [];
83
+ if (cell && !cell.ok) {
84
+ logs.push(cell.message);
85
+ return {
86
+ dir,
87
+ filesDir,
88
+ logs,
89
+ exitCode: EXIT.inputInvalid,
90
+ status: 'failed',
91
+ error: { kind: 'input', message: cell.message },
92
+ };
93
+ }
71
94
  const run: RunInfo = {
72
95
  pipeline: 'test',
73
96
  id: 'test',
@@ -75,7 +98,7 @@ export async function testHandler<S extends AnyStep>(
75
98
  attempt: 1,
76
99
  version: '0.0.0-test',
77
100
  ...opts.run,
78
- step: step.name,
101
+ step: cell?.ok ? cellName(step.name, cell.cell) : step.name,
79
102
  };
80
103
  const real = createCtx({
81
104
  run,
@@ -85,6 +108,7 @@ export async function testHandler<S extends AnyStep>(
85
108
  signal: opts.signal ?? new AbortController().signal,
86
109
  logSink: (l) => logs.push(l),
87
110
  callbackUrls: opts.callbackUrls,
111
+ ...(cell?.ok ? { cell: cell.cell } : {}),
88
112
  });
89
113
  const ctx = { ...real, ...opts.ctx } as Ctx;
90
114
  // Round-trip through JSON like input.json does (drops undefined, rejects non-JSON values early).