@olivervea/pipelines 2.0.0-beta.3 → 2.0.0-beta.31

package/manifest.schema.json

@@ -5,6 +5,11 @@
         {
           "additionalProperties": false,
           "properties": {
+            "cells": {
+              "const": true,
+              "description": "cells(Model): every cell of a matrix step whose output is the model, mapped with a whole-step ref; the value is [{cell, output}, …] in cell order. Omitted otherwise.",
+              "type": "boolean"
+            },
             "model": {
               "description": "Model name (key of `models`).",
               "pattern": "^[A-Za-z][A-Za-z0-9_]*$",
@@ -47,6 +52,47 @@
           "const": "2.0",
           "type": "string"
         },
+        "codeHash": {
+          "description": "Hash of the whole build: the pipeline and its code bundled, plus the harness. Any code change makes a new pipeline version.",
+          "pattern": "^sha256:[0-9a-f]{64}$",
+          "type": "string"
+        },
+        "config": {
+          "description": "Config entries used by steps, sorted by name (omitted when none). Fields of kind secretRef ({secret: NAME}) name a secret.",
+          "items": {
+            "additionalProperties": false,
+            "properties": {
+              "description": {
+                "type": "string"
+              },
+              "name": {
+                "description": "Config entry name; a key of the pipeline's config document.",
+                "pattern": "^[a-z][a-zA-Z0-9]*$",
+                "type": "string"
+              },
+              "required": {
+                "description": "False when the schema accepts a missing value (optional, or a default).",
+                "type": "boolean"
+              },
+              "schema": {
+                "additionalProperties": {},
+                "description": "z.input view: validates the config document's value for this entry (defaults listed).",
+                "propertyNames": {
+                  "type": "string"
+                },
+                "type": "object"
+              }
+            },
+            "required": [
+              "name",
+              "description",
+              "required",
+              "schema"
+            ],
+            "type": "object"
+          },
+          "type": "array"
+        },
         "description": {
           "type": "string"
         },
@@ -85,7 +131,7 @@
         },
         "harness": {
           "additionalProperties": false,
-          "description": "Harness identity. Already folded into every handlerHash.",
+          "description": "Harness identity. Already folded into codeHash.",
           "properties": {
             "bun": {
               "description": "Bun runtime version compiled into the binary.",
@@ -129,6 +175,28 @@
           "minLength": 1,
           "type": "string"
         },
+        "retention": {
+          "additionalProperties": false,
+          "description": "Retention overrides for this pipeline (omitted when none). Each field replaces the instance setting of the same meaning.",
+          "properties": {
+            "buildGraceDays": {
+              "maximum": 9007199254740991,
+              "minimum": 0,
+              "type": "integer"
+            },
+            "keepLatestRuns": {
+              "maximum": 9007199254740991,
+              "minimum": 0,
+              "type": "integer"
+            },
+            "minRunAgeDays": {
+              "maximum": 9007199254740991,
+              "minimum": 0,
+              "type": "integer"
+            }
+          },
+          "type": "object"
+        },
         "secrets": {
           "description": "All declared secrets (steps + poll triggers), sorted by name.",
           "items": {
@@ -169,6 +237,7 @@
         "name",
         "input",
         "harness",
+        "codeHash",
         "models",
         "secrets",
         "triggers",
@@ -220,6 +289,84 @@
         }
       ]
     },
+    "Matrix": {
+      "additionalProperties": false,
+      "description": "A matrix step runs once per cell, named '<step>[<axis>=<value>,…]' in axis order; OLVE_CELL holds the cell as compact JSON.",
+      "properties": {
+        "axes": {
+          "description": "In order: the cells are the cross product, first axis outermost, minus the excludes (1 to 64 cells).",
+          "items": {
+            "additionalProperties": false,
+            "properties": {
+              "name": {
+                "pattern": "^[a-z][a-zA-Z0-9]{0,30}$",
+                "type": "string"
+              },
+              "values": {
+                "description": "Unique per axis.",
+                "items": {
+                  "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$",
+                  "type": "string"
+                },
+                "minItems": 1,
+                "type": "array"
+              }
+            },
+            "required": [
+              "name",
+              "values"
+            ],
+            "type": "object"
+          },
+          "maxItems": 4,
+          "minItems": 1,
+          "type": "array"
+        },
+        "exclude": {
+          "description": "Each entry names one or more axes with one of their values; a cell matching every named axis is left out (omitted when none).",
+          "items": {
+            "additionalProperties": {
+              "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$",
+              "type": "string"
+            },
+            "propertyNames": {
+              "pattern": "^[a-z][a-zA-Z0-9]{0,30}$",
+              "type": "string"
+            },
+            "type": "object"
+          },
+          "type": "array"
+        },
+        "failure": {
+          "additionalProperties": false,
+          "description": "keepRunning: every cell runs to the end. cascadeFailure: a cell whose last allowed attempt failed cancels the step's other scheduled or running cells.",
+          "properties": {
+            "kind": {
+              "enum": [
+                "keepRunning",
+                "cascadeFailure"
+              ],
+              "type": "string"
+            }
+          },
+          "required": [
+            "kind"
+          ],
+          "type": "object"
+        },
+        "maxParallel": {
+          "description": "At most this many of the step's cells running at once in a run.",
+          "maximum": 9007199254740991,
+          "minimum": 1,
+          "type": "integer"
+        }
+      },
+      "required": [
+        "axes",
+        "failure"
+      ],
+      "type": "object"
+    },
     "ModelSchemas": {
       "additionalProperties": false,
       "properties": {
@@ -297,30 +444,170 @@
             "stages"
           ],
           "type": "object"
+        },
+        {
+          "additionalProperties": false,
+          "properties": {
+            "cases": {
+              "additionalProperties": {
+                "items": {
+                  "$ref": "#/$defs/Stage"
+                },
+                "type": "array"
+              },
+              "description": "Discriminator value → stages. Exactly one case runs; a step name may repeat across cases, nowhere else.",
+              "propertyNames": {
+                "type": "string"
+              },
+              "type": "object"
+            },
+            "discriminator": {
+              "description": "The union's literal field whose value names the case.",
+              "minLength": 1,
+              "type": "string"
+            },
+            "select": {
+              "description": "Config reference (`<entry>.<field>…`) to a discriminated union; the run's config snapshot picks the case.",
+              "minLength": 1,
+              "type": "string"
+            }
+          },
+          "required": [
+            "select",
+            "discriminator",
+            "cases"
+          ],
+          "type": "object"
         }
       ]
     },
     "Step": {
       "additionalProperties": false,
       "properties": {
+        "clusterAccess": {
+          "const": true,
+          "description": "The step pod gets the instance's step ServiceAccount token (Kubernetes:StepServiceAccount), if the instance grants one. Omitted = no token.",
+          "type": "boolean"
+        },
         "config": {
           "additionalProperties": {
             "$ref": "#/$defs/Json"
           },
-          "description": "Library-step configuration (display only; already covered by handlerHash).",
+          "description": "The step's settings (display only). A config reference is recorded as {\"$config\": \"<entry>.<field>…\"}; the entry is in configEntries.",
           "propertyNames": {
             "type": "string"
           },
           "type": "object"
         },
+        "configEntries": {
+          "description": "Config entries the step receives (sorted; omitted when none). The service delivers them in input.json under '$config'.",
+          "items": {
+            "description": "Config entry name; a key of the pipeline's config document.",
+            "pattern": "^[a-z][a-zA-Z0-9]*$",
+            "type": "string"
+          },
+          "type": "array"
+        },
         "description": {
           "description": "What the step does, shown in the UI (may be empty).",
           "type": "string"
         },
-        "handlerHash": {
-          "description": "Implementation identity: handler bundle (contract files excluded) + harness (bun + sdkHash); for library steps kind + config + harness.",
-          "pattern": "^sha256:[0-9a-f]{64}$",
-          "type": "string"
+        "externalInputs": {
+          "additionalProperties": {
+            "additionalProperties": false,
+            "properties": {
+              "description": {
+                "type": "string"
+              },
+              "from": {
+                "description": "Who or what may answer.",
+                "items": {
+                  "oneOf": [
+                    {
+                      "additionalProperties": false,
+                      "properties": {
+                        "kind": {
+                          "const": "person",
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "kind"
+                      ],
+                      "type": "object"
+                    },
+                    {
+                      "additionalProperties": false,
+                      "properties": {
+                        "kind": {
+                          "const": "callback",
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "kind"
+                      ],
+                      "type": "object"
+                    },
+                    {
+                      "additionalProperties": false,
+                      "properties": {
+                        "intervalSeconds": {
+                          "maximum": 9007199254740991,
+                          "minimum": 10,
+                          "type": "integer"
+                        },
+                        "kind": {
+                          "const": "poll",
+                          "type": "string"
+                        },
+                        "url": {
+                          "description": "GET while waiting; {run.id} and {run.number} are filled in.",
+                          "type": "string"
+                        },
+                        "valuePath": {
+                          "description": "Dot path to the value; '.' is the whole body.",
+                          "minLength": 1,
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "kind",
+                        "url",
+                        "valuePath",
+                        "intervalSeconds"
+                      ],
+                      "type": "object"
+                    }
+                  ]
+                },
+                "minItems": 1,
+                "type": "array"
+              },
+              "model": {
+                "description": "The value is validated against this model's input view.",
+                "pattern": "^[A-Za-z][A-Za-z0-9_]*$",
+                "type": "string"
+              },
+              "title": {
+                "description": "What is asked, shown to whoever answers.",
+                "minLength": 1,
+                "type": "string"
+              }
+            },
+            "required": [
+              "model",
+              "title",
+              "description",
+              "from"
+            ],
+            "type": "object"
+          },
+          "description": "Inputs answered from outside the run (omitted when none): the step waits until every one has a value, which the service writes into input.json under its key.",
+          "propertyNames": {
+            "type": "string"
+          },
+          "type": "object"
         },
         "image": {
           "description": "Container image the step runs in.",
@@ -348,16 +635,14 @@
           "type": "object"
         },
         "kind": {
-          "description": "'custom' = handlers/<name>.ts; others are SDK library steps.",
-          "enum": [
-            "custom",
-            "script",
-            "gitHubSource",
-            "imageBuild",
-            "helmDeployOverSsh"
-          ],
+          "description": "A label for the step's kind: 'custom' unless the step names one.",
+          "minLength": 1,
           "type": "string"
         },
+        "matrix": {
+          "$ref": "#/$defs/Matrix",
+          "description": "Set on a matrix step (never on a failure handler); omitted otherwise."
+        },
         "name": {
           "description": "Step name: lower-case kebab, 1-63 chars. The step's identity.",
           "pattern": "^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$",
@@ -404,8 +689,7 @@
         "inputTypes",
         "input",
         "output",
-        "secrets",
-        "handlerHash"
+        "secrets"
       ],
       "type": "object"
     },

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.3",
+  "version": "2.0.0-beta.31",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",
   "author": "Oliver Vea",
-  "homepage": "https://github.com/OliverVea/Olve.Pipelines/tree/v2/sdk#readme",
+  "homepage": "https://github.com/OliverVea/Olve.Pipelines/tree/main/sdk#readme",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/OliverVea/Olve.Pipelines.git",
-    "directory": "sdk"
+    "directory": "src/sdk"
   },
   "bugs": {
     "url": "https://github.com/OliverVea/Olve.Pipelines/issues"
@@ -50,13 +50,15 @@
     "test": "bun test",
     "check-compile-fail": "bun scripts/check-compile-fail.ts",
     "gen-schema": "bun scripts/gen-manifest-schema.ts",
-    "check": "tsc --noEmit -p . && bun scripts/check-compile-fail.ts && bun test"
+    "check": "prettier --check . && tsc --noEmit -p . && bun scripts/check-compile-fail.ts && bun test",
+    "format": "prettier --write ."
   },
   "dependencies": {
     "zod": "4.6.7"
   },
   "devDependencies": {
     "@types/bun": "1.4.3",
+    "prettier": "^3.5.0",
     "typescript": "7.0.2"
   }
 }

package/src/assets.d.ts

@@ -1,10 +1,10 @@
 // `import script from "./scripts/x.sh" with { type: "file" }` yields a path string
 // (inside the compiled binary it points into the embedded /$bunfs filesystem).
-declare module "*.sh" {
+declare module '*.sh' {
   const path: string;
   export default path;
 }
-declare module "*.bash" {
+declare module '*.bash' {
   const path: string;
   export default path;
 }

package/src/build/build.ts

@@ -1,21 +1,29 @@
-// olve-build: pipeline dir → single executable + step.tar.gz + manifest.json.
-import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
-import { join, resolve } from "node:path";
-import { canonicalJson } from "../util/canonical.ts";
-import { sha256File } from "../util/hash.ts";
-import type { HarnessInfo } from "../util/handler-hash.ts";
-import { writeTarGz } from "../util/tar.ts";
-import { MANIFEST_BEGIN, MANIFEST_END, uploadFile } from "../harness/result.ts";
-import { GLIBC_INTERP, MUSL_INTERP, readInterpreter } from "./elf.ts";
-import { handlerHashes, type HandlerHash } from "./hashes.ts";
-import { harnessInfo, loadProject, relPosix, type Project } from "./project.ts";
+// olve-build: pipeline dir → single executable + step.tar.zst + manifest.json.
+import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { canonicalJson } from '../util/canonical.ts';
+import { sha256File } from '../util/hash.ts';
+import type { HarnessInfo } from '../util/code-hash.ts';
+import { writeTarZst } from '../util/tar.ts';
+import { MANIFEST_BEGIN, MANIFEST_END, uploadFile } from '../harness/result.ts';
+import { loaderProblem } from './elf.ts';
+import { codeHash } from './hashes.ts';
+import { harnessInfo, loadProject, relPosix, type Project } from './project.ts';
 
-export const BUILD_FLAGS = ["--no-compile-autoload-dotenv", "--no-compile-autoload-bunfig", "--bytecode", "--format=esm"] as const;
-export type Target = "bun-linux-x64" | "bun-linux-x64-musl";
+export const BUILD_FLAGS = [
+  '--no-compile-autoload-dotenv',
+  '--no-compile-autoload-bunfig',
+  '--bytecode',
+  '--format=esm',
+] as const;
+export type Target = 'bun-linux-x64' | 'bun-linux-x64-musl';
 
 export class BuildError extends Error {
-  override name = "BuildError";
-  constructor(message: string, readonly exitCode = 1) {
+  override name = 'BuildError';
+  constructor(
+    message: string,
+    readonly exitCode = 1,
+  ) {
     super(message);
   }
 }
@@ -23,7 +31,7 @@ export class BuildError extends Error {
 export interface BuildOptions {
   out: string;
   target?: Target;
-  /** Pre-signed PUT for step.tar.gz. */
+  /** Pre-signed PUT for the step archive. */
   uploadUrl?: string;
   /** Print the manifest between sentinels on stdout (for the build Job's pod log). */
   sentinels?: boolean;
@@ -36,90 +44,116 @@ export interface BuildResult {
   archive: string;
   manifestFile: string;
   manifest: string;
-  hashes: Record<string, HandlerHash>;
+  codeHash: string;
   harness: HarnessInfo;
 }
 
-/** Writes `<dir>/.olve/gen/{handlers,entry}.ts`: the literal-import handler registry + binary entry. */
-export function generateEntry(project: Project, hashes: Record<string, string>, harness: HarnessInfo): string {
-  const gen = join(project.dir, ".olve", "gen");
+/** Writes `<dir>/.olve/gen/entry.ts`, the entry point of the step binary. */
+export function generateEntry(
+  project: Project,
+  code: string,
+  harness: HarnessInfo,
+): string {
+  const gen = join(project.dir, '.olve', 'gen');
   mkdirSync(gen, { recursive: true });
-  const registry = project.handlers.map(([name, file]) => `  ${JSON.stringify(name)}: () => import(${JSON.stringify(relPosix(gen, file))}),`).join("\n");
-  writeFileSync(
-    join(gen, "handlers.ts"),
-    `// GENERATED by olve-build — do not edit.\n` +
-      `import type { HandlerRegistry, HarnessInfo } from "@olivervea/pipelines/harness";\n\n` +
-      `export const handlers: HandlerRegistry = {\n${registry}\n};\n\n` +
-      `export const handlerHashes: Record<string, string> = ${JSON.stringify(hashes, null, 2)};\n\n` +
-      `export const harness: HarnessInfo = ${JSON.stringify(harness, null, 2)};\n`,
-  );
-  const entry = join(gen, "entry.ts");
+  const entry = join(gen, 'entry.ts');
+  const config = `{ pipeline, codeHash: ${JSON.stringify(code)}, harness: ${JSON.stringify(harness)} }`;
   writeFileSync(
     entry,
     `// GENERATED by olve-build — do not edit. Entry point of the step binary.\n` +
       `import pipeline from ${JSON.stringify(relPosix(gen, project.pipelineFile))};\n` +
-      `import { runHarness } from "@olivervea/pipelines/harness";\n` +
-      `import { handlers, handlerHashes, harness } from "./handlers.ts";\n\n` +
-      `process.exit(await runHarness({ pipeline, handlers, handlerHashes, harness }, process.argv.slice(2)));\n`,
+      `import { runHarness } from "@olivervea/pipelines/harness";\n\n` +
+      `process.exit(await runHarness(${config}, process.argv.slice(2)));\n`,
   );
   return entry;
 }
 
 export async function build(dir: string, opts: BuildOptions): Promise<BuildResult> {
-  const log = opts.log ?? ((l: string) => process.stderr.write(l + "\n"));
-  const stdout = opts.stdout ?? (async (t: string) => void (await Bun.write(Bun.stdout, t)));
-  const target = opts.target ?? "bun-linux-x64";
+  const log = opts.log ?? ((l: string) => process.stderr.write(l + '\n'));
+  const stdout =
+    opts.stdout ?? (async (t: string) => void (await Bun.write(Bun.stdout, t)));
+  const target = opts.target ?? 'bun-linux-x64';
   const project = await loadProject(dir);
   const out = resolve(opts.out);
   mkdirSync(out, { recursive: true });
   const harness = harnessInfo();
 
   const t0 = performance.now();
-  const hashes = await handlerHashes(project, harness);
-  const entry = generateEntry(project, Object.fromEntries(Object.entries(hashes).map(([k, v]) => [k, v.handlerHash])), harness);
-  log(`olve-build: ${project.handlers.length} handler(s) hashed in ${((performance.now() - t0) / 1000).toFixed(2)}s`);
+  const code = await codeHash(project, harness);
+  const entry = generateEntry(project, code, harness);
+  log(`olve-build: hashed the code in ${((performance.now() - t0) / 1000).toFixed(2)}s`);
 
   // 1. compile
-  const binary = join(out, "step");
+  const binary = join(out, 'step');
   rmSync(binary, { force: true });
   const t1 = performance.now();
-  const p = Bun.spawnSync([process.execPath, "build", "--compile", `--target=${target}`, ...BUILD_FLAGS, entry, `--outfile=${binary}`], {
-    cwd: project.dir,
-    stdout: "pipe",
-    stderr: "pipe",
-    env: { ...process.env, NO_COLOR: "1" },
-  });
+  const p = Bun.spawnSync(
+    [
+      process.execPath,
+      'build',
+      '--compile',
+      `--target=${target}`,
+      ...BUILD_FLAGS,
+      entry,
+      `--outfile=${binary}`,
+    ],
+    {
+      cwd: project.dir,
+      stdout: 'pipe',
+      stderr: 'pipe',
+      env: { ...process.env, NO_COLOR: '1' },
+    },
+  );
   if (p.exitCode !== 0 || !existsSync(binary))
-    throw new BuildError(`bun build --compile failed (exit ${p.exitCode}):\n${p.stdout.toString()}${p.stderr.toString()}`);
-  log(`olve-build: compiled ${target} in ${((performance.now() - t1) / 1000).toFixed(2)}s`);
+    throw new BuildError(
+      `bun build --compile failed (exit ${p.exitCode}):\n${p.stdout.toString()}${p.stderr.toString()}`,
+    );
+  log(
+    `olve-build: compiled ${target} in ${((performance.now() - t1) / 1000).toFixed(2)}s`,
+  );
 
-  // 2. loader check: the harness launches the binary through /olve/lib/<loader>; a musl binary
-  //    (e.g. compiled on alpine, where bun-linux-x64 silently yields musl) would not start.
-  const expected = target === "bun-linux-x64" ? GLIBC_INTERP : MUSL_INTERP;
-  const interp = readInterpreter(binary);
-  if (interp !== expected)
-    throw new BuildError(`compiled binary has PT_INTERP ${interp ?? "(none)"}, expected ${expected} — compile on a glibc Bun image (oven/bun:<version>-slim), not alpine`);
+  // 2. loader check: on alpine, bun-linux-x64 silently yields a musl binary, which would not start.
+  const problem = loaderProblem(binary, target);
+  if (problem) throw new BuildError(problem);
 
   // 3. describe (runs the real binary: the manifest is exactly what the step pods will see)
-  const d = Bun.spawnSync([binary, "describe"], { cwd: out, stdout: "pipe", stderr: "pipe", env: { PATH: process.env.PATH ?? "/usr/bin:/bin" } });
-  if (d.exitCode !== 0) throw new BuildError(`describe failed (exit ${d.exitCode}):\n${d.stderr.toString()}`, d.exitCode ?? 1);
+  const d = Bun.spawnSync([binary, 'describe'], {
+    cwd: out,
+    stdout: 'pipe',
+    stderr: 'pipe',
+    env: { PATH: process.env.PATH ?? '/usr/bin:/bin' },
+  });
+  if (d.exitCode !== 0)
+    throw new BuildError(
+      `describe failed (exit ${d.exitCode}):\n${d.stderr.toString()}`,
+      d.exitCode ?? 1,
+    );
   const manifest = d.stdout.toString();
-  const manifestFile = join(out, "manifest.json");
+  const manifestFile = join(out, 'manifest.json');
   writeFileSync(manifestFile, manifest);
 
-  // 4. step.tar.gz (deterministic: `step` at the root, 0755, mtime 0)
-  const archive = join(out, "step.tar.gz");
-  await writeTarGz(archive, [{ path: "step", type: "file", mode: 0o755, source: binary }]);
+  // 4. step.tar.zst (deterministic: `step` at the root, 0755, mtime 0)
+  const archive = join(out, 'step.tar.zst');
+  await writeTarZst(archive, [
+    { path: 'step', type: 'file', mode: 0o755, source: binary },
+  ]);
   writeFileSync(
-    join(out, "build.json"),
-    canonicalJson({ target, flags: BUILD_FLAGS, harness, handlers: hashes, binarySha256: sha256File(binary), archiveSha256: sha256File(archive) }),
+    join(out, 'build.json'),
+    canonicalJson({
+      target,
+      flags: BUILD_FLAGS,
+      harness,
+      codeHash: code,
+      binarySha256: sha256File(binary),
+      archiveSha256: sha256File(archive),
+    }),
   );
   log(`olve-build: wrote ${archive}`);
 
   if (opts.uploadUrl) {
     await uploadFile(opts.uploadUrl, archive, log);
-    log("olve-build: uploaded step.tar.gz");
+    log('olve-build: uploaded step.tar.zst');
   }
   if (opts.sentinels) await stdout(`${MANIFEST_BEGIN}\n${manifest}${MANIFEST_END}\n`);
-  return { binary, archive, manifestFile, manifest, hashes, harness };
+  return { binary, archive, manifestFile, manifest, codeHash: code, harness };
 }