@olivervea/pipelines 2.0.0-beta.1 → 2.0.0-beta.10

package/manifest.schema.json

@@ -47,6 +47,42 @@
           "const": "2.0",
           "type": "string"
         },
+        "config": {
+          "description": "Config entries used by steps, sorted by name (omitted when none). Fields of kind secretRef ({secret: NAME}) name a secret.",
+          "items": {
+            "additionalProperties": false,
+            "properties": {
+              "description": {
+                "type": "string"
+              },
+              "name": {
+                "description": "Config entry name; a key of the pipeline's config document.",
+                "pattern": "^[a-z][a-zA-Z0-9]*$",
+                "type": "string"
+              },
+              "required": {
+                "description": "False when the schema accepts a missing value (optional, or a default).",
+                "type": "boolean"
+              },
+              "schema": {
+                "additionalProperties": {},
+                "description": "z.input view: validates the config document's value for this entry (defaults listed).",
+                "propertyNames": {
+                  "type": "string"
+                },
+                "type": "object"
+              }
+            },
+            "required": [
+              "name",
+              "description",
+              "required",
+              "schema"
+            ],
+            "type": "object"
+          },
+          "type": "array"
+        },
         "description": {
           "type": "string"
         },
@@ -92,7 +128,7 @@
               "type": "string"
             },
             "sdk": {
-              "description": "@olve/pipelines package version.",
+              "description": "@olivervea/pipelines package version.",
               "type": "string"
             },
             "sdkHash": {
@@ -297,28 +333,76 @@
             "stages"
           ],
           "type": "object"
+        },
+        {
+          "additionalProperties": false,
+          "properties": {
+            "cases": {
+              "additionalProperties": {
+                "items": {
+                  "$ref": "#/$defs/Stage"
+                },
+                "type": "array"
+              },
+              "description": "Discriminator value → stages. Exactly one case runs; a step name may repeat across cases, nowhere else.",
+              "propertyNames": {
+                "type": "string"
+              },
+              "type": "object"
+            },
+            "discriminator": {
+              "description": "The union's literal field whose value names the case.",
+              "minLength": 1,
+              "type": "string"
+            },
+            "select": {
+              "description": "Config reference (`<entry>.<field>…`) to a discriminated union; the run's config snapshot picks the case.",
+              "minLength": 1,
+              "type": "string"
+            }
+          },
+          "required": [
+            "select",
+            "discriminator",
+            "cases"
+          ],
+          "type": "object"
         }
       ]
     },
     "Step": {
       "additionalProperties": false,
       "properties": {
+        "clusterAccess": {
+          "const": true,
+          "description": "The step pod gets the instance's step ServiceAccount token (Kubernetes:StepServiceAccount), if the instance grants one. Omitted = no token.",
+          "type": "boolean"
+        },
         "config": {
           "additionalProperties": {
             "$ref": "#/$defs/Json"
           },
-          "description": "Library-step configuration (display only; already covered by handlerHash).",
+          "description": "The step's settings (display only; already covered by handlerHash). A config reference is recorded as {\"$config\": \"<entry>.<field>…\"}; the entry is in configEntries.",
           "propertyNames": {
             "type": "string"
           },
           "type": "object"
         },
+        "configEntries": {
+          "description": "Config entries the step receives (sorted; omitted when none). The service delivers them in input.json under '$config'.",
+          "items": {
+            "description": "Config entry name; a key of the pipeline's config document.",
+            "pattern": "^[a-z][a-zA-Z0-9]*$",
+            "type": "string"
+          },
+          "type": "array"
+        },
         "description": {
           "description": "What the step does, shown in the UI (may be empty).",
           "type": "string"
         },
         "handlerHash": {
-          "description": "Implementation identity: handler bundle (contract files excluded) + harness (bun + sdkHash); for library steps kind + config + harness.",
+          "description": "Implementation identity: handler bundle (contract files excluded) + harness (bun + sdkHash); for a step that carries its implementation kind + config + harness + the hash of the code of its kind.",
           "pattern": "^sha256:[0-9a-f]{64}$",
           "type": "string"
         },
@@ -348,14 +432,8 @@
           "type": "object"
         },
         "kind": {
-          "description": "'custom' = handlers/<name>.ts; others are SDK library steps.",
-          "enum": [
-            "custom",
-            "script",
-            "gitHubSource",
-            "imageBuild",
-            "helmDeployOverSsh"
-          ],
+          "description": "'custom' = the handler is a handlers/*.ts file; any other kind is a step that carries its implementation (an SDK or package step). A label: the service accepts any kind.",
+          "minLength": 1,
           "type": "string"
         },
         "name": {
@@ -483,6 +561,78 @@
             "intervalSeconds"
           ],
           "type": "object"
+        },
+        {
+          "additionalProperties": false,
+          "description": "Inbound HTTP delivery to /api/hooks/<pipeline id>/<name>. Event = {headers, body}; a null mapping ignores it.",
+          "properties": {
+            "kind": {
+              "const": "webhook",
+              "type": "string"
+            },
+            "name": {
+              "pattern": "^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$",
+              "type": "string"
+            },
+            "verify": {
+              "oneOf": [
+                {
+                  "additionalProperties": false,
+                  "properties": {
+                    "header": {
+                      "minLength": 1,
+                      "type": "string"
+                    },
+                    "prefix": {
+                      "description": "Text before the hex digest in the header, e.g. 'sha256='.",
+                      "type": "string"
+                    },
+                    "scheme": {
+                      "const": "hmac-sha256",
+                      "type": "string"
+                    },
+                    "secret": {
+                      "description": "Secret name; also the env var name in the step container.",
+                      "pattern": "^[A-Z_][A-Z0-9_]*$",
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "scheme",
+                    "secret",
+                    "header",
+                    "prefix"
+                  ],
+                  "type": "object"
+                },
+                {
+                  "additionalProperties": false,
+                  "properties": {
+                    "scheme": {
+                      "const": "bearer",
+                      "type": "string"
+                    },
+                    "secret": {
+                      "description": "Secret name; also the env var name in the step container.",
+                      "pattern": "^[A-Z_][A-Z0-9_]*$",
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "scheme",
+                    "secret"
+                  ],
+                  "type": "object"
+                }
+              ]
+            }
+          },
+          "required": [
+            "kind",
+            "name",
+            "verify"
+          ],
+          "type": "object"
         }
       ]
     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.1",
+  "version": "2.0.0-beta.10",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/build/build.ts

@@ -1,14 +1,14 @@
-// olve-build: pipeline dir → single executable + step.tar.gz + manifest.json.
+// olve-build: pipeline dir → single executable + step.tar.zst + manifest.json.
 import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { canonicalJson } from "../util/canonical.ts";
 import { sha256File } from "../util/hash.ts";
 import type { HarnessInfo } from "../util/handler-hash.ts";
-import { writeTarGz } from "../util/tar.ts";
+import { writeTarZst } from "../util/tar.ts";
 import { MANIFEST_BEGIN, MANIFEST_END, uploadFile } from "../harness/result.ts";
 import { GLIBC_INTERP, MUSL_INTERP, readInterpreter } from "./elf.ts";
-import { handlerHashes, type HandlerHash } from "./hashes.ts";
-import { harnessInfo, loadProject, relPosix, type Project } from "./project.ts";
+import { handlerHashes, kindHashes, type HandlerHash } from "./hashes.ts";
+import { harnessInfo, loadPipeline, loadProject, relPosix, type Project } from "./project.ts";
 
 export const BUILD_FLAGS = ["--no-compile-autoload-dotenv", "--no-compile-autoload-bunfig", "--bytecode", "--format=esm"] as const;
 export type Target = "bun-linux-x64" | "bun-linux-x64-musl";
@@ -23,7 +23,7 @@ export class BuildError extends Error {
 export interface BuildOptions {
   out: string;
   target?: Target;
-  /** Pre-signed PUT for step.tar.gz. */
+  /** Pre-signed PUT for the step archive. */
   uploadUrl?: string;
   /** Print the manifest between sentinels on stdout (for the build Job's pod log). */
   sentinels?: boolean;
@@ -41,16 +41,17 @@ export interface BuildResult {
 }
 
 /** Writes `<dir>/.olve/gen/{handlers,entry}.ts`: the literal-import handler registry + binary entry. */
-export function generateEntry(project: Project, hashes: Record<string, string>, harness: HarnessInfo): string {
+export function generateEntry(project: Project, hashes: Record<string, string>, harness: HarnessInfo, kinds: Record<string, string> = {}): string {
   const gen = join(project.dir, ".olve", "gen");
   mkdirSync(gen, { recursive: true });
   const registry = project.handlers.map(([name, file]) => `  ${JSON.stringify(name)}: () => import(${JSON.stringify(relPosix(gen, file))}),`).join("\n");
   writeFileSync(
     join(gen, "handlers.ts"),
     `// GENERATED by olve-build — do not edit.\n` +
-      `import type { HandlerRegistry, HarnessInfo } from "@olve/pipelines/harness";\n\n` +
+      `import type { HandlerRegistry, HarnessInfo } from "@olivervea/pipelines/harness";\n\n` +
       `export const handlers: HandlerRegistry = {\n${registry}\n};\n\n` +
       `export const handlerHashes: Record<string, string> = ${JSON.stringify(hashes, null, 2)};\n\n` +
+      `export const kindHashes: Record<string, string> = ${JSON.stringify(kinds, null, 2)};\n\n` +
       `export const harness: HarnessInfo = ${JSON.stringify(harness, null, 2)};\n`,
   );
   const entry = join(gen, "entry.ts");
@@ -58,9 +59,9 @@ export function generateEntry(project: Project, hashes: Record<string, string>,
     entry,
     `// GENERATED by olve-build — do not edit. Entry point of the step binary.\n` +
       `import pipeline from ${JSON.stringify(relPosix(gen, project.pipelineFile))};\n` +
-      `import { runHarness } from "@olve/pipelines/harness";\n` +
-      `import { handlers, handlerHashes, harness } from "./handlers.ts";\n\n` +
-      `process.exit(await runHarness({ pipeline, handlers, handlerHashes, harness }, process.argv.slice(2)));\n`,
+      `import { runHarness } from "@olivervea/pipelines/harness";\n` +
+      `import { handlers, handlerHashes, kindHashes, harness } from "./handlers.ts";\n\n` +
+      `process.exit(await runHarness({ pipeline, handlers, handlerHashes, kindHashes, harness }, process.argv.slice(2)));\n`,
   );
   return entry;
 }
@@ -69,14 +70,15 @@ export async function build(dir: string, opts: BuildOptions): Promise<BuildResul
   const log = opts.log ?? ((l: string) => process.stderr.write(l + "\n"));
   const stdout = opts.stdout ?? (async (t: string) => void (await Bun.write(Bun.stdout, t)));
   const target = opts.target ?? "bun-linux-x64";
-  const project = loadProject(dir);
+  const project = await loadProject(dir);
   const out = resolve(opts.out);
   mkdirSync(out, { recursive: true });
   const harness = harnessInfo();
 
   const t0 = performance.now();
   const hashes = await handlerHashes(project, harness);
-  const entry = generateEntry(project, Object.fromEntries(Object.entries(hashes).map(([k, v]) => [k, v.handlerHash])), harness);
+  const kinds = await kindHashes(project, await loadPipeline(project));
+  const entry = generateEntry(project, Object.fromEntries(Object.entries(hashes).map(([k, v]) => [k, v.handlerHash])), harness, kinds);
   log(`olve-build: ${project.handlers.length} handler(s) hashed in ${((performance.now() - t0) / 1000).toFixed(2)}s`);
 
   // 1. compile
@@ -107,18 +109,18 @@ export async function build(dir: string, opts: BuildOptions): Promise<BuildResul
   const manifestFile = join(out, "manifest.json");
   writeFileSync(manifestFile, manifest);
 
-  // 4. step.tar.gz (deterministic: `step` at the root, 0755, mtime 0)
-  const archive = join(out, "step.tar.gz");
-  await writeTarGz(archive, [{ path: "step", type: "file", mode: 0o755, source: binary }]);
+  // 4. step.tar.zst (deterministic: `step` at the root, 0755, mtime 0)
+  const archive = join(out, "step.tar.zst");
+  await writeTarZst(archive, [{ path: "step", type: "file", mode: 0o755, source: binary }]);
   writeFileSync(
     join(out, "build.json"),
-    canonicalJson({ target, flags: BUILD_FLAGS, harness, handlers: hashes, binarySha256: sha256File(binary), archiveSha256: sha256File(archive) }),
+    canonicalJson({ target, flags: BUILD_FLAGS, harness, handlers: hashes, kinds, binarySha256: sha256File(binary), archiveSha256: sha256File(archive) }),
   );
   log(`olve-build: wrote ${archive}`);
 
   if (opts.uploadUrl) {
     await uploadFile(opts.uploadUrl, archive, log);
-    log("olve-build: uploaded step.tar.gz");
+    log("olve-build: uploaded step.tar.zst");
   }
   if (opts.sentinels) await stdout(`${MANIFEST_BEGIN}\n${manifest}${MANIFEST_END}\n`);
   return { binary, archive, manifestFile, manifest, hashes, harness };

package/src/build/cli.ts

@@ -32,7 +32,7 @@ async function main(): Promise<number> {
   const dir = positionals[0]!;
   try {
     if (values["hash-only"]) {
-      const h = await handlerHashes(loadProject(dir), harnessInfo());
+      const h = await handlerHashes(await loadProject(dir), harnessInfo());
       await Bun.write(Bun.stdout, canonicalJson({ harness: harnessInfo(), handlers: h }));
       return 0;
     }

package/src/build/hashes.ts

@@ -2,8 +2,12 @@
 // (tree-shaken → function-level granularity; embedded scripts are bundle outputs too), then hash the
 // outputs. Contract changes are versioned through the manifest's model/step entries, SDK changes
 // through `harness`; both are combined in composeHandlerHash.
-import { dirname, join, resolve } from "node:path";
+import { dirname, join, relative, resolve } from "node:path";
 import type { BunPlugin } from "bun";
+import { isImplementation } from "../core/handlers.ts";
+import { originsOf } from "../core/origin.ts";
+import { pipelineData } from "../builder/pipeline.ts";
+import { allSteps } from "../harness/main.ts";
 import { composeHandlerHash, type HarnessInfo } from "../util/handler-hash.ts";
 import { sha256Hex } from "../util/hash.ts";
 import type { Project } from "./project.ts";
@@ -22,7 +26,7 @@ function externals(project: Project): BunPlugin {
   return {
     name: "olve-handler-hash-externals",
     setup(b) {
-      b.onResolve({ filter: /^@olve\/pipelines(\/.*)?$/ }, (args) => ({ path: args.path, external: true }));
+      b.onResolve({ filter: /^@olivervea\/pipelines(\/.*)?$/ }, (args) => ({ path: args.path, external: true }));
       b.onResolve({ filter: /^\.{1,2}\// }, (args) => {
         const base = resolve(dirname(args.importer), args.path);
         for (const cand of [base, `${base}.ts`, join(base, "index.ts")]) if (contracts.has(cand)) return { path: args.path, external: true };
@@ -33,11 +37,12 @@ function externals(project: Project): BunPlugin {
 }
 
 export async function bundleHash(project: Project, entry: string): Promise<{ hash: string; outputs: string[] }> {
+  const inside = !relative(project.dir, entry).startsWith("..");
   const r = await Bun.build({
     entrypoints: [entry],
     target: "bun",
     format: "esm",
-    root: project.dir,
+    root: inside ? project.dir : dirname(entry),
     naming: { entry: "[dir]/[name].[ext]", chunk: "[dir]/[name].[ext]", asset: "[dir]/[name].[ext]" },
     plugins: [externals(project)],
     throw: false,
@@ -65,3 +70,27 @@ export async function handlerHashes(project: Project, harness: HarnessInfo): Pro
   }
   return out;
 }
+
+/**
+ * Kind → hash of the modules that implement the steps of that kind which carry their implementation
+ * (each module bundled like a handler: the SDK and contract files external).
+ */
+export async function kindHashes(project: Project, pipeline: unknown): Promise<Record<string, string>> {
+  const origins = new Map<string, Set<string>>();
+  for (const s of allSteps(pipelineData(pipeline))) {
+    if (!isImplementation(s)) continue;
+    const files = originsOf(s).filter((f) => !project.contractFiles.includes(f));
+    if (files.length === 0)
+      throw new Error(`step '${s.name}' carries its implementation but is defined in ${project.contractFiles.map((f) => relative(project.dir, f)).join(" or ")}; define and implement it in a module of its own (or a package)`);
+    const set = origins.get(s.kind) ?? new Set<string>();
+    for (const f of files) set.add(f);
+    origins.set(s.kind, set);
+  }
+  const out: Record<string, string> = {};
+  for (const [kind, files] of [...origins].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
+    const parts: string[] = [];
+    for (const f of [...files].sort()) parts.push((await bundleHash(project, f)).hash);
+    out[kind] = "sha256:" + sha256Hex(parts.join("\n"));
+  }
+  return out;
+}

package/src/build/project.ts

@@ -1,6 +1,7 @@
 // Layout of a pipeline directory and the identity of the SDK building it.
 import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { dirname, join, relative, resolve } from "node:path";
+import { isImplementation } from "../core/handlers.ts";
 import { sha256Hex } from "../util/hash.ts";
 import type { HarnessInfo } from "../util/handler-hash.ts";
 
@@ -11,7 +12,7 @@ export interface Project {
   pipelineFile: string;
   /** Contract files (excluded from handler bundles): pipeline.ts, models.ts. */
   contractFiles: string[];
-  /** step name → absolute handler file, sorted by name. */
+  /** step name → absolute handler file, sorted by step name. The step is the one the file's `implement(step, …)` names. */
   handlers: [string, string][];
 }
 
@@ -19,20 +20,47 @@ export class ProjectError extends Error {
   override name = "ProjectError";
 }
 
-export function loadProject(dirArg: string): Project {
+export async function loadProject(dirArg: string): Promise<Project> {
   const dir = resolve(dirArg);
   if (!existsSync(dir) || !statSync(dir).isDirectory()) throw new ProjectError(`${dirArg} is not a directory`);
   const pipelineFile = join(dir, "pipeline.ts");
   if (!existsSync(pipelineFile)) throw new ProjectError(`${join(dirArg, "pipeline.ts")} not found (it must default-export pipeline({...}))`);
   const contractFiles = ["pipeline.ts", "models.ts"].map((f) => join(dir, f)).filter((f) => existsSync(f));
   const hdir = join(dir, "handlers");
-  const handlers: [string, string][] = existsSync(hdir)
+  const files = existsSync(hdir)
     ? readdirSync(hdir)
         .filter((f) => f.endsWith(".ts") && !f.endsWith(".test.ts") && !f.endsWith(".d.ts") && statSync(join(hdir, f)).isFile())
         .sort()
-        .map((f) => [f.slice(0, -3), join(hdir, f)])
     : [];
-  return { dir, pipelineFile, contractFiles, handlers };
+  return { dir, pipelineFile, contractFiles, handlers: await resolveHandlers(hdir, files) };
+}
+
+/** The default export of pipeline.ts. */
+export async function loadPipeline(project: Project): Promise<unknown> {
+  const mod = (await import(project.pipelineFile)) as { default?: unknown };
+  if (!mod.default) throw new ProjectError(`${project.pipelineFile} has no default export (export default pipeline({...})...)`);
+  return mod.default;
+}
+
+/** Pairs each handler file with the step its default export implements (file names don't matter). */
+async function resolveHandlers(hdir: string, files: string[]): Promise<[string, string][]> {
+  const byStep = new Map<string, string>();
+  for (const f of files) {
+    const file = join(hdir, f);
+    let impl: unknown;
+    try {
+      impl = ((await import(file)) as { default?: unknown }).default;
+    } catch (e) {
+      throw new ProjectError(`handlers/${f} failed to load: ${(e as Error)?.message ?? String(e)}`);
+    }
+    if (!isImplementation(impl))
+      throw new ProjectError(`handlers/${f} must default-export implement(<step>, …); put shared code in a subfolder such as handlers/lib/`);
+    const step = (impl.step as { name: string }).name;
+    const other = byStep.get(step);
+    if (other) throw new ProjectError(`handlers/${other.slice(hdir.length + 1)} and handlers/${f} both implement step '${step}'`);
+    byStep.set(step, file);
+  }
+  return [...byStep].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
 }
 
 function listFiles(dir: string): string[] {