@olivervea/pipelines 0.0.0-stage → 2.0.0-beta.10

package/package.json

@@ -1,6 +1,62 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "2.0.0-beta.10",
+  "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
+  "type": "module",
+  "license": "MIT",
+  "author": "Oliver Vea",
+  "homepage": "https://github.com/OliverVea/Olve.Pipelines/tree/v2/sdk#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/OliverVea/Olve.Pipelines.git",
+    "directory": "sdk"
+  },
+  "bugs": {
+    "url": "https://github.com/OliverVea/Olve.Pipelines/issues"
+  },
+  "keywords": [
+    "pipelines",
+    "cd",
+    "gitops",
+    "kubernetes",
+    "bun"
+  ],
+  "engines": {
+    "bun": ">=1.4.3"
+  },
+  "exports": {
+    ".": "./src/index.ts",
+    "./harness": "./src/harness/index.ts",
+    "./testing": "./src/testing/index.ts",
+    "./assets": "./src/assets.d.ts",
+    "./manifest.schema.json": "./manifest.schema.json"
+  },
+  "bin": {
+    "olve-build": "./src/build/cli.ts",
+    "olve-dev": "./src/dev/cli.ts"
+  },
+  "files": [
+    "src",
+    "manifest.schema.json",
+    "README.md",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "tag": "beta"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit -p .",
+    "test": "bun test",
+    "check-compile-fail": "bun scripts/check-compile-fail.ts",
+    "gen-schema": "bun scripts/gen-manifest-schema.ts",
+    "check": "tsc --noEmit -p . && bun scripts/check-compile-fail.ts && bun test"
+  },
+  "dependencies": {
+    "zod": "4.6.7"
+  },
+  "devDependencies": {
+    "@types/bun": "1.4.3",
+    "typescript": "7.0.2"
+  }
+}

package/src/assets.d.ts

@@ -0,0 +1,10 @@
+// `import script from "./scripts/x.sh" with { type: "file" }` yields a path string
+// (inside the compiled binary it points into the embedded /$bunfs filesystem).
+declare module "*.sh" {
+  const path: string;
+  export default path;
+}
+declare module "*.bash" {
+  const path: string;
+  export default path;
+}

package/src/build/build.ts

@@ -0,0 +1,127 @@
+// olve-build: pipeline dir → single executable + step.tar.zst + manifest.json.
+import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { canonicalJson } from "../util/canonical.ts";
+import { sha256File } from "../util/hash.ts";
+import type { HarnessInfo } from "../util/handler-hash.ts";
+import { writeTarZst } from "../util/tar.ts";
+import { MANIFEST_BEGIN, MANIFEST_END, uploadFile } from "../harness/result.ts";
+import { GLIBC_INTERP, MUSL_INTERP, readInterpreter } from "./elf.ts";
+import { handlerHashes, kindHashes, type HandlerHash } from "./hashes.ts";
+import { harnessInfo, loadPipeline, loadProject, relPosix, type Project } from "./project.ts";
+
+export const BUILD_FLAGS = ["--no-compile-autoload-dotenv", "--no-compile-autoload-bunfig", "--bytecode", "--format=esm"] as const;
+export type Target = "bun-linux-x64" | "bun-linux-x64-musl";
+
+export class BuildError extends Error {
+  override name = "BuildError";
+  constructor(message: string, readonly exitCode = 1) {
+    super(message);
+  }
+}
+
+export interface BuildOptions {
+  out: string;
+  target?: Target;
+  /** Pre-signed PUT for the step archive. */
+  uploadUrl?: string;
+  /** Print the manifest between sentinels on stdout (for the build Job's pod log). */
+  sentinels?: boolean;
+  log?: (line: string) => void;
+  stdout?: (text: string) => Promise<void>;
+}
+
+export interface BuildResult {
+  binary: string;
+  archive: string;
+  manifestFile: string;
+  manifest: string;
+  hashes: Record<string, HandlerHash>;
+  harness: HarnessInfo;
+}
+
+/** Writes `<dir>/.olve/gen/{handlers,entry}.ts`: the literal-import handler registry + binary entry. */
+export function generateEntry(project: Project, hashes: Record<string, string>, harness: HarnessInfo, kinds: Record<string, string> = {}): string {
+  const gen = join(project.dir, ".olve", "gen");
+  mkdirSync(gen, { recursive: true });
+  const registry = project.handlers.map(([name, file]) => `  ${JSON.stringify(name)}: () => import(${JSON.stringify(relPosix(gen, file))}),`).join("\n");
+  writeFileSync(
+    join(gen, "handlers.ts"),
+    `// GENERATED by olve-build — do not edit.\n` +
+      `import type { HandlerRegistry, HarnessInfo } from "@olivervea/pipelines/harness";\n\n` +
+      `export const handlers: HandlerRegistry = {\n${registry}\n};\n\n` +
+      `export const handlerHashes: Record<string, string> = ${JSON.stringify(hashes, null, 2)};\n\n` +
+      `export const kindHashes: Record<string, string> = ${JSON.stringify(kinds, null, 2)};\n\n` +
+      `export const harness: HarnessInfo = ${JSON.stringify(harness, null, 2)};\n`,
+  );
+  const entry = join(gen, "entry.ts");
+  writeFileSync(
+    entry,
+    `// GENERATED by olve-build — do not edit. Entry point of the step binary.\n` +
+      `import pipeline from ${JSON.stringify(relPosix(gen, project.pipelineFile))};\n` +
+      `import { runHarness } from "@olivervea/pipelines/harness";\n` +
+      `import { handlers, handlerHashes, kindHashes, harness } from "./handlers.ts";\n\n` +
+      `process.exit(await runHarness({ pipeline, handlers, handlerHashes, kindHashes, harness }, process.argv.slice(2)));\n`,
+  );
+  return entry;
+}
+
+export async function build(dir: string, opts: BuildOptions): Promise<BuildResult> {
+  const log = opts.log ?? ((l: string) => process.stderr.write(l + "\n"));
+  const stdout = opts.stdout ?? (async (t: string) => void (await Bun.write(Bun.stdout, t)));
+  const target = opts.target ?? "bun-linux-x64";
+  const project = await loadProject(dir);
+  const out = resolve(opts.out);
+  mkdirSync(out, { recursive: true });
+  const harness = harnessInfo();
+
+  const t0 = performance.now();
+  const hashes = await handlerHashes(project, harness);
+  const kinds = await kindHashes(project, await loadPipeline(project));
+  const entry = generateEntry(project, Object.fromEntries(Object.entries(hashes).map(([k, v]) => [k, v.handlerHash])), harness, kinds);
+  log(`olve-build: ${project.handlers.length} handler(s) hashed in ${((performance.now() - t0) / 1000).toFixed(2)}s`);
+
+  // 1. compile
+  const binary = join(out, "step");
+  rmSync(binary, { force: true });
+  const t1 = performance.now();
+  const p = Bun.spawnSync([process.execPath, "build", "--compile", `--target=${target}`, ...BUILD_FLAGS, entry, `--outfile=${binary}`], {
+    cwd: project.dir,
+    stdout: "pipe",
+    stderr: "pipe",
+    env: { ...process.env, NO_COLOR: "1" },
+  });
+  if (p.exitCode !== 0 || !existsSync(binary))
+    throw new BuildError(`bun build --compile failed (exit ${p.exitCode}):\n${p.stdout.toString()}${p.stderr.toString()}`);
+  log(`olve-build: compiled ${target} in ${((performance.now() - t1) / 1000).toFixed(2)}s`);
+
+  // 2. loader check: the harness launches the binary through /olve/lib/<loader>; a musl binary
+  //    (e.g. compiled on alpine, where bun-linux-x64 silently yields musl) would not start.
+  const expected = target === "bun-linux-x64" ? GLIBC_INTERP : MUSL_INTERP;
+  const interp = readInterpreter(binary);
+  if (interp !== expected)
+    throw new BuildError(`compiled binary has PT_INTERP ${interp ?? "(none)"}, expected ${expected} — compile on a glibc Bun image (oven/bun:<version>-slim), not alpine`);
+
+  // 3. describe (runs the real binary: the manifest is exactly what the step pods will see)
+  const d = Bun.spawnSync([binary, "describe"], { cwd: out, stdout: "pipe", stderr: "pipe", env: { PATH: process.env.PATH ?? "/usr/bin:/bin" } });
+  if (d.exitCode !== 0) throw new BuildError(`describe failed (exit ${d.exitCode}):\n${d.stderr.toString()}`, d.exitCode ?? 1);
+  const manifest = d.stdout.toString();
+  const manifestFile = join(out, "manifest.json");
+  writeFileSync(manifestFile, manifest);
+
+  // 4. step.tar.zst (deterministic: `step` at the root, 0755, mtime 0)
+  const archive = join(out, "step.tar.zst");
+  await writeTarZst(archive, [{ path: "step", type: "file", mode: 0o755, source: binary }]);
+  writeFileSync(
+    join(out, "build.json"),
+    canonicalJson({ target, flags: BUILD_FLAGS, harness, handlers: hashes, kinds, binarySha256: sha256File(binary), archiveSha256: sha256File(archive) }),
+  );
+  log(`olve-build: wrote ${archive}`);
+
+  if (opts.uploadUrl) {
+    await uploadFile(opts.uploadUrl, archive, log);
+    log("olve-build: uploaded step.tar.zst");
+  }
+  if (opts.sentinels) await stdout(`${MANIFEST_BEGIN}\n${manifest}${MANIFEST_END}\n`);
+  return { binary, archive, manifestFile, manifest, hashes, harness };
+}

package/src/build/cli.ts

@@ -0,0 +1,52 @@
+#!/usr/bin/env bun
+// olve-build <pipelineDir> --out <dir> [--target bun-linux-x64|bun-linux-x64-musl] [--upload-url <url>] [--sentinels] [--hash-only]
+import { parseArgs } from "node:util";
+import { canonicalJson } from "../util/canonical.ts";
+import { build, BuildError, type Target } from "./build.ts";
+import { handlerHashes } from "./hashes.ts";
+import { harnessInfo, loadProject, ProjectError } from "./project.ts";
+
+const USAGE = "usage: olve-build <pipelineDir> --out <dir> [--target bun-linux-x64|bun-linux-x64-musl] [--upload-url <url>] [--sentinels]\n       olve-build <pipelineDir> --hash-only";
+
+async function main(): Promise<number> {
+  let parsed;
+  try {
+    parsed = parseArgs({
+      allowPositionals: true,
+      options: {
+        out: { type: "string" },
+        target: { type: "string" },
+        "upload-url": { type: "string" },
+        sentinels: { type: "boolean", default: false },
+        "hash-only": { type: "boolean", default: false },
+        help: { type: "boolean", short: "h", default: false },
+      },
+    });
+  } catch (e) {
+    console.error(`${(e as Error).message}\n${USAGE}`);
+    return 64;
+  }
+  const { values, positionals } = parsed;
+  if (values.help) return console.log(USAGE), 0;
+  if (positionals.length !== 1) return console.error(USAGE), 64;
+  const dir = positionals[0]!;
+  try {
+    if (values["hash-only"]) {
+      const h = await handlerHashes(await loadProject(dir), harnessInfo());
+      await Bun.write(Bun.stdout, canonicalJson({ harness: harnessInfo(), handlers: h }));
+      return 0;
+    }
+    if (!values.out) return console.error(`--out is required\n${USAGE}`), 64;
+    const target = (values.target ?? "bun-linux-x64") as Target;
+    if (target !== "bun-linux-x64" && target !== "bun-linux-x64-musl") return console.error(`unsupported target ${target}`), 64;
+    await build(dir, { out: values.out, target, uploadUrl: values["upload-url"], sentinels: values.sentinels });
+    return 0;
+  } catch (e) {
+    if (e instanceof BuildError) return console.error(e.message), e.exitCode;
+    if (e instanceof ProjectError) return console.error(e.message), 78;
+    console.error((e as Error)?.stack ?? String(e));
+    return 1;
+  }
+}
+
+if (import.meta.main) process.exit(await main());

package/src/build/elf.ts

@@ -0,0 +1,35 @@
+import { closeSync, openSync, readSync } from "node:fs";
+
+export const GLIBC_INTERP = "/lib64/ld-linux-x86-64.so.2";
+export const MUSL_INTERP = "/lib/ld-musl-x86-64.so.1";
+
+function read(fd: number, pos: number, len: number): Buffer {
+  const b = Buffer.alloc(len);
+  const n = readSync(fd, b, 0, len, pos);
+  if (n !== len) throw new Error("truncated ELF file");
+  return b;
+}
+
+/** PT_INTERP of an x86-64 ELF executable, or null if it has none (static). */
+export function readInterpreter(path: string): string | null {
+  const fd = openSync(path, "r");
+  try {
+    const h = read(fd, 0, 64);
+    if (h.readUInt32BE(0) !== 0x7f454c46) throw new Error(`${path} is not an ELF file`);
+    if (h[4] !== 2 || h[5] !== 1) throw new Error(`${path} is not a 64-bit little-endian ELF file`);
+    const phoff = Number(h.readBigUInt64LE(0x20));
+    const phentsize = h.readUInt16LE(0x36);
+    const phnum = h.readUInt16LE(0x38);
+    for (let i = 0; i < phnum; i++) {
+      const ph = read(fd, phoff + i * phentsize, phentsize);
+      if (ph.readUInt32LE(0) === 3 /* PT_INTERP */) {
+        const off = Number(ph.readBigUInt64LE(0x08));
+        const size = Number(ph.readBigUInt64LE(0x20));
+        return read(fd, off, size).toString("latin1").replace(/\0+$/, "");
+      }
+    }
+    return null;
+  } finally {
+    closeSync(fd);
+  }
+}

package/src/build/hashes.ts

@@ -0,0 +1,96 @@
+// Per-handler hashes: bundle each handler on its own with the contract files and the SDK external
+// (tree-shaken → function-level granularity; embedded scripts are bundle outputs too), then hash the
+// outputs. Contract changes are versioned through the manifest's model/step entries, SDK changes
+// through `harness`; both are combined in composeHandlerHash.
+import { dirname, join, relative, resolve } from "node:path";
+import type { BunPlugin } from "bun";
+import { isImplementation } from "../core/handlers.ts";
+import { originsOf } from "../core/origin.ts";
+import { pipelineData } from "../builder/pipeline.ts";
+import { allSteps } from "../harness/main.ts";
+import { composeHandlerHash, type HarnessInfo } from "../util/handler-hash.ts";
+import { sha256Hex } from "../util/hash.ts";
+import type { Project } from "./project.ts";
+
+export interface HandlerHash {
+  /** Hash of the bundle outputs alone. */
+  bundleHash: string;
+  /** composeHandlerHash(bundleHash, harness) — what the manifest carries. */
+  handlerHash: string;
+  /** Bundle output names (for diagnostics). */
+  outputs: string[];
+}
+
+function externals(project: Project): BunPlugin {
+  const contracts = new Set(project.contractFiles);
+  return {
+    name: "olve-handler-hash-externals",
+    setup(b) {
+      b.onResolve({ filter: /^@olivervea\/pipelines(\/.*)?$/ }, (args) => ({ path: args.path, external: true }));
+      b.onResolve({ filter: /^\.{1,2}\// }, (args) => {
+        const base = resolve(dirname(args.importer), args.path);
+        for (const cand of [base, `${base}.ts`, join(base, "index.ts")]) if (contracts.has(cand)) return { path: args.path, external: true };
+        return undefined;
+      });
+    },
+  };
+}
+
+export async function bundleHash(project: Project, entry: string): Promise<{ hash: string; outputs: string[] }> {
+  const inside = !relative(project.dir, entry).startsWith("..");
+  const r = await Bun.build({
+    entrypoints: [entry],
+    target: "bun",
+    format: "esm",
+    root: inside ? project.dir : dirname(entry),
+    naming: { entry: "[dir]/[name].[ext]", chunk: "[dir]/[name].[ext]", asset: "[dir]/[name].[ext]" },
+    plugins: [externals(project)],
+    throw: false,
+  });
+  if (!r.success) throw new AggregateError(r.logs, `bundling ${entry} failed:\n${r.logs.map(String).join("\n")}`);
+  const parts: string[] = [];
+  const outputs: string[] = [];
+  for (const o of [...r.outputs].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))) {
+    const name = o.path.replace(/^\.\//, "");
+    outputs.push(name);
+    const bytes = new Uint8Array(await o.arrayBuffer());
+    // Bun prefixes each module with a `// <path>` comment relative to the *process cwd*; drop those
+    // so the hash depends only on code, not on where the repo or the build tool runs.
+    const content = name.endsWith(".js") ? new TextDecoder().decode(bytes).replace(/^\/\/ \S+\n/gm, "") : bytes;
+    parts.push(`${name}\0${sha256Hex(content)}`);
+  }
+  return { hash: "sha256:" + sha256Hex(parts.join("\n")), outputs };
+}
+
+export async function handlerHashes(project: Project, harness: HarnessInfo): Promise<Record<string, HandlerHash>> {
+  const out: Record<string, HandlerHash> = {};
+  for (const [name, file] of project.handlers) {
+    const b = await bundleHash(project, file);
+    out[name] = { bundleHash: b.hash, handlerHash: composeHandlerHash(b.hash, harness), outputs: b.outputs };
+  }
+  return out;
+}
+
+/**
+ * Kind → hash of the modules that implement the steps of that kind which carry their implementation
+ * (each module bundled like a handler: the SDK and contract files external).
+ */
+export async function kindHashes(project: Project, pipeline: unknown): Promise<Record<string, string>> {
+  const origins = new Map<string, Set<string>>();
+  for (const s of allSteps(pipelineData(pipeline))) {
+    if (!isImplementation(s)) continue;
+    const files = originsOf(s).filter((f) => !project.contractFiles.includes(f));
+    if (files.length === 0)
+      throw new Error(`step '${s.name}' carries its implementation but is defined in ${project.contractFiles.map((f) => relative(project.dir, f)).join(" or ")}; define and implement it in a module of its own (or a package)`);
+    const set = origins.get(s.kind) ?? new Set<string>();
+    for (const f of files) set.add(f);
+    origins.set(s.kind, set);
+  }
+  const out: Record<string, string> = {};
+  for (const [kind, files] of [...origins].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
+    const parts: string[] = [];
+    for (const f of [...files].sort()) parts.push((await bundleHash(project, f)).hash);
+    out[kind] = "sha256:" + sha256Hex(parts.join("\n"));
+  }
+  return out;
+}

package/src/build/project.ts

@@ -0,0 +1,107 @@
+// Layout of a pipeline directory and the identity of the SDK building it.
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import { isImplementation } from "../core/handlers.ts";
+import { sha256Hex } from "../util/hash.ts";
+import type { HarnessInfo } from "../util/handler-hash.ts";
+
+export const SDK_ROOT = resolve(import.meta.dir, "../..");
+
+export interface Project {
+  dir: string;
+  pipelineFile: string;
+  /** Contract files (excluded from handler bundles): pipeline.ts, models.ts. */
+  contractFiles: string[];
+  /** step name → absolute handler file, sorted by step name. The step is the one the file's `implement(step, …)` names. */
+  handlers: [string, string][];
+}
+
+export class ProjectError extends Error {
+  override name = "ProjectError";
+}
+
+export async function loadProject(dirArg: string): Promise<Project> {
+  const dir = resolve(dirArg);
+  if (!existsSync(dir) || !statSync(dir).isDirectory()) throw new ProjectError(`${dirArg} is not a directory`);
+  const pipelineFile = join(dir, "pipeline.ts");
+  if (!existsSync(pipelineFile)) throw new ProjectError(`${join(dirArg, "pipeline.ts")} not found (it must default-export pipeline({...}))`);
+  const contractFiles = ["pipeline.ts", "models.ts"].map((f) => join(dir, f)).filter((f) => existsSync(f));
+  const hdir = join(dir, "handlers");
+  const files = existsSync(hdir)
+    ? readdirSync(hdir)
+        .filter((f) => f.endsWith(".ts") && !f.endsWith(".test.ts") && !f.endsWith(".d.ts") && statSync(join(hdir, f)).isFile())
+        .sort()
+    : [];
+  return { dir, pipelineFile, contractFiles, handlers: await resolveHandlers(hdir, files) };
+}
+
+/** The default export of pipeline.ts. */
+export async function loadPipeline(project: Project): Promise<unknown> {
+  const mod = (await import(project.pipelineFile)) as { default?: unknown };
+  if (!mod.default) throw new ProjectError(`${project.pipelineFile} has no default export (export default pipeline({...})...)`);
+  return mod.default;
+}
+
+/** Pairs each handler file with the step its default export implements (file names don't matter). */
+async function resolveHandlers(hdir: string, files: string[]): Promise<[string, string][]> {
+  const byStep = new Map<string, string>();
+  for (const f of files) {
+    const file = join(hdir, f);
+    let impl: unknown;
+    try {
+      impl = ((await import(file)) as { default?: unknown }).default;
+    } catch (e) {
+      throw new ProjectError(`handlers/${f} failed to load: ${(e as Error)?.message ?? String(e)}`);
+    }
+    if (!isImplementation(impl))
+      throw new ProjectError(`handlers/${f} must default-export implement(<step>, …); put shared code in a subfolder such as handlers/lib/`);
+    const step = (impl.step as { name: string }).name;
+    const other = byStep.get(step);
+    if (other) throw new ProjectError(`handlers/${other.slice(hdir.length + 1)} and handlers/${f} both implement step '${step}'`);
+    byStep.set(step, file);
+  }
+  return [...byStep].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+}
+
+function listFiles(dir: string): string[] {
+  const out: string[] = [];
+  for (const e of readdirSync(dir, { withFileTypes: true })) {
+    const p = join(dir, e.name);
+    if (e.isDirectory()) out.push(...listFiles(p));
+    else if (e.isFile()) out.push(p);
+  }
+  return out;
+}
+
+let cached: HarnessInfo | undefined;
+/**
+ * Harness identity: Bun version, SDK version, and a hash over the SDK's runtime sources (src/,
+ * without the build and dev tools) plus the zod version the SDK resolves.
+ */
+export function harnessInfo(): HarnessInfo {
+  if (cached) return cached;
+  const pkg = JSON.parse(readFileSync(join(SDK_ROOT, "package.json"), "utf8")) as { version: string };
+  const src = join(SDK_ROOT, "src");
+  const files = listFiles(src)
+    .map((f) => relative(src, f).split("\\").join("/"))
+    .filter((f) => !f.startsWith("build/") && !f.startsWith("dev/") && !f.startsWith("testing/") && f.endsWith(".ts"))
+    .sort();
+  const h = new Bun.CryptoHasher("sha256");
+  for (const f of files) h.update(`${f}\0${sha256Hex(readFileSync(join(src, f)))}\n`);
+  let zodVersion = "unknown";
+  try {
+    const zodPkg = Bun.resolveSync("zod/package.json", SDK_ROOT);
+    zodVersion = (JSON.parse(readFileSync(zodPkg, "utf8")) as { version: string }).version;
+  } catch {
+    /* keep unknown */
+  }
+  h.update(`zod@${zodVersion}\n`);
+  cached = { bun: Bun.version, sdk: pkg.version, sdkHash: "sha256:" + h.digest("hex") };
+  return cached;
+}
+
+export const relPosix = (from: string, to: string) => {
+  const r = relative(from, to).split("\\").join("/");
+  return r.startsWith(".") ? r : `./${r}`;
+};
+export { dirname };