@olivervea/pipelines 2.0.0-beta.3 → 2.0.0-beta.30

package/src/build/cli.ts

@@ -1,12 +1,13 @@
 #!/usr/bin/env bun
 // olve-build <pipelineDir> --out <dir> [--target bun-linux-x64|bun-linux-x64-musl] [--upload-url <url>] [--sentinels] [--hash-only]
-import { parseArgs } from "node:util";
-import { canonicalJson } from "../util/canonical.ts";
-import { build, BuildError, type Target } from "./build.ts";
-import { handlerHashes } from "./hashes.ts";
-import { harnessInfo, loadProject, ProjectError } from "./project.ts";
+import { parseArgs } from 'node:util';
+import { canonicalJson } from '../util/canonical.ts';
+import { build, BuildError, type Target } from './build.ts';
+import { codeHash } from './hashes.ts';
+import { harnessInfo, loadProject, ProjectError } from './project.ts';
 
-const USAGE = "usage: olve-build <pipelineDir> --out <dir> [--target bun-linux-x64|bun-linux-x64-musl] [--upload-url <url>] [--sentinels]\n       olve-build <pipelineDir> --hash-only";
+const USAGE =
+  'usage: olve-build <pipelineDir> --out <dir> [--target bun-linux-x64|bun-linux-x64-musl] [--upload-url <url>] [--sentinels]\n       olve-build <pipelineDir> --hash-only';
 
 async function main(): Promise<number> {
   let parsed;
@@ -14,12 +15,12 @@ async function main(): Promise<number> {
     parsed = parseArgs({
       allowPositionals: true,
       options: {
-        out: { type: "string" },
-        target: { type: "string" },
-        "upload-url": { type: "string" },
-        sentinels: { type: "boolean", default: false },
-        "hash-only": { type: "boolean", default: false },
-        help: { type: "boolean", short: "h", default: false },
+        out: { type: 'string' },
+        target: { type: 'string' },
+        'upload-url': { type: 'string' },
+        sentinels: { type: 'boolean', default: false },
+        'hash-only': { type: 'boolean', default: false },
+        help: { type: 'boolean', short: 'h', default: false },
       },
     });
   } catch (e) {
@@ -27,23 +28,32 @@ async function main(): Promise<number> {
     return 64;
   }
   const { values, positionals } = parsed;
-  if (values.help) return console.log(USAGE), 0;
-  if (positionals.length !== 1) return console.error(USAGE), 64;
+  if (values.help) return (console.log(USAGE), 0);
+  if (positionals.length !== 1) return (console.error(USAGE), 64);
   const dir = positionals[0]!;
   try {
-    if (values["hash-only"]) {
-      const h = await handlerHashes(await loadProject(dir), harnessInfo());
-      await Bun.write(Bun.stdout, canonicalJson({ harness: harnessInfo(), handlers: h }));
+    if (values['hash-only']) {
+      const code = await codeHash(await loadProject(dir), harnessInfo());
+      await Bun.write(
+        Bun.stdout,
+        canonicalJson({ harness: harnessInfo(), codeHash: code }),
+      );
       return 0;
     }
-    if (!values.out) return console.error(`--out is required\n${USAGE}`), 64;
-    const target = (values.target ?? "bun-linux-x64") as Target;
-    if (target !== "bun-linux-x64" && target !== "bun-linux-x64-musl") return console.error(`unsupported target ${target}`), 64;
-    await build(dir, { out: values.out, target, uploadUrl: values["upload-url"], sentinels: values.sentinels });
+    if (!values.out) return (console.error(`--out is required\n${USAGE}`), 64);
+    const target = (values.target ?? 'bun-linux-x64') as Target;
+    if (target !== 'bun-linux-x64' && target !== 'bun-linux-x64-musl')
+      return (console.error(`unsupported target ${target}`), 64);
+    await build(dir, {
+      out: values.out,
+      target,
+      uploadUrl: values['upload-url'],
+      sentinels: values.sentinels,
+    });
     return 0;
   } catch (e) {
-    if (e instanceof BuildError) return console.error(e.message), e.exitCode;
-    if (e instanceof ProjectError) return console.error(e.message), 78;
+    if (e instanceof BuildError) return (console.error(e.message), e.exitCode);
+    if (e instanceof ProjectError) return (console.error(e.message), 78);
     console.error((e as Error)?.stack ?? String(e));
     return 1;
   }

package/src/build/elf.ts

@@ -1,22 +1,23 @@
-import { closeSync, openSync, readSync } from "node:fs";
+import { closeSync, openSync, readSync } from 'node:fs';
 
-export const GLIBC_INTERP = "/lib64/ld-linux-x86-64.so.2";
-export const MUSL_INTERP = "/lib/ld-musl-x86-64.so.1";
+export const GLIBC_INTERP = '/lib64/ld-linux-x86-64.so.2';
+export const MUSL_INTERP = '/lib/ld-musl-x86-64.so.1';
 
 function read(fd: number, pos: number, len: number): Buffer {
   const b = Buffer.alloc(len);
   const n = readSync(fd, b, 0, len, pos);
-  if (n !== len) throw new Error("truncated ELF file");
+  if (n !== len) throw new Error('truncated ELF file');
   return b;
 }
 
 /** PT_INTERP of an x86-64 ELF executable, or null if it has none (static). */
 export function readInterpreter(path: string): string | null {
-  const fd = openSync(path, "r");
+  const fd = openSync(path, 'r');
   try {
     const h = read(fd, 0, 64);
     if (h.readUInt32BE(0) !== 0x7f454c46) throw new Error(`${path} is not an ELF file`);
-    if (h[4] !== 2 || h[5] !== 1) throw new Error(`${path} is not a 64-bit little-endian ELF file`);
+    if (h[4] !== 2 || h[5] !== 1)
+      throw new Error(`${path} is not a 64-bit little-endian ELF file`);
     const phoff = Number(h.readBigUInt64LE(0x20));
     const phentsize = h.readUInt16LE(0x36);
     const phnum = h.readUInt16LE(0x38);
@@ -25,7 +26,7 @@ export function readInterpreter(path: string): string | null {
       if (ph.readUInt32LE(0) === 3 /* PT_INTERP */) {
         const off = Number(ph.readBigUInt64LE(0x08));
         const size = Number(ph.readBigUInt64LE(0x20));
-        return read(fd, off, size).toString("latin1").replace(/\0+$/, "");
+        return read(fd, off, size).toString('latin1').replace(/\0+$/, '');
       }
     }
     return null;
@@ -33,3 +34,11 @@ export function readInterpreter(path: string): string | null {
     closeSync(fd);
   }
 }
+
+/** Why `binary` won't start through `target`'s loader (the harness runs it via /olve/lib/<loader>), or undefined. */
+export function loaderProblem(binary: string, target: string): string | undefined {
+  const expected = target === 'bun-linux-x64' ? GLIBC_INTERP : MUSL_INTERP;
+  const interp = readInterpreter(binary);
+  if (interp === expected) return undefined;
+  return `compiled binary has PT_INTERP ${interp ?? '(none)'}, expected ${expected} — compile on a glibc Bun image (oven/bun:<version>-slim), not alpine`;
+}

package/src/build/hashes.ts

@@ -1,67 +1,56 @@
-// Per-handler hashes: bundle each handler on its own with the contract files and the SDK external
-// (tree-shaken → function-level granularity; embedded scripts are bundle outputs too), then hash the
-// outputs. Contract changes are versioned through the manifest's model/step entries, SDK changes
-// through `harness`; both are combined in composeHandlerHash.
-import { dirname, join, resolve } from "node:path";
-import type { BunPlugin } from "bun";
-import { composeHandlerHash, type HarnessInfo } from "../util/handler-hash.ts";
-import { sha256Hex } from "../util/hash.ts";
-import type { Project } from "./project.ts";
+// The build's code hash: everything the pipeline runs, bundled once with the SDK external (the
+// harness identity stands for it).
+import type { BunPlugin } from 'bun';
+import { sha256Hex } from '../util/hash.ts';
+import { composeCodeHash, type HarnessInfo } from '../util/code-hash.ts';
+import type { Project } from './project.ts';
 
-export interface HandlerHash {
-  /** Hash of the bundle outputs alone. */
-  bundleHash: string;
-  /** composeHandlerHash(bundleHash, harness) — what the manifest carries. */
-  handlerHash: string;
-  /** Bundle output names (for diagnostics). */
-  outputs: string[];
-}
-
-function externals(project: Project): BunPlugin {
-  const contracts = new Set(project.contractFiles);
-  return {
-    name: "olve-handler-hash-externals",
-    setup(b) {
-      b.onResolve({ filter: /^@olivervea\/pipelines(\/.*)?$/ }, (args) => ({ path: args.path, external: true }));
-      b.onResolve({ filter: /^\.{1,2}\// }, (args) => {
-        const base = resolve(dirname(args.importer), args.path);
-        for (const cand of [base, `${base}.ts`, join(base, "index.ts")]) if (contracts.has(cand)) return { path: args.path, external: true };
-        return undefined;
-      });
-    },
-  };
-}
+const sdkExternal: BunPlugin = {
+  name: 'olve-code-hash-external',
+  setup(b) {
+    b.onResolve({ filter: /^@olivervea\/pipelines(\/.*)?$/ }, (args) => ({
+      path: args.path,
+      external: true,
+    }));
+  },
+};
 
-export async function bundleHash(project: Project, entry: string): Promise<{ hash: string; outputs: string[] }> {
+async function bundleHash(project: Project): Promise<string> {
+  const entry = project.pipelineFile;
   const r = await Bun.build({
     entrypoints: [entry],
-    target: "bun",
-    format: "esm",
+    target: 'bun',
+    format: 'esm',
     root: project.dir,
-    naming: { entry: "[dir]/[name].[ext]", chunk: "[dir]/[name].[ext]", asset: "[dir]/[name].[ext]" },
-    plugins: [externals(project)],
+    naming: {
+      entry: '[dir]/[name].[ext]',
+      chunk: '[dir]/[name].[ext]',
+      asset: '[dir]/[name].[ext]',
+    },
+    plugins: [sdkExternal],
     throw: false,
   });
-  if (!r.success) throw new AggregateError(r.logs, `bundling ${entry} failed:\n${r.logs.map(String).join("\n")}`);
+  if (!r.success)
+    throw new AggregateError(
+      r.logs,
+      `bundling ${entry} failed:\n${r.logs.map(String).join('\n')}`,
+    );
   const parts: string[] = [];
-  const outputs: string[] = [];
-  for (const o of [...r.outputs].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))) {
-    const name = o.path.replace(/^\.\//, "");
-    outputs.push(name);
+  for (const o of [...r.outputs].sort((a, b) =>
+    a.path < b.path ? -1 : a.path > b.path ? 1 : 0,
+  )) {
+    const name = o.path.replace(/^\.\//, '');
     const bytes = new Uint8Array(await o.arrayBuffer());
     // Bun prefixes each module with a `// <path>` comment relative to the *process cwd*; drop those
     // so the hash depends only on code, not on where the repo or the build tool runs.
-    const content = name.endsWith(".js") ? new TextDecoder().decode(bytes).replace(/^\/\/ \S+\n/gm, "") : bytes;
+    const content = name.endsWith('.js')
+      ? new TextDecoder().decode(bytes).replace(/^\/\/ \S+\n/gm, '')
+      : bytes;
     parts.push(`${name}\0${sha256Hex(content)}`);
   }
-  return { hash: "sha256:" + sha256Hex(parts.join("\n")), outputs };
+  return 'sha256:' + sha256Hex(parts.join('\n'));
 }
 
-export async function handlerHashes(project: Project, harness: HarnessInfo): Promise<Record<string, HandlerHash>> {
-  const out: Record<string, HandlerHash> = {};
-  for (const [name, file] of project.handlers) {
-    const b = await bundleHash(project, file);
-    out[name] = { bundleHash: b.hash, handlerHash: composeHandlerHash(b.hash, harness), outputs: b.outputs };
-  }
-  return out;
+export async function codeHash(project: Project, harness: HarnessInfo): Promise<string> {
+  return composeCodeHash(await bundleHash(project), harness);
 }

package/src/build/project.ts

@@ -1,59 +1,53 @@
 // Layout of a pipeline directory and the identity of the SDK building it.
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { dirname, join, relative, resolve } from "node:path";
-import { isImplementation } from "../core/handlers.ts";
-import { sha256Hex } from "../util/hash.ts";
-import type { HarnessInfo } from "../util/handler-hash.ts";
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { dirname, join, relative, resolve } from 'node:path';
+import { sha256Hex } from '../util/hash.ts';
+import type { HarnessInfo } from '../util/code-hash.ts';
 
-export const SDK_ROOT = resolve(import.meta.dir, "../..");
+export const SDK_ROOT = resolve(import.meta.dir, '../..');
 
 export interface Project {
   dir: string;
   pipelineFile: string;
-  /** Contract files (excluded from handler bundles): pipeline.ts, models.ts. */
-  contractFiles: string[];
-  /** step name → absolute handler file, sorted by step name. The step is the one the file's `implement(step, …)` names. */
-  handlers: [string, string][];
 }
 
 export class ProjectError extends Error {
-  override name = "ProjectError";
+  override name = 'ProjectError';
 }
 
 export async function loadProject(dirArg: string): Promise<Project> {
   const dir = resolve(dirArg);
-  if (!existsSync(dir) || !statSync(dir).isDirectory()) throw new ProjectError(`${dirArg} is not a directory`);
-  const pipelineFile = join(dir, "pipeline.ts");
-  if (!existsSync(pipelineFile)) throw new ProjectError(`${join(dirArg, "pipeline.ts")} not found (it must default-export pipeline({...}))`);
-  const contractFiles = ["pipeline.ts", "models.ts"].map((f) => join(dir, f)).filter((f) => existsSync(f));
-  const hdir = join(dir, "handlers");
-  const files = existsSync(hdir)
-    ? readdirSync(hdir)
-        .filter((f) => f.endsWith(".ts") && !f.endsWith(".test.ts") && !f.endsWith(".d.ts") && statSync(join(hdir, f)).isFile())
-        .sort()
-    : [];
-  return { dir, pipelineFile, contractFiles, handlers: await resolveHandlers(hdir, files) };
+  if (!existsSync(dir) || !statSync(dir).isDirectory())
+    throw new ProjectError(`${dirArg} is not a directory`);
+  return { dir, pipelineFile: entryOf(dirArg, dir) };
 }
 
-/** Pairs each handler file with the step its default export implements (file names don't matter). */
-async function resolveHandlers(hdir: string, files: string[]): Promise<[string, string][]> {
-  const byStep = new Map<string, string>();
-  for (const f of files) {
-    const file = join(hdir, f);
-    let impl: unknown;
-    try {
-      impl = ((await import(file)) as { default?: unknown }).default;
-    } catch (e) {
-      throw new ProjectError(`handlers/${f} failed to load: ${(e as Error)?.message ?? String(e)}`);
-    }
-    if (!isImplementation(impl))
-      throw new ProjectError(`handlers/${f} must default-export implement(<step>, …); put shared code in a subfolder such as handlers/lib/`);
-    const step = (impl.step as { name: string }).name;
-    const other = byStep.get(step);
-    if (other) throw new ProjectError(`handlers/${other.slice(hdir.length + 1)} and handlers/${f} both implement step '${step}'`);
-    byStep.set(step, file);
-  }
-  return [...byStep].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+/** The file `main` in the package's package.json names. */
+function entryOf(dirArg: string, dir: string): string {
+  const pkg = join(dir, 'package.json');
+  const main = existsSync(pkg)
+    ? (JSON.parse(readFileSync(pkg, 'utf8')) as { main?: string }).main
+    : undefined;
+  if (!main)
+    throw new ProjectError(
+      `${join(dirArg, 'package.json')} must name the entry point in \`main\` (a file that default-exports pipeline({...}))`,
+    );
+  const entry = join(dir, main);
+  if (!existsSync(entry))
+    throw new ProjectError(
+      `${join(dirArg, main)} not found (it must default-export pipeline({...}))`,
+    );
+  return entry;
+}
+
+/** The default export of the entry point. */
+export async function loadPipeline(project: Project): Promise<unknown> {
+  const mod = (await import(project.pipelineFile)) as { default?: unknown };
+  if (!mod.default)
+    throw new ProjectError(
+      `${project.pipelineFile} has no default export (export default pipeline({...})...)`,
+    );
+  return mod.default;
 }
 
 function listFiles(dir: string): string[] {
@@ -73,28 +67,37 @@ let cached: HarnessInfo | undefined;
  */
 export function harnessInfo(): HarnessInfo {
   if (cached) return cached;
-  const pkg = JSON.parse(readFileSync(join(SDK_ROOT, "package.json"), "utf8")) as { version: string };
-  const src = join(SDK_ROOT, "src");
+  const pkg = JSON.parse(readFileSync(join(SDK_ROOT, 'package.json'), 'utf8')) as {
+    version: string;
+  };
+  const src = join(SDK_ROOT, 'src');
   const files = listFiles(src)
-    .map((f) => relative(src, f).split("\\").join("/"))
-    .filter((f) => !f.startsWith("build/") && !f.startsWith("dev/") && !f.startsWith("testing/") && f.endsWith(".ts"))
+    .map((f) => relative(src, f).split('\\').join('/'))
+    .filter(
+      (f) =>
+        !f.startsWith('build/') &&
+        !f.startsWith('dev/') &&
+        !f.startsWith('testing/') &&
+        f.endsWith('.ts'),
+    )
     .sort();
-  const h = new Bun.CryptoHasher("sha256");
+  const h = new Bun.CryptoHasher('sha256');
   for (const f of files) h.update(`${f}\0${sha256Hex(readFileSync(join(src, f)))}\n`);
-  let zodVersion = "unknown";
+  let zodVersion = 'unknown';
   try {
-    const zodPkg = Bun.resolveSync("zod/package.json", SDK_ROOT);
-    zodVersion = (JSON.parse(readFileSync(zodPkg, "utf8")) as { version: string }).version;
+    const zodPkg = Bun.resolveSync('zod/package.json', SDK_ROOT);
+    zodVersion = (JSON.parse(readFileSync(zodPkg, 'utf8')) as { version: string })
+      .version;
   } catch {
     /* keep unknown */
   }
   h.update(`zod@${zodVersion}\n`);
-  cached = { bun: Bun.version, sdk: pkg.version, sdkHash: "sha256:" + h.digest("hex") };
+  cached = { bun: Bun.version, sdk: pkg.version, sdkHash: 'sha256:' + h.digest('hex') };
   return cached;
 }
 
 export const relPosix = (from: string, to: string) => {
-  const r = relative(from, to).split("\\").join("/");
-  return r.startsWith(".") ? r : `./${r}`;
+  const r = relative(from, to).split('\\').join('/');
+  return r.startsWith('.') ? r : `./${r}`;
 };
 export { dirname };