@olivervea/pipelines 2.0.0-beta.2 → 2.0.0-beta.20

package/src/build/elf.ts

@@ -1,22 +1,23 @@
-import { closeSync, openSync, readSync } from "node:fs";
+import { closeSync, openSync, readSync } from 'node:fs';
 
-export const GLIBC_INTERP = "/lib64/ld-linux-x86-64.so.2";
-export const MUSL_INTERP = "/lib/ld-musl-x86-64.so.1";
+export const GLIBC_INTERP = '/lib64/ld-linux-x86-64.so.2';
+export const MUSL_INTERP = '/lib/ld-musl-x86-64.so.1';
 
 function read(fd: number, pos: number, len: number): Buffer {
   const b = Buffer.alloc(len);
   const n = readSync(fd, b, 0, len, pos);
-  if (n !== len) throw new Error("truncated ELF file");
+  if (n !== len) throw new Error('truncated ELF file');
   return b;
 }
 
 /** PT_INTERP of an x86-64 ELF executable, or null if it has none (static). */
 export function readInterpreter(path: string): string | null {
-  const fd = openSync(path, "r");
+  const fd = openSync(path, 'r');
   try {
     const h = read(fd, 0, 64);
     if (h.readUInt32BE(0) !== 0x7f454c46) throw new Error(`${path} is not an ELF file`);
-    if (h[4] !== 2 || h[5] !== 1) throw new Error(`${path} is not a 64-bit little-endian ELF file`);
+    if (h[4] !== 2 || h[5] !== 1)
+      throw new Error(`${path} is not a 64-bit little-endian ELF file`);
     const phoff = Number(h.readBigUInt64LE(0x20));
     const phentsize = h.readUInt16LE(0x36);
     const phnum = h.readUInt16LE(0x38);
@@ -25,7 +26,7 @@ export function readInterpreter(path: string): string | null {
       if (ph.readUInt32LE(0) === 3 /* PT_INTERP */) {
         const off = Number(ph.readBigUInt64LE(0x08));
         const size = Number(ph.readBigUInt64LE(0x20));
-        return read(fd, off, size).toString("latin1").replace(/\0+$/, "");
+        return read(fd, off, size).toString('latin1').replace(/\0+$/, '');
       }
     }
     return null;

package/src/build/hashes.ts

@@ -2,11 +2,15 @@
 // (tree-shaken → function-level granularity; embedded scripts are bundle outputs too), then hash the
 // outputs. Contract changes are versioned through the manifest's model/step entries, SDK changes
 // through `harness`; both are combined in composeHandlerHash.
-import { dirname, join, resolve } from "node:path";
-import type { BunPlugin } from "bun";
-import { composeHandlerHash, type HarnessInfo } from "../util/handler-hash.ts";
-import { sha256Hex } from "../util/hash.ts";
-import type { Project } from "./project.ts";
+import { dirname, join, relative, resolve } from 'node:path';
+import type { BunPlugin } from 'bun';
+import { isImplementation } from '../core/handlers.ts';
+import { originsOf } from '../core/origin.ts';
+import { pipelineData } from '../builder/pipeline.ts';
+import { allSteps } from '../harness/main.ts';
+import { composeHandlerHash, type HarnessInfo } from '../util/handler-hash.ts';
+import { sha256Hex } from '../util/hash.ts';
+import type { Project } from './project.ts';
 
 export interface HandlerHash {
   /** Hash of the bundle outputs alone. */
@@ -20,48 +24,106 @@ export interface HandlerHash {
 function externals(project: Project): BunPlugin {
   const contracts = new Set(project.contractFiles);
   return {
-    name: "olve-handler-hash-externals",
+    name: 'olve-handler-hash-externals',
     setup(b) {
-      b.onResolve({ filter: /^@olve\/pipelines(\/.*)?$/ }, (args) => ({ path: args.path, external: true }));
+      b.onResolve({ filter: /^@olivervea\/pipelines(\/.*)?$/ }, (args) => ({
+        path: args.path,
+        external: true,
+      }));
       b.onResolve({ filter: /^\.{1,2}\// }, (args) => {
         const base = resolve(dirname(args.importer), args.path);
-        for (const cand of [base, `${base}.ts`, join(base, "index.ts")]) if (contracts.has(cand)) return { path: args.path, external: true };
+        for (const cand of [base, `${base}.ts`, join(base, 'index.ts')])
+          if (contracts.has(cand)) return { path: args.path, external: true };
         return undefined;
       });
     },
   };
 }
 
-export async function bundleHash(project: Project, entry: string): Promise<{ hash: string; outputs: string[] }> {
+export async function bundleHash(
+  project: Project,
+  entry: string,
+): Promise<{ hash: string; outputs: string[] }> {
+  const inside = !relative(project.dir, entry).startsWith('..');
   const r = await Bun.build({
     entrypoints: [entry],
-    target: "bun",
-    format: "esm",
-    root: project.dir,
-    naming: { entry: "[dir]/[name].[ext]", chunk: "[dir]/[name].[ext]", asset: "[dir]/[name].[ext]" },
+    target: 'bun',
+    format: 'esm',
+    root: inside ? project.dir : dirname(entry),
+    naming: {
+      entry: '[dir]/[name].[ext]',
+      chunk: '[dir]/[name].[ext]',
+      asset: '[dir]/[name].[ext]',
+    },
     plugins: [externals(project)],
     throw: false,
   });
-  if (!r.success) throw new AggregateError(r.logs, `bundling ${entry} failed:\n${r.logs.map(String).join("\n")}`);
+  if (!r.success)
+    throw new AggregateError(
+      r.logs,
+      `bundling ${entry} failed:\n${r.logs.map(String).join('\n')}`,
+    );
   const parts: string[] = [];
   const outputs: string[] = [];
-  for (const o of [...r.outputs].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))) {
-    const name = o.path.replace(/^\.\//, "");
+  for (const o of [...r.outputs].sort((a, b) =>
+    a.path < b.path ? -1 : a.path > b.path ? 1 : 0,
+  )) {
+    const name = o.path.replace(/^\.\//, '');
     outputs.push(name);
     const bytes = new Uint8Array(await o.arrayBuffer());
     // Bun prefixes each module with a `// <path>` comment relative to the *process cwd*; drop those
     // so the hash depends only on code, not on where the repo or the build tool runs.
-    const content = name.endsWith(".js") ? new TextDecoder().decode(bytes).replace(/^\/\/ \S+\n/gm, "") : bytes;
+    const content = name.endsWith('.js')
+      ? new TextDecoder().decode(bytes).replace(/^\/\/ \S+\n/gm, '')
+      : bytes;
     parts.push(`${name}\0${sha256Hex(content)}`);
   }
-  return { hash: "sha256:" + sha256Hex(parts.join("\n")), outputs };
+  return { hash: 'sha256:' + sha256Hex(parts.join('\n')), outputs };
 }
 
-export async function handlerHashes(project: Project, harness: HarnessInfo): Promise<Record<string, HandlerHash>> {
+export async function handlerHashes(
+  project: Project,
+  harness: HarnessInfo,
+): Promise<Record<string, HandlerHash>> {
   const out: Record<string, HandlerHash> = {};
   for (const [name, file] of project.handlers) {
     const b = await bundleHash(project, file);
-    out[name] = { bundleHash: b.hash, handlerHash: composeHandlerHash(b.hash, harness), outputs: b.outputs };
+    out[name] = {
+      bundleHash: b.hash,
+      handlerHash: composeHandlerHash(b.hash, harness),
+      outputs: b.outputs,
+    };
+  }
+  return out;
+}
+
+/**
+ * Kind → hash of the modules that implement the steps of that kind which carry their implementation
+ * (each module bundled like a handler: the SDK and contract files external).
+ */
+export async function kindHashes(
+  project: Project,
+  pipeline: unknown,
+): Promise<Record<string, string>> {
+  const origins = new Map<string, Set<string>>();
+  for (const s of allSteps(pipelineData(pipeline))) {
+    if (!isImplementation(s)) continue;
+    const files = originsOf(s).filter((f) => !project.contractFiles.includes(f));
+    if (files.length === 0)
+      throw new Error(
+        `step '${s.name}' carries its implementation but is defined in ${project.contractFiles.map((f) => relative(project.dir, f)).join(' or ')}; define and implement it in a module of its own (or a package)`,
+      );
+    const set = origins.get(s.kind) ?? new Set<string>();
+    for (const f of files) set.add(f);
+    origins.set(s.kind, set);
+  }
+  const out: Record<string, string> = {};
+  for (const [kind, files] of [...origins].sort(([a], [b]) =>
+    a < b ? -1 : a > b ? 1 : 0,
+  )) {
+    const parts: string[] = [];
+    for (const f of [...files].sort()) parts.push((await bundleHash(project, f)).hash);
+    out[kind] = 'sha256:' + sha256Hex(parts.join('\n'));
   }
   return out;
 }

package/src/build/project.ts

@@ -1,38 +1,96 @@
 // Layout of a pipeline directory and the identity of the SDK building it.
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { dirname, join, relative, resolve } from "node:path";
-import { sha256Hex } from "../util/hash.ts";
-import type { HarnessInfo } from "../util/handler-hash.ts";
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { dirname, join, relative, resolve } from 'node:path';
+import { isImplementation } from '../core/handlers.ts';
+import { sha256Hex } from '../util/hash.ts';
+import type { HarnessInfo } from '../util/handler-hash.ts';
 
-export const SDK_ROOT = resolve(import.meta.dir, "../..");
+export const SDK_ROOT = resolve(import.meta.dir, '../..');
 
 export interface Project {
   dir: string;
   pipelineFile: string;
   /** Contract files (excluded from handler bundles): pipeline.ts, models.ts. */
   contractFiles: string[];
-  /** step name → absolute handler file, sorted by name. */
+  /** step name → absolute handler file, sorted by step name. The step is the one the file's `implement(step, …)` names. */
   handlers: [string, string][];
 }
 
 export class ProjectError extends Error {
-  override name = "ProjectError";
+  override name = 'ProjectError';
 }
 
-export function loadProject(dirArg: string): Project {
+export async function loadProject(dirArg: string): Promise<Project> {
   const dir = resolve(dirArg);
-  if (!existsSync(dir) || !statSync(dir).isDirectory()) throw new ProjectError(`${dirArg} is not a directory`);
-  const pipelineFile = join(dir, "pipeline.ts");
-  if (!existsSync(pipelineFile)) throw new ProjectError(`${join(dirArg, "pipeline.ts")} not found (it must default-export pipeline({...}))`);
-  const contractFiles = ["pipeline.ts", "models.ts"].map((f) => join(dir, f)).filter((f) => existsSync(f));
-  const hdir = join(dir, "handlers");
-  const handlers: [string, string][] = existsSync(hdir)
+  if (!existsSync(dir) || !statSync(dir).isDirectory())
+    throw new ProjectError(`${dirArg} is not a directory`);
+  const pipelineFile = join(dir, 'pipeline.ts');
+  if (!existsSync(pipelineFile))
+    throw new ProjectError(
+      `${join(dirArg, 'pipeline.ts')} not found (it must default-export pipeline({...}))`,
+    );
+  const contractFiles = ['pipeline.ts', 'models.ts']
+    .map((f) => join(dir, f))
+    .filter((f) => existsSync(f));
+  const hdir = join(dir, 'handlers');
+  const files = existsSync(hdir)
     ? readdirSync(hdir)
-        .filter((f) => f.endsWith(".ts") && !f.endsWith(".test.ts") && !f.endsWith(".d.ts") && statSync(join(hdir, f)).isFile())
+        .filter(
+          (f) =>
+            f.endsWith('.ts') &&
+            !f.endsWith('.test.ts') &&
+            !f.endsWith('.d.ts') &&
+            statSync(join(hdir, f)).isFile(),
+        )
         .sort()
-        .map((f) => [f.slice(0, -3), join(hdir, f)])
     : [];
-  return { dir, pipelineFile, contractFiles, handlers };
+  return {
+    dir,
+    pipelineFile,
+    contractFiles,
+    handlers: await resolveHandlers(hdir, files),
+  };
+}
+
+/** The default export of pipeline.ts. */
+export async function loadPipeline(project: Project): Promise<unknown> {
+  const mod = (await import(project.pipelineFile)) as { default?: unknown };
+  if (!mod.default)
+    throw new ProjectError(
+      `${project.pipelineFile} has no default export (export default pipeline({...})...)`,
+    );
+  return mod.default;
+}
+
+/** Pairs each handler file with the step its default export implements (file names don't matter). */
+async function resolveHandlers(
+  hdir: string,
+  files: string[],
+): Promise<[string, string][]> {
+  const byStep = new Map<string, string>();
+  for (const f of files) {
+    const file = join(hdir, f);
+    let impl: unknown;
+    try {
+      impl = ((await import(file)) as { default?: unknown }).default;
+    } catch (e) {
+      throw new ProjectError(
+        `handlers/${f} failed to load: ${(e as Error)?.message ?? String(e)}`,
+      );
+    }
+    if (!isImplementation(impl))
+      throw new ProjectError(
+        `handlers/${f} must default-export implement(<step>, …); put shared code in a subfolder such as handlers/lib/`,
+      );
+    const step = (impl.step as { name: string }).name;
+    const other = byStep.get(step);
+    if (other)
+      throw new ProjectError(
+        `handlers/${other.slice(hdir.length + 1)} and handlers/${f} both implement step '${step}'`,
+      );
+    byStep.set(step, file);
+  }
+  return [...byStep].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
 }
 
 function listFiles(dir: string): string[] {
@@ -52,28 +110,37 @@ let cached: HarnessInfo | undefined;
  */
 export function harnessInfo(): HarnessInfo {
   if (cached) return cached;
-  const pkg = JSON.parse(readFileSync(join(SDK_ROOT, "package.json"), "utf8")) as { version: string };
-  const src = join(SDK_ROOT, "src");
+  const pkg = JSON.parse(readFileSync(join(SDK_ROOT, 'package.json'), 'utf8')) as {
+    version: string;
+  };
+  const src = join(SDK_ROOT, 'src');
   const files = listFiles(src)
-    .map((f) => relative(src, f).split("\\").join("/"))
-    .filter((f) => !f.startsWith("build/") && !f.startsWith("dev/") && !f.startsWith("testing/") && f.endsWith(".ts"))
+    .map((f) => relative(src, f).split('\\').join('/'))
+    .filter(
+      (f) =>
+        !f.startsWith('build/') &&
+        !f.startsWith('dev/') &&
+        !f.startsWith('testing/') &&
+        f.endsWith('.ts'),
+    )
     .sort();
-  const h = new Bun.CryptoHasher("sha256");
+  const h = new Bun.CryptoHasher('sha256');
   for (const f of files) h.update(`${f}\0${sha256Hex(readFileSync(join(src, f)))}\n`);
-  let zodVersion = "unknown";
+  let zodVersion = 'unknown';
   try {
-    const zodPkg = Bun.resolveSync("zod/package.json", SDK_ROOT);
-    zodVersion = (JSON.parse(readFileSync(zodPkg, "utf8")) as { version: string }).version;
+    const zodPkg = Bun.resolveSync('zod/package.json', SDK_ROOT);
+    zodVersion = (JSON.parse(readFileSync(zodPkg, 'utf8')) as { version: string })
+      .version;
   } catch {
     /* keep unknown */
   }
   h.update(`zod@${zodVersion}\n`);
-  cached = { bun: Bun.version, sdk: pkg.version, sdkHash: "sha256:" + h.digest("hex") };
+  cached = { bun: Bun.version, sdk: pkg.version, sdkHash: 'sha256:' + h.digest('hex') };
   return cached;
 }
 
 export const relPosix = (from: string, to: string) => {
-  const r = relative(from, to).split("\\").join("/");
-  return r.startsWith(".") ? r : `./${r}`;
+  const r = relative(from, to).split('\\').join('/');
+  return r.startsWith('.') ? r : `./${r}`;
 };
 export { dirname };