@olivervea/pipelines 2.0.0-beta.3 → 2.0.0-beta.31

package/src/util/tar.ts

@@ -1,27 +1,44 @@
 // Deterministic tar writer (ustar + PAX long names), streaming from disk. Entries are sorted,
 // with mtime 0, uid/gid 0, no user names — so equal content gives byte-identical archives (and
 // sha256s). Extraction uses Bun.Archive.
-import { closeSync, createReadStream, createWriteStream, lstatSync, openSync, readdirSync, readlinkSync, readSync, renameSync, rmSync, writeSync } from "node:fs";
-import { join } from "node:path";
-import { pipeline } from "node:stream/promises";
-import { createGzip } from "node:zlib";
+import {
+  closeSync,
+  createReadStream,
+  createWriteStream,
+  lstatSync,
+  openSync,
+  readdirSync,
+  readlinkSync,
+  readSync,
+  renameSync,
+  rmSync,
+  writeSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import { createGzip } from 'node:zlib';
 
 export type TarEntry =
-  | { path: string; type: "file"; mode: number; source: string }
-  | { path: string; type: "bytes"; mode: number; data: Uint8Array }
-  | { path: string; type: "dir"; mode: number }
-  | { path: string; type: "symlink"; target: string };
+  | { path: string; type: 'file'; mode: number; source: string }
+  | { path: string; type: 'dir'; mode: number }
+  | { path: string; type: 'symlink'; target: string };
 
 const enc = new TextEncoder();
 
 function octal(n: number, len: number): string {
-  return n.toString(8).padStart(len - 1, "0") + "\0";
+  return n.toString(8).padStart(len - 1, '0') + '\0';
 }
 
-function header(name: string, size: number, mode: number, typeflag: string, linkname = ""): Uint8Array {
+function header(
+  name: string,
+  size: number,
+  mode: number,
+  typeflag: string,
+  linkname = '',
+): Uint8Array {
   const h = new Uint8Array(512);
   const put = (s: string | Uint8Array, off: number, len: number) => {
-    const b = typeof s === "string" ? enc.encode(s) : s;
+    const b = typeof s === 'string' ? enc.encode(s) : s;
     h.set(b.subarray(0, len), off);
   };
   put(name, 0, 100);
@@ -30,14 +47,14 @@ function header(name: string, size: number, mode: number, typeflag: string, link
   put(octal(0, 8), 116, 8);
   put(octal(size, 12), 124, 12);
   put(octal(0, 12), 136, 12);
-  put("        ", 148, 8); // checksum placeholder
+  put('        ', 148, 8); // checksum placeholder
   put(typeflag, 156, 1);
   put(linkname, 157, 100);
-  put("ustar\0", 257, 6);
-  put("00", 263, 2);
+  put('ustar\0', 257, 6);
+  put('00', 263, 2);
   let sum = 0;
   for (const b of h) sum += b;
-  put(octal(sum, 7) + " ", 148, 8);
+  put(octal(sum, 7) + ' ', 148, 8);
   return h;
 }
 
@@ -51,9 +68,11 @@ function paxRecord(key: string, value: string): string {
 
 /** Writes `entries` (sorted by path here) as an uncompressed tar to `out`. */
 export function writeTar(out: string, entries: readonly TarEntry[]): void {
-  const sorted = [...entries].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const sorted = [...entries].sort((a, b) =>
+    a.path < b.path ? -1 : a.path > b.path ? 1 : 0,
+  );
   const tmp = `${out}.partial`;
-  const fd = openSync(tmp, "w");
+  const fd = openSync(tmp, 'w');
   const write = (b: Uint8Array) => {
     let off = 0;
     while (off < b.length) off += writeSync(fd, b, off, b.length - off);
@@ -64,29 +83,27 @@ export function writeTar(out: string, entries: readonly TarEntry[]): void {
   };
   try {
     for (const e of sorted) {
-      const name = e.type === "dir" && !e.path.endsWith("/") ? e.path + "/" : e.path;
-      const link = e.type === "symlink" ? e.target : "";
+      const name = e.type === 'dir' && !e.path.endsWith('/') ? e.path + '/' : e.path;
+      const link = e.type === 'symlink' ? e.target : '';
       let size = 0;
-      if (e.type === "file") size = lstatSync(e.source).size;
-      else if (e.type === "bytes") size = e.data.length;
+      if (e.type === 'file') size = lstatSync(e.source).size;
       const pax: string[] = [];
-      if (enc.encode(name).length > 100) pax.push(paxRecord("path", name));
-      if (enc.encode(link).length > 100) pax.push(paxRecord("linkpath", link));
+      if (enc.encode(name).length > 100) pax.push(paxRecord('path', name));
+      if (enc.encode(link).length > 100) pax.push(paxRecord('linkpath', link));
       if (pax.length) {
-        const body = enc.encode(pax.join(""));
-        write(header("././@PaxHeader", body.length, 0o644, "x"));
+        const body = enc.encode(pax.join(''));
+        write(header('././@PaxHeader', body.length, 0o644, 'x'));
         write(body);
         pad(body.length);
       }
       const shortName = enc.encode(name).length > 100 ? name.slice(0, 99) : name;
-      const mode = e.type === "symlink" ? 0o777 : e.mode;
-      const flag = e.type === "dir" ? "5" : e.type === "symlink" ? "2" : "0";
-      write(header(shortName, size, mode, flag, link.length > 100 ? link.slice(0, 99) : link));
-      if (e.type === "bytes") {
-        write(e.data);
-        pad(size);
-      } else if (e.type === "file") {
-        const src = openSync(e.source, "r");
+      const mode = e.type === 'symlink' ? 0o777 : e.mode;
+      const flag = e.type === 'dir' ? '5' : e.type === 'symlink' ? '2' : '0';
+      write(
+        header(shortName, size, mode, flag, link.length > 100 ? link.slice(0, 99) : link),
+      );
+      if (e.type === 'file') {
+        const src = openSync(e.source, 'r');
         try {
           const buf = new Uint8Array(1 << 20);
           let left = size;
@@ -110,18 +127,25 @@ export function writeTar(out: string, entries: readonly TarEntry[]): void {
 }
 
 /** Entries for the contents of `dir` (recursively), with paths prefixed by `prefix` ("" = none). */
-export function dirEntries(dir: string, prefix = ""): TarEntry[] {
+export function dirEntries(dir: string, prefix = ''): TarEntry[] {
   const out: TarEntry[] = [];
   const walk = (abs: string, rel: string) => {
     for (const name of readdirSync(abs).sort()) {
       const a = join(abs, name);
       const r = rel ? `${rel}/${name}` : name;
       const st = lstatSync(a);
-      if (st.isSymbolicLink()) out.push({ path: r, type: "symlink", target: readlinkSync(a) });
+      if (st.isSymbolicLink())
+        out.push({ path: r, type: 'symlink', target: readlinkSync(a) });
       else if (st.isDirectory()) {
-        out.push({ path: r, type: "dir", mode: 0o755 });
+        out.push({ path: r, type: 'dir', mode: 0o755 });
         walk(a, r);
-      } else if (st.isFile()) out.push({ path: r, type: "file", mode: st.mode & 0o111 ? 0o755 : 0o644, source: a });
+      } else if (st.isFile())
+        out.push({
+          path: r,
+          type: 'file',
+          mode: st.mode & 0o111 ? 0o755 : 0o644,
+          source: a,
+        });
       // sockets, fifos, devices are skipped
     }
   };
@@ -137,7 +161,11 @@ export async function gzipFile(src: string, out: string, level = 6): Promise<voi
 }
 
 /** Writes a deterministic `.tar.gz`. */
-export async function writeTarGz(out: string, entries: readonly TarEntry[], level = 6): Promise<void> {
+export async function writeTarGz(
+  out: string,
+  entries: readonly TarEntry[],
+  level = 6,
+): Promise<void> {
   const tar = `${out}.tar-tmp`;
   try {
     writeTar(tar, entries);
@@ -147,6 +175,23 @@ export async function writeTarGz(out: string, entries: readonly TarEntry[], leve
   }
 }
 
+/** Writes a deterministic zstd-compressed tar (level 3: fast to write, and unpacks far faster than gzip). */
+export async function writeTarZst(
+  out: string,
+  entries: readonly TarEntry[],
+  level = 3,
+): Promise<void> {
+  const tar = `${out}.tar-tmp`;
+  const tmp = `${out}.partial`;
+  try {
+    writeTar(tar, entries);
+    await Bun.write(tmp, await Bun.zstdCompress(await Bun.file(tar).bytes(), { level }));
+    renameSync(tmp, out);
+  } finally {
+    rmSync(tar, { force: true });
+  }
+}
+
 /** Extracts a tar or tar.gz file into `dir`. */
 export async function extractTar(archive: string, dir: string): Promise<void> {
   const a = new Bun.Archive(await Bun.file(archive).bytes());

package/src/library/script.ts

@@ -1,85 +0,0 @@
-import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import type { Ctx, RunInfo } from "../ctx/types.ts";
-import type { AnyModel } from "../core/models.ts";
-import type { Secret } from "../core/secrets.ts";
-import { makeStep, type InputSpec, type LibraryStep, type Step, type StepDef, type StepInput } from "../core/steps.ts";
-import { sha256Hex } from "../util/hash.ts";
-
-/** A config value that may depend on the run: a constant or a function of the step's inputs and run info. */
-export type RunValue<I, T = string> = T | ((input: StepInput<I>, run: RunInfo) => T);
-
-export interface ScriptDef<N extends string, I extends InputSpec, O extends AnyModel | null, Sec extends readonly Secret[]>
-  extends StepDef<N, I, O, Sec> {
-  /** An embedded script: `import s from "./scripts/x.sh" with { type: "file" }`. */
-  script: string;
-  /** Extra environment; values may be functions of the inputs (evaluated in the step, not at describe). */
-  env?: Record<string, RunValue<I>>;
-}
-
-/** `deployTo` → `INPUT_DEPLOY_TO`. */
-export function inputEnvName(key: string): string {
-  return "INPUT_" + key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").replace(/[^A-Za-z0-9]+/g, "_").toUpperCase();
-}
-
-/**
- * Runs a shell script (the zero-TypeScript path). Contract for the script:
- *  - `$OLVE_INPUT`: path of the validated input as JSON; each input key is also `INPUT_<KEY>`
- *    (strings raw, everything else JSON).
- *  - `$OLVE_OUTPUT`: if the step has an output model, write the output JSON there. Objects of the
- *    form `{"$file": "<path>"}` / `{"$dir": "<path>"}` are turned into file references.
- *  - Declared secrets are in the environment under their names.
- */
-export function script<
-  const N extends string,
-  const I extends InputSpec = {},
-  O extends AnyModel | null = null,
-  const Sec extends readonly Secret[] = [],
->(def: ScriptDef<N, I, O, Sec>): Step<N, I, O, Sec[number]["name"]> & LibraryStep {
-  const env = def.env ?? {};
-  return makeStep("script", def as never, {
-    manifestConfig: () => ({
-      scriptSha256: "sha256:" + sha256Hex(readFileSync(def.script)),
-      env: Object.fromEntries(Object.entries(env).filter(([, v]) => typeof v === "string")),
-      computedEnv: Object.keys(env).filter((k) => typeof env[k] === "function").sort(),
-    }),
-    async run(input: Record<string, unknown>, ctx: Ctx) {
-      mkdirSync(ctx.tmpDir, { recursive: true });
-      const dir = mkdtempSync(join(ctx.tmpDir, "script-"));
-      const inputPath = join(dir, "input.json");
-      const outputPath = join(dir, "output.json");
-      writeFileSync(inputPath, JSON.stringify(input));
-      const vars: Record<string, string> = { OLVE_INPUT: inputPath };
-      for (const [k, v] of Object.entries(input)) {
-        if (v !== undefined) vars[inputEnvName(k)] = typeof v === "string" ? v : JSON.stringify(v);
-      }
-      for (const [k, v] of Object.entries(env)) {
-        vars[k] = typeof v === "function" ? (v as (i: unknown, r: RunInfo) => string)(input, ctx.run) : v;
-      }
-      if (def.output) vars.OLVE_OUTPUT = outputPath;
-      await ctx.sh({ script: def.script }, { env: vars });
-      if (!def.output) return undefined;
-      let raw: unknown;
-      try {
-        raw = JSON.parse(readFileSync(outputPath, "utf8"));
-      } catch (e) {
-        throw new Error(`script did not write valid JSON to $OLVE_OUTPUT: ${(e as Error).message}`);
-      }
-      return resolveFilePlaceholders(raw, ctx);
-    },
-  }) as never;
-}
-
-async function resolveFilePlaceholders(v: unknown, ctx: Ctx): Promise<unknown> {
-  if (Array.isArray(v)) return Promise.all(v.map((x) => resolveFilePlaceholders(x, ctx)));
-  if (typeof v === "object" && v !== null) {
-    const o = v as Record<string, unknown>;
-    const keys = Object.keys(o);
-    if (keys.length === 1 && typeof o.$file === "string") return ctx.output.file(o.$file);
-    if (keys.length === 1 && typeof o.$dir === "string") return ctx.output.dir(o.$dir);
-    const out: Record<string, unknown> = {};
-    for (const k of keys) out[k] = await resolveFilePlaceholders(o[k], ctx);
-    return out;
-  }
-  return v;
-}

package/src/util/handler-hash.ts

@@ -1,22 +0,0 @@
-import { canonicalJson } from "./canonical.ts";
-import { sha256Hex } from "./hash.ts";
-
-/** Identity of the harness compiled into a binary. A Bun or SDK bump legitimately bumps every handler. */
-export interface HarnessInfo {
-  /** Bun version (runtime compiled into the binary). */
-  bun: string;
-  /** @olivervea/pipelines package version. */
-  sdk: string;
-  /** `sha256:` hash over the SDK runtime sources and the zod version. */
-  sdkHash: string;
-}
-
-/** Custom step: hash of the handler bundle (contract files + SDK external) combined with the harness. */
-export function composeHandlerHash(bundleHash: string, harness: HarnessInfo): string {
-  return "sha256:" + sha256Hex(canonicalJson({ bundle: bundleHash, harness }));
-}
-
-/** Library step: kind + canonical config + harness. */
-export function libraryHandlerHash(kind: string, config: unknown, harness: HarnessInfo): string {
-  return "sha256:" + sha256Hex(canonicalJson({ library: kind, config, harness }));
-}