@olivervea/pipelines 2.0.0-beta.6 → 2.0.0-beta.7

package/README.md

@@ -4,7 +4,7 @@ The [Olve.Pipelines](https://github.com/OliverVea/Olve.Pipelines) v2 SDK. It has
 
 - the **authoring API**: models, steps, handlers and the typed pipeline builder;
 - the **step harness**: the runtime compiled into each pipeline's single executable (`describe`, `run`, `map-trigger`);
-- the **build tool** (`olve-build`), which turns a pipeline directory into `step.tar.gz` and `manifest.json`;
+- the **build tool** (`olve-build`), which turns a pipeline directory into `step.tar.zst` and `manifest.json`;
 - the **dev runner** (`olve-dev`), which runs the same harness path from source (used by `pl dev`).
 
 Design: [`docs/design/pipelines-v2.md`](https://github.com/OliverVea/Olve.Pipelines/blob/v2/docs/design/pipelines-v2.md). Spike findings: [`pipelines-v2-spikes.md`](https://github.com/OliverVea/Olve.Pipelines/blob/v2/docs/design/pipelines-v2-spikes.md).
@@ -418,8 +418,8 @@ olve-build <pipelineDir> --hash-only          # print handler hashes (diagnostic
 3. Run `bun build --compile --target=bun-linux-x64 --no-compile-autoload-dotenv --no-compile-autoload-bunfig --bytecode --format=esm` → `<out>/step`. The output is never minified, because mapper source analysis needs real source.
 4. Check that `PT_INTERP` is `/lib64/ld-linux-x86-64.so.2`. On alpine, Bun silently produces a musl binary; the check fails it.
 5. Run `<out>/step describe` → `<out>/manifest.json`. A describe problem is a build failure with exit 78. **A custom step without a handler file is an error.**
-6. Write `<out>/step.tar.gz` (deterministic, `step` at the root, mode 0755) and `<out>/build.json` (diagnostics).
-7. With `--upload-url`, PUT `step.tar.gz`, using the same retry rules as the result upload. With `--sentinels`, print the manifest between the manifest sentinels.
+6. Write `<out>/step.tar.zst` (a deterministic tar, `step` at the root, mode 0755, zstd level 3: it unpacks in the step pod far faster than gzip; harness-base also still unpacks the gzip archives of older SDKs; from 2.0.0-beta.7 the SDK needs an instance whose harness-base unpacks zstd, i.e. one deployed on or after 2026-10-25) and `<out>/build.json` (diagnostics).
+7. With `--upload-url`, PUT `step.tar.zst`, using the same retry rules as the result upload. With `--sentinels`, print the manifest between the manifest sentinels.
 
 Exit codes: 0 ok, 1 build failure, 64 usage, 78 pipeline invalid.
 
@@ -435,7 +435,7 @@ From npm, nothing else is needed. A local `file:`/`link:` dependency (`"@oliverv
 
 The repo depends on the SDK as a normal package. It resolves through `exports` (`@olivervea/pipelines`, `@olivervea/pipelines/harness`, `@olivervea/pipelines/testing`), and the tests cover this with a `file:` install. `bun` must be on `PATH`, because the bins use `#!/usr/bin/env bun`. A consumer `tsconfig.json` has to include the dot-directory explicitly (`"include": [".pipelines/**/*.ts"]`), and should exclude `.pipelines/.olve`.
 
-How the service runs it (reconcile, `BuildJobBuilder`): one Job per (pipeline, config commit), named `olve-build-<pipeline12>-<sha12>`. The repo tarball comes from `codeload.github.com/<repo>/tar.gz/<sha>` anonymously (public repos), or from the GitHub API tarball endpoint with the binding's credentials key from the pipeline Secret. The binary is PUT to `versions/<pipeline>/builds/<sha>/step.tar.gz`, and a version added from that build keeps this key as its `BinaryKey`. The in-repo fixture `.pipelines-test/` (bound on v2 beta) uses exactly this layout.
+How the service runs it (reconcile, `BuildJobBuilder`): one Job per (pipeline, config commit), named `olve-build-<pipeline12>-<sha12>`. The repo tarball comes from `codeload.github.com/<repo>/tar.gz/<sha>` anonymously (public repos), or from the GitHub API tarball endpoint with the binding's credentials key from the pipeline Secret. The archive is PUT to `versions/<pipeline>/builds/<sha>/step.tar.gz` (the key predates zstd; the init container tells the formats apart by content), and a version added from that build keeps this key as its `BinaryKey`. The in-repo fixture `.pipelines-test/` (bound on v2 beta) uses exactly this layout.
 
 The manifest is the text between the sentinels in the pod log. The manifest text is identical across checkouts. The binary and archive bytes are identical for the same checkout layout.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.6",
+  "version": "2.0.0-beta.7",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/build/build.ts

@@ -1,10 +1,10 @@
-// olve-build: pipeline dir → single executable + step.tar.gz + manifest.json.
+// olve-build: pipeline dir → single executable + step.tar.zst + manifest.json.
 import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { canonicalJson } from "../util/canonical.ts";
 import { sha256File } from "../util/hash.ts";
 import type { HarnessInfo } from "../util/handler-hash.ts";
-import { writeTarGz } from "../util/tar.ts";
+import { writeTarZst } from "../util/tar.ts";
 import { MANIFEST_BEGIN, MANIFEST_END, uploadFile } from "../harness/result.ts";
 import { GLIBC_INTERP, MUSL_INTERP, readInterpreter } from "./elf.ts";
 import { handlerHashes, type HandlerHash } from "./hashes.ts";
@@ -23,7 +23,7 @@ export class BuildError extends Error {
 export interface BuildOptions {
   out: string;
   target?: Target;
-  /** Pre-signed PUT for step.tar.gz. */
+  /** Pre-signed PUT for the step archive. */
   uploadUrl?: string;
   /** Print the manifest between sentinels on stdout (for the build Job's pod log). */
   sentinels?: boolean;
@@ -107,9 +107,9 @@ export async function build(dir: string, opts: BuildOptions): Promise<BuildResul
   const manifestFile = join(out, "manifest.json");
   writeFileSync(manifestFile, manifest);
 
-  // 4. step.tar.gz (deterministic: `step` at the root, 0755, mtime 0)
-  const archive = join(out, "step.tar.gz");
-  await writeTarGz(archive, [{ path: "step", type: "file", mode: 0o755, source: binary }]);
+  // 4. step.tar.zst (deterministic: `step` at the root, 0755, mtime 0)
+  const archive = join(out, "step.tar.zst");
+  await writeTarZst(archive, [{ path: "step", type: "file", mode: 0o755, source: binary }]);
   writeFileSync(
     join(out, "build.json"),
     canonicalJson({ target, flags: BUILD_FLAGS, harness, handlers: hashes, binarySha256: sha256File(binary), archiveSha256: sha256File(archive) }),
@@ -118,7 +118,7 @@ export async function build(dir: string, opts: BuildOptions): Promise<BuildResul
 
   if (opts.uploadUrl) {
     await uploadFile(opts.uploadUrl, archive, log);
-    log("olve-build: uploaded step.tar.gz");
+    log("olve-build: uploaded step.tar.zst");
   }
   if (opts.sentinels) await stdout(`${MANIFEST_BEGIN}\n${manifest}${MANIFEST_END}\n`);
   return { binary, archive, manifestFile, manifest, hashes, harness };

package/src/util/tar.ts

@@ -147,6 +147,19 @@ export async function writeTarGz(out: string, entries: readonly TarEntry[], leve
   }
 }
 
+/** Writes a deterministic zstd-compressed tar (level 3: fast to write, and unpacks far faster than gzip). */
+export async function writeTarZst(out: string, entries: readonly TarEntry[], level = 3): Promise<void> {
+  const tar = `${out}.tar-tmp`;
+  const tmp = `${out}.partial`;
+  try {
+    writeTar(tar, entries);
+    await Bun.write(tmp, await Bun.zstdCompress(await Bun.file(tar).bytes(), { level }));
+    renameSync(tmp, out);
+  } finally {
+    rmSync(tar, { force: true });
+  }
+}
+
 /** Extracts a tar or tar.gz file into `dir`. */
 export async function extractTar(archive: string, dir: string): Promise<void> {
   const a = new Bun.Archive(await Bun.file(archive).bytes());