@olivervea/pipelines 2.0.0-beta.2 → 2.0.0-beta.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +125 -62
  2. package/manifest.schema.json +208 -11
  3. package/package.json +6 -4
  4. package/src/assets.d.ts +2 -2
  5. package/src/build/build.ts +121 -51
  6. package/src/build/cli.ts +29 -22
  7. package/src/build/elf.ts +8 -7
  8. package/src/build/hashes.ts +82 -20
  9. package/src/build/project.ts +95 -28
  10. package/src/builder/pipeline.ts +344 -88
  11. package/src/builder/refs.ts +15 -5
  12. package/src/core/brands.ts +7 -5
  13. package/src/core/config-refs.ts +103 -0
  14. package/src/core/config.ts +224 -0
  15. package/src/core/external.ts +86 -0
  16. package/src/core/handlers.ts +61 -15
  17. package/src/core/models.ts +32 -15
  18. package/src/core/origin.ts +23 -0
  19. package/src/core/secrets.ts +2 -2
  20. package/src/core/steps.ts +89 -38
  21. package/src/core/types.ts +5 -1
  22. package/src/core/webhooks.ts +21 -7
  23. package/src/ctx/files.ts +111 -40
  24. package/src/ctx/index.ts +84 -31
  25. package/src/ctx/sh.ts +105 -44
  26. package/src/ctx/types.ts +12 -4
  27. package/src/describe/describe.ts +701 -133
  28. package/src/describe/manifest-schema.ts +258 -61
  29. package/src/describe/mapper-source.ts +249 -104
  30. package/src/describe/record.ts +67 -37
  31. package/src/dev/cli.ts +72 -35
  32. package/src/harness/env.ts +45 -25
  33. package/src/harness/execute.ts +58 -29
  34. package/src/harness/index.ts +22 -9
  35. package/src/harness/main.ts +239 -78
  36. package/src/harness/result.ts +38 -22
  37. package/src/harness/validate.ts +77 -15
  38. package/src/index.ts +124 -11
  39. package/src/library/checks.ts +28 -0
  40. package/src/library/compose-deploy.ts +113 -0
  41. package/src/library/github-source.ts +100 -47
  42. package/src/library/helm-deploy-ssh.ts +452 -108
  43. package/src/library/helm-deploy.ts +449 -0
  44. package/src/library/http-health.ts +7 -5
  45. package/src/library/image-build.ts +499 -105
  46. package/src/library/index.ts +48 -12
  47. package/src/library/models.ts +33 -9
  48. package/src/library/ssh.ts +67 -0
  49. package/src/testing/index.ts +74 -36
  50. package/src/util/canonical.ts +7 -4
  51. package/src/util/handler-hash.ts +17 -7
  52. package/src/util/hash.ts +5 -5
  53. package/src/util/tar.ts +86 -36
  54. package/src/library/script.ts +0 -85
@@ -1,14 +1,50 @@
1
- import { gitHubSource } from "./github-source.ts";
2
- import { helmDeployOverSsh } from "./helm-deploy-ssh.ts";
3
- import { imageBuild } from "./image-build.ts";
4
- import { script } from "./script.ts";
1
+ import { gitHubSource } from './github-source.ts';
2
+ import { composeDeploy } from './compose-deploy.ts';
3
+ import { helmDeployInCluster } from './helm-deploy.ts';
4
+ import { helmDeployOverSsh } from './helm-deploy-ssh.ts';
5
+ import { imageBuild } from './image-build.ts';
5
6
 
6
- export { Source, BuiltImage, HelmRelease, FailureContext } from "./models.ts";
7
- export { inputEnvName, type RunValue, type ScriptDef } from "./script.ts";
8
- export type { GitHubSourceDef } from "./github-source.ts";
9
- export type { ImageBuildDef } from "./image-build.ts";
10
- export type { HelmDeployOverSshDef } from "./helm-deploy-ssh.ts";
11
- export { httpHealthCheck, HTTP_HEALTH_DEFAULTS, type HttpHealthCheckOptions } from "./http-health.ts";
7
+ export {
8
+ Source,
9
+ BuiltImage,
10
+ PushedImage,
11
+ HelmRelease,
12
+ FailureContext,
13
+ } from './models.ts';
14
+ export type { GitHubSourceDef } from './github-source.ts';
15
+ export type {
16
+ ImageBuildDef,
17
+ ImageBuildTarballDef,
18
+ ImageBuildPushDef,
19
+ RegistryAuth,
20
+ RegistryAuthConfig,
21
+ } from './image-build.ts';
22
+ export type { HelmDeployOverSshDef } from './helm-deploy-ssh.ts';
23
+ export {
24
+ type ComposeDeployDef,
25
+ type ComposeTargetValue,
26
+ type ComposeDeployResult,
27
+ } from './compose-deploy.ts';
28
+ export {
29
+ helmTarget,
30
+ deployTarget,
31
+ type DeployTargetOptions,
32
+ type HelmDeployInClusterDef,
33
+ type HelmDeploySshTargetDef,
34
+ type HelmTargetDeployDef,
35
+ type HelmTargetValue,
36
+ type InClusterTargetValue,
37
+ type SshTargetValue,
38
+ } from './helm-deploy.ts';
39
+ export { sshSession, type SshSession } from './ssh.ts';
40
+ export { serviceUrl, checkSpa } from './checks.ts';
41
+ export { httpHealthCheck, type HttpHealthCheckOptions } from './http-health.ts';
12
42
 
13
- /** SDK-provided steps (step + handler pairs with typed config). */
14
- export const steps = { script, gitHubSource, imageBuild, helmDeployOverSsh };
43
+ /** SDK-provided steps: each is `implement(step(…), …)` with typed settings, staged directly. */
44
+ export const steps = {
45
+ gitHubSource,
46
+ imageBuild,
47
+ helmDeployOverSsh,
48
+ helmDeployInCluster,
49
+ composeDeploy,
50
+ };
@@ -1,8 +1,11 @@
1
- import { z } from "zod";
2
- import { fileRef, model } from "../core/models.ts";
1
+ import { z } from 'zod';
2
+ import { fileRef, model } from '../core/models.ts';
3
3
 
4
4
  /** A source tree at a commit. `tree` references a directory (`path` ends with `/`). */
5
- export const Source = model("Source", z.object({ repo: z.string(), sha: z.string(), tree: fileRef() }));
5
+ export const Source = model(
6
+ 'Source',
7
+ z.object({ repo: z.string(), sha: z.string(), tree: fileRef() }),
8
+ );
6
9
 
7
10
  /**
8
11
  * A container image built by a step, as an image tarball (`docker save` format) in the step's files.
@@ -10,14 +13,35 @@ export const Source = model("Source", z.object({ repo: z.string(), sha: z.string
10
13
  * (`olve-pipelines-v2` → `docker.io/library/olve-pipelines-v2`).
11
14
  */
12
15
  export const BuiltImage = model(
13
- "BuiltImage",
14
- z.object({ repository: z.string(), tag: z.string(), digest: z.string(), tar: fileRef() }),
16
+ 'BuiltImage',
17
+ z.object({
18
+ repository: z.string(),
19
+ tag: z.string(),
20
+ digest: z.string(),
21
+ tar: fileRef(),
22
+ }),
23
+ );
24
+
25
+ /**
26
+ * A container image pushed to a registry by a step (`imageBuild` with `push: true`), pulled from there
27
+ * by the deploy. `repository` includes the registry host.
28
+ */
29
+ export const PushedImage = model(
30
+ 'PushedImage',
31
+ z.object({ repository: z.string(), tag: z.string(), digest: z.string() }),
15
32
  );
16
33
 
17
34
  /** A Helm release that was installed or upgraded. */
18
35
  export const HelmRelease = model(
19
- "HelmRelease",
20
- z.object({ release: z.string(), namespace: z.string(), image: z.string() }),
36
+ 'HelmRelease',
37
+ z.object({
38
+ release: z.string(),
39
+ namespace: z.string(),
40
+ /** The release's main image; absent for a chart deployed without images. */
41
+ image: z.string().optional(),
42
+ /** Where step pods reach the release (the target's `url`); absent = its in-cluster Service URL (`serviceUrl`). */
43
+ url: z.string().optional(),
44
+ }),
21
45
  );
22
46
 
23
47
  /** One failed step of a run, as seen by a failure handler. */
@@ -38,7 +62,7 @@ export const FailedStep = z.object({
38
62
  * the run's failed steps the handler is scoped to (all of them for an unscoped handler); never empty.
39
63
  */
40
64
  export const FailureContext = model(
41
- "FailureContext",
65
+ 'FailureContext',
42
66
  z.object({
43
67
  pipeline: z.object({ id: z.string(), name: z.string() }),
44
68
  run: z.object({
@@ -48,7 +72,7 @@ export const FailureContext = model(
48
72
  address: z.string(),
49
73
  /** The run's derived version, `yyyy.MM.dd.<run>`. */
50
74
  versionStamp: z.string(),
51
- trigger: z.enum(["push", "poll", "manual"]),
75
+ trigger: z.enum(['push', 'poll', 'manual', 'webhook']),
52
76
  /** The run (`GET …/api/pipelines/<id>/runs/<n>`). */
53
77
  url: z.string(),
54
78
  }),
@@ -0,0 +1,67 @@
1
+ // sshSession — run commands on a host over SSH from a handler (installs openssh-client when the image lacks ssh).
2
+ import { chmodSync, mkdirSync, writeFileSync } from 'node:fs';
3
+ import { join } from 'node:path';
4
+ import type { Ctx } from '../ctx/types.ts';
5
+
6
+ const SAFE = /^[A-Za-z0-9_\/.:=@%+,-]+$/;
7
+ /** POSIX shell quoting (safe words are left bare, so logs stay readable). */
8
+ export function shq(s: string): string {
9
+ return SAFE.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`;
10
+ }
11
+
12
+ /** Installs the ssh client unless the image has one: apk (Alpine) or apt-get (Debian, Ubuntu). */
13
+ export const SSH_CLIENT_INSTALL =
14
+ 'command -v ssh >/dev/null 2>&1 || if command -v apk >/dev/null 2>&1; then apk add --no-cache openssh-client; ' +
15
+ 'else apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openssh-client; fi';
16
+
17
+ export interface SshSession {
18
+ readonly host: string;
19
+ /** Runs a remote command; a non-zero exit throws. */
20
+ run(remote: string): Promise<void>;
21
+ /** Runs a remote command and returns its stdout. */
22
+ text(remote: string): Promise<string>;
23
+ /** `scp -r` a local file or directory to a remote path. */
24
+ copy(local: string, remote: string): Promise<void>;
25
+ /** The ssh invocation as a shell command, for pipes and redirects: `ctx.sh(`${s.shell("cat > f")} < ${file}`)`. */
26
+ shell(remote: string): string;
27
+ }
28
+
29
+ /**
30
+ * Opens an SSH "session" to `host` (`user@host`) with a private key: installs openssh-client if needed,
31
+ * writes the key to `<tmpDir>/ssh/id`. The host key is not verified: hosts are reached over the tailnet.
32
+ */
33
+ export async function sshSession(
34
+ ctx: Pick<Ctx, 'exec' | 'sh' | 'tmpDir'>,
35
+ opts: { host: string; key: string },
36
+ ): Promise<SshSession> {
37
+ await ctx.sh(SSH_CLIENT_INSTALL);
38
+ const dir = join(ctx.tmpDir, 'ssh');
39
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
40
+ const keyFile = join(dir, 'id');
41
+ // OpenSSH rejects a key without a trailing newline.
42
+ writeFileSync(keyFile, opts.key.endsWith('\n') ? opts.key : opts.key + '\n', {
43
+ mode: 0o600,
44
+ });
45
+ chmodSync(keyFile, 0o600);
46
+ const sshOpts = [
47
+ '-i',
48
+ keyFile,
49
+ '-o',
50
+ 'StrictHostKeyChecking=no',
51
+ '-o',
52
+ 'UserKnownHostsFile=/dev/null',
53
+ '-o',
54
+ 'LogLevel=ERROR',
55
+ '-o',
56
+ 'BatchMode=yes',
57
+ ];
58
+ const argv = (remote: string) => ['ssh', ...sshOpts, opts.host, remote];
59
+ return {
60
+ host: opts.host,
61
+ run: (remote) => ctx.exec(argv(remote)),
62
+ text: (remote) => ctx.sh.text(argv(remote).map(shq).join(' ')),
63
+ copy: (local, remote) =>
64
+ ctx.exec(['scp', ...sshOpts, '-r', local, `${opts.host}:${remote}`]),
65
+ shell: (remote) => argv(remote).map(shq).join(' '),
66
+ };
67
+ }
@@ -1,26 +1,36 @@
1
- // @olve/pipelines/testing — run a handler under `bun test` through the real harness path
1
+ // @olivervea/pipelines/testing — run a handler under `bun test` through the real harness path
2
2
  // (validate input → handler → strict output validation), with a real ctx you can override.
3
- import { mkdtempSync } from "node:fs";
4
- import { tmpdir } from "node:os";
5
- import { join } from "node:path";
6
- import { isImplementation, type Implementation } from "../core/handlers.ts";
7
- import type { FileRef } from "../core/models.ts";
8
- import { isStep, type AnyStep, type LibraryStep, type OutputValue, type SecretsOf, type StepInput } from "../core/steps.ts";
9
- import { createCtx } from "../ctx/index.ts";
10
- import type { Ctx, RunInfo } from "../ctx/types.ts";
11
- import { executeStep, type FailureKind, type StepFunction } from "../harness/execute.ts";
12
- import { sha256Hex } from "../util/hash.ts";
3
+ import { mkdtempSync } from 'node:fs';
4
+ import { tmpdir } from 'node:os';
5
+ import { join } from 'node:path';
6
+ import { isImplementation, type Implementation } from '../core/handlers.ts';
7
+ import type { FileRef } from '../core/models.ts';
8
+ import type {
9
+ AnyStep,
10
+ ConfigOf,
11
+ OutputValue,
12
+ SecretsOf,
13
+ StepInput,
14
+ } from '../core/steps.ts';
15
+ import { createCtx } from '../ctx/index.ts';
16
+ import type { Ctx, RunInfo } from '../ctx/types.ts';
17
+ import { executeStep, type FailureKind, type StepFunction } from '../harness/execute.ts';
18
+ import { sha256Hex } from '../util/hash.ts';
13
19
 
14
20
  export interface TestHandlerOptions<S extends AnyStep> {
15
21
  /** Replace parts of ctx (e.g. `sh`, `http`, `files`). The rest is the real implementation. */
16
- ctx?: Partial<Ctx<SecretsOf<S>>>;
22
+ ctx?: Partial<Ctx<SecretsOf<S>, ConfigOf<S>>>;
23
+ /** The step's config entries (z.input view: defaults are applied), like `$config` in input.json. */
24
+ config?: Record<string, unknown>;
17
25
  /** Secret values (instead of reading the environment). */
18
26
  secrets?: Partial<Record<SecretsOf<S>, string>>;
19
- run?: Partial<Omit<RunInfo, "step">>;
27
+ run?: Partial<Omit<RunInfo, 'step'>>;
20
28
  /** Directory used as $OLVE_DIR (default: a fresh temp dir). */
21
29
  dir?: string;
22
30
  /** Abort signal handed to ctx.signal. */
23
31
  signal?: AbortSignal;
32
+ /** `<step>/<input>` → URL returned by `ctx.inputUrl`. */
33
+ callbackUrls?: Record<string, string>;
24
34
  }
25
35
 
26
36
  export type TestResult<S extends AnyStep> = {
@@ -30,57 +40,85 @@ export type TestResult<S extends AnyStep> = {
30
40
  logs: string[];
31
41
  exitCode: number;
32
42
  } & (
33
- | { status: "succeeded"; output: OutputValue<S> }
34
- | { status: "skipped"; reason: string }
35
- | { status: "failed"; error: { kind: FailureKind; message: string } }
43
+ | { status: 'succeeded'; output: OutputValue<S> }
44
+ | { status: 'skipped'; reason: string }
45
+ | { status: 'failed'; error: { kind: FailureKind; message: string } }
36
46
  );
37
47
 
38
48
  /**
39
- * Runs a handler (the default export of `handlers/<step>.ts`) or a library step with `input`.
49
+ * Runs an implementation (a handler module's default export, or a step that carries its own) with `input`.
40
50
  * Input is validated like `input.json`, the result like a real run: an invalid output gives
41
51
  * `{ status: "failed", error: { kind: "output" } }`, a throwing handler `kind: "handler"`.
42
52
  */
43
53
  export async function testHandler<S extends AnyStep>(
44
- target: Implementation<S> | (S & LibraryStep),
45
- input: StepInput<S["input"]>,
54
+ target: Implementation<S>,
55
+ input: StepInput<S['input']>,
46
56
  opts: TestHandlerOptions<S> = {},
47
57
  ): Promise<TestResult<S>> {
48
- let step: AnyStep;
49
- let fn: StepFunction;
50
- if (isImplementation(target)) {
51
- step = target.step;
52
- fn = target.handler as unknown as StepFunction;
53
- } else if (isStep(target) && target.builtin) {
54
- step = target;
55
- fn = target.builtin.run as StepFunction;
56
- } else throw new TypeError("testHandler expects implement(...) (a handler module's default export) or a library step");
58
+ if (!isImplementation(target))
59
+ throw new TypeError(
60
+ "testHandler expects implement(…): a handler module's default export or a step that carries its implementation",
61
+ );
62
+ const step: AnyStep = target.step;
63
+ const fn = target.handler as unknown as StepFunction;
57
64
 
58
65
  const dir = opts.dir ?? mkdtempSync(join(tmpdir(), `olve-test-${step.name}-`));
59
- const filesDir = join(dir, "work", "files");
66
+ const filesDir = join(dir, 'work', 'files');
60
67
  const logs: string[] = [];
61
- const run: RunInfo = { pipeline: "test", id: "test", number: 1, attempt: 1, version: "0.0.0-test", ...opts.run, step: step.name };
68
+ const run: RunInfo = {
69
+ pipeline: 'test',
70
+ id: 'test',
71
+ number: 1,
72
+ attempt: 1,
73
+ version: '0.0.0-test',
74
+ ...opts.run,
75
+ step: step.name,
76
+ };
62
77
  const real = createCtx({
63
78
  run,
64
79
  secrets: step.secrets,
65
80
  secretValues: opts.secrets as Record<string, string> | undefined,
66
- paths: { root: dir, workFiles: filesDir, tmp: join(dir, "tmp") },
81
+ paths: { root: dir, workFiles: filesDir, tmp: join(dir, 'tmp') },
67
82
  signal: opts.signal ?? new AbortController().signal,
68
83
  logSink: (l) => logs.push(l),
84
+ callbackUrls: opts.callbackUrls,
69
85
  });
70
86
  const ctx = { ...real, ...opts.ctx } as Ctx;
71
87
  // Round-trip through JSON like input.json does (drops undefined, rejects non-JSON values early).
72
- const raw = JSON.parse(JSON.stringify(input ?? {}));
73
- const exec = await executeStep({ step, fn, rawInput: raw, ctx, workFiles: filesDir, log: (l) => logs.push(l) });
74
- return { dir, filesDir, logs, exitCode: exec.exitCode, ...exec.envelope } as TestResult<S>;
88
+ const raw = JSON.parse(
89
+ JSON.stringify({ ...input, ...(opts.config ? { $config: opts.config } : {}) }),
90
+ );
91
+ const exec = await executeStep({
92
+ step,
93
+ fn,
94
+ rawInput: raw,
95
+ ctx,
96
+ workFiles: filesDir,
97
+ log: (l) => logs.push(l),
98
+ });
99
+ return {
100
+ dir,
101
+ filesDir,
102
+ logs,
103
+ exitCode: exec.exitCode,
104
+ ...exec.envelope,
105
+ } as TestResult<S>;
75
106
  }
76
107
 
77
108
  /** A file reference for test inputs (`sha256` defaults to the hash of `content`). */
78
- export function testFileRef(ref: { step: string; path: string; run?: number; attempt?: number; sha256?: string; content?: string | Uint8Array }): FileRef {
109
+ export function testFileRef(ref: {
110
+ step: string;
111
+ path: string;
112
+ run?: number;
113
+ attempt?: number;
114
+ sha256?: string;
115
+ content?: string | Uint8Array;
116
+ }): FileRef {
79
117
  return {
80
118
  run: ref.run ?? 1,
81
119
  step: ref.step,
82
120
  attempt: ref.attempt ?? 1,
83
121
  path: ref.path,
84
- sha256: ref.sha256 ?? sha256Hex(ref.content ?? ""),
122
+ sha256: ref.sha256 ?? sha256Hex(ref.content ?? ''),
85
123
  } as FileRef;
86
124
  }
@@ -2,7 +2,7 @@
2
2
  // (order is meaningful: stages, steps, enum values), `undefined` members dropped. Stable for hashing.
3
3
  export function canonicalize(v: unknown): unknown {
4
4
  if (Array.isArray(v)) return v.map(canonicalize);
5
- if (v !== null && typeof v === "object") {
5
+ if (v !== null && typeof v === 'object') {
6
6
  const out: Record<string, unknown> = {};
7
7
  for (const k of Object.keys(v).sort()) {
8
8
  const x = (v as Record<string, unknown>)[k];
@@ -10,10 +10,13 @@ export function canonicalize(v: unknown): unknown {
10
10
  }
11
11
  return out;
12
12
  }
13
- if (typeof v === "number" && !Number.isFinite(v)) throw new Error(`non-finite number in canonical JSON: ${v}`);
14
- if (typeof v === "bigint" || typeof v === "function" || typeof v === "symbol") throw new Error(`${typeof v} is not JSON`);
13
+ if (typeof v === 'number' && !Number.isFinite(v))
14
+ throw new Error(`non-finite number in canonical JSON: ${v}`);
15
+ if (typeof v === 'bigint' || typeof v === 'function' || typeof v === 'symbol')
16
+ throw new Error(`${typeof v} is not JSON`);
15
17
  return v;
16
18
  }
17
19
 
18
20
  /** Canonical JSON text, 2-space indented, with a trailing newline. */
19
- export const canonicalJson = (v: unknown): string => JSON.stringify(canonicalize(v), null, 2) + "\n";
21
+ export const canonicalJson = (v: unknown): string =>
22
+ JSON.stringify(canonicalize(v), null, 2) + '\n';
@@ -1,11 +1,11 @@
1
- import { canonicalJson } from "./canonical.ts";
2
- import { sha256Hex } from "./hash.ts";
1
+ import { canonicalJson } from './canonical.ts';
2
+ import { sha256Hex } from './hash.ts';
3
3
 
4
4
  /** Identity of the harness compiled into a binary. A Bun or SDK bump legitimately bumps every handler. */
5
5
  export interface HarnessInfo {
6
6
  /** Bun version (runtime compiled into the binary). */
7
7
  bun: string;
8
- /** @olve/pipelines package version. */
8
+ /** @olivervea/pipelines package version. */
9
9
  sdk: string;
10
10
  /** `sha256:` hash over the SDK runtime sources and the zod version. */
11
11
  sdkHash: string;
@@ -13,10 +13,20 @@ export interface HarnessInfo {
13
13
 
14
14
  /** Custom step: hash of the handler bundle (contract files + SDK external) combined with the harness. */
15
15
  export function composeHandlerHash(bundleHash: string, harness: HarnessInfo): string {
16
- return "sha256:" + sha256Hex(canonicalJson({ bundle: bundleHash, harness }));
16
+ return 'sha256:' + sha256Hex(canonicalJson({ bundle: bundleHash, harness }));
17
17
  }
18
18
 
19
- /** Library step: kind + canonical config + harness. */
20
- export function libraryHandlerHash(kind: string, config: unknown, harness: HarnessInfo): string {
21
- return "sha256:" + sha256Hex(canonicalJson({ library: kind, config, harness }));
19
+ /** A step that carries its implementation: kind + canonical settings + harness, and the hash of the code that implements the kind. */
20
+ export function libraryHandlerHash(
21
+ kind: string,
22
+ config: unknown,
23
+ harness: HarnessInfo,
24
+ code?: string,
25
+ ): string {
26
+ return (
27
+ 'sha256:' +
28
+ sha256Hex(
29
+ canonicalJson({ library: kind, config, harness, ...(code ? { code } : {}) }),
30
+ )
31
+ );
22
32
  }
package/src/util/hash.ts CHANGED
@@ -1,13 +1,13 @@
1
- import { openSync, readSync, closeSync } from "node:fs";
1
+ import { openSync, readSync, closeSync } from 'node:fs';
2
2
 
3
3
  export function sha256Hex(data: string | Uint8Array | ArrayBuffer): string {
4
- return new Bun.CryptoHasher("sha256").update(data).digest("hex");
4
+ return new Bun.CryptoHasher('sha256').update(data).digest('hex');
5
5
  }
6
6
 
7
7
  /** Streaming SHA-256 of a file (constant memory). */
8
8
  export function sha256File(path: string): string {
9
- const h = new Bun.CryptoHasher("sha256");
10
- const fd = openSync(path, "r");
9
+ const h = new Bun.CryptoHasher('sha256');
10
+ const fd = openSync(path, 'r');
11
11
  try {
12
12
  const buf = new Uint8Array(1 << 20);
13
13
  for (;;) {
@@ -18,5 +18,5 @@ export function sha256File(path: string): string {
18
18
  } finally {
19
19
  closeSync(fd);
20
20
  }
21
- return h.digest("hex");
21
+ return h.digest('hex');
22
22
  }
package/src/util/tar.ts CHANGED
@@ -1,27 +1,45 @@
1
1
  // Deterministic tar writer (ustar + PAX long names), streaming from disk. Entries are sorted,
2
2
  // with mtime 0, uid/gid 0, no user names — so equal content gives byte-identical archives (and
3
3
  // sha256s). Extraction uses Bun.Archive.
4
- import { closeSync, createReadStream, createWriteStream, lstatSync, openSync, readdirSync, readlinkSync, readSync, renameSync, rmSync, writeSync } from "node:fs";
5
- import { join } from "node:path";
6
- import { pipeline } from "node:stream/promises";
7
- import { createGzip } from "node:zlib";
4
+ import {
5
+ closeSync,
6
+ createReadStream,
7
+ createWriteStream,
8
+ lstatSync,
9
+ openSync,
10
+ readdirSync,
11
+ readlinkSync,
12
+ readSync,
13
+ renameSync,
14
+ rmSync,
15
+ writeSync,
16
+ } from 'node:fs';
17
+ import { join } from 'node:path';
18
+ import { pipeline } from 'node:stream/promises';
19
+ import { createGzip } from 'node:zlib';
8
20
 
9
21
  export type TarEntry =
10
- | { path: string; type: "file"; mode: number; source: string }
11
- | { path: string; type: "bytes"; mode: number; data: Uint8Array }
12
- | { path: string; type: "dir"; mode: number }
13
- | { path: string; type: "symlink"; target: string };
22
+ | { path: string; type: 'file'; mode: number; source: string }
23
+ | { path: string; type: 'bytes'; mode: number; data: Uint8Array }
24
+ | { path: string; type: 'dir'; mode: number }
25
+ | { path: string; type: 'symlink'; target: string };
14
26
 
15
27
  const enc = new TextEncoder();
16
28
 
17
29
  function octal(n: number, len: number): string {
18
- return n.toString(8).padStart(len - 1, "0") + "\0";
30
+ return n.toString(8).padStart(len - 1, '0') + '\0';
19
31
  }
20
32
 
21
- function header(name: string, size: number, mode: number, typeflag: string, linkname = ""): Uint8Array {
33
+ function header(
34
+ name: string,
35
+ size: number,
36
+ mode: number,
37
+ typeflag: string,
38
+ linkname = '',
39
+ ): Uint8Array {
22
40
  const h = new Uint8Array(512);
23
41
  const put = (s: string | Uint8Array, off: number, len: number) => {
24
- const b = typeof s === "string" ? enc.encode(s) : s;
42
+ const b = typeof s === 'string' ? enc.encode(s) : s;
25
43
  h.set(b.subarray(0, len), off);
26
44
  };
27
45
  put(name, 0, 100);
@@ -30,14 +48,14 @@ function header(name: string, size: number, mode: number, typeflag: string, link
30
48
  put(octal(0, 8), 116, 8);
31
49
  put(octal(size, 12), 124, 12);
32
50
  put(octal(0, 12), 136, 12);
33
- put(" ", 148, 8); // checksum placeholder
51
+ put(' ', 148, 8); // checksum placeholder
34
52
  put(typeflag, 156, 1);
35
53
  put(linkname, 157, 100);
36
- put("ustar\0", 257, 6);
37
- put("00", 263, 2);
54
+ put('ustar\0', 257, 6);
55
+ put('00', 263, 2);
38
56
  let sum = 0;
39
57
  for (const b of h) sum += b;
40
- put(octal(sum, 7) + " ", 148, 8);
58
+ put(octal(sum, 7) + ' ', 148, 8);
41
59
  return h;
42
60
  }
43
61
 
@@ -51,9 +69,11 @@ function paxRecord(key: string, value: string): string {
51
69
 
52
70
  /** Writes `entries` (sorted by path here) as an uncompressed tar to `out`. */
53
71
  export function writeTar(out: string, entries: readonly TarEntry[]): void {
54
- const sorted = [...entries].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
72
+ const sorted = [...entries].sort((a, b) =>
73
+ a.path < b.path ? -1 : a.path > b.path ? 1 : 0,
74
+ );
55
75
  const tmp = `${out}.partial`;
56
- const fd = openSync(tmp, "w");
76
+ const fd = openSync(tmp, 'w');
57
77
  const write = (b: Uint8Array) => {
58
78
  let off = 0;
59
79
  while (off < b.length) off += writeSync(fd, b, off, b.length - off);
@@ -64,29 +84,31 @@ export function writeTar(out: string, entries: readonly TarEntry[]): void {
64
84
  };
65
85
  try {
66
86
  for (const e of sorted) {
67
- const name = e.type === "dir" && !e.path.endsWith("/") ? e.path + "/" : e.path;
68
- const link = e.type === "symlink" ? e.target : "";
87
+ const name = e.type === 'dir' && !e.path.endsWith('/') ? e.path + '/' : e.path;
88
+ const link = e.type === 'symlink' ? e.target : '';
69
89
  let size = 0;
70
- if (e.type === "file") size = lstatSync(e.source).size;
71
- else if (e.type === "bytes") size = e.data.length;
90
+ if (e.type === 'file') size = lstatSync(e.source).size;
91
+ else if (e.type === 'bytes') size = e.data.length;
72
92
  const pax: string[] = [];
73
- if (enc.encode(name).length > 100) pax.push(paxRecord("path", name));
74
- if (enc.encode(link).length > 100) pax.push(paxRecord("linkpath", link));
93
+ if (enc.encode(name).length > 100) pax.push(paxRecord('path', name));
94
+ if (enc.encode(link).length > 100) pax.push(paxRecord('linkpath', link));
75
95
  if (pax.length) {
76
- const body = enc.encode(pax.join(""));
77
- write(header("././@PaxHeader", body.length, 0o644, "x"));
96
+ const body = enc.encode(pax.join(''));
97
+ write(header('././@PaxHeader', body.length, 0o644, 'x'));
78
98
  write(body);
79
99
  pad(body.length);
80
100
  }
81
101
  const shortName = enc.encode(name).length > 100 ? name.slice(0, 99) : name;
82
- const mode = e.type === "symlink" ? 0o777 : e.mode;
83
- const flag = e.type === "dir" ? "5" : e.type === "symlink" ? "2" : "0";
84
- write(header(shortName, size, mode, flag, link.length > 100 ? link.slice(0, 99) : link));
85
- if (e.type === "bytes") {
102
+ const mode = e.type === 'symlink' ? 0o777 : e.mode;
103
+ const flag = e.type === 'dir' ? '5' : e.type === 'symlink' ? '2' : '0';
104
+ write(
105
+ header(shortName, size, mode, flag, link.length > 100 ? link.slice(0, 99) : link),
106
+ );
107
+ if (e.type === 'bytes') {
86
108
  write(e.data);
87
109
  pad(size);
88
- } else if (e.type === "file") {
89
- const src = openSync(e.source, "r");
110
+ } else if (e.type === 'file') {
111
+ const src = openSync(e.source, 'r');
90
112
  try {
91
113
  const buf = new Uint8Array(1 << 20);
92
114
  let left = size;
@@ -110,18 +132,25 @@ export function writeTar(out: string, entries: readonly TarEntry[]): void {
110
132
  }
111
133
 
112
134
  /** Entries for the contents of `dir` (recursively), with paths prefixed by `prefix` ("" = none). */
113
- export function dirEntries(dir: string, prefix = ""): TarEntry[] {
135
+ export function dirEntries(dir: string, prefix = ''): TarEntry[] {
114
136
  const out: TarEntry[] = [];
115
137
  const walk = (abs: string, rel: string) => {
116
138
  for (const name of readdirSync(abs).sort()) {
117
139
  const a = join(abs, name);
118
140
  const r = rel ? `${rel}/${name}` : name;
119
141
  const st = lstatSync(a);
120
- if (st.isSymbolicLink()) out.push({ path: r, type: "symlink", target: readlinkSync(a) });
142
+ if (st.isSymbolicLink())
143
+ out.push({ path: r, type: 'symlink', target: readlinkSync(a) });
121
144
  else if (st.isDirectory()) {
122
- out.push({ path: r, type: "dir", mode: 0o755 });
145
+ out.push({ path: r, type: 'dir', mode: 0o755 });
123
146
  walk(a, r);
124
- } else if (st.isFile()) out.push({ path: r, type: "file", mode: st.mode & 0o111 ? 0o755 : 0o644, source: a });
147
+ } else if (st.isFile())
148
+ out.push({
149
+ path: r,
150
+ type: 'file',
151
+ mode: st.mode & 0o111 ? 0o755 : 0o644,
152
+ source: a,
153
+ });
125
154
  // sockets, fifos, devices are skipped
126
155
  }
127
156
  };
@@ -137,7 +166,11 @@ export async function gzipFile(src: string, out: string, level = 6): Promise<voi
137
166
  }
138
167
 
139
168
  /** Writes a deterministic `.tar.gz`. */
140
- export async function writeTarGz(out: string, entries: readonly TarEntry[], level = 6): Promise<void> {
169
+ export async function writeTarGz(
170
+ out: string,
171
+ entries: readonly TarEntry[],
172
+ level = 6,
173
+ ): Promise<void> {
141
174
  const tar = `${out}.tar-tmp`;
142
175
  try {
143
176
  writeTar(tar, entries);
@@ -147,6 +180,23 @@ export async function writeTarGz(out: string, entries: readonly TarEntry[], leve
147
180
  }
148
181
  }
149
182
 
183
+ /** Writes a deterministic zstd-compressed tar (level 3: fast to write, and unpacks far faster than gzip). */
184
+ export async function writeTarZst(
185
+ out: string,
186
+ entries: readonly TarEntry[],
187
+ level = 3,
188
+ ): Promise<void> {
189
+ const tar = `${out}.tar-tmp`;
190
+ const tmp = `${out}.partial`;
191
+ try {
192
+ writeTar(tar, entries);
193
+ await Bun.write(tmp, await Bun.zstdCompress(await Bun.file(tar).bytes(), { level }));
194
+ renameSync(tmp, out);
195
+ } finally {
196
+ rmSync(tar, { force: true });
197
+ }
198
+ }
199
+
150
200
  /** Extracts a tar or tar.gz file into `dir`. */
151
201
  export async function extractTar(archive: string, dir: string): Promise<void> {
152
202
  const a = new Bun.Archive(await Bun.file(archive).bytes());