@olivervea/pipelines 2.0.0-beta.18 → 2.0.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -2
- package/src/assets.d.ts +2 -2
- package/src/build/build.ts +111 -43
- package/src/build/cli.ts +28 -21
- package/src/build/elf.ts +8 -7
- package/src/build/hashes.ts +61 -28
- package/src/build/project.ts +69 -30
- package/src/builder/pipeline.ts +274 -102
- package/src/builder/refs.ts +12 -6
- package/src/core/brands.ts +6 -6
- package/src/core/config-refs.ts +46 -20
- package/src/core/config.ts +110 -33
- package/src/core/external.ts +30 -12
- package/src/core/handlers.ts +54 -14
- package/src/core/models.ts +29 -14
- package/src/core/origin.ts +7 -4
- package/src/core/secrets.ts +2 -2
- package/src/core/steps.ts +39 -23
- package/src/core/types.ts +5 -1
- package/src/core/webhooks.ts +21 -7
- package/src/ctx/files.ts +111 -40
- package/src/ctx/index.ts +75 -34
- package/src/ctx/sh.ts +104 -44
- package/src/ctx/types.ts +4 -4
- package/src/describe/describe.ts +550 -196
- package/src/describe/manifest-schema.ts +210 -80
- package/src/describe/mapper-source.ts +249 -104
- package/src/describe/record.ts +67 -37
- package/src/dev/cli.ts +64 -34
- package/src/harness/env.ts +33 -27
- package/src/harness/execute.ts +55 -29
- package/src/harness/index.ts +21 -8
- package/src/harness/main.ts +197 -74
- package/src/harness/result.ts +38 -22
- package/src/harness/validate.ts +62 -19
- package/src/index.ts +123 -14
- package/src/library/checks.ts +11 -6
- package/src/library/compose-deploy.ts +40 -23
- package/src/library/github-source.ts +69 -39
- package/src/library/helm-deploy-ssh.ts +213 -76
- package/src/library/helm-deploy.ts +263 -85
- package/src/library/http-health.ts +7 -5
- package/src/library/image-build.ts +293 -117
- package/src/library/index.ts +47 -15
- package/src/library/models.ts +17 -9
- package/src/library/ssh.ts +31 -14
- package/src/testing/index.ts +63 -24
- package/src/util/canonical.ts +7 -4
- package/src/util/handler-hash.ts +15 -5
- package/src/util/hash.ts +5 -5
- package/src/util/tar.ts +74 -37
package/src/library/index.ts
CHANGED
|
@@ -1,18 +1,50 @@
|
|
|
1
|
-
import { gitHubSource } from
|
|
2
|
-
import { composeDeploy } from
|
|
3
|
-
import { helmDeployInCluster } from
|
|
4
|
-
import { helmDeployOverSsh } from
|
|
5
|
-
import { imageBuild } from
|
|
1
|
+
import { gitHubSource } from './github-source.ts';
|
|
2
|
+
import { composeDeploy } from './compose-deploy.ts';
|
|
3
|
+
import { helmDeployInCluster } from './helm-deploy.ts';
|
|
4
|
+
import { helmDeployOverSsh } from './helm-deploy-ssh.ts';
|
|
5
|
+
import { imageBuild } from './image-build.ts';
|
|
6
6
|
|
|
7
|
-
export {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
export {
|
|
15
|
-
export
|
|
7
|
+
export {
|
|
8
|
+
Source,
|
|
9
|
+
BuiltImage,
|
|
10
|
+
PushedImage,
|
|
11
|
+
HelmRelease,
|
|
12
|
+
FailureContext,
|
|
13
|
+
} from './models.ts';
|
|
14
|
+
export type { GitHubSourceDef } from './github-source.ts';
|
|
15
|
+
export type {
|
|
16
|
+
ImageBuildDef,
|
|
17
|
+
ImageBuildTarballDef,
|
|
18
|
+
ImageBuildPushDef,
|
|
19
|
+
RegistryAuth,
|
|
20
|
+
RegistryAuthConfig,
|
|
21
|
+
} from './image-build.ts';
|
|
22
|
+
export type { HelmDeployOverSshDef } from './helm-deploy-ssh.ts';
|
|
23
|
+
export {
|
|
24
|
+
type ComposeDeployDef,
|
|
25
|
+
type ComposeTargetValue,
|
|
26
|
+
type ComposeDeployResult,
|
|
27
|
+
} from './compose-deploy.ts';
|
|
28
|
+
export {
|
|
29
|
+
helmTarget,
|
|
30
|
+
deployTarget,
|
|
31
|
+
type DeployTargetOptions,
|
|
32
|
+
type HelmDeployInClusterDef,
|
|
33
|
+
type HelmDeploySshTargetDef,
|
|
34
|
+
type HelmTargetDeployDef,
|
|
35
|
+
type HelmTargetValue,
|
|
36
|
+
type InClusterTargetValue,
|
|
37
|
+
type SshTargetValue,
|
|
38
|
+
} from './helm-deploy.ts';
|
|
39
|
+
export { sshSession, type SshSession } from './ssh.ts';
|
|
40
|
+
export { serviceUrl, checkSpa } from './checks.ts';
|
|
41
|
+
export { httpHealthCheck, type HttpHealthCheckOptions } from './http-health.ts';
|
|
16
42
|
|
|
17
43
|
/** SDK-provided steps: each is `implement(step(…), …)` with typed settings, staged directly. */
|
|
18
|
-
export const steps = {
|
|
44
|
+
export const steps = {
|
|
45
|
+
gitHubSource,
|
|
46
|
+
imageBuild,
|
|
47
|
+
helmDeployOverSsh,
|
|
48
|
+
helmDeployInCluster,
|
|
49
|
+
composeDeploy,
|
|
50
|
+
};
|
package/src/library/models.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
import { z } from
|
|
2
|
-
import { fileRef, model } from
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { fileRef, model } from '../core/models.ts';
|
|
3
3
|
|
|
4
4
|
/** A source tree at a commit. `tree` references a directory (`path` ends with `/`). */
|
|
5
|
-
export const Source = model(
|
|
5
|
+
export const Source = model(
|
|
6
|
+
'Source',
|
|
7
|
+
z.object({ repo: z.string(), sha: z.string(), tree: fileRef() }),
|
|
8
|
+
);
|
|
6
9
|
|
|
7
10
|
/**
|
|
8
11
|
* A container image built by a step, as an image tarball (`docker save` format) in the step's files.
|
|
@@ -10,8 +13,13 @@ export const Source = model("Source", z.object({ repo: z.string(), sha: z.string
|
|
|
10
13
|
* (`olve-pipelines-v2` → `docker.io/library/olve-pipelines-v2`).
|
|
11
14
|
*/
|
|
12
15
|
export const BuiltImage = model(
|
|
13
|
-
|
|
14
|
-
z.object({
|
|
16
|
+
'BuiltImage',
|
|
17
|
+
z.object({
|
|
18
|
+
repository: z.string(),
|
|
19
|
+
tag: z.string(),
|
|
20
|
+
digest: z.string(),
|
|
21
|
+
tar: fileRef(),
|
|
22
|
+
}),
|
|
15
23
|
);
|
|
16
24
|
|
|
17
25
|
/**
|
|
@@ -19,13 +27,13 @@ export const BuiltImage = model(
|
|
|
19
27
|
* by the deploy. `repository` includes the registry host.
|
|
20
28
|
*/
|
|
21
29
|
export const PushedImage = model(
|
|
22
|
-
|
|
30
|
+
'PushedImage',
|
|
23
31
|
z.object({ repository: z.string(), tag: z.string(), digest: z.string() }),
|
|
24
32
|
);
|
|
25
33
|
|
|
26
34
|
/** A Helm release that was installed or upgraded. */
|
|
27
35
|
export const HelmRelease = model(
|
|
28
|
-
|
|
36
|
+
'HelmRelease',
|
|
29
37
|
z.object({
|
|
30
38
|
release: z.string(),
|
|
31
39
|
namespace: z.string(),
|
|
@@ -54,7 +62,7 @@ export const FailedStep = z.object({
|
|
|
54
62
|
* the run's failed steps the handler is scoped to (all of them for an unscoped handler); never empty.
|
|
55
63
|
*/
|
|
56
64
|
export const FailureContext = model(
|
|
57
|
-
|
|
65
|
+
'FailureContext',
|
|
58
66
|
z.object({
|
|
59
67
|
pipeline: z.object({ id: z.string(), name: z.string() }),
|
|
60
68
|
run: z.object({
|
|
@@ -64,7 +72,7 @@ export const FailureContext = model(
|
|
|
64
72
|
address: z.string(),
|
|
65
73
|
/** The run's derived version, `yyyy.MM.dd.<run>`. */
|
|
66
74
|
versionStamp: z.string(),
|
|
67
|
-
trigger: z.enum([
|
|
75
|
+
trigger: z.enum(['push', 'poll', 'manual', 'webhook']),
|
|
68
76
|
/** The run (`GET …/api/pipelines/<id>/runs/<n>`). */
|
|
69
77
|
url: z.string(),
|
|
70
78
|
}),
|
package/src/library/ssh.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// sshSession — run commands on a host over SSH from a handler (installs openssh-client when the image lacks ssh).
|
|
2
|
-
import { chmodSync, mkdirSync, writeFileSync } from
|
|
3
|
-
import { join } from
|
|
4
|
-
import type { Ctx } from
|
|
2
|
+
import { chmodSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import type { Ctx } from '../ctx/types.ts';
|
|
5
5
|
|
|
6
6
|
const SAFE = /^[A-Za-z0-9_\/.:=@%+,-]+$/;
|
|
7
7
|
/** POSIX shell quoting (safe words are left bare, so logs stay readable). */
|
|
@@ -11,8 +11,8 @@ export function shq(s: string): string {
|
|
|
11
11
|
|
|
12
12
|
/** Installs the ssh client unless the image has one: apk (Alpine) or apt-get (Debian, Ubuntu). */
|
|
13
13
|
export const SSH_CLIENT_INSTALL =
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
'command -v ssh >/dev/null 2>&1 || if command -v apk >/dev/null 2>&1; then apk add --no-cache openssh-client; ' +
|
|
15
|
+
'else apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openssh-client; fi';
|
|
16
16
|
|
|
17
17
|
export interface SshSession {
|
|
18
18
|
readonly host: string;
|
|
@@ -30,21 +30,38 @@ export interface SshSession {
|
|
|
30
30
|
* Opens an SSH "session" to `host` (`user@host`) with a private key: installs openssh-client if needed,
|
|
31
31
|
* writes the key to `<tmpDir>/ssh/id`. The host key is not verified: hosts are reached over the tailnet.
|
|
32
32
|
*/
|
|
33
|
-
export async function sshSession(
|
|
33
|
+
export async function sshSession(
|
|
34
|
+
ctx: Pick<Ctx, 'exec' | 'sh' | 'tmpDir'>,
|
|
35
|
+
opts: { host: string; key: string },
|
|
36
|
+
): Promise<SshSession> {
|
|
34
37
|
await ctx.sh(SSH_CLIENT_INSTALL);
|
|
35
|
-
const dir = join(ctx.tmpDir,
|
|
38
|
+
const dir = join(ctx.tmpDir, 'ssh');
|
|
36
39
|
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
37
|
-
const keyFile = join(dir,
|
|
40
|
+
const keyFile = join(dir, 'id');
|
|
38
41
|
// OpenSSH rejects a key without a trailing newline.
|
|
39
|
-
writeFileSync(keyFile, opts.key.endsWith(
|
|
42
|
+
writeFileSync(keyFile, opts.key.endsWith('\n') ? opts.key : opts.key + '\n', {
|
|
43
|
+
mode: 0o600,
|
|
44
|
+
});
|
|
40
45
|
chmodSync(keyFile, 0o600);
|
|
41
|
-
const sshOpts = [
|
|
42
|
-
|
|
46
|
+
const sshOpts = [
|
|
47
|
+
'-i',
|
|
48
|
+
keyFile,
|
|
49
|
+
'-o',
|
|
50
|
+
'StrictHostKeyChecking=no',
|
|
51
|
+
'-o',
|
|
52
|
+
'UserKnownHostsFile=/dev/null',
|
|
53
|
+
'-o',
|
|
54
|
+
'LogLevel=ERROR',
|
|
55
|
+
'-o',
|
|
56
|
+
'BatchMode=yes',
|
|
57
|
+
];
|
|
58
|
+
const argv = (remote: string) => ['ssh', ...sshOpts, opts.host, remote];
|
|
43
59
|
return {
|
|
44
60
|
host: opts.host,
|
|
45
61
|
run: (remote) => ctx.exec(argv(remote)),
|
|
46
|
-
text: (remote) => ctx.sh.text(argv(remote).map(shq).join(
|
|
47
|
-
copy: (local, remote) =>
|
|
48
|
-
|
|
62
|
+
text: (remote) => ctx.sh.text(argv(remote).map(shq).join(' ')),
|
|
63
|
+
copy: (local, remote) =>
|
|
64
|
+
ctx.exec(['scp', ...sshOpts, '-r', local, `${opts.host}:${remote}`]),
|
|
65
|
+
shell: (remote) => argv(remote).map(shq).join(' '),
|
|
49
66
|
};
|
|
50
67
|
}
|
package/src/testing/index.ts
CHANGED
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
// @olivervea/pipelines/testing — run a handler under `bun test` through the real harness path
|
|
2
2
|
// (validate input → handler → strict output validation), with a real ctx you can override.
|
|
3
|
-
import { mkdtempSync } from
|
|
4
|
-
import { tmpdir } from
|
|
5
|
-
import { join } from
|
|
6
|
-
import { isImplementation, type Implementation } from
|
|
7
|
-
import type { FileRef } from
|
|
8
|
-
import type {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
3
|
+
import { mkdtempSync } from 'node:fs';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { isImplementation, type Implementation } from '../core/handlers.ts';
|
|
7
|
+
import type { FileRef } from '../core/models.ts';
|
|
8
|
+
import type {
|
|
9
|
+
AnyStep,
|
|
10
|
+
ConfigOf,
|
|
11
|
+
OutputValue,
|
|
12
|
+
SecretsOf,
|
|
13
|
+
StepInput,
|
|
14
|
+
} from '../core/steps.ts';
|
|
15
|
+
import { createCtx } from '../ctx/index.ts';
|
|
16
|
+
import type { Ctx, RunInfo } from '../ctx/types.ts';
|
|
17
|
+
import { executeStep, type FailureKind, type StepFunction } from '../harness/execute.ts';
|
|
18
|
+
import { sha256Hex } from '../util/hash.ts';
|
|
13
19
|
|
|
14
20
|
export interface TestHandlerOptions<S extends AnyStep> {
|
|
15
21
|
/** Replace parts of ctx (e.g. `sh`, `http`, `files`). The rest is the real implementation. */
|
|
@@ -18,7 +24,7 @@ export interface TestHandlerOptions<S extends AnyStep> {
|
|
|
18
24
|
config?: Record<string, unknown>;
|
|
19
25
|
/** Secret values (instead of reading the environment). */
|
|
20
26
|
secrets?: Partial<Record<SecretsOf<S>, string>>;
|
|
21
|
-
run?: Partial<Omit<RunInfo,
|
|
27
|
+
run?: Partial<Omit<RunInfo, 'step'>>;
|
|
22
28
|
/** Directory used as $OLVE_DIR (default: a fresh temp dir). */
|
|
23
29
|
dir?: string;
|
|
24
30
|
/** Abort signal handed to ctx.signal. */
|
|
@@ -34,9 +40,9 @@ export type TestResult<S extends AnyStep> = {
|
|
|
34
40
|
logs: string[];
|
|
35
41
|
exitCode: number;
|
|
36
42
|
} & (
|
|
37
|
-
| { status:
|
|
38
|
-
| { status:
|
|
39
|
-
| { status:
|
|
43
|
+
| { status: 'succeeded'; output: OutputValue<S> }
|
|
44
|
+
| { status: 'skipped'; reason: string }
|
|
45
|
+
| { status: 'failed'; error: { kind: FailureKind; message: string } }
|
|
40
46
|
);
|
|
41
47
|
|
|
42
48
|
/**
|
|
@@ -46,40 +52,73 @@ export type TestResult<S extends AnyStep> = {
|
|
|
46
52
|
*/
|
|
47
53
|
export async function testHandler<S extends AnyStep>(
|
|
48
54
|
target: Implementation<S>,
|
|
49
|
-
input: StepInput<S[
|
|
55
|
+
input: StepInput<S['input']>,
|
|
50
56
|
opts: TestHandlerOptions<S> = {},
|
|
51
57
|
): Promise<TestResult<S>> {
|
|
52
|
-
if (!isImplementation(target))
|
|
58
|
+
if (!isImplementation(target))
|
|
59
|
+
throw new TypeError(
|
|
60
|
+
"testHandler expects implement(…): a handler module's default export or a step that carries its implementation",
|
|
61
|
+
);
|
|
53
62
|
const step: AnyStep = target.step;
|
|
54
63
|
const fn = target.handler as unknown as StepFunction;
|
|
55
64
|
|
|
56
65
|
const dir = opts.dir ?? mkdtempSync(join(tmpdir(), `olve-test-${step.name}-`));
|
|
57
|
-
const filesDir = join(dir,
|
|
66
|
+
const filesDir = join(dir, 'work', 'files');
|
|
58
67
|
const logs: string[] = [];
|
|
59
|
-
const run: RunInfo = {
|
|
68
|
+
const run: RunInfo = {
|
|
69
|
+
pipeline: 'test',
|
|
70
|
+
id: 'test',
|
|
71
|
+
number: 1,
|
|
72
|
+
attempt: 1,
|
|
73
|
+
version: '0.0.0-test',
|
|
74
|
+
...opts.run,
|
|
75
|
+
step: step.name,
|
|
76
|
+
};
|
|
60
77
|
const real = createCtx({
|
|
61
78
|
run,
|
|
62
79
|
secrets: step.secrets,
|
|
63
80
|
secretValues: opts.secrets as Record<string, string> | undefined,
|
|
64
|
-
paths: { root: dir, workFiles: filesDir, tmp: join(dir,
|
|
81
|
+
paths: { root: dir, workFiles: filesDir, tmp: join(dir, 'tmp') },
|
|
65
82
|
signal: opts.signal ?? new AbortController().signal,
|
|
66
83
|
logSink: (l) => logs.push(l),
|
|
67
84
|
callbackUrls: opts.callbackUrls,
|
|
68
85
|
});
|
|
69
86
|
const ctx = { ...real, ...opts.ctx } as Ctx;
|
|
70
87
|
// Round-trip through JSON like input.json does (drops undefined, rejects non-JSON values early).
|
|
71
|
-
const raw = JSON.parse(
|
|
72
|
-
|
|
73
|
-
|
|
88
|
+
const raw = JSON.parse(
|
|
89
|
+
JSON.stringify({ ...input, ...(opts.config ? { $config: opts.config } : {}) }),
|
|
90
|
+
);
|
|
91
|
+
const exec = await executeStep({
|
|
92
|
+
step,
|
|
93
|
+
fn,
|
|
94
|
+
rawInput: raw,
|
|
95
|
+
ctx,
|
|
96
|
+
workFiles: filesDir,
|
|
97
|
+
log: (l) => logs.push(l),
|
|
98
|
+
});
|
|
99
|
+
return {
|
|
100
|
+
dir,
|
|
101
|
+
filesDir,
|
|
102
|
+
logs,
|
|
103
|
+
exitCode: exec.exitCode,
|
|
104
|
+
...exec.envelope,
|
|
105
|
+
} as TestResult<S>;
|
|
74
106
|
}
|
|
75
107
|
|
|
76
108
|
/** A file reference for test inputs (`sha256` defaults to the hash of `content`). */
|
|
77
|
-
export function testFileRef(ref: {
|
|
109
|
+
export function testFileRef(ref: {
|
|
110
|
+
step: string;
|
|
111
|
+
path: string;
|
|
112
|
+
run?: number;
|
|
113
|
+
attempt?: number;
|
|
114
|
+
sha256?: string;
|
|
115
|
+
content?: string | Uint8Array;
|
|
116
|
+
}): FileRef {
|
|
78
117
|
return {
|
|
79
118
|
run: ref.run ?? 1,
|
|
80
119
|
step: ref.step,
|
|
81
120
|
attempt: ref.attempt ?? 1,
|
|
82
121
|
path: ref.path,
|
|
83
|
-
sha256: ref.sha256 ?? sha256Hex(ref.content ??
|
|
122
|
+
sha256: ref.sha256 ?? sha256Hex(ref.content ?? ''),
|
|
84
123
|
} as FileRef;
|
|
85
124
|
}
|
package/src/util/canonical.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// (order is meaningful: stages, steps, enum values), `undefined` members dropped. Stable for hashing.
|
|
3
3
|
export function canonicalize(v: unknown): unknown {
|
|
4
4
|
if (Array.isArray(v)) return v.map(canonicalize);
|
|
5
|
-
if (v !== null && typeof v ===
|
|
5
|
+
if (v !== null && typeof v === 'object') {
|
|
6
6
|
const out: Record<string, unknown> = {};
|
|
7
7
|
for (const k of Object.keys(v).sort()) {
|
|
8
8
|
const x = (v as Record<string, unknown>)[k];
|
|
@@ -10,10 +10,13 @@ export function canonicalize(v: unknown): unknown {
|
|
|
10
10
|
}
|
|
11
11
|
return out;
|
|
12
12
|
}
|
|
13
|
-
if (typeof v ===
|
|
14
|
-
|
|
13
|
+
if (typeof v === 'number' && !Number.isFinite(v))
|
|
14
|
+
throw new Error(`non-finite number in canonical JSON: ${v}`);
|
|
15
|
+
if (typeof v === 'bigint' || typeof v === 'function' || typeof v === 'symbol')
|
|
16
|
+
throw new Error(`${typeof v} is not JSON`);
|
|
15
17
|
return v;
|
|
16
18
|
}
|
|
17
19
|
|
|
18
20
|
/** Canonical JSON text, 2-space indented, with a trailing newline. */
|
|
19
|
-
export const canonicalJson = (v: unknown): string =>
|
|
21
|
+
export const canonicalJson = (v: unknown): string =>
|
|
22
|
+
JSON.stringify(canonicalize(v), null, 2) + '\n';
|
package/src/util/handler-hash.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { canonicalJson } from
|
|
2
|
-
import { sha256Hex } from
|
|
1
|
+
import { canonicalJson } from './canonical.ts';
|
|
2
|
+
import { sha256Hex } from './hash.ts';
|
|
3
3
|
|
|
4
4
|
/** Identity of the harness compiled into a binary. A Bun or SDK bump legitimately bumps every handler. */
|
|
5
5
|
export interface HarnessInfo {
|
|
@@ -13,10 +13,20 @@ export interface HarnessInfo {
|
|
|
13
13
|
|
|
14
14
|
/** Custom step: hash of the handler bundle (contract files + SDK external) combined with the harness. */
|
|
15
15
|
export function composeHandlerHash(bundleHash: string, harness: HarnessInfo): string {
|
|
16
|
-
return
|
|
16
|
+
return 'sha256:' + sha256Hex(canonicalJson({ bundle: bundleHash, harness }));
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
/** A step that carries its implementation: kind + canonical settings + harness, and the hash of the code that implements the kind. */
|
|
20
|
-
export function libraryHandlerHash(
|
|
21
|
-
|
|
20
|
+
export function libraryHandlerHash(
|
|
21
|
+
kind: string,
|
|
22
|
+
config: unknown,
|
|
23
|
+
harness: HarnessInfo,
|
|
24
|
+
code?: string,
|
|
25
|
+
): string {
|
|
26
|
+
return (
|
|
27
|
+
'sha256:' +
|
|
28
|
+
sha256Hex(
|
|
29
|
+
canonicalJson({ library: kind, config, harness, ...(code ? { code } : {}) }),
|
|
30
|
+
)
|
|
31
|
+
);
|
|
22
32
|
}
|
package/src/util/hash.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { openSync, readSync, closeSync } from
|
|
1
|
+
import { openSync, readSync, closeSync } from 'node:fs';
|
|
2
2
|
|
|
3
3
|
export function sha256Hex(data: string | Uint8Array | ArrayBuffer): string {
|
|
4
|
-
return new Bun.CryptoHasher(
|
|
4
|
+
return new Bun.CryptoHasher('sha256').update(data).digest('hex');
|
|
5
5
|
}
|
|
6
6
|
|
|
7
7
|
/** Streaming SHA-256 of a file (constant memory). */
|
|
8
8
|
export function sha256File(path: string): string {
|
|
9
|
-
const h = new Bun.CryptoHasher(
|
|
10
|
-
const fd = openSync(path,
|
|
9
|
+
const h = new Bun.CryptoHasher('sha256');
|
|
10
|
+
const fd = openSync(path, 'r');
|
|
11
11
|
try {
|
|
12
12
|
const buf = new Uint8Array(1 << 20);
|
|
13
13
|
for (;;) {
|
|
@@ -18,5 +18,5 @@ export function sha256File(path: string): string {
|
|
|
18
18
|
} finally {
|
|
19
19
|
closeSync(fd);
|
|
20
20
|
}
|
|
21
|
-
return h.digest(
|
|
21
|
+
return h.digest('hex');
|
|
22
22
|
}
|
package/src/util/tar.ts
CHANGED
|
@@ -1,27 +1,45 @@
|
|
|
1
1
|
// Deterministic tar writer (ustar + PAX long names), streaming from disk. Entries are sorted,
|
|
2
2
|
// with mtime 0, uid/gid 0, no user names — so equal content gives byte-identical archives (and
|
|
3
3
|
// sha256s). Extraction uses Bun.Archive.
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
import {
|
|
5
|
+
closeSync,
|
|
6
|
+
createReadStream,
|
|
7
|
+
createWriteStream,
|
|
8
|
+
lstatSync,
|
|
9
|
+
openSync,
|
|
10
|
+
readdirSync,
|
|
11
|
+
readlinkSync,
|
|
12
|
+
readSync,
|
|
13
|
+
renameSync,
|
|
14
|
+
rmSync,
|
|
15
|
+
writeSync,
|
|
16
|
+
} from 'node:fs';
|
|
17
|
+
import { join } from 'node:path';
|
|
18
|
+
import { pipeline } from 'node:stream/promises';
|
|
19
|
+
import { createGzip } from 'node:zlib';
|
|
8
20
|
|
|
9
21
|
export type TarEntry =
|
|
10
|
-
| { path: string; type:
|
|
11
|
-
| { path: string; type:
|
|
12
|
-
| { path: string; type:
|
|
13
|
-
| { path: string; type:
|
|
22
|
+
| { path: string; type: 'file'; mode: number; source: string }
|
|
23
|
+
| { path: string; type: 'bytes'; mode: number; data: Uint8Array }
|
|
24
|
+
| { path: string; type: 'dir'; mode: number }
|
|
25
|
+
| { path: string; type: 'symlink'; target: string };
|
|
14
26
|
|
|
15
27
|
const enc = new TextEncoder();
|
|
16
28
|
|
|
17
29
|
function octal(n: number, len: number): string {
|
|
18
|
-
return n.toString(8).padStart(len - 1,
|
|
30
|
+
return n.toString(8).padStart(len - 1, '0') + '\0';
|
|
19
31
|
}
|
|
20
32
|
|
|
21
|
-
function header(
|
|
33
|
+
function header(
|
|
34
|
+
name: string,
|
|
35
|
+
size: number,
|
|
36
|
+
mode: number,
|
|
37
|
+
typeflag: string,
|
|
38
|
+
linkname = '',
|
|
39
|
+
): Uint8Array {
|
|
22
40
|
const h = new Uint8Array(512);
|
|
23
41
|
const put = (s: string | Uint8Array, off: number, len: number) => {
|
|
24
|
-
const b = typeof s ===
|
|
42
|
+
const b = typeof s === 'string' ? enc.encode(s) : s;
|
|
25
43
|
h.set(b.subarray(0, len), off);
|
|
26
44
|
};
|
|
27
45
|
put(name, 0, 100);
|
|
@@ -30,14 +48,14 @@ function header(name: string, size: number, mode: number, typeflag: string, link
|
|
|
30
48
|
put(octal(0, 8), 116, 8);
|
|
31
49
|
put(octal(size, 12), 124, 12);
|
|
32
50
|
put(octal(0, 12), 136, 12);
|
|
33
|
-
put(
|
|
51
|
+
put(' ', 148, 8); // checksum placeholder
|
|
34
52
|
put(typeflag, 156, 1);
|
|
35
53
|
put(linkname, 157, 100);
|
|
36
|
-
put(
|
|
37
|
-
put(
|
|
54
|
+
put('ustar\0', 257, 6);
|
|
55
|
+
put('00', 263, 2);
|
|
38
56
|
let sum = 0;
|
|
39
57
|
for (const b of h) sum += b;
|
|
40
|
-
put(octal(sum, 7) +
|
|
58
|
+
put(octal(sum, 7) + ' ', 148, 8);
|
|
41
59
|
return h;
|
|
42
60
|
}
|
|
43
61
|
|
|
@@ -51,9 +69,11 @@ function paxRecord(key: string, value: string): string {
|
|
|
51
69
|
|
|
52
70
|
/** Writes `entries` (sorted by path here) as an uncompressed tar to `out`. */
|
|
53
71
|
export function writeTar(out: string, entries: readonly TarEntry[]): void {
|
|
54
|
-
const sorted = [...entries].sort((a, b) =>
|
|
72
|
+
const sorted = [...entries].sort((a, b) =>
|
|
73
|
+
a.path < b.path ? -1 : a.path > b.path ? 1 : 0,
|
|
74
|
+
);
|
|
55
75
|
const tmp = `${out}.partial`;
|
|
56
|
-
const fd = openSync(tmp,
|
|
76
|
+
const fd = openSync(tmp, 'w');
|
|
57
77
|
const write = (b: Uint8Array) => {
|
|
58
78
|
let off = 0;
|
|
59
79
|
while (off < b.length) off += writeSync(fd, b, off, b.length - off);
|
|
@@ -64,29 +84,31 @@ export function writeTar(out: string, entries: readonly TarEntry[]): void {
|
|
|
64
84
|
};
|
|
65
85
|
try {
|
|
66
86
|
for (const e of sorted) {
|
|
67
|
-
const name = e.type ===
|
|
68
|
-
const link = e.type ===
|
|
87
|
+
const name = e.type === 'dir' && !e.path.endsWith('/') ? e.path + '/' : e.path;
|
|
88
|
+
const link = e.type === 'symlink' ? e.target : '';
|
|
69
89
|
let size = 0;
|
|
70
|
-
if (e.type ===
|
|
71
|
-
else if (e.type ===
|
|
90
|
+
if (e.type === 'file') size = lstatSync(e.source).size;
|
|
91
|
+
else if (e.type === 'bytes') size = e.data.length;
|
|
72
92
|
const pax: string[] = [];
|
|
73
|
-
if (enc.encode(name).length > 100) pax.push(paxRecord(
|
|
74
|
-
if (enc.encode(link).length > 100) pax.push(paxRecord(
|
|
93
|
+
if (enc.encode(name).length > 100) pax.push(paxRecord('path', name));
|
|
94
|
+
if (enc.encode(link).length > 100) pax.push(paxRecord('linkpath', link));
|
|
75
95
|
if (pax.length) {
|
|
76
|
-
const body = enc.encode(pax.join(
|
|
77
|
-
write(header(
|
|
96
|
+
const body = enc.encode(pax.join(''));
|
|
97
|
+
write(header('././@PaxHeader', body.length, 0o644, 'x'));
|
|
78
98
|
write(body);
|
|
79
99
|
pad(body.length);
|
|
80
100
|
}
|
|
81
101
|
const shortName = enc.encode(name).length > 100 ? name.slice(0, 99) : name;
|
|
82
|
-
const mode = e.type ===
|
|
83
|
-
const flag = e.type ===
|
|
84
|
-
write(
|
|
85
|
-
|
|
102
|
+
const mode = e.type === 'symlink' ? 0o777 : e.mode;
|
|
103
|
+
const flag = e.type === 'dir' ? '5' : e.type === 'symlink' ? '2' : '0';
|
|
104
|
+
write(
|
|
105
|
+
header(shortName, size, mode, flag, link.length > 100 ? link.slice(0, 99) : link),
|
|
106
|
+
);
|
|
107
|
+
if (e.type === 'bytes') {
|
|
86
108
|
write(e.data);
|
|
87
109
|
pad(size);
|
|
88
|
-
} else if (e.type ===
|
|
89
|
-
const src = openSync(e.source,
|
|
110
|
+
} else if (e.type === 'file') {
|
|
111
|
+
const src = openSync(e.source, 'r');
|
|
90
112
|
try {
|
|
91
113
|
const buf = new Uint8Array(1 << 20);
|
|
92
114
|
let left = size;
|
|
@@ -110,18 +132,25 @@ export function writeTar(out: string, entries: readonly TarEntry[]): void {
|
|
|
110
132
|
}
|
|
111
133
|
|
|
112
134
|
/** Entries for the contents of `dir` (recursively), with paths prefixed by `prefix` ("" = none). */
|
|
113
|
-
export function dirEntries(dir: string, prefix =
|
|
135
|
+
export function dirEntries(dir: string, prefix = ''): TarEntry[] {
|
|
114
136
|
const out: TarEntry[] = [];
|
|
115
137
|
const walk = (abs: string, rel: string) => {
|
|
116
138
|
for (const name of readdirSync(abs).sort()) {
|
|
117
139
|
const a = join(abs, name);
|
|
118
140
|
const r = rel ? `${rel}/${name}` : name;
|
|
119
141
|
const st = lstatSync(a);
|
|
120
|
-
if (st.isSymbolicLink())
|
|
142
|
+
if (st.isSymbolicLink())
|
|
143
|
+
out.push({ path: r, type: 'symlink', target: readlinkSync(a) });
|
|
121
144
|
else if (st.isDirectory()) {
|
|
122
|
-
out.push({ path: r, type:
|
|
145
|
+
out.push({ path: r, type: 'dir', mode: 0o755 });
|
|
123
146
|
walk(a, r);
|
|
124
|
-
} else if (st.isFile())
|
|
147
|
+
} else if (st.isFile())
|
|
148
|
+
out.push({
|
|
149
|
+
path: r,
|
|
150
|
+
type: 'file',
|
|
151
|
+
mode: st.mode & 0o111 ? 0o755 : 0o644,
|
|
152
|
+
source: a,
|
|
153
|
+
});
|
|
125
154
|
// sockets, fifos, devices are skipped
|
|
126
155
|
}
|
|
127
156
|
};
|
|
@@ -137,7 +166,11 @@ export async function gzipFile(src: string, out: string, level = 6): Promise<voi
|
|
|
137
166
|
}
|
|
138
167
|
|
|
139
168
|
/** Writes a deterministic `.tar.gz`. */
|
|
140
|
-
export async function writeTarGz(
|
|
169
|
+
export async function writeTarGz(
|
|
170
|
+
out: string,
|
|
171
|
+
entries: readonly TarEntry[],
|
|
172
|
+
level = 6,
|
|
173
|
+
): Promise<void> {
|
|
141
174
|
const tar = `${out}.tar-tmp`;
|
|
142
175
|
try {
|
|
143
176
|
writeTar(tar, entries);
|
|
@@ -148,7 +181,11 @@ export async function writeTarGz(out: string, entries: readonly TarEntry[], leve
|
|
|
148
181
|
}
|
|
149
182
|
|
|
150
183
|
/** Writes a deterministic zstd-compressed tar (level 3: fast to write, and unpacks far faster than gzip). */
|
|
151
|
-
export async function writeTarZst(
|
|
184
|
+
export async function writeTarZst(
|
|
185
|
+
out: string,
|
|
186
|
+
entries: readonly TarEntry[],
|
|
187
|
+
level = 3,
|
|
188
|
+
): Promise<void> {
|
|
152
189
|
const tar = `${out}.tar-tmp`;
|
|
153
190
|
const tmp = `${out}.partial`;
|
|
154
191
|
try {
|