@olivervea/pipelines 2.0.0-beta.17 → 2.0.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/manifest.schema.json +22 -0
- package/package.json +4 -2
- package/src/assets.d.ts +2 -2
- package/src/build/build.ts +111 -43
- package/src/build/cli.ts +28 -21
- package/src/build/elf.ts +8 -7
- package/src/build/hashes.ts +61 -28
- package/src/build/project.ts +69 -30
- package/src/builder/pipeline.ts +288 -102
- package/src/builder/refs.ts +12 -6
- package/src/core/brands.ts +6 -6
- package/src/core/config-refs.ts +46 -20
- package/src/core/config.ts +110 -33
- package/src/core/external.ts +30 -12
- package/src/core/handlers.ts +54 -14
- package/src/core/models.ts +29 -14
- package/src/core/origin.ts +7 -4
- package/src/core/secrets.ts +2 -2
- package/src/core/steps.ts +39 -23
- package/src/core/types.ts +5 -1
- package/src/core/webhooks.ts +21 -7
- package/src/ctx/files.ts +111 -40
- package/src/ctx/index.ts +75 -34
- package/src/ctx/sh.ts +104 -44
- package/src/ctx/types.ts +4 -4
- package/src/describe/describe.ts +558 -194
- package/src/describe/manifest-schema.ts +217 -79
- package/src/describe/mapper-source.ts +249 -104
- package/src/describe/record.ts +67 -37
- package/src/dev/cli.ts +64 -34
- package/src/harness/env.ts +33 -27
- package/src/harness/execute.ts +55 -29
- package/src/harness/index.ts +21 -8
- package/src/harness/main.ts +197 -74
- package/src/harness/result.ts +38 -22
- package/src/harness/validate.ts +62 -19
- package/src/index.ts +123 -14
- package/src/library/checks.ts +11 -6
- package/src/library/compose-deploy.ts +40 -23
- package/src/library/github-source.ts +69 -39
- package/src/library/helm-deploy-ssh.ts +213 -76
- package/src/library/helm-deploy.ts +263 -85
- package/src/library/http-health.ts +7 -5
- package/src/library/image-build.ts +293 -117
- package/src/library/index.ts +47 -15
- package/src/library/models.ts +17 -9
- package/src/library/ssh.ts +31 -14
- package/src/testing/index.ts +63 -24
- package/src/util/canonical.ts +7 -4
- package/src/util/handler-hash.ts +15 -5
- package/src/util/hash.ts +5 -5
- package/src/util/tar.ts +74 -37
package/src/core/origin.ts
CHANGED
|
@@ -1,20 +1,23 @@
|
|
|
1
|
-
import { ORIGIN } from
|
|
1
|
+
import { ORIGIN } from './brands.ts';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* The file that called the function calling this one, from the stack. A tail call drops its caller's
|
|
5
5
|
* frame, so this can name a module further up; the build only uses it to choose what to hash.
|
|
6
6
|
*/
|
|
7
7
|
export function callerFile(): string | undefined {
|
|
8
|
-
const files = (new Error().stack ??
|
|
8
|
+
const files = (new Error().stack ?? '')
|
|
9
|
+
.split('\n')
|
|
10
|
+
.slice(1)
|
|
11
|
+
.map((f) => f.match(FRAME)?.[1]);
|
|
9
12
|
const own = files[0];
|
|
10
13
|
const callee = files[1];
|
|
11
14
|
const file = files.slice(2).find((f) => f && f !== own && f !== callee) ?? undefined;
|
|
12
|
-
return file?.startsWith(
|
|
15
|
+
return file?.startsWith('file://') ? new URL(file).pathname : file;
|
|
13
16
|
}
|
|
14
17
|
const FRAME = /\(?((?:file:\/\/)?\/[^()]*?):\d+(?::\d+)?\)?\s*$/;
|
|
15
18
|
|
|
16
19
|
/** The modules that defined and implemented a step (where `step` and `implement` were called). */
|
|
17
20
|
export function originsOf(v: unknown): string[] {
|
|
18
21
|
const o = (v as Record<symbol, unknown>)[ORIGIN];
|
|
19
|
-
return Array.isArray(o) ? o.filter((x): x is string => typeof x ===
|
|
22
|
+
return Array.isArray(o) ? o.filter((x): x is string => typeof x === 'string') : [];
|
|
20
23
|
}
|
package/src/core/secrets.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** A named pipeline secret. Steps (and poll triggers) declare the secrets they use; only those are mounted. */
|
|
2
2
|
export interface Secret<N extends string = string> {
|
|
3
|
-
readonly kind:
|
|
3
|
+
readonly kind: 'secret';
|
|
4
4
|
readonly name: N;
|
|
5
5
|
readonly description: string;
|
|
6
6
|
}
|
|
@@ -8,6 +8,6 @@ export interface Secret<N extends string = string> {
|
|
|
8
8
|
export const SECRET_NAME = /^[A-Z_][A-Z0-9_]*$/;
|
|
9
9
|
|
|
10
10
|
export function secret<const N extends string>(name: N, description: string): Secret<N> {
|
|
11
|
-
return Object.freeze({ kind:
|
|
11
|
+
return Object.freeze({ kind: 'secret' as const, name, description });
|
|
12
12
|
}
|
|
13
13
|
export type DeclaredSecrets<N extends string> = { readonly [K in N]: string };
|
package/src/core/steps.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import type { z } from
|
|
2
|
-
import { ORIGIN, STEP, type StepBrand } from
|
|
3
|
-
import { callerFile } from
|
|
4
|
-
import type { AnyModel, Infer, Optional } from
|
|
5
|
-
import type { External } from
|
|
6
|
-
import type { AnyConfigEntry } from
|
|
7
|
-
import { configRefsIn } from
|
|
8
|
-
import type { Secret } from
|
|
9
|
-
import type { Simplify } from
|
|
1
|
+
import type { z } from 'zod';
|
|
2
|
+
import { ORIGIN, STEP, type StepBrand } from './brands.ts';
|
|
3
|
+
import { callerFile } from './origin.ts';
|
|
4
|
+
import type { AnyModel, Infer, Optional } from './models.ts';
|
|
5
|
+
import type { External } from './external.ts';
|
|
6
|
+
import type { AnyConfigEntry } from './config.ts';
|
|
7
|
+
import { configRefsIn } from './config-refs.ts';
|
|
8
|
+
import type { Secret } from './secrets.ts';
|
|
9
|
+
import type { Simplify } from './types.ts';
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
12
|
* A step input entry: a model (matchable by model), `optional(model)`, a plain zod schema (mapper required),
|
|
@@ -16,11 +16,15 @@ export type InputEntry = AnyModel | Optional | External | z.ZodType;
|
|
|
16
16
|
export type InputSpec = { readonly [key: string]: InputEntry };
|
|
17
17
|
|
|
18
18
|
export type InferEntry<E> =
|
|
19
|
-
E extends Optional<infer M>
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
19
|
+
E extends Optional<infer M>
|
|
20
|
+
? Infer<M> | undefined
|
|
21
|
+
: E extends External<infer M>
|
|
22
|
+
? Infer<M>
|
|
23
|
+
: E extends AnyModel
|
|
24
|
+
? Infer<E>
|
|
25
|
+
: E extends z.ZodType
|
|
26
|
+
? z.output<E>
|
|
27
|
+
: never;
|
|
24
28
|
/** What a handler receives: the output view of every input entry. */
|
|
25
29
|
export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
|
|
26
30
|
|
|
@@ -62,16 +66,25 @@ export type AnyStep = Step<string, any, AnyModel | null, any, any>;
|
|
|
62
66
|
export type SecretsOf<S> = S extends Step<any, any, any, infer Sec, any> ? Sec : never;
|
|
63
67
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
64
68
|
export type ConfigOf<S> = S extends Step<any, any, any, any, infer C> ? C : never;
|
|
65
|
-
export type OutputValue<S extends AnyStep> = S[
|
|
69
|
+
export type OutputValue<S extends AnyStep> = S['output'] extends AnyModel
|
|
70
|
+
? Infer<S['output']>
|
|
71
|
+
: void;
|
|
66
72
|
|
|
67
73
|
/** Step names must be literals (identity, duplicate detection and mapper keys depend on it). */
|
|
68
|
-
export type LiteralName<N extends string> = N &
|
|
74
|
+
export type LiteralName<N extends string> = N &
|
|
75
|
+
(string extends N ? '❌ step name must be a string literal' : unknown);
|
|
69
76
|
|
|
70
|
-
export { STEP_NAME } from
|
|
77
|
+
export { STEP_NAME } from './models.ts';
|
|
71
78
|
/** Names that cannot be steps: `input` is the pipeline input's root in mappings. */
|
|
72
|
-
export const RESERVED_STEP_NAMES: readonly string[] = [
|
|
79
|
+
export const RESERVED_STEP_NAMES: readonly string[] = ['input'];
|
|
73
80
|
|
|
74
|
-
export interface StepDef<
|
|
81
|
+
export interface StepDef<
|
|
82
|
+
N extends string,
|
|
83
|
+
I extends InputSpec,
|
|
84
|
+
O extends AnyModel | null,
|
|
85
|
+
Sec extends readonly Secret[],
|
|
86
|
+
Cfg extends readonly AnyConfigEntry[] = [],
|
|
87
|
+
> {
|
|
75
88
|
name: LiteralName<N>;
|
|
76
89
|
/** What the step does, shown in the UI. Required; may be empty. */
|
|
77
90
|
description: string;
|
|
@@ -102,11 +115,12 @@ export interface StepDef<N extends string, I extends InputSpec, O extends AnyMod
|
|
|
102
115
|
|
|
103
116
|
function makeStep(def: Record<string, unknown>, origin: string | undefined): AnyStep {
|
|
104
117
|
const config = new Map<string, AnyConfigEntry>();
|
|
105
|
-
for (const e of (def.config as readonly AnyConfigEntry[] | undefined) ?? [])
|
|
118
|
+
for (const e of (def.config as readonly AnyConfigEntry[] | undefined) ?? [])
|
|
119
|
+
if (!config.has(e.name)) config.set(e.name, e);
|
|
106
120
|
configRefsIn(def.settings, config);
|
|
107
121
|
const s = {
|
|
108
122
|
[STEP]: true,
|
|
109
|
-
kind: def.kind ??
|
|
123
|
+
kind: def.kind ?? 'custom',
|
|
110
124
|
name: def.name,
|
|
111
125
|
description: def.description,
|
|
112
126
|
image: def.image,
|
|
@@ -124,7 +138,9 @@ function makeStep(def: Record<string, unknown>, origin: string | undefined): Any
|
|
|
124
138
|
}
|
|
125
139
|
|
|
126
140
|
export function isStep(v: unknown): v is AnyStep {
|
|
127
|
-
return
|
|
141
|
+
return (
|
|
142
|
+
typeof v === 'object' && v !== null && (v as Record<symbol, unknown>)[STEP] === true
|
|
143
|
+
);
|
|
128
144
|
}
|
|
129
145
|
|
|
130
146
|
/**
|
|
@@ -137,6 +153,6 @@ export function step<
|
|
|
137
153
|
O extends AnyModel | null = null,
|
|
138
154
|
const Sec extends readonly Secret[] = [],
|
|
139
155
|
const Cfg extends readonly AnyConfigEntry[] = [],
|
|
140
|
-
>(def: StepDef<N, I, O, Sec, Cfg>): Step<N, I, O, Sec[number][
|
|
156
|
+
>(def: StepDef<N, I, O, Sec, Cfg>): Step<N, I, O, Sec[number]['name'], Cfg[number]> {
|
|
141
157
|
return makeStep(def as never, callerFile()) as never;
|
|
142
158
|
}
|
package/src/core/types.ts
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
export type Primitive = string | number | boolean | bigint | null | undefined;
|
|
2
2
|
export type Simplify<T> = { [K in keyof T]: T[K] } & {};
|
|
3
3
|
export type Awaitable<T> = T | Promise<T>;
|
|
4
|
-
export type IsUnion<T, U = T> = T extends unknown
|
|
4
|
+
export type IsUnion<T, U = T> = T extends unknown
|
|
5
|
+
? [U] extends [T]
|
|
6
|
+
? false
|
|
7
|
+
: true
|
|
8
|
+
: never;
|
|
5
9
|
/** `Ok` when there is no error, else the error (a readable string-literal type). */
|
|
6
10
|
export type OrError<Err, Ok> = [Err] extends [never] ? Ok : Err;
|
package/src/core/webhooks.ts
CHANGED
|
@@ -1,23 +1,37 @@
|
|
|
1
|
-
import type { Secret } from
|
|
1
|
+
import type { Secret } from './secrets.ts';
|
|
2
2
|
|
|
3
3
|
/** How a webhook trigger authenticates deliveries. Build one with `hmacSha256`, `githubSignature` or `bearer`. */
|
|
4
4
|
export type WebhookVerify =
|
|
5
|
-
| {
|
|
6
|
-
|
|
5
|
+
| {
|
|
6
|
+
readonly scheme: 'hmac-sha256';
|
|
7
|
+
readonly secret: Secret;
|
|
8
|
+
readonly header: string;
|
|
9
|
+
readonly prefix: string;
|
|
10
|
+
}
|
|
11
|
+
| { readonly scheme: 'bearer'; readonly secret: Secret };
|
|
7
12
|
|
|
8
13
|
/** HMAC-SHA256 of the raw body keyed by `secret`, hex, in `header` after `prefix`. */
|
|
9
|
-
export function hmacSha256(o: {
|
|
10
|
-
|
|
14
|
+
export function hmacSha256(o: {
|
|
15
|
+
secret: Secret;
|
|
16
|
+
header: string;
|
|
17
|
+
prefix?: string;
|
|
18
|
+
}): WebhookVerify {
|
|
19
|
+
return Object.freeze({
|
|
20
|
+
scheme: 'hmac-sha256',
|
|
21
|
+
secret: o.secret,
|
|
22
|
+
header: o.header,
|
|
23
|
+
prefix: o.prefix ?? '',
|
|
24
|
+
});
|
|
11
25
|
}
|
|
12
26
|
|
|
13
27
|
/** GitHub's delivery signature: `X-Hub-Signature-256: sha256=<hex>`. */
|
|
14
28
|
export function githubSignature(secret: Secret): WebhookVerify {
|
|
15
|
-
return hmacSha256({ secret, header:
|
|
29
|
+
return hmacSha256({ secret, header: 'X-Hub-Signature-256', prefix: 'sha256=' });
|
|
16
30
|
}
|
|
17
31
|
|
|
18
32
|
/** `Authorization: Bearer <secret>`. */
|
|
19
33
|
export function bearer(secret: Secret): WebhookVerify {
|
|
20
|
-
return Object.freeze({ scheme:
|
|
34
|
+
return Object.freeze({ scheme: 'bearer', secret });
|
|
21
35
|
}
|
|
22
36
|
|
|
23
37
|
/**
|
package/src/ctx/files.ts
CHANGED
|
@@ -4,19 +4,31 @@
|
|
|
4
4
|
// `path` minus the trailing "/" plus ".tar" for directories (a deterministic tar). The step writes
|
|
5
5
|
// objects to `$OLVE_DIR/work/files/<object path>`; the service stores them under
|
|
6
6
|
// `runs/<p>/<run>/<step>/<attempt>/files/<object path>`.
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
7
|
+
import {
|
|
8
|
+
copyFileSync,
|
|
9
|
+
existsSync,
|
|
10
|
+
mkdirSync,
|
|
11
|
+
readFileSync,
|
|
12
|
+
renameSync,
|
|
13
|
+
rmSync,
|
|
14
|
+
statSync,
|
|
15
|
+
writeFileSync,
|
|
16
|
+
} from 'node:fs';
|
|
17
|
+
import { basename, dirname, join, resolve } from 'node:path';
|
|
18
|
+
import { fileRefSchema, FILE_PATH, type FileRef } from '../core/models.ts';
|
|
19
|
+
import { ENV } from '../harness/env.ts';
|
|
20
|
+
import { sha256File } from '../util/hash.ts';
|
|
21
|
+
import { dirEntries, extractTar, writeTar } from '../util/tar.ts';
|
|
22
|
+
import type { OutputFileOptions, RunInfo } from './types.ts';
|
|
14
23
|
|
|
15
|
-
export const objectPath = (refPath: string): string =>
|
|
16
|
-
|
|
24
|
+
export const objectPath = (refPath: string): string =>
|
|
25
|
+
refPath.endsWith('/') ? refPath.slice(0, -1) + '.tar' : refPath;
|
|
26
|
+
export const fileKey = (
|
|
27
|
+
ref: Pick<FileRef, 'run' | 'step' | 'attempt' | 'path'>,
|
|
28
|
+
): string => `${ref.run}/${ref.step}/${ref.attempt}/${ref.path}`;
|
|
17
29
|
|
|
18
30
|
/** Strips everything from `?` on (pre-signed credentials). */
|
|
19
|
-
export const redactUrl = (url: string): string => url.replace(/[?#].*$/,
|
|
31
|
+
export const redactUrl = (url: string): string => url.replace(/[?#].*$/, '?…');
|
|
20
32
|
|
|
21
33
|
export interface FilesDeps {
|
|
22
34
|
run: RunInfo;
|
|
@@ -28,42 +40,54 @@ export interface FilesDeps {
|
|
|
28
40
|
}
|
|
29
41
|
|
|
30
42
|
function checkName(name: string, what: string): void {
|
|
31
|
-
if (!FILE_PATH.test(name) || name.endsWith(
|
|
43
|
+
if (!FILE_PATH.test(name) || name.endsWith('/'))
|
|
44
|
+
throw new Error(
|
|
45
|
+
`${what}: '${name}' is not a valid relative file name (no leading '/', no '..', no trailing '/')`,
|
|
46
|
+
);
|
|
32
47
|
}
|
|
33
48
|
|
|
34
49
|
export function createOutput(deps: FilesDeps) {
|
|
35
50
|
const used = new Set<string>();
|
|
36
51
|
const claim = (obj: string) => {
|
|
37
|
-
if (used.has(obj) || existsSync(join(deps.workFiles, obj)))
|
|
52
|
+
if (used.has(obj) || existsSync(join(deps.workFiles, obj)))
|
|
53
|
+
throw new Error(`output file '${obj}' already exists`);
|
|
38
54
|
used.add(obj);
|
|
39
55
|
const dest = join(deps.workFiles, obj);
|
|
40
56
|
mkdirSync(dirname(dest), { recursive: true });
|
|
41
57
|
return dest;
|
|
42
58
|
};
|
|
43
59
|
const ref = (path: string, dest: string): FileRef =>
|
|
44
|
-
({
|
|
60
|
+
({
|
|
61
|
+
run: deps.run.number,
|
|
62
|
+
step: deps.run.step,
|
|
63
|
+
attempt: deps.run.attempt,
|
|
64
|
+
path,
|
|
65
|
+
sha256: sha256File(dest),
|
|
66
|
+
}) as FileRef;
|
|
45
67
|
|
|
46
68
|
return {
|
|
47
69
|
async file(path: string, opts: OutputFileOptions = {}): Promise<FileRef> {
|
|
48
70
|
const src = resolve(path);
|
|
49
|
-
if (!statSync(src).isFile())
|
|
71
|
+
if (!statSync(src).isFile())
|
|
72
|
+
throw new Error(`ctx.output.file: '${path}' is not a file`);
|
|
50
73
|
const name = opts.name ?? basename(src);
|
|
51
|
-
checkName(name,
|
|
74
|
+
checkName(name, 'ctx.output.file');
|
|
52
75
|
const dest = claim(name);
|
|
53
76
|
copyFileSync(src, dest);
|
|
54
77
|
return ref(name, dest);
|
|
55
78
|
},
|
|
56
79
|
async dir(path: string, opts: OutputFileOptions = {}): Promise<FileRef> {
|
|
57
80
|
const src = resolve(path);
|
|
58
|
-
if (!statSync(src).isDirectory())
|
|
81
|
+
if (!statSync(src).isDirectory())
|
|
82
|
+
throw new Error(`ctx.output.dir: '${path}' is not a directory`);
|
|
59
83
|
const name = opts.name ?? basename(src);
|
|
60
|
-
checkName(name,
|
|
61
|
-
const dest = claim(objectPath(name +
|
|
84
|
+
checkName(name, 'ctx.output.dir');
|
|
85
|
+
const dest = claim(objectPath(name + '/'));
|
|
62
86
|
writeTar(dest, dirEntries(src));
|
|
63
|
-
return ref(name +
|
|
87
|
+
return ref(name + '/', dest);
|
|
64
88
|
},
|
|
65
89
|
async write(name: string, content: string | Uint8Array): Promise<FileRef> {
|
|
66
|
-
checkName(name,
|
|
90
|
+
checkName(name, 'ctx.output.write');
|
|
67
91
|
const dest = claim(name);
|
|
68
92
|
writeFileSync(dest, content);
|
|
69
93
|
return ref(name, dest);
|
|
@@ -72,20 +96,33 @@ export function createOutput(deps: FilesDeps) {
|
|
|
72
96
|
}
|
|
73
97
|
|
|
74
98
|
function urlMap(root: string): Record<string, string> {
|
|
75
|
-
const raw =
|
|
99
|
+
const raw =
|
|
100
|
+
process.env[ENV.fileUrls] ??
|
|
101
|
+
(existsSync(join(root, 'files.json'))
|
|
102
|
+
? readFileSync(join(root, 'files.json'), 'utf8')
|
|
103
|
+
: undefined);
|
|
76
104
|
if (!raw) return {};
|
|
77
105
|
try {
|
|
78
106
|
const m = JSON.parse(raw) as unknown;
|
|
79
|
-
if (typeof m !==
|
|
107
|
+
if (typeof m !== 'object' || m === null || Array.isArray(m))
|
|
108
|
+
throw new Error('not an object');
|
|
80
109
|
return m as Record<string, string>;
|
|
81
110
|
} catch (e) {
|
|
82
|
-
throw new Error(
|
|
111
|
+
throw new Error(
|
|
112
|
+
`${ENV.fileUrls} / files.json is not a JSON object of file key → URL (${(e as Error).message})`,
|
|
113
|
+
);
|
|
83
114
|
}
|
|
84
115
|
}
|
|
85
116
|
|
|
86
117
|
const TRANSIENT = (status: number) => status >= 500 || status === 408 || status === 429;
|
|
87
118
|
|
|
88
|
-
async function download(
|
|
119
|
+
async function download(
|
|
120
|
+
url: string,
|
|
121
|
+
dest: string,
|
|
122
|
+
headers: Record<string, string>,
|
|
123
|
+
signal: AbortSignal,
|
|
124
|
+
log: FilesDeps['log'],
|
|
125
|
+
): Promise<void> {
|
|
89
126
|
const attempts = 4;
|
|
90
127
|
for (let i = 1; ; i++) {
|
|
91
128
|
let status = 0;
|
|
@@ -93,14 +130,19 @@ async function download(url: string, dest: string, headers: Record<string, strin
|
|
|
93
130
|
const res = await fetch(url, { headers, signal });
|
|
94
131
|
status = res.status;
|
|
95
132
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
96
|
-
await Bun.write(dest +
|
|
97
|
-
renameSync(dest +
|
|
133
|
+
await Bun.write(dest + '.partial', res);
|
|
134
|
+
renameSync(dest + '.partial', dest);
|
|
98
135
|
return;
|
|
99
136
|
} catch (e) {
|
|
100
|
-
rmSync(dest +
|
|
137
|
+
rmSync(dest + '.partial', { force: true });
|
|
101
138
|
const transient = status === 0 || TRANSIENT(status);
|
|
102
|
-
if (signal.aborted || !transient || i >= attempts)
|
|
103
|
-
|
|
139
|
+
if (signal.aborted || !transient || i >= attempts)
|
|
140
|
+
throw new Error(`download of ${redactUrl(url)} failed: ${(e as Error).message}`);
|
|
141
|
+
log('download failed, retrying', {
|
|
142
|
+
url: redactUrl(url),
|
|
143
|
+
attempt: i,
|
|
144
|
+
error: (e as Error).message,
|
|
145
|
+
});
|
|
104
146
|
await Bun.sleep(1000 * 2 ** (i - 1));
|
|
105
147
|
}
|
|
106
148
|
}
|
|
@@ -110,29 +152,56 @@ export function createFetch(deps: FilesDeps) {
|
|
|
110
152
|
const cache = new Map<string, string>();
|
|
111
153
|
return async (input: FileRef): Promise<string> => {
|
|
112
154
|
const parsed = fileRefSchema.safeParse(input);
|
|
113
|
-
if (!parsed.success)
|
|
155
|
+
if (!parsed.success)
|
|
156
|
+
throw new Error(
|
|
157
|
+
`ctx.files.fetch: not a valid file reference: ${JSON.stringify(input)}`,
|
|
158
|
+
);
|
|
114
159
|
const ref = parsed.data;
|
|
115
160
|
const key = fileKey(ref);
|
|
116
161
|
const hit = cache.get(key);
|
|
117
162
|
if (hit) return hit;
|
|
118
163
|
|
|
119
164
|
const obj = objectPath(ref.path);
|
|
120
|
-
const base = join(
|
|
121
|
-
|
|
165
|
+
const base = join(
|
|
166
|
+
deps.tmpDir,
|
|
167
|
+
'files',
|
|
168
|
+
String(ref.run),
|
|
169
|
+
ref.step,
|
|
170
|
+
String(ref.attempt),
|
|
171
|
+
);
|
|
172
|
+
const objFile = join(base, '.objects', obj);
|
|
122
173
|
mkdirSync(dirname(objFile), { recursive: true });
|
|
123
174
|
|
|
124
|
-
const own =
|
|
175
|
+
const own =
|
|
176
|
+
ref.run === deps.run.number &&
|
|
177
|
+
ref.step === deps.run.step &&
|
|
178
|
+
ref.attempt === deps.run.attempt;
|
|
125
179
|
const urls = urlMap(deps.root);
|
|
126
180
|
const localDir = process.env[ENV.filesDir];
|
|
127
181
|
const baseUrl = process.env[ENV.filesBaseUrl];
|
|
128
|
-
if (own && existsSync(join(deps.workFiles, obj)))
|
|
182
|
+
if (own && existsSync(join(deps.workFiles, obj)))
|
|
183
|
+
copyFileSync(join(deps.workFiles, obj), objFile);
|
|
129
184
|
else if (urls[key]) await download(urls[key], objFile, {}, deps.signal, deps.log);
|
|
130
|
-
else if (
|
|
131
|
-
|
|
185
|
+
else if (
|
|
186
|
+
localDir &&
|
|
187
|
+
existsSync(
|
|
188
|
+
join(localDir, String(ref.run), ref.step, String(ref.attempt), 'files', obj),
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
copyFileSync(
|
|
192
|
+
join(localDir, String(ref.run), ref.step, String(ref.attempt), 'files', obj),
|
|
193
|
+
objFile,
|
|
194
|
+
);
|
|
132
195
|
else if (baseUrl) {
|
|
133
196
|
const token = process.env[ENV.filesToken];
|
|
134
|
-
const url = `${baseUrl.replace(/\/+$/,
|
|
135
|
-
await download(
|
|
197
|
+
const url = `${baseUrl.replace(/\/+$/, '')}/${ref.run}/${encodeURIComponent(ref.step)}/${ref.attempt}/${obj.split('/').map(encodeURIComponent).join('/')}`;
|
|
198
|
+
await download(
|
|
199
|
+
url,
|
|
200
|
+
objFile,
|
|
201
|
+
token ? { authorization: `Bearer ${token}` } : {},
|
|
202
|
+
deps.signal,
|
|
203
|
+
deps.log,
|
|
204
|
+
);
|
|
136
205
|
} else {
|
|
137
206
|
throw new Error(
|
|
138
207
|
`ctx.files.fetch: no source for ${key} (set ${ENV.fileUrls} / $OLVE_DIR/files.json, ${ENV.filesBaseUrl} or ${ENV.filesDir})`,
|
|
@@ -142,10 +211,12 @@ export function createFetch(deps: FilesDeps) {
|
|
|
142
211
|
const actual = sha256File(objFile);
|
|
143
212
|
if (actual !== ref.sha256) {
|
|
144
213
|
rmSync(objFile, { force: true });
|
|
145
|
-
throw new Error(
|
|
214
|
+
throw new Error(
|
|
215
|
+
`ctx.files.fetch: ${key} has sha256 ${actual}, expected ${ref.sha256}`,
|
|
216
|
+
);
|
|
146
217
|
}
|
|
147
218
|
let result = objFile;
|
|
148
|
-
if (ref.path.endsWith(
|
|
219
|
+
if (ref.path.endsWith('/')) {
|
|
149
220
|
result = join(base, ref.path.slice(0, -1));
|
|
150
221
|
rmSync(result, { recursive: true, force: true });
|
|
151
222
|
mkdirSync(result, { recursive: true });
|
package/src/ctx/index.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { mkdirSync, readFileSync } from
|
|
2
|
-
import { resolve } from
|
|
3
|
-
import { skipped } from
|
|
4
|
-
import type { Secret } from
|
|
5
|
-
import { createFetch, createOutput, redactUrl } from
|
|
6
|
-
import { createSh } from
|
|
7
|
-
import type { Ctx, Http, RetryOptions, RunInfo } from
|
|
1
|
+
import { mkdirSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
import { skipped } from '../core/handlers.ts';
|
|
4
|
+
import type { Secret } from '../core/secrets.ts';
|
|
5
|
+
import { createFetch, createOutput, redactUrl } from './files.ts';
|
|
6
|
+
import { createSh } from './sh.ts';
|
|
7
|
+
import type { Ctx, Http, RetryOptions, RunInfo } from './types.ts';
|
|
8
8
|
|
|
9
9
|
export interface CtxOptions {
|
|
10
10
|
run: RunInfo;
|
|
@@ -23,18 +23,27 @@ export interface CtxOptions {
|
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
export function formatLog(message: string, fields?: Record<string, unknown>): string {
|
|
26
|
-
const parts = Object.entries(fields ?? {}).map(
|
|
27
|
-
|
|
26
|
+
const parts = Object.entries(fields ?? {}).map(
|
|
27
|
+
([k, v]) =>
|
|
28
|
+
`${k}=${typeof v === 'string' && !/\s|"/.test(v) ? v : JSON.stringify(v)}`,
|
|
29
|
+
);
|
|
30
|
+
return parts.length ? `${message} ${parts.join(' ')}` : message;
|
|
28
31
|
}
|
|
29
32
|
|
|
30
|
-
export function createSecrets(
|
|
33
|
+
export function createSecrets(
|
|
34
|
+
declared: readonly Secret[],
|
|
35
|
+
values?: Readonly<Record<string, string>>,
|
|
36
|
+
): Record<string, string> {
|
|
31
37
|
const obj: Record<string, string> = {};
|
|
32
38
|
for (const s of declared) {
|
|
33
39
|
Object.defineProperty(obj, s.name, {
|
|
34
40
|
enumerable: true,
|
|
35
41
|
get() {
|
|
36
42
|
const v = values?.[s.name] ?? process.env[s.name];
|
|
37
|
-
if (v === undefined)
|
|
43
|
+
if (v === undefined)
|
|
44
|
+
throw new Error(
|
|
45
|
+
`secret ${s.name} is not set (the harness reads it from the environment variable ${s.name})`,
|
|
46
|
+
);
|
|
38
47
|
return v;
|
|
39
48
|
},
|
|
40
49
|
});
|
|
@@ -43,52 +52,74 @@ export function createSecrets(declared: readonly Secret[], values?: Readonly<Rec
|
|
|
43
52
|
}
|
|
44
53
|
|
|
45
54
|
async function httpError(res: Response, url: string): Promise<Error> {
|
|
46
|
-
let body =
|
|
55
|
+
let body = '';
|
|
47
56
|
try {
|
|
48
57
|
body = (await res.text()).slice(0, 500);
|
|
49
58
|
} catch {
|
|
50
59
|
/* ignore */
|
|
51
60
|
}
|
|
52
|
-
return new Error(
|
|
61
|
+
return new Error(
|
|
62
|
+
`HTTP ${res.status} ${res.statusText} from ${redactUrl(url)}${body ? `: ${body}` : ''}`,
|
|
63
|
+
);
|
|
53
64
|
}
|
|
54
65
|
|
|
55
66
|
export function createHttp(signal: AbortSignal): Http {
|
|
56
|
-
const f = (url: string, init?: RequestInit) =>
|
|
67
|
+
const f = (url: string, init?: RequestInit) =>
|
|
68
|
+
fetch(url, { ...init, signal: init?.signal ?? signal });
|
|
57
69
|
return {
|
|
58
70
|
fetch: f,
|
|
59
|
-
async getJson<T>(
|
|
60
|
-
|
|
71
|
+
async getJson<T>(
|
|
72
|
+
url: string,
|
|
73
|
+
init?: { headers?: Record<string, string> },
|
|
74
|
+
): Promise<T> {
|
|
75
|
+
const res = await f(url, {
|
|
76
|
+
headers: { accept: 'application/json', ...init?.headers },
|
|
77
|
+
});
|
|
61
78
|
if (!res.ok) throw await httpError(res, url);
|
|
62
79
|
return (await res.json()) as T;
|
|
63
80
|
},
|
|
64
|
-
async expectOk(
|
|
65
|
-
|
|
81
|
+
async expectOk(
|
|
82
|
+
url: string,
|
|
83
|
+
init?: { headers?: Record<string, string>; method?: string },
|
|
84
|
+
): Promise<Response> {
|
|
85
|
+
const res = await f(url, { method: init?.method ?? 'GET', headers: init?.headers });
|
|
66
86
|
if (!res.ok) throw await httpError(res, url);
|
|
67
87
|
return res;
|
|
68
88
|
},
|
|
69
89
|
};
|
|
70
90
|
}
|
|
71
91
|
|
|
72
|
-
export function createRetry(
|
|
73
|
-
|
|
74
|
-
|
|
92
|
+
export function createRetry(
|
|
93
|
+
signal: AbortSignal,
|
|
94
|
+
log: (m: string, f?: Record<string, unknown>) => void,
|
|
95
|
+
) {
|
|
96
|
+
return async <T>(
|
|
97
|
+
opts: RetryOptions,
|
|
98
|
+
fn: (attempt: number) => Promise<T>,
|
|
99
|
+
): Promise<T> => {
|
|
100
|
+
if (!(Number.isInteger(opts.attempts) && opts.attempts >= 1))
|
|
101
|
+
throw new Error('retry: attempts must be a positive integer');
|
|
75
102
|
let delay = Math.max(0, opts.delaySeconds) * 1000;
|
|
76
103
|
for (let attempt = 1; ; attempt++) {
|
|
77
104
|
try {
|
|
78
105
|
return await fn(attempt);
|
|
79
106
|
} catch (e) {
|
|
80
107
|
if (attempt >= opts.attempts || signal.aborted) throw e;
|
|
81
|
-
log(
|
|
108
|
+
log('retrying', {
|
|
109
|
+
attempt,
|
|
110
|
+
of: opts.attempts,
|
|
111
|
+
error: (e as Error)?.message ?? String(e),
|
|
112
|
+
});
|
|
82
113
|
await new Promise<void>((res, rej) => {
|
|
83
114
|
const t = setTimeout(() => {
|
|
84
|
-
signal.removeEventListener(
|
|
115
|
+
signal.removeEventListener('abort', onAbort);
|
|
85
116
|
res();
|
|
86
117
|
}, delay);
|
|
87
118
|
const onAbort = () => {
|
|
88
119
|
clearTimeout(t);
|
|
89
|
-
rej(new Error(
|
|
120
|
+
rej(new Error('retry cancelled: the step was cancelled'));
|
|
90
121
|
};
|
|
91
|
-
signal.addEventListener(
|
|
122
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
92
123
|
});
|
|
93
124
|
delay *= opts.backoff ?? 1;
|
|
94
125
|
}
|
|
@@ -100,17 +131,25 @@ export function createRetry(signal: AbortSignal, log: (m: string, f?: Record<str
|
|
|
100
131
|
export function createCtx(o: CtxOptions): Ctx {
|
|
101
132
|
mkdirSync(o.paths.tmp, { recursive: true });
|
|
102
133
|
mkdirSync(o.paths.workFiles, { recursive: true });
|
|
103
|
-
const sink = o.logSink ?? ((line: string) => process.stderr.write(line +
|
|
104
|
-
const log = (message: string, fields?: Record<string, unknown>) =>
|
|
134
|
+
const sink = o.logSink ?? ((line: string) => process.stderr.write(line + '\n'));
|
|
135
|
+
const log = (message: string, fields?: Record<string, unknown>) =>
|
|
136
|
+
sink(formatLog(message, fields));
|
|
105
137
|
const { sh, exec } = createSh({ tmpDir: o.paths.tmp, signal: o.signal });
|
|
106
|
-
const filesDeps = {
|
|
138
|
+
const filesDeps = {
|
|
139
|
+
run: o.run,
|
|
140
|
+
workFiles: o.paths.workFiles,
|
|
141
|
+
tmpDir: o.paths.tmp,
|
|
142
|
+
root: o.paths.root,
|
|
143
|
+
signal: o.signal,
|
|
144
|
+
log,
|
|
145
|
+
};
|
|
107
146
|
return {
|
|
108
|
-
secrets: createSecrets(o.secrets, o.secretValues) as Ctx[
|
|
109
|
-
config: Object.freeze({ ...o.config }) as Ctx[
|
|
147
|
+
secrets: createSecrets(o.secrets, o.secretValues) as Ctx['secrets'],
|
|
148
|
+
config: Object.freeze({ ...o.config }) as Ctx['config'],
|
|
110
149
|
sh,
|
|
111
150
|
exec,
|
|
112
|
-
readText: async (p) => readFileSync(resolve(p),
|
|
113
|
-
readJson: async <T>(p: string) => JSON.parse(readFileSync(resolve(p),
|
|
151
|
+
readText: async (p) => readFileSync(resolve(p), 'utf8'),
|
|
152
|
+
readJson: async <T>(p: string) => JSON.parse(readFileSync(resolve(p), 'utf8')) as T,
|
|
114
153
|
files: { fetch: createFetch(filesDeps) },
|
|
115
154
|
output: createOutput(filesDeps),
|
|
116
155
|
http: createHttp(o.signal),
|
|
@@ -121,10 +160,12 @@ export function createCtx(o: CtxOptions): Ctx {
|
|
|
121
160
|
tmpDir: o.paths.tmp,
|
|
122
161
|
skip: skipped,
|
|
123
162
|
inputUrl: (step, input) => {
|
|
124
|
-
const name = typeof step ===
|
|
163
|
+
const name = typeof step === 'string' ? step : step.name;
|
|
125
164
|
const url = o.callbackUrls?.[`${name}/${input}`];
|
|
126
165
|
if (!url)
|
|
127
|
-
throw new Error(
|
|
166
|
+
throw new Error(
|
|
167
|
+
`no callback URL for input '${input}' of step '${name}': it must list callback() in from, and the instance needs a public base URL`,
|
|
168
|
+
);
|
|
128
169
|
return url;
|
|
129
170
|
},
|
|
130
171
|
};
|