@olivervea/pipelines 2.0.0-beta.17 → 2.0.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +2 -0
  2. package/manifest.schema.json +22 -0
  3. package/package.json +4 -2
  4. package/src/assets.d.ts +2 -2
  5. package/src/build/build.ts +111 -43
  6. package/src/build/cli.ts +28 -21
  7. package/src/build/elf.ts +8 -7
  8. package/src/build/hashes.ts +61 -28
  9. package/src/build/project.ts +69 -30
  10. package/src/builder/pipeline.ts +288 -102
  11. package/src/builder/refs.ts +12 -6
  12. package/src/core/brands.ts +6 -6
  13. package/src/core/config-refs.ts +46 -20
  14. package/src/core/config.ts +110 -33
  15. package/src/core/external.ts +30 -12
  16. package/src/core/handlers.ts +54 -14
  17. package/src/core/models.ts +29 -14
  18. package/src/core/origin.ts +7 -4
  19. package/src/core/secrets.ts +2 -2
  20. package/src/core/steps.ts +39 -23
  21. package/src/core/types.ts +5 -1
  22. package/src/core/webhooks.ts +21 -7
  23. package/src/ctx/files.ts +111 -40
  24. package/src/ctx/index.ts +75 -34
  25. package/src/ctx/sh.ts +104 -44
  26. package/src/ctx/types.ts +4 -4
  27. package/src/describe/describe.ts +558 -194
  28. package/src/describe/manifest-schema.ts +217 -79
  29. package/src/describe/mapper-source.ts +249 -104
  30. package/src/describe/record.ts +67 -37
  31. package/src/dev/cli.ts +64 -34
  32. package/src/harness/env.ts +33 -27
  33. package/src/harness/execute.ts +55 -29
  34. package/src/harness/index.ts +21 -8
  35. package/src/harness/main.ts +197 -74
  36. package/src/harness/result.ts +38 -22
  37. package/src/harness/validate.ts +62 -19
  38. package/src/index.ts +123 -14
  39. package/src/library/checks.ts +11 -6
  40. package/src/library/compose-deploy.ts +40 -23
  41. package/src/library/github-source.ts +69 -39
  42. package/src/library/helm-deploy-ssh.ts +213 -76
  43. package/src/library/helm-deploy.ts +263 -85
  44. package/src/library/http-health.ts +7 -5
  45. package/src/library/image-build.ts +293 -117
  46. package/src/library/index.ts +47 -15
  47. package/src/library/models.ts +17 -9
  48. package/src/library/ssh.ts +31 -14
  49. package/src/testing/index.ts +63 -24
  50. package/src/util/canonical.ts +7 -4
  51. package/src/util/handler-hash.ts +15 -5
  52. package/src/util/hash.ts +5 -5
  53. package/src/util/tar.ts +74 -37
@@ -1,20 +1,23 @@
1
- import { ORIGIN } from "./brands.ts";
1
+ import { ORIGIN } from './brands.ts';
2
2
 
3
3
  /**
4
4
  * The file that called the function calling this one, from the stack. A tail call drops its caller's
5
5
  * frame, so this can name a module further up; the build only uses it to choose what to hash.
6
6
  */
7
7
  export function callerFile(): string | undefined {
8
- const files = (new Error().stack ?? "").split("\n").slice(1).map((f) => f.match(FRAME)?.[1]);
8
+ const files = (new Error().stack ?? '')
9
+ .split('\n')
10
+ .slice(1)
11
+ .map((f) => f.match(FRAME)?.[1]);
9
12
  const own = files[0];
10
13
  const callee = files[1];
11
14
  const file = files.slice(2).find((f) => f && f !== own && f !== callee) ?? undefined;
12
- return file?.startsWith("file://") ? new URL(file).pathname : file;
15
+ return file?.startsWith('file://') ? new URL(file).pathname : file;
13
16
  }
14
17
  const FRAME = /\(?((?:file:\/\/)?\/[^()]*?):\d+(?::\d+)?\)?\s*$/;
15
18
 
16
19
  /** The modules that defined and implemented a step (where `step` and `implement` were called). */
17
20
  export function originsOf(v: unknown): string[] {
18
21
  const o = (v as Record<symbol, unknown>)[ORIGIN];
19
- return Array.isArray(o) ? o.filter((x): x is string => typeof x === "string") : [];
22
+ return Array.isArray(o) ? o.filter((x): x is string => typeof x === 'string') : [];
20
23
  }
@@ -1,6 +1,6 @@
1
1
  /** A named pipeline secret. Steps (and poll triggers) declare the secrets they use; only those are mounted. */
2
2
  export interface Secret<N extends string = string> {
3
- readonly kind: "secret";
3
+ readonly kind: 'secret';
4
4
  readonly name: N;
5
5
  readonly description: string;
6
6
  }
@@ -8,6 +8,6 @@ export interface Secret<N extends string = string> {
8
8
  export const SECRET_NAME = /^[A-Z_][A-Z0-9_]*$/;
9
9
 
10
10
  export function secret<const N extends string>(name: N, description: string): Secret<N> {
11
- return Object.freeze({ kind: "secret" as const, name, description });
11
+ return Object.freeze({ kind: 'secret' as const, name, description });
12
12
  }
13
13
  export type DeclaredSecrets<N extends string> = { readonly [K in N]: string };
package/src/core/steps.ts CHANGED
@@ -1,12 +1,12 @@
1
- import type { z } from "zod";
2
- import { ORIGIN, STEP, type StepBrand } from "./brands.ts";
3
- import { callerFile } from "./origin.ts";
4
- import type { AnyModel, Infer, Optional } from "./models.ts";
5
- import type { External } from "./external.ts";
6
- import type { AnyConfigEntry } from "./config.ts";
7
- import { configRefsIn } from "./config-refs.ts";
8
- import type { Secret } from "./secrets.ts";
9
- import type { Simplify } from "./types.ts";
1
+ import type { z } from 'zod';
2
+ import { ORIGIN, STEP, type StepBrand } from './brands.ts';
3
+ import { callerFile } from './origin.ts';
4
+ import type { AnyModel, Infer, Optional } from './models.ts';
5
+ import type { External } from './external.ts';
6
+ import type { AnyConfigEntry } from './config.ts';
7
+ import { configRefsIn } from './config-refs.ts';
8
+ import type { Secret } from './secrets.ts';
9
+ import type { Simplify } from './types.ts';
10
10
 
11
11
  /**
12
12
  * A step input entry: a model (matchable by model), `optional(model)`, a plain zod schema (mapper required),
@@ -16,11 +16,15 @@ export type InputEntry = AnyModel | Optional | External | z.ZodType;
16
16
  export type InputSpec = { readonly [key: string]: InputEntry };
17
17
 
18
18
  export type InferEntry<E> =
19
- E extends Optional<infer M> ? Infer<M> | undefined
20
- : E extends External<infer M> ? Infer<M>
21
- : E extends AnyModel ? Infer<E>
22
- : E extends z.ZodType ? z.output<E>
23
- : never;
19
+ E extends Optional<infer M>
20
+ ? Infer<M> | undefined
21
+ : E extends External<infer M>
22
+ ? Infer<M>
23
+ : E extends AnyModel
24
+ ? Infer<E>
25
+ : E extends z.ZodType
26
+ ? z.output<E>
27
+ : never;
24
28
  /** What a handler receives: the output view of every input entry. */
25
29
  export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
26
30
 
@@ -62,16 +66,25 @@ export type AnyStep = Step<string, any, AnyModel | null, any, any>;
62
66
  export type SecretsOf<S> = S extends Step<any, any, any, infer Sec, any> ? Sec : never;
63
67
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
64
68
  export type ConfigOf<S> = S extends Step<any, any, any, any, infer C> ? C : never;
65
- export type OutputValue<S extends AnyStep> = S["output"] extends AnyModel ? Infer<S["output"]> : void;
69
+ export type OutputValue<S extends AnyStep> = S['output'] extends AnyModel
70
+ ? Infer<S['output']>
71
+ : void;
66
72
 
67
73
  /** Step names must be literals (identity, duplicate detection and mapper keys depend on it). */
68
- export type LiteralName<N extends string> = N & (string extends N ? "❌ step name must be a string literal" : unknown);
74
+ export type LiteralName<N extends string> = N &
75
+ (string extends N ? '❌ step name must be a string literal' : unknown);
69
76
 
70
- export { STEP_NAME } from "./models.ts";
77
+ export { STEP_NAME } from './models.ts';
71
78
  /** Names that cannot be steps: `input` is the pipeline input's root in mappings. */
72
- export const RESERVED_STEP_NAMES: readonly string[] = ["input"];
79
+ export const RESERVED_STEP_NAMES: readonly string[] = ['input'];
73
80
 
74
- export interface StepDef<N extends string, I extends InputSpec, O extends AnyModel | null, Sec extends readonly Secret[], Cfg extends readonly AnyConfigEntry[] = []> {
81
+ export interface StepDef<
82
+ N extends string,
83
+ I extends InputSpec,
84
+ O extends AnyModel | null,
85
+ Sec extends readonly Secret[],
86
+ Cfg extends readonly AnyConfigEntry[] = [],
87
+ > {
75
88
  name: LiteralName<N>;
76
89
  /** What the step does, shown in the UI. Required; may be empty. */
77
90
  description: string;
@@ -102,11 +115,12 @@ export interface StepDef<N extends string, I extends InputSpec, O extends AnyMod
102
115
 
103
116
  function makeStep(def: Record<string, unknown>, origin: string | undefined): AnyStep {
104
117
  const config = new Map<string, AnyConfigEntry>();
105
- for (const e of (def.config as readonly AnyConfigEntry[] | undefined) ?? []) if (!config.has(e.name)) config.set(e.name, e);
118
+ for (const e of (def.config as readonly AnyConfigEntry[] | undefined) ?? [])
119
+ if (!config.has(e.name)) config.set(e.name, e);
106
120
  configRefsIn(def.settings, config);
107
121
  const s = {
108
122
  [STEP]: true,
109
- kind: def.kind ?? "custom",
123
+ kind: def.kind ?? 'custom',
110
124
  name: def.name,
111
125
  description: def.description,
112
126
  image: def.image,
@@ -124,7 +138,9 @@ function makeStep(def: Record<string, unknown>, origin: string | undefined): Any
124
138
  }
125
139
 
126
140
  export function isStep(v: unknown): v is AnyStep {
127
- return typeof v === "object" && v !== null && (v as Record<symbol, unknown>)[STEP] === true;
141
+ return (
142
+ typeof v === 'object' && v !== null && (v as Record<symbol, unknown>)[STEP] === true
143
+ );
128
144
  }
129
145
 
130
146
  /**
@@ -137,6 +153,6 @@ export function step<
137
153
  O extends AnyModel | null = null,
138
154
  const Sec extends readonly Secret[] = [],
139
155
  const Cfg extends readonly AnyConfigEntry[] = [],
140
- >(def: StepDef<N, I, O, Sec, Cfg>): Step<N, I, O, Sec[number]["name"], Cfg[number]> {
156
+ >(def: StepDef<N, I, O, Sec, Cfg>): Step<N, I, O, Sec[number]['name'], Cfg[number]> {
141
157
  return makeStep(def as never, callerFile()) as never;
142
158
  }
package/src/core/types.ts CHANGED
@@ -1,6 +1,10 @@
1
1
  export type Primitive = string | number | boolean | bigint | null | undefined;
2
2
  export type Simplify<T> = { [K in keyof T]: T[K] } & {};
3
3
  export type Awaitable<T> = T | Promise<T>;
4
- export type IsUnion<T, U = T> = T extends unknown ? ([U] extends [T] ? false : true) : never;
4
+ export type IsUnion<T, U = T> = T extends unknown
5
+ ? [U] extends [T]
6
+ ? false
7
+ : true
8
+ : never;
5
9
  /** `Ok` when there is no error, else the error (a readable string-literal type). */
6
10
  export type OrError<Err, Ok> = [Err] extends [never] ? Ok : Err;
@@ -1,23 +1,37 @@
1
- import type { Secret } from "./secrets.ts";
1
+ import type { Secret } from './secrets.ts';
2
2
 
3
3
  /** How a webhook trigger authenticates deliveries. Build one with `hmacSha256`, `githubSignature` or `bearer`. */
4
4
  export type WebhookVerify =
5
- | { readonly scheme: "hmac-sha256"; readonly secret: Secret; readonly header: string; readonly prefix: string }
6
- | { readonly scheme: "bearer"; readonly secret: Secret };
5
+ | {
6
+ readonly scheme: 'hmac-sha256';
7
+ readonly secret: Secret;
8
+ readonly header: string;
9
+ readonly prefix: string;
10
+ }
11
+ | { readonly scheme: 'bearer'; readonly secret: Secret };
7
12
 
8
13
  /** HMAC-SHA256 of the raw body keyed by `secret`, hex, in `header` after `prefix`. */
9
- export function hmacSha256(o: { secret: Secret; header: string; prefix?: string }): WebhookVerify {
10
- return Object.freeze({ scheme: "hmac-sha256", secret: o.secret, header: o.header, prefix: o.prefix ?? "" });
14
+ export function hmacSha256(o: {
15
+ secret: Secret;
16
+ header: string;
17
+ prefix?: string;
18
+ }): WebhookVerify {
19
+ return Object.freeze({
20
+ scheme: 'hmac-sha256',
21
+ secret: o.secret,
22
+ header: o.header,
23
+ prefix: o.prefix ?? '',
24
+ });
11
25
  }
12
26
 
13
27
  /** GitHub's delivery signature: `X-Hub-Signature-256: sha256=<hex>`. */
14
28
  export function githubSignature(secret: Secret): WebhookVerify {
15
- return hmacSha256({ secret, header: "X-Hub-Signature-256", prefix: "sha256=" });
29
+ return hmacSha256({ secret, header: 'X-Hub-Signature-256', prefix: 'sha256=' });
16
30
  }
17
31
 
18
32
  /** `Authorization: Bearer <secret>`. */
19
33
  export function bearer(secret: Secret): WebhookVerify {
20
- return Object.freeze({ scheme: "bearer", secret });
34
+ return Object.freeze({ scheme: 'bearer', secret });
21
35
  }
22
36
 
23
37
  /**
package/src/ctx/files.ts CHANGED
@@ -4,19 +4,31 @@
4
4
  // `path` minus the trailing "/" plus ".tar" for directories (a deterministic tar). The step writes
5
5
  // objects to `$OLVE_DIR/work/files/<object path>`; the service stores them under
6
6
  // `runs/<p>/<run>/<step>/<attempt>/files/<object path>`.
7
- import { copyFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
8
- import { basename, dirname, join, resolve } from "node:path";
9
- import { fileRefSchema, FILE_PATH, type FileRef } from "../core/models.ts";
10
- import { ENV } from "../harness/env.ts";
11
- import { sha256File } from "../util/hash.ts";
12
- import { dirEntries, extractTar, writeTar } from "../util/tar.ts";
13
- import type { OutputFileOptions, RunInfo } from "./types.ts";
7
+ import {
8
+ copyFileSync,
9
+ existsSync,
10
+ mkdirSync,
11
+ readFileSync,
12
+ renameSync,
13
+ rmSync,
14
+ statSync,
15
+ writeFileSync,
16
+ } from 'node:fs';
17
+ import { basename, dirname, join, resolve } from 'node:path';
18
+ import { fileRefSchema, FILE_PATH, type FileRef } from '../core/models.ts';
19
+ import { ENV } from '../harness/env.ts';
20
+ import { sha256File } from '../util/hash.ts';
21
+ import { dirEntries, extractTar, writeTar } from '../util/tar.ts';
22
+ import type { OutputFileOptions, RunInfo } from './types.ts';
14
23
 
15
- export const objectPath = (refPath: string): string => (refPath.endsWith("/") ? refPath.slice(0, -1) + ".tar" : refPath);
16
- export const fileKey = (ref: Pick<FileRef, "run" | "step" | "attempt" | "path">): string => `${ref.run}/${ref.step}/${ref.attempt}/${ref.path}`;
24
+ export const objectPath = (refPath: string): string =>
25
+ refPath.endsWith('/') ? refPath.slice(0, -1) + '.tar' : refPath;
26
+ export const fileKey = (
27
+ ref: Pick<FileRef, 'run' | 'step' | 'attempt' | 'path'>,
28
+ ): string => `${ref.run}/${ref.step}/${ref.attempt}/${ref.path}`;
17
29
 
18
30
  /** Strips everything from `?` on (pre-signed credentials). */
19
- export const redactUrl = (url: string): string => url.replace(/[?#].*$/, "?…");
31
+ export const redactUrl = (url: string): string => url.replace(/[?#].*$/, '?…');
20
32
 
21
33
  export interface FilesDeps {
22
34
  run: RunInfo;
@@ -28,42 +40,54 @@ export interface FilesDeps {
28
40
  }
29
41
 
30
42
  function checkName(name: string, what: string): void {
31
- if (!FILE_PATH.test(name) || name.endsWith("/")) throw new Error(`${what}: '${name}' is not a valid relative file name (no leading '/', no '..', no trailing '/')`);
43
+ if (!FILE_PATH.test(name) || name.endsWith('/'))
44
+ throw new Error(
45
+ `${what}: '${name}' is not a valid relative file name (no leading '/', no '..', no trailing '/')`,
46
+ );
32
47
  }
33
48
 
34
49
  export function createOutput(deps: FilesDeps) {
35
50
  const used = new Set<string>();
36
51
  const claim = (obj: string) => {
37
- if (used.has(obj) || existsSync(join(deps.workFiles, obj))) throw new Error(`output file '${obj}' already exists`);
52
+ if (used.has(obj) || existsSync(join(deps.workFiles, obj)))
53
+ throw new Error(`output file '${obj}' already exists`);
38
54
  used.add(obj);
39
55
  const dest = join(deps.workFiles, obj);
40
56
  mkdirSync(dirname(dest), { recursive: true });
41
57
  return dest;
42
58
  };
43
59
  const ref = (path: string, dest: string): FileRef =>
44
- ({ run: deps.run.number, step: deps.run.step, attempt: deps.run.attempt, path, sha256: sha256File(dest) }) as FileRef;
60
+ ({
61
+ run: deps.run.number,
62
+ step: deps.run.step,
63
+ attempt: deps.run.attempt,
64
+ path,
65
+ sha256: sha256File(dest),
66
+ }) as FileRef;
45
67
 
46
68
  return {
47
69
  async file(path: string, opts: OutputFileOptions = {}): Promise<FileRef> {
48
70
  const src = resolve(path);
49
- if (!statSync(src).isFile()) throw new Error(`ctx.output.file: '${path}' is not a file`);
71
+ if (!statSync(src).isFile())
72
+ throw new Error(`ctx.output.file: '${path}' is not a file`);
50
73
  const name = opts.name ?? basename(src);
51
- checkName(name, "ctx.output.file");
74
+ checkName(name, 'ctx.output.file');
52
75
  const dest = claim(name);
53
76
  copyFileSync(src, dest);
54
77
  return ref(name, dest);
55
78
  },
56
79
  async dir(path: string, opts: OutputFileOptions = {}): Promise<FileRef> {
57
80
  const src = resolve(path);
58
- if (!statSync(src).isDirectory()) throw new Error(`ctx.output.dir: '${path}' is not a directory`);
81
+ if (!statSync(src).isDirectory())
82
+ throw new Error(`ctx.output.dir: '${path}' is not a directory`);
59
83
  const name = opts.name ?? basename(src);
60
- checkName(name, "ctx.output.dir");
61
- const dest = claim(objectPath(name + "/"));
84
+ checkName(name, 'ctx.output.dir');
85
+ const dest = claim(objectPath(name + '/'));
62
86
  writeTar(dest, dirEntries(src));
63
- return ref(name + "/", dest);
87
+ return ref(name + '/', dest);
64
88
  },
65
89
  async write(name: string, content: string | Uint8Array): Promise<FileRef> {
66
- checkName(name, "ctx.output.write");
90
+ checkName(name, 'ctx.output.write');
67
91
  const dest = claim(name);
68
92
  writeFileSync(dest, content);
69
93
  return ref(name, dest);
@@ -72,20 +96,33 @@ export function createOutput(deps: FilesDeps) {
72
96
  }
73
97
 
74
98
  function urlMap(root: string): Record<string, string> {
75
- const raw = process.env[ENV.fileUrls] ?? (existsSync(join(root, "files.json")) ? readFileSync(join(root, "files.json"), "utf8") : undefined);
99
+ const raw =
100
+ process.env[ENV.fileUrls] ??
101
+ (existsSync(join(root, 'files.json'))
102
+ ? readFileSync(join(root, 'files.json'), 'utf8')
103
+ : undefined);
76
104
  if (!raw) return {};
77
105
  try {
78
106
  const m = JSON.parse(raw) as unknown;
79
- if (typeof m !== "object" || m === null || Array.isArray(m)) throw new Error("not an object");
107
+ if (typeof m !== 'object' || m === null || Array.isArray(m))
108
+ throw new Error('not an object');
80
109
  return m as Record<string, string>;
81
110
  } catch (e) {
82
- throw new Error(`${ENV.fileUrls} / files.json is not a JSON object of file key → URL (${(e as Error).message})`);
111
+ throw new Error(
112
+ `${ENV.fileUrls} / files.json is not a JSON object of file key → URL (${(e as Error).message})`,
113
+ );
83
114
  }
84
115
  }
85
116
 
86
117
  const TRANSIENT = (status: number) => status >= 500 || status === 408 || status === 429;
87
118
 
88
- async function download(url: string, dest: string, headers: Record<string, string>, signal: AbortSignal, log: FilesDeps["log"]): Promise<void> {
119
+ async function download(
120
+ url: string,
121
+ dest: string,
122
+ headers: Record<string, string>,
123
+ signal: AbortSignal,
124
+ log: FilesDeps['log'],
125
+ ): Promise<void> {
89
126
  const attempts = 4;
90
127
  for (let i = 1; ; i++) {
91
128
  let status = 0;
@@ -93,14 +130,19 @@ async function download(url: string, dest: string, headers: Record<string, strin
93
130
  const res = await fetch(url, { headers, signal });
94
131
  status = res.status;
95
132
  if (!res.ok) throw new Error(`HTTP ${res.status}`);
96
- await Bun.write(dest + ".partial", res);
97
- renameSync(dest + ".partial", dest);
133
+ await Bun.write(dest + '.partial', res);
134
+ renameSync(dest + '.partial', dest);
98
135
  return;
99
136
  } catch (e) {
100
- rmSync(dest + ".partial", { force: true });
137
+ rmSync(dest + '.partial', { force: true });
101
138
  const transient = status === 0 || TRANSIENT(status);
102
- if (signal.aborted || !transient || i >= attempts) throw new Error(`download of ${redactUrl(url)} failed: ${(e as Error).message}`);
103
- log("download failed, retrying", { url: redactUrl(url), attempt: i, error: (e as Error).message });
139
+ if (signal.aborted || !transient || i >= attempts)
140
+ throw new Error(`download of ${redactUrl(url)} failed: ${(e as Error).message}`);
141
+ log('download failed, retrying', {
142
+ url: redactUrl(url),
143
+ attempt: i,
144
+ error: (e as Error).message,
145
+ });
104
146
  await Bun.sleep(1000 * 2 ** (i - 1));
105
147
  }
106
148
  }
@@ -110,29 +152,56 @@ export function createFetch(deps: FilesDeps) {
110
152
  const cache = new Map<string, string>();
111
153
  return async (input: FileRef): Promise<string> => {
112
154
  const parsed = fileRefSchema.safeParse(input);
113
- if (!parsed.success) throw new Error(`ctx.files.fetch: not a valid file reference: ${JSON.stringify(input)}`);
155
+ if (!parsed.success)
156
+ throw new Error(
157
+ `ctx.files.fetch: not a valid file reference: ${JSON.stringify(input)}`,
158
+ );
114
159
  const ref = parsed.data;
115
160
  const key = fileKey(ref);
116
161
  const hit = cache.get(key);
117
162
  if (hit) return hit;
118
163
 
119
164
  const obj = objectPath(ref.path);
120
- const base = join(deps.tmpDir, "files", String(ref.run), ref.step, String(ref.attempt));
121
- const objFile = join(base, ".objects", obj);
165
+ const base = join(
166
+ deps.tmpDir,
167
+ 'files',
168
+ String(ref.run),
169
+ ref.step,
170
+ String(ref.attempt),
171
+ );
172
+ const objFile = join(base, '.objects', obj);
122
173
  mkdirSync(dirname(objFile), { recursive: true });
123
174
 
124
- const own = ref.run === deps.run.number && ref.step === deps.run.step && ref.attempt === deps.run.attempt;
175
+ const own =
176
+ ref.run === deps.run.number &&
177
+ ref.step === deps.run.step &&
178
+ ref.attempt === deps.run.attempt;
125
179
  const urls = urlMap(deps.root);
126
180
  const localDir = process.env[ENV.filesDir];
127
181
  const baseUrl = process.env[ENV.filesBaseUrl];
128
- if (own && existsSync(join(deps.workFiles, obj))) copyFileSync(join(deps.workFiles, obj), objFile);
182
+ if (own && existsSync(join(deps.workFiles, obj)))
183
+ copyFileSync(join(deps.workFiles, obj), objFile);
129
184
  else if (urls[key]) await download(urls[key], objFile, {}, deps.signal, deps.log);
130
- else if (localDir && existsSync(join(localDir, String(ref.run), ref.step, String(ref.attempt), "files", obj)))
131
- copyFileSync(join(localDir, String(ref.run), ref.step, String(ref.attempt), "files", obj), objFile);
185
+ else if (
186
+ localDir &&
187
+ existsSync(
188
+ join(localDir, String(ref.run), ref.step, String(ref.attempt), 'files', obj),
189
+ )
190
+ )
191
+ copyFileSync(
192
+ join(localDir, String(ref.run), ref.step, String(ref.attempt), 'files', obj),
193
+ objFile,
194
+ );
132
195
  else if (baseUrl) {
133
196
  const token = process.env[ENV.filesToken];
134
- const url = `${baseUrl.replace(/\/+$/, "")}/${ref.run}/${encodeURIComponent(ref.step)}/${ref.attempt}/${obj.split("/").map(encodeURIComponent).join("/")}`;
135
- await download(url, objFile, token ? { authorization: `Bearer ${token}` } : {}, deps.signal, deps.log);
197
+ const url = `${baseUrl.replace(/\/+$/, '')}/${ref.run}/${encodeURIComponent(ref.step)}/${ref.attempt}/${obj.split('/').map(encodeURIComponent).join('/')}`;
198
+ await download(
199
+ url,
200
+ objFile,
201
+ token ? { authorization: `Bearer ${token}` } : {},
202
+ deps.signal,
203
+ deps.log,
204
+ );
136
205
  } else {
137
206
  throw new Error(
138
207
  `ctx.files.fetch: no source for ${key} (set ${ENV.fileUrls} / $OLVE_DIR/files.json, ${ENV.filesBaseUrl} or ${ENV.filesDir})`,
@@ -142,10 +211,12 @@ export function createFetch(deps: FilesDeps) {
142
211
  const actual = sha256File(objFile);
143
212
  if (actual !== ref.sha256) {
144
213
  rmSync(objFile, { force: true });
145
- throw new Error(`ctx.files.fetch: ${key} has sha256 ${actual}, expected ${ref.sha256}`);
214
+ throw new Error(
215
+ `ctx.files.fetch: ${key} has sha256 ${actual}, expected ${ref.sha256}`,
216
+ );
146
217
  }
147
218
  let result = objFile;
148
- if (ref.path.endsWith("/")) {
219
+ if (ref.path.endsWith('/')) {
149
220
  result = join(base, ref.path.slice(0, -1));
150
221
  rmSync(result, { recursive: true, force: true });
151
222
  mkdirSync(result, { recursive: true });
package/src/ctx/index.ts CHANGED
@@ -1,10 +1,10 @@
1
- import { mkdirSync, readFileSync } from "node:fs";
2
- import { resolve } from "node:path";
3
- import { skipped } from "../core/handlers.ts";
4
- import type { Secret } from "../core/secrets.ts";
5
- import { createFetch, createOutput, redactUrl } from "./files.ts";
6
- import { createSh } from "./sh.ts";
7
- import type { Ctx, Http, RetryOptions, RunInfo } from "./types.ts";
1
+ import { mkdirSync, readFileSync } from 'node:fs';
2
+ import { resolve } from 'node:path';
3
+ import { skipped } from '../core/handlers.ts';
4
+ import type { Secret } from '../core/secrets.ts';
5
+ import { createFetch, createOutput, redactUrl } from './files.ts';
6
+ import { createSh } from './sh.ts';
7
+ import type { Ctx, Http, RetryOptions, RunInfo } from './types.ts';
8
8
 
9
9
  export interface CtxOptions {
10
10
  run: RunInfo;
@@ -23,18 +23,27 @@ export interface CtxOptions {
23
23
  }
24
24
 
25
25
  export function formatLog(message: string, fields?: Record<string, unknown>): string {
26
- const parts = Object.entries(fields ?? {}).map(([k, v]) => `${k}=${typeof v === "string" && !/\s|"/.test(v) ? v : JSON.stringify(v)}`);
27
- return parts.length ? `${message} ${parts.join(" ")}` : message;
26
+ const parts = Object.entries(fields ?? {}).map(
27
+ ([k, v]) =>
28
+ `${k}=${typeof v === 'string' && !/\s|"/.test(v) ? v : JSON.stringify(v)}`,
29
+ );
30
+ return parts.length ? `${message} ${parts.join(' ')}` : message;
28
31
  }
29
32
 
30
- export function createSecrets(declared: readonly Secret[], values?: Readonly<Record<string, string>>): Record<string, string> {
33
+ export function createSecrets(
34
+ declared: readonly Secret[],
35
+ values?: Readonly<Record<string, string>>,
36
+ ): Record<string, string> {
31
37
  const obj: Record<string, string> = {};
32
38
  for (const s of declared) {
33
39
  Object.defineProperty(obj, s.name, {
34
40
  enumerable: true,
35
41
  get() {
36
42
  const v = values?.[s.name] ?? process.env[s.name];
37
- if (v === undefined) throw new Error(`secret ${s.name} is not set (the harness reads it from the environment variable ${s.name})`);
43
+ if (v === undefined)
44
+ throw new Error(
45
+ `secret ${s.name} is not set (the harness reads it from the environment variable ${s.name})`,
46
+ );
38
47
  return v;
39
48
  },
40
49
  });
@@ -43,52 +52,74 @@ export function createSecrets(declared: readonly Secret[], values?: Readonly<Rec
43
52
  }
44
53
 
45
54
  async function httpError(res: Response, url: string): Promise<Error> {
46
- let body = "";
55
+ let body = '';
47
56
  try {
48
57
  body = (await res.text()).slice(0, 500);
49
58
  } catch {
50
59
  /* ignore */
51
60
  }
52
- return new Error(`HTTP ${res.status} ${res.statusText} from ${redactUrl(url)}${body ? `: ${body}` : ""}`);
61
+ return new Error(
62
+ `HTTP ${res.status} ${res.statusText} from ${redactUrl(url)}${body ? `: ${body}` : ''}`,
63
+ );
53
64
  }
54
65
 
55
66
  export function createHttp(signal: AbortSignal): Http {
56
- const f = (url: string, init?: RequestInit) => fetch(url, { ...init, signal: init?.signal ?? signal });
67
+ const f = (url: string, init?: RequestInit) =>
68
+ fetch(url, { ...init, signal: init?.signal ?? signal });
57
69
  return {
58
70
  fetch: f,
59
- async getJson<T>(url: string, init?: { headers?: Record<string, string> }): Promise<T> {
60
- const res = await f(url, { headers: { accept: "application/json", ...init?.headers } });
71
+ async getJson<T>(
72
+ url: string,
73
+ init?: { headers?: Record<string, string> },
74
+ ): Promise<T> {
75
+ const res = await f(url, {
76
+ headers: { accept: 'application/json', ...init?.headers },
77
+ });
61
78
  if (!res.ok) throw await httpError(res, url);
62
79
  return (await res.json()) as T;
63
80
  },
64
- async expectOk(url: string, init?: { headers?: Record<string, string>; method?: string }): Promise<Response> {
65
- const res = await f(url, { method: init?.method ?? "GET", headers: init?.headers });
81
+ async expectOk(
82
+ url: string,
83
+ init?: { headers?: Record<string, string>; method?: string },
84
+ ): Promise<Response> {
85
+ const res = await f(url, { method: init?.method ?? 'GET', headers: init?.headers });
66
86
  if (!res.ok) throw await httpError(res, url);
67
87
  return res;
68
88
  },
69
89
  };
70
90
  }
71
91
 
72
- export function createRetry(signal: AbortSignal, log: (m: string, f?: Record<string, unknown>) => void) {
73
- return async <T>(opts: RetryOptions, fn: (attempt: number) => Promise<T>): Promise<T> => {
74
- if (!(Number.isInteger(opts.attempts) && opts.attempts >= 1)) throw new Error("retry: attempts must be a positive integer");
92
+ export function createRetry(
93
+ signal: AbortSignal,
94
+ log: (m: string, f?: Record<string, unknown>) => void,
95
+ ) {
96
+ return async <T>(
97
+ opts: RetryOptions,
98
+ fn: (attempt: number) => Promise<T>,
99
+ ): Promise<T> => {
100
+ if (!(Number.isInteger(opts.attempts) && opts.attempts >= 1))
101
+ throw new Error('retry: attempts must be a positive integer');
75
102
  let delay = Math.max(0, opts.delaySeconds) * 1000;
76
103
  for (let attempt = 1; ; attempt++) {
77
104
  try {
78
105
  return await fn(attempt);
79
106
  } catch (e) {
80
107
  if (attempt >= opts.attempts || signal.aborted) throw e;
81
- log("retrying", { attempt, of: opts.attempts, error: (e as Error)?.message ?? String(e) });
108
+ log('retrying', {
109
+ attempt,
110
+ of: opts.attempts,
111
+ error: (e as Error)?.message ?? String(e),
112
+ });
82
113
  await new Promise<void>((res, rej) => {
83
114
  const t = setTimeout(() => {
84
- signal.removeEventListener("abort", onAbort);
115
+ signal.removeEventListener('abort', onAbort);
85
116
  res();
86
117
  }, delay);
87
118
  const onAbort = () => {
88
119
  clearTimeout(t);
89
- rej(new Error("retry cancelled: the step was cancelled"));
120
+ rej(new Error('retry cancelled: the step was cancelled'));
90
121
  };
91
- signal.addEventListener("abort", onAbort, { once: true });
122
+ signal.addEventListener('abort', onAbort, { once: true });
92
123
  });
93
124
  delay *= opts.backoff ?? 1;
94
125
  }
@@ -100,17 +131,25 @@ export function createRetry(signal: AbortSignal, log: (m: string, f?: Record<str
100
131
  export function createCtx(o: CtxOptions): Ctx {
101
132
  mkdirSync(o.paths.tmp, { recursive: true });
102
133
  mkdirSync(o.paths.workFiles, { recursive: true });
103
- const sink = o.logSink ?? ((line: string) => process.stderr.write(line + "\n"));
104
- const log = (message: string, fields?: Record<string, unknown>) => sink(formatLog(message, fields));
134
+ const sink = o.logSink ?? ((line: string) => process.stderr.write(line + '\n'));
135
+ const log = (message: string, fields?: Record<string, unknown>) =>
136
+ sink(formatLog(message, fields));
105
137
  const { sh, exec } = createSh({ tmpDir: o.paths.tmp, signal: o.signal });
106
- const filesDeps = { run: o.run, workFiles: o.paths.workFiles, tmpDir: o.paths.tmp, root: o.paths.root, signal: o.signal, log };
138
+ const filesDeps = {
139
+ run: o.run,
140
+ workFiles: o.paths.workFiles,
141
+ tmpDir: o.paths.tmp,
142
+ root: o.paths.root,
143
+ signal: o.signal,
144
+ log,
145
+ };
107
146
  return {
108
- secrets: createSecrets(o.secrets, o.secretValues) as Ctx["secrets"],
109
- config: Object.freeze({ ...o.config }) as Ctx["config"],
147
+ secrets: createSecrets(o.secrets, o.secretValues) as Ctx['secrets'],
148
+ config: Object.freeze({ ...o.config }) as Ctx['config'],
110
149
  sh,
111
150
  exec,
112
- readText: async (p) => readFileSync(resolve(p), "utf8"),
113
- readJson: async <T>(p: string) => JSON.parse(readFileSync(resolve(p), "utf8")) as T,
151
+ readText: async (p) => readFileSync(resolve(p), 'utf8'),
152
+ readJson: async <T>(p: string) => JSON.parse(readFileSync(resolve(p), 'utf8')) as T,
114
153
  files: { fetch: createFetch(filesDeps) },
115
154
  output: createOutput(filesDeps),
116
155
  http: createHttp(o.signal),
@@ -121,10 +160,12 @@ export function createCtx(o: CtxOptions): Ctx {
121
160
  tmpDir: o.paths.tmp,
122
161
  skip: skipped,
123
162
  inputUrl: (step, input) => {
124
- const name = typeof step === "string" ? step : step.name;
163
+ const name = typeof step === 'string' ? step : step.name;
125
164
  const url = o.callbackUrls?.[`${name}/${input}`];
126
165
  if (!url)
127
- throw new Error(`no callback URL for input '${input}' of step '${name}': it must list callback() in from, and the instance needs a public base URL`);
166
+ throw new Error(
167
+ `no callback URL for input '${input}' of step '${name}': it must list callback() in from, and the instance needs a public base URL`,
168
+ );
128
169
  return url;
129
170
  },
130
171
  };