@olivervea/pipelines 2.0.0-beta.2 → 2.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -26
- package/manifest.schema.json +1 -1
- package/package.json +1 -1
- package/src/build/build.ts +3 -3
- package/src/build/cli.ts +1 -1
- package/src/build/hashes.ts +1 -1
- package/src/build/project.ts +26 -5
- package/src/builder/pipeline.ts +22 -7
- package/src/ctx/sh.ts +2 -1
- package/src/describe/describe.ts +5 -5
- package/src/describe/manifest-schema.ts +1 -1
- package/src/dev/cli.ts +1 -1
- package/src/harness/index.ts +1 -1
- package/src/harness/main.ts +4 -4
- package/src/index.ts +2 -2
- package/src/library/checks.ts +23 -0
- package/src/library/helm-deploy-ssh.ts +107 -62
- package/src/library/image-build.ts +112 -16
- package/src/library/index.ts +4 -2
- package/src/library/models.ts +9 -0
- package/src/library/ssh.ts +45 -0
- package/src/testing/index.ts +1 -1
- package/src/util/handler-hash.ts +1 -1
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @olivervea/pipelines
|
|
2
2
|
|
|
3
|
-
The [Olve.Pipelines](https://github.com/OliverVea/Olve.Pipelines) v2 SDK
|
|
3
|
+
The [Olve.Pipelines](https://github.com/OliverVea/Olve.Pipelines) v2 SDK. It has four parts:
|
|
4
4
|
|
|
5
5
|
- the **authoring API**: models, steps, handlers and the typed pipeline builder;
|
|
6
6
|
- the **step harness**: the runtime compiled into each pipeline's single executable (`describe`, `run`, `map-trigger`);
|
|
@@ -12,14 +12,14 @@ The manifest format is in [`manifest.schema.json`](manifest.schema.json), genera
|
|
|
12
12
|
|
|
13
13
|
```
|
|
14
14
|
sdk/
|
|
15
|
-
src/index.ts authoring API (`@
|
|
15
|
+
src/index.ts authoring API (`@olivervea/pipelines`)
|
|
16
16
|
src/core/ models, secrets, steps, implement()
|
|
17
17
|
src/builder/ pipeline builder (types + runtime records), Ref<T>
|
|
18
18
|
src/library/ library steps (script, gitHubSource, imageBuild, helmDeployOverSsh), their exported helpers (httpHealthCheck) and SDK models
|
|
19
19
|
src/describe/ describe: manifest, checks, mapper recording + source analysis, manifest schema
|
|
20
|
-
src/harness/ harness CLI modes, env contract, envelope, result upload (`@
|
|
20
|
+
src/harness/ harness CLI modes, env contract, envelope, result upload (`@olivervea/pipelines/harness`)
|
|
21
21
|
src/ctx/ the ctx handed to handlers
|
|
22
|
-
src/testing/ testHandler (`@
|
|
22
|
+
src/testing/ testHandler (`@olivervea/pipelines/testing`)
|
|
23
23
|
src/build/ olve-build
|
|
24
24
|
src/dev/ olve-dev
|
|
25
25
|
examples/self-deploy/ fixture pipeline (the self-deploy shape) used by the tests
|
|
@@ -30,13 +30,12 @@ sdk/
|
|
|
30
30
|
|
|
31
31
|
## Install
|
|
32
32
|
|
|
33
|
-
The package is `@olivervea/pipelines` on npm;
|
|
34
|
-
in the pipeline directory's `package.json`:
|
|
33
|
+
The package is `@olivervea/pipelines` on npm; a pipeline directory's `package.json` depends on it by that name:
|
|
35
34
|
|
|
36
35
|
```json
|
|
37
36
|
{
|
|
38
37
|
"type": "module",
|
|
39
|
-
"dependencies": { "@
|
|
38
|
+
"dependencies": { "@olivervea/pipelines": "2.0.0-beta.3" },
|
|
40
39
|
"devDependencies": { "@types/bun": "1.4.3", "typescript": "7.0.2" }
|
|
41
40
|
}
|
|
42
41
|
```
|
|
@@ -47,11 +46,9 @@ bunx olve-dev describe .
|
|
|
47
46
|
bunx olve-build . --out .olve/out
|
|
48
47
|
```
|
|
49
48
|
|
|
50
|
-
- Keep the alias name: `olve-build` generates imports of `@olve/pipelines/harness`, so an unaliased
|
|
51
|
-
`@olivervea/pipelines` dependency does not build.
|
|
52
49
|
- The package ships TypeScript sources (Bun runs them directly; `harness.sdkHash` hashes them), so the
|
|
53
50
|
same version gives the same `sdkHash` and handler hashes from npm or from `file:../sdk`.
|
|
54
|
-
- Inside this repo, `.pipelines/` and `.pipelines-test/` use `"@
|
|
51
|
+
- Inside this repo, `.pipelines/` and `.pipelines-test/` use `"@olivervea/pipelines": "file:../sdk"`.
|
|
55
52
|
|
|
56
53
|
### Releases
|
|
57
54
|
|
|
@@ -67,7 +64,7 @@ A pipeline directory (the binding path, `.pipelines/` by default) looks like thi
|
|
|
67
64
|
```
|
|
68
65
|
models.ts data models (contract file)
|
|
69
66
|
pipeline.ts steps + `export default pipeline(…)` (contract file)
|
|
70
|
-
handlers
|
|
67
|
+
handlers/*.ts one per custom step: `export default implement(step, fn)` (pairs by the step, not the file name)
|
|
71
68
|
handlers/lib/… shared helpers (not handlers)
|
|
72
69
|
scripts/*.sh embedded with `import s from "../scripts/x.sh" with { type: "file" }`
|
|
73
70
|
```
|
|
@@ -77,7 +74,7 @@ Contract files hold contracts only. Describe evaluates them without loading hand
|
|
|
77
74
|
### Models, secrets, steps
|
|
78
75
|
|
|
79
76
|
```ts
|
|
80
|
-
import { model, z, fileRef, optional, secret, step, steps, Source } from "@
|
|
77
|
+
import { model, z, fileRef, optional, secret, step, steps, Source } from "@olivervea/pipelines";
|
|
81
78
|
|
|
82
79
|
export const BuildArtifact = model("BuildArtifact", z.object({
|
|
83
80
|
version: z.string(),
|
|
@@ -123,28 +120,38 @@ In the output JSON, an object of the form `{"$file": "<path>"}` or `{"$dir": "<p
|
|
|
123
120
|
|
|
124
121
|
### Library steps
|
|
125
122
|
|
|
126
|
-
Library steps carry their own implementation and image; they take typed config and use the SDK models `Source`, `BuiltImage` and `HelmRelease`. The set grows from real use: a step or an option exists because a pipeline uses it.
|
|
123
|
+
Library steps carry their own implementation and image; they take typed config and use the SDK models `Source`, `BuiltImage`, `PushedImage` and `HelmRelease`. The set grows from real use: a step or an option exists because a pipeline uses it.
|
|
127
124
|
|
|
128
125
|
| Step | Image | Input → output | Config |
|
|
129
126
|
|---|---|---|---|
|
|
130
127
|
| `steps.script` | yours | yours | `script` (embedded), `env` (see above) |
|
|
131
128
|
| `steps.gitHubSource` | `alpine:3.22` (work happens in the harness) | `{ repo: "owner/name", sha }` (plain schemas: needs a mapper) → `Source` | `token?: Secret` (sent as `Authorization: Bearer`; becomes the step's secret) |
|
|
132
|
-
| `steps.imageBuild` | `gcr.io/kaniko-project/executor:debug` | `{ source: Source }` → `BuiltImage { repository, tag, digest, tar }` | `repository` (image name; tag = run version, `-attempt<n>` on reruns), `cacheRepo?` (kaniko layer cache), `registryMap?` (kaniko `--registry-map`: registry → mirror prefix; default: the homelab pull-through caches, `{}` = off), `stallTimeoutSeconds?` (default 300, 0 = off), `retries?` (default 2) |
|
|
133
|
-
| `steps.helmDeployOverSsh` | `alpine:3.22` (+ `apk add openssh-client` at run time) | `{ source: Source,
|
|
129
|
+
| `steps.imageBuild` | `gcr.io/kaniko-project/executor:debug` | `{ source: Source }` → `BuiltImage { repository, tag, digest, tar }`; with `push`: `PushedImage { repository, tag, digest }` | `repository` (image name; tag = run version, `-attempt<n>` on reruns), `context?` (dir in the source tree; default the root), `dockerfile?` (relative to the context; default `Dockerfile`), `push?` (push to `repository` instead of a tarball), `contentTag?` (with `push`: tag `src-<context hash>`, skip when the registry has it), `cacheRepo?` (kaniko layer cache), `registryMap?` (kaniko `--registry-map`: registry → mirror prefix; default: the homelab pull-through caches, `{}` = off), `stallTimeoutSeconds?` (default 300, 0 = off), `retries?` (default 2) |
|
|
130
|
+
| `steps.helmDeployOverSsh` | `alpine:3.22` (+ `apk add openssh-client` at run time) | `{ source: Source, <key>: BuiltImage }` per `images` (`PushedImage` with `fromRegistry`) → `HelmRelease { release, namespace, image }` (`image`: the first image) | `host` (`user@host`), `sshKey: Secret`, `release`, `namespace`, `chart` (dir in the source), `values?` (chart-relative `-f`), `set?`, `images?` (input key → values prefix; default `{ image: "image" }`), `fromRegistry?`, `deployments?` (default `[release]`), `rolloutTimeoutSeconds?` (default 120), `healthUrl?` (URL or `{ url, attempts?, delaySeconds? }`) |
|
|
134
131
|
|
|
135
132
|
- `gitHubSource` downloads `https://api.github.com/repos/<repo>/tarball/<sha>` (retrying 5xx), strips the top-level directory and outputs it as `tree`.
|
|
136
|
-
- `imageBuild` copies the tree to `/kaniko/build-context` (kaniko wipes other directories between multi-stage builds), builds
|
|
133
|
+
- `imageBuild` copies the tree to `/kaniko/build-context` (kaniko wipes other directories between multi-stage builds), builds `--context=/kaniko/build-context[/<context>] --dockerfile=<that>/<dockerfile>` with `--no-push --tar-path --digest-file`, and outputs the tarball. A missing context or Dockerfile fails the step before kaniko starts. `repository` in the output is fully qualified the way a runtime names a loaded image (`x` → `docker.io/library/x`). A **stall watchdog** kills kaniko when it prints nothing for `stallTimeoutSeconds` (a base-image pull can stall on an idle connection at "Unpacking rootfs", and kaniko has no read timeout), failing the attempt; `retries` then starts a fresh attempt. The retry is deliberately a new pod, never a second kaniko in the same container: kaniko unpacks the base image over the container root and does not clean up when killed.
|
|
137
134
|
- **Caches never block a build.** `registryMap` defaults to the homelab pull-through caches (`DEFAULT_REGISTRY_MAP`: `mcr.microsoft.com` → `mcr-cache.ovea.pro`, `index.docker.io` → `hub-cache.ovea.pro`; kaniko under gVisor stalls pulling from mcr.microsoft.com directly). A map you pass replaces the default; `{}` turns mapping off. Before kaniko starts, every mirror host and the `cacheRepo` registry gets a 5 s `GET https://<host>/v2/` (200 or 401 = up); one that doesn't answer is dropped with a log line (`mcr-cache.ovea.pro unreachable; pulling from mcr.microsoft.com directly`, or no `--cache` for the layer cache). After that, kaniko itself copes: it falls back to the original registry when a mirror can't serve an image's manifest (`RetrieveRemoteImage`; `--skip-default-registry-fallback` is never set), a layer-cache lookup error is a cache miss, a cache push error is a warning, and `--skip-push-permission-check` stops its startup check of the cache repo from exiting. A cache that dies mid-download (after the manifest came from it) fails or stalls the attempt; `retries` starts a fresh one, which probes again.
|
|
138
|
-
- `
|
|
135
|
+
- **Registry mode** (`push: true`, for images too big for the step's ~1 GiB scratch space and Garage): kaniko runs without `--no-push`/`--tar-path`, pushes to `repository` (it must start with a registry host; pushes are anonymous) and the output is a `PushedImage` with the digest from `--digest-file`. With `contentTag`, the tag is `src-` + the first 12 hex of the sha256 of the context directory's deterministic tar (the bytes `ctx.output.dir` would write), computed from the fetched tree; an anonymous `HEAD https://<host>/v2/<path>/manifests/<tag>` that answers a `Docker-Content-Digest` skips kaniko and outputs that digest.
|
|
136
|
+
- `helmDeployOverSsh` gets each image into the host's **k3s** containerd: a `BuiltImage` tar is streamed in (`nerdctl --address /run/k3s/containerd/containerd.sock --namespace k8s.io load`), a `PushedImage` (`fromRegistry`) is pulled there (`… pull -q`, so the rollout doesn't wait on the pull). It copies the chart to `/tmp/<release>-helm-<namespace>`, runs `helm upgrade --install` from inside it (values, then `set`, then per image `<prefix>.repository`/`.tag`/`.pullPolicy` — `Never` for tarballs, `IfNotPresent` for registry images — which always win), waits for `kubectl rollout status` of each of `deployments` in order (`deploy/<name>` for a bare name), and with `healthUrl` polls it (5 × 5 s unless given) from the harness. Several images (one chart, two Deployments):
|
|
137
|
+
|
|
138
|
+
```ts
|
|
139
|
+
steps.helmDeployOverSsh({ name: "deploy-beta", description: "", host: "oliver@bulwark-m2", sshKey: SSH_PRIVATE_KEY,
|
|
140
|
+
release: "olve-slimerepublics", namespace: "apps-beta", chart: "helm",
|
|
141
|
+
images: { server: "image", web: "web.image" }, deployments: ["olve-slimerepublics-server", "olve-slimerepublics-web"] });
|
|
142
|
+
```
|
|
139
143
|
|
|
140
144
|
**Helpers.** General functions the library steps are built on, exported for custom handlers:
|
|
141
145
|
|
|
142
146
|
| Helper | What it does |
|
|
143
147
|
|---|---|
|
|
144
148
|
| `httpHealthCheck(ctx, url, { attempts?, delaySeconds?, headers? })` | GETs `url` until it returns 2xx (redirects followed), retrying network errors and non-2xx statuses; defaults 5 attempts, 5 s apart (v1's deploy-beta loop). Throws the last error when the attempts run out. Runs in the harness, so the image needs no curl. `helmDeployOverSsh`'s `healthUrl` uses it. |
|
|
149
|
+
| `serviceUrl(release)` | `http://<release>.<namespace>.svc.cluster.local` for a `HelmRelease` (charts that name the Service after the release). |
|
|
150
|
+
| `checkSpa(ctx, url)` | `GET <url>/` must reference an `/assets/*.js` bundle (Vite's layout) and the bundle must be served as JavaScript; returns `{ html, bundle }` for app-specific checks. |
|
|
151
|
+
| `sshSession(ctx, { host, key })` | Installs openssh-client (apk), writes the key to `<tmpDir>/ssh/id` (trailing newline, mode 600; host key not verified) and returns `{ run(remote), text(remote), copy(local, remote), shell(remote) }`; `shell` is the quoted ssh command for pipes (`ctx.sh(`printf '%s' "$V" \| ${s.shell("kubectl apply -f -")}`)`). `helmDeployOverSsh` uses it. `shq(s)` is the POSIX quoting it uses. |
|
|
145
152
|
|
|
146
153
|
```ts
|
|
147
|
-
import { httpHealthCheck, implement } from "@
|
|
154
|
+
import { httpHealthCheck, implement } from "@olivervea/pipelines";
|
|
148
155
|
export default implement(smoke, async (_, ctx) => { await httpHealthCheck(ctx, "http://app.ns.svc.cluster.local/api/health", { attempts: 10 }); });
|
|
149
156
|
```
|
|
150
157
|
|
|
@@ -152,7 +159,7 @@ export default implement(smoke, async (_, ctx) => { await httpHealthCheck(ctx, "
|
|
|
152
159
|
|
|
153
160
|
```ts
|
|
154
161
|
// handlers/deploy-beta.ts
|
|
155
|
-
import { implement } from "@
|
|
162
|
+
import { implement } from "@olivervea/pipelines";
|
|
156
163
|
import { deployBeta } from "../pipeline";
|
|
157
164
|
import deployScript from "../scripts/deploy.sh" with { type: "file" };
|
|
158
165
|
|
|
@@ -165,7 +172,7 @@ export default implement(deployBeta, async ({ artifact }, ctx) => {
|
|
|
165
172
|
|
|
166
173
|
| `ctx.` | |
|
|
167
174
|
|---|---|
|
|
168
|
-
| `sh(cmd \| script, { env, cwd, args })`, `sh.text(…)` | Runs a command string, or an embedded script, through the image's shell. The script is copied to `$OLVE_DIR/tmp/scripts/` first. The shell is the first of `/bin/sh`, `/busybox/sh`, `/bin/bash` that exists; a shell-less image gets a clear `ShellNotFoundError`. A script's shebang is honoured when its interpreter exists. A string counts as an embedded script when it is a `/$bunfs/…` path, or an absolute path to an existing `*.sh` or `*.bash` file. Pass `{ script }` to force script mode. A non-zero exit throws a `CommandError` that carries `exitCode`. |
|
|
175
|
+
| `sh(cmd \| script, { env, cwd, args })`, `sh.text(…)` | Runs a command string, or an embedded script, through the image's shell. The script is copied to `$OLVE_DIR/tmp/scripts/` first. The shell is the first of `/bin/sh`, `/busybox/sh`, `/bin/bash` that exists; a shell-less image gets a clear `ShellNotFoundError`. A script's shebang is honoured when its interpreter exists. A string counts as an embedded script when it is a `/$bunfs/…` path, or an absolute path to an existing `*.sh` or `*.bash` file. Pass `{ script }` to force script mode. `args` are the script's arguments, or `$1`, `$2`… of a command string. A non-zero exit throws a `CommandError` that carries `exitCode`. |
|
|
169
176
|
| `exec([argv…], { env, cwd, idleTimeoutSeconds })` | Runs a binary directly, with no shell (works on distroless). `idleTimeoutSeconds` (also on `sh`) is an inactivity watchdog: no stdout/stderr output for that long → SIGTERM, SIGKILL 10 s later, and a `CommandStalledError`. |
|
|
170
177
|
| `readText(path)`, `readJson(path)` | Paths are relative to the cwd. |
|
|
171
178
|
| `files.fetch(ref)` | Downloads a file reference and verifies its sha256. Directory refs are extracted. Returns a local path. |
|
|
@@ -204,7 +211,7 @@ export default pipeline({ name: "Olve.Pipelines", description: "…", input: M.D
|
|
|
204
211
|
### Testing handlers
|
|
205
212
|
|
|
206
213
|
```ts
|
|
207
|
-
import { testHandler, testFileRef } from "@
|
|
214
|
+
import { testHandler, testFileRef } from "@olivervea/pipelines/testing";
|
|
208
215
|
import deployBeta from "../handlers/deploy-beta";
|
|
209
216
|
|
|
210
217
|
const r = await testHandler(deployBeta, { artifact }, { secrets: { SSH_PRIVATE_KEY: "k" }, ctx: { http: fakeHttp }, run: { number: 412 } });
|
|
@@ -235,7 +242,7 @@ This section is everything the C# side needs in order to run the binary.
|
|
|
235
242
|
|
|
236
243
|
| Path | Who | What |
|
|
237
244
|
|---|---|---|
|
|
238
|
-
| `input.json` | service (via olve-init) | **run**: a JSON object with exactly the step's input keys, built from the manifest's `input` mappings. Leave out a key whose ref root step was skipped; this is allowed only for `optional` inputs. Unknown keys → exit 65. **map-trigger push**:
|
|
245
|
+
| `input.json` | service (via olve-init) | **run**: a JSON object with exactly the step's input keys, built from the manifest's `input` mappings. Leave out a key whose ref root step was skipped; this is allowed only for `optional` inputs. Unknown keys → exit 65. **map-trigger push**: `{"ref": "refs/heads/<branch>", "after": "<sha>", "repository": {"full_name": "<owner>/<repo>"}}` (`PushEvent`). **map-trigger poll**: `{"value": "<extracted value as string>"}`. **map-trigger webhook**: `{"headers": {…}, "body": …}` (`WebhookEvent`). |
|
|
239
246
|
| `work/output.json` | harness | The envelope. Any stale copy is deleted at start, and the new file is written atomically. |
|
|
240
247
|
| `work/files/<object path>` | harness | Output files and directory tars (see "File references"). |
|
|
241
248
|
| `tmp/` | harness | Scratch space. Point `TMPDIR` here. |
|
|
@@ -330,7 +337,7 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
|
|
|
330
337
|
An object whose only key is `ref` (with a string value) or `const` is a tag. Describe never emits a nested object with exactly that key set. Model-matched inputs are emitted already resolved (`{"ref": "build"}`).
|
|
331
338
|
- **`inputTypes`** declares each input key: `{model, optional}`, or `{schema: {output, input?}}` for a plain schema.
|
|
332
339
|
- **`handlerHash`** is the step's implementation identity. A step version can be computed from image + `inputTypes`/model versions + `output` model version + `secrets` + `input` mappings + `handlerHash`. Stage placement, `when` and the step's `description` are not part of it.
|
|
333
|
-
- Custom steps: `sha256(canonical {bundle, harness})`. `bundle` hashes the handler bundled on its own, with `pipeline.ts`, `models.ts` and `@
|
|
340
|
+
- Custom steps: `sha256(canonical {bundle, harness})`. `bundle` hashes the handler bundled on its own, with `pipeline.ts`, `models.ts` and `@olivervea/pipelines` external. It is tree-shaken and includes embedded scripts, so it changes only when that handler's code (or code it uses) changes.
|
|
334
341
|
- Library steps: `sha256(canonical {library: kind, config, harness})`.
|
|
335
342
|
- Because `harness` is part of both hashes, a Bun or SDK bump changes every handler.
|
|
336
343
|
- **`stages`** is an array whose items are either `{steps: [Step…]}` (parallel, with a barrier) or `{when: {field: [values]}, stages: [...]}`.
|
|
@@ -360,7 +367,7 @@ olve-build <pipelineDir> --out <dir> [--target bun-linux-x64|bun-linux-x64-musl]
|
|
|
360
367
|
olve-build <pipelineDir> --hash-only # print handler hashes (diagnostics)
|
|
361
368
|
```
|
|
362
369
|
|
|
363
|
-
1.
|
|
370
|
+
1. Pair each `handlers/*.ts` with the step its `implement(step, …)` names, and hash it (see `handlerHash`).
|
|
364
371
|
2. Generate `<pipelineDir>/.olve/gen/handlers.ts` (a registry of literal `import()` paths, plus hashes and harness info) and `entry.ts`. Add `.olve/` to `.gitignore`.
|
|
365
372
|
3. Run `bun build --compile --target=bun-linux-x64 --no-compile-autoload-dotenv --no-compile-autoload-bunfig --bytecode --format=esm` → `<out>/step`. The output is never minified, because mapper source analysis needs real source.
|
|
366
373
|
4. Check that `PT_INTERP` is `/lib64/ld-linux-x86-64.so.2`. On alpine, Bun silently produces a musl binary; the check fails it.
|
|
@@ -378,9 +385,9 @@ bun install --frozen-lockfile
|
|
|
378
385
|
bunx olve-build .pipelines --out /work/out --upload-url "$STEP_PUT_URL" --sentinels
|
|
379
386
|
```
|
|
380
387
|
|
|
381
|
-
From npm
|
|
388
|
+
From npm, nothing else is needed. A local `file:`/`link:` dependency (`"@olivervea/pipelines": "file:../sdk"`, as Olve.Pipelines' own `.pipelines/` does) resolves to its **source directory**, not the copy in `node_modules`, so its own dependencies must be installed there too: the build Job runs `bun install --frozen-lockfile --production` in every local dependency of the binding's `package.json` that is inside the repo, before building.
|
|
382
389
|
|
|
383
|
-
The repo depends on the SDK as a normal package. It resolves through `exports` (`@
|
|
390
|
+
The repo depends on the SDK as a normal package. It resolves through `exports` (`@olivervea/pipelines`, `@olivervea/pipelines/harness`, `@olivervea/pipelines/testing`), and the tests cover this with a `file:` install. `bun` must be on `PATH`, because the bins use `#!/usr/bin/env bun`. A consumer `tsconfig.json` has to include the dot-directory explicitly (`"include": [".pipelines/**/*.ts"]`), and should exclude `.pipelines/.olve`.
|
|
384
391
|
|
|
385
392
|
How the service runs it (reconcile, `BuildJobBuilder`): one Job per (pipeline, config commit), named `olve-build-<pipeline12>-<sha12>`. The repo tarball comes from `codeload.github.com/<repo>/tar.gz/<sha>` anonymously (public repos), or from the GitHub API tarball endpoint with the binding's credentials key from the pipeline Secret. The binary is PUT to `versions/<pipeline>/builds/<sha>/step.tar.gz`, and a version added from that build keeps this key as its `BinaryKey`. The in-repo fixture `.pipelines-test/` (bound on v2 beta) uses exactly this layout.
|
|
386
393
|
|
package/manifest.schema.json
CHANGED
package/package.json
CHANGED
package/src/build/build.ts
CHANGED
|
@@ -48,7 +48,7 @@ export function generateEntry(project: Project, hashes: Record<string, string>,
|
|
|
48
48
|
writeFileSync(
|
|
49
49
|
join(gen, "handlers.ts"),
|
|
50
50
|
`// GENERATED by olve-build — do not edit.\n` +
|
|
51
|
-
`import type { HandlerRegistry, HarnessInfo } from "@
|
|
51
|
+
`import type { HandlerRegistry, HarnessInfo } from "@olivervea/pipelines/harness";\n\n` +
|
|
52
52
|
`export const handlers: HandlerRegistry = {\n${registry}\n};\n\n` +
|
|
53
53
|
`export const handlerHashes: Record<string, string> = ${JSON.stringify(hashes, null, 2)};\n\n` +
|
|
54
54
|
`export const harness: HarnessInfo = ${JSON.stringify(harness, null, 2)};\n`,
|
|
@@ -58,7 +58,7 @@ export function generateEntry(project: Project, hashes: Record<string, string>,
|
|
|
58
58
|
entry,
|
|
59
59
|
`// GENERATED by olve-build — do not edit. Entry point of the step binary.\n` +
|
|
60
60
|
`import pipeline from ${JSON.stringify(relPosix(gen, project.pipelineFile))};\n` +
|
|
61
|
-
`import { runHarness } from "@
|
|
61
|
+
`import { runHarness } from "@olivervea/pipelines/harness";\n` +
|
|
62
62
|
`import { handlers, handlerHashes, harness } from "./handlers.ts";\n\n` +
|
|
63
63
|
`process.exit(await runHarness({ pipeline, handlers, handlerHashes, harness }, process.argv.slice(2)));\n`,
|
|
64
64
|
);
|
|
@@ -69,7 +69,7 @@ export async function build(dir: string, opts: BuildOptions): Promise<BuildResul
|
|
|
69
69
|
const log = opts.log ?? ((l: string) => process.stderr.write(l + "\n"));
|
|
70
70
|
const stdout = opts.stdout ?? (async (t: string) => void (await Bun.write(Bun.stdout, t)));
|
|
71
71
|
const target = opts.target ?? "bun-linux-x64";
|
|
72
|
-
const project = loadProject(dir);
|
|
72
|
+
const project = await loadProject(dir);
|
|
73
73
|
const out = resolve(opts.out);
|
|
74
74
|
mkdirSync(out, { recursive: true });
|
|
75
75
|
const harness = harnessInfo();
|
package/src/build/cli.ts
CHANGED
|
@@ -32,7 +32,7 @@ async function main(): Promise<number> {
|
|
|
32
32
|
const dir = positionals[0]!;
|
|
33
33
|
try {
|
|
34
34
|
if (values["hash-only"]) {
|
|
35
|
-
const h = await handlerHashes(loadProject(dir), harnessInfo());
|
|
35
|
+
const h = await handlerHashes(await loadProject(dir), harnessInfo());
|
|
36
36
|
await Bun.write(Bun.stdout, canonicalJson({ harness: harnessInfo(), handlers: h }));
|
|
37
37
|
return 0;
|
|
38
38
|
}
|
package/src/build/hashes.ts
CHANGED
|
@@ -22,7 +22,7 @@ function externals(project: Project): BunPlugin {
|
|
|
22
22
|
return {
|
|
23
23
|
name: "olve-handler-hash-externals",
|
|
24
24
|
setup(b) {
|
|
25
|
-
b.onResolve({ filter: /^@
|
|
25
|
+
b.onResolve({ filter: /^@olivervea\/pipelines(\/.*)?$/ }, (args) => ({ path: args.path, external: true }));
|
|
26
26
|
b.onResolve({ filter: /^\.{1,2}\// }, (args) => {
|
|
27
27
|
const base = resolve(dirname(args.importer), args.path);
|
|
28
28
|
for (const cand of [base, `${base}.ts`, join(base, "index.ts")]) if (contracts.has(cand)) return { path: args.path, external: true };
|
package/src/build/project.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Layout of a pipeline directory and the identity of the SDK building it.
|
|
2
2
|
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
3
3
|
import { dirname, join, relative, resolve } from "node:path";
|
|
4
|
+
import { isImplementation } from "../core/handlers.ts";
|
|
4
5
|
import { sha256Hex } from "../util/hash.ts";
|
|
5
6
|
import type { HarnessInfo } from "../util/handler-hash.ts";
|
|
6
7
|
|
|
@@ -11,7 +12,7 @@ export interface Project {
|
|
|
11
12
|
pipelineFile: string;
|
|
12
13
|
/** Contract files (excluded from handler bundles): pipeline.ts, models.ts. */
|
|
13
14
|
contractFiles: string[];
|
|
14
|
-
/** step name → absolute handler file, sorted by name. */
|
|
15
|
+
/** step name → absolute handler file, sorted by step name. The step is the one the file's `implement(step, …)` names. */
|
|
15
16
|
handlers: [string, string][];
|
|
16
17
|
}
|
|
17
18
|
|
|
@@ -19,20 +20,40 @@ export class ProjectError extends Error {
|
|
|
19
20
|
override name = "ProjectError";
|
|
20
21
|
}
|
|
21
22
|
|
|
22
|
-
export function loadProject(dirArg: string): Project {
|
|
23
|
+
export async function loadProject(dirArg: string): Promise<Project> {
|
|
23
24
|
const dir = resolve(dirArg);
|
|
24
25
|
if (!existsSync(dir) || !statSync(dir).isDirectory()) throw new ProjectError(`${dirArg} is not a directory`);
|
|
25
26
|
const pipelineFile = join(dir, "pipeline.ts");
|
|
26
27
|
if (!existsSync(pipelineFile)) throw new ProjectError(`${join(dirArg, "pipeline.ts")} not found (it must default-export pipeline({...}))`);
|
|
27
28
|
const contractFiles = ["pipeline.ts", "models.ts"].map((f) => join(dir, f)).filter((f) => existsSync(f));
|
|
28
29
|
const hdir = join(dir, "handlers");
|
|
29
|
-
const
|
|
30
|
+
const files = existsSync(hdir)
|
|
30
31
|
? readdirSync(hdir)
|
|
31
32
|
.filter((f) => f.endsWith(".ts") && !f.endsWith(".test.ts") && !f.endsWith(".d.ts") && statSync(join(hdir, f)).isFile())
|
|
32
33
|
.sort()
|
|
33
|
-
.map((f) => [f.slice(0, -3), join(hdir, f)])
|
|
34
34
|
: [];
|
|
35
|
-
return { dir, pipelineFile, contractFiles, handlers };
|
|
35
|
+
return { dir, pipelineFile, contractFiles, handlers: await resolveHandlers(hdir, files) };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Pairs each handler file with the step its default export implements (file names don't matter). */
|
|
39
|
+
async function resolveHandlers(hdir: string, files: string[]): Promise<[string, string][]> {
|
|
40
|
+
const byStep = new Map<string, string>();
|
|
41
|
+
for (const f of files) {
|
|
42
|
+
const file = join(hdir, f);
|
|
43
|
+
let impl: unknown;
|
|
44
|
+
try {
|
|
45
|
+
impl = ((await import(file)) as { default?: unknown }).default;
|
|
46
|
+
} catch (e) {
|
|
47
|
+
throw new ProjectError(`handlers/${f} failed to load: ${(e as Error)?.message ?? String(e)}`);
|
|
48
|
+
}
|
|
49
|
+
if (!isImplementation(impl))
|
|
50
|
+
throw new ProjectError(`handlers/${f} must default-export implement(<step>, …); put shared code in a subfolder such as handlers/lib/`);
|
|
51
|
+
const step = (impl.step as { name: string }).name;
|
|
52
|
+
const other = byStep.get(step);
|
|
53
|
+
if (other) throw new ProjectError(`handlers/${other.slice(hdir.length + 1)} and handlers/${f} both implement step '${step}'`);
|
|
54
|
+
byStep.set(step, file);
|
|
55
|
+
}
|
|
56
|
+
return [...byStep].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
|
36
57
|
}
|
|
37
58
|
|
|
38
59
|
function listFiles(dir: string): string[] {
|
package/src/builder/pipeline.ts
CHANGED
|
@@ -178,6 +178,11 @@ export function isPipeline(p: unknown): boolean {
|
|
|
178
178
|
|
|
179
179
|
// ─── Pipeline builder ────────────────────────────────────────────────────────────────────────
|
|
180
180
|
|
|
181
|
+
// Builder calls record into shared data and hand back a fresh handle; a handle that was already
|
|
182
|
+
// extended no longer matches its type (it would claim fewer stages than the data has), so reuse is a problem.
|
|
183
|
+
const extended = new WeakSet<object>();
|
|
184
|
+
const ReusedProblem = "a pipeline value was extended twice: chain the calls (pipeline({...}).stage(a).stage(b)) instead of reusing an earlier value";
|
|
185
|
+
|
|
181
186
|
export interface PipelineDef<In extends AnyModel> {
|
|
182
187
|
/** Display name. */
|
|
183
188
|
name: string;
|
|
@@ -203,10 +208,19 @@ export class Pipeline<In extends AnyModel, A extends AvailMap> {
|
|
|
203
208
|
};
|
|
204
209
|
}
|
|
205
210
|
|
|
211
|
+
private next(): never {
|
|
212
|
+
const data = this[PIPELINE];
|
|
213
|
+
if (extended.has(this) && !data.problems.includes(ReusedProblem)) data.problems.push(ReusedProblem);
|
|
214
|
+
extended.add(this);
|
|
215
|
+
const handle = Object.create(Object.getPrototypeOf(this)) as Pipeline<In, A>;
|
|
216
|
+
(handle as { [PIPELINE]: PipelineData })[PIPELINE] = data;
|
|
217
|
+
return handle as never;
|
|
218
|
+
}
|
|
219
|
+
|
|
206
220
|
/** Map a GitHub push to the pipeline input (`z.input` view). Runs as code in a `map-trigger` Job per event. */
|
|
207
221
|
onPush<R extends InferInput<In>>(map: (event: PushEvent) => Exact<R, InferInput<In>>): this {
|
|
208
222
|
this[PIPELINE].triggers.push({ kind: "push", name: "push", map: map as (event: unknown) => unknown });
|
|
209
|
-
return this;
|
|
223
|
+
return this.next();
|
|
210
224
|
}
|
|
211
225
|
|
|
212
226
|
/**
|
|
@@ -235,7 +249,7 @@ export class Pipeline<In extends AnyModel, A extends AvailMap> {
|
|
|
235
249
|
intervalSeconds: cfg.intervalSeconds,
|
|
236
250
|
map: map as (value: string) => unknown,
|
|
237
251
|
});
|
|
238
|
-
return this;
|
|
252
|
+
return this.next();
|
|
239
253
|
}
|
|
240
254
|
|
|
241
255
|
/**
|
|
@@ -247,7 +261,7 @@ export class Pipeline<In extends AnyModel, A extends AvailMap> {
|
|
|
247
261
|
map: (event: WebhookEvent) => Exact<R, InferInput<In>> | null,
|
|
248
262
|
): this {
|
|
249
263
|
this[PIPELINE].triggers.push({ kind: "webhook", name: cfg.name, verify: cfg.verify, map: map as (event: WebhookEvent) => unknown });
|
|
250
|
-
return this;
|
|
264
|
+
return this.next();
|
|
251
265
|
}
|
|
252
266
|
|
|
253
267
|
/**
|
|
@@ -272,7 +286,7 @@ export class Pipeline<In extends AnyModel, A extends AvailMap> {
|
|
|
272
286
|
});
|
|
273
287
|
if (entries.length === 0) data.problems.push("empty stage");
|
|
274
288
|
data.nodes.push({ kind: "stage", steps });
|
|
275
|
-
return this;
|
|
289
|
+
return this.next();
|
|
276
290
|
}
|
|
277
291
|
|
|
278
292
|
/** Declarative condition on literal-like pipeline-input fields. Outputs produced inside become maybe-available after. */
|
|
@@ -280,9 +294,10 @@ export class Pipeline<In extends AnyModel, A extends AvailMap> {
|
|
|
280
294
|
const data = this[PIPELINE];
|
|
281
295
|
const inner = new Pipeline<In, A>({ name: data.name, input: data.input as In }, data);
|
|
282
296
|
const returned = block(inner);
|
|
283
|
-
if (returned
|
|
297
|
+
if (!isPipeline(returned) || pipelineData(returned) !== inner[PIPELINE])
|
|
298
|
+
data.problems.push("a when block must return the pipeline it was given (p => p.stage(...))");
|
|
284
299
|
data.nodes.push({ kind: "when", cond: { ...(cond as object) }, nodes: inner[PIPELINE].nodes });
|
|
285
|
-
return this
|
|
300
|
+
return this.next();
|
|
286
301
|
}
|
|
287
302
|
|
|
288
303
|
/** Run `handler` (a step whose inputs are all FailureContext) when one of `steps` (default: any) fails. Untracked, best-effort. */
|
|
@@ -295,7 +310,7 @@ export class Pipeline<In extends AnyModel, A extends AvailMap> {
|
|
|
295
310
|
opts?: { steps?: readonly Step<StepNames<A>, any, any, any>[] },
|
|
296
311
|
): this {
|
|
297
312
|
this[PIPELINE].failureHandlers.push({ step: handler as AnyStep, steps: opts?.steps ? [...(opts.steps as readonly AnyStep[])] : null });
|
|
298
|
-
return this;
|
|
313
|
+
return this.next();
|
|
299
314
|
}
|
|
300
315
|
}
|
|
301
316
|
|
package/src/ctx/sh.ts
CHANGED
|
@@ -85,7 +85,8 @@ export function createSh(deps: ShDeps): { sh: Sh; exec: (argv: readonly string[]
|
|
|
85
85
|
const scriptPath = typeof cmd === "object" ? cmd.script : isEmbeddedScript(cmd) ? cmd : undefined;
|
|
86
86
|
if (scriptPath === undefined) {
|
|
87
87
|
const shell = detectShell(deps.shellCandidates);
|
|
88
|
-
|
|
88
|
+
// `sh -c cmd sh a b`: the args are $1, $2… of the command.
|
|
89
|
+
return { argv: [shell, "-c", cmd as string, "sh", ...args], what: `command \`${truncate(cmd as string)}\`` };
|
|
89
90
|
}
|
|
90
91
|
const file = materialize(scriptPath);
|
|
91
92
|
const first = readFileSync(file, "utf8").split("\n", 1)[0] ?? "";
|
package/src/describe/describe.ts
CHANGED
|
@@ -20,7 +20,7 @@ export class DescribeError extends Error {
|
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
export interface DescribeOptions {
|
|
23
|
-
/** Step names that have a handler
|
|
23
|
+
/** Step names that have a handler (a `handlers/*.ts` default-exporting `implement(step, …)`). */
|
|
24
24
|
handlerNames: Iterable<string>;
|
|
25
25
|
/** Step name → handler hash (from the build; includes the harness identity). */
|
|
26
26
|
handlerHashes: Readonly<Record<string, string>>;
|
|
@@ -198,7 +198,7 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
198
198
|
let handlerHash: string;
|
|
199
199
|
let config: Record<string, unknown> | undefined;
|
|
200
200
|
if (s.kind === "custom") {
|
|
201
|
-
if (!handlerNames.has(s.name)) problems.push(`${at} has no handler (
|
|
201
|
+
if (!handlerNames.has(s.name)) problems.push(`${at} has no handler (add a handlers/*.ts default-exporting implement(${camel(s.name)}, …))`);
|
|
202
202
|
handlerHash = opts.handlerHashes[s.name] ?? "";
|
|
203
203
|
if (handlerNames.has(s.name) && !handlerHash) problems.push(`${at}: no handler hash was computed`);
|
|
204
204
|
} else if (s.builtin) {
|
|
@@ -329,11 +329,11 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
329
329
|
return { step: describeStep({ step: fh.step }, new Map(), true), steps: scope };
|
|
330
330
|
}).filter((x) => x !== undefined);
|
|
331
331
|
|
|
332
|
-
// ──
|
|
332
|
+
// ── handlers without a step ──
|
|
333
333
|
for (const h of handlerNames) {
|
|
334
334
|
const s = steps.get(h);
|
|
335
|
-
if (!s) problems.push(`
|
|
336
|
-
else if (s.kind !== "custom") problems.push(`
|
|
335
|
+
if (!s) problems.push(`a handler implements step '${h}', which is not in the pipeline`);
|
|
336
|
+
else if (s.kind !== "custom") problems.push(`a handler implements '${h}', a library step (${s.kind}) that carries its own implementation`);
|
|
337
337
|
}
|
|
338
338
|
|
|
339
339
|
// ── triggers ──
|
|
@@ -117,7 +117,7 @@ export const Manifest = z
|
|
|
117
117
|
harness: z
|
|
118
118
|
.strictObject({
|
|
119
119
|
bun: z.string().describe("Bun runtime version compiled into the binary."),
|
|
120
|
-
sdk: z.string().describe("@
|
|
120
|
+
sdk: z.string().describe("@olivervea/pipelines package version."),
|
|
121
121
|
sdkHash: hash.describe("Hash of the SDK runtime sources + zod version."),
|
|
122
122
|
})
|
|
123
123
|
.describe("Harness identity. Already folded into every handlerHash."),
|
package/src/dev/cli.ts
CHANGED
|
@@ -17,7 +17,7 @@ const USAGE = `usage: olve-dev describe <dir> [--sentinels]
|
|
|
17
17
|
olve-dev map-trigger <dir> <trigger> --event-file f [--olve-dir d]`;
|
|
18
18
|
|
|
19
19
|
export async function devConfig(dir: string, withHashes: boolean): Promise<HarnessConfig> {
|
|
20
|
-
const project = loadProject(dir);
|
|
20
|
+
const project = await loadProject(dir);
|
|
21
21
|
const mod = (await import(project.pipelineFile)) as { default?: unknown };
|
|
22
22
|
if (!mod.default) throw new ProjectError(`${project.pipelineFile} has no default export (export default pipeline({...})...)`);
|
|
23
23
|
const harness = harnessInfo();
|
package/src/harness/index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// @
|
|
1
|
+
// @olivervea/pipelines/harness — the runtime the build tool compiles into the step binary.
|
|
2
2
|
export { runHarness, defaultIO, allSteps, resolveStepFunction, type HarnessConfig, type HandlerRegistry, type HarnessIO } from "./main.ts";
|
|
3
3
|
export { EXIT } from "./exit.ts";
|
|
4
4
|
export { ENV, paths, runInfoFromEnv } from "./env.ts";
|
package/src/harness/main.ts
CHANGED
|
@@ -77,17 +77,17 @@ export async function resolveStepFunction(step: AnyStep, handlers: HandlerRegist
|
|
|
77
77
|
return step.builtin.run as StepFunction;
|
|
78
78
|
}
|
|
79
79
|
const load = handlers[step.name];
|
|
80
|
-
if (!load) return `step '${step.name}' has no handler
|
|
80
|
+
if (!load) return `step '${step.name}' has no handler`;
|
|
81
81
|
let mod: { default: unknown };
|
|
82
82
|
try {
|
|
83
83
|
mod = await load();
|
|
84
84
|
} catch (e) {
|
|
85
|
-
return `
|
|
85
|
+
return `the handler of step '${step.name}' failed to load: ${(e as Error)?.stack ?? String(e)}`;
|
|
86
86
|
}
|
|
87
87
|
const impl = mod.default;
|
|
88
|
-
if (!isImplementation(impl)) return `
|
|
88
|
+
if (!isImplementation(impl)) return `the handler of step '${step.name}' must default-export implement(<step '${step.name}'>, …)`;
|
|
89
89
|
if (impl.step !== step)
|
|
90
|
-
return `
|
|
90
|
+
return `the handler of step '${step.name}' implements a different step object (it must be implement(<the '${step.name}' step from pipeline.ts>, …))`;
|
|
91
91
|
return impl.handler as unknown as StepFunction;
|
|
92
92
|
}
|
|
93
93
|
|
package/src/index.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/// <reference path="./assets.d.ts" />
|
|
2
|
-
// @
|
|
2
|
+
// @olivervea/pipelines — authoring API (contracts, handlers, pipeline builder).
|
|
3
3
|
export { z } from "zod";
|
|
4
4
|
export { model, fileRef, optional, type Model, type AnyModel, type Infer, type InferInput, type Plain, type FileRef, type Optional } from "./core/models.ts";
|
|
5
5
|
export { secret, type Secret, type DeclaredSecrets } from "./core/secrets.ts";
|
|
@@ -9,4 +9,4 @@ export { implement, type Implementation, type Handler, type HandlerResult, type
|
|
|
9
9
|
export type { Ctx, RunInfo, Sh, ShOptions, ExecOptions, Http, HttpInit, RetryOptions, OutputFileOptions } from "./ctx/types.ts";
|
|
10
10
|
export type { Ref, Mappable, MapperResult } from "./builder/refs.ts";
|
|
11
11
|
export { pipeline, Pipeline, type PipelineDef, type InRefs, type InitialAvail, type WhenCond, type PushEvent, type StageEntry, type CheckStage, type MapperFn } from "./builder/pipeline.ts";
|
|
12
|
-
export { steps, Source, BuiltImage, HelmRelease, FailureContext, type RunValue, type ScriptDef, type GitHubSourceDef, type ImageBuildDef, type HelmDeployOverSshDef, httpHealthCheck, type HttpHealthCheckOptions } from "./library/index.ts";
|
|
12
|
+
export { steps, Source, BuiltImage, PushedImage, HelmRelease, FailureContext, type RunValue, type ScriptDef, type GitHubSourceDef, type ImageBuildDef, type ImageBuildTarballDef, type ImageBuildPushDef, type HelmDeployOverSshDef, httpHealthCheck, type HttpHealthCheckOptions, sshSession, shq, type SshSession, serviceUrl, checkSpa } from "./library/index.ts";
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Checks for deployed apps, used from custom test handlers.
|
|
2
|
+
import type { Ctx } from "../ctx/types.ts";
|
|
3
|
+
|
|
4
|
+
/** The in-cluster URL of a release's Service, assuming the chart names it after the release. */
|
|
5
|
+
export function serviceUrl(release: { release: string; namespace: string }): string {
|
|
6
|
+
return `http://${release.release}.${release.namespace}.svc.cluster.local`;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Checks `url` serves a single-page app: `GET <url>/` references a `/assets/*.js` bundle (Vite's
|
|
11
|
+
* layout), and the bundle is served as JavaScript. Returns the page, for app-specific checks.
|
|
12
|
+
*/
|
|
13
|
+
export async function checkSpa(ctx: Pick<Ctx, "http" | "log">, url: string): Promise<{ html: string; bundle: string }> {
|
|
14
|
+
const base = url.replace(/\/$/, "");
|
|
15
|
+
const html = await (await ctx.http.expectOk(`${base}/`)).text();
|
|
16
|
+
const bundle = html.match(/src="(\/assets\/[^"]+\.js)"/)?.[1];
|
|
17
|
+
if (bundle === undefined) throw new Error(`${base}/ does not reference an /assets/*.js bundle`);
|
|
18
|
+
const js = await ctx.http.expectOk(`${base}${bundle}`);
|
|
19
|
+
await js.body?.cancel().catch(() => {});
|
|
20
|
+
if (!(js.headers.get("content-type")?.includes("javascript") ?? false)) throw new Error(`${base}${bundle} is not served as JavaScript`);
|
|
21
|
+
ctx.log("SPA served", { url: base, bundle });
|
|
22
|
+
return { html, bundle };
|
|
23
|
+
}
|
|
@@ -1,16 +1,20 @@
|
|
|
1
|
-
// steps.helmDeployOverSsh —
|
|
2
|
-
//
|
|
1
|
+
// steps.helmDeployOverSsh — get built images onto a k3s host and helm-upgrade a release there, over
|
|
2
|
+
// SSH; optionally verify an HTTP health endpoint afterwards.
|
|
3
3
|
//
|
|
4
4
|
// Ported from v1's olve-lib.sh (olve_ssh_host, olve_image_import, olve_helm_deploy) and
|
|
5
5
|
// deploy-beta.sh (rollout wait + health loop); the footguns are kept as comments below.
|
|
6
|
-
import {
|
|
6
|
+
import { existsSync, statSync } from "node:fs";
|
|
7
7
|
import { join } from "node:path";
|
|
8
8
|
import type { Ctx } from "../ctx/types.ts";
|
|
9
9
|
import type { FileRef } from "../core/models.ts";
|
|
10
10
|
import type { Secret } from "../core/secrets.ts";
|
|
11
11
|
import { makeStep, type LibraryStep, type LiteralName, type Step } from "../core/steps.ts";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
12
|
+
import type { Simplify } from "../core/types.ts";
|
|
13
|
+
import { httpHealthCheck, type HttpHealthCheckOptions } from "./http-health.ts";
|
|
14
|
+
import { BuiltImage, HelmRelease, PushedImage, Source } from "./models.ts";
|
|
15
|
+
import { shq, sshSession } from "./ssh.ts";
|
|
16
|
+
|
|
17
|
+
export { shq } from "./ssh.ts";
|
|
14
18
|
|
|
15
19
|
/** Alpine: `apk add openssh-client` at run time gives ssh/scp (the health check runs in the harness). */
|
|
16
20
|
export const HELM_DEPLOY_IMAGE = "alpine:3.22";
|
|
@@ -20,13 +24,18 @@ export const HELM_DEPLOY_IMAGE = "alpine:3.22";
|
|
|
20
24
|
* the k3s instance's `k8s.io` namespace (otherwise pods fail with ErrImageNeverPull).
|
|
21
25
|
*/
|
|
22
26
|
export const K3S_IMAGE_IMPORT = "sudo nerdctl --address /run/k3s/containerd/containerd.sock --namespace k8s.io load";
|
|
27
|
+
/** Registry images are pulled the same way before the rollout, so the rollout doesn't wait on the pull. */
|
|
28
|
+
export const K3S_IMAGE_PULL = "sudo nerdctl --address /run/k3s/containerd/containerd.sock --namespace k8s.io pull -q";
|
|
23
29
|
|
|
24
|
-
const
|
|
30
|
+
export const DEFAULT_ROLLOUT_TIMEOUT_SECONDS = 120;
|
|
25
31
|
|
|
26
|
-
|
|
27
|
-
|
|
32
|
+
type DefaultImages = { readonly image: "image" };
|
|
33
|
+
type ImageModel<R extends boolean> = R extends true ? typeof PushedImage : typeof BuiltImage;
|
|
34
|
+
export type HelmDeployOverSshInput<Im extends Record<string, string> = DefaultImages, R extends boolean = false> = Simplify<
|
|
35
|
+
{ readonly source: typeof Source } & { readonly [K in keyof Im]: ImageModel<R> }
|
|
36
|
+
>;
|
|
28
37
|
|
|
29
|
-
export interface HelmDeployOverSshDef<N extends string, K extends string> {
|
|
38
|
+
export interface HelmDeployOverSshDef<N extends string, K extends string, Im extends Record<string, string> = DefaultImages, R extends boolean = false> {
|
|
30
39
|
name: LiteralName<N>;
|
|
31
40
|
/** What the step does, shown in the UI. Required; may be empty. */
|
|
32
41
|
description: string;
|
|
@@ -34,7 +43,7 @@ export interface HelmDeployOverSshDef<N extends string, K extends string> {
|
|
|
34
43
|
host: string;
|
|
35
44
|
/** Private key for `host`. */
|
|
36
45
|
sshKey: Secret<K>;
|
|
37
|
-
/** Helm release name
|
|
46
|
+
/** Helm release name. */
|
|
38
47
|
release: string;
|
|
39
48
|
namespace: string;
|
|
40
49
|
/** Chart directory inside the source tree, e.g. `helm`. */
|
|
@@ -42,19 +51,36 @@ export interface HelmDeployOverSshDef<N extends string, K extends string> {
|
|
|
42
51
|
/** Values files, relative to the chart directory (`-f`, in order). */
|
|
43
52
|
values?: readonly string[];
|
|
44
53
|
/**
|
|
45
|
-
* Chart-specific `--set` flags (e.g. `{ "
|
|
46
|
-
* the step and always win:
|
|
54
|
+
* Chart-specific `--set` flags (e.g. `{ "ingress.enabled": "false" }`). The image flags are added by
|
|
55
|
+
* the step and always win: `<prefix>.repository`, `<prefix>.tag`, `<prefix>.pullPolicy`.
|
|
47
56
|
*/
|
|
48
57
|
set?: Readonly<Record<string, string>>;
|
|
49
|
-
/**
|
|
50
|
-
|
|
58
|
+
/**
|
|
59
|
+
* The images to deploy: step input key → the chart's values prefix for that image. Each input is a
|
|
60
|
+
* `BuiltImage` (or `PushedImage` with `fromRegistry`). Default `{ image: "image" }`; several images
|
|
61
|
+
* need a mapper, e.g. `{ server: "image", web: "web.image" }`.
|
|
62
|
+
*/
|
|
63
|
+
images?: Im & { readonly source?: never };
|
|
64
|
+
/**
|
|
65
|
+
* The images are `PushedImage`s from a registry (`imageBuild` with `push`): pulled on the host
|
|
66
|
+
* first, then `pullPolicy=IfNotPresent`. Default: `BuiltImage` tarballs, imported on the host, then
|
|
67
|
+
* `pullPolicy=Never`.
|
|
68
|
+
*/
|
|
69
|
+
fromRegistry?: R & (boolean extends R ? "❌ fromRegistry must be a literal true or false (use `as const` on a shared config object)" : unknown);
|
|
70
|
+
/** Rollouts to wait for, in order: `deploy/<name>` for a bare name. Default: the release name. */
|
|
71
|
+
deployments?: readonly string[];
|
|
72
|
+
/** Per rollout (`kubectl rollout status --timeout`). Default 120. */
|
|
73
|
+
rolloutTimeoutSeconds?: number;
|
|
74
|
+
/** After the rollouts, GET this URL until it returns 2xx (`httpHealthCheck`: 5 × 5 s unless given), or fail the step. */
|
|
75
|
+
healthUrl?: string | ({ url: string } & Pick<HttpHealthCheckOptions, "attempts" | "delaySeconds">);
|
|
51
76
|
timeoutSeconds?: number;
|
|
52
77
|
}
|
|
53
78
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
79
|
+
export interface HelmImageFlags {
|
|
80
|
+
prefix: string;
|
|
81
|
+
repository: string;
|
|
82
|
+
tag: string;
|
|
83
|
+
pullPolicy: "Never" | "IfNotPresent";
|
|
58
84
|
}
|
|
59
85
|
|
|
60
86
|
/** Remote `helm upgrade --install`, run from inside the copied chart directory. */
|
|
@@ -64,18 +90,18 @@ export function helmUpgradeCommand(o: {
|
|
|
64
90
|
namespace: string;
|
|
65
91
|
values: readonly string[];
|
|
66
92
|
set: Readonly<Record<string, string>>;
|
|
67
|
-
|
|
68
|
-
tag: string;
|
|
93
|
+
images: readonly HelmImageFlags[];
|
|
69
94
|
}): string {
|
|
70
95
|
// Chart-relative values paths (`-f values-beta.yaml`) resolve because helm runs from inside the
|
|
71
|
-
// copied chart dir. Extra flags come before the image flags, so the image overrides win.
|
|
72
|
-
// is local to the node (imported above), never pulled: pullPolicy=Never.
|
|
96
|
+
// copied chart dir. Extra flags come before the image flags, so the image overrides win.
|
|
73
97
|
const flags = [
|
|
74
98
|
...o.values.flatMap((v) => ["-f", v]),
|
|
75
99
|
...Object.entries(o.set).flatMap(([k, v]) => ["--set", `${k}=${v}`]),
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
100
|
+
...o.images.flatMap((i) => [
|
|
101
|
+
"--set", `${i.prefix}.repository=${i.repository}`,
|
|
102
|
+
"--set", `${i.prefix}.tag=${i.tag}`,
|
|
103
|
+
"--set", `${i.prefix}.pullPolicy=${i.pullPolicy}`,
|
|
104
|
+
]),
|
|
79
105
|
];
|
|
80
106
|
return [
|
|
81
107
|
`cd ${shq(o.remoteChart)}`,
|
|
@@ -85,15 +111,31 @@ export function helmUpgradeCommand(o: {
|
|
|
85
111
|
}
|
|
86
112
|
|
|
87
113
|
/**
|
|
88
|
-
* Deploys
|
|
89
|
-
*
|
|
90
|
-
* wait for
|
|
114
|
+
* Deploys images with the Helm chart at `<source>/<chart>` on a k3s host over SSH: get each image
|
|
115
|
+
* into the host's k3s containerd (stream a tarball in, or pull from the registry), copy the chart,
|
|
116
|
+
* `helm upgrade --install`, wait for each rollout, then (with `healthUrl`) poll the health endpoint.
|
|
91
117
|
*/
|
|
92
|
-
export function helmDeployOverSsh<
|
|
93
|
-
|
|
94
|
-
|
|
118
|
+
export function helmDeployOverSsh<
|
|
119
|
+
const N extends string,
|
|
120
|
+
const K extends string,
|
|
121
|
+
const Im extends Record<string, string> = DefaultImages,
|
|
122
|
+
const R extends boolean = false,
|
|
123
|
+
>(def: HelmDeployOverSshDef<N, K, Im, R>): Step<N, HelmDeployOverSshInput<Im, R>, typeof HelmRelease, K> & LibraryStep {
|
|
95
124
|
const values = def.values ?? [];
|
|
96
125
|
const set = def.set ?? {};
|
|
126
|
+
const images: Record<string, string> = def.images ?? { image: "image" };
|
|
127
|
+
const imageEntries = Object.entries(images);
|
|
128
|
+
const fromRegistry = def.fromRegistry === true;
|
|
129
|
+
const deployments = def.deployments ?? [def.release];
|
|
130
|
+
const rolloutTimeout = def.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT_SECONDS;
|
|
131
|
+
const health = typeof def.healthUrl === "string" ? { url: def.healthUrl } : def.healthUrl;
|
|
132
|
+
if (imageEntries.length === 0) throw new Error(`helmDeployOverSsh '${def.name}': images must not be empty`);
|
|
133
|
+
if ("source" in images) throw new Error(`helmDeployOverSsh '${def.name}': 'source' can't be an image input`);
|
|
134
|
+
if (deployments.length === 0) throw new Error(`helmDeployOverSsh '${def.name}': deployments must not be empty`);
|
|
135
|
+
const imageModel = fromRegistry ? PushedImage : BuiltImage;
|
|
136
|
+
const input = { source: Source, ...Object.fromEntries(imageEntries.map(([key]) => [key, imageModel])) };
|
|
137
|
+
const isDefaultImages = imageEntries.length === 1 && images.image === "image";
|
|
138
|
+
|
|
97
139
|
return makeStep(
|
|
98
140
|
"helmDeployOverSsh",
|
|
99
141
|
{ name: def.name, description: def.description, image: HELM_DEPLOY_IMAGE, input, output: HelmRelease, secrets: [def.sshKey], timeoutSeconds: def.timeoutSeconds },
|
|
@@ -106,45 +148,48 @@ export function helmDeployOverSsh<const N extends string, const K extends string
|
|
|
106
148
|
chart: def.chart,
|
|
107
149
|
values: [...values],
|
|
108
150
|
set: { ...set },
|
|
109
|
-
...(
|
|
151
|
+
...(isDefaultImages ? {} : { images: { ...images } }),
|
|
152
|
+
...(fromRegistry ? { fromRegistry: true } : {}),
|
|
153
|
+
...(def.deployments ? { deployments: [...def.deployments] } : {}),
|
|
154
|
+
...(def.rolloutTimeoutSeconds !== undefined ? { rolloutTimeoutSeconds: def.rolloutTimeoutSeconds } : {}),
|
|
155
|
+
...(def.healthUrl !== undefined ? { healthUrl: typeof def.healthUrl === "string" ? def.healthUrl : { ...def.healthUrl } } : {}),
|
|
110
156
|
}),
|
|
111
|
-
async run(
|
|
112
|
-
|
|
113
|
-
ctx: Ctx,
|
|
114
|
-
) {
|
|
115
|
-
const tree = await ctx.files.fetch(source.tree);
|
|
157
|
+
async run(input: Record<string, { repository: string; tag: string; tar?: FileRef }> & { source: { tree: FileRef } }, ctx: Ctx) {
|
|
158
|
+
const tree = await ctx.files.fetch(input.source.tree);
|
|
116
159
|
const chartDir = join(tree, def.chart);
|
|
117
160
|
if (!existsSync(chartDir) || !statSync(chartDir).isDirectory()) throw new Error(`chart directory '${def.chart}' not found in the source tree`);
|
|
118
|
-
const imageTar = await ctx.files.fetch(image.tar);
|
|
119
|
-
|
|
120
|
-
// SSH setup. The key file needs a trailing newline (OpenSSH rejects the key otherwise) and
|
|
121
|
-
// mode 600. The host key is not verified (as in v1): the host is reached over the tailnet.
|
|
122
|
-
await ctx.exec(["apk", "add", "--no-cache", "openssh-client"]);
|
|
123
|
-
const sshDir = join(ctx.tmpDir, "ssh");
|
|
124
|
-
mkdirSync(sshDir, { recursive: true, mode: 0o700 });
|
|
125
|
-
const keyFile = join(sshDir, "id");
|
|
126
|
-
const key = ctx.secrets[def.sshKey.name]!;
|
|
127
|
-
writeFileSync(keyFile, key.endsWith("\n") ? key : key + "\n", { mode: 0o600 });
|
|
128
|
-
chmodSync(keyFile, 0o600);
|
|
129
|
-
const opts = ["-i", keyFile, "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", "-o", "BatchMode=yes"];
|
|
130
|
-
const ssh = (remote: string) => ctx.exec(["ssh", ...opts, def.host, remote]);
|
|
131
161
|
|
|
132
|
-
const
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
162
|
+
const ssh = await sshSession(ctx, { host: def.host, key: ctx.secrets[def.sshKey.name]! });
|
|
163
|
+
const flags: HelmImageFlags[] = [];
|
|
164
|
+
for (const [key, prefix] of imageEntries) {
|
|
165
|
+
const image = input[key]!;
|
|
166
|
+
const ref = `${image.repository}:${image.tag}`;
|
|
167
|
+
if (fromRegistry) {
|
|
168
|
+
ctx.log("pulling image", { image: ref, host: def.host });
|
|
169
|
+
await ssh.run(`${K3S_IMAGE_PULL} ${shq(ref)}`);
|
|
170
|
+
} else {
|
|
171
|
+
const tar = await ctx.files.fetch(image.tar!);
|
|
172
|
+
ctx.log("importing image", { image: ref, host: def.host });
|
|
173
|
+
// Streamed over ssh's stdin (ctx.exec has none, hence the shell redirect).
|
|
174
|
+
await ctx.sh(`${ssh.shell(K3S_IMAGE_IMPORT)} < ${shq(tar)}`);
|
|
175
|
+
}
|
|
176
|
+
flags.push({ prefix, repository: image.repository, tag: image.tag, pullPolicy: fromRegistry ? "IfNotPresent" : "Never" });
|
|
177
|
+
}
|
|
136
178
|
|
|
137
179
|
const remoteChart = `/tmp/${def.release}-helm-${def.namespace}`;
|
|
138
|
-
ctx.log("helm upgrade", { release: def.release, namespace: def.namespace,
|
|
139
|
-
await ssh(`rm -rf ${shq(remoteChart)}`);
|
|
140
|
-
await
|
|
141
|
-
await ssh(helmUpgradeCommand({ remoteChart, release: def.release, namespace: def.namespace, values, set,
|
|
180
|
+
ctx.log("helm upgrade", { release: def.release, namespace: def.namespace, images: flags.map((f) => `${f.repository}:${f.tag}`).join(" ") });
|
|
181
|
+
await ssh.run(`rm -rf ${shq(remoteChart)}`);
|
|
182
|
+
await ssh.copy(chartDir, remoteChart);
|
|
183
|
+
await ssh.run(helmUpgradeCommand({ remoteChart, release: def.release, namespace: def.namespace, values, set, images: flags }));
|
|
142
184
|
|
|
143
|
-
|
|
144
|
-
|
|
185
|
+
for (const d of deployments) {
|
|
186
|
+
const target = d.includes("/") ? d : `deploy/${d}`;
|
|
187
|
+
ctx.log("waiting for rollout", { deployment: target });
|
|
188
|
+
await ssh.run(`kubectl -n ${shq(def.namespace)} rollout status ${shq(target)} --timeout=${rolloutTimeout}s`);
|
|
189
|
+
}
|
|
145
190
|
|
|
146
|
-
if (
|
|
147
|
-
return { release: def.release, namespace: def.namespace, image:
|
|
191
|
+
if (health) await httpHealthCheck(ctx, health.url, { attempts: health.attempts, delaySeconds: health.delaySeconds });
|
|
192
|
+
return { release: def.release, namespace: def.namespace, image: `${flags[0]!.repository}:${flags[0]!.tag}` };
|
|
148
193
|
},
|
|
149
194
|
},
|
|
150
195
|
) as never;
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
// steps.imageBuild — build a container image from a `Source` with kaniko
|
|
2
|
-
//
|
|
3
|
-
// No registry push: the tarball travels as a file ref and a deploy step imports it on the host.
|
|
1
|
+
// steps.imageBuild — build a container image from a `Source` with kaniko: to an image tarball that
|
|
2
|
+
// travels as a file ref (a deploy imports it on the host), or pushed to a registry (`push: true`).
|
|
4
3
|
// Ported from v1's olve-lib.sh (olve_kaniko_build + build.sh's layer cache); footguns kept below.
|
|
5
|
-
import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
4
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
6
5
|
import { join } from "node:path";
|
|
7
6
|
import type { Ctx, RunInfo } from "../ctx/types.ts";
|
|
8
7
|
import type { FileRef } from "../core/models.ts";
|
|
9
8
|
import { makeStep, type LibraryStep, type LiteralName, type Step } from "../core/steps.ts";
|
|
10
|
-
import {
|
|
9
|
+
import { sha256File } from "../util/hash.ts";
|
|
10
|
+
import { dirEntries, writeTar } from "../util/tar.ts";
|
|
11
|
+
import { BuiltImage, PushedImage, Source } from "./models.ts";
|
|
11
12
|
|
|
12
13
|
/** kaniko's debug image (it has a busybox shell for debugging; the executor itself needs none). */
|
|
13
14
|
export const KANIKO_IMAGE = "gcr.io/kaniko-project/executor:debug";
|
|
@@ -26,8 +27,15 @@ export interface ImageBuildDef<N extends string> {
|
|
|
26
27
|
name: LiteralName<N>;
|
|
27
28
|
/** What the step does, shown in the UI. Required; may be empty. */
|
|
28
29
|
description: string;
|
|
29
|
-
/**
|
|
30
|
+
/**
|
|
31
|
+
* Image name, e.g. `olve-pipelines-v2`; with `push`, the registry repository to push to
|
|
32
|
+
* (`registry.ovea.pro/app/api`). The tag is the run's version (see `imageTag`) unless `contentTag`.
|
|
33
|
+
*/
|
|
30
34
|
repository: string;
|
|
35
|
+
/** Build context: a directory of the source tree (default: the root). */
|
|
36
|
+
context?: string;
|
|
37
|
+
/** Dockerfile path, relative to the context (default `Dockerfile`). */
|
|
38
|
+
dockerfile?: string;
|
|
31
39
|
/**
|
|
32
40
|
* Registry repository for kaniko's layer cache (`--cache=true --cache-repo=…`), so unchanged
|
|
33
41
|
* layers (restore, apt, npm, …) are reused. kaniko pushes cache layers even with `--no-push`.
|
|
@@ -56,6 +64,25 @@ export interface ImageBuildDef<N extends string> {
|
|
|
56
64
|
retries?: number;
|
|
57
65
|
}
|
|
58
66
|
|
|
67
|
+
/** Tarball mode (default): output `BuiltImage` with the image tar as a file. */
|
|
68
|
+
export interface ImageBuildTarballDef<N extends string> extends ImageBuildDef<N> {
|
|
69
|
+
push?: false;
|
|
70
|
+
contentTag?: never;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Registry mode: push to `repository` and output `PushedImage`, for images too big to travel as a
|
|
75
|
+
* tarball (the step's scratch space is ~1 GiB).
|
|
76
|
+
*/
|
|
77
|
+
export interface ImageBuildPushDef<N extends string> extends ImageBuildDef<N> {
|
|
78
|
+
push: true;
|
|
79
|
+
/**
|
|
80
|
+
* Tag the image `src-<hash>`, the hash of the build context's content, and skip the build when the
|
|
81
|
+
* registry already has that tag: the image is rebuilt only when its context changes.
|
|
82
|
+
*/
|
|
83
|
+
contentTag?: boolean;
|
|
84
|
+
}
|
|
85
|
+
|
|
59
86
|
/**
|
|
60
87
|
* Default `registryMap`: pull-through caches for mcr.microsoft.com and Docker Hub. kaniko under gVisor
|
|
61
88
|
* can stall forever pulling from mcr.microsoft.com directly.
|
|
@@ -132,28 +159,91 @@ export function qualifyRepository(repository: string): string {
|
|
|
132
159
|
return `docker.io/${repository}`;
|
|
133
160
|
}
|
|
134
161
|
|
|
162
|
+
/** A relative path inside the source tree (no `..`, not absolute). */
|
|
163
|
+
function checkRelative(what: string, path: string): string {
|
|
164
|
+
const clean = path.replace(/^\.\/+/, "").replace(/\/+$/, "");
|
|
165
|
+
if (clean.startsWith("/") || clean.split("/").includes("..")) throw new Error(`imageBuild ${what} '${path}' must be a relative path inside the source tree`);
|
|
166
|
+
return clean === "." ? "" : clean;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** The `src-<12 hex>` tag of a directory: the sha256 of its deterministic tar (as `ctx.output.dir` writes it). */
|
|
170
|
+
export function contentTagOf(dir: string, tmpDir: string): string {
|
|
171
|
+
mkdirSync(tmpDir, { recursive: true });
|
|
172
|
+
const work = mkdtempSync(join(tmpDir, "content-tag-"));
|
|
173
|
+
try {
|
|
174
|
+
const tar = join(work, "context.tar");
|
|
175
|
+
writeTar(tar, dirEntries(dir));
|
|
176
|
+
return `src-${sha256File(tar).slice(0, 12)}`;
|
|
177
|
+
} finally {
|
|
178
|
+
rmSync(work, { recursive: true, force: true });
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const MANIFEST_TYPES = [
|
|
183
|
+
"application/vnd.oci.image.index.v1+json",
|
|
184
|
+
"application/vnd.oci.image.manifest.v1+json",
|
|
185
|
+
"application/vnd.docker.distribution.manifest.list.v2+json",
|
|
186
|
+
"application/vnd.docker.distribution.manifest.v2+json",
|
|
187
|
+
].join(", ");
|
|
188
|
+
|
|
189
|
+
/** The digest of `repository:tag` in its registry (anonymous `HEAD …/manifests/<tag>`), or undefined if it has none. */
|
|
190
|
+
export async function registryDigest(http: Ctx["http"], repository: string, tag: string): Promise<string | undefined> {
|
|
191
|
+
const [host, ...path] = repository.split("/");
|
|
192
|
+
const res = await http.fetch(`https://${host}/v2/${path.join("/")}/manifests/${tag}`, { method: "HEAD", headers: { Accept: MANIFEST_TYPES } });
|
|
193
|
+
await res.body?.cancel().catch(() => {});
|
|
194
|
+
if (res.status === 404) return undefined;
|
|
195
|
+
if (!res.ok) throw new Error(`registry lookup of ${repository}:${tag} answered HTTP ${res.status}`);
|
|
196
|
+
const digest = res.headers.get("docker-content-digest") ?? "";
|
|
197
|
+
return /^sha256:[0-9a-f]{64}$/.test(digest) ? digest : undefined;
|
|
198
|
+
}
|
|
199
|
+
|
|
135
200
|
/**
|
|
136
|
-
* Builds
|
|
137
|
-
*
|
|
201
|
+
* Builds an image with kaniko from `<context>/<dockerfile>` in the source tree (default: `Dockerfile`
|
|
202
|
+
* at the root). Default: into an image tarball, output `BuiltImage { repository, tag, digest, tar }`.
|
|
203
|
+
* With `push: true`: pushed to `repository`, output `PushedImage { repository, tag, digest }`.
|
|
138
204
|
*/
|
|
139
|
-
export function imageBuild<const N extends string>(def:
|
|
205
|
+
export function imageBuild<const N extends string>(def: ImageBuildTarballDef<N>): Step<N, ImageBuildInput, typeof BuiltImage, never> & LibraryStep;
|
|
206
|
+
export function imageBuild<const N extends string>(def: ImageBuildPushDef<N>): Step<N, ImageBuildInput, typeof PushedImage, never> & LibraryStep;
|
|
207
|
+
export function imageBuild<const N extends string>(def: ImageBuildTarballDef<N> | ImageBuildPushDef<N>): AnyImageBuild {
|
|
140
208
|
const registryMap = def.registryMap ?? DEFAULT_REGISTRY_MAP;
|
|
141
209
|
const mapped = Object.keys(registryMap).length > 0;
|
|
210
|
+
const push = def.push === true;
|
|
211
|
+
const contentTag = push && def.contentTag === true;
|
|
212
|
+
const context = checkRelative("context", def.context ?? "");
|
|
213
|
+
const dockerfile = checkRelative("dockerfile", def.dockerfile ?? "Dockerfile");
|
|
214
|
+
if (push && !/[.:]|^localhost$/.test(def.repository.split("/")[0]!)) throw new Error(`imageBuild '${def.name}': push needs a repository with a registry host, got '${def.repository}'`);
|
|
142
215
|
return makeStep(
|
|
143
216
|
"imageBuild",
|
|
144
|
-
{ name: def.name, description: def.description, image: KANIKO_IMAGE, input, output: BuiltImage, secrets: [], timeoutSeconds: def.timeoutSeconds, retries: def.retries ?? DEFAULT_RETRIES },
|
|
217
|
+
{ name: def.name, description: def.description, image: KANIKO_IMAGE, input, output: push ? PushedImage : BuiltImage, secrets: [], timeoutSeconds: def.timeoutSeconds, retries: def.retries ?? DEFAULT_RETRIES },
|
|
145
218
|
{
|
|
146
219
|
manifestConfig: () => ({
|
|
147
220
|
repository: def.repository,
|
|
221
|
+
...(context ? { context } : {}),
|
|
222
|
+
...(dockerfile !== "Dockerfile" ? { dockerfile } : {}),
|
|
223
|
+
...(push ? { push: true } : {}),
|
|
224
|
+
...(contentTag ? { contentTag: true } : {}),
|
|
148
225
|
...(def.cacheRepo ? { cacheRepo: def.cacheRepo } : {}),
|
|
149
226
|
...(mapped ? { registryMap: { ...registryMap } } : {}),
|
|
150
227
|
}),
|
|
151
228
|
async run({ source }: { source: { tree: FileRef } }, ctx: Ctx) {
|
|
152
|
-
const tag = imageTag(ctx.run);
|
|
153
|
-
const caches = await reachableCaches(ctx, { registryMap, cacheRepo: def.cacheRepo });
|
|
154
229
|
const tree = await ctx.files.fetch(source.tree);
|
|
230
|
+
const contextDir = context ? join(tree, context) : tree;
|
|
231
|
+
if (!existsSync(contextDir) || !statSync(contextDir).isDirectory()) throw new Error(`build context '${context}' not found in the source tree`);
|
|
232
|
+
if (!existsSync(join(contextDir, dockerfile))) throw new Error(`${dockerfile} not found in the build context${context ? ` '${context}'` : ""}`);
|
|
233
|
+
|
|
234
|
+
const tag = contentTag ? contentTagOf(contextDir, ctx.tmpDir) : imageTag(ctx.run);
|
|
235
|
+
if (contentTag) {
|
|
236
|
+
const digest = await registryDigest(ctx.http, def.repository, tag);
|
|
237
|
+
if (digest) {
|
|
238
|
+
ctx.log("image already in the registry", { image: `${def.repository}:${tag}`, digest });
|
|
239
|
+
return { repository: def.repository, tag, digest };
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
const caches = await reachableCaches(ctx, { registryMap, cacheRepo: def.cacheRepo });
|
|
155
244
|
await ctx.exec(["mkdir", "-p", KANIKO_CONTEXT]);
|
|
156
245
|
await ctx.exec(["cp", "-r", `${tree}/.`, KANIKO_CONTEXT]);
|
|
246
|
+
const kanikoContext = context ? `${KANIKO_CONTEXT}/${context}` : KANIKO_CONTEXT;
|
|
157
247
|
|
|
158
248
|
// Outputs go to the harness's scratch dir (/olve/tmp, an emptyDir mount kaniko leaves alone;
|
|
159
249
|
// kaniko images have no /tmp), then into the step's files.
|
|
@@ -161,16 +251,16 @@ export function imageBuild<const N extends string>(def: ImageBuildDef<N>): Step<
|
|
|
161
251
|
const out = mkdtempSync(join(ctx.tmpDir, "image-build-"));
|
|
162
252
|
const tarPath = join(out, "image.tar");
|
|
163
253
|
const digestPath = join(out, "digest");
|
|
254
|
+
const output = push ? [] : ["--no-push", `--tar-path=${tarPath}`];
|
|
164
255
|
// No --single-snapshot: it is incompatible with multi-stage Dockerfiles whose later stages need
|
|
165
256
|
// the build context intact. --skip-push-permission-check: with --no-push kaniko otherwise checks
|
|
166
257
|
// the cache repo at startup and exits if it is down; later cache lookup and push errors are
|
|
167
258
|
// only warnings.
|
|
168
259
|
await ctx.exec([
|
|
169
260
|
"/kaniko/executor",
|
|
170
|
-
`--context=${
|
|
171
|
-
`--dockerfile=${
|
|
172
|
-
|
|
173
|
-
`--tar-path=${tarPath}`,
|
|
261
|
+
`--context=${kanikoContext}`,
|
|
262
|
+
`--dockerfile=${kanikoContext}/${dockerfile}`,
|
|
263
|
+
...output,
|
|
174
264
|
`--digest-file=${digestPath}`,
|
|
175
265
|
`--destination=${def.repository}:${tag}`,
|
|
176
266
|
...(caches.cacheRepo ? ["--cache=true", `--cache-repo=${caches.cacheRepo}`, "--skip-push-permission-check"] : []),
|
|
@@ -184,6 +274,10 @@ export function imageBuild<const N extends string>(def: ImageBuildDef<N>): Step<
|
|
|
184
274
|
throw new Error(`kaniko did not write the image digest to ${digestPath}`);
|
|
185
275
|
}
|
|
186
276
|
if (!/^sha256:[0-9a-f]{64}$/.test(digest)) throw new Error(`kaniko wrote an unexpected digest: '${digest}'`);
|
|
277
|
+
if (push) {
|
|
278
|
+
ctx.log("image pushed", { image: `${def.repository}:${tag}`, digest });
|
|
279
|
+
return { repository: def.repository, tag, digest };
|
|
280
|
+
}
|
|
187
281
|
const tar = await ctx.output.file(tarPath, { name: "image.tar" });
|
|
188
282
|
rmSync(tarPath); // the copy in the step's files is the one that counts; don't hold the image twice in /olve
|
|
189
283
|
ctx.log("image built", { image: `${def.repository}:${tag}`, digest });
|
|
@@ -192,3 +286,5 @@ export function imageBuild<const N extends string>(def: ImageBuildDef<N>): Step<
|
|
|
192
286
|
},
|
|
193
287
|
) as never;
|
|
194
288
|
}
|
|
289
|
+
|
|
290
|
+
type AnyImageBuild = Step<string, ImageBuildInput, typeof BuiltImage | typeof PushedImage, never> & LibraryStep;
|
package/src/library/index.ts
CHANGED
|
@@ -3,11 +3,13 @@ import { helmDeployOverSsh } from "./helm-deploy-ssh.ts";
|
|
|
3
3
|
import { imageBuild } from "./image-build.ts";
|
|
4
4
|
import { script } from "./script.ts";
|
|
5
5
|
|
|
6
|
-
export { Source, BuiltImage, HelmRelease, FailureContext } from "./models.ts";
|
|
6
|
+
export { Source, BuiltImage, PushedImage, HelmRelease, FailureContext } from "./models.ts";
|
|
7
7
|
export { inputEnvName, type RunValue, type ScriptDef } from "./script.ts";
|
|
8
8
|
export type { GitHubSourceDef } from "./github-source.ts";
|
|
9
|
-
export type { ImageBuildDef } from "./image-build.ts";
|
|
9
|
+
export type { ImageBuildDef, ImageBuildTarballDef, ImageBuildPushDef } from "./image-build.ts";
|
|
10
10
|
export type { HelmDeployOverSshDef } from "./helm-deploy-ssh.ts";
|
|
11
|
+
export { sshSession, shq, type SshSession } from "./ssh.ts";
|
|
12
|
+
export { serviceUrl, checkSpa } from "./checks.ts";
|
|
11
13
|
export { httpHealthCheck, HTTP_HEALTH_DEFAULTS, type HttpHealthCheckOptions } from "./http-health.ts";
|
|
12
14
|
|
|
13
15
|
/** SDK-provided steps (step + handler pairs with typed config). */
|
package/src/library/models.ts
CHANGED
|
@@ -14,6 +14,15 @@ export const BuiltImage = model(
|
|
|
14
14
|
z.object({ repository: z.string(), tag: z.string(), digest: z.string(), tar: fileRef() }),
|
|
15
15
|
);
|
|
16
16
|
|
|
17
|
+
/**
|
|
18
|
+
* A container image pushed to a registry by a step (`imageBuild` with `push: true`), pulled from there
|
|
19
|
+
* by the deploy. `repository` includes the registry host.
|
|
20
|
+
*/
|
|
21
|
+
export const PushedImage = model(
|
|
22
|
+
"PushedImage",
|
|
23
|
+
z.object({ repository: z.string(), tag: z.string(), digest: z.string() }),
|
|
24
|
+
);
|
|
25
|
+
|
|
17
26
|
/** A Helm release that was installed or upgraded. */
|
|
18
27
|
export const HelmRelease = model(
|
|
19
28
|
"HelmRelease",
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// sshSession — run commands on a host over SSH from a handler (alpine image: installs openssh-client).
|
|
2
|
+
import { chmodSync, mkdirSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import type { Ctx } from "../ctx/types.ts";
|
|
5
|
+
|
|
6
|
+
const SAFE = /^[A-Za-z0-9_\/.:=@%+,-]+$/;
|
|
7
|
+
/** POSIX shell quoting (safe words are left bare, so logs stay readable). */
|
|
8
|
+
export function shq(s: string): string {
|
|
9
|
+
return SAFE.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface SshSession {
|
|
13
|
+
readonly host: string;
|
|
14
|
+
/** Runs a remote command; a non-zero exit throws. */
|
|
15
|
+
run(remote: string): Promise<void>;
|
|
16
|
+
/** Runs a remote command and returns its stdout. */
|
|
17
|
+
text(remote: string): Promise<string>;
|
|
18
|
+
/** `scp -r` a local file or directory to a remote path. */
|
|
19
|
+
copy(local: string, remote: string): Promise<void>;
|
|
20
|
+
/** The ssh invocation as a shell command, for pipes and redirects: `ctx.sh(`${s.shell("cat > f")} < ${file}`)`. */
|
|
21
|
+
shell(remote: string): string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Opens an SSH "session" to `host` (`user@host`) with a private key: installs openssh-client (apk),
|
|
26
|
+
* writes the key to `<tmpDir>/ssh/id`. The host key is not verified: hosts are reached over the tailnet.
|
|
27
|
+
*/
|
|
28
|
+
export async function sshSession(ctx: Pick<Ctx, "exec" | "sh" | "tmpDir">, opts: { host: string; key: string }): Promise<SshSession> {
|
|
29
|
+
await ctx.exec(["apk", "add", "--no-cache", "openssh-client"]);
|
|
30
|
+
const dir = join(ctx.tmpDir, "ssh");
|
|
31
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
32
|
+
const keyFile = join(dir, "id");
|
|
33
|
+
// OpenSSH rejects a key without a trailing newline.
|
|
34
|
+
writeFileSync(keyFile, opts.key.endsWith("\n") ? opts.key : opts.key + "\n", { mode: 0o600 });
|
|
35
|
+
chmodSync(keyFile, 0o600);
|
|
36
|
+
const sshOpts = ["-i", keyFile, "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", "-o", "BatchMode=yes"];
|
|
37
|
+
const argv = (remote: string) => ["ssh", ...sshOpts, opts.host, remote];
|
|
38
|
+
return {
|
|
39
|
+
host: opts.host,
|
|
40
|
+
run: (remote) => ctx.exec(argv(remote)),
|
|
41
|
+
text: (remote) => ctx.sh.text(argv(remote).map(shq).join(" ")),
|
|
42
|
+
copy: (local, remote) => ctx.exec(["scp", ...sshOpts, "-r", local, `${opts.host}:${remote}`]),
|
|
43
|
+
shell: (remote) => argv(remote).map(shq).join(" "),
|
|
44
|
+
};
|
|
45
|
+
}
|
package/src/testing/index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// @
|
|
1
|
+
// @olivervea/pipelines/testing — run a handler under `bun test` through the real harness path
|
|
2
2
|
// (validate input → handler → strict output validation), with a real ctx you can override.
|
|
3
3
|
import { mkdtempSync } from "node:fs";
|
|
4
4
|
import { tmpdir } from "node:os";
|
package/src/util/handler-hash.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { sha256Hex } from "./hash.ts";
|
|
|
5
5
|
export interface HarnessInfo {
|
|
6
6
|
/** Bun version (runtime compiled into the binary). */
|
|
7
7
|
bun: string;
|
|
8
|
-
/** @
|
|
8
|
+
/** @olivervea/pipelines package version. */
|
|
9
9
|
sdk: string;
|
|
10
10
|
/** `sha256:` hash over the SDK runtime sources and the zod version. */
|
|
11
11
|
sdkHash: string;
|