@olivervea/pipelines 2.0.0-beta.21 → 2.0.0-beta.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -18
- package/package.json +1 -1
- package/src/build/build.ts +4 -18
- package/src/build/hashes.ts +4 -18
- package/src/build/project.ts +8 -55
- package/src/builder/pipeline.ts +2 -2
- package/src/core/handlers.ts +3 -3
- package/src/core/steps.ts +3 -4
- package/src/describe/describe.ts +6 -36
- package/src/dev/cli.ts +0 -6
- package/src/harness/exit.ts +1 -1
- package/src/harness/index.ts +0 -1
- package/src/harness/main.ts +4 -27
- package/src/testing/index.ts +2 -2
package/README.md
CHANGED
|
@@ -59,7 +59,7 @@ skips when the version exists, or when the pipeline secret `NPM_TOKEN` is unset.
|
|
|
59
59
|
|
|
60
60
|
## Authoring
|
|
61
61
|
|
|
62
|
-
A pipeline directory (the binding path, `.pipelines/` by default) is an ordinary Bun package. `main` in `package.json` names the entry point, which default-exports the pipeline
|
|
62
|
+
A pipeline directory (the binding path, `.pipelines/` by default) is an ordinary Bun package. `main` in `package.json` is required and names the entry point, which default-exports the pipeline; every other name is yours. A step carries its implementation, `implement(step({ … }), fn)`, and is imported and staged like any value.
|
|
63
63
|
|
|
64
64
|
Describe evaluates the entry point and everything it imports, so top-level code must be deterministic: no env, time or network. Scripts are embedded with `import s from "./x.sh" with { type: "file" }`.
|
|
65
65
|
|
|
@@ -89,6 +89,7 @@ export const deployBeta = step({
|
|
|
89
89
|
|
|
90
90
|
Rules:
|
|
91
91
|
|
|
92
|
+
- `step()` is the contract. A pipeline stages `implement(step, fn)`; a staged step without an implementation is a describe error.
|
|
92
93
|
- Models are **nominal**: two models with the same shape are not interchangeable.
|
|
93
94
|
- There is **one schema per model name**. Two different schemas registered under the same name is a describe error.
|
|
94
95
|
- **Outputs are validated strictly**: unknown keys are rejected at any depth, not stripped.
|
|
@@ -168,13 +169,13 @@ Library steps carry their own implementation and image; they take typed settings
|
|
|
168
169
|
| `sshSession(ctx, { host, key })` | Installs openssh-client when the image has no ssh (apk or apt-get), writes the key to `<tmpDir>/ssh/id` (trailing newline, mode 600; host key not verified) and returns `{ run(remote), text(remote), copy(local, remote), shell(remote) }`; `shell` is the quoted ssh command for pipes (`ctx.sh(`printf '%s' "$V" \| ${s.shell("kubectl apply -f -")}`)`). `helmDeployOverSsh` uses it. |
|
|
169
170
|
|
|
170
171
|
```ts
|
|
171
|
-
import { httpHealthCheck, implement } from "@olivervea/pipelines";
|
|
172
|
-
export
|
|
172
|
+
import { httpHealthCheck, implement, step } from "@olivervea/pipelines";
|
|
173
|
+
export const smoke = implement(step({ name: "smoke", description: "", image: "alpine:3.22" }), async (_, ctx) => { await httpHealthCheck(ctx, "http://app.ns.svc.cluster.local/api/health", { attempts: 10 }); });
|
|
173
174
|
```
|
|
174
175
|
|
|
175
176
|
### Reusable steps
|
|
176
177
|
|
|
177
|
-
A step several pipelines use is a function returning `implement(step(…), handler)`, staged directly
|
|
178
|
+
A step several pipelines use is a function returning `implement(step(…), handler)`, staged directly. The library steps are written this way, so a module in a repo or a package of its own works the same.
|
|
178
179
|
|
|
179
180
|
```ts
|
|
180
181
|
import { implement, resolveConfigRefs, serviceUrl, step, HelmRelease, type Configurable, type LiteralName } from "@olivervea/pipelines";
|
|
@@ -189,17 +190,15 @@ export function smokeTest<const N extends string>(def: { name: LiteralName<N>; d
|
|
|
189
190
|
|
|
190
191
|
- **`kind`**: a label, `custom` unless the step names one. Prefix a package's kinds (`acme/smokeTest`).
|
|
191
192
|
- **`settings`**: what the step was set up with, recorded as the step's manifest `config` (so part of its version). JSON values and config references; the step receives every entry its settings reference.
|
|
192
|
-
- `implement` refuses a step that already carries an implementation
|
|
193
|
+
- `implement` refuses a step that already carries an implementation.
|
|
193
194
|
|
|
194
195
|
### Handlers
|
|
195
196
|
|
|
196
197
|
```ts
|
|
197
|
-
|
|
198
|
-
import
|
|
199
|
-
import { deployBeta } from "../pipeline";
|
|
200
|
-
import deployScript from "../scripts/deploy.sh" with { type: "file" };
|
|
198
|
+
import { implement, step } from "@olivervea/pipelines";
|
|
199
|
+
import deployScript from "./scripts/deploy.sh" with { type: "file" };
|
|
201
200
|
|
|
202
|
-
export
|
|
201
|
+
export const deployBeta = implement(step({ name: "deploy-beta", /* … */ }), async ({ artifact }, ctx) => {
|
|
203
202
|
await ctx.sh(deployScript, { env: { IMAGE_TAR: await ctx.files.fetch(artifact.image), CHART_DIR: await ctx.files.fetch(artifact.chart) } });
|
|
204
203
|
await ctx.retry({ attempts: 5, delaySeconds: 5 }, () => ctx.http.expectOk("https://pipelines-beta.ovea.pro/api/health"));
|
|
205
204
|
return { release: "olve-pipelines", version: artifact.version, url: "https://pipelines-beta.ovea.pro" }; // or ctx.skip("reason")
|
|
@@ -251,13 +250,13 @@ export default pipeline({ name: "Olve.Pipelines", description: "…", input: M.D
|
|
|
251
250
|
|
|
252
251
|
```ts
|
|
253
252
|
import { testHandler, testFileRef } from "@olivervea/pipelines/testing";
|
|
254
|
-
import deployBeta from "../
|
|
253
|
+
import { deployBeta } from "../steps/deploy-beta";
|
|
255
254
|
|
|
256
255
|
const r = await testHandler(deployBeta, { artifact }, { secrets: { SSH_PRIVATE_KEY: "k" }, config: { target: { host: "h" } }, ctx: { http: fakeHttp }, run: { number: 412 } });
|
|
257
256
|
expect(r).toMatchObject({ status: "succeeded", output: { release: "olve-pipelines" } }); // r.filesDir, r.logs, r.exitCode
|
|
258
257
|
```
|
|
259
258
|
|
|
260
|
-
`testHandler` uses the same validation path as a real run. Invalid input gives `status: "failed"` with `kind: "input"`; an invalid return value gives `kind: "output"`. Any part of `ctx` can be replaced; whatever you don't replace is the real implementation. `testHandler` takes any implementation: a
|
|
259
|
+
`testHandler` uses the same validation path as a real run. Invalid input gives `status: "failed"` with `kind: "input"`; an invalid return value gives `kind: "output"`. Any part of `ctx` can be replaced; whatever you don't replace is the real implementation. `testHandler` takes any implementation: a custom step, a library step or a reusable step.
|
|
261
260
|
|
|
262
261
|
## Harness contract (service ⇄ step binary)
|
|
263
262
|
|
|
@@ -337,7 +336,7 @@ For `map-trigger`, `output` is the pipeline input after it has been parsed by th
|
|
|
337
336
|
| 65 | `input.json` missing, not JSON, or invalid for the step (or event), including its `$config` | failed / `input` |
|
|
338
337
|
| 66 | the return value failed strict validation, or an own file ref is missing or changed | failed / `output` |
|
|
339
338
|
| 74 | the `result.tar.gz` upload failed after retries. Used only when the step itself succeeded; otherwise the step's own code wins. | yes (printed) |
|
|
340
|
-
| 78 | the pipeline is invalid (describe problems, or a
|
|
339
|
+
| 78 | the pipeline is invalid (describe problems, or a step without an implementation) | none |
|
|
341
340
|
| 130 / 143 | the handler failed after SIGINT / SIGTERM (cancellation, `activeDeadlineSeconds`) | failed / `handler` |
|
|
342
341
|
|
|
343
342
|
### Result upload (`OLVE_RESULT_URL`)
|
|
@@ -368,7 +367,7 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
|
|
|
368
367
|
- **`apiVersion`** is `"2.0"`.
|
|
369
368
|
- **`harness`** is `{ bun, sdk, sdkHash }`.
|
|
370
369
|
- **`models[name]`** is `{ output, input? }`.
|
|
371
|
-
- **Step `kind`** is `custom`
|
|
370
|
+
- **Step `kind`** is `custom` by default, or what a reusable step names (`imageBuild`, `composeDeploy`, a package's `acme/…`). To the service it is a label: any non-empty kind is accepted, so a new library or package step needs no service change.
|
|
372
371
|
- `output` is the `z.output` view, with `additionalProperties: false` on every object. It validates outputs, the pool and trigger results.
|
|
373
372
|
- `input` is the `z.input` view (defaults optional). Use it for trigger forms and manual input. It is emitted only when it differs from `output` beyond `additionalProperties`.
|
|
374
373
|
- **Step `input`** maps each input key to a mapping:
|
|
@@ -379,7 +378,7 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
|
|
|
379
378
|
An object whose only key is `ref` (with a string value) or `const` is a tag. Describe never emits a nested object with exactly that key set. Model-matched inputs are emitted already resolved (`{"ref": "build"}`).
|
|
380
379
|
- **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from}`, `from` being `{kind: "person"}`, `{kind: "callback"}` or `{kind: "poll", url, valuePath, intervalSeconds}`. Those keys are not in `input` or `inputTypes`; the service merges the answers into `input.json`.
|
|
381
380
|
- **`inputTypes`** declares each input key: `{model, optional}`, or `{schema: {output, input?}}` for a plain schema.
|
|
382
|
-
- **`codeHash`** is the build's identity: `sha256(canonical {bundle, harness})`. `bundle` hashes the entry point
|
|
381
|
+
- **`codeHash`** is the build's identity: `sha256(canonical {bundle, harness})`. `bundle` hashes the entry point bundled with everything it imports, with `@olivervea/pipelines` external. It is tree-shaken and includes embedded scripts and dependencies, so any change to code in use changes it, and with it the manifest, which makes a new pipeline version. A step version covers what the step declares: image, `inputTypes`/model versions, `output` model version, `secrets`, `configEntries` (and those entries' schemas), `input` mappings and `config`.
|
|
383
382
|
- **`stages`** is an array whose items are either `{steps: [Step…]}` (parallel, with a barrier) or `{when: {field: [values]}, stages: [...]}`.
|
|
384
383
|
- **`triggers`**:
|
|
385
384
|
- `{kind: "push", name: "push"}`;
|
|
@@ -395,7 +394,7 @@ Describe runs these checks; reconcile should re-run the manifest-level ones:
|
|
|
395
394
|
|
|
396
395
|
- step name format and reserved names (`input`, `failure`), and duplicate step names;
|
|
397
396
|
- one schema per model name, and JSON-representable models;
|
|
398
|
-
-
|
|
397
|
+
- a staged step without an implementation, and an empty step kind;
|
|
399
398
|
- availability, ambiguity, maybe-available as required, `optional` without a producer, plain schema without a mapper;
|
|
400
399
|
- mapper source analysis and recording, unknown fields in refs, mapper keys that are missing or extra;
|
|
401
400
|
- `when` on an unknown or non-literal field, or with an invalid value;
|
|
@@ -411,8 +410,8 @@ olve-build <pipelineDir> --out <dir> [--target bun-linux-x64|bun-linux-x64-musl]
|
|
|
411
410
|
olve-build <pipelineDir> --hash-only # print the code hash (diagnostics)
|
|
412
411
|
```
|
|
413
412
|
|
|
414
|
-
1.
|
|
415
|
-
2. Generate `<pipelineDir>/.olve/gen/
|
|
413
|
+
1. Read the entry point from `main` in `package.json` (a missing `main`, or a `main` whose file is missing, is an error) and hash the build (see `codeHash`).
|
|
414
|
+
2. Generate `<pipelineDir>/.olve/gen/entry.ts`, which imports the pipeline and calls `runHarness` with it, the code hash and the harness info. Add `.olve/` to `.gitignore`.
|
|
416
415
|
3. Run `bun build --compile --target=bun-linux-x64 --no-compile-autoload-dotenv --no-compile-autoload-bunfig --bytecode --format=esm` → `<out>/step`. The output is never minified, because mapper source analysis needs real source.
|
|
417
416
|
4. Check that `PT_INTERP` is `/lib64/ld-linux-x86-64.so.2`. On alpine, Bun silently produces a musl binary; the check fails it.
|
|
418
417
|
5. Run `<out>/step describe` → `<out>/manifest.json`. A describe problem is a build failure with exit 78. **A step with no implementation is an error.**
|
package/package.json
CHANGED
package/src/build/build.ts
CHANGED
|
@@ -48,7 +48,7 @@ export interface BuildResult {
|
|
|
48
48
|
harness: HarnessInfo;
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
/** Writes `<dir>/.olve/gen/
|
|
51
|
+
/** Writes `<dir>/.olve/gen/entry.ts`, the entry point of the step binary. */
|
|
52
52
|
export function generateEntry(
|
|
53
53
|
project: Project,
|
|
54
54
|
code: string,
|
|
@@ -56,28 +56,14 @@ export function generateEntry(
|
|
|
56
56
|
): string {
|
|
57
57
|
const gen = join(project.dir, '.olve', 'gen');
|
|
58
58
|
mkdirSync(gen, { recursive: true });
|
|
59
|
-
const registry = project.handlers
|
|
60
|
-
.map(
|
|
61
|
-
([name, file]) =>
|
|
62
|
-
` ${JSON.stringify(name)}: () => import(${JSON.stringify(relPosix(gen, file))}),`,
|
|
63
|
-
)
|
|
64
|
-
.join('\n');
|
|
65
|
-
writeFileSync(
|
|
66
|
-
join(gen, 'handlers.ts'),
|
|
67
|
-
`// GENERATED by olve-build — do not edit.\n` +
|
|
68
|
-
`import type { HandlerRegistry, HarnessInfo } from "@olivervea/pipelines/harness";\n\n` +
|
|
69
|
-
`export const handlers: HandlerRegistry = {\n${registry}\n};\n\n` +
|
|
70
|
-
`export const codeHash = ${JSON.stringify(code)};\n\n` +
|
|
71
|
-
`export const harness: HarnessInfo = ${JSON.stringify(harness, null, 2)};\n`,
|
|
72
|
-
);
|
|
73
59
|
const entry = join(gen, 'entry.ts');
|
|
60
|
+
const config = `{ pipeline, codeHash: ${JSON.stringify(code)}, harness: ${JSON.stringify(harness)} }`;
|
|
74
61
|
writeFileSync(
|
|
75
62
|
entry,
|
|
76
63
|
`// GENERATED by olve-build — do not edit. Entry point of the step binary.\n` +
|
|
77
64
|
`import pipeline from ${JSON.stringify(relPosix(gen, project.pipelineFile))};\n` +
|
|
78
|
-
`import { runHarness } from "@olivervea/pipelines/harness";\n` +
|
|
79
|
-
`
|
|
80
|
-
`process.exit(await runHarness({ pipeline, handlers, codeHash, harness }, process.argv.slice(2)));\n`,
|
|
65
|
+
`import { runHarness } from "@olivervea/pipelines/harness";\n\n` +
|
|
66
|
+
`process.exit(await runHarness(${config}, process.argv.slice(2)));\n`,
|
|
81
67
|
);
|
|
82
68
|
return entry;
|
|
83
69
|
}
|
package/src/build/hashes.ts
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
// The build's code hash: everything the pipeline runs, bundled once with the SDK external (the
|
|
2
2
|
// harness identity stands for it).
|
|
3
|
-
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
4
|
-
import { join } from 'node:path';
|
|
5
3
|
import type { BunPlugin } from 'bun';
|
|
6
4
|
import { sha256Hex } from '../util/hash.ts';
|
|
7
5
|
import { composeCodeHash, type HarnessInfo } from '../util/code-hash.ts';
|
|
8
|
-
import {
|
|
6
|
+
import type { Project } from './project.ts';
|
|
9
7
|
|
|
10
8
|
const sdkExternal: BunPlugin = {
|
|
11
9
|
name: 'olve-code-hash-external',
|
|
@@ -17,7 +15,8 @@ const sdkExternal: BunPlugin = {
|
|
|
17
15
|
},
|
|
18
16
|
};
|
|
19
17
|
|
|
20
|
-
async function bundleHash(project: Project
|
|
18
|
+
async function bundleHash(project: Project): Promise<string> {
|
|
19
|
+
const entry = project.pipelineFile;
|
|
21
20
|
const r = await Bun.build({
|
|
22
21
|
entrypoints: [entry],
|
|
23
22
|
target: 'bun',
|
|
@@ -53,18 +52,5 @@ async function bundleHash(project: Project, entry: string): Promise<string> {
|
|
|
53
52
|
}
|
|
54
53
|
|
|
55
54
|
export async function codeHash(project: Project, harness: HarnessInfo): Promise<string> {
|
|
56
|
-
|
|
57
|
-
mkdirSync(gen, { recursive: true });
|
|
58
|
-
const entry = join(gen, 'code.ts');
|
|
59
|
-
const files = [project.pipelineFile, ...project.handlers.map(([, file]) => file)];
|
|
60
|
-
writeFileSync(
|
|
61
|
-
entry,
|
|
62
|
-
files
|
|
63
|
-
.map(
|
|
64
|
-
(f, i) =>
|
|
65
|
-
`export { default as m${i} } from ${JSON.stringify(relPosix(gen, f))};\n`,
|
|
66
|
-
)
|
|
67
|
-
.join(''),
|
|
68
|
-
);
|
|
69
|
-
return composeCodeHash(await bundleHash(project, entry), harness);
|
|
55
|
+
return composeCodeHash(await bundleHash(project), harness);
|
|
70
56
|
}
|
package/src/build/project.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
// Layout of a pipeline directory and the identity of the SDK building it.
|
|
2
2
|
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
|
3
3
|
import { dirname, join, relative, resolve } from 'node:path';
|
|
4
|
-
import { isImplementation } from '../core/handlers.ts';
|
|
5
4
|
import { sha256Hex } from '../util/hash.ts';
|
|
6
5
|
import type { HarnessInfo } from '../util/code-hash.ts';
|
|
7
6
|
|
|
@@ -10,8 +9,6 @@ export const SDK_ROOT = resolve(import.meta.dir, '../..');
|
|
|
10
9
|
export interface Project {
|
|
11
10
|
dir: string;
|
|
12
11
|
pipelineFile: string;
|
|
13
|
-
/** step name → absolute handler file, sorted by step name. The step is the one the file's `implement(step, …)` names. */
|
|
14
|
-
handlers: [string, string][];
|
|
15
12
|
}
|
|
16
13
|
|
|
17
14
|
export class ProjectError extends Error {
|
|
@@ -22,36 +19,23 @@ export async function loadProject(dirArg: string): Promise<Project> {
|
|
|
22
19
|
const dir = resolve(dirArg);
|
|
23
20
|
if (!existsSync(dir) || !statSync(dir).isDirectory())
|
|
24
21
|
throw new ProjectError(`${dirArg} is not a directory`);
|
|
25
|
-
|
|
26
|
-
const hdir = join(dir, 'handlers');
|
|
27
|
-
const files = existsSync(hdir)
|
|
28
|
-
? readdirSync(hdir)
|
|
29
|
-
.filter(
|
|
30
|
-
(f) =>
|
|
31
|
-
f.endsWith('.ts') &&
|
|
32
|
-
!f.endsWith('.test.ts') &&
|
|
33
|
-
!f.endsWith('.d.ts') &&
|
|
34
|
-
statSync(join(hdir, f)).isFile(),
|
|
35
|
-
)
|
|
36
|
-
.sort()
|
|
37
|
-
: [];
|
|
38
|
-
return {
|
|
39
|
-
dir,
|
|
40
|
-
pipelineFile,
|
|
41
|
-
handlers: await resolveHandlers(hdir, files),
|
|
42
|
-
};
|
|
22
|
+
return { dir, pipelineFile: entryOf(dirArg, dir) };
|
|
43
23
|
}
|
|
44
24
|
|
|
45
|
-
/** The
|
|
25
|
+
/** The file `main` in the package's package.json names. */
|
|
46
26
|
function entryOf(dirArg: string, dir: string): string {
|
|
47
27
|
const pkg = join(dir, 'package.json');
|
|
48
28
|
const main = existsSync(pkg)
|
|
49
29
|
? (JSON.parse(readFileSync(pkg, 'utf8')) as { main?: string }).main
|
|
50
30
|
: undefined;
|
|
51
|
-
|
|
31
|
+
if (!main)
|
|
32
|
+
throw new ProjectError(
|
|
33
|
+
`${join(dirArg, 'package.json')} must name the entry point in \`main\` (a file that default-exports pipeline({...}))`,
|
|
34
|
+
);
|
|
35
|
+
const entry = join(dir, main);
|
|
52
36
|
if (!existsSync(entry))
|
|
53
37
|
throw new ProjectError(
|
|
54
|
-
`${join(dirArg, main
|
|
38
|
+
`${join(dirArg, main)} not found (it must default-export pipeline({...}))`,
|
|
55
39
|
);
|
|
56
40
|
return entry;
|
|
57
41
|
}
|
|
@@ -66,37 +50,6 @@ export async function loadPipeline(project: Project): Promise<unknown> {
|
|
|
66
50
|
return mod.default;
|
|
67
51
|
}
|
|
68
52
|
|
|
69
|
-
/** Pairs each handler file with the step its default export implements (file names don't matter). */
|
|
70
|
-
async function resolveHandlers(
|
|
71
|
-
hdir: string,
|
|
72
|
-
files: string[],
|
|
73
|
-
): Promise<[string, string][]> {
|
|
74
|
-
const byStep = new Map<string, string>();
|
|
75
|
-
for (const f of files) {
|
|
76
|
-
const file = join(hdir, f);
|
|
77
|
-
let impl: unknown;
|
|
78
|
-
try {
|
|
79
|
-
impl = ((await import(file)) as { default?: unknown }).default;
|
|
80
|
-
} catch (e) {
|
|
81
|
-
throw new ProjectError(
|
|
82
|
-
`handlers/${f} failed to load: ${(e as Error)?.message ?? String(e)}`,
|
|
83
|
-
);
|
|
84
|
-
}
|
|
85
|
-
if (!isImplementation(impl))
|
|
86
|
-
throw new ProjectError(
|
|
87
|
-
`handlers/${f} must default-export implement(<step>, …); put shared code in a subfolder such as handlers/lib/`,
|
|
88
|
-
);
|
|
89
|
-
const step = (impl.step as { name: string }).name;
|
|
90
|
-
const other = byStep.get(step);
|
|
91
|
-
if (other)
|
|
92
|
-
throw new ProjectError(
|
|
93
|
-
`handlers/${other.slice(hdir.length + 1)} and handlers/${f} both implement step '${step}'`,
|
|
94
|
-
);
|
|
95
|
-
byStep.set(step, file);
|
|
96
|
-
}
|
|
97
|
-
return [...byStep].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
|
98
|
-
}
|
|
99
|
-
|
|
100
53
|
function listFiles(dir: string): string[] {
|
|
101
54
|
const out: string[] = [];
|
|
102
55
|
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
package/src/builder/pipeline.ts
CHANGED
|
@@ -314,7 +314,7 @@ export function pipelineData(p: unknown): PipelineData {
|
|
|
314
314
|
: undefined;
|
|
315
315
|
if (!d)
|
|
316
316
|
throw new TypeError(
|
|
317
|
-
'not a pipeline (
|
|
317
|
+
'not a pipeline (the entry point must default-export pipeline({...})...)',
|
|
318
318
|
);
|
|
319
319
|
return d as PipelineData;
|
|
320
320
|
}
|
|
@@ -553,7 +553,7 @@ export class Pipeline<In extends AnyModel, A extends AvailMap> {
|
|
|
553
553
|
}
|
|
554
554
|
}
|
|
555
555
|
|
|
556
|
-
/** Start a pipeline definition.
|
|
556
|
+
/** Start a pipeline definition. The entry point must default-export the result. */
|
|
557
557
|
export function pipeline<In extends AnyModel>(
|
|
558
558
|
def: PipelineDef<In>,
|
|
559
559
|
): Pipeline<In, InitialAvail<In>> {
|
package/src/core/handlers.ts
CHANGED
|
@@ -30,7 +30,7 @@ export type Handler<S extends AnyStep> = (
|
|
|
30
30
|
ctx: Ctx<SecretsOf<S>, ConfigOf<S>>,
|
|
31
31
|
) => Awaitable<HandlerResult<S>>;
|
|
32
32
|
|
|
33
|
-
/** A step with its handler. It is itself a step: stage it
|
|
33
|
+
/** A step with its handler. It is itself a step: stage it like any value. */
|
|
34
34
|
export type Implementation<S extends AnyStep = AnyStep> = S & {
|
|
35
35
|
readonly [ImplementationBrand]: true;
|
|
36
36
|
/** The step as defined. */
|
|
@@ -48,8 +48,8 @@ type NoExtraFields<R, T> = R extends Skipped | void
|
|
|
48
48
|
};
|
|
49
49
|
|
|
50
50
|
/**
|
|
51
|
-
* Implement a step. Input, ctx and return types come from the step definition alone.
|
|
52
|
-
*
|
|
51
|
+
* Implement a step. Input, ctx and return types come from the step definition alone. Stage the
|
|
52
|
+
* result directly; a reusable step gives itself a `kind`.
|
|
53
53
|
*/
|
|
54
54
|
export function implement<S extends AnyStep, R extends HandlerResult<S>>(
|
|
55
55
|
step: S &
|
package/src/core/steps.ts
CHANGED
|
@@ -28,8 +28,7 @@ export type InferEntry<E> =
|
|
|
28
28
|
export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
|
|
29
29
|
|
|
30
30
|
/**
|
|
31
|
-
* A label for what a step is, shown in the UI. `custom`
|
|
32
|
-
* `handlers/`; a step that carries its implementation (`implement(step, …)` staged directly) names its
|
|
31
|
+
* A label for what a step is, shown in the UI. `custom` is the default; a reusable step names its
|
|
33
32
|
* kind, e.g. `gitHubSource`. Packages prefix theirs (`acme/deploy`).
|
|
34
33
|
*/
|
|
35
34
|
export type StepKind = string;
|
|
@@ -142,8 +141,8 @@ export function isStep(v: unknown): v is AnyStep {
|
|
|
142
141
|
}
|
|
143
142
|
|
|
144
143
|
/**
|
|
145
|
-
* Define a step: a contract (name, image, inputs, output, secrets).
|
|
146
|
-
*
|
|
144
|
+
* Define a step: a contract (name, image, inputs, output, secrets). `implement(step, fn)` gives it
|
|
145
|
+
* its handler; that is what a pipeline stages.
|
|
147
146
|
*/
|
|
148
147
|
export function step<
|
|
149
148
|
const N extends string,
|
package/src/describe/describe.ts
CHANGED
|
@@ -40,8 +40,6 @@ export class DescribeError extends Error {
|
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
export interface DescribeOptions {
|
|
43
|
-
/** Step names that have a handler (a `handlers/*.ts` default-exporting `implement(step, …)`). */
|
|
44
|
-
handlerNames: Iterable<string>;
|
|
45
43
|
/** Hash of the build (from the build; includes the harness identity). */
|
|
46
44
|
codeHash?: string;
|
|
47
45
|
harness: HarnessInfo;
|
|
@@ -126,7 +124,6 @@ function literalValues(field: JsonSchema | undefined): unknown[] | undefined {
|
|
|
126
124
|
export function describePipeline(pipeline: unknown, opts: DescribeOptions): ManifestT {
|
|
127
125
|
const data = pipelineData(pipeline);
|
|
128
126
|
const problems: string[] = [...data.problems];
|
|
129
|
-
const handlerNames = new Set(opts.handlerNames);
|
|
130
127
|
|
|
131
128
|
// ── models ──
|
|
132
129
|
const models = new Map<string, { schemas: ModelSchemasT; key: string }>();
|
|
@@ -229,20 +226,10 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
229
226
|
const prev = placements.get(s.name) ?? [];
|
|
230
227
|
if (prev.some((p) => !exclusive(p.cases, cases)))
|
|
231
228
|
problems.push(`duplicate step name '${s.name}'`);
|
|
232
|
-
else if (prev.
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
)
|
|
237
|
-
)
|
|
238
|
-
problems.push(
|
|
239
|
-
`step '${s.name}': two different steps with handler files share the name across select cases; handlers pair by name, so use one step or different names`,
|
|
240
|
-
);
|
|
241
|
-
if (prev.some((p) => (p.step.output?.name ?? null) !== (s.output?.name ?? null)))
|
|
242
|
-
problems.push(
|
|
243
|
-
`step '${s.name}': its select cases must give it the same output model`,
|
|
244
|
-
);
|
|
245
|
-
}
|
|
229
|
+
else if (prev.some((p) => (p.step.output?.name ?? null) !== (s.output?.name ?? null)))
|
|
230
|
+
problems.push(
|
|
231
|
+
`step '${s.name}': its select cases must give it the same output model`,
|
|
232
|
+
);
|
|
246
233
|
placements.set(s.name, [...prev, { step: s, cases }]);
|
|
247
234
|
if (!steps.has(s.name)) steps.set(s.name, s);
|
|
248
235
|
};
|
|
@@ -455,10 +442,8 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
455
442
|
`${at}: config reference '${ref}': config '${entry}' has no field '${path.join('.')}'`,
|
|
456
443
|
);
|
|
457
444
|
}
|
|
458
|
-
if (!carried
|
|
459
|
-
problems.push(
|
|
460
|
-
`${at} has no implementation (implement(${camel(s.name)}, …) and stage that, or add a handlers/*.ts default-exporting it)`,
|
|
461
|
-
);
|
|
445
|
+
if (!carried)
|
|
446
|
+
problems.push(`${at} has no implementation (stage implement(step({...}), fn))`);
|
|
462
447
|
if (typeof s.description !== 'string')
|
|
463
448
|
problems.push(`${at}: description must be a string (it may be empty)`);
|
|
464
449
|
if (
|
|
@@ -718,17 +703,6 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
718
703
|
})
|
|
719
704
|
.filter((x) => x !== undefined);
|
|
720
705
|
|
|
721
|
-
// ── handlers without a step ──
|
|
722
|
-
for (const h of handlerNames) {
|
|
723
|
-
const s = steps.get(h);
|
|
724
|
-
if (!s)
|
|
725
|
-
problems.push(`a handler implements step '${h}', which is not in the pipeline`);
|
|
726
|
-
else if (isImplementation(s))
|
|
727
|
-
problems.push(
|
|
728
|
-
`a handler implements '${h}', a ${s.kind} step that carries its own implementation`,
|
|
729
|
-
);
|
|
730
|
-
}
|
|
731
|
-
|
|
732
706
|
// ── triggers ──
|
|
733
707
|
const triggerNames = new Set<string>();
|
|
734
708
|
const triggers: TriggerT[] = data.triggers.map((t) => {
|
|
@@ -930,7 +904,3 @@ interface CaseOf {
|
|
|
930
904
|
value: string;
|
|
931
905
|
cond: string;
|
|
932
906
|
}
|
|
933
|
-
|
|
934
|
-
function camel(name: string): string {
|
|
935
|
-
return name.replace(/-([a-z0-9])/g, (_, c: string) => c.toUpperCase());
|
|
936
|
-
}
|
package/src/dev/cli.ts
CHANGED
|
@@ -36,12 +36,6 @@ export async function devConfig(
|
|
|
36
36
|
const harness = harnessInfo();
|
|
37
37
|
return {
|
|
38
38
|
pipeline,
|
|
39
|
-
handlers: Object.fromEntries(
|
|
40
|
-
project.handlers.map(([name, file]) => [
|
|
41
|
-
name,
|
|
42
|
-
() => import(file) as Promise<{ default: unknown }>,
|
|
43
|
-
]),
|
|
44
|
-
),
|
|
45
39
|
...(withHashes ? { codeHash: await codeHash(project, harness) } : {}),
|
|
46
40
|
harness,
|
|
47
41
|
};
|
package/src/harness/exit.ts
CHANGED
|
@@ -11,7 +11,7 @@ export const EXIT = {
|
|
|
11
11
|
outputInvalid: 66,
|
|
12
12
|
/** Uploading result.tar.gz to OLVE_RESULT_URL failed (after retries). The envelope was still printed. */
|
|
13
13
|
uploadFailed: 74,
|
|
14
|
-
/** The pipeline is invalid: describe problems,
|
|
14
|
+
/** The pipeline is invalid: describe problems, a step without an implementation. */
|
|
15
15
|
configInvalid: 78,
|
|
16
16
|
/** Cancelled by SIGINT / SIGTERM (128 + signal). */
|
|
17
17
|
sigint: 130,
|
package/src/harness/index.ts
CHANGED
package/src/harness/main.ts
CHANGED
|
@@ -31,15 +31,9 @@ import {
|
|
|
31
31
|
} from './result.ts';
|
|
32
32
|
import { formatIssues, parseStrict } from './validate.ts';
|
|
33
33
|
|
|
34
|
-
/** step name → lazy import of `handlers/<name>.ts`. Literal import() paths so Bun bundles them. */
|
|
35
|
-
export type HandlerRegistry = Readonly<
|
|
36
|
-
Record<string, () => Promise<{ default: unknown }>>
|
|
37
|
-
>;
|
|
38
|
-
|
|
39
34
|
export interface HarnessConfig {
|
|
40
35
|
/** Default export of the entry point. */
|
|
41
36
|
pipeline: unknown;
|
|
42
|
-
handlers: HandlerRegistry;
|
|
43
37
|
/** Hash of the build (from the build; includes the harness identity). */
|
|
44
38
|
codeHash?: string;
|
|
45
39
|
harness: HarnessInfo;
|
|
@@ -129,26 +123,10 @@ export function cancellation(): { signal: AbortSignal; dispose(): void } {
|
|
|
129
123
|
};
|
|
130
124
|
}
|
|
131
125
|
|
|
132
|
-
/**
|
|
133
|
-
export
|
|
134
|
-
step: AnyStep,
|
|
135
|
-
handlers: HandlerRegistry,
|
|
136
|
-
): Promise<StepFunction | string> {
|
|
126
|
+
/** The function `step` carries, or what is wrong. */
|
|
127
|
+
export function resolveStepFunction(step: AnyStep): StepFunction | string {
|
|
137
128
|
if (isImplementation(step)) return step.handler as unknown as StepFunction;
|
|
138
|
-
|
|
139
|
-
if (!load) return `step '${step.name}' has no handler`;
|
|
140
|
-
let mod: { default: unknown };
|
|
141
|
-
try {
|
|
142
|
-
mod = await load();
|
|
143
|
-
} catch (e) {
|
|
144
|
-
return `the handler of step '${step.name}' failed to load: ${(e as Error)?.stack ?? String(e)}`;
|
|
145
|
-
}
|
|
146
|
-
const impl = mod.default;
|
|
147
|
-
if (!isImplementation(impl))
|
|
148
|
-
return `the handler of step '${step.name}' must default-export implement(<step '${step.name}'>, …)`;
|
|
149
|
-
if (impl.step !== step)
|
|
150
|
-
return `the handler of step '${step.name}' implements a different step object (it must be implement(<the '${step.name}' step from pipeline.ts>, …))`;
|
|
151
|
-
return impl.handler as unknown as StepFunction;
|
|
129
|
+
return `step '${step.name}' has no implementation (stage implement(step({...}), fn))`;
|
|
152
130
|
}
|
|
153
131
|
|
|
154
132
|
/** Writes output.json, prints the sentinel block and uploads result.tar.gz when OLVE_RESULT_URL is set. */
|
|
@@ -189,7 +167,6 @@ async function describeMode(
|
|
|
189
167
|
try {
|
|
190
168
|
const text = manifestText(
|
|
191
169
|
describePipeline(cfg.pipeline, {
|
|
192
|
-
handlerNames: Object.keys(cfg.handlers),
|
|
193
170
|
codeHash: cfg.codeHash,
|
|
194
171
|
harness: cfg.harness,
|
|
195
172
|
}),
|
|
@@ -214,7 +191,7 @@ async function runMode(
|
|
|
214
191
|
const data = pipelineData(cfg.pipeline);
|
|
215
192
|
const step = findStep(data, name, selectedCases());
|
|
216
193
|
if (typeof step === 'string') return (io.stderr(step), EXIT.usage);
|
|
217
|
-
const fn =
|
|
194
|
+
const fn = resolveStepFunction(step);
|
|
218
195
|
if (typeof fn === 'string') return (io.stderr(fn), EXIT.configInvalid);
|
|
219
196
|
|
|
220
197
|
const p = paths();
|
package/src/testing/index.ts
CHANGED
|
@@ -46,7 +46,7 @@ export type TestResult<S extends AnyStep> = {
|
|
|
46
46
|
);
|
|
47
47
|
|
|
48
48
|
/**
|
|
49
|
-
* Runs
|
|
49
|
+
* Runs a step that carries its implementation with `input`.
|
|
50
50
|
* Input is validated like `input.json`, the result like a real run: an invalid output gives
|
|
51
51
|
* `{ status: "failed", error: { kind: "output" } }`, a throwing handler `kind: "handler"`.
|
|
52
52
|
*/
|
|
@@ -57,7 +57,7 @@ export async function testHandler<S extends AnyStep>(
|
|
|
57
57
|
): Promise<TestResult<S>> {
|
|
58
58
|
if (!isImplementation(target))
|
|
59
59
|
throw new TypeError(
|
|
60
|
-
|
|
60
|
+
'testHandler expects implement(…): a step that carries its implementation',
|
|
61
61
|
);
|
|
62
62
|
const step: AnyStep = target.step;
|
|
63
63
|
const fn = target.handler as unknown as StepFunction;
|