@olivervea/pipelines 2.0.0-beta.18 → 2.0.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -2
- package/src/assets.d.ts +2 -2
- package/src/build/build.ts +111 -43
- package/src/build/cli.ts +28 -21
- package/src/build/elf.ts +8 -7
- package/src/build/hashes.ts +61 -28
- package/src/build/project.ts +69 -30
- package/src/builder/pipeline.ts +274 -102
- package/src/builder/refs.ts +12 -6
- package/src/core/brands.ts +6 -6
- package/src/core/config-refs.ts +46 -20
- package/src/core/config.ts +110 -33
- package/src/core/external.ts +30 -12
- package/src/core/handlers.ts +54 -14
- package/src/core/models.ts +29 -14
- package/src/core/origin.ts +7 -4
- package/src/core/secrets.ts +2 -2
- package/src/core/steps.ts +39 -23
- package/src/core/types.ts +5 -1
- package/src/core/webhooks.ts +21 -7
- package/src/ctx/files.ts +111 -40
- package/src/ctx/index.ts +75 -34
- package/src/ctx/sh.ts +104 -44
- package/src/ctx/types.ts +4 -4
- package/src/describe/describe.ts +550 -196
- package/src/describe/manifest-schema.ts +210 -80
- package/src/describe/mapper-source.ts +249 -104
- package/src/describe/record.ts +67 -37
- package/src/dev/cli.ts +64 -34
- package/src/harness/env.ts +33 -27
- package/src/harness/execute.ts +55 -29
- package/src/harness/index.ts +21 -8
- package/src/harness/main.ts +197 -74
- package/src/harness/result.ts +38 -22
- package/src/harness/validate.ts +62 -19
- package/src/index.ts +123 -14
- package/src/library/checks.ts +11 -6
- package/src/library/compose-deploy.ts +40 -23
- package/src/library/github-source.ts +69 -39
- package/src/library/helm-deploy-ssh.ts +213 -76
- package/src/library/helm-deploy.ts +263 -85
- package/src/library/http-health.ts +7 -5
- package/src/library/image-build.ts +293 -117
- package/src/library/index.ts +47 -15
- package/src/library/models.ts +17 -9
- package/src/library/ssh.ts +31 -14
- package/src/testing/index.ts +63 -24
- package/src/util/canonical.ts +7 -4
- package/src/util/handler-hash.ts +15 -5
- package/src/util/hash.ts +5 -5
- package/src/util/tar.ts +74 -37
package/src/describe/describe.ts
CHANGED
|
@@ -1,27 +1,42 @@
|
|
|
1
1
|
// `describe`: evaluates the contracts (models + pipeline, never handlers) into the manifest and runs
|
|
2
2
|
// every describe-time check. All problems are collected and reported together.
|
|
3
|
-
import { z } from
|
|
4
|
-
import { CONFIG_NAME, isConfigEntry, SECRET_REF_KIND } from
|
|
5
|
-
import { CONFIG_CASE_KEY, CONFIG_REF_KEY } from
|
|
6
|
-
import { configRefPath, configRefTarget, isConfigRef, unionAt } from
|
|
7
|
-
import { isModel, isOptional, MODEL_NAME, type AnyModel } from
|
|
8
|
-
import { isExternal } from
|
|
9
|
-
import { SECRET_NAME, type Secret } from
|
|
10
|
-
import { RESERVED_STEP_NAMES, STEP_NAME, isStep, type AnyStep } from
|
|
11
|
-
import { isImplementation } from
|
|
12
|
-
import { recordConfigRefs } from
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
import {
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import { CONFIG_NAME, isConfigEntry, SECRET_REF_KIND } from '../core/config.ts';
|
|
5
|
+
import { CONFIG_CASE_KEY, CONFIG_REF_KEY } from '../core/config-refs.ts';
|
|
6
|
+
import { configRefPath, configRefTarget, isConfigRef, unionAt } from '../core/config.ts';
|
|
7
|
+
import { isModel, isOptional, MODEL_NAME, type AnyModel } from '../core/models.ts';
|
|
8
|
+
import { isExternal } from '../core/external.ts';
|
|
9
|
+
import { SECRET_NAME, type Secret } from '../core/secrets.ts';
|
|
10
|
+
import { RESERVED_STEP_NAMES, STEP_NAME, isStep, type AnyStep } from '../core/steps.ts';
|
|
11
|
+
import { isImplementation } from '../core/handlers.ts';
|
|
12
|
+
import { recordConfigRefs } from '../core/config-refs.ts';
|
|
13
|
+
import {
|
|
14
|
+
pipelineData,
|
|
15
|
+
type MapperFunction,
|
|
16
|
+
type StageNode,
|
|
17
|
+
type StageStepNode,
|
|
18
|
+
} from '../builder/pipeline.ts';
|
|
19
|
+
import { FailureContext } from '../library/models.ts';
|
|
20
|
+
import { canonicalize, canonicalJson } from '../util/canonical.ts';
|
|
21
|
+
import { libraryHandlerHash, type HarnessInfo } from '../util/handler-hash.ts';
|
|
22
|
+
import { analyzeMapperSource } from './mapper-source.ts';
|
|
23
|
+
import { MapperError, recordMapper, type Mapping } from './record.ts';
|
|
24
|
+
import {
|
|
25
|
+
API_VERSION,
|
|
26
|
+
Manifest,
|
|
27
|
+
type ManifestT,
|
|
28
|
+
type ModelSchemasT,
|
|
29
|
+
type Stage,
|
|
30
|
+
type StepDescriptorT,
|
|
31
|
+
type TriggerT,
|
|
32
|
+
} from './manifest-schema.ts';
|
|
20
33
|
|
|
21
34
|
export class DescribeError extends Error {
|
|
22
|
-
override name =
|
|
35
|
+
override name = 'DescribeError';
|
|
23
36
|
constructor(readonly problems: readonly string[]) {
|
|
24
|
-
super(
|
|
37
|
+
super(
|
|
38
|
+
`describe found ${problems.length} problem(s):\n${problems.map((p) => ` - ${p}`).join('\n')}`,
|
|
39
|
+
);
|
|
25
40
|
}
|
|
26
41
|
}
|
|
27
42
|
|
|
@@ -36,7 +51,7 @@ export interface DescribeOptions {
|
|
|
36
51
|
}
|
|
37
52
|
|
|
38
53
|
/** Root name used in failure-handler mappings (the service supplies FailureContext under it). */
|
|
39
|
-
export const FAILURE_ROOT =
|
|
54
|
+
export const FAILURE_ROOT = 'failure';
|
|
40
55
|
const RESERVED = new Set([...RESERVED_STEP_NAMES, FAILURE_ROOT]);
|
|
41
56
|
|
|
42
57
|
type JsonSchema = Record<string, unknown>;
|
|
@@ -47,9 +62,10 @@ interface Avail {
|
|
|
47
62
|
|
|
48
63
|
function stripStrict(v: unknown): unknown {
|
|
49
64
|
if (Array.isArray(v)) return v.map(stripStrict);
|
|
50
|
-
if (v !== null && typeof v ===
|
|
65
|
+
if (v !== null && typeof v === 'object') {
|
|
51
66
|
const out: Record<string, unknown> = {};
|
|
52
|
-
for (const [k, x] of Object.entries(v))
|
|
67
|
+
for (const [k, x] of Object.entries(v))
|
|
68
|
+
if (!(k === 'additionalProperties' && x === false)) out[k] = stripStrict(x);
|
|
53
69
|
return out;
|
|
54
70
|
}
|
|
55
71
|
return v;
|
|
@@ -57,38 +73,46 @@ function stripStrict(v: unknown): unknown {
|
|
|
57
73
|
|
|
58
74
|
/** JSON Schema views of a zod schema. Throws if it is not representable (dates, transforms, …). */
|
|
59
75
|
export function schemaViews(schema: z.ZodType): ModelSchemasT {
|
|
60
|
-
const view = (io:
|
|
61
|
-
const js = z.toJSONSchema(schema, { io, target:
|
|
76
|
+
const view = (io: 'input' | 'output') => {
|
|
77
|
+
const js = z.toJSONSchema(schema, { io, target: 'draft-2020-12' }) as JsonSchema;
|
|
62
78
|
delete js.$schema; // constant noise; apiVersion pins the dialect
|
|
63
79
|
return canonicalize(js) as JsonSchema;
|
|
64
80
|
};
|
|
65
|
-
const output = view(
|
|
66
|
-
const input = view(
|
|
81
|
+
const output = view('output');
|
|
82
|
+
const input = view('input');
|
|
67
83
|
const same = JSON.stringify(stripStrict(output)) === JSON.stringify(input);
|
|
68
84
|
return same ? { output } : { output, input };
|
|
69
85
|
}
|
|
70
86
|
|
|
71
87
|
/** Follows `segments` through a JSON Schema; undefined if the field doesn't exist. */
|
|
72
|
-
function resolveField(
|
|
88
|
+
function resolveField(
|
|
89
|
+
schema: JsonSchema | undefined,
|
|
90
|
+
segments: readonly string[],
|
|
91
|
+
): boolean {
|
|
73
92
|
if (!schema) return false;
|
|
74
93
|
if (segments.length === 0) return true;
|
|
75
94
|
const [seg, ...rest] = segments as [string, ...string[]];
|
|
76
|
-
if (Array.isArray(schema.anyOf))
|
|
77
|
-
|
|
78
|
-
if (Array.isArray(schema.
|
|
95
|
+
if (Array.isArray(schema.anyOf))
|
|
96
|
+
return (schema.anyOf as JsonSchema[]).some((b) => resolveField(b, segments));
|
|
97
|
+
if (Array.isArray(schema.oneOf))
|
|
98
|
+
return (schema.oneOf as JsonSchema[]).some((b) => resolveField(b, segments));
|
|
99
|
+
if (Array.isArray(schema.allOf))
|
|
100
|
+
return (schema.allOf as JsonSchema[]).some((b) => resolveField(b, segments));
|
|
79
101
|
const props = schema.properties as Record<string, JsonSchema> | undefined;
|
|
80
102
|
if (props && Object.hasOwn(props, seg)) return resolveField(props[seg], rest);
|
|
81
103
|
const extra = schema.additionalProperties;
|
|
82
|
-
if (extra && typeof extra ===
|
|
104
|
+
if (extra && typeof extra === 'object') return resolveField(extra as JsonSchema, rest);
|
|
83
105
|
// A schema without type/properties/anyOf accepts anything (z.unknown / z.any).
|
|
84
|
-
return Object.keys(schema).every(
|
|
106
|
+
return Object.keys(schema).every(
|
|
107
|
+
(k) => k === 'description' || k === 'title' || k === 'default',
|
|
108
|
+
);
|
|
85
109
|
}
|
|
86
110
|
|
|
87
111
|
function literalValues(field: JsonSchema | undefined): unknown[] | undefined {
|
|
88
112
|
if (!field) return undefined;
|
|
89
113
|
if (Array.isArray(field.enum)) return field.enum as unknown[];
|
|
90
114
|
if (field.const !== undefined) return [field.const];
|
|
91
|
-
if (field.type ===
|
|
115
|
+
if (field.type === 'boolean') return [true, false];
|
|
92
116
|
const alts = (field.anyOf ?? field.oneOf) as JsonSchema[] | undefined;
|
|
93
117
|
if (Array.isArray(alts)) {
|
|
94
118
|
const all: unknown[] = [];
|
|
@@ -110,49 +134,85 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
110
134
|
// ── models ──
|
|
111
135
|
const models = new Map<string, { schemas: ModelSchemasT; key: string }>();
|
|
112
136
|
const useModel = (m: AnyModel, at: string): string => {
|
|
113
|
-
if (!MODEL_NAME.test(m.name))
|
|
137
|
+
if (!MODEL_NAME.test(m.name))
|
|
138
|
+
problems.push(`${at}: model name '${m.name}' must match ${MODEL_NAME}`);
|
|
114
139
|
let schemas: ModelSchemasT;
|
|
115
140
|
try {
|
|
116
141
|
schemas = schemaViews(m.schema);
|
|
117
142
|
} catch (e) {
|
|
118
|
-
problems.push(
|
|
143
|
+
problems.push(
|
|
144
|
+
`${at}: model '${m.name}' is not representable as JSON Schema (${(e as Error).message})`,
|
|
145
|
+
);
|
|
119
146
|
return m.name;
|
|
120
147
|
}
|
|
121
148
|
const key = JSON.stringify(schemas);
|
|
122
149
|
const prev = models.get(m.name);
|
|
123
150
|
if (!prev) models.set(m.name, { schemas, key });
|
|
124
|
-
else if (prev.key !== key)
|
|
151
|
+
else if (prev.key !== key)
|
|
152
|
+
problems.push(
|
|
153
|
+
`two different schemas are registered under the model name '${m.name}' (one schema per model name)`,
|
|
154
|
+
);
|
|
125
155
|
return m.name;
|
|
126
156
|
};
|
|
127
157
|
|
|
128
158
|
// ── secrets ──
|
|
129
159
|
const secrets = new Map<string, string>();
|
|
130
160
|
const useSecret = (s: Secret, at: string): string => {
|
|
131
|
-
if (!SECRET_NAME.test(s.name))
|
|
161
|
+
if (!SECRET_NAME.test(s.name))
|
|
162
|
+
problems.push(`${at}: secret name '${s.name}' must match ${SECRET_NAME}`);
|
|
132
163
|
const prev = secrets.get(s.name);
|
|
133
164
|
if (prev === undefined) secrets.set(s.name, s.description);
|
|
134
|
-
else if (prev !== s.description)
|
|
165
|
+
else if (prev !== s.description)
|
|
166
|
+
problems.push(`secret '${s.name}' is declared twice with different descriptions`);
|
|
135
167
|
return s.name;
|
|
136
168
|
};
|
|
137
169
|
|
|
138
170
|
// ── config entries ──
|
|
139
|
-
const configs = new Map<
|
|
171
|
+
const configs = new Map<
|
|
172
|
+
string,
|
|
173
|
+
{
|
|
174
|
+
description: string;
|
|
175
|
+
required: boolean;
|
|
176
|
+
schema: JsonSchema;
|
|
177
|
+
key: string;
|
|
178
|
+
secretRefs: string[];
|
|
179
|
+
}
|
|
180
|
+
>();
|
|
140
181
|
const useConfig = (e: unknown, at: string): string | undefined => {
|
|
141
|
-
if (!isConfigEntry(e))
|
|
142
|
-
|
|
182
|
+
if (!isConfigEntry(e))
|
|
183
|
+
return void problems.push(
|
|
184
|
+
`${at}: config entries must be config("name", "description", schema)`,
|
|
185
|
+
);
|
|
186
|
+
if (!CONFIG_NAME.test(e.name))
|
|
187
|
+
problems.push(`${at}: config name '${e.name}' must match ${CONFIG_NAME}`);
|
|
143
188
|
let schema: JsonSchema;
|
|
144
189
|
try {
|
|
145
|
-
schema = z.toJSONSchema(e.schema, {
|
|
190
|
+
schema = z.toJSONSchema(e.schema, {
|
|
191
|
+
io: 'input',
|
|
192
|
+
target: 'draft-2020-12',
|
|
193
|
+
}) as JsonSchema;
|
|
146
194
|
delete schema.$schema;
|
|
147
195
|
schema = canonicalize(schema) as JsonSchema;
|
|
148
196
|
} catch (err) {
|
|
149
|
-
problems.push(
|
|
197
|
+
problems.push(
|
|
198
|
+
`${at}: config '${e.name}' is not representable as JSON Schema (${(err as Error).message})`,
|
|
199
|
+
);
|
|
150
200
|
return e.name;
|
|
151
201
|
}
|
|
152
202
|
const key = JSON.stringify([e.description, schema]);
|
|
153
203
|
const prev = configs.get(e.name);
|
|
154
|
-
if (!prev)
|
|
155
|
-
|
|
204
|
+
if (!prev)
|
|
205
|
+
configs.set(e.name, {
|
|
206
|
+
description: e.description,
|
|
207
|
+
required: !e.schema.safeParse(undefined).success,
|
|
208
|
+
schema,
|
|
209
|
+
key,
|
|
210
|
+
secretRefs: secretRefsOf(schema),
|
|
211
|
+
});
|
|
212
|
+
else if (prev.key !== key)
|
|
213
|
+
problems.push(
|
|
214
|
+
`two different config entries are named '${e.name}' (one description and schema per config name)`,
|
|
215
|
+
);
|
|
156
216
|
return e.name;
|
|
157
217
|
};
|
|
158
218
|
|
|
@@ -160,51 +220,96 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
160
220
|
const steps = new Map<string, AnyStep>();
|
|
161
221
|
const placements = new Map<string, { step: AnyStep; cases: readonly CaseOf[] }[]>();
|
|
162
222
|
// Two places are mutually exclusive when they sit in different cases of the same select.
|
|
163
|
-
const exclusive = (a: readonly CaseOf[], b: readonly CaseOf[]) =>
|
|
223
|
+
const exclusive = (a: readonly CaseOf[], b: readonly CaseOf[]) =>
|
|
224
|
+
a.some((x) => b.some((y) => y.select === x.select && y.value !== x.value));
|
|
164
225
|
const registerStep = (s: AnyStep, where: string, cases: readonly CaseOf[] = []) => {
|
|
165
|
-
if (typeof s.name !==
|
|
166
|
-
problems.push(
|
|
167
|
-
|
|
226
|
+
if (typeof s.name !== 'string' || !STEP_NAME.test(s.name))
|
|
227
|
+
problems.push(
|
|
228
|
+
`${where}: step name '${String(s.name)}' must be lower-case kebab (${STEP_NAME})`,
|
|
229
|
+
);
|
|
230
|
+
if (RESERVED.has(s.name))
|
|
231
|
+
problems.push(`${where}: '${s.name}' is reserved and cannot be a step name`);
|
|
168
232
|
const prev = placements.get(s.name) ?? [];
|
|
169
|
-
if (prev.some((p) => !exclusive(p.cases, cases)))
|
|
233
|
+
if (prev.some((p) => !exclusive(p.cases, cases)))
|
|
234
|
+
problems.push(`duplicate step name '${s.name}'`);
|
|
170
235
|
else if (prev.length) {
|
|
171
|
-
if (
|
|
172
|
-
|
|
236
|
+
if (
|
|
237
|
+
prev.some(
|
|
238
|
+
(p) => p.step !== s && !isImplementation(p.step) && !isImplementation(s),
|
|
239
|
+
)
|
|
240
|
+
)
|
|
241
|
+
problems.push(
|
|
242
|
+
`step '${s.name}': two different steps with handler files share the name across select cases; handlers pair by name, so use one step or different names`,
|
|
243
|
+
);
|
|
173
244
|
if (prev.some((p) => (p.step.output?.name ?? null) !== (s.output?.name ?? null)))
|
|
174
|
-
problems.push(
|
|
245
|
+
problems.push(
|
|
246
|
+
`step '${s.name}': its select cases must give it the same output model`,
|
|
247
|
+
);
|
|
175
248
|
}
|
|
176
249
|
placements.set(s.name, [...prev, { step: s, cases }]);
|
|
177
250
|
if (!steps.has(s.name)) steps.set(s.name, s);
|
|
178
251
|
};
|
|
179
252
|
|
|
180
253
|
const inputModel = data.input;
|
|
181
|
-
if (!isModel(inputModel))
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
254
|
+
if (!isModel(inputModel))
|
|
255
|
+
throw new DescribeError([
|
|
256
|
+
'pipeline input must be a model (model("Name", z.object({...})))',
|
|
257
|
+
]);
|
|
258
|
+
useModel(inputModel, 'pipeline input');
|
|
259
|
+
const inputOutputView = (): JsonSchema | undefined =>
|
|
260
|
+
models.get(inputModel.name)?.schemas.output;
|
|
261
|
+
const outputViewOf = (modelName: string | null): JsonSchema | undefined =>
|
|
262
|
+
modelName ? models.get(modelName)?.schemas.output : undefined;
|
|
185
263
|
|
|
186
264
|
// ── one step ──
|
|
187
|
-
const describeStep = (
|
|
265
|
+
const describeStep = (
|
|
266
|
+
node: StageStepNode,
|
|
267
|
+
avail: Map<string, Avail>,
|
|
268
|
+
failureHandler: boolean,
|
|
269
|
+
cases: readonly CaseOf[] = [],
|
|
270
|
+
): StepDescriptorT => {
|
|
188
271
|
const s = node.step;
|
|
189
272
|
const at = `step '${s.name}'`;
|
|
190
|
-
if (typeof s.image !==
|
|
273
|
+
if (typeof s.image !== 'string' || s.image.trim() === '')
|
|
274
|
+
problems.push(`${at}: image must be a non-empty string`);
|
|
191
275
|
|
|
192
|
-
const inputTypes: StepDescriptorT[
|
|
193
|
-
const externalInputs: NonNullable<StepDescriptorT[
|
|
276
|
+
const inputTypes: StepDescriptorT['inputTypes'] = {};
|
|
277
|
+
const externalInputs: NonNullable<StepDescriptorT['externalInputs']> = {};
|
|
194
278
|
const allEntries = Object.entries(s.input as Record<string, unknown>);
|
|
195
279
|
for (const [key, e] of allEntries) {
|
|
196
280
|
if (!isExternal(e)) continue;
|
|
197
|
-
if (failureHandler)
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
if (e.
|
|
202
|
-
problems.push(`${at} external input '${key}':
|
|
281
|
+
if (failureHandler)
|
|
282
|
+
problems.push(
|
|
283
|
+
`${at}: failure handlers run unattended and cannot wait for input ('${key}')`,
|
|
284
|
+
);
|
|
285
|
+
if (typeof e.title !== 'string' || e.title.trim() === '')
|
|
286
|
+
problems.push(`${at} external input '${key}': title must not be empty`);
|
|
287
|
+
if (e.from.length === 0)
|
|
288
|
+
problems.push(
|
|
289
|
+
`${at} external input '${key}': from must list at least one source`,
|
|
290
|
+
);
|
|
291
|
+
if (new Set(e.from.map((f) => f.kind)).size !== e.from.length)
|
|
292
|
+
problems.push(
|
|
293
|
+
`${at} external input '${key}': each source kind may be listed once`,
|
|
294
|
+
);
|
|
295
|
+
if (e.from.some((f) => f.kind === 'callback') && !/^[A-Za-z0-9_-]+$/.test(key))
|
|
296
|
+
problems.push(
|
|
297
|
+
`${at} external input '${key}': a callback input's key must be letters, digits, '_' or '-' (it is part of the URL)`,
|
|
298
|
+
);
|
|
203
299
|
for (const f of e.from) {
|
|
204
|
-
if (f.kind !==
|
|
205
|
-
if (!/^https?:\/\//.test(f.url))
|
|
206
|
-
|
|
207
|
-
|
|
300
|
+
if (f.kind !== 'poll') continue;
|
|
301
|
+
if (!/^https?:\/\//.test(f.url))
|
|
302
|
+
problems.push(
|
|
303
|
+
`${at} external input '${key}': poll url must be an absolute http(s) URL`,
|
|
304
|
+
);
|
|
305
|
+
if (!Number.isInteger(f.intervalSeconds) || f.intervalSeconds < 10)
|
|
306
|
+
problems.push(
|
|
307
|
+
`${at} external input '${key}': poll intervalSeconds must be an integer of at least 10`,
|
|
308
|
+
);
|
|
309
|
+
if (f.valuePath.trim() === '')
|
|
310
|
+
problems.push(
|
|
311
|
+
`${at} external input '${key}': poll valuePath must not be empty`,
|
|
312
|
+
);
|
|
208
313
|
}
|
|
209
314
|
externalInputs[key] = {
|
|
210
315
|
model: useModel(e.model, `${at} external input '${key}'`),
|
|
@@ -215,22 +320,34 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
215
320
|
}
|
|
216
321
|
const entries = allEntries.filter(([, e]) => !isExternal(e));
|
|
217
322
|
for (const [key, e] of entries) {
|
|
218
|
-
if (isOptional(e))
|
|
219
|
-
|
|
323
|
+
if (isOptional(e))
|
|
324
|
+
inputTypes[key] = {
|
|
325
|
+
model: useModel(e.model, `${at} input '${key}'`),
|
|
326
|
+
optional: true,
|
|
327
|
+
};
|
|
328
|
+
else if (isModel(e))
|
|
329
|
+
inputTypes[key] = { model: useModel(e, `${at} input '${key}'`), optional: false };
|
|
220
330
|
else if (e instanceof z.ZodType) {
|
|
221
331
|
try {
|
|
222
332
|
inputTypes[key] = { schema: schemaViews(e) };
|
|
223
333
|
} catch (err) {
|
|
224
|
-
problems.push(
|
|
334
|
+
problems.push(
|
|
335
|
+
`${at} input '${key}': schema is not representable as JSON Schema (${(err as Error).message})`,
|
|
336
|
+
);
|
|
225
337
|
}
|
|
226
|
-
} else
|
|
338
|
+
} else
|
|
339
|
+
problems.push(
|
|
340
|
+
`${at} input '${key}': must be a model, optional(model) or a zod schema`,
|
|
341
|
+
);
|
|
227
342
|
}
|
|
228
343
|
|
|
229
344
|
let input: Record<string, Mapping> = {};
|
|
230
345
|
if (failureHandler) {
|
|
231
346
|
for (const [key, e] of entries) {
|
|
232
347
|
if (e !== FailureContext && !(isModel(e) && e.name === FailureContext.name))
|
|
233
|
-
problems.push(
|
|
348
|
+
problems.push(
|
|
349
|
+
`${at}: a failure handler's inputs must all be FailureContext (input '${key}' is not)`,
|
|
350
|
+
);
|
|
234
351
|
input[key] = { ref: FAILURE_ROOT };
|
|
235
352
|
}
|
|
236
353
|
if (node.mapper) problems.push(`${at}: failure handlers take no mapper`);
|
|
@@ -241,32 +358,54 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
241
358
|
const optional = isOptional(e);
|
|
242
359
|
const m = optional ? e.model : isModel(e) ? e : undefined;
|
|
243
360
|
if (!m) {
|
|
244
|
-
problems.push(
|
|
361
|
+
problems.push(
|
|
362
|
+
`${at}: input '${key}' is a plain schema, not a model, so it cannot be matched by model; add a mapper ([step, from => ({ ... })])`,
|
|
363
|
+
);
|
|
245
364
|
continue;
|
|
246
365
|
}
|
|
247
366
|
const producers = [...avail].filter(([, a]) => a.model === m.name);
|
|
248
367
|
if (producers.length === 0) {
|
|
249
|
-
problems.push(
|
|
368
|
+
problems.push(
|
|
369
|
+
`${at}: ${m.name} (input '${key}') is not available at this stage${optional ? ' (optional() means maybe-skipped; there must still be an earlier producer)' : ''}`,
|
|
370
|
+
);
|
|
250
371
|
} else if (producers.length > 1) {
|
|
251
|
-
problems.push(
|
|
372
|
+
problems.push(
|
|
373
|
+
`${at}: ${m.name} (input '${key}') has several producers (${producers
|
|
374
|
+
.map(([n]) => n)
|
|
375
|
+
.sort()
|
|
376
|
+
.join(', ')}); use a mapper to pick one`,
|
|
377
|
+
);
|
|
252
378
|
} else {
|
|
253
379
|
const [producer, a] = producers[0]!;
|
|
254
|
-
if (a.maybe && !optional)
|
|
380
|
+
if (a.maybe && !optional)
|
|
381
|
+
problems.push(
|
|
382
|
+
`${at}: ${m.name} (input '${key}') is only maybe-available (produced inside a when block); declare it optional(${m.name})`,
|
|
383
|
+
);
|
|
255
384
|
input[key] = { ref: producer };
|
|
256
385
|
}
|
|
257
386
|
}
|
|
258
387
|
}
|
|
259
388
|
|
|
260
389
|
for (const key of Object.keys(s.input as Record<string, unknown>))
|
|
261
|
-
if (key.startsWith(
|
|
390
|
+
if (key.startsWith('$'))
|
|
391
|
+
problems.push(`${at}: input '${key}': keys starting with '$' are reserved`);
|
|
262
392
|
const secretNames = s.secrets.map((x) => useSecret(x, at)).sort();
|
|
263
|
-
const configNames = [
|
|
393
|
+
const configNames = [
|
|
394
|
+
...new Set(
|
|
395
|
+
(s.config ?? [])
|
|
396
|
+
.map((c) => useConfig(c, at))
|
|
397
|
+
.filter((n): n is string => n !== undefined),
|
|
398
|
+
),
|
|
399
|
+
].sort();
|
|
264
400
|
const carried = isImplementation(s);
|
|
265
401
|
let settings: Record<string, unknown> | undefined;
|
|
266
402
|
let settingsProblem: string | undefined;
|
|
267
403
|
if (s.settings !== undefined || carried) {
|
|
268
404
|
try {
|
|
269
|
-
settings = canonicalize(recordConfigRefs(s.settings ?? {})) as Record<
|
|
405
|
+
settings = canonicalize(recordConfigRefs(s.settings ?? {})) as Record<
|
|
406
|
+
string,
|
|
407
|
+
unknown
|
|
408
|
+
>;
|
|
270
409
|
} catch (e) {
|
|
271
410
|
settingsProblem = (e as Error).message;
|
|
272
411
|
settings = {};
|
|
@@ -276,60 +415,109 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
276
415
|
// A step whose settings reference parts of an entry needs the secrets those parts name; any other
|
|
277
416
|
// step receiving an entry needs every secret the entry names.
|
|
278
417
|
for (const name of configNames) {
|
|
279
|
-
const into = refs.filter((r) => r.ref.split(
|
|
280
|
-
const needed =
|
|
281
|
-
|
|
282
|
-
|
|
418
|
+
const into = refs.filter((r) => r.ref.split('.')[0] === name);
|
|
419
|
+
const needed =
|
|
420
|
+
into.length === 0
|
|
421
|
+
? (configs.get(name)?.secretRefs ?? [])
|
|
422
|
+
: [
|
|
423
|
+
...new Set(
|
|
424
|
+
into.flatMap((r) =>
|
|
425
|
+
secretRefsOf(
|
|
426
|
+
subSchemas(
|
|
427
|
+
configs.get(name)?.schema,
|
|
428
|
+
r.ref.split('.').slice(1),
|
|
429
|
+
r.case,
|
|
430
|
+
name,
|
|
431
|
+
),
|
|
432
|
+
),
|
|
433
|
+
),
|
|
434
|
+
),
|
|
435
|
+
].sort();
|
|
283
436
|
for (const ref of needed)
|
|
284
|
-
if (!secretNames.includes(ref))
|
|
437
|
+
if (!secretNames.includes(ref))
|
|
438
|
+
problems.push(
|
|
439
|
+
`${at}: config '${name}' references secret '${ref}' (secretRef), so the step must declare it in secrets`,
|
|
440
|
+
);
|
|
285
441
|
}
|
|
286
442
|
let handlerHash: string;
|
|
287
|
-
if (settingsProblem !== undefined)
|
|
288
|
-
|
|
289
|
-
if (
|
|
443
|
+
if (settingsProblem !== undefined)
|
|
444
|
+
problems.push(`${at}: invalid ${s.kind} settings (${settingsProblem})`);
|
|
445
|
+
if (typeof s.kind !== 'string' || s.kind === '')
|
|
446
|
+
problems.push(`${at}: kind must be a non-empty string`);
|
|
447
|
+
if (carried && s.kind === 'custom')
|
|
448
|
+
problems.push(
|
|
449
|
+
`${at} carries its implementation, so it needs a kind (step({ kind: "…", … })); a step with kind custom has its handler in handlers/`,
|
|
450
|
+
);
|
|
290
451
|
for (const { ref, case: kase } of refs) {
|
|
291
|
-
const [entry, ...path] = ref.split(
|
|
452
|
+
const [entry, ...path] = ref.split('.') as [string, ...string[]];
|
|
292
453
|
if (kase !== undefined && !cases.some((c) => c.cond === kase))
|
|
293
|
-
problems.push(
|
|
294
|
-
|
|
295
|
-
|
|
454
|
+
problems.push(
|
|
455
|
+
`${at}: config reference '${ref}' is narrowed to the select case '${kase}' and can only be used inside it`,
|
|
456
|
+
);
|
|
457
|
+
if (!configNames.includes(entry))
|
|
458
|
+
problems.push(
|
|
459
|
+
`${at}: config reference '${ref}': the step does not receive config entry '${entry}'`,
|
|
460
|
+
);
|
|
461
|
+
else if (!resolveField(configs.get(entry)?.schema, path))
|
|
462
|
+
problems.push(
|
|
463
|
+
`${at}: config reference '${ref}': config '${entry}' has no field '${path.join('.')}'`,
|
|
464
|
+
);
|
|
296
465
|
}
|
|
297
466
|
if (carried) {
|
|
298
467
|
const code = opts.kindHashes?.[s.kind];
|
|
299
|
-
if (opts.kindHashes && !code)
|
|
468
|
+
if (opts.kindHashes && !code)
|
|
469
|
+
problems.push(`${at}: no hash was computed for the code of kind '${s.kind}'`);
|
|
300
470
|
handlerHash = libraryHandlerHash(s.kind, settings, opts.harness, code);
|
|
301
471
|
} else {
|
|
302
|
-
if (!handlerNames.has(s.name))
|
|
303
|
-
|
|
304
|
-
|
|
472
|
+
if (!handlerNames.has(s.name))
|
|
473
|
+
problems.push(
|
|
474
|
+
`${at} has no handler (add a handlers/*.ts default-exporting implement(${camel(s.name)}, …))`,
|
|
475
|
+
);
|
|
476
|
+
handlerHash = opts.handlerHashes[s.name] ?? '';
|
|
477
|
+
if (handlerNames.has(s.name) && !handlerHash)
|
|
478
|
+
problems.push(`${at}: no handler hash was computed`);
|
|
305
479
|
}
|
|
306
|
-
if (typeof s.description !==
|
|
307
|
-
|
|
308
|
-
if (
|
|
480
|
+
if (typeof s.description !== 'string')
|
|
481
|
+
problems.push(`${at}: description must be a string (it may be empty)`);
|
|
482
|
+
if (
|
|
483
|
+
s.timeoutSeconds !== undefined &&
|
|
484
|
+
!(Number.isInteger(s.timeoutSeconds) && s.timeoutSeconds > 0)
|
|
485
|
+
)
|
|
486
|
+
problems.push(`${at}: timeoutSeconds must be a positive integer`);
|
|
487
|
+
if (s.retries !== undefined && !(Number.isInteger(s.retries) && s.retries >= 0))
|
|
488
|
+
problems.push(`${at}: retries must be a non-negative integer`);
|
|
309
489
|
|
|
310
490
|
const out: StepDescriptorT = {
|
|
311
491
|
name: s.name,
|
|
312
|
-
description: typeof s.description ===
|
|
492
|
+
description: typeof s.description === 'string' ? s.description : '',
|
|
313
493
|
kind: s.kind,
|
|
314
494
|
image: s.image,
|
|
315
495
|
inputTypes,
|
|
316
496
|
input,
|
|
317
497
|
output: s.output ? useModel(s.output, `${at} output`) : null,
|
|
318
498
|
secrets: secretNames,
|
|
319
|
-
handlerHash: handlerHash ||
|
|
499
|
+
handlerHash: handlerHash || 'sha256:' + '0'.repeat(64),
|
|
320
500
|
...(s.timeoutSeconds !== undefined ? { timeoutSeconds: s.timeoutSeconds } : {}),
|
|
321
501
|
...(s.retries !== undefined ? { retries: s.retries } : {}),
|
|
322
|
-
...(settings !== undefined
|
|
502
|
+
...(settings !== undefined
|
|
503
|
+
? { config: settings as Record<string, z.core.util.JSONType> }
|
|
504
|
+
: {}),
|
|
323
505
|
...(configNames.length ? { configEntries: configNames } : {}),
|
|
324
506
|
...(s.clusterAccess === true ? { clusterAccess: true as const } : {}),
|
|
325
507
|
...(Object.keys(externalInputs).length ? { externalInputs } : {}),
|
|
326
508
|
};
|
|
327
|
-
if (s.clusterAccess !== undefined && typeof s.clusterAccess !==
|
|
509
|
+
if (s.clusterAccess !== undefined && typeof s.clusterAccess !== 'boolean')
|
|
510
|
+
problems.push(`${at}: clusterAccess must be a boolean`);
|
|
328
511
|
if (s.output && !isModel(s.output)) problems.push(`${at}: output must be a model`);
|
|
329
512
|
return out;
|
|
330
513
|
};
|
|
331
514
|
|
|
332
|
-
const mapStep = (
|
|
515
|
+
const mapStep = (
|
|
516
|
+
at: string,
|
|
517
|
+
s: AnyStep,
|
|
518
|
+
mapper: MapperFunction,
|
|
519
|
+
avail: Map<string, Avail>,
|
|
520
|
+
): Record<string, Mapping> => {
|
|
333
521
|
try {
|
|
334
522
|
analyzeMapperSource(Function.prototype.toString.call(mapper));
|
|
335
523
|
} catch (e) {
|
|
@@ -346,154 +534,278 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
346
534
|
else problems.push(`${at}: mapper threw: ${(e as Error)?.message ?? String(e)}`);
|
|
347
535
|
return {};
|
|
348
536
|
}
|
|
349
|
-
const declared = Object.fromEntries(
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
537
|
+
const declared = Object.fromEntries(
|
|
538
|
+
Object.entries(s.input as Record<string, unknown>).filter(
|
|
539
|
+
([, e]) => !isExternal(e),
|
|
540
|
+
),
|
|
541
|
+
);
|
|
542
|
+
for (const k of Object.keys(declared))
|
|
543
|
+
if (!(k in recorded)) problems.push(`${at}: mapper does not provide input '${k}'`);
|
|
544
|
+
for (const k of Object.keys(recorded))
|
|
545
|
+
if (!(k in declared))
|
|
546
|
+
problems.push(`${at}: mapper provides '${k}', which is not an input of the step`);
|
|
547
|
+
for (const [k, m] of Object.entries(recorded))
|
|
548
|
+
checkMapping(`${at} input '${k}'`, m, avail, isOptional(declared[k]), true);
|
|
353
549
|
return recorded;
|
|
354
550
|
};
|
|
355
551
|
|
|
356
|
-
const checkMapping = (
|
|
552
|
+
const checkMapping = (
|
|
553
|
+
at: string,
|
|
554
|
+
m: Mapping,
|
|
555
|
+
avail: Map<string, Avail>,
|
|
556
|
+
optionalTarget: boolean,
|
|
557
|
+
topLevel: boolean,
|
|
558
|
+
) => {
|
|
357
559
|
const keys = Object.keys(m);
|
|
358
|
-
if (
|
|
359
|
-
|
|
560
|
+
if (
|
|
561
|
+
keys.length === 1 &&
|
|
562
|
+
keys[0] === 'ref' &&
|
|
563
|
+
typeof (m as { ref: unknown }).ref === 'string'
|
|
564
|
+
) {
|
|
565
|
+
const [root, ...path] = (m as { ref: string }).ref.split('.') as [
|
|
566
|
+
string,
|
|
567
|
+
...string[],
|
|
568
|
+
];
|
|
360
569
|
const a = avail.get(root);
|
|
361
570
|
if (!a) return problems.push(`${at}: '${root}' is not available at this stage`);
|
|
362
|
-
if (!resolveField(outputViewOf(a.model), path))
|
|
571
|
+
if (!resolveField(outputViewOf(a.model), path))
|
|
572
|
+
problems.push(
|
|
573
|
+
`${at}: '${(m as { ref: string }).ref}' — model ${a.model} has no field '${path.join('.')}'`,
|
|
574
|
+
);
|
|
363
575
|
if (a.maybe && !(optionalTarget && topLevel))
|
|
364
|
-
problems.push(
|
|
576
|
+
problems.push(
|
|
577
|
+
`${at}: '${(m as { ref: string }).ref}' is only maybe-available (produced inside a when block); map it whole into an optional(...) input`,
|
|
578
|
+
);
|
|
365
579
|
return;
|
|
366
580
|
}
|
|
367
|
-
if (keys.length === 1 && keys[0] ===
|
|
368
|
-
for (const [k, x] of Object.entries(m as Record<string, Mapping>))
|
|
581
|
+
if (keys.length === 1 && keys[0] === 'const') return;
|
|
582
|
+
for (const [k, x] of Object.entries(m as Record<string, Mapping>))
|
|
583
|
+
checkMapping(`${at}.${k}`, x, avail, optionalTarget, false);
|
|
369
584
|
};
|
|
370
585
|
|
|
371
586
|
// ── stages ──
|
|
372
587
|
let selects = 0;
|
|
373
|
-
const walk = (
|
|
588
|
+
const walk = (
|
|
589
|
+
nodes: readonly StageNode[],
|
|
590
|
+
avail: Map<string, Avail>,
|
|
591
|
+
cases: readonly CaseOf[] = [],
|
|
592
|
+
): Stage[] => {
|
|
374
593
|
const out: Stage[] = [];
|
|
375
594
|
for (const n of nodes) {
|
|
376
|
-
if (n.kind ===
|
|
595
|
+
if (n.kind === 'select') {
|
|
377
596
|
out.push(describeSelect(n, avail, cases, ++selects));
|
|
378
597
|
continue;
|
|
379
598
|
}
|
|
380
|
-
if (n.kind ===
|
|
381
|
-
const descs = n.steps
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
599
|
+
if (n.kind === 'stage') {
|
|
600
|
+
const descs = n.steps
|
|
601
|
+
.map((ss) => {
|
|
602
|
+
if (!isStep(ss.step)) {
|
|
603
|
+
problems.push('a stage entry is not a step');
|
|
604
|
+
return undefined;
|
|
605
|
+
}
|
|
606
|
+
registerStep(ss.step, 'stage', cases);
|
|
607
|
+
return describeStep(ss, avail, false, cases);
|
|
608
|
+
})
|
|
609
|
+
.filter((d): d is StepDescriptorT => d !== undefined);
|
|
389
610
|
// Outputs become available only after the barrier (same-stage siblings can't see each other).
|
|
390
|
-
for (const ss of n.steps)
|
|
611
|
+
for (const ss of n.steps)
|
|
612
|
+
if (isStep(ss.step))
|
|
613
|
+
avail.set(ss.step.name, {
|
|
614
|
+
model: ss.step.output?.name ?? null,
|
|
615
|
+
maybe: false,
|
|
616
|
+
});
|
|
391
617
|
if (descs.length) out.push({ steps: descs });
|
|
392
618
|
} else {
|
|
393
619
|
const when: Record<string, (string | number | boolean)[]> = {};
|
|
394
620
|
const props = (inputOutputView()?.properties ?? {}) as Record<string, JsonSchema>;
|
|
395
621
|
for (const [field, raw] of Object.entries(n.cond)) {
|
|
396
|
-
const values = (Array.isArray(raw) ? raw : [raw]) as (
|
|
397
|
-
|
|
622
|
+
const values = (Array.isArray(raw) ? raw : [raw]) as (
|
|
623
|
+
string | number | boolean
|
|
624
|
+
)[];
|
|
625
|
+
if (!Object.hasOwn(props, field))
|
|
626
|
+
problems.push(
|
|
627
|
+
`when: '${field}' is not a field of the pipeline input ${inputModel.name}`,
|
|
628
|
+
);
|
|
398
629
|
else {
|
|
399
630
|
const literals = literalValues(props[field]);
|
|
400
|
-
if (!literals)
|
|
401
|
-
|
|
631
|
+
if (!literals)
|
|
632
|
+
problems.push(
|
|
633
|
+
`when: '${field}' is not a literal-like field (enum, boolean or literal)`,
|
|
634
|
+
);
|
|
635
|
+
else
|
|
636
|
+
for (const v of values)
|
|
637
|
+
if (!literals.includes(v))
|
|
638
|
+
problems.push(
|
|
639
|
+
`when: ${JSON.stringify(v)} is not a valid value of '${field}' (${literals.map((l) => JSON.stringify(l)).join(', ')})`,
|
|
640
|
+
);
|
|
402
641
|
}
|
|
403
642
|
if (values.length === 0) problems.push(`when: '${field}' has no values`);
|
|
404
643
|
when[field] = values;
|
|
405
644
|
}
|
|
406
645
|
const inner = new Map(avail);
|
|
407
646
|
const stages = walk(n.nodes, inner, cases);
|
|
408
|
-
for (const [name, a] of inner)
|
|
647
|
+
for (const [name, a] of inner)
|
|
648
|
+
if (!avail.has(name)) avail.set(name, { model: a.model, maybe: true });
|
|
409
649
|
out.push({ when, stages });
|
|
410
650
|
}
|
|
411
651
|
}
|
|
412
652
|
return out;
|
|
413
653
|
};
|
|
414
|
-
const describeSelect = (
|
|
654
|
+
const describeSelect = (
|
|
655
|
+
n: Extract<StageNode, { kind: 'select' }>,
|
|
656
|
+
avail: Map<string, Avail>,
|
|
657
|
+
cases: readonly CaseOf[],
|
|
658
|
+
id: number,
|
|
659
|
+
): Stage => {
|
|
415
660
|
if (!isConfigRef(n.on)) {
|
|
416
|
-
problems.push(
|
|
417
|
-
return { select:
|
|
661
|
+
problems.push('select: the first argument must be a config reference');
|
|
662
|
+
return { select: '', discriminator: '', cases: {} };
|
|
418
663
|
}
|
|
419
664
|
const ref = configRefPath(n.on);
|
|
420
665
|
const { entry, case: kase } = configRefTarget(n.on);
|
|
421
666
|
useConfig(entry, `select '${ref}'`);
|
|
422
667
|
if (kase !== undefined && !cases.some((c) => c.cond === kase))
|
|
423
|
-
problems.push(
|
|
668
|
+
problems.push(
|
|
669
|
+
`select '${ref}': the reference is narrowed to the case '${kase}' and can only be used inside it`,
|
|
670
|
+
);
|
|
424
671
|
const union = unionAt(n.on);
|
|
425
|
-
if (!union)
|
|
672
|
+
if (!union)
|
|
673
|
+
problems.push(
|
|
674
|
+
`select '${ref}': it must reference a z.discriminatedUnion in config '${entry.name}'`,
|
|
675
|
+
);
|
|
426
676
|
else {
|
|
427
|
-
for (const v of union.values)
|
|
428
|
-
|
|
677
|
+
for (const v of union.values)
|
|
678
|
+
if (!Object.hasOwn(n.cases, v))
|
|
679
|
+
problems.push(`select '${ref}': no case for ${union.discriminator} '${v}'`);
|
|
680
|
+
for (const v of Object.keys(n.cases))
|
|
681
|
+
if (!union.values.includes(v))
|
|
682
|
+
problems.push(
|
|
683
|
+
`select '${ref}': '${v}' is not a value of ${union.discriminator} (${union.values.join(', ')})`,
|
|
684
|
+
);
|
|
429
685
|
}
|
|
430
|
-
const discriminator = union?.discriminator ??
|
|
686
|
+
const discriminator = union?.discriminator ?? '';
|
|
431
687
|
const branches = Object.entries(n.cases).map(([value, nodes]) => {
|
|
432
688
|
const inner = new Map(avail);
|
|
433
|
-
const stages = walk(nodes, inner, [
|
|
689
|
+
const stages = walk(nodes, inner, [
|
|
690
|
+
...cases,
|
|
691
|
+
{ select: id, value, cond: `${ref}.${discriminator}=${value}` },
|
|
692
|
+
]);
|
|
434
693
|
return { value, inner, stages };
|
|
435
694
|
});
|
|
436
695
|
// A step every case has is as available as in its cases; one only some cases have is maybe-available.
|
|
437
|
-
const added = new Set(
|
|
696
|
+
const added = new Set(
|
|
697
|
+
branches.flatMap((b) => [...b.inner.keys()].filter((k) => !avail.has(k))),
|
|
698
|
+
);
|
|
438
699
|
for (const name of added) {
|
|
439
|
-
const present = branches
|
|
440
|
-
|
|
700
|
+
const present = branches
|
|
701
|
+
.map((b) => b.inner.get(name))
|
|
702
|
+
.filter((a): a is Avail => a !== undefined);
|
|
703
|
+
avail.set(name, {
|
|
704
|
+
model: present[0]!.model,
|
|
705
|
+
maybe: present.length < branches.length || present.some((a) => a.maybe),
|
|
706
|
+
});
|
|
441
707
|
}
|
|
442
|
-
return {
|
|
708
|
+
return {
|
|
709
|
+
select: ref,
|
|
710
|
+
discriminator,
|
|
711
|
+
cases: Object.fromEntries(branches.map((b) => [b.value, b.stages])),
|
|
712
|
+
};
|
|
443
713
|
};
|
|
444
|
-
const stages = walk(
|
|
714
|
+
const stages = walk(
|
|
715
|
+
data.nodes,
|
|
716
|
+
new Map([['input', { model: inputModel.name, maybe: false }]]),
|
|
717
|
+
);
|
|
445
718
|
|
|
446
719
|
// ── failure handlers ──
|
|
447
720
|
const stageStepNames = new Set(steps.keys());
|
|
448
721
|
const seenHandlers = new Map<string, AnyStep>();
|
|
449
|
-
const failureHandlers = data.failureHandlers
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
722
|
+
const failureHandlers = data.failureHandlers
|
|
723
|
+
.map((fh) => {
|
|
724
|
+
if (!isStep(fh.step)) {
|
|
725
|
+
problems.push('onFailure: handler is not a step');
|
|
726
|
+
return undefined;
|
|
727
|
+
}
|
|
728
|
+
if (seenHandlers.get(fh.step.name) !== fh.step) registerStep(fh.step, 'onFailure');
|
|
729
|
+
seenHandlers.set(fh.step.name, fh.step);
|
|
730
|
+
const scope = fh.steps === null ? null : fh.steps.map((x) => x.name);
|
|
731
|
+
for (const n of scope ?? [])
|
|
732
|
+
if (!stageStepNames.has(n))
|
|
733
|
+
problems.push(
|
|
734
|
+
`onFailure(${fh.step.name}): '${n}' is not a step of this pipeline`,
|
|
735
|
+
);
|
|
736
|
+
return { step: describeStep({ step: fh.step }, new Map(), true), steps: scope };
|
|
737
|
+
})
|
|
738
|
+
.filter((x) => x !== undefined);
|
|
460
739
|
|
|
461
740
|
// ── handlers without a step ──
|
|
462
741
|
for (const h of handlerNames) {
|
|
463
742
|
const s = steps.get(h);
|
|
464
|
-
if (!s)
|
|
465
|
-
|
|
743
|
+
if (!s)
|
|
744
|
+
problems.push(`a handler implements step '${h}', which is not in the pipeline`);
|
|
745
|
+
else if (isImplementation(s))
|
|
746
|
+
problems.push(
|
|
747
|
+
`a handler implements '${h}', a ${s.kind} step that carries its own implementation`,
|
|
748
|
+
);
|
|
466
749
|
}
|
|
467
750
|
|
|
468
751
|
// ── triggers ──
|
|
469
752
|
const triggerNames = new Set<string>();
|
|
470
753
|
const triggers: TriggerT[] = data.triggers.map((t) => {
|
|
471
|
-
if (triggerNames.has(t.name))
|
|
754
|
+
if (triggerNames.has(t.name))
|
|
755
|
+
problems.push(
|
|
756
|
+
`duplicate trigger '${t.name}'${t.kind === 'push' ? ' (onPush may be used once)' : ' (give each onPoll and onWebhook a distinct name)'}`,
|
|
757
|
+
);
|
|
472
758
|
triggerNames.add(t.name);
|
|
473
|
-
if (t.kind ===
|
|
474
|
-
if (t.kind ===
|
|
475
|
-
if (!STEP_NAME.test(t.name))
|
|
759
|
+
if (t.kind === 'push') return { kind: 'push', name: 'push' };
|
|
760
|
+
if (t.kind === 'webhook') {
|
|
761
|
+
if (!STEP_NAME.test(t.name))
|
|
762
|
+
problems.push(`webhook trigger name '${t.name}' must be lower-case kebab`);
|
|
476
763
|
const secret = useSecret(t.verify.secret, `webhook '${t.name}'`);
|
|
477
|
-
if (t.verify.scheme ===
|
|
478
|
-
|
|
479
|
-
|
|
764
|
+
if (t.verify.scheme === 'bearer')
|
|
765
|
+
return { kind: 'webhook', name: t.name, verify: { scheme: 'bearer', secret } };
|
|
766
|
+
if (!t.verify.header)
|
|
767
|
+
problems.push(`webhook '${t.name}': the signature header must not be empty`);
|
|
768
|
+
return {
|
|
769
|
+
kind: 'webhook',
|
|
770
|
+
name: t.name,
|
|
771
|
+
verify: {
|
|
772
|
+
scheme: 'hmac-sha256',
|
|
773
|
+
secret,
|
|
774
|
+
header: t.verify.header,
|
|
775
|
+
prefix: t.verify.prefix,
|
|
776
|
+
},
|
|
777
|
+
};
|
|
480
778
|
}
|
|
481
|
-
if (!STEP_NAME.test(t.name))
|
|
482
|
-
|
|
779
|
+
if (!STEP_NAME.test(t.name))
|
|
780
|
+
problems.push(`poll trigger name '${t.name}' must be lower-case kebab`);
|
|
781
|
+
if (!(Number.isInteger(t.intervalSeconds) && t.intervalSeconds > 0))
|
|
782
|
+
problems.push(`poll '${t.name}': intervalSeconds must be a positive integer`);
|
|
483
783
|
const declared = new Set(t.secrets.map((s) => useSecret(s, `poll '${t.name}'`)));
|
|
484
784
|
for (const [h, v] of Object.entries(t.headers)) {
|
|
485
785
|
for (const m of v.matchAll(/\$SECRET:([A-Za-z0-9_]+)/g))
|
|
486
|
-
if (!declared.has(m[1]!))
|
|
786
|
+
if (!declared.has(m[1]!))
|
|
787
|
+
problems.push(
|
|
788
|
+
`poll '${t.name}': header '${h}' references secret '${m[1]}', which is not in the poll's secrets`,
|
|
789
|
+
);
|
|
487
790
|
}
|
|
488
|
-
return {
|
|
791
|
+
return {
|
|
792
|
+
kind: 'poll',
|
|
793
|
+
name: t.name,
|
|
794
|
+
url: t.url,
|
|
795
|
+
headers: { ...t.headers },
|
|
796
|
+
secrets: [...declared].sort(),
|
|
797
|
+
valuePath: t.valuePath,
|
|
798
|
+
intervalSeconds: t.intervalSeconds,
|
|
799
|
+
};
|
|
489
800
|
});
|
|
490
801
|
|
|
491
802
|
// ── retention ──
|
|
492
803
|
const retention: Record<string, number> = {};
|
|
493
|
-
for (const key of [
|
|
804
|
+
for (const key of ['keepLatestRuns', 'minRunAgeDays', 'buildGraceDays'] as const) {
|
|
494
805
|
const v = data.retention?.[key];
|
|
495
806
|
if (v === undefined) continue;
|
|
496
|
-
if (!(Number.isInteger(v) && v >= 0))
|
|
807
|
+
if (!(Number.isInteger(v) && v >= 0))
|
|
808
|
+
problems.push(`retention.${key} must be a non-negative integer`);
|
|
497
809
|
else retention[key] = v;
|
|
498
810
|
}
|
|
499
811
|
|
|
@@ -507,12 +819,19 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
507
819
|
...(Object.keys(retention).length ? { retention } : {}),
|
|
508
820
|
harness: opts.harness,
|
|
509
821
|
models: Object.fromEntries([...models].map(([k, v]) => [k, v.schemas])),
|
|
510
|
-
secrets: [...secrets]
|
|
822
|
+
secrets: [...secrets]
|
|
823
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
824
|
+
.map(([name, description]) => ({ name, description })),
|
|
511
825
|
...(configs.size
|
|
512
826
|
? {
|
|
513
827
|
config: [...configs]
|
|
514
828
|
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
515
|
-
.map(([name, c]) => ({
|
|
829
|
+
.map(([name, c]) => ({
|
|
830
|
+
name,
|
|
831
|
+
description: c.description,
|
|
832
|
+
required: c.required,
|
|
833
|
+
schema: c.schema,
|
|
834
|
+
})),
|
|
516
835
|
}
|
|
517
836
|
: {}),
|
|
518
837
|
triggers,
|
|
@@ -520,7 +839,10 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
|
|
|
520
839
|
failureHandlers,
|
|
521
840
|
});
|
|
522
841
|
const checked = Manifest.safeParse(manifest);
|
|
523
|
-
if (!checked.success)
|
|
842
|
+
if (!checked.success)
|
|
843
|
+
throw new DescribeError([
|
|
844
|
+
`internal: manifest does not match its schema: ${z.prettifyError(checked.error)}`,
|
|
845
|
+
]);
|
|
524
846
|
return manifest as ManifestT;
|
|
525
847
|
}
|
|
526
848
|
|
|
@@ -531,27 +853,45 @@ export const manifestText = (m: ManifestT): string => canonicalJson(m);
|
|
|
531
853
|
* The schemas at `path` inside an entry's schema: every union branch that has the field, except where
|
|
532
854
|
* `kase` (`<entry>.<path>.<discriminator>=<value>`) narrows a union to one branch.
|
|
533
855
|
*/
|
|
534
|
-
export function subSchemas(
|
|
856
|
+
export function subSchemas(
|
|
857
|
+
schema: unknown,
|
|
858
|
+
path: readonly string[],
|
|
859
|
+
kase: string | undefined,
|
|
860
|
+
entry: string,
|
|
861
|
+
): unknown[] {
|
|
535
862
|
let narrow: { at: string; discriminator: string; value: string } | undefined;
|
|
536
863
|
if (kase) {
|
|
537
|
-
const eq = kase.lastIndexOf(
|
|
864
|
+
const eq = kase.lastIndexOf('=');
|
|
538
865
|
const left = kase.slice(0, eq);
|
|
539
|
-
const dot = left.lastIndexOf(
|
|
540
|
-
narrow = {
|
|
866
|
+
const dot = left.lastIndexOf('.');
|
|
867
|
+
narrow = {
|
|
868
|
+
at: left.slice(0, dot),
|
|
869
|
+
discriminator: left.slice(dot + 1),
|
|
870
|
+
value: kase.slice(eq + 1),
|
|
871
|
+
};
|
|
541
872
|
}
|
|
542
873
|
const walk = (node: unknown, rest: readonly string[], here: string): unknown[] => {
|
|
543
|
-
if (node === null || typeof node !==
|
|
874
|
+
if (node === null || typeof node !== 'object') return [];
|
|
544
875
|
const o = node as JsonSchema;
|
|
545
876
|
const branches = (o.oneOf ?? o.anyOf) as JsonSchema[] | undefined;
|
|
546
877
|
if (Array.isArray(branches)) {
|
|
547
|
-
const chosen =
|
|
548
|
-
|
|
549
|
-
|
|
878
|
+
const chosen =
|
|
879
|
+
narrow && narrow.at === here
|
|
880
|
+
? branches.filter(
|
|
881
|
+
(b) =>
|
|
882
|
+
(
|
|
883
|
+
(b.properties as Record<string, JsonSchema> | undefined)?.[
|
|
884
|
+
narrow!.discriminator
|
|
885
|
+
] as JsonSchema | undefined
|
|
886
|
+
)?.const === narrow!.value,
|
|
887
|
+
)
|
|
888
|
+
: branches;
|
|
550
889
|
return chosen.flatMap((b) => walk(b, rest, here));
|
|
551
890
|
}
|
|
552
891
|
if (rest.length === 0) return [node];
|
|
553
892
|
const props = o.properties as Record<string, JsonSchema> | undefined;
|
|
554
|
-
if (props && Object.hasOwn(props, rest[0]!))
|
|
893
|
+
if (props && Object.hasOwn(props, rest[0]!))
|
|
894
|
+
return walk(props[rest[0]!], rest.slice(1), `${here}.${rest[0]}`);
|
|
555
895
|
return [];
|
|
556
896
|
};
|
|
557
897
|
return walk(schema, path, entry);
|
|
@@ -562,11 +902,12 @@ export function secretRefsOf(schema: unknown): string[] {
|
|
|
562
902
|
const out = new Set<string>();
|
|
563
903
|
const walk = (v: unknown) => {
|
|
564
904
|
if (Array.isArray(v)) return v.forEach(walk);
|
|
565
|
-
if (v === null || typeof v !==
|
|
905
|
+
if (v === null || typeof v !== 'object') return;
|
|
566
906
|
const o = v as JsonSchema;
|
|
567
|
-
if (o[
|
|
568
|
-
const name = (o.properties as { secret?: { const?: unknown } } | undefined)?.secret
|
|
569
|
-
|
|
907
|
+
if (o['x-olve-kind'] === SECRET_REF_KIND) {
|
|
908
|
+
const name = (o.properties as { secret?: { const?: unknown } } | undefined)?.secret
|
|
909
|
+
?.const;
|
|
910
|
+
if (typeof name === 'string') out.add(name);
|
|
570
911
|
}
|
|
571
912
|
Object.values(o).forEach(walk);
|
|
572
913
|
};
|
|
@@ -575,14 +916,27 @@ export function secretRefsOf(schema: unknown): string[] {
|
|
|
575
916
|
}
|
|
576
917
|
|
|
577
918
|
/** The `{"$config": "<entry>.<path>"[, "$case": …]}` references in a step's recorded settings. */
|
|
578
|
-
export function recordedConfigRefs(
|
|
919
|
+
export function recordedConfigRefs(
|
|
920
|
+
v: unknown,
|
|
921
|
+
out: { ref: string; case?: string }[] = [],
|
|
922
|
+
): { ref: string; case?: string }[] {
|
|
579
923
|
if (Array.isArray(v)) v.forEach((x) => recordedConfigRefs(x, out));
|
|
580
|
-
else if (v !== null && typeof v ===
|
|
924
|
+
else if (v !== null && typeof v === 'object') {
|
|
581
925
|
const o = v as Record<string, unknown>;
|
|
582
926
|
const keys = Object.keys(o).sort();
|
|
583
|
-
const isRef =
|
|
584
|
-
|
|
585
|
-
|
|
927
|
+
const isRef =
|
|
928
|
+
typeof o[CONFIG_REF_KEY] === 'string' &&
|
|
929
|
+
(keys.length === 1
|
|
930
|
+
? keys[0] === CONFIG_REF_KEY
|
|
931
|
+
: keys.length === 2 &&
|
|
932
|
+
keys[0] === CONFIG_CASE_KEY &&
|
|
933
|
+
keys[1] === CONFIG_REF_KEY &&
|
|
934
|
+
typeof o[CONFIG_CASE_KEY] === 'string');
|
|
935
|
+
if (isRef)
|
|
936
|
+
out.push({
|
|
937
|
+
ref: o[CONFIG_REF_KEY] as string,
|
|
938
|
+
...(keys.length === 2 ? { case: o[CONFIG_CASE_KEY] as string } : {}),
|
|
939
|
+
});
|
|
586
940
|
else Object.values(o).forEach((x) => recordedConfigRefs(x, out));
|
|
587
941
|
}
|
|
588
942
|
return out;
|