@ontrails/testing 1.0.0-beta.18 → 1.0.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/README.md +31 -29
- package/package.json +24 -8
- package/src/all-established.ts +168 -0
- package/src/all.ts +3 -130
- package/src/assertions.ts +2 -2
- package/src/cli.ts +6 -0
- package/src/{crosses.ts → composes.ts} +87 -79
- package/src/context.ts +22 -21
- package/src/contracts.ts +19 -17
- package/src/effective-examples.ts +67 -3
- package/src/errors.ts +47 -0
- package/src/examples.ts +131 -118
- package/src/harness-cli.ts +35 -7
- package/src/harness-http.ts +87 -9
- package/src/harness-mcp.ts +30 -6
- package/src/http.ts +10 -0
- package/src/index.ts +5 -34
- package/src/mcp.ts +6 -0
- package/src/scenario.ts +67 -60
- package/src/surface-parity.ts +43 -10
- package/src/types.ts +11 -232
|
@@ -1,20 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* testComposes — composing-aware scenario testing for trails with compositions.
|
|
3
3
|
*
|
|
4
|
-
* Tests the
|
|
5
|
-
* and supports failure injection from
|
|
4
|
+
* Tests the composing graph: which trails were composed, in what order,
|
|
5
|
+
* and supports failure injection from composed trail examples.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import { describe, expect, test } from 'bun:test';
|
|
9
9
|
|
|
10
10
|
import type {
|
|
11
11
|
AnyTrail,
|
|
12
|
-
|
|
12
|
+
ComposeFn,
|
|
13
|
+
ExecuteTrailOptions,
|
|
13
14
|
ResourceOverrideMap,
|
|
14
15
|
TrailContext,
|
|
15
16
|
} from '@ontrails/core';
|
|
16
17
|
import {
|
|
17
|
-
|
|
18
|
+
buildComposeValidationSchema,
|
|
18
19
|
executeTrail,
|
|
19
20
|
InternalError,
|
|
20
21
|
Result,
|
|
@@ -28,13 +29,18 @@ import {
|
|
|
28
29
|
assertSchemaMatch,
|
|
29
30
|
} from './assertions.js';
|
|
30
31
|
import { mergeResourceOverrides, mergeTestContext } from './context.js';
|
|
31
|
-
import
|
|
32
|
+
import { createErrorFromName } from './errors.js';
|
|
33
|
+
import type { ComposeScenario } from './types.js';
|
|
34
|
+
|
|
35
|
+
type TestingExecuteTrailOptions = ExecuteTrailOptions & {
|
|
36
|
+
readonly validationSchema?: ReturnType<typeof buildComposeValidationSchema>;
|
|
37
|
+
};
|
|
32
38
|
|
|
33
39
|
// ---------------------------------------------------------------------------
|
|
34
|
-
//
|
|
40
|
+
// Compose trace
|
|
35
41
|
// ---------------------------------------------------------------------------
|
|
36
42
|
|
|
37
|
-
interface
|
|
43
|
+
interface ComposeRecord {
|
|
38
44
|
readonly id: string;
|
|
39
45
|
readonly input: unknown;
|
|
40
46
|
}
|
|
@@ -63,10 +69,10 @@ const collectDeclaredResources = (
|
|
|
63
69
|
}
|
|
64
70
|
seenTrailIds.add(candidate.id);
|
|
65
71
|
collect(candidate);
|
|
66
|
-
for (const
|
|
67
|
-
const
|
|
68
|
-
if (
|
|
69
|
-
visit(
|
|
72
|
+
for (const composedId of candidate.composes) {
|
|
73
|
+
const composedTrail = trailsMap?.get(composedId);
|
|
74
|
+
if (composedTrail) {
|
|
75
|
+
visit(composedTrail);
|
|
70
76
|
}
|
|
71
77
|
}
|
|
72
78
|
};
|
|
@@ -75,7 +81,7 @@ const collectDeclaredResources = (
|
|
|
75
81
|
return resources;
|
|
76
82
|
};
|
|
77
83
|
|
|
78
|
-
const
|
|
84
|
+
const resolveComposeMockResources = async (
|
|
79
85
|
trailDef: AnyTrail,
|
|
80
86
|
trailsMap: ReadonlyMap<string, AnyTrail> | undefined
|
|
81
87
|
): Promise<ResourceOverrideMap> => {
|
|
@@ -114,12 +120,12 @@ const findErrorExample = (
|
|
|
114
120
|
};
|
|
115
121
|
|
|
116
122
|
/**
|
|
117
|
-
* Try to inject an error from a
|
|
123
|
+
* Try to inject an error from a composed trail's example.
|
|
118
124
|
* Returns undefined when no injection is configured for this trail ID.
|
|
119
125
|
*/
|
|
120
126
|
const tryInjectError = (
|
|
121
127
|
id: string,
|
|
122
|
-
scenario:
|
|
128
|
+
scenario: ComposeScenario,
|
|
123
129
|
trailsMap: ReadonlyMap<string, AnyTrail> | undefined
|
|
124
130
|
): Result<unknown, Error> | undefined => {
|
|
125
131
|
const injection = scenario.injectFromExample?.[id];
|
|
@@ -141,7 +147,7 @@ const tryInjectError = (
|
|
|
141
147
|
)
|
|
142
148
|
);
|
|
143
149
|
}
|
|
144
|
-
return Result.err(
|
|
150
|
+
return Result.err(createErrorFromName(errorName));
|
|
145
151
|
};
|
|
146
152
|
|
|
147
153
|
const executeFromMap = (
|
|
@@ -150,65 +156,67 @@ const executeFromMap = (
|
|
|
150
156
|
trailsMap: ReadonlyMap<string, AnyTrail> | undefined,
|
|
151
157
|
ctx: TrailContext,
|
|
152
158
|
resources: ResourceOverrideMap | undefined,
|
|
153
|
-
|
|
159
|
+
compose?: ComposeFn
|
|
154
160
|
): Result<unknown, Error> | Promise<Result<unknown, Error>> | undefined => {
|
|
155
161
|
const trailDef = trailsMap?.get(id);
|
|
156
162
|
if (trailDef === undefined) {
|
|
157
163
|
return undefined;
|
|
158
164
|
}
|
|
159
165
|
|
|
160
|
-
const nestedCtx =
|
|
161
|
-
|
|
166
|
+
const nestedCtx = compose ? { ...ctx, compose } : ctx;
|
|
167
|
+
const options: TestingExecuteTrailOptions = {
|
|
162
168
|
ctx: nestedCtx,
|
|
163
169
|
resources,
|
|
164
|
-
validationSchema:
|
|
165
|
-
}
|
|
170
|
+
validationSchema: buildComposeValidationSchema(trailDef),
|
|
171
|
+
};
|
|
172
|
+
return executeTrail(trailDef, input, options);
|
|
166
173
|
};
|
|
167
174
|
|
|
168
175
|
/** Extract trail ID from either a trail object or a string. */
|
|
169
|
-
const
|
|
170
|
-
|
|
176
|
+
const resolveComposeId = (
|
|
177
|
+
idOrTrail: string | { readonly id: string }
|
|
178
|
+
): string => (typeof idOrTrail === 'string' ? idOrTrail : idOrTrail.id);
|
|
171
179
|
|
|
172
180
|
// ---------------------------------------------------------------------------
|
|
173
|
-
//
|
|
181
|
+
// Compose factory
|
|
174
182
|
// ---------------------------------------------------------------------------
|
|
175
183
|
|
|
176
|
-
/** Delegate to
|
|
177
|
-
const
|
|
184
|
+
/** Delegate to baseCompose, executeFromMap, or fall back to Result.ok(). */
|
|
185
|
+
const delegateCompose = (
|
|
178
186
|
id: string,
|
|
179
187
|
input: unknown,
|
|
180
|
-
|
|
188
|
+
baseCompose: ComposeFn | undefined,
|
|
181
189
|
trailsMap: ReadonlyMap<string, AnyTrail> | undefined,
|
|
182
190
|
ctx: TrailContext,
|
|
183
191
|
resources: ResourceOverrideMap | undefined,
|
|
184
|
-
self:
|
|
192
|
+
self: ComposeFn
|
|
185
193
|
): Promise<Result<unknown, Error>> => {
|
|
186
|
-
if (
|
|
187
|
-
return
|
|
194
|
+
if (baseCompose !== undefined) {
|
|
195
|
+
return baseCompose(id, input);
|
|
188
196
|
}
|
|
189
197
|
const executed = executeFromMap(id, input, trailsMap, ctx, resources, self);
|
|
190
198
|
return Promise.resolve(executed ?? Result.ok());
|
|
191
199
|
};
|
|
192
200
|
|
|
193
201
|
/**
|
|
194
|
-
* Build a recording
|
|
202
|
+
* Build a recording compose function that optionally injects errors.
|
|
195
203
|
*/
|
|
196
|
-
const
|
|
197
|
-
trace:
|
|
198
|
-
scenario:
|
|
204
|
+
const createRecordingCompose = (
|
|
205
|
+
trace: ComposeRecord[],
|
|
206
|
+
scenario: ComposeScenario,
|
|
199
207
|
trailsMap: ReadonlyMap<string, AnyTrail> | undefined,
|
|
200
|
-
|
|
208
|
+
baseCompose: ComposeFn | undefined,
|
|
201
209
|
ctx: TrailContext,
|
|
202
210
|
resources: ResourceOverrideMap | undefined
|
|
203
|
-
):
|
|
204
|
-
// The generic O on
|
|
211
|
+
): ComposeFn => {
|
|
212
|
+
// The generic O on ComposeFn is erased at runtime; the cast is safe
|
|
205
213
|
// because callers narrow via isOk/isErr before accessing the value.
|
|
206
|
-
const
|
|
214
|
+
const invokeCompose = async (
|
|
207
215
|
idOrTrail: string | { readonly id: string },
|
|
208
216
|
input: unknown,
|
|
209
|
-
self:
|
|
217
|
+
self: ComposeFn
|
|
210
218
|
) => {
|
|
211
|
-
const id =
|
|
219
|
+
const id = resolveComposeId(idOrTrail);
|
|
212
220
|
trace.push({ id, input });
|
|
213
221
|
|
|
214
222
|
const injected = tryInjectError(id, scenario, trailsMap);
|
|
@@ -216,10 +224,10 @@ const createRecordingCross = (
|
|
|
216
224
|
return injected;
|
|
217
225
|
}
|
|
218
226
|
|
|
219
|
-
return await
|
|
227
|
+
return await delegateCompose(
|
|
220
228
|
id,
|
|
221
229
|
input,
|
|
222
|
-
|
|
230
|
+
baseCompose,
|
|
223
231
|
trailsMap,
|
|
224
232
|
ctx,
|
|
225
233
|
resources,
|
|
@@ -227,9 +235,9 @@ const createRecordingCross = (
|
|
|
227
235
|
);
|
|
228
236
|
};
|
|
229
237
|
|
|
230
|
-
// Accepts either a trail object (typed
|
|
238
|
+
// Accepts either a trail object (typed compose), a string id (untyped),
|
|
231
239
|
// or a batch of `[target, input]` tuples.
|
|
232
|
-
const
|
|
240
|
+
const compose = async function compose(
|
|
233
241
|
idOrTrail:
|
|
234
242
|
| string
|
|
235
243
|
| { readonly id: string }
|
|
@@ -239,19 +247,19 @@ const createRecordingCross = (
|
|
|
239
247
|
if (Array.isArray(idOrTrail)) {
|
|
240
248
|
return await Promise.all(
|
|
241
249
|
idOrTrail.map(([target, batchInput]) =>
|
|
242
|
-
|
|
250
|
+
invokeCompose(target, batchInput, compose as ComposeFn)
|
|
243
251
|
)
|
|
244
252
|
);
|
|
245
253
|
}
|
|
246
254
|
|
|
247
|
-
return await
|
|
255
|
+
return await invokeCompose(
|
|
248
256
|
idOrTrail as string | { readonly id: string },
|
|
249
257
|
input,
|
|
250
|
-
|
|
258
|
+
compose as ComposeFn
|
|
251
259
|
);
|
|
252
|
-
} as
|
|
260
|
+
} as ComposeFn;
|
|
253
261
|
|
|
254
|
-
return
|
|
262
|
+
return compose;
|
|
255
263
|
};
|
|
256
264
|
|
|
257
265
|
// ---------------------------------------------------------------------------
|
|
@@ -260,7 +268,7 @@ const createRecordingCross = (
|
|
|
260
268
|
|
|
261
269
|
const assertScenarioResult = (
|
|
262
270
|
result: Result<unknown, Error>,
|
|
263
|
-
scenario:
|
|
271
|
+
scenario: ComposeScenario,
|
|
264
272
|
trailDef: AnyTrail
|
|
265
273
|
): void => {
|
|
266
274
|
if (scenario.expectValue !== undefined) {
|
|
@@ -278,26 +286,26 @@ const assertScenarioResult = (
|
|
|
278
286
|
}
|
|
279
287
|
};
|
|
280
288
|
|
|
281
|
-
const
|
|
282
|
-
trace: readonly
|
|
283
|
-
scenario:
|
|
289
|
+
const assertComposeTrace = (
|
|
290
|
+
trace: readonly ComposeRecord[],
|
|
291
|
+
scenario: ComposeScenario
|
|
284
292
|
): void => {
|
|
285
|
-
if (scenario.
|
|
286
|
-
const
|
|
287
|
-
expect(
|
|
293
|
+
if (scenario.expectComposed !== undefined) {
|
|
294
|
+
const composedIds = trace.map((r) => r.id);
|
|
295
|
+
expect(composedIds).toEqual([...scenario.expectComposed]);
|
|
288
296
|
}
|
|
289
|
-
if (scenario.
|
|
297
|
+
if (scenario.expectComposedCount !== undefined) {
|
|
290
298
|
const counts: Record<string, number> = {};
|
|
291
299
|
for (const record of trace) {
|
|
292
300
|
counts[record.id] = (counts[record.id] ?? 0) + 1;
|
|
293
301
|
}
|
|
294
|
-
expect(counts).toEqual({ ...scenario.
|
|
302
|
+
expect(counts).toEqual({ ...scenario.expectComposedCount });
|
|
295
303
|
}
|
|
296
304
|
};
|
|
297
305
|
|
|
298
306
|
const handleValidationError = (
|
|
299
307
|
validated: Result<unknown, Error>,
|
|
300
|
-
scenario:
|
|
308
|
+
scenario: ComposeScenario
|
|
301
309
|
): boolean => {
|
|
302
310
|
if (!validated.isErr()) {
|
|
303
311
|
return false;
|
|
@@ -319,27 +327,27 @@ const handleValidationError = (
|
|
|
319
327
|
// ---------------------------------------------------------------------------
|
|
320
328
|
|
|
321
329
|
const buildTestContext = (
|
|
322
|
-
scenario:
|
|
330
|
+
scenario: ComposeScenario,
|
|
323
331
|
ctx: Partial<TrailContext> | undefined,
|
|
324
332
|
trailsMap: ReadonlyMap<string, AnyTrail> | undefined,
|
|
325
333
|
resources: ResourceOverrideMap | undefined
|
|
326
|
-
): { trace:
|
|
327
|
-
const trace:
|
|
334
|
+
): { trace: ComposeRecord[]; testCtx: TrailContext } => {
|
|
335
|
+
const trace: ComposeRecord[] = [];
|
|
328
336
|
const baseCtx = mergeTestContext(ctx);
|
|
329
|
-
const
|
|
337
|
+
const compose = createRecordingCompose(
|
|
330
338
|
trace,
|
|
331
339
|
scenario,
|
|
332
340
|
trailsMap,
|
|
333
|
-
baseCtx.
|
|
341
|
+
baseCtx.compose,
|
|
334
342
|
baseCtx,
|
|
335
343
|
resources
|
|
336
344
|
);
|
|
337
|
-
return { testCtx: { ...baseCtx,
|
|
345
|
+
return { testCtx: { ...baseCtx, compose }, trace };
|
|
338
346
|
};
|
|
339
347
|
|
|
340
348
|
const runScenario = async (
|
|
341
349
|
trailDef: AnyTrail,
|
|
342
|
-
scenario:
|
|
350
|
+
scenario: ComposeScenario,
|
|
343
351
|
ctx: Partial<TrailContext> | undefined,
|
|
344
352
|
trailsMap: ReadonlyMap<string, AnyTrail> | undefined,
|
|
345
353
|
resources: ResourceOverrideMap | undefined
|
|
@@ -359,16 +367,16 @@ const runScenario = async (
|
|
|
359
367
|
ctx: testCtx,
|
|
360
368
|
resources,
|
|
361
369
|
});
|
|
362
|
-
|
|
370
|
+
assertComposeTrace(trace, scenario);
|
|
363
371
|
assertScenarioResult(result, scenario, trailDef);
|
|
364
372
|
};
|
|
365
373
|
|
|
366
374
|
// ---------------------------------------------------------------------------
|
|
367
|
-
//
|
|
375
|
+
// testComposes
|
|
368
376
|
// ---------------------------------------------------------------------------
|
|
369
377
|
|
|
370
|
-
/** Options for
|
|
371
|
-
export interface
|
|
378
|
+
/** Options for testComposes that provide trail definitions for injection. */
|
|
379
|
+
export interface TestComposeOptions {
|
|
372
380
|
/** Partial context overrides. */
|
|
373
381
|
readonly ctx?: Partial<TrailContext> | undefined;
|
|
374
382
|
/**
|
|
@@ -382,33 +390,33 @@ export interface TestCrossOptions {
|
|
|
382
390
|
}
|
|
383
391
|
|
|
384
392
|
/**
|
|
385
|
-
* Generate a describe block for a trail with
|
|
393
|
+
* Generate a describe block for a trail with compositions with one test per scenario.
|
|
386
394
|
*
|
|
387
395
|
* @example
|
|
388
396
|
* ```ts
|
|
389
|
-
*
|
|
397
|
+
* testComposes(onboardTrail, [
|
|
390
398
|
* {
|
|
391
|
-
* description: "
|
|
399
|
+
* description: "composes add then relate",
|
|
392
400
|
* input: { name: "Alpha" },
|
|
393
401
|
* expectOk: true,
|
|
394
|
-
*
|
|
402
|
+
* expectComposed: ["entity.add", "entity.relate"],
|
|
395
403
|
* },
|
|
396
404
|
* ]);
|
|
397
405
|
* ```
|
|
398
406
|
*/
|
|
399
|
-
export const
|
|
407
|
+
export const testComposes = (
|
|
400
408
|
trailDef: AnyTrail,
|
|
401
|
-
scenarios: readonly
|
|
402
|
-
options?:
|
|
409
|
+
scenarios: readonly ComposeScenario[],
|
|
410
|
+
options?: TestComposeOptions
|
|
403
411
|
): void => {
|
|
404
412
|
const explicitResources = options?.resources;
|
|
405
413
|
|
|
406
414
|
describe(trailDef.id, () => {
|
|
407
415
|
test.each([...scenarios])(
|
|
408
416
|
'$description',
|
|
409
|
-
async (scenario:
|
|
417
|
+
async (scenario: ComposeScenario) => {
|
|
410
418
|
const resources = mergeResourceOverrides(
|
|
411
|
-
await
|
|
419
|
+
await resolveComposeMockResources(trailDef, options?.trails),
|
|
412
420
|
options?.ctx,
|
|
413
421
|
explicitResources
|
|
414
422
|
);
|
package/src/context.ts
CHANGED
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import type {
|
|
6
|
-
|
|
6
|
+
ComposeFn,
|
|
7
7
|
ResourceOverrideMap,
|
|
8
8
|
Topo,
|
|
9
9
|
TrailContext,
|
|
10
10
|
} from '@ontrails/core';
|
|
11
11
|
import {
|
|
12
12
|
Result,
|
|
13
|
-
|
|
13
|
+
buildComposeValidationSchema,
|
|
14
14
|
createResourceLookup,
|
|
15
15
|
passthroughTrace,
|
|
16
16
|
} from '@ontrails/core';
|
|
@@ -53,10 +53,10 @@ export const createTestContext = (
|
|
|
53
53
|
};
|
|
54
54
|
|
|
55
55
|
// ---------------------------------------------------------------------------
|
|
56
|
-
//
|
|
56
|
+
// createComposeContext
|
|
57
57
|
// ---------------------------------------------------------------------------
|
|
58
58
|
|
|
59
|
-
export interface
|
|
59
|
+
export interface CreateComposeContextOptions {
|
|
60
60
|
readonly responses?: Record<string, Result<unknown, Error>> | undefined;
|
|
61
61
|
}
|
|
62
62
|
|
|
@@ -94,39 +94,38 @@ export interface TestExecutionOptions {
|
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
/**
|
|
97
|
-
* Create a mock `
|
|
97
|
+
* Create a mock `ComposeFn` for testing composite trails.
|
|
98
98
|
*
|
|
99
99
|
* Returns preconfigured `Result` values keyed by trail ID. Calls to
|
|
100
100
|
* unregistered IDs return `Result.err` with a descriptive message.
|
|
101
101
|
*
|
|
102
102
|
* @example
|
|
103
103
|
* ```ts
|
|
104
|
-
* const
|
|
104
|
+
* const compose = createComposeContext({
|
|
105
105
|
* responses: { 'entity.add': Result.ok({ id: '1', name: 'Alpha' }) },
|
|
106
106
|
* });
|
|
107
|
-
* const ctx = { ...createTestContext(),
|
|
107
|
+
* const ctx = { ...createTestContext(), compose };
|
|
108
108
|
* ```
|
|
109
109
|
*/
|
|
110
|
-
export const
|
|
111
|
-
options?:
|
|
112
|
-
):
|
|
110
|
+
export const createComposeContext = (
|
|
111
|
+
options?: CreateComposeContextOptions
|
|
112
|
+
): ComposeFn => {
|
|
113
113
|
const responses = options?.responses ?? {};
|
|
114
|
-
const
|
|
114
|
+
const respondToCompose = <O>(
|
|
115
115
|
idOrTrail: string | { readonly id: string }
|
|
116
116
|
): Promise<Result<O, Error>> => {
|
|
117
117
|
const id = typeof idOrTrail === 'string' ? idOrTrail : idOrTrail.id;
|
|
118
118
|
const response = responses[id];
|
|
119
119
|
if (response === undefined) {
|
|
120
120
|
return Promise.resolve(
|
|
121
|
-
Result.err(
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
>
|
|
121
|
+
Result.err(
|
|
122
|
+
new Error(`No mock response for compose("${id}")`)
|
|
123
|
+
) as Result<O, Error>
|
|
125
124
|
);
|
|
126
125
|
}
|
|
127
126
|
return Promise.resolve(response as Result<O, Error>);
|
|
128
127
|
};
|
|
129
|
-
const
|
|
128
|
+
const compose = (async (
|
|
130
129
|
idOrTrail:
|
|
131
130
|
| string
|
|
132
131
|
| { readonly id: string }
|
|
@@ -135,13 +134,15 @@ export const createCrossContext = (
|
|
|
135
134
|
) => {
|
|
136
135
|
if (Array.isArray(idOrTrail)) {
|
|
137
136
|
return await Promise.all(
|
|
138
|
-
idOrTrail.map(([target]) =>
|
|
137
|
+
idOrTrail.map(([target]) => respondToCompose(target))
|
|
139
138
|
);
|
|
140
139
|
}
|
|
141
140
|
|
|
142
|
-
return await
|
|
143
|
-
|
|
144
|
-
|
|
141
|
+
return await respondToCompose(
|
|
142
|
+
idOrTrail as string | { readonly id: string }
|
|
143
|
+
);
|
|
144
|
+
}) as ComposeFn;
|
|
145
|
+
return compose;
|
|
145
146
|
};
|
|
146
147
|
|
|
147
148
|
/**
|
|
@@ -200,7 +201,7 @@ export const createMockResources = async (
|
|
|
200
201
|
): Promise<ResourceOverrideMap> => await buildMockResources(app);
|
|
201
202
|
|
|
202
203
|
// Re-export from core so existing consumers of this module continue to work.
|
|
203
|
-
export {
|
|
204
|
+
export { buildComposeValidationSchema };
|
|
204
205
|
|
|
205
206
|
/**
|
|
206
207
|
* Merge a Partial<TrailContext> into a test context.
|
package/src/contracts.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* testContracts — output schema verification.
|
|
3
3
|
*
|
|
4
4
|
* For every trail that has both examples and an output schema,
|
|
5
|
-
* run each example and validate the
|
|
5
|
+
* run each example and validate the Result.ok value against
|
|
6
6
|
* the declared schema.
|
|
7
7
|
*/
|
|
8
8
|
|
|
@@ -20,7 +20,8 @@ import {
|
|
|
20
20
|
createMockResources,
|
|
21
21
|
} from './context.js';
|
|
22
22
|
import type { TestExecutionOptions } from './context.js';
|
|
23
|
-
import {
|
|
23
|
+
import type { TrailExampleTarget } from './effective-examples.js';
|
|
24
|
+
import { deriveTrailExampleTargets } from './effective-examples.js';
|
|
24
25
|
|
|
25
26
|
// ---------------------------------------------------------------------------
|
|
26
27
|
// Helpers
|
|
@@ -41,13 +42,22 @@ const validateOutputSchema = (
|
|
|
41
42
|
}
|
|
42
43
|
};
|
|
43
44
|
|
|
45
|
+
type ContractExampleTarget = TrailExampleTarget & {
|
|
46
|
+
readonly output: z.ZodType;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const hasContractExamples = (
|
|
50
|
+
target: TrailExampleTarget
|
|
51
|
+
): target is ContractExampleTarget =>
|
|
52
|
+
target.output !== undefined && target.examples.length > 0;
|
|
53
|
+
|
|
44
54
|
// ---------------------------------------------------------------------------
|
|
45
55
|
// testContracts
|
|
46
56
|
// ---------------------------------------------------------------------------
|
|
47
57
|
|
|
48
58
|
/**
|
|
49
|
-
* Verify that every trail
|
|
50
|
-
* output schema. Catches
|
|
59
|
+
* Verify that every successful trail result matches its declared
|
|
60
|
+
* output schema. Catches output-schema drift.
|
|
51
61
|
*
|
|
52
62
|
* Trails without output schemas or examples are skipped.
|
|
53
63
|
*/
|
|
@@ -60,21 +70,12 @@ export const testContracts = (
|
|
|
60
70
|
): void => {
|
|
61
71
|
const resolveInput =
|
|
62
72
|
typeof ctxOrFactory === 'function' ? ctxOrFactory : () => ctxOrFactory;
|
|
63
|
-
const allEntries = (app.list() as Trail<unknown, unknown, unknown>[])
|
|
64
|
-
(
|
|
65
|
-
|
|
66
|
-
examples: deriveTrailExamples(trailDef),
|
|
67
|
-
})
|
|
68
|
-
);
|
|
73
|
+
const allEntries = (app.list() as Trail<unknown, unknown, unknown>[])
|
|
74
|
+
.flatMap(deriveTrailExampleTargets)
|
|
75
|
+
.filter(hasContractExamples);
|
|
69
76
|
|
|
70
77
|
describe('contracts', () => {
|
|
71
78
|
describe.each(allEntries)('$id', (t) => {
|
|
72
|
-
if (t.output === undefined) {
|
|
73
|
-
return;
|
|
74
|
-
}
|
|
75
|
-
if (t.examples.length === 0) {
|
|
76
|
-
return;
|
|
77
|
-
}
|
|
78
79
|
const { examples, output: outputSchema } = t;
|
|
79
80
|
const successExamples = examples.filter((e) => e.error === undefined);
|
|
80
81
|
|
|
@@ -92,10 +93,11 @@ export const testContracts = (
|
|
|
92
93
|
const validated = validateInput(t.input, example.input);
|
|
93
94
|
expectOk(validated);
|
|
94
95
|
|
|
95
|
-
const result = await executeTrail(t, example.input, {
|
|
96
|
+
const result = await executeTrail(t.trail, example.input, {
|
|
96
97
|
ctx: testCtx,
|
|
97
98
|
resources,
|
|
98
99
|
topo: app,
|
|
100
|
+
...(t.version === undefined ? {} : { version: t.version }),
|
|
99
101
|
});
|
|
100
102
|
const resultValue = expectOk(result);
|
|
101
103
|
|
|
@@ -1,16 +1,34 @@
|
|
|
1
1
|
import type { AnyContour, Trail, TrailExample } from '@ontrails/core';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
getContourReferences,
|
|
4
|
+
getTrailVersionEntryKind,
|
|
5
|
+
isArchivedTrailVersionEntry,
|
|
6
|
+
} from '@ontrails/core';
|
|
3
7
|
import { z } from 'zod';
|
|
4
8
|
|
|
5
9
|
type ExampleRecord = Readonly<Record<string, unknown>>;
|
|
6
10
|
|
|
11
|
+
export interface TrailExampleTarget {
|
|
12
|
+
readonly composes: readonly string[];
|
|
13
|
+
readonly current: boolean;
|
|
14
|
+
readonly examples: readonly TrailExample<unknown, unknown>[];
|
|
15
|
+
readonly id: string;
|
|
16
|
+
readonly input: Trail<unknown, unknown, unknown>['input'];
|
|
17
|
+
readonly output: Trail<unknown, unknown, unknown>['output'];
|
|
18
|
+
readonly trail: Trail<unknown, unknown, unknown>;
|
|
19
|
+
readonly version?: number | undefined;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const normalizeComposeRef = (value: string | { readonly id: string }): string =>
|
|
23
|
+
typeof value === 'string' ? value : value.id;
|
|
24
|
+
|
|
7
25
|
/**
|
|
8
26
|
* Tracks examples that `deriveTrailExamples` synthesizes from contour
|
|
9
27
|
* fixtures. Authored examples are passed through untouched and never
|
|
10
28
|
* appear here, so consumers can distinguish the two by identity.
|
|
11
29
|
*
|
|
12
30
|
* Exposed via `isDerivedExample` so downstream testing helpers (e.g.
|
|
13
|
-
* `testExamples`
|
|
31
|
+
* `testExamples` composing coverage) can relax invariants that only make
|
|
14
32
|
* sense for authored inputs.
|
|
15
33
|
*/
|
|
16
34
|
const derivedExamples = new WeakSet<TrailExample<unknown, unknown>>();
|
|
@@ -288,7 +306,7 @@ const formatFixtureName = (
|
|
|
288
306
|
* Examples returned by this helper come from one of two provenances:
|
|
289
307
|
* - **Authored.** When `trail.examples` is non-empty, its entries are
|
|
290
308
|
* returned verbatim. These are the developer's stated intent and carry
|
|
291
|
-
* full invariants — including
|
|
309
|
+
* full invariants — including composing-coverage assertions in
|
|
292
310
|
* `testExamples`.
|
|
293
311
|
* - **Derived.** When there are no authored examples but the trail has
|
|
294
312
|
* contours with examples, candidate inputs are synthesized from contour
|
|
@@ -348,3 +366,49 @@ export const deriveTrailExamples = (
|
|
|
348
366
|
return [derived];
|
|
349
367
|
});
|
|
350
368
|
};
|
|
369
|
+
|
|
370
|
+
export const deriveTrailExampleTargets = (
|
|
371
|
+
trail: Trail<unknown, unknown, unknown>
|
|
372
|
+
): readonly TrailExampleTarget[] => {
|
|
373
|
+
const targets: TrailExampleTarget[] = [];
|
|
374
|
+
const currentExamples = deriveTrailExamples(trail);
|
|
375
|
+
if (currentExamples.length > 0) {
|
|
376
|
+
targets.push({
|
|
377
|
+
composes: trail.composes,
|
|
378
|
+
current: true,
|
|
379
|
+
examples: currentExamples,
|
|
380
|
+
id: trail.id,
|
|
381
|
+
input: trail.input,
|
|
382
|
+
output: trail.output,
|
|
383
|
+
trail,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
for (const [rawVersion, entry] of Object.entries(
|
|
388
|
+
trail.versions ?? {}
|
|
389
|
+
).toSorted(([left], [right]) => Number(left) - Number(right))) {
|
|
390
|
+
if (isArchivedTrailVersionEntry(entry)) {
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
const examples = entry.examples ?? [];
|
|
394
|
+
if (examples.length === 0) {
|
|
395
|
+
continue;
|
|
396
|
+
}
|
|
397
|
+
const kind = getTrailVersionEntryKind(entry);
|
|
398
|
+
targets.push({
|
|
399
|
+
composes:
|
|
400
|
+
kind === 'fork'
|
|
401
|
+
? (entry.composes ?? []).map(normalizeComposeRef)
|
|
402
|
+
: trail.composes,
|
|
403
|
+
current: false,
|
|
404
|
+
examples,
|
|
405
|
+
id: `${trail.id}@${rawVersion}`,
|
|
406
|
+
input: entry.input,
|
|
407
|
+
output: entry.output,
|
|
408
|
+
trail,
|
|
409
|
+
version: Number(rawVersion),
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
return targets;
|
|
414
|
+
};
|