@ontrails/testing 1.0.0-beta.12 → 1.0.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -27
- package/README.md +9 -9
- package/dist/context.d.ts +11 -11
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +23 -21
- package/dist/context.js.map +1 -1
- package/dist/contracts.js +8 -8
- package/dist/contracts.js.map +1 -1
- package/dist/crosses.d.ts +38 -0
- package/dist/crosses.d.ts.map +1 -0
- package/dist/crosses.js +213 -0
- package/dist/crosses.js.map +1 -0
- package/dist/examples.d.ts +4 -4
- package/dist/examples.js +31 -31
- package/dist/examples.js.map +1 -1
- package/dist/harness-mcp.d.ts +1 -1
- package/dist/harness-mcp.js +2 -2
- package/dist/harness-mcp.js.map +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/trail.js +1 -1
- package/dist/trail.js.map +1 -1
- package/dist/types.d.ts +8 -8
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/__tests__/all.test.ts +20 -18
- package/src/__tests__/context.test.ts +27 -27
- package/src/__tests__/contracts.test.ts +29 -29
- package/src/__tests__/{follows.test.ts → crosses.test.ts} +193 -195
- package/src/__tests__/detours.test.ts +3 -3
- package/src/__tests__/examples.test.ts +145 -144
- package/src/__tests__/trail.test.ts +4 -4
- package/src/context.ts +36 -32
- package/src/contracts.ts +12 -12
- package/src/{follows.ts → crosses.ts} +97 -92
- package/src/examples.ts +43 -43
- package/src/harness-mcp.ts +2 -2
- package/src/index.ts +6 -6
- package/src/trail.ts +1 -1
- package/src/types.ts +9 -9
- package/tsconfig.tsbuildinfo +1 -1
package/src/examples.ts
CHANGED
|
@@ -3,15 +3,15 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Iterates every trail in the app's topo. For each trail with examples,
|
|
5
5
|
* generates describe/test blocks using bun:test. Progressive assertion
|
|
6
|
-
* determines which check to run per example. For trails with `
|
|
7
|
-
* declarations, checks that every declared
|
|
6
|
+
* determines which check to run per example. For trails with `crosses`
|
|
7
|
+
* declarations, checks that every declared crossing was called at least once.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { describe, expect, test } from 'bun:test';
|
|
11
11
|
|
|
12
12
|
import type {
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
CrossFn,
|
|
14
|
+
ProvisionOverrideMap,
|
|
15
15
|
Topo,
|
|
16
16
|
TrailExample,
|
|
17
17
|
Trail,
|
|
@@ -46,10 +46,10 @@ import {
|
|
|
46
46
|
} from './assertions.js';
|
|
47
47
|
import {
|
|
48
48
|
defaultMintPermit,
|
|
49
|
-
|
|
49
|
+
mergeProvisionOverrides,
|
|
50
50
|
mergeTestContext,
|
|
51
51
|
normalizeTestExecutionOptions,
|
|
52
|
-
|
|
52
|
+
resolveMockProvisions,
|
|
53
53
|
} from './context.js';
|
|
54
54
|
import type { MintableTrail, TestExecutionOptions } from './context.js';
|
|
55
55
|
|
|
@@ -160,7 +160,7 @@ const runExample = async (
|
|
|
160
160
|
example: TrailExample<unknown, unknown>,
|
|
161
161
|
output: z.ZodType | undefined,
|
|
162
162
|
testCtx: TrailContext,
|
|
163
|
-
|
|
163
|
+
provisions?: ProvisionOverrideMap,
|
|
164
164
|
opts?: TestExecutionOptions
|
|
165
165
|
): Promise<void> => {
|
|
166
166
|
const validated = validateInput(t.input, example.input);
|
|
@@ -173,51 +173,51 @@ const runExample = async (
|
|
|
173
173
|
|
|
174
174
|
const result = await executeTrail(t, example.input, {
|
|
175
175
|
ctx,
|
|
176
|
-
|
|
176
|
+
provisions: provisions ?? opts?.provisions,
|
|
177
177
|
});
|
|
178
178
|
assertProgressiveMatch(result, example, output);
|
|
179
179
|
};
|
|
180
180
|
|
|
181
181
|
// ---------------------------------------------------------------------------
|
|
182
|
-
//
|
|
182
|
+
// Crossing coverage for trails with crossings
|
|
183
183
|
// ---------------------------------------------------------------------------
|
|
184
184
|
|
|
185
185
|
/**
|
|
186
|
-
* Build a recording
|
|
186
|
+
* Build a recording cross function that tracks which trail IDs are called.
|
|
187
187
|
*
|
|
188
|
-
* Delegates to `
|
|
188
|
+
* Delegates to `baseCross` when available, otherwise looks up the trail
|
|
189
189
|
* in the topo and executes it with validated input. Falls back to
|
|
190
190
|
* `Result.ok()` when neither is available.
|
|
191
191
|
*/
|
|
192
|
-
const
|
|
192
|
+
const createCoverageCross = (
|
|
193
193
|
called: Set<string>,
|
|
194
|
-
|
|
194
|
+
baseCross: CrossFn | undefined,
|
|
195
195
|
topo: Topo,
|
|
196
196
|
ctx: TrailContext,
|
|
197
|
-
|
|
198
|
-
):
|
|
199
|
-
const
|
|
197
|
+
provisions?: ProvisionOverrideMap
|
|
198
|
+
): CrossFn => {
|
|
199
|
+
const cross = (id: string, input: unknown) => {
|
|
200
200
|
called.add(id);
|
|
201
201
|
|
|
202
|
-
if (
|
|
203
|
-
return
|
|
202
|
+
if (baseCross !== undefined) {
|
|
203
|
+
return baseCross(id, input);
|
|
204
204
|
}
|
|
205
205
|
|
|
206
206
|
const trailDef = topo.get(id);
|
|
207
207
|
if (trailDef !== undefined) {
|
|
208
208
|
return executeTrail(trailDef, input, {
|
|
209
|
-
ctx: { ...ctx,
|
|
210
|
-
|
|
209
|
+
ctx: { ...ctx, cross },
|
|
210
|
+
provisions,
|
|
211
211
|
});
|
|
212
212
|
}
|
|
213
213
|
|
|
214
214
|
return Promise.resolve(Result.ok());
|
|
215
215
|
};
|
|
216
|
-
return
|
|
216
|
+
return cross as CrossFn;
|
|
217
217
|
};
|
|
218
218
|
|
|
219
219
|
/**
|
|
220
|
-
* Run a single example against a
|
|
220
|
+
* Run a single example against a trail with crossings, recording cross calls.
|
|
221
221
|
*/
|
|
222
222
|
const runCompositionExample = async (
|
|
223
223
|
trailDef: Trail<unknown, unknown>,
|
|
@@ -226,7 +226,7 @@ const runCompositionExample = async (
|
|
|
226
226
|
baseCtx: TrailContext,
|
|
227
227
|
called: Set<string>,
|
|
228
228
|
topo: Topo,
|
|
229
|
-
|
|
229
|
+
provisions?: ProvisionOverrideMap,
|
|
230
230
|
opts?: TestExecutionOptions
|
|
231
231
|
): Promise<void> => {
|
|
232
232
|
const validated = validateInput(trailDef.input, example.input);
|
|
@@ -236,18 +236,18 @@ const runCompositionExample = async (
|
|
|
236
236
|
}
|
|
237
237
|
|
|
238
238
|
const mintedCtx = opts ? applyAutoMint(baseCtx, trailDef, opts) : baseCtx;
|
|
239
|
-
const
|
|
239
|
+
const cross = createCoverageCross(
|
|
240
240
|
called,
|
|
241
|
-
mintedCtx.
|
|
241
|
+
mintedCtx.cross,
|
|
242
242
|
topo,
|
|
243
243
|
mintedCtx,
|
|
244
|
-
|
|
244
|
+
provisions
|
|
245
245
|
);
|
|
246
|
-
const testCtx: TrailContext = { ...mintedCtx,
|
|
246
|
+
const testCtx: TrailContext = { ...mintedCtx, cross };
|
|
247
247
|
|
|
248
248
|
const result = await executeTrail(trailDef, example.input, {
|
|
249
249
|
ctx: testCtx,
|
|
250
|
-
|
|
250
|
+
provisions: provisions ?? opts?.provisions,
|
|
251
251
|
});
|
|
252
252
|
assertProgressiveMatch(result, example, output);
|
|
253
253
|
};
|
|
@@ -259,8 +259,8 @@ const runCompositionExample = async (
|
|
|
259
259
|
/**
|
|
260
260
|
* Generate describe/test blocks for every trail example in the app.
|
|
261
261
|
*
|
|
262
|
-
* For trails with `
|
|
263
|
-
* every declared
|
|
262
|
+
* For trails with `crosses` declarations and examples, also verifies that
|
|
263
|
+
* every declared crossed ID was called at least once across all examples.
|
|
264
264
|
*
|
|
265
265
|
* One line in your test file:
|
|
266
266
|
* ```ts
|
|
@@ -281,8 +281,8 @@ export const testExamples = (
|
|
|
281
281
|
const withExamples = allTrails.filter(
|
|
282
282
|
(t) => t.examples !== undefined && t.examples.length > 0
|
|
283
283
|
);
|
|
284
|
-
const simpleTrails = withExamples.filter((t) => t.
|
|
285
|
-
const compositionTrails = withExamples.filter((t) => t.
|
|
284
|
+
const simpleTrails = withExamples.filter((t) => t.crosses.length === 0);
|
|
285
|
+
const compositionTrails = withExamples.filter((t) => t.crosses.length > 0);
|
|
286
286
|
|
|
287
287
|
// Simple trails: run examples directly
|
|
288
288
|
if (simpleTrails.length > 0) {
|
|
@@ -296,19 +296,19 @@ export const testExamples = (
|
|
|
296
296
|
'example: $name',
|
|
297
297
|
async (example: TrailExample<unknown, unknown>) => {
|
|
298
298
|
const resolved = normalizeTestExecutionOptions(resolveInput());
|
|
299
|
-
const
|
|
300
|
-
await
|
|
299
|
+
const provisions = mergeProvisionOverrides(
|
|
300
|
+
await resolveMockProvisions(app),
|
|
301
301
|
resolved.ctx,
|
|
302
|
-
resolved.
|
|
302
|
+
resolved.provisions
|
|
303
303
|
);
|
|
304
304
|
const testCtx = mergeTestContext(resolved.ctx);
|
|
305
|
-
await runExample(t, example, output, testCtx,
|
|
305
|
+
await runExample(t, example, output, testCtx, provisions, resolved);
|
|
306
306
|
}
|
|
307
307
|
);
|
|
308
308
|
});
|
|
309
309
|
}
|
|
310
310
|
|
|
311
|
-
// Composition trails: use recording
|
|
311
|
+
// Composition trails: use recording cross and check coverage
|
|
312
312
|
if (compositionTrails.length > 0) {
|
|
313
313
|
describe.each(compositionTrails)('$id', (t) => {
|
|
314
314
|
const { examples, output } = t;
|
|
@@ -322,10 +322,10 @@ export const testExamples = (
|
|
|
322
322
|
'example: $name',
|
|
323
323
|
async (example: TrailExample<unknown, unknown>) => {
|
|
324
324
|
const resolved = normalizeTestExecutionOptions(resolveInput());
|
|
325
|
-
const
|
|
326
|
-
await
|
|
325
|
+
const provisions = mergeProvisionOverrides(
|
|
326
|
+
await resolveMockProvisions(app),
|
|
327
327
|
resolved.ctx,
|
|
328
|
-
resolved.
|
|
328
|
+
resolved.provisions
|
|
329
329
|
);
|
|
330
330
|
const baseCtx = mergeTestContext(resolved.ctx);
|
|
331
331
|
await runCompositionExample(
|
|
@@ -335,14 +335,14 @@ export const testExamples = (
|
|
|
335
335
|
baseCtx,
|
|
336
336
|
called,
|
|
337
337
|
app,
|
|
338
|
-
|
|
338
|
+
provisions,
|
|
339
339
|
resolved
|
|
340
340
|
);
|
|
341
341
|
}
|
|
342
342
|
);
|
|
343
343
|
|
|
344
|
-
test('
|
|
345
|
-
const uncovered = t.
|
|
344
|
+
test('crossing coverage', () => {
|
|
345
|
+
const uncovered = t.crosses.filter((id) => !called.has(id));
|
|
346
346
|
expect(uncovered).toEqual([]);
|
|
347
347
|
});
|
|
348
348
|
});
|
package/src/harness-mcp.ts
CHANGED
|
@@ -22,7 +22,7 @@ import type {
|
|
|
22
22
|
* Create an MCP harness for integration testing.
|
|
23
23
|
*
|
|
24
24
|
* Builds MCP tools from the app's topo and provides a `callTool()` method
|
|
25
|
-
* that invokes tools directly without any transport
|
|
25
|
+
* that invokes tools directly without any transport boundary.
|
|
26
26
|
*
|
|
27
27
|
* ```ts
|
|
28
28
|
* const harness = createMcpHarness({ app });
|
|
@@ -54,9 +54,9 @@ export const createMcpHarness = (options: McpHarnessOptions): McpHarness => {
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
const result = await tool.handler(args, {
|
|
57
|
+
abortSignal: undefined,
|
|
57
58
|
progressToken: undefined,
|
|
58
59
|
sendProgress: undefined,
|
|
59
|
-
signal: undefined,
|
|
60
60
|
});
|
|
61
61
|
|
|
62
62
|
return {
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Contract-driven testing
|
|
2
2
|
export { testAll } from './all.js';
|
|
3
3
|
export { testExamples } from './examples.js';
|
|
4
|
-
export {
|
|
4
|
+
export { testCrosses } from './crosses.js';
|
|
5
5
|
export { testTrail } from './trail.js';
|
|
6
6
|
export { testContracts } from './contracts.js';
|
|
7
7
|
export { testDetours } from './detours.js';
|
|
@@ -17,27 +17,27 @@ export {
|
|
|
17
17
|
|
|
18
18
|
// Mock factories
|
|
19
19
|
export {
|
|
20
|
-
|
|
20
|
+
createCrossContext,
|
|
21
21
|
createTestContext,
|
|
22
22
|
defaultMintPermit,
|
|
23
23
|
} from './context.js';
|
|
24
24
|
export { createTestLogger } from './logger.js';
|
|
25
25
|
|
|
26
|
-
//
|
|
26
|
+
// Trailhead harnesses
|
|
27
27
|
export { createCliHarness } from './harness-cli.js';
|
|
28
28
|
export { createMcpHarness } from './harness-mcp.js';
|
|
29
29
|
|
|
30
30
|
// Types
|
|
31
|
-
export type {
|
|
31
|
+
export type { CreateCrossContextOptions } from './context.js';
|
|
32
32
|
export type {
|
|
33
33
|
MintableTrail,
|
|
34
34
|
MintedPermit,
|
|
35
35
|
TestExecutionOptions,
|
|
36
36
|
} from './context.js';
|
|
37
|
-
export type {
|
|
37
|
+
export type { TestCrossOptions } from './crosses.js';
|
|
38
38
|
|
|
39
39
|
export type {
|
|
40
|
-
|
|
40
|
+
CrossScenario,
|
|
41
41
|
TestScenario,
|
|
42
42
|
TestLogger,
|
|
43
43
|
TestTrailContextOptions,
|
package/src/trail.ts
CHANGED
|
@@ -82,7 +82,7 @@ const runScenario = async (
|
|
|
82
82
|
}
|
|
83
83
|
const validatedInput = expectOk(validated);
|
|
84
84
|
|
|
85
|
-
const result = await trailDef.
|
|
85
|
+
const result = await trailDef.blaze(validatedInput, testCtx);
|
|
86
86
|
assertScenarioResult(result, scenario, trailDef);
|
|
87
87
|
};
|
|
88
88
|
|
package/src/types.ts
CHANGED
|
@@ -26,16 +26,16 @@ export interface TestScenario {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
// ---------------------------------------------------------------------------
|
|
29
|
-
//
|
|
29
|
+
// Cross Scenario (for testCrosses)
|
|
30
30
|
// ---------------------------------------------------------------------------
|
|
31
31
|
|
|
32
|
-
/** A test scenario for a trail's
|
|
33
|
-
export interface
|
|
34
|
-
/** Assert these trail IDs were
|
|
35
|
-
readonly
|
|
36
|
-
/** Assert
|
|
37
|
-
readonly
|
|
38
|
-
/** Inject failure from a
|
|
32
|
+
/** A test scenario for a trail's crossing graph. */
|
|
33
|
+
export interface CrossScenario extends TestScenario {
|
|
34
|
+
/** Assert these trail IDs were crossed, in order. */
|
|
35
|
+
readonly expectCrossed?: readonly string[] | undefined;
|
|
36
|
+
/** Assert crossing counts per trail ID. */
|
|
37
|
+
readonly expectCrossedCount?: Readonly<Record<string, number>> | undefined;
|
|
38
|
+
/** Inject failure from a crossed trail's example by description. */
|
|
39
39
|
readonly injectFromExample?: Readonly<Record<string, string>> | undefined;
|
|
40
40
|
}
|
|
41
41
|
|
|
@@ -65,7 +65,7 @@ export interface TestTrailContextOptions {
|
|
|
65
65
|
readonly env?: Record<string, string> | undefined;
|
|
66
66
|
readonly logger?: Logger | undefined;
|
|
67
67
|
readonly requestId?: string | undefined;
|
|
68
|
-
readonly
|
|
68
|
+
readonly abortSignal?: AbortSignal | undefined;
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
// ---------------------------------------------------------------------------
|
package/tsconfig.tsbuildinfo
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"root":["./src/all.ts","./src/assertions.ts","./src/context.ts","./src/contracts.ts","./src/
|
|
1
|
+
{"root":["./src/all.ts","./src/assertions.ts","./src/context.ts","./src/contracts.ts","./src/crosses.ts","./src/detours.ts","./src/examples.ts","./src/harness-cli.ts","./src/harness-mcp.ts","./src/index.ts","./src/logger.ts","./src/trail.ts","./src/types.ts"],"version":"5.9.3"}
|