@ontrails/testing 1.0.0-beta.11 → 1.0.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-lint.log +1 -1
- package/CHANGELOG.md +72 -22
- package/README.md +9 -9
- package/dist/context.d.ts +41 -11
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +36 -21
- package/dist/context.js.map +1 -1
- package/dist/contracts.js +8 -8
- package/dist/contracts.js.map +1 -1
- package/dist/crosses.d.ts +38 -0
- package/dist/crosses.d.ts.map +1 -0
- package/dist/crosses.js +213 -0
- package/dist/crosses.js.map +1 -0
- package/dist/examples.d.ts +4 -4
- package/dist/examples.d.ts.map +1 -1
- package/dist/examples.js +52 -32
- package/dist/examples.js.map +1 -1
- package/dist/harness-mcp.d.ts +1 -1
- package/dist/harness-mcp.js +2 -2
- package/dist/harness-mcp.js.map +1 -1
- package/dist/index.d.ts +6 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/trail.js +1 -1
- package/dist/trail.js.map +1 -1
- package/dist/types.d.ts +8 -8
- package/dist/types.d.ts.map +1 -1
- package/package.json +5 -5
- package/src/__tests__/all.test.ts +20 -18
- package/src/__tests__/context.test.ts +27 -27
- package/src/__tests__/contracts.test.ts +29 -29
- package/src/__tests__/{follows.test.ts → crosses.test.ts} +193 -195
- package/src/__tests__/detours.test.ts +3 -3
- package/src/__tests__/examples.test.ts +221 -126
- package/src/__tests__/trail.test.ts +4 -4
- package/src/context.ts +80 -32
- package/src/contracts.ts +12 -12
- package/src/{follows.ts → crosses.ts} +97 -92
- package/src/examples.ts +76 -46
- package/src/harness-mcp.ts +2 -2
- package/src/index.ts +15 -7
- package/src/trail.ts +1 -1
- package/src/types.ts +9 -9
- package/tsconfig.tsbuildinfo +1 -1
package/src/examples.ts
CHANGED
|
@@ -3,15 +3,15 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Iterates every trail in the app's topo. For each trail with examples,
|
|
5
5
|
* generates describe/test blocks using bun:test. Progressive assertion
|
|
6
|
-
* determines which check to run per example. For trails with `
|
|
7
|
-
* declarations, checks that every declared
|
|
6
|
+
* determines which check to run per example. For trails with `crosses`
|
|
7
|
+
* declarations, checks that every declared crossing was called at least once.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { describe, expect, test } from 'bun:test';
|
|
11
11
|
|
|
12
12
|
import type {
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
CrossFn,
|
|
14
|
+
ProvisionOverrideMap,
|
|
15
15
|
Topo,
|
|
16
16
|
TrailExample,
|
|
17
17
|
Trail,
|
|
@@ -45,12 +45,13 @@ import {
|
|
|
45
45
|
assertSchemaMatch,
|
|
46
46
|
} from './assertions.js';
|
|
47
47
|
import {
|
|
48
|
-
|
|
48
|
+
defaultMintPermit,
|
|
49
|
+
mergeProvisionOverrides,
|
|
49
50
|
mergeTestContext,
|
|
50
51
|
normalizeTestExecutionOptions,
|
|
51
|
-
|
|
52
|
+
resolveMockProvisions,
|
|
52
53
|
} from './context.js';
|
|
53
|
-
import type { TestExecutionOptions } from './context.js';
|
|
54
|
+
import type { MintableTrail, TestExecutionOptions } from './context.js';
|
|
54
55
|
|
|
55
56
|
// ---------------------------------------------------------------------------
|
|
56
57
|
// Error class name -> constructor map
|
|
@@ -127,6 +128,29 @@ const handleValidationError = (
|
|
|
127
128
|
);
|
|
128
129
|
};
|
|
129
130
|
|
|
131
|
+
/**
|
|
132
|
+
* Apply auto-minting: if the trail declares scoped permits and the context
|
|
133
|
+
* doesn't already have a permit, mint one and merge it into the context.
|
|
134
|
+
*/
|
|
135
|
+
const applyAutoMint = (
|
|
136
|
+
ctx: TrailContext,
|
|
137
|
+
trailDef: MintableTrail,
|
|
138
|
+
opts: TestExecutionOptions
|
|
139
|
+
): TrailContext => {
|
|
140
|
+
if (opts.strictPermits) {
|
|
141
|
+
return ctx;
|
|
142
|
+
}
|
|
143
|
+
if (ctx.permit !== undefined) {
|
|
144
|
+
return ctx;
|
|
145
|
+
}
|
|
146
|
+
const mint = opts.mintPermit ?? defaultMintPermit;
|
|
147
|
+
const permit = mint(trailDef);
|
|
148
|
+
if (!permit) {
|
|
149
|
+
return ctx;
|
|
150
|
+
}
|
|
151
|
+
return { ...ctx, permit };
|
|
152
|
+
};
|
|
153
|
+
|
|
130
154
|
/**
|
|
131
155
|
* Run a single example against a trail.
|
|
132
156
|
* Handles validation, execution, and assertions.
|
|
@@ -136,7 +160,8 @@ const runExample = async (
|
|
|
136
160
|
example: TrailExample<unknown, unknown>,
|
|
137
161
|
output: z.ZodType | undefined,
|
|
138
162
|
testCtx: TrailContext,
|
|
139
|
-
|
|
163
|
+
provisions?: ProvisionOverrideMap,
|
|
164
|
+
opts?: TestExecutionOptions
|
|
140
165
|
): Promise<void> => {
|
|
141
166
|
const validated = validateInput(t.input, example.input);
|
|
142
167
|
|
|
@@ -144,53 +169,55 @@ const runExample = async (
|
|
|
144
169
|
return;
|
|
145
170
|
}
|
|
146
171
|
|
|
172
|
+
const ctx = opts ? applyAutoMint(testCtx, t, opts) : testCtx;
|
|
173
|
+
|
|
147
174
|
const result = await executeTrail(t, example.input, {
|
|
148
|
-
ctx
|
|
149
|
-
|
|
175
|
+
ctx,
|
|
176
|
+
provisions: provisions ?? opts?.provisions,
|
|
150
177
|
});
|
|
151
178
|
assertProgressiveMatch(result, example, output);
|
|
152
179
|
};
|
|
153
180
|
|
|
154
181
|
// ---------------------------------------------------------------------------
|
|
155
|
-
//
|
|
182
|
+
// Crossing coverage for trails with crossings
|
|
156
183
|
// ---------------------------------------------------------------------------
|
|
157
184
|
|
|
158
185
|
/**
|
|
159
|
-
* Build a recording
|
|
186
|
+
* Build a recording cross function that tracks which trail IDs are called.
|
|
160
187
|
*
|
|
161
|
-
* Delegates to `
|
|
188
|
+
* Delegates to `baseCross` when available, otherwise looks up the trail
|
|
162
189
|
* in the topo and executes it with validated input. Falls back to
|
|
163
190
|
* `Result.ok()` when neither is available.
|
|
164
191
|
*/
|
|
165
|
-
const
|
|
192
|
+
const createCoverageCross = (
|
|
166
193
|
called: Set<string>,
|
|
167
|
-
|
|
194
|
+
baseCross: CrossFn | undefined,
|
|
168
195
|
topo: Topo,
|
|
169
196
|
ctx: TrailContext,
|
|
170
|
-
|
|
171
|
-
):
|
|
172
|
-
const
|
|
197
|
+
provisions?: ProvisionOverrideMap
|
|
198
|
+
): CrossFn => {
|
|
199
|
+
const cross = (id: string, input: unknown) => {
|
|
173
200
|
called.add(id);
|
|
174
201
|
|
|
175
|
-
if (
|
|
176
|
-
return
|
|
202
|
+
if (baseCross !== undefined) {
|
|
203
|
+
return baseCross(id, input);
|
|
177
204
|
}
|
|
178
205
|
|
|
179
206
|
const trailDef = topo.get(id);
|
|
180
207
|
if (trailDef !== undefined) {
|
|
181
208
|
return executeTrail(trailDef, input, {
|
|
182
|
-
ctx: { ...ctx,
|
|
183
|
-
|
|
209
|
+
ctx: { ...ctx, cross },
|
|
210
|
+
provisions,
|
|
184
211
|
});
|
|
185
212
|
}
|
|
186
213
|
|
|
187
214
|
return Promise.resolve(Result.ok());
|
|
188
215
|
};
|
|
189
|
-
return
|
|
216
|
+
return cross as CrossFn;
|
|
190
217
|
};
|
|
191
218
|
|
|
192
219
|
/**
|
|
193
|
-
* Run a single example against a
|
|
220
|
+
* Run a single example against a trail with crossings, recording cross calls.
|
|
194
221
|
*/
|
|
195
222
|
const runCompositionExample = async (
|
|
196
223
|
trailDef: Trail<unknown, unknown>,
|
|
@@ -199,7 +226,8 @@ const runCompositionExample = async (
|
|
|
199
226
|
baseCtx: TrailContext,
|
|
200
227
|
called: Set<string>,
|
|
201
228
|
topo: Topo,
|
|
202
|
-
|
|
229
|
+
provisions?: ProvisionOverrideMap,
|
|
230
|
+
opts?: TestExecutionOptions
|
|
203
231
|
): Promise<void> => {
|
|
204
232
|
const validated = validateInput(trailDef.input, example.input);
|
|
205
233
|
|
|
@@ -207,18 +235,19 @@ const runCompositionExample = async (
|
|
|
207
235
|
return;
|
|
208
236
|
}
|
|
209
237
|
|
|
210
|
-
const
|
|
238
|
+
const mintedCtx = opts ? applyAutoMint(baseCtx, trailDef, opts) : baseCtx;
|
|
239
|
+
const cross = createCoverageCross(
|
|
211
240
|
called,
|
|
212
|
-
|
|
241
|
+
mintedCtx.cross,
|
|
213
242
|
topo,
|
|
214
|
-
|
|
215
|
-
|
|
243
|
+
mintedCtx,
|
|
244
|
+
provisions
|
|
216
245
|
);
|
|
217
|
-
const testCtx: TrailContext = { ...
|
|
246
|
+
const testCtx: TrailContext = { ...mintedCtx, cross };
|
|
218
247
|
|
|
219
248
|
const result = await executeTrail(trailDef, example.input, {
|
|
220
249
|
ctx: testCtx,
|
|
221
|
-
|
|
250
|
+
provisions: provisions ?? opts?.provisions,
|
|
222
251
|
});
|
|
223
252
|
assertProgressiveMatch(result, example, output);
|
|
224
253
|
};
|
|
@@ -230,8 +259,8 @@ const runCompositionExample = async (
|
|
|
230
259
|
/**
|
|
231
260
|
* Generate describe/test blocks for every trail example in the app.
|
|
232
261
|
*
|
|
233
|
-
* For trails with `
|
|
234
|
-
* every declared
|
|
262
|
+
* For trails with `crosses` declarations and examples, also verifies that
|
|
263
|
+
* every declared crossed ID was called at least once across all examples.
|
|
235
264
|
*
|
|
236
265
|
* One line in your test file:
|
|
237
266
|
* ```ts
|
|
@@ -252,8 +281,8 @@ export const testExamples = (
|
|
|
252
281
|
const withExamples = allTrails.filter(
|
|
253
282
|
(t) => t.examples !== undefined && t.examples.length > 0
|
|
254
283
|
);
|
|
255
|
-
const simpleTrails = withExamples.filter((t) => t.
|
|
256
|
-
const compositionTrails = withExamples.filter((t) => t.
|
|
284
|
+
const simpleTrails = withExamples.filter((t) => t.crosses.length === 0);
|
|
285
|
+
const compositionTrails = withExamples.filter((t) => t.crosses.length > 0);
|
|
257
286
|
|
|
258
287
|
// Simple trails: run examples directly
|
|
259
288
|
if (simpleTrails.length > 0) {
|
|
@@ -267,19 +296,19 @@ export const testExamples = (
|
|
|
267
296
|
'example: $name',
|
|
268
297
|
async (example: TrailExample<unknown, unknown>) => {
|
|
269
298
|
const resolved = normalizeTestExecutionOptions(resolveInput());
|
|
270
|
-
const
|
|
271
|
-
await
|
|
299
|
+
const provisions = mergeProvisionOverrides(
|
|
300
|
+
await resolveMockProvisions(app),
|
|
272
301
|
resolved.ctx,
|
|
273
|
-
resolved.
|
|
302
|
+
resolved.provisions
|
|
274
303
|
);
|
|
275
304
|
const testCtx = mergeTestContext(resolved.ctx);
|
|
276
|
-
await runExample(t, example, output, testCtx,
|
|
305
|
+
await runExample(t, example, output, testCtx, provisions, resolved);
|
|
277
306
|
}
|
|
278
307
|
);
|
|
279
308
|
});
|
|
280
309
|
}
|
|
281
310
|
|
|
282
|
-
// Composition trails: use recording
|
|
311
|
+
// Composition trails: use recording cross and check coverage
|
|
283
312
|
if (compositionTrails.length > 0) {
|
|
284
313
|
describe.each(compositionTrails)('$id', (t) => {
|
|
285
314
|
const { examples, output } = t;
|
|
@@ -293,10 +322,10 @@ export const testExamples = (
|
|
|
293
322
|
'example: $name',
|
|
294
323
|
async (example: TrailExample<unknown, unknown>) => {
|
|
295
324
|
const resolved = normalizeTestExecutionOptions(resolveInput());
|
|
296
|
-
const
|
|
297
|
-
await
|
|
325
|
+
const provisions = mergeProvisionOverrides(
|
|
326
|
+
await resolveMockProvisions(app),
|
|
298
327
|
resolved.ctx,
|
|
299
|
-
resolved.
|
|
328
|
+
resolved.provisions
|
|
300
329
|
);
|
|
301
330
|
const baseCtx = mergeTestContext(resolved.ctx);
|
|
302
331
|
await runCompositionExample(
|
|
@@ -306,13 +335,14 @@ export const testExamples = (
|
|
|
306
335
|
baseCtx,
|
|
307
336
|
called,
|
|
308
337
|
app,
|
|
309
|
-
|
|
338
|
+
provisions,
|
|
339
|
+
resolved
|
|
310
340
|
);
|
|
311
341
|
}
|
|
312
342
|
);
|
|
313
343
|
|
|
314
|
-
test('
|
|
315
|
-
const uncovered = t.
|
|
344
|
+
test('crossing coverage', () => {
|
|
345
|
+
const uncovered = t.crosses.filter((id) => !called.has(id));
|
|
316
346
|
expect(uncovered).toEqual([]);
|
|
317
347
|
});
|
|
318
348
|
});
|
package/src/harness-mcp.ts
CHANGED
|
@@ -22,7 +22,7 @@ import type {
|
|
|
22
22
|
* Create an MCP harness for integration testing.
|
|
23
23
|
*
|
|
24
24
|
* Builds MCP tools from the app's topo and provides a `callTool()` method
|
|
25
|
-
* that invokes tools directly without any transport
|
|
25
|
+
* that invokes tools directly without any transport boundary.
|
|
26
26
|
*
|
|
27
27
|
* ```ts
|
|
28
28
|
* const harness = createMcpHarness({ app });
|
|
@@ -54,9 +54,9 @@ export const createMcpHarness = (options: McpHarnessOptions): McpHarness => {
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
const result = await tool.handler(args, {
|
|
57
|
+
abortSignal: undefined,
|
|
57
58
|
progressToken: undefined,
|
|
58
59
|
sendProgress: undefined,
|
|
59
|
-
signal: undefined,
|
|
60
60
|
});
|
|
61
61
|
|
|
62
62
|
return {
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Contract-driven testing
|
|
2
2
|
export { testAll } from './all.js';
|
|
3
3
|
export { testExamples } from './examples.js';
|
|
4
|
-
export {
|
|
4
|
+
export { testCrosses } from './crosses.js';
|
|
5
5
|
export { testTrail } from './trail.js';
|
|
6
6
|
export { testContracts } from './contracts.js';
|
|
7
7
|
export { testDetours } from './detours.js';
|
|
@@ -16,20 +16,28 @@ export {
|
|
|
16
16
|
} from './assertions.js';
|
|
17
17
|
|
|
18
18
|
// Mock factories
|
|
19
|
-
export {
|
|
19
|
+
export {
|
|
20
|
+
createCrossContext,
|
|
21
|
+
createTestContext,
|
|
22
|
+
defaultMintPermit,
|
|
23
|
+
} from './context.js';
|
|
20
24
|
export { createTestLogger } from './logger.js';
|
|
21
25
|
|
|
22
|
-
//
|
|
26
|
+
// Trailhead harnesses
|
|
23
27
|
export { createCliHarness } from './harness-cli.js';
|
|
24
28
|
export { createMcpHarness } from './harness-mcp.js';
|
|
25
29
|
|
|
26
30
|
// Types
|
|
27
|
-
export type {
|
|
28
|
-
export type {
|
|
29
|
-
|
|
31
|
+
export type { CreateCrossContextOptions } from './context.js';
|
|
32
|
+
export type {
|
|
33
|
+
MintableTrail,
|
|
34
|
+
MintedPermit,
|
|
35
|
+
TestExecutionOptions,
|
|
36
|
+
} from './context.js';
|
|
37
|
+
export type { TestCrossOptions } from './crosses.js';
|
|
30
38
|
|
|
31
39
|
export type {
|
|
32
|
-
|
|
40
|
+
CrossScenario,
|
|
33
41
|
TestScenario,
|
|
34
42
|
TestLogger,
|
|
35
43
|
TestTrailContextOptions,
|
package/src/trail.ts
CHANGED
|
@@ -82,7 +82,7 @@ const runScenario = async (
|
|
|
82
82
|
}
|
|
83
83
|
const validatedInput = expectOk(validated);
|
|
84
84
|
|
|
85
|
-
const result = await trailDef.
|
|
85
|
+
const result = await trailDef.blaze(validatedInput, testCtx);
|
|
86
86
|
assertScenarioResult(result, scenario, trailDef);
|
|
87
87
|
};
|
|
88
88
|
|
package/src/types.ts
CHANGED
|
@@ -26,16 +26,16 @@ export interface TestScenario {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
// ---------------------------------------------------------------------------
|
|
29
|
-
//
|
|
29
|
+
// Cross Scenario (for testCrosses)
|
|
30
30
|
// ---------------------------------------------------------------------------
|
|
31
31
|
|
|
32
|
-
/** A test scenario for a trail's
|
|
33
|
-
export interface
|
|
34
|
-
/** Assert these trail IDs were
|
|
35
|
-
readonly
|
|
36
|
-
/** Assert
|
|
37
|
-
readonly
|
|
38
|
-
/** Inject failure from a
|
|
32
|
+
/** A test scenario for a trail's crossing graph. */
|
|
33
|
+
export interface CrossScenario extends TestScenario {
|
|
34
|
+
/** Assert these trail IDs were crossed, in order. */
|
|
35
|
+
readonly expectCrossed?: readonly string[] | undefined;
|
|
36
|
+
/** Assert crossing counts per trail ID. */
|
|
37
|
+
readonly expectCrossedCount?: Readonly<Record<string, number>> | undefined;
|
|
38
|
+
/** Inject failure from a crossed trail's example by description. */
|
|
39
39
|
readonly injectFromExample?: Readonly<Record<string, string>> | undefined;
|
|
40
40
|
}
|
|
41
41
|
|
|
@@ -65,7 +65,7 @@ export interface TestTrailContextOptions {
|
|
|
65
65
|
readonly env?: Record<string, string> | undefined;
|
|
66
66
|
readonly logger?: Logger | undefined;
|
|
67
67
|
readonly requestId?: string | undefined;
|
|
68
|
-
readonly
|
|
68
|
+
readonly abortSignal?: AbortSignal | undefined;
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
// ---------------------------------------------------------------------------
|
package/tsconfig.tsbuildinfo
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"root":["./src/all.ts","./src/assertions.ts","./src/context.ts","./src/contracts.ts","./src/
|
|
1
|
+
{"root":["./src/all.ts","./src/assertions.ts","./src/context.ts","./src/contracts.ts","./src/crosses.ts","./src/detours.ts","./src/examples.ts","./src/harness-cli.ts","./src/harness-mcp.ts","./src/index.ts","./src/logger.ts","./src/trail.ts","./src/types.ts"],"version":"5.9.3"}
|