@ontrails/testing 1.0.0-beta.2 → 1.0.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +396 -5
- package/README.md +86 -22
- package/package.json +34 -5
- package/src/all-established.ts +168 -0
- package/src/all.ts +51 -12
- package/src/assertions.ts +253 -0
- package/src/cli.ts +6 -0
- package/src/composes.ts +433 -0
- package/src/context.ts +200 -14
- package/src/contracts.ts +45 -21
- package/src/detours.ts +155 -18
- package/src/effective-examples.ts +414 -0
- package/src/errors.ts +47 -0
- package/src/examples.ts +302 -177
- package/src/harness-cli.ts +83 -58
- package/src/harness-http.ts +341 -0
- package/src/harness-mcp.ts +46 -16
- package/src/http.ts +10 -0
- package/src/index.ts +22 -14
- package/src/logger.ts +3 -1
- package/src/mcp.ts +6 -0
- package/src/scenario.ts +375 -0
- package/src/signals.ts +221 -0
- package/src/surface-parity.ts +389 -0
- package/src/trail.ts +1 -1
- package/src/types.ts +24 -52
- package/.turbo/turbo-build.log +0 -1
- package/.turbo/turbo-lint.log +0 -3
- package/.turbo/turbo-typecheck.log +0 -1
- package/dist/all.d.ts +0 -30
- package/dist/all.d.ts.map +0 -1
- package/dist/all.js +0 -47
- package/dist/all.js.map +0 -1
- package/dist/assertions.d.ts +0 -49
- package/dist/assertions.d.ts.map +0 -1
- package/dist/assertions.js +0 -84
- package/dist/assertions.js.map +0 -1
- package/dist/context.d.ts +0 -19
- package/dist/context.d.ts.map +0 -1
- package/dist/context.js +0 -33
- package/dist/context.js.map +0 -1
- package/dist/contracts.d.ts +0 -16
- package/dist/contracts.d.ts.map +0 -1
- package/dist/contracts.js +0 -56
- package/dist/contracts.js.map +0 -1
- package/dist/detours.d.ts +0 -12
- package/dist/detours.d.ts.map +0 -1
- package/dist/detours.js +0 -30
- package/dist/detours.js.map +0 -1
- package/dist/examples.d.ts +0 -22
- package/dist/examples.d.ts.map +0 -1
- package/dist/examples.js +0 -187
- package/dist/examples.js.map +0 -1
- package/dist/harness-cli.d.ts +0 -21
- package/dist/harness-cli.d.ts.map +0 -1
- package/dist/harness-cli.js +0 -213
- package/dist/harness-cli.js.map +0 -1
- package/dist/harness-mcp.d.ts +0 -21
- package/dist/harness-mcp.d.ts.map +0 -1
- package/dist/harness-mcp.js +0 -50
- package/dist/harness-mcp.js.map +0 -1
- package/dist/hike.d.ts +0 -32
- package/dist/hike.d.ts.map +0 -1
- package/dist/hike.js +0 -169
- package/dist/hike.js.map +0 -1
- package/dist/index.d.ts +0 -14
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -16
- package/dist/index.js.map +0 -1
- package/dist/logger.d.ts +0 -15
- package/dist/logger.d.ts.map +0 -1
- package/dist/logger.js +0 -87
- package/dist/logger.js.map +0 -1
- package/dist/trail.d.ts +0 -20
- package/dist/trail.d.ts.map +0 -1
- package/dist/trail.js +0 -80
- package/dist/trail.js.map +0 -1
- package/dist/types.d.ts +0 -80
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -5
- package/dist/types.js.map +0 -1
- package/src/__tests__/context.test.ts +0 -60
- package/src/__tests__/contracts.test.ts +0 -68
- package/src/__tests__/detours.test.ts +0 -55
- package/src/__tests__/examples.test.ts +0 -176
- package/src/__tests__/hike.test.ts +0 -164
- package/src/__tests__/logger.test.ts +0 -136
- package/src/__tests__/trail.test.ts +0 -99
- package/src/hike.ts +0 -283
- package/tsconfig.json +0 -9
- package/tsconfig.tsbuildinfo +0 -1
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Example-driven parity checks across shipped surfaces.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { describe, expect, test } from 'bun:test';
|
|
6
|
+
|
|
7
|
+
import { deriveCliPath, filterSurfaceTrails } from '@ontrails/core';
|
|
8
|
+
import type {
|
|
9
|
+
ResourceOverrideMap,
|
|
10
|
+
Topo,
|
|
11
|
+
Trail,
|
|
12
|
+
TrailContext,
|
|
13
|
+
TrailExample,
|
|
14
|
+
} from '@ontrails/core';
|
|
15
|
+
import { deriveHttpInputSource, deriveHttpMethod } from '@ontrails/http';
|
|
16
|
+
import type { HttpMethod } from '@ontrails/http';
|
|
17
|
+
import { MCP_TOOL_ERROR_META_KEY, deriveToolName } from '@ontrails/mcp';
|
|
18
|
+
|
|
19
|
+
import type { TestAllEstablishedOptions } from './all-established.js';
|
|
20
|
+
import {
|
|
21
|
+
createMockResources,
|
|
22
|
+
defaultCreatePermit,
|
|
23
|
+
mergeResourceOverrides,
|
|
24
|
+
mergeTestContext,
|
|
25
|
+
} from './context.js';
|
|
26
|
+
import { deriveTrailExamples } from './effective-examples.js';
|
|
27
|
+
import { createCliHarness } from './harness-cli.js';
|
|
28
|
+
import type { CliHarnessResult } from './harness-cli.js';
|
|
29
|
+
import { createHttpHarness } from './harness-http.js';
|
|
30
|
+
import type { HttpHarnessResult } from './harness-http.js';
|
|
31
|
+
import { createMcpHarness } from './harness-mcp.js';
|
|
32
|
+
import type { McpHarnessResult } from './harness-mcp.js';
|
|
33
|
+
|
|
34
|
+
type ParityTrail = Trail<unknown, unknown, unknown>;
|
|
35
|
+
|
|
36
|
+
export type SurfaceParitySurface = 'cli' | 'mcp' | 'http';
|
|
37
|
+
|
|
38
|
+
export interface SurfaceParityExclusion {
|
|
39
|
+
/** Optional example name. Omit to exclude every example for the trail. */
|
|
40
|
+
readonly example?: string | undefined;
|
|
41
|
+
/** Human-readable reason shown in the skipped test name. */
|
|
42
|
+
readonly reason: string;
|
|
43
|
+
/** Trail ID to exclude. */
|
|
44
|
+
readonly trailId: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface SurfaceParityOptions extends TestAllEstablishedOptions {
|
|
48
|
+
readonly createResources?:
|
|
49
|
+
| (() => ResourceOverrideMap | Promise<ResourceOverrideMap>)
|
|
50
|
+
| undefined;
|
|
51
|
+
readonly exclusions?: readonly SurfaceParityExclusion[] | undefined;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export type NormalizedSurfaceParityResult =
|
|
55
|
+
| {
|
|
56
|
+
readonly ok: true;
|
|
57
|
+
readonly value: unknown;
|
|
58
|
+
}
|
|
59
|
+
| {
|
|
60
|
+
readonly error: {
|
|
61
|
+
readonly category: string;
|
|
62
|
+
readonly code: string;
|
|
63
|
+
};
|
|
64
|
+
readonly ok: false;
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
export interface SurfaceParityComparison {
|
|
68
|
+
readonly cli: NormalizedSurfaceParityResult;
|
|
69
|
+
readonly http: NormalizedSurfaceParityResult;
|
|
70
|
+
readonly mcp: NormalizedSurfaceParityResult;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const httpPathForTrail = (trailId: string): string =>
|
|
74
|
+
`/${trailId.replaceAll('.', '/')}`;
|
|
75
|
+
|
|
76
|
+
const escapeWhitespaceForCliToken = (value: string): string =>
|
|
77
|
+
value.replaceAll(/\s/gu, (char) => {
|
|
78
|
+
let escaped = '';
|
|
79
|
+
|
|
80
|
+
for (let index = 0; index < char.length; index += 1) {
|
|
81
|
+
const codePoint = char.codePointAt(index);
|
|
82
|
+
|
|
83
|
+
if (codePoint !== undefined) {
|
|
84
|
+
escaped += `\\u${codePoint.toString(16).padStart(4, '0')}`;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return escaped;
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
const cliCommandForExample = (
|
|
92
|
+
trail: ParityTrail,
|
|
93
|
+
example: TrailExample<unknown, unknown>
|
|
94
|
+
): string => {
|
|
95
|
+
const path = deriveCliPath(trail.id).join(' ');
|
|
96
|
+
const inputJson = escapeWhitespaceForCliToken(
|
|
97
|
+
JSON.stringify(example.input ?? {})
|
|
98
|
+
);
|
|
99
|
+
return `${path} --input-json ${inputJson} --output json`;
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
const isObjectRecord = (value: unknown): value is Record<string, unknown> =>
|
|
103
|
+
value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
104
|
+
|
|
105
|
+
const readTextContent = (content: unknown): string | undefined => {
|
|
106
|
+
if (!Array.isArray(content)) {
|
|
107
|
+
return undefined;
|
|
108
|
+
}
|
|
109
|
+
const firstText = content.find(
|
|
110
|
+
(item): item is { readonly text: string; readonly type: string } =>
|
|
111
|
+
isObjectRecord(item) &&
|
|
112
|
+
item['type'] === 'text' &&
|
|
113
|
+
typeof item['text'] === 'string'
|
|
114
|
+
);
|
|
115
|
+
return firstText?.text;
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
const parseJsonText = (text: string | undefined): unknown => {
|
|
119
|
+
if (text === undefined) {
|
|
120
|
+
return undefined;
|
|
121
|
+
}
|
|
122
|
+
try {
|
|
123
|
+
return JSON.parse(text);
|
|
124
|
+
} catch {
|
|
125
|
+
return text;
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
const structuredContentValue = (
|
|
130
|
+
structuredContent: Record<string, unknown> | undefined
|
|
131
|
+
): unknown => {
|
|
132
|
+
if (structuredContent === undefined) {
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
135
|
+
const keys = Object.keys(structuredContent);
|
|
136
|
+
return keys.length === 1 && keys[0] === 'data'
|
|
137
|
+
? structuredContent['data']
|
|
138
|
+
: structuredContent;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
const normalizeCliResult = (
|
|
142
|
+
result: CliHarnessResult
|
|
143
|
+
): NormalizedSurfaceParityResult =>
|
|
144
|
+
result.exitCode === 0
|
|
145
|
+
? { ok: true, value: result.json }
|
|
146
|
+
: {
|
|
147
|
+
error: {
|
|
148
|
+
category: result.error?.category ?? 'internal',
|
|
149
|
+
code: result.error?.code ?? 'InternalError',
|
|
150
|
+
},
|
|
151
|
+
ok: false,
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
const normalizeHttpResult = (
|
|
155
|
+
result: HttpHarnessResult
|
|
156
|
+
): NormalizedSurfaceParityResult =>
|
|
157
|
+
result.ok
|
|
158
|
+
? { ok: true, value: result.data }
|
|
159
|
+
: {
|
|
160
|
+
error: {
|
|
161
|
+
category: result.error?.category ?? 'internal',
|
|
162
|
+
code: result.error?.code ?? 'InternalError',
|
|
163
|
+
},
|
|
164
|
+
ok: false,
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
const readMcpError = (
|
|
168
|
+
result: McpHarnessResult
|
|
169
|
+
): { readonly category: string; readonly code: string } => {
|
|
170
|
+
const errorMeta = result.meta?.[MCP_TOOL_ERROR_META_KEY];
|
|
171
|
+
if (isObjectRecord(errorMeta)) {
|
|
172
|
+
return {
|
|
173
|
+
category:
|
|
174
|
+
typeof errorMeta['category'] === 'string'
|
|
175
|
+
? errorMeta['category']
|
|
176
|
+
: 'internal',
|
|
177
|
+
code:
|
|
178
|
+
typeof errorMeta['name'] === 'string'
|
|
179
|
+
? errorMeta['name']
|
|
180
|
+
: 'InternalError',
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
return { category: 'internal', code: 'InternalError' };
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
const normalizeMcpResult = (
|
|
187
|
+
result: McpHarnessResult
|
|
188
|
+
): NormalizedSurfaceParityResult =>
|
|
189
|
+
result.isError
|
|
190
|
+
? { error: readMcpError(result), ok: false }
|
|
191
|
+
: {
|
|
192
|
+
ok: true,
|
|
193
|
+
value:
|
|
194
|
+
result.structuredContent === undefined
|
|
195
|
+
? parseJsonText(readTextContent(result.content))
|
|
196
|
+
: structuredContentValue(result.structuredContent),
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
const applyAutoPermit = (
|
|
200
|
+
ctx: TrailContext,
|
|
201
|
+
trail: ParityTrail,
|
|
202
|
+
options: SurfaceParityOptions
|
|
203
|
+
): TrailContext => {
|
|
204
|
+
if (options.strictPermits || ctx.permit !== undefined) {
|
|
205
|
+
return ctx;
|
|
206
|
+
}
|
|
207
|
+
const permit = (options.createPermit ?? defaultCreatePermit)(trail);
|
|
208
|
+
return permit === undefined ? ctx : { ...ctx, permit };
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
const createInvocationContext = async (
|
|
212
|
+
app: Topo,
|
|
213
|
+
trail: ParityTrail,
|
|
214
|
+
options: SurfaceParityOptions
|
|
215
|
+
) => {
|
|
216
|
+
const autoResources =
|
|
217
|
+
options.createResources === undefined
|
|
218
|
+
? await createMockResources(app)
|
|
219
|
+
: await options.createResources();
|
|
220
|
+
const resources = mergeResourceOverrides(
|
|
221
|
+
autoResources,
|
|
222
|
+
options.ctx,
|
|
223
|
+
options.resources
|
|
224
|
+
);
|
|
225
|
+
const ctx = applyAutoPermit(mergeTestContext(options.ctx), trail, options);
|
|
226
|
+
return { ctx, resources };
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
const mergeSurfaceContext = (
|
|
230
|
+
ctx: TrailContext,
|
|
231
|
+
surfaceCtx: Partial<TrailContext> | undefined
|
|
232
|
+
): TrailContext =>
|
|
233
|
+
surfaceCtx === undefined ? ctx : mergeTestContext({ ...ctx, ...surfaceCtx });
|
|
234
|
+
|
|
235
|
+
const runCliExample = async (
|
|
236
|
+
app: Topo,
|
|
237
|
+
trail: ParityTrail,
|
|
238
|
+
example: TrailExample<unknown, unknown>,
|
|
239
|
+
options: SurfaceParityOptions
|
|
240
|
+
): Promise<NormalizedSurfaceParityResult> => {
|
|
241
|
+
const { ctx, resources } = await createInvocationContext(app, trail, options);
|
|
242
|
+
const cliOptions = options.cli;
|
|
243
|
+
const harness = createCliHarness({
|
|
244
|
+
graph: app,
|
|
245
|
+
...cliOptions,
|
|
246
|
+
ctx: mergeSurfaceContext(ctx, cliOptions?.ctx),
|
|
247
|
+
resources,
|
|
248
|
+
});
|
|
249
|
+
return normalizeCliResult(
|
|
250
|
+
await harness.run(cliCommandForExample(trail, example))
|
|
251
|
+
);
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
const runMcpExample = async (
|
|
255
|
+
app: Topo,
|
|
256
|
+
trail: ParityTrail,
|
|
257
|
+
example: TrailExample<unknown, unknown>,
|
|
258
|
+
options: SurfaceParityOptions
|
|
259
|
+
): Promise<NormalizedSurfaceParityResult> => {
|
|
260
|
+
const { ctx, resources } = await createInvocationContext(app, trail, options);
|
|
261
|
+
const mcpOptions = options.mcp;
|
|
262
|
+
const harness = createMcpHarness({
|
|
263
|
+
graph: app,
|
|
264
|
+
...mcpOptions,
|
|
265
|
+
createContext: async () =>
|
|
266
|
+
mergeSurfaceContext(ctx, await mcpOptions?.createContext?.()),
|
|
267
|
+
resources,
|
|
268
|
+
});
|
|
269
|
+
const toolName = deriveToolName(app.name, trail.id);
|
|
270
|
+
return normalizeMcpResult(
|
|
271
|
+
await harness.callTool(
|
|
272
|
+
toolName,
|
|
273
|
+
isObjectRecord(example.input) ? example.input : {}
|
|
274
|
+
)
|
|
275
|
+
);
|
|
276
|
+
};
|
|
277
|
+
|
|
278
|
+
const httpRequestForExample = (
|
|
279
|
+
method: HttpMethod,
|
|
280
|
+
trailId: string,
|
|
281
|
+
example: TrailExample<unknown, unknown>
|
|
282
|
+
) => {
|
|
283
|
+
const input = isObjectRecord(example.input) ? example.input : {};
|
|
284
|
+
return deriveHttpInputSource(method) === 'query'
|
|
285
|
+
? { method, path: httpPathForTrail(trailId), query: input }
|
|
286
|
+
: { body: input, method, path: httpPathForTrail(trailId) };
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
const runHttpExample = async (
|
|
290
|
+
app: Topo,
|
|
291
|
+
trail: ParityTrail,
|
|
292
|
+
example: TrailExample<unknown, unknown>,
|
|
293
|
+
options: SurfaceParityOptions
|
|
294
|
+
): Promise<NormalizedSurfaceParityResult> => {
|
|
295
|
+
const { ctx, resources } = await createInvocationContext(app, trail, options);
|
|
296
|
+
const httpOptions = options.http;
|
|
297
|
+
const harness = createHttpHarness({
|
|
298
|
+
graph: app,
|
|
299
|
+
...httpOptions,
|
|
300
|
+
ctx: mergeSurfaceContext(ctx, httpOptions?.ctx),
|
|
301
|
+
resources,
|
|
302
|
+
});
|
|
303
|
+
const method = deriveHttpMethod(trail.intent);
|
|
304
|
+
return normalizeHttpResult(
|
|
305
|
+
await harness.request(httpRequestForExample(method, trail.id, example))
|
|
306
|
+
);
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
export const runSurfaceParityExample = async (
|
|
310
|
+
app: Topo,
|
|
311
|
+
trail: ParityTrail,
|
|
312
|
+
example: TrailExample<unknown, unknown>,
|
|
313
|
+
options: SurfaceParityOptions = {}
|
|
314
|
+
): Promise<SurfaceParityComparison> => {
|
|
315
|
+
// CLI harness output capture is process-scoped, so keep surface execution
|
|
316
|
+
// ordered even though MCP and HTTP do not share that constraint.
|
|
317
|
+
const cli = await runCliExample(app, trail, example, options);
|
|
318
|
+
const mcp = await runMcpExample(app, trail, example, options);
|
|
319
|
+
const http = await runHttpExample(app, trail, example, options);
|
|
320
|
+
return { cli, http, mcp };
|
|
321
|
+
};
|
|
322
|
+
|
|
323
|
+
const findExclusion = (
|
|
324
|
+
exclusions: readonly SurfaceParityExclusion[] | undefined,
|
|
325
|
+
trail: ParityTrail,
|
|
326
|
+
example: TrailExample<unknown, unknown>
|
|
327
|
+
): SurfaceParityExclusion | undefined =>
|
|
328
|
+
exclusions?.find(
|
|
329
|
+
(exclusion) =>
|
|
330
|
+
exclusion.trailId === trail.id &&
|
|
331
|
+
(exclusion.example === undefined || exclusion.example === example.name)
|
|
332
|
+
);
|
|
333
|
+
|
|
334
|
+
const parityTrails = (app: Topo): readonly ParityTrail[] =>
|
|
335
|
+
filterSurfaceTrails(app.list()).filter(
|
|
336
|
+
(trail) => deriveTrailExamples(trail).length > 0
|
|
337
|
+
);
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Register example-driven parity tests for CLI, MCP, and HTTP.
|
|
341
|
+
*
|
|
342
|
+
* @example
|
|
343
|
+
* ```ts
|
|
344
|
+
* import { testSurfaceParity } from '@ontrails/testing/surface-parity';
|
|
345
|
+
* import { graph } from '../src/app.js';
|
|
346
|
+
*
|
|
347
|
+
* testSurfaceParity(graph);
|
|
348
|
+
* ```
|
|
349
|
+
*/
|
|
350
|
+
export const testSurfaceParity = (
|
|
351
|
+
app: Topo,
|
|
352
|
+
optionsOrFactory?:
|
|
353
|
+
| SurfaceParityOptions
|
|
354
|
+
| (() => SurfaceParityOptions | undefined)
|
|
355
|
+
): void => {
|
|
356
|
+
const resolveOptions =
|
|
357
|
+
typeof optionsOrFactory === 'function'
|
|
358
|
+
? optionsOrFactory
|
|
359
|
+
: () => optionsOrFactory;
|
|
360
|
+
|
|
361
|
+
describe('surface parity', () => {
|
|
362
|
+
for (const trail of parityTrails(app)) {
|
|
363
|
+
describe(trail.id, () => {
|
|
364
|
+
for (const example of deriveTrailExamples(trail)) {
|
|
365
|
+
const options = resolveOptions() ?? {};
|
|
366
|
+
const exclusion = findExclusion(options.exclusions, trail, example);
|
|
367
|
+
const testName = `example: ${example.name}`;
|
|
368
|
+
if (exclusion !== undefined) {
|
|
369
|
+
test.skip(`${testName} (excluded: ${exclusion.reason})`, () => {
|
|
370
|
+
throw new Error('Skipped parity exclusion should not execute');
|
|
371
|
+
});
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
test(testName, async () => {
|
|
376
|
+
const comparison = await runSurfaceParityExample(
|
|
377
|
+
app,
|
|
378
|
+
trail,
|
|
379
|
+
example,
|
|
380
|
+
resolveOptions() ?? {}
|
|
381
|
+
);
|
|
382
|
+
expect(comparison.mcp).toEqual(comparison.cli);
|
|
383
|
+
expect(comparison.http).toEqual(comparison.cli);
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
});
|
|
389
|
+
};
|
package/src/trail.ts
CHANGED
|
@@ -82,7 +82,7 @@ const runScenario = async (
|
|
|
82
82
|
}
|
|
83
83
|
const validatedInput = expectOk(validated);
|
|
84
84
|
|
|
85
|
-
const result = await trailDef.
|
|
85
|
+
const result = await trailDef.blaze(validatedInput, testCtx);
|
|
86
86
|
assertScenarioResult(result, scenario, trailDef);
|
|
87
87
|
};
|
|
88
88
|
|
package/src/types.ts
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* Shared types for @ontrails/testing.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import type { Logger,
|
|
6
|
-
import type { LogLevel, LogRecord } from '@ontrails/
|
|
5
|
+
import type { AnyTrail, Logger, TraceFn } from '@ontrails/core';
|
|
6
|
+
import type { LogLevel, LogRecord } from '@ontrails/observe';
|
|
7
7
|
|
|
8
8
|
// ---------------------------------------------------------------------------
|
|
9
9
|
// Test Scenario (for testTrail)
|
|
@@ -26,16 +26,16 @@ export interface TestScenario {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
// ---------------------------------------------------------------------------
|
|
29
|
-
//
|
|
29
|
+
// Compose Scenario (for testComposes)
|
|
30
30
|
// ---------------------------------------------------------------------------
|
|
31
31
|
|
|
32
|
-
/** A test scenario for a
|
|
33
|
-
export interface
|
|
34
|
-
/** Assert these trail IDs were
|
|
35
|
-
readonly
|
|
36
|
-
/** Assert
|
|
37
|
-
readonly
|
|
38
|
-
/** Inject failure from a
|
|
32
|
+
/** A test scenario for a trail's composing graph. */
|
|
33
|
+
export interface ComposeScenario extends TestScenario {
|
|
34
|
+
/** Assert these trail IDs were composed, in order. */
|
|
35
|
+
readonly expectComposed?: readonly string[] | undefined;
|
|
36
|
+
/** Assert composing counts per trail ID. */
|
|
37
|
+
readonly expectComposedCount?: Readonly<Record<string, number>> | undefined;
|
|
38
|
+
/** Inject failure from a composed trail's example by description. */
|
|
39
39
|
readonly injectFromExample?: Readonly<Record<string, string>> | undefined;
|
|
40
40
|
}
|
|
41
41
|
|
|
@@ -65,53 +65,25 @@ export interface TestTrailContextOptions {
|
|
|
65
65
|
readonly env?: Record<string, string> | undefined;
|
|
66
66
|
readonly logger?: Logger | undefined;
|
|
67
67
|
readonly requestId?: string | undefined;
|
|
68
|
-
readonly
|
|
68
|
+
readonly abortSignal?: AbortSignal | undefined;
|
|
69
|
+
readonly trace?: TraceFn | undefined;
|
|
69
70
|
}
|
|
70
71
|
|
|
71
72
|
// ---------------------------------------------------------------------------
|
|
72
|
-
//
|
|
73
|
+
// Scenario (for composition testing)
|
|
73
74
|
// ---------------------------------------------------------------------------
|
|
74
75
|
|
|
75
|
-
/**
|
|
76
|
-
export interface
|
|
77
|
-
readonly
|
|
76
|
+
/** Marker for compose-step references in scenario inputs. */
|
|
77
|
+
export interface RefToken {
|
|
78
|
+
readonly __ref: true;
|
|
79
|
+
readonly path: string;
|
|
78
80
|
}
|
|
79
81
|
|
|
80
|
-
/** A
|
|
81
|
-
export interface
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
export interface CliHarnessResult {
|
|
88
|
-
readonly exitCode: number;
|
|
89
|
-
/** Parsed JSON output if --output json was used. */
|
|
90
|
-
readonly json?: unknown | undefined;
|
|
91
|
-
readonly stderr: string;
|
|
92
|
-
readonly stdout: string;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
// ---------------------------------------------------------------------------
|
|
96
|
-
// MCP Harness
|
|
97
|
-
// ---------------------------------------------------------------------------
|
|
98
|
-
|
|
99
|
-
/** Options for creating an MCP harness. */
|
|
100
|
-
export interface McpHarnessOptions {
|
|
101
|
-
readonly app: Topo;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
/** A test harness for MCP tools. */
|
|
105
|
-
export interface McpHarness {
|
|
106
|
-
/** Call an MCP tool by name with arguments. */
|
|
107
|
-
callTool(
|
|
108
|
-
name: string,
|
|
109
|
-
args: Record<string, unknown>
|
|
110
|
-
): Promise<McpHarnessResult>;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
/** The result of an MCP harness tool invocation. */
|
|
114
|
-
export interface McpHarnessResult {
|
|
115
|
-
readonly content: unknown;
|
|
116
|
-
readonly isError: boolean;
|
|
82
|
+
/** A single step in a scenario. */
|
|
83
|
+
export interface ScenarioStep {
|
|
84
|
+
readonly compose: AnyTrail;
|
|
85
|
+
readonly input: Record<string, unknown>;
|
|
86
|
+
readonly as?: string | undefined;
|
|
87
|
+
readonly expected?: unknown | undefined;
|
|
88
|
+
readonly expectedMatch?: unknown | undefined;
|
|
117
89
|
}
|
package/.turbo/turbo-build.log
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
$ tsc -b
|
package/.turbo/turbo-lint.log
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
$ tsc --noEmit
|
package/dist/all.d.ts
DELETED
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* testAll — single-line governance suite for any Topo.
|
|
3
|
-
*
|
|
4
|
-
* Wraps topo validation, example execution, contract checks, and detour
|
|
5
|
-
* verification into one describe block.
|
|
6
|
-
*/
|
|
7
|
-
import type { Topo, TrailContext } from '@ontrails/core';
|
|
8
|
-
/**
|
|
9
|
-
* Run the full governance test suite for a Topo.
|
|
10
|
-
*
|
|
11
|
-
* Generates a `governance` describe block containing:
|
|
12
|
-
* - Structural validation via `validateTopo`
|
|
13
|
-
* - Example execution via `testExamples`
|
|
14
|
-
* - Output contract checks via `testContracts`
|
|
15
|
-
* - Detour target verification via `testDetours`
|
|
16
|
-
*
|
|
17
|
-
* Accepts either a static context or a factory function that produces a
|
|
18
|
-
* fresh context per test (useful when the context contains mutable state
|
|
19
|
-
* like an in-memory store).
|
|
20
|
-
*
|
|
21
|
-
* @example
|
|
22
|
-
* ```ts
|
|
23
|
-
* import { testAll } from '@ontrails/testing';
|
|
24
|
-
* import { app } from '../src/app.js';
|
|
25
|
-
*
|
|
26
|
-
* testAll(app);
|
|
27
|
-
* ```
|
|
28
|
-
*/
|
|
29
|
-
export declare const testAll: (topo: Topo, ctxOrFactory?: Partial<TrailContext> | (() => Partial<TrailContext>)) => void;
|
|
30
|
-
//# sourceMappingURL=all.d.ts.map
|
package/dist/all.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"all.d.ts","sourceRoot":"","sources":["../src/all.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAOzD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,OAAO,GAClB,MAAM,IAAI,EACV,eAAe,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC,KACnE,IAcF,CAAC"}
|
package/dist/all.js
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* testAll — single-line governance suite for any Topo.
|
|
3
|
-
*
|
|
4
|
-
* Wraps topo validation, example execution, contract checks, and detour
|
|
5
|
-
* verification into one describe block.
|
|
6
|
-
*/
|
|
7
|
-
import { describe, expect, test } from 'bun:test';
|
|
8
|
-
import { validateTopo } from '@ontrails/core';
|
|
9
|
-
import { testContracts } from './contracts.js';
|
|
10
|
-
import { testDetours } from './detours.js';
|
|
11
|
-
import { testExamples } from './examples.js';
|
|
12
|
-
/**
|
|
13
|
-
* Run the full governance test suite for a Topo.
|
|
14
|
-
*
|
|
15
|
-
* Generates a `governance` describe block containing:
|
|
16
|
-
* - Structural validation via `validateTopo`
|
|
17
|
-
* - Example execution via `testExamples`
|
|
18
|
-
* - Output contract checks via `testContracts`
|
|
19
|
-
* - Detour target verification via `testDetours`
|
|
20
|
-
*
|
|
21
|
-
* Accepts either a static context or a factory function that produces a
|
|
22
|
-
* fresh context per test (useful when the context contains mutable state
|
|
23
|
-
* like an in-memory store).
|
|
24
|
-
*
|
|
25
|
-
* @example
|
|
26
|
-
* ```ts
|
|
27
|
-
* import { testAll } from '@ontrails/testing';
|
|
28
|
-
* import { app } from '../src/app.js';
|
|
29
|
-
*
|
|
30
|
-
* testAll(app);
|
|
31
|
-
* ```
|
|
32
|
-
*/
|
|
33
|
-
export const testAll = (topo, ctxOrFactory) => {
|
|
34
|
-
describe('governance', () => {
|
|
35
|
-
test('topo validates', () => {
|
|
36
|
-
const result = validateTopo(topo);
|
|
37
|
-
expect(result.isOk()).toBe(true);
|
|
38
|
-
});
|
|
39
|
-
// oxlint-disable-next-line jest/require-hook -- these generate describe/test blocks, not setup code
|
|
40
|
-
testExamples(topo, ctxOrFactory);
|
|
41
|
-
// oxlint-disable-next-line jest/require-hook -- these generate describe/test blocks, not setup code
|
|
42
|
-
testContracts(topo, ctxOrFactory);
|
|
43
|
-
// oxlint-disable-next-line jest/require-hook -- these generate describe/test blocks, not setup code
|
|
44
|
-
testDetours(topo);
|
|
45
|
-
});
|
|
46
|
-
};
|
|
47
|
-
//# sourceMappingURL=all.js.map
|
package/dist/all.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"all.js","sourceRoot":"","sources":["../src/all.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAGlD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,IAAU,EACV,YAAoE,EAC9D,EAAE;IACR,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,IAAI,CAAC,gBAAgB,EAAE,GAAG,EAAE;YAC1B,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,oGAAoG;QACpG,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QACjC,oGAAoG;QACpG,aAAa,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAClC,oGAAoG;QACpG,WAAW,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC"}
|
package/dist/assertions.d.ts
DELETED
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Progressive assertion logic for example-driven testing.
|
|
3
|
-
*
|
|
4
|
-
* Three tiers:
|
|
5
|
-
* 1. Full match — example has `expected` output
|
|
6
|
-
* 2. Schema-only — no expected output, no error
|
|
7
|
-
* 3. Error match — example declares an error class name
|
|
8
|
-
*/
|
|
9
|
-
import type { Result } from '@ontrails/core';
|
|
10
|
-
import type { z } from 'zod';
|
|
11
|
-
/**
|
|
12
|
-
* Assert that a Result is Ok and return its value.
|
|
13
|
-
*
|
|
14
|
-
* Eliminates the `if (result.isOk())` / `as unknown as` dance in tests.
|
|
15
|
-
*
|
|
16
|
-
* @example
|
|
17
|
-
* ```typescript
|
|
18
|
-
* const value = expectOk(result);
|
|
19
|
-
* expect(value.name).toBe('Alice');
|
|
20
|
-
* ```
|
|
21
|
-
*/
|
|
22
|
-
export declare const expectOk: <T, E>(result: Result<T, E>) => T;
|
|
23
|
-
/**
|
|
24
|
-
* Assert that a Result is Err and return its error.
|
|
25
|
-
*
|
|
26
|
-
* Eliminates the `if (result.isErr())` / `as unknown as` dance in tests.
|
|
27
|
-
*
|
|
28
|
-
* @example
|
|
29
|
-
* ```typescript
|
|
30
|
-
* const error = expectErr(result);
|
|
31
|
-
* expect(error).toBeInstanceOf(ValidationError);
|
|
32
|
-
* ```
|
|
33
|
-
*/
|
|
34
|
-
export declare const expectErr: <T, E>(result: Result<T, E>) => E;
|
|
35
|
-
/**
|
|
36
|
-
* Assert that the result is ok and its value deep-equals the expected output.
|
|
37
|
-
*/
|
|
38
|
-
export declare const assertFullMatch: (result: Result<unknown, Error>, expected: unknown) => void;
|
|
39
|
-
/**
|
|
40
|
-
* Assert that the result is ok and, if an output schema is provided,
|
|
41
|
-
* the value parses against it.
|
|
42
|
-
*/
|
|
43
|
-
export declare const assertSchemaMatch: (result: Result<unknown, Error>, outputSchema: z.ZodType | undefined) => void;
|
|
44
|
-
/**
|
|
45
|
-
* Assert that the result is an error of the specified type, with optional
|
|
46
|
-
* message substring matching.
|
|
47
|
-
*/
|
|
48
|
-
export declare const assertErrorMatch: (result: Result<unknown, Error>, expectedError: new (...args: never[]) => Error, expectedMessage?: string) => void;
|
|
49
|
-
//# sourceMappingURL=assertions.d.ts.map
|
package/dist/assertions.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../src/assertions.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAM7B;;;;;;;;;;GAUG;AACH,eAAO,MAAM,QAAQ,GAAI,CAAC,EAAE,CAAC,EAAE,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,KAAG,CAGrD,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,SAAS,GAAI,CAAC,EAAE,CAAC,EAAE,QAAQ,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,KAAG,CAGtD,CAAC;AAMF;;GAEG;AACH,eAAO,MAAM,eAAe,GAC1B,QAAQ,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,EAC9B,UAAU,OAAO,KAChB,IAGF,CAAC;AAMF;;;GAGG;AACH,eAAO,MAAM,iBAAiB,GAC5B,QAAQ,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,EAC9B,cAAc,CAAC,CAAC,OAAO,GAAG,SAAS,KAClC,IAUF,CAAC;AAMF;;;GAGG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,EAC9B,eAAe,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,KAAK,KAAK,EAC9C,kBAAkB,MAAM,KACvB,IAMF,CAAC"}
|