@ontrails/testing 1.0.0-beta.16 → 1.0.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +45 -1
- package/package.json +8 -7
- package/src/all.ts +25 -0
- package/src/harness-cli.ts +16 -3
- package/src/harness-http.ts +263 -0
- package/src/harness-mcp.ts +2 -0
- package/src/index.ts +17 -0
- package/src/surface-parity.ts +356 -0
- package/src/types.ts +135 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,34 @@
|
|
|
1
1
|
# @ontrails/testing
|
|
2
2
|
|
|
3
|
+
## 1.0.0-beta.18
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies [c0b2948]
|
|
8
|
+
- Updated dependencies [fc3219c]
|
|
9
|
+
- Updated dependencies [bc2d327]
|
|
10
|
+
- Updated dependencies [bf44972]
|
|
11
|
+
- Updated dependencies [e0ae995]
|
|
12
|
+
- @ontrails/http@1.0.0-beta.18
|
|
13
|
+
- @ontrails/observe@1.0.0-beta.18
|
|
14
|
+
- @ontrails/cli@1.0.0-beta.18
|
|
15
|
+
- @ontrails/core@1.0.0-beta.18
|
|
16
|
+
- @ontrails/mcp@1.0.0-beta.18
|
|
17
|
+
|
|
18
|
+
## 1.0.0-beta.17
|
|
19
|
+
|
|
20
|
+
### Patch Changes
|
|
21
|
+
|
|
22
|
+
- ce42573: Add an example-driven CLI/MCP/HTTP surface parity helper.
|
|
23
|
+
- e1eb4ee: Add an HTTP surface harness and include HTTP projection validation in `testAllEstablished`.
|
|
24
|
+
- Updated dependencies [3dc8254]
|
|
25
|
+
- Updated dependencies [61497c5]
|
|
26
|
+
- @ontrails/core@1.0.0-beta.17
|
|
27
|
+
- @ontrails/cli@1.0.0-beta.17
|
|
28
|
+
- @ontrails/http@1.0.0-beta.17
|
|
29
|
+
- @ontrails/mcp@1.0.0-beta.17
|
|
30
|
+
- @ontrails/observe@1.0.0-beta.17
|
|
31
|
+
|
|
3
32
|
## 1.0.0-beta.16
|
|
4
33
|
|
|
5
34
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -32,11 +32,13 @@ testDetours(graph); // Validate detour constructor, recover, and ordering sem
|
|
|
32
32
|
| `testTrail(trail, scenarios)` | Custom scenarios for edge cases, error paths, and cross chains |
|
|
33
33
|
| `testContracts(topo, ctx?)` | Validate output against declared schemas |
|
|
34
34
|
| `testDetours(topo)` | Validate detour constructor, recover, and shadowing semantics |
|
|
35
|
+
| `testSurfaceParity(topo, options?)` | Run trail examples through CLI, MCP, and HTTP and compare normalized semantics |
|
|
35
36
|
| `createCrossContext(options?)` | Mock `CrossFn` for testing composite trails; returns preconfigured `Result` values keyed by trail ID |
|
|
36
37
|
| `createTestContext(options?)` | `TrailContext` with sensible test defaults |
|
|
37
38
|
| `createTestLogger()` | Logger that captures entries in memory for assertions |
|
|
38
39
|
| `createCliHarness(options)` | Execute CLI commands in-process, capture stdout/stderr |
|
|
39
40
|
| `createMcpHarness(options)` | Invoke MCP tools directly without transport |
|
|
41
|
+
| `createHttpHarness(options)` | Execute HTTP route projections in-process without a server |
|
|
40
42
|
|
|
41
43
|
See the [API Reference](../../docs/api-reference.md) for the full list.
|
|
42
44
|
|
|
@@ -89,7 +91,11 @@ Calls to unregistered trail IDs return `Result.err` with a descriptive message,
|
|
|
89
91
|
## Surface Harnesses
|
|
90
92
|
|
|
91
93
|
```typescript
|
|
92
|
-
import {
|
|
94
|
+
import {
|
|
95
|
+
createCliHarness,
|
|
96
|
+
createHttpHarness,
|
|
97
|
+
createMcpHarness,
|
|
98
|
+
} from '@ontrails/testing';
|
|
93
99
|
|
|
94
100
|
// CLI
|
|
95
101
|
const cli = createCliHarness({ graph });
|
|
@@ -100,8 +106,46 @@ expect(result.exitCode).toBe(0);
|
|
|
100
106
|
const mcp = createMcpHarness({ graph });
|
|
101
107
|
const tool = await mcp.callTool('myapp_entity_show', { name: 'Alpha' });
|
|
102
108
|
expect(tool.isError).toBe(false);
|
|
109
|
+
|
|
110
|
+
// HTTP
|
|
111
|
+
const http = createHttpHarness({ graph });
|
|
112
|
+
const response = await http.get('/entity/show', { name: 'Alpha' });
|
|
113
|
+
expect(response.status).toBe(200);
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Surface Parity
|
|
117
|
+
|
|
118
|
+
`testSurfaceParity()` is a focused gate for established apps that want to prove
|
|
119
|
+
their examples behave the same through every shipped surface.
|
|
120
|
+
|
|
121
|
+
```typescript
|
|
122
|
+
import { testSurfaceParity } from '@ontrails/testing';
|
|
123
|
+
import { graph } from '../app';
|
|
124
|
+
|
|
125
|
+
const createDeterministicTestDb = () => ({});
|
|
126
|
+
|
|
127
|
+
testSurfaceParity(graph, {
|
|
128
|
+
createResources: () => ({
|
|
129
|
+
'db.main': createDeterministicTestDb(),
|
|
130
|
+
}),
|
|
131
|
+
exclusions: [
|
|
132
|
+
{
|
|
133
|
+
example: 'Creates generated data',
|
|
134
|
+
reason: 'output includes generated IDs that intentionally differ per surface run',
|
|
135
|
+
trailId: 'entity.add',
|
|
136
|
+
},
|
|
137
|
+
],
|
|
138
|
+
});
|
|
103
139
|
```
|
|
104
140
|
|
|
141
|
+
The helper compares normalized success payloads and normalized TrailsError
|
|
142
|
+
category/code pairs. CLI command names, MCP tool names, HTTP method/path values,
|
|
143
|
+
transport envelopes, activation consumers, internal trails, and planned
|
|
144
|
+
WebSocket work stay outside the equality check. Use `createResources` when
|
|
145
|
+
examples need deterministic fixtures for each surface invocation, and use
|
|
146
|
+
exclusions for intentional semantic differences that should not be silently
|
|
147
|
+
skipped.
|
|
148
|
+
|
|
105
149
|
## Installation
|
|
106
150
|
|
|
107
151
|
```bash
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ontrails/testing",
|
|
3
|
-
"version": "1.0.0-beta.
|
|
3
|
+
"version": "1.0.0-beta.18",
|
|
4
4
|
"files": [
|
|
5
5
|
"src/**/*.ts",
|
|
6
6
|
"!src/**/__tests__/**",
|
|
@@ -22,14 +22,15 @@
|
|
|
22
22
|
"clean": "rm -rf dist *.tsbuildinfo"
|
|
23
23
|
},
|
|
24
24
|
"devDependencies": {
|
|
25
|
-
"@ontrails/drizzle": "^1.0.0-beta.
|
|
26
|
-
"@ontrails/store": "^1.0.0-beta.
|
|
25
|
+
"@ontrails/drizzle": "^1.0.0-beta.17",
|
|
26
|
+
"@ontrails/store": "^1.0.0-beta.17"
|
|
27
27
|
},
|
|
28
28
|
"peerDependencies": {
|
|
29
|
-
"@ontrails/cli": "^1.0.0-beta.
|
|
30
|
-
"@ontrails/core": "^1.0.0-beta.
|
|
31
|
-
"@ontrails/
|
|
32
|
-
"@ontrails/
|
|
29
|
+
"@ontrails/cli": "^1.0.0-beta.17",
|
|
30
|
+
"@ontrails/core": "^1.0.0-beta.17",
|
|
31
|
+
"@ontrails/http": "^1.0.0-beta.17",
|
|
32
|
+
"@ontrails/mcp": "^1.0.0-beta.17",
|
|
33
|
+
"@ontrails/observe": "^1.0.0-beta.17",
|
|
33
34
|
"zod": "^4.3.5"
|
|
34
35
|
}
|
|
35
36
|
}
|
package/src/all.ts
CHANGED
|
@@ -11,6 +11,7 @@ import type { Topo, TrailContext } from '@ontrails/core';
|
|
|
11
11
|
import { validateEstablishedTopo, validateTopo } from '@ontrails/core';
|
|
12
12
|
|
|
13
13
|
import { createCliHarness } from './harness-cli.js';
|
|
14
|
+
import { createHttpHarness } from './harness-http.js';
|
|
14
15
|
import { createMcpHarness } from './harness-mcp.js';
|
|
15
16
|
import { testContracts } from './contracts.js';
|
|
16
17
|
import type { TestExecutionOptions } from './context.js';
|
|
@@ -108,6 +109,7 @@ const isEstablishedOptions = (
|
|
|
108
109
|
(Object.hasOwn(input, 'cli') ||
|
|
109
110
|
Object.hasOwn(input, 'createPermit') ||
|
|
110
111
|
Object.hasOwn(input, 'ctx') ||
|
|
112
|
+
Object.hasOwn(input, 'http') ||
|
|
111
113
|
Object.hasOwn(input, 'mcp') ||
|
|
112
114
|
Object.hasOwn(input, 'resources') ||
|
|
113
115
|
Object.hasOwn(input, 'strictPermits'));
|
|
@@ -154,6 +156,22 @@ const toMcpHarnessOptions = (
|
|
|
154
156
|
...options.mcp,
|
|
155
157
|
});
|
|
156
158
|
|
|
159
|
+
const toHttpHarnessOptions = (
|
|
160
|
+
topo: Topo,
|
|
161
|
+
options: TestAllEstablishedOptions
|
|
162
|
+
) => {
|
|
163
|
+
const httpOptions = {
|
|
164
|
+
graph: topo,
|
|
165
|
+
...options.http,
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
if (options.ctx !== undefined) {
|
|
169
|
+
httpOptions.ctx = options.ctx;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return httpOptions;
|
|
173
|
+
};
|
|
174
|
+
|
|
157
175
|
const registerEstablishedSurfaceSuite = (
|
|
158
176
|
topo: Topo,
|
|
159
177
|
resolveInput: () =>
|
|
@@ -175,6 +193,13 @@ const registerEstablishedSurfaceSuite = (
|
|
|
175
193
|
createMcpHarness(toMcpHarnessOptions(topo, options))
|
|
176
194
|
).not.toThrow();
|
|
177
195
|
});
|
|
196
|
+
|
|
197
|
+
test('HTTP projection validates established topo', () => {
|
|
198
|
+
const options = normalizeEstablishedOptions(resolveInput());
|
|
199
|
+
expect(() =>
|
|
200
|
+
createHttpHarness(toHttpHarnessOptions(topo, options))
|
|
201
|
+
).not.toThrow();
|
|
202
|
+
});
|
|
178
203
|
});
|
|
179
204
|
};
|
|
180
205
|
|
package/src/harness-cli.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { deriveCliCommands } from '@ontrails/cli';
|
|
9
9
|
import type { CliCommand } from '@ontrails/cli';
|
|
10
10
|
import type { TrailContext } from '@ontrails/core';
|
|
11
|
+
import { projectPublicSurfaceError } from '@ontrails/core';
|
|
11
12
|
|
|
12
13
|
import { mergeTestContext } from './context.js';
|
|
13
14
|
import type {
|
|
@@ -207,10 +208,16 @@ const buildErrorResult = (
|
|
|
207
208
|
streams: CapturedStreams
|
|
208
209
|
): CliHarnessResult => {
|
|
209
210
|
streams.restore();
|
|
210
|
-
const
|
|
211
|
+
const actualError = error instanceof Error ? error : new Error(String(error));
|
|
212
|
+
const projection = projectPublicSurfaceError('cli', actualError);
|
|
211
213
|
return {
|
|
214
|
+
error: {
|
|
215
|
+
category: projection.category,
|
|
216
|
+
code: projection.name,
|
|
217
|
+
message: projection.message,
|
|
218
|
+
},
|
|
212
219
|
exitCode: 1,
|
|
213
|
-
stderr: streams.getStderr() || message,
|
|
220
|
+
stderr: streams.getStderr() || projection.message,
|
|
214
221
|
stdout: streams.getStdout(),
|
|
215
222
|
};
|
|
216
223
|
};
|
|
@@ -227,9 +234,15 @@ const executeCommand = async (
|
|
|
227
234
|
streams.restore();
|
|
228
235
|
|
|
229
236
|
if (result.isErr()) {
|
|
237
|
+
const projection = projectPublicSurfaceError('cli', result.error);
|
|
230
238
|
return {
|
|
239
|
+
error: {
|
|
240
|
+
category: projection.category,
|
|
241
|
+
code: projection.name,
|
|
242
|
+
message: projection.message,
|
|
243
|
+
},
|
|
231
244
|
exitCode: 1,
|
|
232
|
-
stderr: streams.getStderr() ||
|
|
245
|
+
stderr: streams.getStderr() || projection.message,
|
|
233
246
|
stdout: streams.getStdout(),
|
|
234
247
|
};
|
|
235
248
|
}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTTP integration test harness.
|
|
3
|
+
*
|
|
4
|
+
* Builds framework-agnostic HTTP routes from a graph and executes them
|
|
5
|
+
* directly, without Hono or a listening server.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { deriveHttpRoutes } from '@ontrails/http';
|
|
9
|
+
import type { HttpMethod, HttpRouteDefinition } from '@ontrails/http';
|
|
10
|
+
import type { TrailContext, TrailContextInit } from '@ontrails/core';
|
|
11
|
+
import { NotFoundError, projectPublicSurfaceError } from '@ontrails/core';
|
|
12
|
+
|
|
13
|
+
import { mergeTestContext } from './context.js';
|
|
14
|
+
import type {
|
|
15
|
+
HttpHarness,
|
|
16
|
+
HttpHarnessOptions,
|
|
17
|
+
HttpHarnessRequest,
|
|
18
|
+
HttpHarnessRequestOptions,
|
|
19
|
+
HttpHarnessResult,
|
|
20
|
+
} from './types.js';
|
|
21
|
+
|
|
22
|
+
const TEST_ORIGIN = 'http://ontrails.test';
|
|
23
|
+
|
|
24
|
+
const normalizeMethod = (method: HttpMethod): HttpMethod =>
|
|
25
|
+
method.toUpperCase() as HttpMethod;
|
|
26
|
+
|
|
27
|
+
const collectQueryParams = (url: URL): Record<string, unknown> => {
|
|
28
|
+
const query: Record<string, unknown> = {};
|
|
29
|
+
const seen = new Set<string>();
|
|
30
|
+
|
|
31
|
+
for (const key of url.searchParams.keys()) {
|
|
32
|
+
if (seen.has(key)) {
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
seen.add(key);
|
|
36
|
+
const values = url.searchParams.getAll(key);
|
|
37
|
+
query[key] = values.length > 1 ? values : values[0];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return query;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const findRoute = (
|
|
44
|
+
routes: readonly HttpRouteDefinition[],
|
|
45
|
+
method: HttpMethod,
|
|
46
|
+
path: string
|
|
47
|
+
): HttpRouteDefinition | undefined =>
|
|
48
|
+
routes.find((route) => route.method === method && route.path === path);
|
|
49
|
+
|
|
50
|
+
const mapError = (error: Error): HttpHarnessResult => {
|
|
51
|
+
const projection = projectPublicSurfaceError('http', error);
|
|
52
|
+
const body = {
|
|
53
|
+
error: {
|
|
54
|
+
category: projection.category,
|
|
55
|
+
code: projection.name,
|
|
56
|
+
message: projection.message,
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
return {
|
|
60
|
+
body,
|
|
61
|
+
error: body.error,
|
|
62
|
+
ok: false,
|
|
63
|
+
status: projection.code,
|
|
64
|
+
};
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
const mapSuccess = (data: unknown): HttpHarnessResult => ({
|
|
68
|
+
body: { data },
|
|
69
|
+
data,
|
|
70
|
+
ok: true,
|
|
71
|
+
status: 200,
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
const isWebhookParseResult = (
|
|
75
|
+
input: unknown
|
|
76
|
+
): input is {
|
|
77
|
+
readonly error?: Error | undefined;
|
|
78
|
+
isErr(): boolean;
|
|
79
|
+
readonly value?: unknown | undefined;
|
|
80
|
+
} =>
|
|
81
|
+
typeof input === 'object' &&
|
|
82
|
+
input !== null &&
|
|
83
|
+
'isErr' in input &&
|
|
84
|
+
typeof input.isErr === 'function';
|
|
85
|
+
|
|
86
|
+
const mergeContextInit = (
|
|
87
|
+
base: TrailContextInit | undefined,
|
|
88
|
+
ctx: Partial<TrailContext> | undefined
|
|
89
|
+
): TrailContextInit => ({
|
|
90
|
+
...base,
|
|
91
|
+
...mergeTestContext({ ...base, ...ctx }),
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const createHarnessContextFactory = (
|
|
95
|
+
options: HttpHarnessOptions
|
|
96
|
+
): (() => TrailContextInit | Promise<TrailContextInit>) => {
|
|
97
|
+
const { createContext, ctx } = options;
|
|
98
|
+
return async () => {
|
|
99
|
+
const base = await createContext?.();
|
|
100
|
+
return mergeContextInit(base, ctx);
|
|
101
|
+
};
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
const buildInput = (
|
|
105
|
+
route: HttpRouteDefinition,
|
|
106
|
+
url: URL,
|
|
107
|
+
request: HttpHarnessRequest
|
|
108
|
+
): unknown => {
|
|
109
|
+
if (route.inputSource === 'query') {
|
|
110
|
+
return {
|
|
111
|
+
...collectQueryParams(url),
|
|
112
|
+
...request.query,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
return request.body ?? {};
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
const executeRouteWithInput = async (
|
|
119
|
+
route: HttpRouteDefinition,
|
|
120
|
+
input: unknown,
|
|
121
|
+
request: HttpHarnessRequest
|
|
122
|
+
): Promise<HttpHarnessResult> => {
|
|
123
|
+
const result = await route.execute(
|
|
124
|
+
input,
|
|
125
|
+
request.requestId,
|
|
126
|
+
request.abortSignal,
|
|
127
|
+
{ headers: request.headers }
|
|
128
|
+
);
|
|
129
|
+
|
|
130
|
+
if (result.isErr()) {
|
|
131
|
+
return mapError(result.error);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return mapSuccess(result.value);
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
const executeRoute = async (
|
|
138
|
+
route: HttpRouteDefinition,
|
|
139
|
+
url: URL,
|
|
140
|
+
request: HttpHarnessRequest
|
|
141
|
+
): Promise<HttpHarnessResult> => {
|
|
142
|
+
const parsedInput = buildInput(route, url, request);
|
|
143
|
+
if (route.inputSource === 'webhook' && route.parseWebhookInput) {
|
|
144
|
+
const parsed = route.parseWebhookInput(parsedInput);
|
|
145
|
+
if (isWebhookParseResult(parsed)) {
|
|
146
|
+
if (parsed.isErr()) {
|
|
147
|
+
return mapError(parsed.error ?? new Error('Invalid webhook input'));
|
|
148
|
+
}
|
|
149
|
+
return await executeRouteWithInput(route, parsed.value, request);
|
|
150
|
+
}
|
|
151
|
+
return await executeRouteWithInput(route, parsed, request);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return await executeRouteWithInput(route, parsedInput, request);
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
// ---------------------------------------------------------------------------
|
|
158
|
+
// createHttpHarness
|
|
159
|
+
// ---------------------------------------------------------------------------
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Create an HTTP harness for integration testing.
|
|
163
|
+
*
|
|
164
|
+
* @example
|
|
165
|
+
* ```ts
|
|
166
|
+
* import { createHttpHarness } from '@ontrails/testing';
|
|
167
|
+
*
|
|
168
|
+
* const http = createHttpHarness({ graph });
|
|
169
|
+
* const result = await http.get('/entity/show', { name: 'Alpha' });
|
|
170
|
+
* expect(result.status).toBe(200);
|
|
171
|
+
* ```
|
|
172
|
+
*/
|
|
173
|
+
export const createHttpHarness = (
|
|
174
|
+
harnessOptions: HttpHarnessOptions
|
|
175
|
+
): HttpHarness => {
|
|
176
|
+
const { ctx: _ctx, graph, ...deriveOptions } = harnessOptions;
|
|
177
|
+
const routesResult = deriveHttpRoutes(graph, {
|
|
178
|
+
...deriveOptions,
|
|
179
|
+
createContext: createHarnessContextFactory(harnessOptions),
|
|
180
|
+
});
|
|
181
|
+
if (routesResult.isErr()) {
|
|
182
|
+
throw routesResult.error;
|
|
183
|
+
}
|
|
184
|
+
const routes = routesResult.value;
|
|
185
|
+
|
|
186
|
+
const request = async (
|
|
187
|
+
rawRequest: HttpHarnessRequest
|
|
188
|
+
): Promise<HttpHarnessResult> => {
|
|
189
|
+
const method = normalizeMethod(rawRequest.method);
|
|
190
|
+
const url = new URL(rawRequest.path, TEST_ORIGIN);
|
|
191
|
+
const route = findRoute(routes, method, url.pathname);
|
|
192
|
+
|
|
193
|
+
if (!route) {
|
|
194
|
+
return mapError(
|
|
195
|
+
new NotFoundError(`No HTTP route found for ${method} ${url.pathname}`)
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
return await executeRoute(route, url, { ...rawRequest, method });
|
|
200
|
+
};
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
delete: async (
|
|
204
|
+
path: string,
|
|
205
|
+
body?: unknown,
|
|
206
|
+
requestOptions?: HttpHarnessRequestOptions
|
|
207
|
+
) =>
|
|
208
|
+
await request({
|
|
209
|
+
...requestOptions,
|
|
210
|
+
body,
|
|
211
|
+
method: 'DELETE',
|
|
212
|
+
path,
|
|
213
|
+
}),
|
|
214
|
+
get: async (
|
|
215
|
+
path: string,
|
|
216
|
+
query?: Record<string, unknown>,
|
|
217
|
+
requestOptions?: HttpHarnessRequestOptions
|
|
218
|
+
) =>
|
|
219
|
+
await request({
|
|
220
|
+
...requestOptions,
|
|
221
|
+
method: 'GET',
|
|
222
|
+
path,
|
|
223
|
+
query: {
|
|
224
|
+
...requestOptions?.query,
|
|
225
|
+
...query,
|
|
226
|
+
},
|
|
227
|
+
}),
|
|
228
|
+
patch: async (
|
|
229
|
+
path: string,
|
|
230
|
+
body?: unknown,
|
|
231
|
+
requestOptions?: HttpHarnessRequestOptions
|
|
232
|
+
) =>
|
|
233
|
+
await request({
|
|
234
|
+
...requestOptions,
|
|
235
|
+
body,
|
|
236
|
+
method: 'PATCH',
|
|
237
|
+
path,
|
|
238
|
+
}),
|
|
239
|
+
post: async (
|
|
240
|
+
path: string,
|
|
241
|
+
body?: unknown,
|
|
242
|
+
requestOptions?: HttpHarnessRequestOptions
|
|
243
|
+
) =>
|
|
244
|
+
await request({
|
|
245
|
+
...requestOptions,
|
|
246
|
+
body,
|
|
247
|
+
method: 'POST',
|
|
248
|
+
path,
|
|
249
|
+
}),
|
|
250
|
+
put: async (
|
|
251
|
+
path: string,
|
|
252
|
+
body?: unknown,
|
|
253
|
+
requestOptions?: HttpHarnessRequestOptions
|
|
254
|
+
) =>
|
|
255
|
+
await request({
|
|
256
|
+
...requestOptions,
|
|
257
|
+
body,
|
|
258
|
+
method: 'PUT',
|
|
259
|
+
path,
|
|
260
|
+
}),
|
|
261
|
+
request,
|
|
262
|
+
};
|
|
263
|
+
};
|
package/src/harness-mcp.ts
CHANGED
package/src/index.ts
CHANGED
|
@@ -6,6 +6,10 @@ export { testCrosses } from './crosses.js';
|
|
|
6
6
|
export { testTrail } from './trail.js';
|
|
7
7
|
export { testContracts } from './contracts.js';
|
|
8
8
|
export { testDetours } from './detours.js';
|
|
9
|
+
export {
|
|
10
|
+
runSurfaceParityExample,
|
|
11
|
+
testSurfaceParity,
|
|
12
|
+
} from './surface-parity.js';
|
|
9
13
|
|
|
10
14
|
// Assertions
|
|
11
15
|
export {
|
|
@@ -32,6 +36,7 @@ export { createTestLogger } from './logger.js';
|
|
|
32
36
|
|
|
33
37
|
// Surface harnesses
|
|
34
38
|
export { createCliHarness } from './harness-cli.js';
|
|
39
|
+
export { createHttpHarness } from './harness-http.js';
|
|
35
40
|
export { createMcpHarness } from './harness-mcp.js';
|
|
36
41
|
|
|
37
42
|
// Types
|
|
@@ -54,7 +59,19 @@ export type {
|
|
|
54
59
|
CliHarness,
|
|
55
60
|
CliHarnessOptions,
|
|
56
61
|
CliHarnessResult,
|
|
62
|
+
HttpHarness,
|
|
63
|
+
HttpHarnessErrorBody,
|
|
64
|
+
HttpHarnessOptions,
|
|
65
|
+
HttpHarnessRequest,
|
|
66
|
+
HttpHarnessRequestOptions,
|
|
67
|
+
HttpHarnessResult,
|
|
68
|
+
HttpHarnessSuccessBody,
|
|
57
69
|
McpHarness,
|
|
58
70
|
McpHarnessOptions,
|
|
59
71
|
McpHarnessResult,
|
|
72
|
+
NormalizedSurfaceParityResult,
|
|
73
|
+
SurfaceParityExclusion,
|
|
74
|
+
SurfaceParityOptions,
|
|
75
|
+
SurfaceParitySurface,
|
|
60
76
|
} from './types.js';
|
|
77
|
+
export type { SurfaceParityComparison } from './surface-parity.js';
|
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Example-driven parity checks across shipped surfaces.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { describe, expect, test } from 'bun:test';
|
|
6
|
+
|
|
7
|
+
import { deriveCliPath, filterSurfaceTrails } from '@ontrails/core';
|
|
8
|
+
import type { Topo, Trail, TrailContext, TrailExample } from '@ontrails/core';
|
|
9
|
+
import { deriveHttpInputSource, deriveHttpMethod } from '@ontrails/http';
|
|
10
|
+
import type { HttpMethod } from '@ontrails/http';
|
|
11
|
+
import { MCP_TOOL_ERROR_META_KEY, deriveToolName } from '@ontrails/mcp';
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
createMockResources,
|
|
15
|
+
defaultCreatePermit,
|
|
16
|
+
mergeResourceOverrides,
|
|
17
|
+
mergeTestContext,
|
|
18
|
+
} from './context.js';
|
|
19
|
+
import { deriveTrailExamples } from './effective-examples.js';
|
|
20
|
+
import { createCliHarness } from './harness-cli.js';
|
|
21
|
+
import { createHttpHarness } from './harness-http.js';
|
|
22
|
+
import { createMcpHarness } from './harness-mcp.js';
|
|
23
|
+
import type {
|
|
24
|
+
CliHarnessResult,
|
|
25
|
+
HttpHarnessResult,
|
|
26
|
+
McpHarnessResult,
|
|
27
|
+
NormalizedSurfaceParityResult,
|
|
28
|
+
SurfaceParityExclusion,
|
|
29
|
+
SurfaceParityOptions,
|
|
30
|
+
} from './types.js';
|
|
31
|
+
|
|
32
|
+
type ParityTrail = Trail<unknown, unknown, unknown>;
|
|
33
|
+
|
|
34
|
+
export interface SurfaceParityComparison {
|
|
35
|
+
readonly cli: NormalizedSurfaceParityResult;
|
|
36
|
+
readonly http: NormalizedSurfaceParityResult;
|
|
37
|
+
readonly mcp: NormalizedSurfaceParityResult;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const httpPathForTrail = (trailId: string): string =>
|
|
41
|
+
`/${trailId.replaceAll('.', '/')}`;
|
|
42
|
+
|
|
43
|
+
const escapeWhitespaceForCliToken = (value: string): string =>
|
|
44
|
+
value.replaceAll(/\s/gu, (char) => {
|
|
45
|
+
let escaped = '';
|
|
46
|
+
|
|
47
|
+
for (let index = 0; index < char.length; index += 1) {
|
|
48
|
+
const codePoint = char.codePointAt(index);
|
|
49
|
+
|
|
50
|
+
if (codePoint !== undefined) {
|
|
51
|
+
escaped += `\\u${codePoint.toString(16).padStart(4, '0')}`;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return escaped;
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
const cliCommandForExample = (
|
|
59
|
+
trail: ParityTrail,
|
|
60
|
+
example: TrailExample<unknown, unknown>
|
|
61
|
+
): string => {
|
|
62
|
+
const path = deriveCliPath(trail.id).join(' ');
|
|
63
|
+
const inputJson = escapeWhitespaceForCliToken(
|
|
64
|
+
JSON.stringify(example.input ?? {})
|
|
65
|
+
);
|
|
66
|
+
return `${path} --input-json ${inputJson} --output json`;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const isObjectRecord = (value: unknown): value is Record<string, unknown> =>
|
|
70
|
+
value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
71
|
+
|
|
72
|
+
const readTextContent = (content: unknown): string | undefined => {
|
|
73
|
+
if (!Array.isArray(content)) {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
const firstText = content.find(
|
|
77
|
+
(item): item is { readonly text: string; readonly type: string } =>
|
|
78
|
+
isObjectRecord(item) &&
|
|
79
|
+
item['type'] === 'text' &&
|
|
80
|
+
typeof item['text'] === 'string'
|
|
81
|
+
);
|
|
82
|
+
return firstText?.text;
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
const parseJsonText = (text: string | undefined): unknown => {
|
|
86
|
+
if (text === undefined) {
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
try {
|
|
90
|
+
return JSON.parse(text);
|
|
91
|
+
} catch {
|
|
92
|
+
return text;
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
const structuredContentValue = (
|
|
97
|
+
structuredContent: Record<string, unknown> | undefined
|
|
98
|
+
): unknown => {
|
|
99
|
+
if (structuredContent === undefined) {
|
|
100
|
+
return undefined;
|
|
101
|
+
}
|
|
102
|
+
const keys = Object.keys(structuredContent);
|
|
103
|
+
return keys.length === 1 && keys[0] === 'data'
|
|
104
|
+
? structuredContent['data']
|
|
105
|
+
: structuredContent;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const normalizeCliResult = (
|
|
109
|
+
result: CliHarnessResult
|
|
110
|
+
): NormalizedSurfaceParityResult =>
|
|
111
|
+
result.exitCode === 0
|
|
112
|
+
? { ok: true, value: result.json }
|
|
113
|
+
: {
|
|
114
|
+
error: {
|
|
115
|
+
category: result.error?.category ?? 'internal',
|
|
116
|
+
code: result.error?.code ?? 'InternalError',
|
|
117
|
+
},
|
|
118
|
+
ok: false,
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
const normalizeHttpResult = (
|
|
122
|
+
result: HttpHarnessResult
|
|
123
|
+
): NormalizedSurfaceParityResult =>
|
|
124
|
+
result.ok
|
|
125
|
+
? { ok: true, value: result.data }
|
|
126
|
+
: {
|
|
127
|
+
error: {
|
|
128
|
+
category: result.error?.category ?? 'internal',
|
|
129
|
+
code: result.error?.code ?? 'InternalError',
|
|
130
|
+
},
|
|
131
|
+
ok: false,
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
const readMcpError = (
|
|
135
|
+
result: McpHarnessResult
|
|
136
|
+
): { readonly category: string; readonly code: string } => {
|
|
137
|
+
const errorMeta = result.meta?.[MCP_TOOL_ERROR_META_KEY];
|
|
138
|
+
if (isObjectRecord(errorMeta)) {
|
|
139
|
+
return {
|
|
140
|
+
category:
|
|
141
|
+
typeof errorMeta['category'] === 'string'
|
|
142
|
+
? errorMeta['category']
|
|
143
|
+
: 'internal',
|
|
144
|
+
code:
|
|
145
|
+
typeof errorMeta['name'] === 'string'
|
|
146
|
+
? errorMeta['name']
|
|
147
|
+
: 'InternalError',
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
return { category: 'internal', code: 'InternalError' };
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
const normalizeMcpResult = (
|
|
154
|
+
result: McpHarnessResult
|
|
155
|
+
): NormalizedSurfaceParityResult =>
|
|
156
|
+
result.isError
|
|
157
|
+
? { error: readMcpError(result), ok: false }
|
|
158
|
+
: {
|
|
159
|
+
ok: true,
|
|
160
|
+
value:
|
|
161
|
+
result.structuredContent === undefined
|
|
162
|
+
? parseJsonText(readTextContent(result.content))
|
|
163
|
+
: structuredContentValue(result.structuredContent),
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const applyAutoPermit = (
|
|
167
|
+
ctx: TrailContext,
|
|
168
|
+
trail: ParityTrail,
|
|
169
|
+
options: SurfaceParityOptions
|
|
170
|
+
): TrailContext => {
|
|
171
|
+
if (options.strictPermits || ctx.permit !== undefined) {
|
|
172
|
+
return ctx;
|
|
173
|
+
}
|
|
174
|
+
const permit = (options.createPermit ?? defaultCreatePermit)(trail);
|
|
175
|
+
return permit === undefined ? ctx : { ...ctx, permit };
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
const createInvocationContext = async (
|
|
179
|
+
app: Topo,
|
|
180
|
+
trail: ParityTrail,
|
|
181
|
+
options: SurfaceParityOptions
|
|
182
|
+
) => {
|
|
183
|
+
const autoResources =
|
|
184
|
+
options.createResources === undefined
|
|
185
|
+
? await createMockResources(app)
|
|
186
|
+
: await options.createResources();
|
|
187
|
+
const resources = mergeResourceOverrides(
|
|
188
|
+
autoResources,
|
|
189
|
+
options.ctx,
|
|
190
|
+
options.resources
|
|
191
|
+
);
|
|
192
|
+
const ctx = applyAutoPermit(mergeTestContext(options.ctx), trail, options);
|
|
193
|
+
return { ctx, resources };
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
const mergeSurfaceContext = (
|
|
197
|
+
ctx: TrailContext,
|
|
198
|
+
surfaceCtx: Partial<TrailContext> | undefined
|
|
199
|
+
): TrailContext =>
|
|
200
|
+
surfaceCtx === undefined ? ctx : mergeTestContext({ ...ctx, ...surfaceCtx });
|
|
201
|
+
|
|
202
|
+
const runCliExample = async (
|
|
203
|
+
app: Topo,
|
|
204
|
+
trail: ParityTrail,
|
|
205
|
+
example: TrailExample<unknown, unknown>,
|
|
206
|
+
options: SurfaceParityOptions
|
|
207
|
+
): Promise<NormalizedSurfaceParityResult> => {
|
|
208
|
+
const { ctx, resources } = await createInvocationContext(app, trail, options);
|
|
209
|
+
const cliOptions = options.cli;
|
|
210
|
+
const harness = createCliHarness({
|
|
211
|
+
graph: app,
|
|
212
|
+
...cliOptions,
|
|
213
|
+
ctx: mergeSurfaceContext(ctx, cliOptions?.ctx),
|
|
214
|
+
resources,
|
|
215
|
+
});
|
|
216
|
+
return normalizeCliResult(
|
|
217
|
+
await harness.run(cliCommandForExample(trail, example))
|
|
218
|
+
);
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
const runMcpExample = async (
|
|
222
|
+
app: Topo,
|
|
223
|
+
trail: ParityTrail,
|
|
224
|
+
example: TrailExample<unknown, unknown>,
|
|
225
|
+
options: SurfaceParityOptions
|
|
226
|
+
): Promise<NormalizedSurfaceParityResult> => {
|
|
227
|
+
const { ctx, resources } = await createInvocationContext(app, trail, options);
|
|
228
|
+
const mcpOptions = options.mcp;
|
|
229
|
+
const harness = createMcpHarness({
|
|
230
|
+
graph: app,
|
|
231
|
+
...mcpOptions,
|
|
232
|
+
createContext: async () =>
|
|
233
|
+
mergeSurfaceContext(ctx, await mcpOptions?.createContext?.()),
|
|
234
|
+
resources,
|
|
235
|
+
});
|
|
236
|
+
const toolName = deriveToolName(app.name, trail.id);
|
|
237
|
+
return normalizeMcpResult(
|
|
238
|
+
await harness.callTool(
|
|
239
|
+
toolName,
|
|
240
|
+
isObjectRecord(example.input) ? example.input : {}
|
|
241
|
+
)
|
|
242
|
+
);
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
const httpRequestForExample = (
|
|
246
|
+
method: HttpMethod,
|
|
247
|
+
trailId: string,
|
|
248
|
+
example: TrailExample<unknown, unknown>
|
|
249
|
+
) => {
|
|
250
|
+
const input = isObjectRecord(example.input) ? example.input : {};
|
|
251
|
+
return deriveHttpInputSource(method) === 'query'
|
|
252
|
+
? { method, path: httpPathForTrail(trailId), query: input }
|
|
253
|
+
: { body: input, method, path: httpPathForTrail(trailId) };
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
const runHttpExample = async (
|
|
257
|
+
app: Topo,
|
|
258
|
+
trail: ParityTrail,
|
|
259
|
+
example: TrailExample<unknown, unknown>,
|
|
260
|
+
options: SurfaceParityOptions
|
|
261
|
+
): Promise<NormalizedSurfaceParityResult> => {
|
|
262
|
+
const { ctx, resources } = await createInvocationContext(app, trail, options);
|
|
263
|
+
const httpOptions = options.http;
|
|
264
|
+
const harness = createHttpHarness({
|
|
265
|
+
graph: app,
|
|
266
|
+
...httpOptions,
|
|
267
|
+
ctx: mergeSurfaceContext(ctx, httpOptions?.ctx),
|
|
268
|
+
resources,
|
|
269
|
+
});
|
|
270
|
+
const method = deriveHttpMethod(trail.intent);
|
|
271
|
+
return normalizeHttpResult(
|
|
272
|
+
await harness.request(httpRequestForExample(method, trail.id, example))
|
|
273
|
+
);
|
|
274
|
+
};
|
|
275
|
+
|
|
276
|
+
export const runSurfaceParityExample = async (
|
|
277
|
+
app: Topo,
|
|
278
|
+
trail: ParityTrail,
|
|
279
|
+
example: TrailExample<unknown, unknown>,
|
|
280
|
+
options: SurfaceParityOptions = {}
|
|
281
|
+
): Promise<SurfaceParityComparison> => {
|
|
282
|
+
// CLI harness output capture is process-scoped, so keep surface execution
|
|
283
|
+
// ordered even though MCP and HTTP do not share that constraint.
|
|
284
|
+
const cli = await runCliExample(app, trail, example, options);
|
|
285
|
+
const mcp = await runMcpExample(app, trail, example, options);
|
|
286
|
+
const http = await runHttpExample(app, trail, example, options);
|
|
287
|
+
return { cli, http, mcp };
|
|
288
|
+
};
|
|
289
|
+
|
|
290
|
+
const findExclusion = (
|
|
291
|
+
exclusions: readonly SurfaceParityExclusion[] | undefined,
|
|
292
|
+
trail: ParityTrail,
|
|
293
|
+
example: TrailExample<unknown, unknown>
|
|
294
|
+
): SurfaceParityExclusion | undefined =>
|
|
295
|
+
exclusions?.find(
|
|
296
|
+
(exclusion) =>
|
|
297
|
+
exclusion.trailId === trail.id &&
|
|
298
|
+
(exclusion.example === undefined || exclusion.example === example.name)
|
|
299
|
+
);
|
|
300
|
+
|
|
301
|
+
const parityTrails = (app: Topo): readonly ParityTrail[] =>
|
|
302
|
+
filterSurfaceTrails(app.list()).filter(
|
|
303
|
+
(trail) => deriveTrailExamples(trail).length > 0
|
|
304
|
+
);
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Register example-driven parity tests for CLI, MCP, and HTTP.
|
|
308
|
+
*
|
|
309
|
+
* @example
|
|
310
|
+
* ```ts
|
|
311
|
+
* import { testSurfaceParity } from '@ontrails/testing';
|
|
312
|
+
* import { graph } from '../src/app.js';
|
|
313
|
+
*
|
|
314
|
+
* testSurfaceParity(graph);
|
|
315
|
+
* ```
|
|
316
|
+
*/
|
|
317
|
+
export const testSurfaceParity = (
|
|
318
|
+
app: Topo,
|
|
319
|
+
optionsOrFactory?:
|
|
320
|
+
| SurfaceParityOptions
|
|
321
|
+
| (() => SurfaceParityOptions | undefined)
|
|
322
|
+
): void => {
|
|
323
|
+
const resolveOptions =
|
|
324
|
+
typeof optionsOrFactory === 'function'
|
|
325
|
+
? optionsOrFactory
|
|
326
|
+
: () => optionsOrFactory;
|
|
327
|
+
|
|
328
|
+
describe('surface parity', () => {
|
|
329
|
+
for (const trail of parityTrails(app)) {
|
|
330
|
+
describe(trail.id, () => {
|
|
331
|
+
for (const example of deriveTrailExamples(trail)) {
|
|
332
|
+
const options = resolveOptions() ?? {};
|
|
333
|
+
const exclusion = findExclusion(options.exclusions, trail, example);
|
|
334
|
+
const testName = `example: ${example.name}`;
|
|
335
|
+
if (exclusion !== undefined) {
|
|
336
|
+
test.skip(`${testName} (excluded: ${exclusion.reason})`, () => {
|
|
337
|
+
throw new Error('Skipped parity exclusion should not execute');
|
|
338
|
+
});
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
test(testName, async () => {
|
|
343
|
+
const comparison = await runSurfaceParityExample(
|
|
344
|
+
app,
|
|
345
|
+
trail,
|
|
346
|
+
example,
|
|
347
|
+
resolveOptions() ?? {}
|
|
348
|
+
);
|
|
349
|
+
expect(comparison.mcp).toEqual(comparison.cli);
|
|
350
|
+
expect(comparison.http).toEqual(comparison.cli);
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
});
|
|
356
|
+
};
|
package/src/types.ts
CHANGED
|
@@ -3,10 +3,16 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import type { DeriveCliCommandsOptions } from '@ontrails/cli';
|
|
6
|
+
import type {
|
|
7
|
+
DeriveHttpRoutesOptions,
|
|
8
|
+
HttpHeaderSource,
|
|
9
|
+
HttpMethod,
|
|
10
|
+
} from '@ontrails/http';
|
|
6
11
|
import type { McpExtra, DeriveMcpToolsOptions } from '@ontrails/mcp';
|
|
7
12
|
import type {
|
|
8
13
|
AnyTrail,
|
|
9
14
|
Logger,
|
|
15
|
+
ResourceOverrideMap,
|
|
10
16
|
Topo,
|
|
11
17
|
TraceFn,
|
|
12
18
|
TrailContext,
|
|
@@ -98,6 +104,13 @@ export interface CliHarness {
|
|
|
98
104
|
|
|
99
105
|
/** The result of a CLI harness command execution. */
|
|
100
106
|
export interface CliHarnessResult {
|
|
107
|
+
readonly error?:
|
|
108
|
+
| {
|
|
109
|
+
readonly category: string;
|
|
110
|
+
readonly code: string;
|
|
111
|
+
readonly message: string;
|
|
112
|
+
}
|
|
113
|
+
| undefined;
|
|
101
114
|
readonly exitCode: number;
|
|
102
115
|
/** Parsed JSON output if --output json was used. */
|
|
103
116
|
readonly json?: unknown | undefined;
|
|
@@ -128,8 +141,129 @@ export interface McpHarness {
|
|
|
128
141
|
export interface McpHarnessResult {
|
|
129
142
|
readonly content: unknown;
|
|
130
143
|
readonly isError: boolean;
|
|
144
|
+
readonly meta?: Record<string, unknown> | undefined;
|
|
145
|
+
readonly structuredContent?: Record<string, unknown> | undefined;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// ---------------------------------------------------------------------------
|
|
149
|
+
// HTTP Harness
|
|
150
|
+
// ---------------------------------------------------------------------------
|
|
151
|
+
|
|
152
|
+
/** Options for creating an HTTP harness. */
|
|
153
|
+
export interface HttpHarnessOptions extends DeriveHttpRoutesOptions {
|
|
154
|
+
readonly ctx?: Partial<TrailContext> | undefined;
|
|
155
|
+
readonly graph: Topo;
|
|
131
156
|
}
|
|
132
157
|
|
|
158
|
+
export interface HttpHarnessRequest {
|
|
159
|
+
readonly abortSignal?: AbortSignal | undefined;
|
|
160
|
+
readonly body?: unknown | undefined;
|
|
161
|
+
readonly headers?: HttpHeaderSource | undefined;
|
|
162
|
+
readonly method: HttpMethod;
|
|
163
|
+
readonly path: string;
|
|
164
|
+
readonly query?: Record<string, unknown> | undefined;
|
|
165
|
+
readonly requestId?: string | undefined;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export interface HttpHarnessRequestOptions extends Omit<
|
|
169
|
+
HttpHarnessRequest,
|
|
170
|
+
'body' | 'method' | 'path' | 'query'
|
|
171
|
+
> {
|
|
172
|
+
readonly query?: Record<string, unknown> | undefined;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** A test harness for HTTP route projections. */
|
|
176
|
+
export interface HttpHarness {
|
|
177
|
+
/** Execute a raw HTTP-style harness request. */
|
|
178
|
+
request(request: HttpHarnessRequest): Promise<HttpHarnessResult>;
|
|
179
|
+
/** Execute a GET request, reading input from query params. */
|
|
180
|
+
get(
|
|
181
|
+
path: string,
|
|
182
|
+
query?: Record<string, unknown>,
|
|
183
|
+
options?: HttpHarnessRequestOptions
|
|
184
|
+
): Promise<HttpHarnessResult>;
|
|
185
|
+
/** Execute a POST request, reading input from the JSON-like body value. */
|
|
186
|
+
post(
|
|
187
|
+
path: string,
|
|
188
|
+
body?: unknown,
|
|
189
|
+
options?: HttpHarnessRequestOptions
|
|
190
|
+
): Promise<HttpHarnessResult>;
|
|
191
|
+
/** Execute a PUT request. */
|
|
192
|
+
put(
|
|
193
|
+
path: string,
|
|
194
|
+
body?: unknown,
|
|
195
|
+
options?: HttpHarnessRequestOptions
|
|
196
|
+
): Promise<HttpHarnessResult>;
|
|
197
|
+
/** Execute a PATCH request. */
|
|
198
|
+
patch(
|
|
199
|
+
path: string,
|
|
200
|
+
body?: unknown,
|
|
201
|
+
options?: HttpHarnessRequestOptions
|
|
202
|
+
): Promise<HttpHarnessResult>;
|
|
203
|
+
/** Execute a DELETE request. */
|
|
204
|
+
delete(
|
|
205
|
+
path: string,
|
|
206
|
+
body?: unknown,
|
|
207
|
+
options?: HttpHarnessRequestOptions
|
|
208
|
+
): Promise<HttpHarnessResult>;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export interface HttpHarnessErrorBody {
|
|
212
|
+
readonly error: {
|
|
213
|
+
readonly category: string;
|
|
214
|
+
readonly code: string;
|
|
215
|
+
readonly message: string;
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export interface HttpHarnessSuccessBody {
|
|
220
|
+
readonly data: unknown;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** The result of an HTTP harness request. */
|
|
224
|
+
export interface HttpHarnessResult {
|
|
225
|
+
readonly body: HttpHarnessErrorBody | HttpHarnessSuccessBody;
|
|
226
|
+
readonly data?: unknown | undefined;
|
|
227
|
+
readonly error?: HttpHarnessErrorBody['error'] | undefined;
|
|
228
|
+
readonly ok: boolean;
|
|
229
|
+
readonly status: number;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// ---------------------------------------------------------------------------
|
|
233
|
+
// Surface parity
|
|
234
|
+
// ---------------------------------------------------------------------------
|
|
235
|
+
|
|
236
|
+
export type SurfaceParitySurface = 'cli' | 'mcp' | 'http';
|
|
237
|
+
|
|
238
|
+
export interface SurfaceParityExclusion {
|
|
239
|
+
/** Optional example name. Omit to exclude every example for the trail. */
|
|
240
|
+
readonly example?: string | undefined;
|
|
241
|
+
/** Human-readable reason shown in the skipped test name. */
|
|
242
|
+
readonly reason: string;
|
|
243
|
+
/** Trail ID to exclude. */
|
|
244
|
+
readonly trailId: string;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
export interface SurfaceParityOptions extends TestAllEstablishedOptions {
|
|
248
|
+
readonly createResources?:
|
|
249
|
+
| (() => ResourceOverrideMap | Promise<ResourceOverrideMap>)
|
|
250
|
+
| undefined;
|
|
251
|
+
readonly exclusions?: readonly SurfaceParityExclusion[] | undefined;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
export type NormalizedSurfaceParityResult =
|
|
255
|
+
| {
|
|
256
|
+
readonly ok: true;
|
|
257
|
+
readonly value: unknown;
|
|
258
|
+
}
|
|
259
|
+
| {
|
|
260
|
+
readonly error: {
|
|
261
|
+
readonly category: string;
|
|
262
|
+
readonly code: string;
|
|
263
|
+
};
|
|
264
|
+
readonly ok: false;
|
|
265
|
+
};
|
|
266
|
+
|
|
133
267
|
// ---------------------------------------------------------------------------
|
|
134
268
|
// Established verification
|
|
135
269
|
// ---------------------------------------------------------------------------
|
|
@@ -150,6 +284,7 @@ export interface TestAllEstablishedOptions {
|
|
|
150
284
|
| undefined)
|
|
151
285
|
| undefined;
|
|
152
286
|
readonly ctx?: Partial<TrailContext> | undefined;
|
|
287
|
+
readonly http?: Omit<HttpHarnessOptions, 'graph'> | undefined;
|
|
153
288
|
readonly mcp?: Omit<McpHarnessOptions, 'graph'> | undefined;
|
|
154
289
|
readonly resources?: Record<string, unknown> | undefined;
|
|
155
290
|
readonly strictPermits?: boolean | undefined;
|