@ontrails/testing 1.0.0-beta.16 → 1.0.0-beta.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,19 @@
1
1
  # @ontrails/testing
2
2
 
3
+ ## 1.0.0-beta.17
4
+
5
+ ### Patch Changes
6
+
7
+ - ce42573: Add an example-driven CLI/MCP/HTTP surface parity helper.
8
+ - e1eb4ee: Add an HTTP surface harness and include HTTP projection validation in `testAllEstablished`.
9
+ - Updated dependencies [3dc8254]
10
+ - Updated dependencies [61497c5]
11
+ - @ontrails/core@1.0.0-beta.17
12
+ - @ontrails/cli@1.0.0-beta.17
13
+ - @ontrails/http@1.0.0-beta.17
14
+ - @ontrails/mcp@1.0.0-beta.17
15
+ - @ontrails/observe@1.0.0-beta.17
16
+
3
17
  ## 1.0.0-beta.16
4
18
 
5
19
  ### Minor Changes
package/README.md CHANGED
@@ -32,11 +32,13 @@ testDetours(graph); // Validate detour constructor, recover, and ordering sem
32
32
  | `testTrail(trail, scenarios)` | Custom scenarios for edge cases, error paths, and cross chains |
33
33
  | `testContracts(topo, ctx?)` | Validate output against declared schemas |
34
34
  | `testDetours(topo)` | Validate detour constructor, recover, and shadowing semantics |
35
+ | `testSurfaceParity(topo, options?)` | Run trail examples through CLI, MCP, and HTTP and compare normalized semantics |
35
36
  | `createCrossContext(options?)` | Mock `CrossFn` for testing composite trails; returns preconfigured `Result` values keyed by trail ID |
36
37
  | `createTestContext(options?)` | `TrailContext` with sensible test defaults |
37
38
  | `createTestLogger()` | Logger that captures entries in memory for assertions |
38
39
  | `createCliHarness(options)` | Execute CLI commands in-process, capture stdout/stderr |
39
40
  | `createMcpHarness(options)` | Invoke MCP tools directly without transport |
41
+ | `createHttpHarness(options)` | Execute HTTP route projections in-process without a server |
40
42
 
41
43
  See the [API Reference](../../docs/api-reference.md) for the full list.
42
44
 
@@ -89,7 +91,11 @@ Calls to unregistered trail IDs return `Result.err` with a descriptive message,
89
91
  ## Surface Harnesses
90
92
 
91
93
  ```typescript
92
- import { createCliHarness, createMcpHarness } from '@ontrails/testing';
94
+ import {
95
+ createCliHarness,
96
+ createHttpHarness,
97
+ createMcpHarness,
98
+ } from '@ontrails/testing';
93
99
 
94
100
  // CLI
95
101
  const cli = createCliHarness({ graph });
@@ -100,8 +106,46 @@ expect(result.exitCode).toBe(0);
100
106
  const mcp = createMcpHarness({ graph });
101
107
  const tool = await mcp.callTool('myapp_entity_show', { name: 'Alpha' });
102
108
  expect(tool.isError).toBe(false);
109
+
110
+ // HTTP
111
+ const http = createHttpHarness({ graph });
112
+ const response = await http.get('/entity/show', { name: 'Alpha' });
113
+ expect(response.status).toBe(200);
114
+ ```
115
+
116
+ ## Surface Parity
117
+
118
+ `testSurfaceParity()` is a focused gate for established apps that want to prove
119
+ their examples behave the same through every shipped surface.
120
+
121
+ ```typescript
122
+ import { testSurfaceParity } from '@ontrails/testing';
123
+ import { graph } from '../app';
124
+
125
+ const createDeterministicTestDb = () => ({});
126
+
127
+ testSurfaceParity(graph, {
128
+ createResources: () => ({
129
+ 'db.main': createDeterministicTestDb(),
130
+ }),
131
+ exclusions: [
132
+ {
133
+ example: 'Creates generated data',
134
+ reason: 'output includes generated IDs that intentionally differ per surface run',
135
+ trailId: 'entity.add',
136
+ },
137
+ ],
138
+ });
103
139
  ```
104
140
 
141
+ The helper compares normalized success payloads and normalized TrailsError
142
+ category/code pairs. CLI command names, MCP tool names, HTTP method/path values,
143
+ transport envelopes, activation consumers, internal trails, and planned
144
+ WebSocket work stay outside the equality check. Use `createResources` when
145
+ examples need deterministic fixtures for each surface invocation, and use
146
+ exclusions for intentional semantic differences that should not be silently
147
+ skipped.
148
+
105
149
  ## Installation
106
150
 
107
151
  ```bash
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ontrails/testing",
3
- "version": "1.0.0-beta.16",
3
+ "version": "1.0.0-beta.17",
4
4
  "files": [
5
5
  "src/**/*.ts",
6
6
  "!src/**/__tests__/**",
@@ -22,14 +22,15 @@
22
22
  "clean": "rm -rf dist *.tsbuildinfo"
23
23
  },
24
24
  "devDependencies": {
25
- "@ontrails/drizzle": "^1.0.0-beta.15",
26
- "@ontrails/store": "^1.0.0-beta.15"
25
+ "@ontrails/drizzle": "^1.0.0-beta.16",
26
+ "@ontrails/store": "^1.0.0-beta.16"
27
27
  },
28
28
  "peerDependencies": {
29
- "@ontrails/cli": "^1.0.0-beta.15",
30
- "@ontrails/core": "^1.0.0-beta.15",
31
- "@ontrails/mcp": "^1.0.0-beta.15",
32
- "@ontrails/observe": "^1.0.0-beta.15",
29
+ "@ontrails/cli": "^1.0.0-beta.16",
30
+ "@ontrails/core": "^1.0.0-beta.16",
31
+ "@ontrails/http": "^1.0.0-beta.16",
32
+ "@ontrails/mcp": "^1.0.0-beta.16",
33
+ "@ontrails/observe": "^1.0.0-beta.16",
33
34
  "zod": "^4.3.5"
34
35
  }
35
36
  }
package/src/all.ts CHANGED
@@ -11,6 +11,7 @@ import type { Topo, TrailContext } from '@ontrails/core';
11
11
  import { validateEstablishedTopo, validateTopo } from '@ontrails/core';
12
12
 
13
13
  import { createCliHarness } from './harness-cli.js';
14
+ import { createHttpHarness } from './harness-http.js';
14
15
  import { createMcpHarness } from './harness-mcp.js';
15
16
  import { testContracts } from './contracts.js';
16
17
  import type { TestExecutionOptions } from './context.js';
@@ -108,6 +109,7 @@ const isEstablishedOptions = (
108
109
  (Object.hasOwn(input, 'cli') ||
109
110
  Object.hasOwn(input, 'createPermit') ||
110
111
  Object.hasOwn(input, 'ctx') ||
112
+ Object.hasOwn(input, 'http') ||
111
113
  Object.hasOwn(input, 'mcp') ||
112
114
  Object.hasOwn(input, 'resources') ||
113
115
  Object.hasOwn(input, 'strictPermits'));
@@ -154,6 +156,22 @@ const toMcpHarnessOptions = (
154
156
  ...options.mcp,
155
157
  });
156
158
 
159
+ const toHttpHarnessOptions = (
160
+ topo: Topo,
161
+ options: TestAllEstablishedOptions
162
+ ) => {
163
+ const httpOptions = {
164
+ graph: topo,
165
+ ...options.http,
166
+ };
167
+
168
+ if (options.ctx !== undefined) {
169
+ httpOptions.ctx = options.ctx;
170
+ }
171
+
172
+ return httpOptions;
173
+ };
174
+
157
175
  const registerEstablishedSurfaceSuite = (
158
176
  topo: Topo,
159
177
  resolveInput: () =>
@@ -175,6 +193,13 @@ const registerEstablishedSurfaceSuite = (
175
193
  createMcpHarness(toMcpHarnessOptions(topo, options))
176
194
  ).not.toThrow();
177
195
  });
196
+
197
+ test('HTTP projection validates established topo', () => {
198
+ const options = normalizeEstablishedOptions(resolveInput());
199
+ expect(() =>
200
+ createHttpHarness(toHttpHarnessOptions(topo, options))
201
+ ).not.toThrow();
202
+ });
178
203
  });
179
204
  };
180
205
 
@@ -8,6 +8,7 @@
8
8
  import { deriveCliCommands } from '@ontrails/cli';
9
9
  import type { CliCommand } from '@ontrails/cli';
10
10
  import type { TrailContext } from '@ontrails/core';
11
+ import { projectPublicSurfaceError } from '@ontrails/core';
11
12
 
12
13
  import { mergeTestContext } from './context.js';
13
14
  import type {
@@ -207,10 +208,16 @@ const buildErrorResult = (
207
208
  streams: CapturedStreams
208
209
  ): CliHarnessResult => {
209
210
  streams.restore();
210
- const message = error instanceof Error ? error.message : String(error);
211
+ const actualError = error instanceof Error ? error : new Error(String(error));
212
+ const projection = projectPublicSurfaceError('cli', actualError);
211
213
  return {
214
+ error: {
215
+ category: projection.category,
216
+ code: projection.name,
217
+ message: projection.message,
218
+ },
212
219
  exitCode: 1,
213
- stderr: streams.getStderr() || message,
220
+ stderr: streams.getStderr() || projection.message,
214
221
  stdout: streams.getStdout(),
215
222
  };
216
223
  };
@@ -227,9 +234,15 @@ const executeCommand = async (
227
234
  streams.restore();
228
235
 
229
236
  if (result.isErr()) {
237
+ const projection = projectPublicSurfaceError('cli', result.error);
230
238
  return {
239
+ error: {
240
+ category: projection.category,
241
+ code: projection.name,
242
+ message: projection.message,
243
+ },
231
244
  exitCode: 1,
232
- stderr: streams.getStderr() || result.error.message,
245
+ stderr: streams.getStderr() || projection.message,
233
246
  stdout: streams.getStdout(),
234
247
  };
235
248
  }
@@ -0,0 +1,263 @@
1
+ /**
2
+ * HTTP integration test harness.
3
+ *
4
+ * Builds framework-agnostic HTTP routes from a graph and executes them
5
+ * directly, without Hono or a listening server.
6
+ */
7
+
8
+ import { deriveHttpRoutes } from '@ontrails/http';
9
+ import type { HttpMethod, HttpRouteDefinition } from '@ontrails/http';
10
+ import type { TrailContext, TrailContextInit } from '@ontrails/core';
11
+ import { NotFoundError, projectPublicSurfaceError } from '@ontrails/core';
12
+
13
+ import { mergeTestContext } from './context.js';
14
+ import type {
15
+ HttpHarness,
16
+ HttpHarnessOptions,
17
+ HttpHarnessRequest,
18
+ HttpHarnessRequestOptions,
19
+ HttpHarnessResult,
20
+ } from './types.js';
21
+
22
+ const TEST_ORIGIN = 'http://ontrails.test';
23
+
24
+ const normalizeMethod = (method: HttpMethod): HttpMethod =>
25
+ method.toUpperCase() as HttpMethod;
26
+
27
+ const collectQueryParams = (url: URL): Record<string, unknown> => {
28
+ const query: Record<string, unknown> = {};
29
+ const seen = new Set<string>();
30
+
31
+ for (const key of url.searchParams.keys()) {
32
+ if (seen.has(key)) {
33
+ continue;
34
+ }
35
+ seen.add(key);
36
+ const values = url.searchParams.getAll(key);
37
+ query[key] = values.length > 1 ? values : values[0];
38
+ }
39
+
40
+ return query;
41
+ };
42
+
43
+ const findRoute = (
44
+ routes: readonly HttpRouteDefinition[],
45
+ method: HttpMethod,
46
+ path: string
47
+ ): HttpRouteDefinition | undefined =>
48
+ routes.find((route) => route.method === method && route.path === path);
49
+
50
+ const mapError = (error: Error): HttpHarnessResult => {
51
+ const projection = projectPublicSurfaceError('http', error);
52
+ const body = {
53
+ error: {
54
+ category: projection.category,
55
+ code: projection.name,
56
+ message: projection.message,
57
+ },
58
+ };
59
+ return {
60
+ body,
61
+ error: body.error,
62
+ ok: false,
63
+ status: projection.code,
64
+ };
65
+ };
66
+
67
+ const mapSuccess = (data: unknown): HttpHarnessResult => ({
68
+ body: { data },
69
+ data,
70
+ ok: true,
71
+ status: 200,
72
+ });
73
+
74
+ const isWebhookParseResult = (
75
+ input: unknown
76
+ ): input is {
77
+ readonly error?: Error | undefined;
78
+ isErr(): boolean;
79
+ readonly value?: unknown | undefined;
80
+ } =>
81
+ typeof input === 'object' &&
82
+ input !== null &&
83
+ 'isErr' in input &&
84
+ typeof input.isErr === 'function';
85
+
86
+ const mergeContextInit = (
87
+ base: TrailContextInit | undefined,
88
+ ctx: Partial<TrailContext> | undefined
89
+ ): TrailContextInit => ({
90
+ ...base,
91
+ ...mergeTestContext({ ...base, ...ctx }),
92
+ });
93
+
94
+ const createHarnessContextFactory = (
95
+ options: HttpHarnessOptions
96
+ ): (() => TrailContextInit | Promise<TrailContextInit>) => {
97
+ const { createContext, ctx } = options;
98
+ return async () => {
99
+ const base = await createContext?.();
100
+ return mergeContextInit(base, ctx);
101
+ };
102
+ };
103
+
104
+ const buildInput = (
105
+ route: HttpRouteDefinition,
106
+ url: URL,
107
+ request: HttpHarnessRequest
108
+ ): unknown => {
109
+ if (route.inputSource === 'query') {
110
+ return {
111
+ ...collectQueryParams(url),
112
+ ...request.query,
113
+ };
114
+ }
115
+ return request.body ?? {};
116
+ };
117
+
118
+ const executeRouteWithInput = async (
119
+ route: HttpRouteDefinition,
120
+ input: unknown,
121
+ request: HttpHarnessRequest
122
+ ): Promise<HttpHarnessResult> => {
123
+ const result = await route.execute(
124
+ input,
125
+ request.requestId,
126
+ request.abortSignal,
127
+ { headers: request.headers }
128
+ );
129
+
130
+ if (result.isErr()) {
131
+ return mapError(result.error);
132
+ }
133
+
134
+ return mapSuccess(result.value);
135
+ };
136
+
137
+ const executeRoute = async (
138
+ route: HttpRouteDefinition,
139
+ url: URL,
140
+ request: HttpHarnessRequest
141
+ ): Promise<HttpHarnessResult> => {
142
+ const parsedInput = buildInput(route, url, request);
143
+ if (route.inputSource === 'webhook' && route.parseWebhookInput) {
144
+ const parsed = route.parseWebhookInput(parsedInput);
145
+ if (isWebhookParseResult(parsed)) {
146
+ if (parsed.isErr()) {
147
+ return mapError(parsed.error ?? new Error('Invalid webhook input'));
148
+ }
149
+ return await executeRouteWithInput(route, parsed.value, request);
150
+ }
151
+ return await executeRouteWithInput(route, parsed, request);
152
+ }
153
+
154
+ return await executeRouteWithInput(route, parsedInput, request);
155
+ };
156
+
157
+ // ---------------------------------------------------------------------------
158
+ // createHttpHarness
159
+ // ---------------------------------------------------------------------------
160
+
161
+ /**
162
+ * Create an HTTP harness for integration testing.
163
+ *
164
+ * @example
165
+ * ```ts
166
+ * import { createHttpHarness } from '@ontrails/testing';
167
+ *
168
+ * const http = createHttpHarness({ graph });
169
+ * const result = await http.get('/entity/show', { name: 'Alpha' });
170
+ * expect(result.status).toBe(200);
171
+ * ```
172
+ */
173
+ export const createHttpHarness = (
174
+ harnessOptions: HttpHarnessOptions
175
+ ): HttpHarness => {
176
+ const { ctx: _ctx, graph, ...deriveOptions } = harnessOptions;
177
+ const routesResult = deriveHttpRoutes(graph, {
178
+ ...deriveOptions,
179
+ createContext: createHarnessContextFactory(harnessOptions),
180
+ });
181
+ if (routesResult.isErr()) {
182
+ throw routesResult.error;
183
+ }
184
+ const routes = routesResult.value;
185
+
186
+ const request = async (
187
+ rawRequest: HttpHarnessRequest
188
+ ): Promise<HttpHarnessResult> => {
189
+ const method = normalizeMethod(rawRequest.method);
190
+ const url = new URL(rawRequest.path, TEST_ORIGIN);
191
+ const route = findRoute(routes, method, url.pathname);
192
+
193
+ if (!route) {
194
+ return mapError(
195
+ new NotFoundError(`No HTTP route found for ${method} ${url.pathname}`)
196
+ );
197
+ }
198
+
199
+ return await executeRoute(route, url, { ...rawRequest, method });
200
+ };
201
+
202
+ return {
203
+ delete: async (
204
+ path: string,
205
+ body?: unknown,
206
+ requestOptions?: HttpHarnessRequestOptions
207
+ ) =>
208
+ await request({
209
+ ...requestOptions,
210
+ body,
211
+ method: 'DELETE',
212
+ path,
213
+ }),
214
+ get: async (
215
+ path: string,
216
+ query?: Record<string, unknown>,
217
+ requestOptions?: HttpHarnessRequestOptions
218
+ ) =>
219
+ await request({
220
+ ...requestOptions,
221
+ method: 'GET',
222
+ path,
223
+ query: {
224
+ ...requestOptions?.query,
225
+ ...query,
226
+ },
227
+ }),
228
+ patch: async (
229
+ path: string,
230
+ body?: unknown,
231
+ requestOptions?: HttpHarnessRequestOptions
232
+ ) =>
233
+ await request({
234
+ ...requestOptions,
235
+ body,
236
+ method: 'PATCH',
237
+ path,
238
+ }),
239
+ post: async (
240
+ path: string,
241
+ body?: unknown,
242
+ requestOptions?: HttpHarnessRequestOptions
243
+ ) =>
244
+ await request({
245
+ ...requestOptions,
246
+ body,
247
+ method: 'POST',
248
+ path,
249
+ }),
250
+ put: async (
251
+ path: string,
252
+ body?: unknown,
253
+ requestOptions?: HttpHarnessRequestOptions
254
+ ) =>
255
+ await request({
256
+ ...requestOptions,
257
+ body,
258
+ method: 'PUT',
259
+ path,
260
+ }),
261
+ request,
262
+ };
263
+ };
@@ -63,6 +63,8 @@ export const createMcpHarness = (options: McpHarnessOptions): McpHarness => {
63
63
  return {
64
64
  content: result.content,
65
65
  isError: result.isError ?? false,
66
+ meta: result._meta,
67
+ structuredContent: result.structuredContent,
66
68
  };
67
69
  },
68
70
  };
package/src/index.ts CHANGED
@@ -6,6 +6,10 @@ export { testCrosses } from './crosses.js';
6
6
  export { testTrail } from './trail.js';
7
7
  export { testContracts } from './contracts.js';
8
8
  export { testDetours } from './detours.js';
9
+ export {
10
+ runSurfaceParityExample,
11
+ testSurfaceParity,
12
+ } from './surface-parity.js';
9
13
 
10
14
  // Assertions
11
15
  export {
@@ -32,6 +36,7 @@ export { createTestLogger } from './logger.js';
32
36
 
33
37
  // Surface harnesses
34
38
  export { createCliHarness } from './harness-cli.js';
39
+ export { createHttpHarness } from './harness-http.js';
35
40
  export { createMcpHarness } from './harness-mcp.js';
36
41
 
37
42
  // Types
@@ -54,7 +59,19 @@ export type {
54
59
  CliHarness,
55
60
  CliHarnessOptions,
56
61
  CliHarnessResult,
62
+ HttpHarness,
63
+ HttpHarnessErrorBody,
64
+ HttpHarnessOptions,
65
+ HttpHarnessRequest,
66
+ HttpHarnessRequestOptions,
67
+ HttpHarnessResult,
68
+ HttpHarnessSuccessBody,
57
69
  McpHarness,
58
70
  McpHarnessOptions,
59
71
  McpHarnessResult,
72
+ NormalizedSurfaceParityResult,
73
+ SurfaceParityExclusion,
74
+ SurfaceParityOptions,
75
+ SurfaceParitySurface,
60
76
  } from './types.js';
77
+ export type { SurfaceParityComparison } from './surface-parity.js';
@@ -0,0 +1,356 @@
1
+ /**
2
+ * Example-driven parity checks across shipped surfaces.
3
+ */
4
+
5
+ import { describe, expect, test } from 'bun:test';
6
+
7
+ import { deriveCliPath, filterSurfaceTrails } from '@ontrails/core';
8
+ import type { Topo, Trail, TrailContext, TrailExample } from '@ontrails/core';
9
+ import { deriveHttpInputSource, deriveHttpMethod } from '@ontrails/http';
10
+ import type { HttpMethod } from '@ontrails/http';
11
+ import { MCP_TOOL_ERROR_META_KEY, deriveToolName } from '@ontrails/mcp';
12
+
13
+ import {
14
+ createMockResources,
15
+ defaultCreatePermit,
16
+ mergeResourceOverrides,
17
+ mergeTestContext,
18
+ } from './context.js';
19
+ import { deriveTrailExamples } from './effective-examples.js';
20
+ import { createCliHarness } from './harness-cli.js';
21
+ import { createHttpHarness } from './harness-http.js';
22
+ import { createMcpHarness } from './harness-mcp.js';
23
+ import type {
24
+ CliHarnessResult,
25
+ HttpHarnessResult,
26
+ McpHarnessResult,
27
+ NormalizedSurfaceParityResult,
28
+ SurfaceParityExclusion,
29
+ SurfaceParityOptions,
30
+ } from './types.js';
31
+
32
+ type ParityTrail = Trail<unknown, unknown, unknown>;
33
+
34
+ export interface SurfaceParityComparison {
35
+ readonly cli: NormalizedSurfaceParityResult;
36
+ readonly http: NormalizedSurfaceParityResult;
37
+ readonly mcp: NormalizedSurfaceParityResult;
38
+ }
39
+
40
+ const httpPathForTrail = (trailId: string): string =>
41
+ `/${trailId.replaceAll('.', '/')}`;
42
+
43
+ const escapeWhitespaceForCliToken = (value: string): string =>
44
+ value.replaceAll(/\s/gu, (char) => {
45
+ let escaped = '';
46
+
47
+ for (let index = 0; index < char.length; index += 1) {
48
+ const codePoint = char.codePointAt(index);
49
+
50
+ if (codePoint !== undefined) {
51
+ escaped += `\\u${codePoint.toString(16).padStart(4, '0')}`;
52
+ }
53
+ }
54
+
55
+ return escaped;
56
+ });
57
+
58
+ const cliCommandForExample = (
59
+ trail: ParityTrail,
60
+ example: TrailExample<unknown, unknown>
61
+ ): string => {
62
+ const path = deriveCliPath(trail.id).join(' ');
63
+ const inputJson = escapeWhitespaceForCliToken(
64
+ JSON.stringify(example.input ?? {})
65
+ );
66
+ return `${path} --input-json ${inputJson} --output json`;
67
+ };
68
+
69
+ const isObjectRecord = (value: unknown): value is Record<string, unknown> =>
70
+ value !== null && typeof value === 'object' && !Array.isArray(value);
71
+
72
+ const readTextContent = (content: unknown): string | undefined => {
73
+ if (!Array.isArray(content)) {
74
+ return undefined;
75
+ }
76
+ const firstText = content.find(
77
+ (item): item is { readonly text: string; readonly type: string } =>
78
+ isObjectRecord(item) &&
79
+ item['type'] === 'text' &&
80
+ typeof item['text'] === 'string'
81
+ );
82
+ return firstText?.text;
83
+ };
84
+
85
+ const parseJsonText = (text: string | undefined): unknown => {
86
+ if (text === undefined) {
87
+ return undefined;
88
+ }
89
+ try {
90
+ return JSON.parse(text);
91
+ } catch {
92
+ return text;
93
+ }
94
+ };
95
+
96
+ const structuredContentValue = (
97
+ structuredContent: Record<string, unknown> | undefined
98
+ ): unknown => {
99
+ if (structuredContent === undefined) {
100
+ return undefined;
101
+ }
102
+ const keys = Object.keys(structuredContent);
103
+ return keys.length === 1 && keys[0] === 'data'
104
+ ? structuredContent['data']
105
+ : structuredContent;
106
+ };
107
+
108
+ const normalizeCliResult = (
109
+ result: CliHarnessResult
110
+ ): NormalizedSurfaceParityResult =>
111
+ result.exitCode === 0
112
+ ? { ok: true, value: result.json }
113
+ : {
114
+ error: {
115
+ category: result.error?.category ?? 'internal',
116
+ code: result.error?.code ?? 'InternalError',
117
+ },
118
+ ok: false,
119
+ };
120
+
121
+ const normalizeHttpResult = (
122
+ result: HttpHarnessResult
123
+ ): NormalizedSurfaceParityResult =>
124
+ result.ok
125
+ ? { ok: true, value: result.data }
126
+ : {
127
+ error: {
128
+ category: result.error?.category ?? 'internal',
129
+ code: result.error?.code ?? 'InternalError',
130
+ },
131
+ ok: false,
132
+ };
133
+
134
+ const readMcpError = (
135
+ result: McpHarnessResult
136
+ ): { readonly category: string; readonly code: string } => {
137
+ const errorMeta = result.meta?.[MCP_TOOL_ERROR_META_KEY];
138
+ if (isObjectRecord(errorMeta)) {
139
+ return {
140
+ category:
141
+ typeof errorMeta['category'] === 'string'
142
+ ? errorMeta['category']
143
+ : 'internal',
144
+ code:
145
+ typeof errorMeta['name'] === 'string'
146
+ ? errorMeta['name']
147
+ : 'InternalError',
148
+ };
149
+ }
150
+ return { category: 'internal', code: 'InternalError' };
151
+ };
152
+
153
+ const normalizeMcpResult = (
154
+ result: McpHarnessResult
155
+ ): NormalizedSurfaceParityResult =>
156
+ result.isError
157
+ ? { error: readMcpError(result), ok: false }
158
+ : {
159
+ ok: true,
160
+ value:
161
+ result.structuredContent === undefined
162
+ ? parseJsonText(readTextContent(result.content))
163
+ : structuredContentValue(result.structuredContent),
164
+ };
165
+
166
+ const applyAutoPermit = (
167
+ ctx: TrailContext,
168
+ trail: ParityTrail,
169
+ options: SurfaceParityOptions
170
+ ): TrailContext => {
171
+ if (options.strictPermits || ctx.permit !== undefined) {
172
+ return ctx;
173
+ }
174
+ const permit = (options.createPermit ?? defaultCreatePermit)(trail);
175
+ return permit === undefined ? ctx : { ...ctx, permit };
176
+ };
177
+
178
+ const createInvocationContext = async (
179
+ app: Topo,
180
+ trail: ParityTrail,
181
+ options: SurfaceParityOptions
182
+ ) => {
183
+ const autoResources =
184
+ options.createResources === undefined
185
+ ? await createMockResources(app)
186
+ : await options.createResources();
187
+ const resources = mergeResourceOverrides(
188
+ autoResources,
189
+ options.ctx,
190
+ options.resources
191
+ );
192
+ const ctx = applyAutoPermit(mergeTestContext(options.ctx), trail, options);
193
+ return { ctx, resources };
194
+ };
195
+
196
+ const mergeSurfaceContext = (
197
+ ctx: TrailContext,
198
+ surfaceCtx: Partial<TrailContext> | undefined
199
+ ): TrailContext =>
200
+ surfaceCtx === undefined ? ctx : mergeTestContext({ ...ctx, ...surfaceCtx });
201
+
202
+ const runCliExample = async (
203
+ app: Topo,
204
+ trail: ParityTrail,
205
+ example: TrailExample<unknown, unknown>,
206
+ options: SurfaceParityOptions
207
+ ): Promise<NormalizedSurfaceParityResult> => {
208
+ const { ctx, resources } = await createInvocationContext(app, trail, options);
209
+ const cliOptions = options.cli;
210
+ const harness = createCliHarness({
211
+ graph: app,
212
+ ...cliOptions,
213
+ ctx: mergeSurfaceContext(ctx, cliOptions?.ctx),
214
+ resources,
215
+ });
216
+ return normalizeCliResult(
217
+ await harness.run(cliCommandForExample(trail, example))
218
+ );
219
+ };
220
+
221
+ const runMcpExample = async (
222
+ app: Topo,
223
+ trail: ParityTrail,
224
+ example: TrailExample<unknown, unknown>,
225
+ options: SurfaceParityOptions
226
+ ): Promise<NormalizedSurfaceParityResult> => {
227
+ const { ctx, resources } = await createInvocationContext(app, trail, options);
228
+ const mcpOptions = options.mcp;
229
+ const harness = createMcpHarness({
230
+ graph: app,
231
+ ...mcpOptions,
232
+ createContext: async () =>
233
+ mergeSurfaceContext(ctx, await mcpOptions?.createContext?.()),
234
+ resources,
235
+ });
236
+ const toolName = deriveToolName(app.name, trail.id);
237
+ return normalizeMcpResult(
238
+ await harness.callTool(
239
+ toolName,
240
+ isObjectRecord(example.input) ? example.input : {}
241
+ )
242
+ );
243
+ };
244
+
245
+ const httpRequestForExample = (
246
+ method: HttpMethod,
247
+ trailId: string,
248
+ example: TrailExample<unknown, unknown>
249
+ ) => {
250
+ const input = isObjectRecord(example.input) ? example.input : {};
251
+ return deriveHttpInputSource(method) === 'query'
252
+ ? { method, path: httpPathForTrail(trailId), query: input }
253
+ : { body: input, method, path: httpPathForTrail(trailId) };
254
+ };
255
+
256
+ const runHttpExample = async (
257
+ app: Topo,
258
+ trail: ParityTrail,
259
+ example: TrailExample<unknown, unknown>,
260
+ options: SurfaceParityOptions
261
+ ): Promise<NormalizedSurfaceParityResult> => {
262
+ const { ctx, resources } = await createInvocationContext(app, trail, options);
263
+ const httpOptions = options.http;
264
+ const harness = createHttpHarness({
265
+ graph: app,
266
+ ...httpOptions,
267
+ ctx: mergeSurfaceContext(ctx, httpOptions?.ctx),
268
+ resources,
269
+ });
270
+ const method = deriveHttpMethod(trail.intent);
271
+ return normalizeHttpResult(
272
+ await harness.request(httpRequestForExample(method, trail.id, example))
273
+ );
274
+ };
275
+
276
+ export const runSurfaceParityExample = async (
277
+ app: Topo,
278
+ trail: ParityTrail,
279
+ example: TrailExample<unknown, unknown>,
280
+ options: SurfaceParityOptions = {}
281
+ ): Promise<SurfaceParityComparison> => {
282
+ // CLI harness output capture is process-scoped, so keep surface execution
283
+ // ordered even though MCP and HTTP do not share that constraint.
284
+ const cli = await runCliExample(app, trail, example, options);
285
+ const mcp = await runMcpExample(app, trail, example, options);
286
+ const http = await runHttpExample(app, trail, example, options);
287
+ return { cli, http, mcp };
288
+ };
289
+
290
+ const findExclusion = (
291
+ exclusions: readonly SurfaceParityExclusion[] | undefined,
292
+ trail: ParityTrail,
293
+ example: TrailExample<unknown, unknown>
294
+ ): SurfaceParityExclusion | undefined =>
295
+ exclusions?.find(
296
+ (exclusion) =>
297
+ exclusion.trailId === trail.id &&
298
+ (exclusion.example === undefined || exclusion.example === example.name)
299
+ );
300
+
301
+ const parityTrails = (app: Topo): readonly ParityTrail[] =>
302
+ filterSurfaceTrails(app.list()).filter(
303
+ (trail) => deriveTrailExamples(trail).length > 0
304
+ );
305
+
306
+ /**
307
+ * Register example-driven parity tests for CLI, MCP, and HTTP.
308
+ *
309
+ * @example
310
+ * ```ts
311
+ * import { testSurfaceParity } from '@ontrails/testing';
312
+ * import { graph } from '../src/app.js';
313
+ *
314
+ * testSurfaceParity(graph);
315
+ * ```
316
+ */
317
+ export const testSurfaceParity = (
318
+ app: Topo,
319
+ optionsOrFactory?:
320
+ | SurfaceParityOptions
321
+ | (() => SurfaceParityOptions | undefined)
322
+ ): void => {
323
+ const resolveOptions =
324
+ typeof optionsOrFactory === 'function'
325
+ ? optionsOrFactory
326
+ : () => optionsOrFactory;
327
+
328
+ describe('surface parity', () => {
329
+ for (const trail of parityTrails(app)) {
330
+ describe(trail.id, () => {
331
+ for (const example of deriveTrailExamples(trail)) {
332
+ const options = resolveOptions() ?? {};
333
+ const exclusion = findExclusion(options.exclusions, trail, example);
334
+ const testName = `example: ${example.name}`;
335
+ if (exclusion !== undefined) {
336
+ test.skip(`${testName} (excluded: ${exclusion.reason})`, () => {
337
+ throw new Error('Skipped parity exclusion should not execute');
338
+ });
339
+ continue;
340
+ }
341
+
342
+ test(testName, async () => {
343
+ const comparison = await runSurfaceParityExample(
344
+ app,
345
+ trail,
346
+ example,
347
+ resolveOptions() ?? {}
348
+ );
349
+ expect(comparison.mcp).toEqual(comparison.cli);
350
+ expect(comparison.http).toEqual(comparison.cli);
351
+ });
352
+ }
353
+ });
354
+ }
355
+ });
356
+ };
package/src/types.ts CHANGED
@@ -3,10 +3,16 @@
3
3
  */
4
4
 
5
5
  import type { DeriveCliCommandsOptions } from '@ontrails/cli';
6
+ import type {
7
+ DeriveHttpRoutesOptions,
8
+ HttpHeaderSource,
9
+ HttpMethod,
10
+ } from '@ontrails/http';
6
11
  import type { McpExtra, DeriveMcpToolsOptions } from '@ontrails/mcp';
7
12
  import type {
8
13
  AnyTrail,
9
14
  Logger,
15
+ ResourceOverrideMap,
10
16
  Topo,
11
17
  TraceFn,
12
18
  TrailContext,
@@ -98,6 +104,13 @@ export interface CliHarness {
98
104
 
99
105
  /** The result of a CLI harness command execution. */
100
106
  export interface CliHarnessResult {
107
+ readonly error?:
108
+ | {
109
+ readonly category: string;
110
+ readonly code: string;
111
+ readonly message: string;
112
+ }
113
+ | undefined;
101
114
  readonly exitCode: number;
102
115
  /** Parsed JSON output if --output json was used. */
103
116
  readonly json?: unknown | undefined;
@@ -128,8 +141,129 @@ export interface McpHarness {
128
141
  export interface McpHarnessResult {
129
142
  readonly content: unknown;
130
143
  readonly isError: boolean;
144
+ readonly meta?: Record<string, unknown> | undefined;
145
+ readonly structuredContent?: Record<string, unknown> | undefined;
146
+ }
147
+
148
+ // ---------------------------------------------------------------------------
149
+ // HTTP Harness
150
+ // ---------------------------------------------------------------------------
151
+
152
+ /** Options for creating an HTTP harness. */
153
+ export interface HttpHarnessOptions extends DeriveHttpRoutesOptions {
154
+ readonly ctx?: Partial<TrailContext> | undefined;
155
+ readonly graph: Topo;
131
156
  }
132
157
 
158
+ export interface HttpHarnessRequest {
159
+ readonly abortSignal?: AbortSignal | undefined;
160
+ readonly body?: unknown | undefined;
161
+ readonly headers?: HttpHeaderSource | undefined;
162
+ readonly method: HttpMethod;
163
+ readonly path: string;
164
+ readonly query?: Record<string, unknown> | undefined;
165
+ readonly requestId?: string | undefined;
166
+ }
167
+
168
+ export interface HttpHarnessRequestOptions extends Omit<
169
+ HttpHarnessRequest,
170
+ 'body' | 'method' | 'path' | 'query'
171
+ > {
172
+ readonly query?: Record<string, unknown> | undefined;
173
+ }
174
+
175
+ /** A test harness for HTTP route projections. */
176
+ export interface HttpHarness {
177
+ /** Execute a raw HTTP-style harness request. */
178
+ request(request: HttpHarnessRequest): Promise<HttpHarnessResult>;
179
+ /** Execute a GET request, reading input from query params. */
180
+ get(
181
+ path: string,
182
+ query?: Record<string, unknown>,
183
+ options?: HttpHarnessRequestOptions
184
+ ): Promise<HttpHarnessResult>;
185
+ /** Execute a POST request, reading input from the JSON-like body value. */
186
+ post(
187
+ path: string,
188
+ body?: unknown,
189
+ options?: HttpHarnessRequestOptions
190
+ ): Promise<HttpHarnessResult>;
191
+ /** Execute a PUT request. */
192
+ put(
193
+ path: string,
194
+ body?: unknown,
195
+ options?: HttpHarnessRequestOptions
196
+ ): Promise<HttpHarnessResult>;
197
+ /** Execute a PATCH request. */
198
+ patch(
199
+ path: string,
200
+ body?: unknown,
201
+ options?: HttpHarnessRequestOptions
202
+ ): Promise<HttpHarnessResult>;
203
+ /** Execute a DELETE request. */
204
+ delete(
205
+ path: string,
206
+ body?: unknown,
207
+ options?: HttpHarnessRequestOptions
208
+ ): Promise<HttpHarnessResult>;
209
+ }
210
+
211
+ export interface HttpHarnessErrorBody {
212
+ readonly error: {
213
+ readonly category: string;
214
+ readonly code: string;
215
+ readonly message: string;
216
+ };
217
+ }
218
+
219
+ export interface HttpHarnessSuccessBody {
220
+ readonly data: unknown;
221
+ }
222
+
223
+ /** The result of an HTTP harness request. */
224
+ export interface HttpHarnessResult {
225
+ readonly body: HttpHarnessErrorBody | HttpHarnessSuccessBody;
226
+ readonly data?: unknown | undefined;
227
+ readonly error?: HttpHarnessErrorBody['error'] | undefined;
228
+ readonly ok: boolean;
229
+ readonly status: number;
230
+ }
231
+
232
+ // ---------------------------------------------------------------------------
233
+ // Surface parity
234
+ // ---------------------------------------------------------------------------
235
+
236
+ export type SurfaceParitySurface = 'cli' | 'mcp' | 'http';
237
+
238
+ export interface SurfaceParityExclusion {
239
+ /** Optional example name. Omit to exclude every example for the trail. */
240
+ readonly example?: string | undefined;
241
+ /** Human-readable reason shown in the skipped test name. */
242
+ readonly reason: string;
243
+ /** Trail ID to exclude. */
244
+ readonly trailId: string;
245
+ }
246
+
247
+ export interface SurfaceParityOptions extends TestAllEstablishedOptions {
248
+ readonly createResources?:
249
+ | (() => ResourceOverrideMap | Promise<ResourceOverrideMap>)
250
+ | undefined;
251
+ readonly exclusions?: readonly SurfaceParityExclusion[] | undefined;
252
+ }
253
+
254
+ export type NormalizedSurfaceParityResult =
255
+ | {
256
+ readonly ok: true;
257
+ readonly value: unknown;
258
+ }
259
+ | {
260
+ readonly error: {
261
+ readonly category: string;
262
+ readonly code: string;
263
+ };
264
+ readonly ok: false;
265
+ };
266
+
133
267
  // ---------------------------------------------------------------------------
134
268
  // Established verification
135
269
  // ---------------------------------------------------------------------------
@@ -150,6 +284,7 @@ export interface TestAllEstablishedOptions {
150
284
  | undefined)
151
285
  | undefined;
152
286
  readonly ctx?: Partial<TrailContext> | undefined;
287
+ readonly http?: Omit<HttpHarnessOptions, 'graph'> | undefined;
153
288
  readonly mcp?: Omit<McpHarnessOptions, 'graph'> | undefined;
154
289
  readonly resources?: Record<string, unknown> | undefined;
155
290
  readonly strictPermits?: boolean | undefined;