@ontrails/testing 1.0.0-beta.17 → 1.0.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/README.md +31 -29
- package/package.json +24 -8
- package/src/all-established.ts +168 -0
- package/src/all.ts +3 -130
- package/src/assertions.ts +2 -2
- package/src/cli.ts +6 -0
- package/src/{crosses.ts → composes.ts} +87 -79
- package/src/context.ts +22 -21
- package/src/contracts.ts +19 -17
- package/src/effective-examples.ts +67 -3
- package/src/errors.ts +47 -0
- package/src/examples.ts +131 -118
- package/src/harness-cli.ts +35 -7
- package/src/harness-http.ts +87 -9
- package/src/harness-mcp.ts +30 -6
- package/src/http.ts +10 -0
- package/src/index.ts +5 -34
- package/src/mcp.ts +6 -0
- package/src/scenario.ts +67 -60
- package/src/surface-parity.ts +43 -10
- package/src/types.ts +11 -232
package/src/types.ts
CHANGED
|
@@ -2,21 +2,7 @@
|
|
|
2
2
|
* Shared types for @ontrails/testing.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import type {
|
|
6
|
-
import type {
|
|
7
|
-
DeriveHttpRoutesOptions,
|
|
8
|
-
HttpHeaderSource,
|
|
9
|
-
HttpMethod,
|
|
10
|
-
} from '@ontrails/http';
|
|
11
|
-
import type { McpExtra, DeriveMcpToolsOptions } from '@ontrails/mcp';
|
|
12
|
-
import type {
|
|
13
|
-
AnyTrail,
|
|
14
|
-
Logger,
|
|
15
|
-
ResourceOverrideMap,
|
|
16
|
-
Topo,
|
|
17
|
-
TraceFn,
|
|
18
|
-
TrailContext,
|
|
19
|
-
} from '@ontrails/core';
|
|
5
|
+
import type { AnyTrail, Logger, TraceFn } from '@ontrails/core';
|
|
20
6
|
import type { LogLevel, LogRecord } from '@ontrails/observe';
|
|
21
7
|
|
|
22
8
|
// ---------------------------------------------------------------------------
|
|
@@ -40,16 +26,16 @@ export interface TestScenario {
|
|
|
40
26
|
}
|
|
41
27
|
|
|
42
28
|
// ---------------------------------------------------------------------------
|
|
43
|
-
//
|
|
29
|
+
// Compose Scenario (for testComposes)
|
|
44
30
|
// ---------------------------------------------------------------------------
|
|
45
31
|
|
|
46
|
-
/** A test scenario for a trail's
|
|
47
|
-
export interface
|
|
48
|
-
/** Assert these trail IDs were
|
|
49
|
-
readonly
|
|
50
|
-
/** Assert
|
|
51
|
-
readonly
|
|
52
|
-
/** Inject failure from a
|
|
32
|
+
/** A test scenario for a trail's composing graph. */
|
|
33
|
+
export interface ComposeScenario extends TestScenario {
|
|
34
|
+
/** Assert these trail IDs were composed, in order. */
|
|
35
|
+
readonly expectComposed?: readonly string[] | undefined;
|
|
36
|
+
/** Assert composing counts per trail ID. */
|
|
37
|
+
readonly expectComposedCount?: Readonly<Record<string, number>> | undefined;
|
|
38
|
+
/** Inject failure from a composed trail's example by description. */
|
|
53
39
|
readonly injectFromExample?: Readonly<Record<string, string>> | undefined;
|
|
54
40
|
}
|
|
55
41
|
|
|
@@ -83,218 +69,11 @@ export interface TestTrailContextOptions {
|
|
|
83
69
|
readonly trace?: TraceFn | undefined;
|
|
84
70
|
}
|
|
85
71
|
|
|
86
|
-
// ---------------------------------------------------------------------------
|
|
87
|
-
// CLI Harness
|
|
88
|
-
// ---------------------------------------------------------------------------
|
|
89
|
-
|
|
90
|
-
/** Options for creating a CLI harness. */
|
|
91
|
-
export interface CliHarnessOptions extends Omit<
|
|
92
|
-
DeriveCliCommandsOptions,
|
|
93
|
-
'onResult' | 'presets' | 'resolveInput'
|
|
94
|
-
> {
|
|
95
|
-
readonly ctx?: Partial<TrailContext> | undefined;
|
|
96
|
-
readonly graph: Topo;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
/** A test harness for CLI commands. */
|
|
100
|
-
export interface CliHarness {
|
|
101
|
-
/** Execute a CLI command string and capture output. */
|
|
102
|
-
run(command: string): Promise<CliHarnessResult>;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
/** The result of a CLI harness command execution. */
|
|
106
|
-
export interface CliHarnessResult {
|
|
107
|
-
readonly error?:
|
|
108
|
-
| {
|
|
109
|
-
readonly category: string;
|
|
110
|
-
readonly code: string;
|
|
111
|
-
readonly message: string;
|
|
112
|
-
}
|
|
113
|
-
| undefined;
|
|
114
|
-
readonly exitCode: number;
|
|
115
|
-
/** Parsed JSON output if --output json was used. */
|
|
116
|
-
readonly json?: unknown | undefined;
|
|
117
|
-
readonly stderr: string;
|
|
118
|
-
readonly stdout: string;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
// ---------------------------------------------------------------------------
|
|
122
|
-
// MCP Harness
|
|
123
|
-
// ---------------------------------------------------------------------------
|
|
124
|
-
|
|
125
|
-
/** Options for creating an MCP harness. */
|
|
126
|
-
export interface McpHarnessOptions extends DeriveMcpToolsOptions {
|
|
127
|
-
readonly extra?: Partial<McpExtra> | undefined;
|
|
128
|
-
readonly graph: Topo;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
/** A test harness for MCP tools. */
|
|
132
|
-
export interface McpHarness {
|
|
133
|
-
/** Call an MCP tool by name with arguments. */
|
|
134
|
-
callTool(
|
|
135
|
-
name: string,
|
|
136
|
-
args: Record<string, unknown>
|
|
137
|
-
): Promise<McpHarnessResult>;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/** The result of an MCP harness tool invocation. */
|
|
141
|
-
export interface McpHarnessResult {
|
|
142
|
-
readonly content: unknown;
|
|
143
|
-
readonly isError: boolean;
|
|
144
|
-
readonly meta?: Record<string, unknown> | undefined;
|
|
145
|
-
readonly structuredContent?: Record<string, unknown> | undefined;
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
// ---------------------------------------------------------------------------
|
|
149
|
-
// HTTP Harness
|
|
150
|
-
// ---------------------------------------------------------------------------
|
|
151
|
-
|
|
152
|
-
/** Options for creating an HTTP harness. */
|
|
153
|
-
export interface HttpHarnessOptions extends DeriveHttpRoutesOptions {
|
|
154
|
-
readonly ctx?: Partial<TrailContext> | undefined;
|
|
155
|
-
readonly graph: Topo;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
export interface HttpHarnessRequest {
|
|
159
|
-
readonly abortSignal?: AbortSignal | undefined;
|
|
160
|
-
readonly body?: unknown | undefined;
|
|
161
|
-
readonly headers?: HttpHeaderSource | undefined;
|
|
162
|
-
readonly method: HttpMethod;
|
|
163
|
-
readonly path: string;
|
|
164
|
-
readonly query?: Record<string, unknown> | undefined;
|
|
165
|
-
readonly requestId?: string | undefined;
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
export interface HttpHarnessRequestOptions extends Omit<
|
|
169
|
-
HttpHarnessRequest,
|
|
170
|
-
'body' | 'method' | 'path' | 'query'
|
|
171
|
-
> {
|
|
172
|
-
readonly query?: Record<string, unknown> | undefined;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
/** A test harness for HTTP route projections. */
|
|
176
|
-
export interface HttpHarness {
|
|
177
|
-
/** Execute a raw HTTP-style harness request. */
|
|
178
|
-
request(request: HttpHarnessRequest): Promise<HttpHarnessResult>;
|
|
179
|
-
/** Execute a GET request, reading input from query params. */
|
|
180
|
-
get(
|
|
181
|
-
path: string,
|
|
182
|
-
query?: Record<string, unknown>,
|
|
183
|
-
options?: HttpHarnessRequestOptions
|
|
184
|
-
): Promise<HttpHarnessResult>;
|
|
185
|
-
/** Execute a POST request, reading input from the JSON-like body value. */
|
|
186
|
-
post(
|
|
187
|
-
path: string,
|
|
188
|
-
body?: unknown,
|
|
189
|
-
options?: HttpHarnessRequestOptions
|
|
190
|
-
): Promise<HttpHarnessResult>;
|
|
191
|
-
/** Execute a PUT request. */
|
|
192
|
-
put(
|
|
193
|
-
path: string,
|
|
194
|
-
body?: unknown,
|
|
195
|
-
options?: HttpHarnessRequestOptions
|
|
196
|
-
): Promise<HttpHarnessResult>;
|
|
197
|
-
/** Execute a PATCH request. */
|
|
198
|
-
patch(
|
|
199
|
-
path: string,
|
|
200
|
-
body?: unknown,
|
|
201
|
-
options?: HttpHarnessRequestOptions
|
|
202
|
-
): Promise<HttpHarnessResult>;
|
|
203
|
-
/** Execute a DELETE request. */
|
|
204
|
-
delete(
|
|
205
|
-
path: string,
|
|
206
|
-
body?: unknown,
|
|
207
|
-
options?: HttpHarnessRequestOptions
|
|
208
|
-
): Promise<HttpHarnessResult>;
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
export interface HttpHarnessErrorBody {
|
|
212
|
-
readonly error: {
|
|
213
|
-
readonly category: string;
|
|
214
|
-
readonly code: string;
|
|
215
|
-
readonly message: string;
|
|
216
|
-
};
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
export interface HttpHarnessSuccessBody {
|
|
220
|
-
readonly data: unknown;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
/** The result of an HTTP harness request. */
|
|
224
|
-
export interface HttpHarnessResult {
|
|
225
|
-
readonly body: HttpHarnessErrorBody | HttpHarnessSuccessBody;
|
|
226
|
-
readonly data?: unknown | undefined;
|
|
227
|
-
readonly error?: HttpHarnessErrorBody['error'] | undefined;
|
|
228
|
-
readonly ok: boolean;
|
|
229
|
-
readonly status: number;
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
// ---------------------------------------------------------------------------
|
|
233
|
-
// Surface parity
|
|
234
|
-
// ---------------------------------------------------------------------------
|
|
235
|
-
|
|
236
|
-
export type SurfaceParitySurface = 'cli' | 'mcp' | 'http';
|
|
237
|
-
|
|
238
|
-
export interface SurfaceParityExclusion {
|
|
239
|
-
/** Optional example name. Omit to exclude every example for the trail. */
|
|
240
|
-
readonly example?: string | undefined;
|
|
241
|
-
/** Human-readable reason shown in the skipped test name. */
|
|
242
|
-
readonly reason: string;
|
|
243
|
-
/** Trail ID to exclude. */
|
|
244
|
-
readonly trailId: string;
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
export interface SurfaceParityOptions extends TestAllEstablishedOptions {
|
|
248
|
-
readonly createResources?:
|
|
249
|
-
| (() => ResourceOverrideMap | Promise<ResourceOverrideMap>)
|
|
250
|
-
| undefined;
|
|
251
|
-
readonly exclusions?: readonly SurfaceParityExclusion[] | undefined;
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
export type NormalizedSurfaceParityResult =
|
|
255
|
-
| {
|
|
256
|
-
readonly ok: true;
|
|
257
|
-
readonly value: unknown;
|
|
258
|
-
}
|
|
259
|
-
| {
|
|
260
|
-
readonly error: {
|
|
261
|
-
readonly category: string;
|
|
262
|
-
readonly code: string;
|
|
263
|
-
};
|
|
264
|
-
readonly ok: false;
|
|
265
|
-
};
|
|
266
|
-
|
|
267
|
-
// ---------------------------------------------------------------------------
|
|
268
|
-
// Established verification
|
|
269
|
-
// ---------------------------------------------------------------------------
|
|
270
|
-
|
|
271
|
-
export interface TestAllEstablishedOptions {
|
|
272
|
-
readonly cli?: Omit<CliHarnessOptions, 'graph'> | undefined;
|
|
273
|
-
readonly createPermit?:
|
|
274
|
-
| ((trail: {
|
|
275
|
-
readonly permit?:
|
|
276
|
-
| { readonly scopes: readonly string[] }
|
|
277
|
-
| 'public'
|
|
278
|
-
| undefined;
|
|
279
|
-
}) =>
|
|
280
|
-
| {
|
|
281
|
-
readonly id: string;
|
|
282
|
-
readonly scopes: readonly string[];
|
|
283
|
-
}
|
|
284
|
-
| undefined)
|
|
285
|
-
| undefined;
|
|
286
|
-
readonly ctx?: Partial<TrailContext> | undefined;
|
|
287
|
-
readonly http?: Omit<HttpHarnessOptions, 'graph'> | undefined;
|
|
288
|
-
readonly mcp?: Omit<McpHarnessOptions, 'graph'> | undefined;
|
|
289
|
-
readonly resources?: Record<string, unknown> | undefined;
|
|
290
|
-
readonly strictPermits?: boolean | undefined;
|
|
291
|
-
}
|
|
292
|
-
|
|
293
72
|
// ---------------------------------------------------------------------------
|
|
294
73
|
// Scenario (for composition testing)
|
|
295
74
|
// ---------------------------------------------------------------------------
|
|
296
75
|
|
|
297
|
-
/** Marker for
|
|
76
|
+
/** Marker for compose-step references in scenario inputs. */
|
|
298
77
|
export interface RefToken {
|
|
299
78
|
readonly __ref: true;
|
|
300
79
|
readonly path: string;
|
|
@@ -302,7 +81,7 @@ export interface RefToken {
|
|
|
302
81
|
|
|
303
82
|
/** A single step in a scenario. */
|
|
304
83
|
export interface ScenarioStep {
|
|
305
|
-
readonly
|
|
84
|
+
readonly compose: AnyTrail;
|
|
306
85
|
readonly input: Record<string, unknown>;
|
|
307
86
|
readonly as?: string | undefined;
|
|
308
87
|
readonly expected?: unknown | undefined;
|