@ontrails/testing 1.0.0-beta.0 → 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-lint.log +1 -1
- package/CHANGELOG.md +13 -0
- package/README.md +43 -173
- package/package.json +5 -5
package/.turbo/turbo-lint.log
CHANGED
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# @ontrails/testing
|
|
2
2
|
|
|
3
|
+
## 1.0.0-beta.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Fix two blocking bugs from real-world migration:
|
|
8
|
+
- Published packages now resolve correctly (workspace:^ instead of workspace:\*)
|
|
9
|
+
- Error forwarding works across different success types (Err no longer carries phantom T)
|
|
10
|
+
- Updated dependencies
|
|
11
|
+
- @ontrails/core@1.0.0-beta.1
|
|
12
|
+
- @ontrails/cli@1.0.0-beta.1
|
|
13
|
+
- @ontrails/mcp@1.0.0-beta.1
|
|
14
|
+
- @ontrails/logging@1.0.0-beta.1
|
|
15
|
+
|
|
3
16
|
## 1.0.0-beta.0
|
|
4
17
|
|
|
5
18
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -1,40 +1,8 @@
|
|
|
1
1
|
# @ontrails/testing
|
|
2
2
|
|
|
3
|
-
Contract-driven testing
|
|
3
|
+
Contract-driven testing for Trails. Add examples to your trails, then `testAll(app)` runs them as assertions, validates output schemas, checks composition graphs, and verifies structural integrity. One line of test code, full governance.
|
|
4
4
|
|
|
5
|
-
##
|
|
6
|
-
|
|
7
|
-
```bash
|
|
8
|
-
bun add -d @ontrails/testing
|
|
9
|
-
```
|
|
10
|
-
|
|
11
|
-
Peer dependencies: `@ontrails/core`, `@ontrails/cli`, `@ontrails/mcp`, `@ontrails/logging`, `zod`.
|
|
12
|
-
|
|
13
|
-
## Quick Start
|
|
14
|
-
|
|
15
|
-
```typescript
|
|
16
|
-
import { testExamples } from '@ontrails/testing';
|
|
17
|
-
import { app } from '../app';
|
|
18
|
-
|
|
19
|
-
testExamples(app);
|
|
20
|
-
```
|
|
21
|
-
|
|
22
|
-
One line. The entire topo is tested. Every trail, every example: input validation, implementation execution, output verification.
|
|
23
|
-
|
|
24
|
-
```text
|
|
25
|
-
PASS src/__tests__/app.test.ts
|
|
26
|
-
greet
|
|
27
|
-
example: Basic greeting
|
|
28
|
-
example: Loud greeting
|
|
29
|
-
entity.show
|
|
30
|
-
example: Show entity by name
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
## API Overview
|
|
34
|
-
|
|
35
|
-
### `testAll(topo, ctx?)`
|
|
36
|
-
|
|
37
|
-
Single-line governance suite. Wraps topo validation, example execution, contract checks, and detour verification into one `governance` describe block:
|
|
5
|
+
## Usage
|
|
38
6
|
|
|
39
7
|
```typescript
|
|
40
8
|
import { testAll } from '@ontrails/testing';
|
|
@@ -43,26 +11,38 @@ import { app } from '../app';
|
|
|
43
11
|
testAll(app);
|
|
44
12
|
```
|
|
45
13
|
|
|
46
|
-
For most apps,
|
|
14
|
+
That single call covers example execution, contract validation, detour checks, and topo validation. For most apps, this is all you need.
|
|
47
15
|
|
|
48
|
-
|
|
16
|
+
If you want finer control:
|
|
49
17
|
|
|
50
|
-
|
|
18
|
+
```typescript
|
|
19
|
+
import { testExamples, testContracts, testDetours } from '@ontrails/testing';
|
|
51
20
|
|
|
52
|
-
|
|
21
|
+
testExamples(app); // Run every trail's examples as tests
|
|
22
|
+
testContracts(app); // Validate outputs against declared schemas
|
|
23
|
+
testDetours(app); // Verify detour targets exist
|
|
24
|
+
```
|
|
53
25
|
|
|
54
|
-
|
|
55
|
-
2. Calls the implementation with validated input
|
|
56
|
-
3. Applies progressive assertion (see below)
|
|
57
|
-
4. Validates output against the trail's output schema (if present)
|
|
26
|
+
## API
|
|
58
27
|
|
|
59
|
-
|
|
28
|
+
| Export | What it does |
|
|
29
|
+
| --- | --- |
|
|
30
|
+
| `testAll(topo, ctx?)` | Single-line governance suite: validation + examples + contracts + detours |
|
|
31
|
+
| `testExamples(topo, ctx?)` | Run trail examples as `describe`/`test` blocks |
|
|
32
|
+
| `testTrail(trail, scenarios)` | Custom scenarios for edge cases and error paths |
|
|
33
|
+
| `testHike(hike, scenarios)` | Test composition graphs -- follow chains, failure injection |
|
|
34
|
+
| `testContracts(topo, ctx?)` | Validate implementation output against declared schemas |
|
|
35
|
+
| `testDetours(topo)` | Verify every detour target exists in the topo |
|
|
36
|
+
| `createTestContext(options?)` | `TrailContext` with sensible test defaults |
|
|
37
|
+
| `createTestLogger()` | Logger that captures entries in memory for assertions |
|
|
38
|
+
| `createCliHarness(options)` | Execute CLI commands in-process, capture stdout/stderr |
|
|
39
|
+
| `createMcpHarness(options)` | Invoke MCP tools directly without transport |
|
|
60
40
|
|
|
61
|
-
|
|
41
|
+
See the [API Reference](../../docs/api-reference.md) for the full list.
|
|
62
42
|
|
|
63
|
-
|
|
43
|
+
## testTrail
|
|
64
44
|
|
|
65
|
-
|
|
45
|
+
For edge cases that do not belong in agent-facing examples:
|
|
66
46
|
|
|
67
47
|
```typescript
|
|
68
48
|
import { testTrail } from '@ontrails/testing';
|
|
@@ -71,151 +51,41 @@ import { ValidationError, NotFoundError } from '@ontrails/core';
|
|
|
71
51
|
testTrail(showTrail, [
|
|
72
52
|
{ description: 'empty name', input: { name: '' }, expectOk: true },
|
|
73
53
|
{ description: 'missing name', input: {}, expectErr: ValidationError },
|
|
74
|
-
{
|
|
75
|
-
description: 'exact match',
|
|
76
|
-
input: { name: 'Alpha' },
|
|
77
|
-
expectValue: { name: 'Alpha', type: 'concept' },
|
|
78
|
-
},
|
|
79
|
-
{
|
|
80
|
-
description: 'not found',
|
|
81
|
-
input: { name: 'missing' },
|
|
82
|
-
expectErr: NotFoundError,
|
|
83
|
-
expectErrMessage: 'not found',
|
|
84
|
-
},
|
|
54
|
+
{ description: 'not found', input: { name: 'missing' }, expectErr: NotFoundError },
|
|
85
55
|
]);
|
|
86
56
|
```
|
|
87
57
|
|
|
88
|
-
|
|
58
|
+
## testHike
|
|
89
59
|
|
|
90
|
-
|
|
60
|
+
Where `testTrail` exercises a single trail, `testHike` exercises the follow graph:
|
|
91
61
|
|
|
92
62
|
```typescript
|
|
93
63
|
import { testHike } from '@ontrails/testing';
|
|
94
64
|
|
|
95
65
|
testHike(onboardHike, [
|
|
96
|
-
{ description: '
|
|
97
|
-
{ description: '
|
|
66
|
+
{ description: 'happy path', input: { name: 'Delta', type: 'tool' }, expectOk: true },
|
|
67
|
+
{ description: 'add fails', input: { name: 'Alpha' }, expectErr: AlreadyExistsError },
|
|
98
68
|
]);
|
|
99
69
|
```
|
|
100
70
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
### `testContracts(app, ctx?)`
|
|
104
|
-
|
|
105
|
-
Catches implementation-schema drift. Runs every example through the implementation, then validates the result against the trail's `output` schema. Reports detailed Zod errors on mismatch.
|
|
71
|
+
## Surface harnesses
|
|
106
72
|
|
|
107
73
|
```typescript
|
|
108
|
-
import {
|
|
109
|
-
|
|
110
|
-
testContracts(app);
|
|
111
|
-
// Fails if any implementation returns data that doesn't match its declared output schema
|
|
112
|
-
```
|
|
113
|
-
|
|
114
|
-
### `testDetours(app)`
|
|
115
|
-
|
|
116
|
-
Structural validation of detour declarations. Verifies every detour target trail exists in the topo. No implementation execution needed.
|
|
117
|
-
|
|
118
|
-
```typescript
|
|
119
|
-
import { testDetours } from '@ontrails/testing';
|
|
120
|
-
|
|
121
|
-
testDetours(app);
|
|
122
|
-
// Fails: Trail "entity.show" has detour target "entity.search" which does not exist in the topo
|
|
123
|
-
```
|
|
124
|
-
|
|
125
|
-
### Progressive Assertion
|
|
126
|
-
|
|
127
|
-
What `testExamples` checks depends on what the example declares:
|
|
128
|
-
|
|
129
|
-
**Full match** -- example has `expected`:
|
|
130
|
-
|
|
131
|
-
```typescript
|
|
132
|
-
{
|
|
133
|
-
name: 'Found',
|
|
134
|
-
input: { name: 'Alpha' },
|
|
135
|
-
expected: { name: 'Alpha', type: 'concept' },
|
|
136
|
-
}
|
|
137
|
-
```
|
|
138
|
-
|
|
139
|
-
Asserts `result.isOk()` and `result.value` deep-equals `expected`.
|
|
140
|
-
|
|
141
|
-
**Schema-only match** -- example has no `expected` and no `error`:
|
|
142
|
-
|
|
143
|
-
```typescript
|
|
144
|
-
{ name: 'Returns something valid', input: { name: 'Alpha' } }
|
|
145
|
-
```
|
|
146
|
-
|
|
147
|
-
Asserts `result.isOk()` and validates against the trail's output schema.
|
|
148
|
-
|
|
149
|
-
**Error match** -- example has `error`:
|
|
150
|
-
|
|
151
|
-
```typescript
|
|
152
|
-
{ name: 'Not found', input: { name: 'missing' }, error: 'NotFoundError' }
|
|
153
|
-
```
|
|
154
|
-
|
|
155
|
-
Asserts `result.isErr()` and `instanceof` check.
|
|
156
|
-
|
|
157
|
-
### Test Context and Mocks
|
|
158
|
-
|
|
159
|
-
```typescript
|
|
160
|
-
import { createTestContext, createTestLogger } from '@ontrails/testing';
|
|
161
|
-
|
|
162
|
-
// TrailContext with sensible test defaults
|
|
163
|
-
const ctx = createTestContext({
|
|
164
|
-
requestId: 'test-001',
|
|
165
|
-
env: { TRAILS_ENV: 'test' },
|
|
166
|
-
});
|
|
167
|
-
|
|
168
|
-
// Logger that captures entries in memory
|
|
169
|
-
const logger = createTestLogger();
|
|
170
|
-
logger.info('hello');
|
|
171
|
-
logger.entries; // All captured records
|
|
172
|
-
logger.assertLogged('info', 'hello'); // Passes if any entry matches
|
|
173
|
-
logger.clear(); // Reset
|
|
174
|
-
```
|
|
175
|
-
|
|
176
|
-
### Surface Harnesses
|
|
177
|
-
|
|
178
|
-
**CLI harness** -- execute commands in-process and capture stdout/stderr:
|
|
179
|
-
|
|
180
|
-
```typescript
|
|
181
|
-
import { createCliHarness } from '@ontrails/testing';
|
|
182
|
-
|
|
183
|
-
const harness = createCliHarness({ app });
|
|
184
|
-
const result = await harness.run('entity show --name Alpha --output json');
|
|
74
|
+
import { createCliHarness, createMcpHarness } from '@ontrails/testing';
|
|
185
75
|
|
|
76
|
+
// CLI
|
|
77
|
+
const cli = createCliHarness({ app });
|
|
78
|
+
const result = await cli.run('entity show --name Alpha --output json');
|
|
186
79
|
expect(result.exitCode).toBe(0);
|
|
187
|
-
expect(result.json).toMatchObject({ name: 'Alpha' });
|
|
188
|
-
```
|
|
189
|
-
|
|
190
|
-
**MCP harness** -- invoke tools directly without transport:
|
|
191
|
-
|
|
192
|
-
```typescript
|
|
193
|
-
import { createMcpHarness } from '@ontrails/testing';
|
|
194
|
-
|
|
195
|
-
const harness = createMcpHarness({ app });
|
|
196
|
-
const result = await harness.callTool('myapp_entity_show', { name: 'Alpha' });
|
|
197
80
|
|
|
198
|
-
|
|
81
|
+
// MCP
|
|
82
|
+
const mcp = createMcpHarness({ app });
|
|
83
|
+
const tool = await mcp.callTool('myapp_entity_show', { name: 'Alpha' });
|
|
84
|
+
expect(tool.isError).toBe(false);
|
|
199
85
|
```
|
|
200
86
|
|
|
201
|
-
##
|
|
87
|
+
## Installation
|
|
202
88
|
|
|
203
|
-
```
|
|
204
|
-
|
|
205
|
-
testAll,
|
|
206
|
-
testExamples,
|
|
207
|
-
testTrail,
|
|
208
|
-
testHike,
|
|
209
|
-
testContracts,
|
|
210
|
-
testDetours,
|
|
211
|
-
createTestContext,
|
|
212
|
-
createTestLogger,
|
|
213
|
-
createCliHarness,
|
|
214
|
-
createMcpHarness,
|
|
215
|
-
} from '@ontrails/testing';
|
|
89
|
+
```bash
|
|
90
|
+
bun add -d @ontrails/testing
|
|
216
91
|
```
|
|
217
|
-
|
|
218
|
-
## Further Reading
|
|
219
|
-
|
|
220
|
-
- [Testing Guide](../../docs/testing.md)
|
|
221
|
-
- [Getting Started](../../docs/getting-started.md)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ontrails/testing",
|
|
3
|
-
"version": "1.0.0-beta.
|
|
3
|
+
"version": "1.0.0-beta.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./src/index.ts",
|
|
@@ -14,10 +14,10 @@
|
|
|
14
14
|
"clean": "rm -rf dist *.tsbuildinfo"
|
|
15
15
|
},
|
|
16
16
|
"peerDependencies": {
|
|
17
|
-
"@ontrails/cli": "workspace
|
|
18
|
-
"@ontrails/core": "workspace
|
|
19
|
-
"@ontrails/logging": "workspace
|
|
20
|
-
"@ontrails/mcp": "workspace
|
|
17
|
+
"@ontrails/cli": "workspace:^",
|
|
18
|
+
"@ontrails/core": "workspace:^",
|
|
19
|
+
"@ontrails/logging": "workspace:^",
|
|
20
|
+
"@ontrails/mcp": "workspace:^",
|
|
21
21
|
"zod": "catalog:"
|
|
22
22
|
}
|
|
23
23
|
}
|