mcp-integration-harness 0.1.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +126 -0
- package/dist/coverage.d.ts +56 -0
- package/dist/coverage.js +58 -0
- package/dist/coverage.js.map +1 -0
- package/dist/harness.d.ts +88 -0
- package/dist/harness.js +110 -0
- package/dist/harness.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/loopback.d.ts +22 -0
- package/dist/loopback.js +50 -0
- package/dist/loopback.js.map +1 -0
- package/dist/wait.d.ts +49 -0
- package/dist/wait.js +111 -0
- package/dist/wait.js.map +1 -0
- package/package.json +66 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Willi Thiel
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# mcp-integration-harness
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/mcp-integration-harness)
|
|
4
|
+
[](https://nodejs.org)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Run your built [Model Context Protocol](https://modelcontextprotocol.io) server
|
|
8
|
+
as a real process, against a real backend in Docker, and fail the build unless
|
|
9
|
+
**every** tool was either exercised or excused in writing.
|
|
10
|
+
|
|
11
|
+
```ts
|
|
12
|
+
const harness = await startServer({
|
|
13
|
+
env: { WIKIJS_URL: backendUrl, WIKIJS_TOKEN: token },
|
|
14
|
+
elicit: 'accept',
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
const page = await harness.call('create_page', {
|
|
18
|
+
path: 'test',
|
|
19
|
+
content: '# hi',
|
|
20
|
+
});
|
|
21
|
+
await harness.call('get_page', { path: 'test' });
|
|
22
|
+
await harness.confirmed('delete_page', { id });
|
|
23
|
+
|
|
24
|
+
it('exercises every tool in the catalogue', () => {
|
|
25
|
+
expectEveryToolExercised(harness, ALL_TOOLS, {
|
|
26
|
+
get_page_conflict: 'needs two concurrent edits; see the conflict test',
|
|
27
|
+
reset_user_password: 'needs a local user and a working mail transport',
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Why not the in-memory transport you already have
|
|
33
|
+
|
|
34
|
+
Because it answers a different question. A unit suite links server to client
|
|
35
|
+
with `InMemoryTransport` and replaces `fetch` underneath, which tests whether
|
|
36
|
+
the server does what you believe the remote API does — not whether the API does
|
|
37
|
+
it. Every genuinely surprising bug in this family was found by hand against a
|
|
38
|
+
running instance: a `limit` parameter that counts join rows rather than pages,
|
|
39
|
+
an empty `files` entry that deletes, a `filter` that is silently ignored. A
|
|
40
|
+
stub cannot find those, because the stub encodes the same belief the code does.
|
|
41
|
+
|
|
42
|
+
Four things are additionally untouched by any in-process test: `src/index.ts`,
|
|
43
|
+
`loadConfig` reading a real environment, the stdio transport's framing, and
|
|
44
|
+
elicitation across a **process boundary**. This spawns the built artifact, so
|
|
45
|
+
all four are on the path.
|
|
46
|
+
|
|
47
|
+
## `expectEveryToolExercised`
|
|
48
|
+
|
|
49
|
+
The part worth copying even if you write the rest yourself.
|
|
50
|
+
|
|
51
|
+
`skipped` is a `Record<tool, reason>`, never a `string[]`. A bare list lets a
|
|
52
|
+
tool leave the suite by adding six characters, and nothing afterwards
|
|
53
|
+
distinguishes a deliberate omission from a forgotten one. A reason has to be
|
|
54
|
+
written by a person — a small cost, in exactly the place a small cost is
|
|
55
|
+
useful.
|
|
56
|
+
|
|
57
|
+
It fails in three directions, not one:
|
|
58
|
+
|
|
59
|
+
1. **A tool neither called nor excused.** The gap everyone expects.
|
|
60
|
+
2. **An excused tool that _was_ called.** The reason is now false, and a false
|
|
61
|
+
reason is worse than none: the next person reads it and believes the tool
|
|
62
|
+
cannot be tested.
|
|
63
|
+
3. **An excused tool that no longer exists.** Renamed or removed, its excuse
|
|
64
|
+
left behind, quietly making the exception list look longer than the real one.
|
|
65
|
+
|
|
66
|
+
All three are reported together, so a fourteen-repository rollout is one
|
|
67
|
+
afternoon rather than fourteen CI rounds.
|
|
68
|
+
|
|
69
|
+
## `assertLoopback`
|
|
70
|
+
|
|
71
|
+
The guard that matters more than the tests it protects.
|
|
72
|
+
|
|
73
|
+
An integration suite calls every tool, deletes included, and the machine it is
|
|
74
|
+
written on is usually the same machine that has the _real_ servers configured —
|
|
75
|
+
a wiki people write in, a CI instance that builds things, a VPN. One inherited
|
|
76
|
+
`WIKIJS_URL` is all it takes.
|
|
77
|
+
|
|
78
|
+
So: a **hard throw**, never a skip. A skipped test reports "nothing to do
|
|
79
|
+
here", which is the wrong sentence when the reason is "this was pointed at
|
|
80
|
+
production". Hosts are compared numerically via
|
|
81
|
+
[`mcp-internal-hosts`](https://www.npmjs.com/package/mcp-internal-hosts), so
|
|
82
|
+
`[::ffff:127.0.0.1]` and `localhost.` count and `127.example.com` — a hostname
|
|
83
|
+
anybody can register — does not.
|
|
84
|
+
|
|
85
|
+
`startServer` is the same idea as a property: the child gets `PATH` and the
|
|
86
|
+
variables you passed. Nothing is inherited, so nothing can be inherited by
|
|
87
|
+
accident.
|
|
88
|
+
|
|
89
|
+
## API
|
|
90
|
+
|
|
91
|
+
| Export | What it does |
|
|
92
|
+
| -------------------------- | ------------------------------------------------------------------------------- |
|
|
93
|
+
| `startServer(options)` | Spawns `dist/index.js` over real stdio and returns a `LiveHarness` |
|
|
94
|
+
| `harness.call(name, args)` | Calls a tool, records it for coverage, returns the joined text parts |
|
|
95
|
+
| `harness.raw(name, args)` | The same, returning the whole result — for a tool that answers with an image |
|
|
96
|
+
| `harness.confirmed(…)` | Drives **both halves** of the two-call token, for the no-dialog fallback path |
|
|
97
|
+
| `harness.prompts` | Every message the server put in front of the user, in order |
|
|
98
|
+
| `harness.stderr()` | Everything the server wrote to stderr, including before the handshake completed |
|
|
99
|
+
| `expectEveryToolExercised` | The three-way coverage assertion above |
|
|
100
|
+
| `toolCoverage` | The same comparison without asserting, for printing the numbers |
|
|
101
|
+
| `assertLoopback(url)` | Throws unless the URL is on this machine |
|
|
102
|
+
| `waitForHttp(url, opts)` | Polls until an HTTP backend is ready, and says what the last attempt got |
|
|
103
|
+
| `waitForTcp(host, port)` | The same for a backend that is not HTTP — IMAP, SMTP — optionally on a greeting |
|
|
104
|
+
|
|
105
|
+
`elicit: 'accept' | 'decline' | 'cancel'` makes the harness declare the
|
|
106
|
+
elicitation capability and answer the dialog, which is the path a real client
|
|
107
|
+
takes. **Omitting** it declares no capability at all, which is what makes a
|
|
108
|
+
guarded tool fall back to the token — so `confirmed()` only works on a harness
|
|
109
|
+
started without `elicit`, and says so when it does not.
|
|
110
|
+
|
|
111
|
+
## Requirements
|
|
112
|
+
|
|
113
|
+
Node 22 or newer, and `@modelcontextprotocol/client` 2.x as a peer. Both belong
|
|
114
|
+
in `devDependencies`: this is test infrastructure and has no business in a
|
|
115
|
+
server's runtime tree.
|
|
116
|
+
|
|
117
|
+
## Related
|
|
118
|
+
|
|
119
|
+
- [`mcp-approval`](https://www.npmjs.com/package/mcp-approval) — the
|
|
120
|
+
human-in-the-loop guard whose two paths this drives
|
|
121
|
+
- [`mcp-internal-hosts`](https://www.npmjs.com/package/mcp-internal-hosts) —
|
|
122
|
+
the SSRF host classifier the loopback guard is built on
|
|
123
|
+
|
|
124
|
+
## Licence
|
|
125
|
+
|
|
126
|
+
MIT
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { LiveHarness } from './harness.js';
|
|
2
|
+
/**
|
|
3
|
+
* The reason a tool could not be exercised against a real backend.
|
|
4
|
+
*
|
|
5
|
+
* A `Record<tool, reason>` rather than a `string[]`, and that is the whole
|
|
6
|
+
* design. A bare list lets a tool be dropped from the suite by adding six
|
|
7
|
+
* characters, and nothing afterwards can tell a deliberate omission from a
|
|
8
|
+
* forgotten one. A reason has to be written by a person, which is a small cost
|
|
9
|
+
* exactly where a small cost is useful.
|
|
10
|
+
*
|
|
11
|
+
* Reasons that have earned their place look like:
|
|
12
|
+
*
|
|
13
|
+
* list_listening_sessions: 'needs playback sessions; no tool can create one'
|
|
14
|
+
* approve_pipeline: 'needs a fork pipeline blocked on approval'
|
|
15
|
+
* get_page_conflict: 'needs two concurrent edits; see the conflict test'
|
|
16
|
+
*
|
|
17
|
+
* A reason that has not:
|
|
18
|
+
*
|
|
19
|
+
* delete_everything: 'skipped'
|
|
20
|
+
*/
|
|
21
|
+
export type SkipReasons = Readonly<Record<string, string>>;
|
|
22
|
+
export interface CoverageReport {
|
|
23
|
+
called: readonly string[];
|
|
24
|
+
skipped: readonly string[];
|
|
25
|
+
/** In the catalogue, neither called nor given a reason. */
|
|
26
|
+
missing: readonly string[];
|
|
27
|
+
/** Given a reason, but called anyway — the reason is stale. */
|
|
28
|
+
staleReasons: readonly string[];
|
|
29
|
+
/** Given a reason, but no longer a tool — the reason outlived its tool. */
|
|
30
|
+
unknownReasons: readonly string[];
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Compares what ran against the catalogue, without asserting.
|
|
34
|
+
*
|
|
35
|
+
* Exported separately so a caller can print the numbers — "48 of 62 against a
|
|
36
|
+
* real backend, 14 skipped with reasons" is worth having in a CI log even on a
|
|
37
|
+
* green run.
|
|
38
|
+
*/
|
|
39
|
+
export declare function toolCoverage(harness: Pick<LiveHarness, 'called'>, allTools: readonly string[], skipped: SkipReasons): CoverageReport;
|
|
40
|
+
/**
|
|
41
|
+
* Fails unless every tool in the catalogue was called or excused.
|
|
42
|
+
*
|
|
43
|
+
* Three directions, not one, because a coverage check that only looks for gaps
|
|
44
|
+
* rots from the other end:
|
|
45
|
+
*
|
|
46
|
+
* 1. A tool neither called nor excused — the gap everyone expects.
|
|
47
|
+
* 2. An excused tool that *was* called. The reason is now false, and a false
|
|
48
|
+
* reason is worse than none: the next person reads it and believes the
|
|
49
|
+
* tool cannot be tested.
|
|
50
|
+
* 3. An excused tool that no longer exists. The tool was renamed or removed
|
|
51
|
+
* and its excuse stayed behind, quietly making the exception list look
|
|
52
|
+
* longer than the real one.
|
|
53
|
+
*
|
|
54
|
+
* Throws rather than returning, so it reads as one line in a test.
|
|
55
|
+
*/
|
|
56
|
+
export declare function expectEveryToolExercised(harness: Pick<LiveHarness, 'called'>, allTools: readonly string[], skipped?: SkipReasons): void;
|
package/dist/coverage.js
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compares what ran against the catalogue, without asserting.
|
|
3
|
+
*
|
|
4
|
+
* Exported separately so a caller can print the numbers — "48 of 62 against a
|
|
5
|
+
* real backend, 14 skipped with reasons" is worth having in a CI log even on a
|
|
6
|
+
* green run.
|
|
7
|
+
*/
|
|
8
|
+
export function toolCoverage(harness, allTools, skipped) {
|
|
9
|
+
const catalogue = new Set(allTools);
|
|
10
|
+
const reasons = Object.keys(skipped);
|
|
11
|
+
return {
|
|
12
|
+
called: [...harness.called].sort(),
|
|
13
|
+
skipped: reasons.sort(),
|
|
14
|
+
missing: allTools
|
|
15
|
+
.filter((tool) => !harness.called.has(tool) && !(tool in skipped))
|
|
16
|
+
.sort(),
|
|
17
|
+
staleReasons: reasons.filter((tool) => harness.called.has(tool)).sort(),
|
|
18
|
+
unknownReasons: reasons.filter((tool) => !catalogue.has(tool)).sort(),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Fails unless every tool in the catalogue was called or excused.
|
|
23
|
+
*
|
|
24
|
+
* Three directions, not one, because a coverage check that only looks for gaps
|
|
25
|
+
* rots from the other end:
|
|
26
|
+
*
|
|
27
|
+
* 1. A tool neither called nor excused — the gap everyone expects.
|
|
28
|
+
* 2. An excused tool that *was* called. The reason is now false, and a false
|
|
29
|
+
* reason is worse than none: the next person reads it and believes the
|
|
30
|
+
* tool cannot be tested.
|
|
31
|
+
* 3. An excused tool that no longer exists. The tool was renamed or removed
|
|
32
|
+
* and its excuse stayed behind, quietly making the exception list look
|
|
33
|
+
* longer than the real one.
|
|
34
|
+
*
|
|
35
|
+
* Throws rather than returning, so it reads as one line in a test.
|
|
36
|
+
*/
|
|
37
|
+
export function expectEveryToolExercised(harness, allTools, skipped = {}) {
|
|
38
|
+
const report = toolCoverage(harness, allTools, skipped);
|
|
39
|
+
const problems = [];
|
|
40
|
+
if (report.missing.length > 0) {
|
|
41
|
+
problems.push(`${report.missing.length} tool(s) never called and not excused: ` +
|
|
42
|
+
`${report.missing.join(', ')}. Call them, or give each a reason in the ` +
|
|
43
|
+
'skip map saying what a real backend cannot provide.');
|
|
44
|
+
}
|
|
45
|
+
if (report.staleReasons.length > 0) {
|
|
46
|
+
problems.push(`${report.staleReasons.length} excused tool(s) were called after all: ` +
|
|
47
|
+
`${report.staleReasons.join(', ')}. Remove the reason — it is no longer true.`);
|
|
48
|
+
}
|
|
49
|
+
if (report.unknownReasons.length > 0) {
|
|
50
|
+
problems.push(`${report.unknownReasons.length} reason(s) name a tool that no longer ` +
|
|
51
|
+
`exists: ${report.unknownReasons.join(', ')}.`);
|
|
52
|
+
}
|
|
53
|
+
if (problems.length > 0) {
|
|
54
|
+
throw new Error(`${report.called.length} of ${allTools.length} tools exercised, ` +
|
|
55
|
+
`${report.skipped.length} excused.\n\n${problems.join('\n\n')}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=coverage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coverage.js","sourceRoot":"","sources":["../src/coverage.ts"],"names":[],"mappings":"AAkCA;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAC1B,OAAoC,EACpC,QAA2B,EAC3B,OAAoB;IAEpB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,OAAO;QACL,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;QAClC,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;QACvB,OAAO,EAAE,QAAQ;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,CAAC;aACjE,IAAI,EAAE;QACT,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;QACvE,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;KACtE,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,wBAAwB,CACtC,OAAoC,EACpC,QAA2B,EAC3B,OAAO,GAAgB,EAAE;IAEzB,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CACX,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,yCAAyC;YAC/D,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,4CAA4C;YACxE,qDAAqD,CACxD,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,QAAQ,CAAC,IAAI,CACX,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,0CAA0C;YACrE,GAAG,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,6CAA6C,CACjF,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,QAAQ,CAAC,IAAI,CACX,GAAG,MAAM,CAAC,cAAc,CAAC,MAAM,wCAAwC;YACrE,WAAW,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACjD,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,OAAO,QAAQ,CAAC,MAAM,oBAAoB;YAC/D,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,gBAAgB,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAClE,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { Client } from '@modelcontextprotocol/client';
|
|
2
|
+
/**
|
|
3
|
+
* Driving a built MCP server over real stdio, against a real backend.
|
|
4
|
+
*
|
|
5
|
+
* Every unit suite in this family links a server to a client with
|
|
6
|
+
* `InMemoryTransport` and replaces the network underneath. That is the right
|
|
7
|
+
* trade for a unit test, and it leaves four things untested in every repository:
|
|
8
|
+
* `src/index.ts`, `loadConfig` reading a real environment, the stdio transport's
|
|
9
|
+
* framing, and elicitation across a process boundary. This spawns the built
|
|
10
|
+
* artifact instead, so all four are on the path.
|
|
11
|
+
*/
|
|
12
|
+
/** How a client that can show a dialog answers it. */
|
|
13
|
+
export type ElicitBehaviour = 'accept' | 'decline' | 'cancel';
|
|
14
|
+
export interface StartServerOptions {
|
|
15
|
+
/** The built entry point. Relative paths resolve against `cwd`. */
|
|
16
|
+
entry?: string;
|
|
17
|
+
/** Where to run it. Defaults to the current working directory. */
|
|
18
|
+
cwd?: string;
|
|
19
|
+
/**
|
|
20
|
+
* The server's entire environment, `PATH` aside.
|
|
21
|
+
*
|
|
22
|
+
* Deliberately not merged with `process.env`. A `WIKIJS_URL` left in a shell
|
|
23
|
+
* is otherwise enough to point an integration run — deletes included — at
|
|
24
|
+
* whatever that variable happens to name. Nothing is inherited, so nothing
|
|
25
|
+
* can be inherited by accident.
|
|
26
|
+
*/
|
|
27
|
+
env: Record<string, string>;
|
|
28
|
+
/**
|
|
29
|
+
* How the client answers a confirmation dialog. Omitted means the client
|
|
30
|
+
* declares no elicitation capability at all, which is what makes a guarded
|
|
31
|
+
* tool fall back to the two-call token — see {@link LiveHarness.confirmed}.
|
|
32
|
+
*/
|
|
33
|
+
elicit?: ElicitBehaviour;
|
|
34
|
+
/** Seconds to wait for the handshake. Default 30. */
|
|
35
|
+
timeoutSeconds?: number;
|
|
36
|
+
}
|
|
37
|
+
export interface CallOptions {
|
|
38
|
+
/** Assert that the call fails. Refusals are behaviour worth pinning too. */
|
|
39
|
+
expectError?: boolean;
|
|
40
|
+
}
|
|
41
|
+
export interface ToolResult {
|
|
42
|
+
content?: {
|
|
43
|
+
type: string;
|
|
44
|
+
text?: string;
|
|
45
|
+
mimeType?: string;
|
|
46
|
+
data?: string;
|
|
47
|
+
}[];
|
|
48
|
+
isError?: boolean;
|
|
49
|
+
}
|
|
50
|
+
export interface LiveHarness {
|
|
51
|
+
client: Client;
|
|
52
|
+
/**
|
|
53
|
+
* Calls a tool, records it against the coverage set, returns the text.
|
|
54
|
+
*
|
|
55
|
+
* Throws when the outcome is not the expected one, with the server's own
|
|
56
|
+
* message attached — a failure here is nearly always the backend saying
|
|
57
|
+
* something the stubbed unit tests never had to say.
|
|
58
|
+
*/
|
|
59
|
+
call(name: string, args?: Record<string, unknown>, options?: CallOptions): Promise<string>;
|
|
60
|
+
/**
|
|
61
|
+
* The same, returning the whole result rather than its text.
|
|
62
|
+
*
|
|
63
|
+
* For a tool that answers with an image or a resource — a cover, a QR code,
|
|
64
|
+
* an uploaded asset — where the parts are the point. Reaching for
|
|
65
|
+
* `harness.client` instead would skip the coverage bookkeeping, and the
|
|
66
|
+
* missing tool would then have to be added to `called` by hand, which is
|
|
67
|
+
* exactly the sort of thing that stops being done.
|
|
68
|
+
*/
|
|
69
|
+
raw(name: string, args?: Record<string, unknown>, options?: CallOptions): Promise<ToolResult>;
|
|
70
|
+
/**
|
|
71
|
+
* Drives both halves of the two-call token for one guarded tool.
|
|
72
|
+
*
|
|
73
|
+
* Only meaningful on a harness started **without** `elicit`: with a dialog
|
|
74
|
+
* available the server refuses to offer a token at all, which is the whole
|
|
75
|
+
* point of the dialog. Use it to prove the fallback path still works.
|
|
76
|
+
*/
|
|
77
|
+
confirmed(name: string, args?: Record<string, unknown>): Promise<string>;
|
|
78
|
+
/** Every message the server put in front of the user, in order. */
|
|
79
|
+
prompts: string[];
|
|
80
|
+
/** The tools that were called. What {@link expectEveryToolExercised} reads. */
|
|
81
|
+
called: ReadonlySet<string>;
|
|
82
|
+
/** Everything the server wrote to stderr, for a failure report. */
|
|
83
|
+
stderr(): string;
|
|
84
|
+
close(): Promise<void>;
|
|
85
|
+
}
|
|
86
|
+
/** Pulls the fallback token out of a refusal. */
|
|
87
|
+
export declare function tokenOf(text: string): string;
|
|
88
|
+
export declare function startServer(options: StartServerOptions): Promise<LiveHarness>;
|
package/dist/harness.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { Client } from '@modelcontextprotocol/client';
|
|
2
|
+
import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';
|
|
3
|
+
/** Pulls the fallback token out of a refusal. */
|
|
4
|
+
export function tokenOf(text) {
|
|
5
|
+
const match = /confirm_token="([0-9a-f]+)"/.exec(text);
|
|
6
|
+
if (!match?.[1]) {
|
|
7
|
+
throw new Error(`no confirm_token in the result — did the client declare elicitation? ` +
|
|
8
|
+
`Got: ${text.slice(0, 300)}`);
|
|
9
|
+
}
|
|
10
|
+
return match[1];
|
|
11
|
+
}
|
|
12
|
+
/** The text parts of a tool result, joined. */
|
|
13
|
+
function textOf(result) {
|
|
14
|
+
const parts = (result.content ?? []);
|
|
15
|
+
return parts
|
|
16
|
+
.filter((part) => part.type === 'text')
|
|
17
|
+
.map((part) => part.text)
|
|
18
|
+
.join('\n');
|
|
19
|
+
}
|
|
20
|
+
export async function startServer(options) {
|
|
21
|
+
const entry = options.entry ?? 'dist/index.js';
|
|
22
|
+
const prompts = [];
|
|
23
|
+
const called = new Set();
|
|
24
|
+
const errors = [];
|
|
25
|
+
const client = new Client({ name: 'mcp-integration-harness', version: '0.1.0' }, options.elicit === undefined ? {} : { capabilities: { elicitation: {} } });
|
|
26
|
+
if (options.elicit !== undefined) {
|
|
27
|
+
const behaviour = options.elicit;
|
|
28
|
+
client.setRequestHandler('elicitation/create', (request) => {
|
|
29
|
+
// `message` is required by the elicitation schema, so no fallback: a
|
|
30
|
+
// server that omitted it should surface as an empty prompt in the
|
|
31
|
+
// assertion, not be papered over here.
|
|
32
|
+
prompts.push(request.params.message);
|
|
33
|
+
if (behaviour === 'cancel')
|
|
34
|
+
return { action: 'cancel' };
|
|
35
|
+
if (behaviour === 'decline')
|
|
36
|
+
return { action: 'decline' };
|
|
37
|
+
return { action: 'accept', content: { confirm: true } };
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
const transport = new StdioClientTransport({
|
|
41
|
+
command: process.execPath,
|
|
42
|
+
args: [entry],
|
|
43
|
+
// PATH only. See the comment on StartServerOptions.env.
|
|
44
|
+
env: { PATH: process.env.PATH ?? '', ...options.env },
|
|
45
|
+
...(options.cwd === undefined ? {} : { cwd: options.cwd }),
|
|
46
|
+
stderr: 'pipe',
|
|
47
|
+
});
|
|
48
|
+
// Attached before connect, not after: the startup banner and any refusal to
|
|
49
|
+
// start are written during the handshake, and a listener added afterwards
|
|
50
|
+
// has already missed them. The transport hands back a PassThrough
|
|
51
|
+
// immediately for exactly this reason.
|
|
52
|
+
transport.stderr?.on('data', (chunk) => {
|
|
53
|
+
errors.push(chunk.toString());
|
|
54
|
+
});
|
|
55
|
+
try {
|
|
56
|
+
await client.connect(transport);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
// A server that dies during the handshake reports "Connection closed" and
|
|
60
|
+
// nothing else, while the reason — a missing dist/, a config error, a
|
|
61
|
+
// refused credential — is sitting in the stderr just captured. Without this
|
|
62
|
+
// the most common first failure of a new suite is also the least legible.
|
|
63
|
+
throw new Error(`mcp-integration-harness: ${process.execPath} ${entry} did not start.\n` +
|
|
64
|
+
`${String(error)}\n\nIts stderr:\n${errors.join('') || '(nothing)'}`);
|
|
65
|
+
}
|
|
66
|
+
const raw = async (name, args = {}, callOptions = {}) => {
|
|
67
|
+
called.add(name);
|
|
68
|
+
let result;
|
|
69
|
+
try {
|
|
70
|
+
result = (await client.callTool({
|
|
71
|
+
name,
|
|
72
|
+
arguments: args,
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
// The same reasoning as at connect time, for the other moment a server
|
|
77
|
+
// can die: a crash mid-suite surfaces as a bare "Not connected" on the
|
|
78
|
+
// next call, while what actually happened is in the stderr captured
|
|
79
|
+
// since it started.
|
|
80
|
+
throw new Error(`mcp-integration-harness: calling ${name} failed at the transport.\n` +
|
|
81
|
+
`${String(error)}\n\nThe server's stderr so far:\n` +
|
|
82
|
+
`${errors.join('') || '(nothing)'}`);
|
|
83
|
+
}
|
|
84
|
+
const failed = result.isError === true;
|
|
85
|
+
if (failed !== (callOptions.expectError ?? false)) {
|
|
86
|
+
const text = textOf(result);
|
|
87
|
+
throw new Error(callOptions.expectError
|
|
88
|
+
? `${name} was expected to fail and did not: ${text.slice(0, 500)}`
|
|
89
|
+
: `${name} failed: ${text.slice(0, 500)}`);
|
|
90
|
+
}
|
|
91
|
+
return result;
|
|
92
|
+
};
|
|
93
|
+
const call = async (name, args = {}, callOptions = {}) => textOf(await raw(name, args, callOptions));
|
|
94
|
+
return {
|
|
95
|
+
client,
|
|
96
|
+
call,
|
|
97
|
+
raw,
|
|
98
|
+
prompts,
|
|
99
|
+
called,
|
|
100
|
+
stderr: () => errors.join(''),
|
|
101
|
+
confirmed: async (name, args = {}) => {
|
|
102
|
+
const first = await call(name, args);
|
|
103
|
+
return call(name, { ...args, confirm_token: tokenOf(first) });
|
|
104
|
+
},
|
|
105
|
+
close: async () => {
|
|
106
|
+
await client.close();
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
//# sourceMappingURL=harness.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harness.js","sourceRoot":"","sources":["../src/harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AA+F1E,iDAAiD;AACjD,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CACb,uEAAuE;YACrE,QAAQ,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC/B,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,+CAA+C;AAC/C,SAAS,MAAM,CAAC,MAA6B;IAC3C,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAsC,CAAC;IAC1E,OAAO,KAAK;SACT,MAAM,CACL,CAAC,IAAI,EAA0C,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CACvE;SACA,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;SACxB,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAA2B;IAE3B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,eAAe,CAAC;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,yBAAyB,EAAE,OAAO,EAAE,OAAO,EAAE,EACrD,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,EAAE,CAC1E,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;QACjC,MAAM,CAAC,iBAAiB,CAAC,oBAAoB,EAAE,CAAC,OAAO,EAAE,EAAE;YACzD,qEAAqE;YACrE,kEAAkE;YAClE,uCAAuC;YACvC,OAAO,CAAC,IAAI,CAAE,OAAO,CAAC,MAA8B,CAAC,OAAO,CAAC,CAAC;YAC9D,IAAI,SAAS,KAAK,QAAQ;gBAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;YACxD,IAAI,SAAS,KAAK,SAAS;gBAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;YAC1D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,oBAAoB,CAAC;QACzC,OAAO,EAAE,OAAO,CAAC,QAAQ;QACzB,IAAI,EAAE,CAAC,KAAK,CAAC;QACb,wDAAwD;QACxD,GAAG,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE;QACrD,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;QAC1D,MAAM,EAAE,MAAM;KACf,CAAC,CAAC;IAEH,4EAA4E;IAC5E,0EAA0E;IAC1E,kEAAkE;IAClE,uCAAuC;IACvC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;QAC7C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,0EAA0E;QAC1E,sEAAsE;QACtE,4EAA4E;QAC5E,0EAA0E;QAC1E,MAAM,IAAI,KAAK,CACb,4BAA4B,OAAO,CAAC,QAAQ,IAAI,KAAK,mBAAmB;YACtE,GAAG,MAAM,CAAC,KAAK,CAAC,oBAAoB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,WAAW,EAAE,CACvE,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,KAAK,EACf,IAAY,EACZ,IAAI,GAA4B,EAAE,EAClC,WAAW,GAAgB,EAAE,EACR,EAAE;QACvB,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,MAAkB,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC;gBAC9B,IAAI;gBACJ,SAAS,EAAE,IAAI;aAChB,CAAC,CAAe,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,uEAAuE;YACvE,uEAAuE;YACvE,oEAAoE;YACpE,oBAAoB;YACpB,MAAM,IAAI,KAAK,CACb,oCAAoC,IAAI,6BAA6B;gBACnE,GAAG,MAAM,CAAC,KAAK,CAAC,mCAAmC;gBACnD,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,WAAW,EAAE,CACtC,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC;QACvC,IAAI,MAAM,KAAK,CAAC,WAAW,CAAC,WAAW,IAAI,KAAK,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,WAAW,CAAC,WAAW;gBACrB,CAAC,CAAC,GAAG,IAAI,sCAAsC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;gBACnE,CAAC,CAAC,GAAG,IAAI,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC5C,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,EAChB,IAAY,EACZ,IAAI,GAA4B,EAAE,EAClC,WAAW,GAAgB,EAAE,EACZ,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;IAEjE,OAAO;QACL,MAAM;QACN,IAAI;QACJ,GAAG;QACH,OAAO;QACP,MAAM;QACN,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,KAAK,EAAE,KAAK,IAAI,EAAE;YAChB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { startServer, tokenOf, type CallOptions, type ElicitBehaviour, type LiveHarness, type StartServerOptions, type ToolResult, } from './harness.js';
|
|
2
|
+
export { expectEveryToolExercised, toolCoverage, type CoverageReport, type SkipReasons, } from './coverage.js';
|
|
3
|
+
export { assertLoopback, assertLoopbackHost } from './loopback.js';
|
|
4
|
+
export { waitForHttp, waitForTcp, type TcpWaitOptions, type WaitOptions, } from './wait.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { startServer, tokenOf, } from './harness.js';
|
|
2
|
+
export { expectEveryToolExercised, toolCoverage, } from './coverage.js';
|
|
3
|
+
export { assertLoopback, assertLoopbackHost } from './loopback.js';
|
|
4
|
+
export { waitForHttp, waitForTcp, } from './wait.js';
|
|
5
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EACX,OAAO,GAMR,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,wBAAwB,EACxB,YAAY,GAGb,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnE,OAAO,EACL,WAAW,EACX,UAAU,GAGX,MAAM,WAAW,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Refuses any backend that is not on this machine.
|
|
3
|
+
*
|
|
4
|
+
* This is the one guard that matters more than the tests it protects. An
|
|
5
|
+
* integration suite calls every tool, including the deletes, and the machine it
|
|
6
|
+
* is written on is usually the machine that also has the *real* servers
|
|
7
|
+
* configured — a wiki people write in, a CI instance that builds things, a VPN.
|
|
8
|
+
* Getting `WIKIJS_URL` from the ambient environment once is all it takes.
|
|
9
|
+
*
|
|
10
|
+
* So: a hard throw, never a skip. A skipped test reads as "nothing to do here",
|
|
11
|
+
* which is exactly the wrong report when the reason is "this was pointed at
|
|
12
|
+
* production". The suite must stop and say so.
|
|
13
|
+
*
|
|
14
|
+
* `internalHostKind` rather than a string comparison, so every spelling of the
|
|
15
|
+
* same address counts: `[::ffff:127.0.0.1]`, which `URL` canonicalises to
|
|
16
|
+
* `[::ffff:7f00:1]` before anything else sees it, and `localhost.` with its root
|
|
17
|
+
* label. A prefix check on `127.` would also call `127.example.com` local, which
|
|
18
|
+
* is a public hostname anyone can register.
|
|
19
|
+
*/
|
|
20
|
+
export declare function assertLoopbackHost(host: string): void;
|
|
21
|
+
/** {@link assertLoopbackHost}, for a backend addressed by URL. */
|
|
22
|
+
export declare function assertLoopback(url: string): void;
|
package/dist/loopback.js
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { internalHostKind } from 'mcp-internal-hosts';
|
|
2
|
+
/**
|
|
3
|
+
* Refuses any backend that is not on this machine.
|
|
4
|
+
*
|
|
5
|
+
* This is the one guard that matters more than the tests it protects. An
|
|
6
|
+
* integration suite calls every tool, including the deletes, and the machine it
|
|
7
|
+
* is written on is usually the machine that also has the *real* servers
|
|
8
|
+
* configured — a wiki people write in, a CI instance that builds things, a VPN.
|
|
9
|
+
* Getting `WIKIJS_URL` from the ambient environment once is all it takes.
|
|
10
|
+
*
|
|
11
|
+
* So: a hard throw, never a skip. A skipped test reads as "nothing to do here",
|
|
12
|
+
* which is exactly the wrong report when the reason is "this was pointed at
|
|
13
|
+
* production". The suite must stop and say so.
|
|
14
|
+
*
|
|
15
|
+
* `internalHostKind` rather than a string comparison, so every spelling of the
|
|
16
|
+
* same address counts: `[::ffff:127.0.0.1]`, which `URL` canonicalises to
|
|
17
|
+
* `[::ffff:7f00:1]` before anything else sees it, and `localhost.` with its root
|
|
18
|
+
* label. A prefix check on `127.` would also call `127.example.com` local, which
|
|
19
|
+
* is a public hostname anyone can register.
|
|
20
|
+
*/
|
|
21
|
+
export function assertLoopbackHost(host) {
|
|
22
|
+
if (internalHostKind(host) !== 'loopback') {
|
|
23
|
+
throw new Error(`mcp-integration-harness: refusing to talk to ${host} — the integration ` +
|
|
24
|
+
'suite calls every tool, deletes included, and may only ever talk to a ' +
|
|
25
|
+
'throwaway backend on this machine. Expected a loopback host; got one ' +
|
|
26
|
+
'that resolves somewhere else.');
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/** {@link assertLoopbackHost}, for a backend addressed by URL. */
|
|
30
|
+
export function assertLoopback(url) {
|
|
31
|
+
let parsed;
|
|
32
|
+
try {
|
|
33
|
+
parsed = new URL(url);
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
throw new Error(`mcp-integration-harness: refusing to run against "${url}" — not a URL. ` +
|
|
37
|
+
'The backend URL must come from the throwaway compose stack.');
|
|
38
|
+
}
|
|
39
|
+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
40
|
+
// `new URL('localhost:3000')` succeeds: protocol `localhost:`, hostname
|
|
41
|
+
// empty. Without this branch the next check rejects it for having no host,
|
|
42
|
+
// and the message reads "refusing to talk to " with a hole where the
|
|
43
|
+
// hostname should be — true, but no help at all to whoever forgot `http://`.
|
|
44
|
+
throw new Error(`mcp-integration-harness: refusing to run against "${url}" — not an ` +
|
|
45
|
+
'http(s) URL. A scheme-less "localhost:3000" parses as a URL whose ' +
|
|
46
|
+
'protocol is "localhost:", which is not the same thing as a backend.');
|
|
47
|
+
}
|
|
48
|
+
assertLoopbackHost(parsed.hostname);
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=loopback.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loopback.js","sourceRoot":"","sources":["../src/loopback.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,UAAU,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,gDAAgD,IAAI,qBAAqB;YACvE,wEAAwE;YACxE,uEAAuE;YACvE,+BAA+B,CAClC,CAAC;IACJ,CAAC;AACH,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,cAAc,CAAC,GAAW;IACxC,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,qDAAqD,GAAG,iBAAiB;YACvE,6DAA6D,CAChE,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAChE,wEAAwE;QACxE,2EAA2E;QAC3E,qEAAqE;QACrE,6EAA6E;QAC7E,MAAM,IAAI,KAAK,CACb,qDAAqD,GAAG,aAAa;YACnE,oEAAoE;YACpE,qEAAqE,CACxE,CAAC;IACJ,CAAC;IACD,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC"}
|
package/dist/wait.d.ts
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export interface WaitOptions {
|
|
2
|
+
/** How long to keep trying. Default 120 s — a cold Postgres is slow. */
|
|
3
|
+
timeoutSeconds?: number;
|
|
4
|
+
/** Between attempts. Default 1 s. */
|
|
5
|
+
intervalMs?: number;
|
|
6
|
+
/** What counts as ready. Default: any response at all. */
|
|
7
|
+
ready?: (response: Response) => boolean;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Waits for a backend to answer, or explains why it never did.
|
|
11
|
+
*
|
|
12
|
+
* `docker compose up --wait` covers the services that declare a healthcheck,
|
|
13
|
+
* and most upstream images do not. The alternative people reach for is a fixed
|
|
14
|
+
* `sleep 30`, which is both too long on a warm machine and too short on a cold
|
|
15
|
+
* one — and when it is too short the failure surfaces as the first tool call
|
|
16
|
+
* returning ECONNREFUSED, which reads like a bug in the tool.
|
|
17
|
+
*
|
|
18
|
+
* The last error is kept and thrown, because "timed out" on its own does not
|
|
19
|
+
* distinguish "not listening yet" from "listening and answering 500".
|
|
20
|
+
*
|
|
21
|
+
* For a backend that does not speak HTTP, use {@link waitForTcp}. `fetch`
|
|
22
|
+
* against an IMAP or SMTP port does not resolve — the greeting is not an HTTP
|
|
23
|
+
* response, so it rejects, and this would report a timeout for a server that
|
|
24
|
+
* came up immediately.
|
|
25
|
+
*/
|
|
26
|
+
export declare function waitForHttp(url: string, options?: WaitOptions): Promise<void>;
|
|
27
|
+
export interface TcpWaitOptions {
|
|
28
|
+
timeoutSeconds?: number;
|
|
29
|
+
intervalMs?: number;
|
|
30
|
+
/**
|
|
31
|
+
* Text the server must send unprompted, if it greets.
|
|
32
|
+
*
|
|
33
|
+
* IMAP answers `* OK`, SMTP answers `220`. Checking the greeting rather than
|
|
34
|
+
* only the connection is what tells "the port is open" from "the service
|
|
35
|
+
* behind it has finished starting" — Docker publishes the port before the
|
|
36
|
+
* process inside is listening on it, so a bare connect can succeed against
|
|
37
|
+
* nothing.
|
|
38
|
+
*/
|
|
39
|
+
expect?: string;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Waits for a plain TCP service — IMAP, SMTP, anything not HTTP.
|
|
43
|
+
*
|
|
44
|
+
* Several backends in this family do not speak HTTP at all, and `fetch`
|
|
45
|
+
* against them rejects rather than answering: undici cannot parse an IMAP
|
|
46
|
+
* greeting as an HTTP response, so {@link waitForHttp} reports a timeout for a
|
|
47
|
+
* server that was ready in a second.
|
|
48
|
+
*/
|
|
49
|
+
export declare function waitForTcp(host: string, port: number, options?: TcpWaitOptions): Promise<void>;
|
package/dist/wait.js
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { createConnection } from 'node:net';
|
|
2
|
+
import { assertLoopback, assertLoopbackHost } from './loopback.js';
|
|
3
|
+
/**
|
|
4
|
+
* Waits for a backend to answer, or explains why it never did.
|
|
5
|
+
*
|
|
6
|
+
* `docker compose up --wait` covers the services that declare a healthcheck,
|
|
7
|
+
* and most upstream images do not. The alternative people reach for is a fixed
|
|
8
|
+
* `sleep 30`, which is both too long on a warm machine and too short on a cold
|
|
9
|
+
* one — and when it is too short the failure surfaces as the first tool call
|
|
10
|
+
* returning ECONNREFUSED, which reads like a bug in the tool.
|
|
11
|
+
*
|
|
12
|
+
* The last error is kept and thrown, because "timed out" on its own does not
|
|
13
|
+
* distinguish "not listening yet" from "listening and answering 500".
|
|
14
|
+
*
|
|
15
|
+
* For a backend that does not speak HTTP, use {@link waitForTcp}. `fetch`
|
|
16
|
+
* against an IMAP or SMTP port does not resolve — the greeting is not an HTTP
|
|
17
|
+
* response, so it rejects, and this would report a timeout for a server that
|
|
18
|
+
* came up immediately.
|
|
19
|
+
*/
|
|
20
|
+
export async function waitForHttp(url, options = {}) {
|
|
21
|
+
assertLoopback(url);
|
|
22
|
+
const timeoutSeconds = options.timeoutSeconds ?? 120;
|
|
23
|
+
const deadline = Date.now() + timeoutSeconds * 1000;
|
|
24
|
+
const interval = options.intervalMs ?? 1000;
|
|
25
|
+
const ready = options.ready ?? (() => true);
|
|
26
|
+
let last = 'no attempt completed';
|
|
27
|
+
for (;;) {
|
|
28
|
+
try {
|
|
29
|
+
const response = await fetch(url, {
|
|
30
|
+
signal: AbortSignal.timeout(5000),
|
|
31
|
+
redirect: 'manual',
|
|
32
|
+
});
|
|
33
|
+
if (ready(response))
|
|
34
|
+
return;
|
|
35
|
+
last = `HTTP ${response.status}`;
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
// `String(error)` rather than `error.message`: the message alone is often
|
|
39
|
+
// just "fetch failed", and the constructor name is the half that says
|
|
40
|
+
// whether nothing was listening or the request timed out.
|
|
41
|
+
last = String(error);
|
|
42
|
+
}
|
|
43
|
+
if (Date.now() >= deadline) {
|
|
44
|
+
throw new Error(`mcp-integration-harness: ${url} did not become ready within ` +
|
|
45
|
+
`${timeoutSeconds}s. Last attempt: ${last}. ` +
|
|
46
|
+
'Is the compose stack up? `docker compose logs` usually says why.');
|
|
47
|
+
}
|
|
48
|
+
await new Promise((resolve) => setTimeout(resolve, interval));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Waits for a plain TCP service — IMAP, SMTP, anything not HTTP.
|
|
53
|
+
*
|
|
54
|
+
* Several backends in this family do not speak HTTP at all, and `fetch`
|
|
55
|
+
* against them rejects rather than answering: undici cannot parse an IMAP
|
|
56
|
+
* greeting as an HTTP response, so {@link waitForHttp} reports a timeout for a
|
|
57
|
+
* server that was ready in a second.
|
|
58
|
+
*/
|
|
59
|
+
export async function waitForTcp(host, port, options = {}) {
|
|
60
|
+
assertLoopbackHost(host);
|
|
61
|
+
const timeoutSeconds = options.timeoutSeconds ?? 120;
|
|
62
|
+
const deadline = Date.now() + timeoutSeconds * 1000;
|
|
63
|
+
const interval = options.intervalMs ?? 1000;
|
|
64
|
+
let last = 'no attempt completed';
|
|
65
|
+
for (;;) {
|
|
66
|
+
try {
|
|
67
|
+
await attempt(host, port, options.expect);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
last = String(error);
|
|
72
|
+
}
|
|
73
|
+
if (Date.now() >= deadline) {
|
|
74
|
+
throw new Error(`mcp-integration-harness: ${host}:${port} did not become ready within ` +
|
|
75
|
+
`${timeoutSeconds}s. Last attempt: ${last}. ` +
|
|
76
|
+
'Is the compose stack up? `docker compose logs` usually says why. ' +
|
|
77
|
+
'A service that binds 127.0.0.1 *inside* its container publishes a ' +
|
|
78
|
+
'port that reaches nothing — check what its log says it bound to.');
|
|
79
|
+
}
|
|
80
|
+
await new Promise((resolve) => setTimeout(resolve, interval));
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
function attempt(host, port, expect) {
|
|
84
|
+
return new Promise((resolve, reject) => {
|
|
85
|
+
const socket = createConnection({ host, port });
|
|
86
|
+
const done = (error) => {
|
|
87
|
+
socket.removeAllListeners();
|
|
88
|
+
socket.destroy();
|
|
89
|
+
if (error)
|
|
90
|
+
reject(error);
|
|
91
|
+
else
|
|
92
|
+
resolve();
|
|
93
|
+
};
|
|
94
|
+
socket.setTimeout(5000, () => done(new Error('timed out')));
|
|
95
|
+
socket.on('error', done);
|
|
96
|
+
if (expect === undefined) {
|
|
97
|
+
socket.on('connect', () => done());
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
let greeting = '';
|
|
101
|
+
socket.on('data', (chunk) => {
|
|
102
|
+
greeting += chunk.toString('utf8');
|
|
103
|
+
if (greeting.includes(expect))
|
|
104
|
+
done();
|
|
105
|
+
else if (greeting.length > 4096) {
|
|
106
|
+
done(new Error(`greeting did not contain ${expect}`));
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
//# sourceMappingURL=wait.js.map
|
package/dist/wait.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wait.js","sourceRoot":"","sources":["../src/wait.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAE5C,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAWnE;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAW,EACX,OAAO,GAAgB,EAAE;IAEzB,cAAc,CAAC,GAAG,CAAC,CAAC;IACpB,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,GAAG,CAAC;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,IAAI,CAAC;IACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;IAC5C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,IAAI,GAAG,sBAAsB,CAAC;IAElC,SAAS,CAAC;QACR,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;gBACjC,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YACH,IAAI,KAAK,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAC5B,IAAI,GAAG,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,0EAA0E;YAC1E,sEAAsE;YACtE,0DAA0D;YAC1D,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACb,4BAA4B,GAAG,+BAA+B;gBAC5D,GAAG,cAAc,oBAAoB,IAAI,IAAI;gBAC7C,kEAAkE,CACrE,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAiBD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,IAAY,EACZ,OAAO,GAAmB,EAAE;IAE5B,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACzB,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,GAAG,CAAC;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,IAAI,CAAC;IACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;IAC5C,IAAI,IAAI,GAAG,sBAAsB,CAAC;IAElC,SAAS,CAAC;QACR,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAC1C,OAAO;QACT,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACb,4BAA4B,IAAI,IAAI,IAAI,+BAA+B;gBACrE,GAAG,cAAc,oBAAoB,IAAI,IAAI;gBAC7C,mEAAmE;gBACnE,oEAAoE;gBACpE,kEAAkE,CACrE,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CACd,IAAY,EACZ,IAAY,EACZ,MAA0B;IAE1B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,gBAAgB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,CAAC,KAAa,EAAQ,EAAE;YACnC,MAAM,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,KAAK;gBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;gBACpB,OAAO,EAAE,CAAC;QACjB,CAAC,CAAC;QACF,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACzB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACnC,OAAO;QACT,CAAC;QACD,IAAI,QAAQ,GAAG,EAAE,CAAC;QAClB,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YAClC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACnC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,IAAI,EAAE,CAAC;iBACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,KAAK,CAAC,4BAA4B,MAAM,EAAE,CAAC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "mcp-integration-harness",
|
|
3
|
+
"version": "0.1.0-rc.1",
|
|
4
|
+
"description": "Drive an MCP server over real stdio against a real backend, and prove every tool was exercised",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"mcp",
|
|
7
|
+
"model-context-protocol",
|
|
8
|
+
"integration-testing",
|
|
9
|
+
"end-to-end",
|
|
10
|
+
"testing"
|
|
11
|
+
],
|
|
12
|
+
"license": "MIT",
|
|
13
|
+
"author": "Willi Thiel",
|
|
14
|
+
"repository": {
|
|
15
|
+
"type": "git",
|
|
16
|
+
"url": "git+https://github.com/ni-c/mcp-integration-harness.git"
|
|
17
|
+
},
|
|
18
|
+
"bugs": {
|
|
19
|
+
"url": "https://github.com/ni-c/mcp-integration-harness/issues"
|
|
20
|
+
},
|
|
21
|
+
"homepage": "https://github.com/ni-c/mcp-integration-harness#readme",
|
|
22
|
+
"funding": "https://github.com/sponsors/ni-c",
|
|
23
|
+
"type": "module",
|
|
24
|
+
"main": "dist/index.js",
|
|
25
|
+
"types": "dist/index.d.ts",
|
|
26
|
+
"exports": {
|
|
27
|
+
".": {
|
|
28
|
+
"types": "./dist/index.d.ts",
|
|
29
|
+
"default": "./dist/index.js"
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
"files": [
|
|
33
|
+
"dist"
|
|
34
|
+
],
|
|
35
|
+
"engines": {
|
|
36
|
+
"node": ">=22"
|
|
37
|
+
},
|
|
38
|
+
"scripts": {
|
|
39
|
+
"build": "tsc",
|
|
40
|
+
"watch": "tsc --watch",
|
|
41
|
+
"lint": "oxlint --deny-warnings && prettier --check .",
|
|
42
|
+
"format": "prettier --write .",
|
|
43
|
+
"test": "vitest run",
|
|
44
|
+
"test:coverage": "vitest run --coverage",
|
|
45
|
+
"prepublishOnly": "npm run build"
|
|
46
|
+
},
|
|
47
|
+
"peerDependencies": {
|
|
48
|
+
"@modelcontextprotocol/client": "^2.0.0"
|
|
49
|
+
},
|
|
50
|
+
"dependencies": {
|
|
51
|
+
"mcp-internal-hosts": "^0.2.0"
|
|
52
|
+
},
|
|
53
|
+
"devDependencies": {
|
|
54
|
+
"@modelcontextprotocol/client": "^2.0.0",
|
|
55
|
+
"@modelcontextprotocol/core": "^2.0.0",
|
|
56
|
+
"@modelcontextprotocol/server": "^2.0.0",
|
|
57
|
+
"@types/node": "^26.2.0",
|
|
58
|
+
"@vitest/coverage-v8": "^4.1.10",
|
|
59
|
+
"mcp-approval": "^0.7.0",
|
|
60
|
+
"oxlint": "^1.80.0",
|
|
61
|
+
"prettier": "^3.6.0",
|
|
62
|
+
"typescript": "^7.0.2",
|
|
63
|
+
"vitest": "^4.1.10",
|
|
64
|
+
"zod": "^4.4.3"
|
|
65
|
+
}
|
|
66
|
+
}
|