@acid-cat/cli 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/conform/check.d.ts +1 -0
- package/dist/conform/check.js +0 -4
- package/dist/conform/index.d.ts +5 -0
- package/dist/conform/index.js +2 -0
- package/dist/conform/mcp-stdio.d.ts +4 -0
- package/dist/conform/mcp-stdio.js +266 -0
- package/dist/conform/mcp.d.ts +7 -2
- package/dist/conform/mcp.js +59 -52
- package/dist/conform/properties.d.ts +5 -0
- package/dist/conform/properties.js +28 -24
- package/dist/json/package.json.js +1 -1
- package/dist/main.js +5 -3
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@ The AcidCat command line. npm package: `@acid-cat/cli`. It gives the `acid-cat`
|
|
|
6
6
|
acid-cat check [--strict] <catalog.json ...> # check catalogs
|
|
7
7
|
acid-cat format [--write] <catalog.json ...> # write catalogs in canonical form
|
|
8
8
|
acid-cat view <catalog.json> <target> # write the view of one target
|
|
9
|
-
acid-cat conform --mcp <url> --http <url> --cli "<command>" --webmcp <url> [--deadline <ms>] # test an implementation
|
|
9
|
+
acid-cat conform --mcp <url> --mcp-stdio "<command>" --http <url> --cli "<command>" --webmcp <url> [--deadline <ms>] # test an implementation
|
|
10
10
|
```
|
|
11
11
|
|
|
12
12
|
Install it as a development dependency, and run it with `pnpm exec acid-cat …`:
|
|
@@ -34,10 +34,11 @@ The exit code is 0 if there are no errors. It is 1 if there are errors. With `--
|
|
|
34
34
|
|
|
35
35
|
`conform` tests a running implementation of the [conformance fixture](conformance/fixture.pkl) through its bindings (see [Conformance](../../spec/conformance.md#acid-cat-conform)):
|
|
36
36
|
|
|
37
|
-
- **MCP:**
|
|
37
|
+
- **MCP through HTTP:** `--mcp <url>`. The official MCP client, with revision `2026-07-28`, for tools and outcomes; raw requests for rejections.
|
|
38
|
+
- **MCP through stdio:** `--mcp-stdio "<command>"`, or `mcpStdio` in code. The checks start the server as a process. The official MCP client tests the tools and outcomes two times: with revision `2026-07-28`, and with the `initialize` handshake of revision `2025-11-25`. It also tests `notifications/tools/list_changed` and `notifications/cancelled`. Raw lines test the version negotiation, a line that is not JSON, a standard output with only JSON-RPC messages, and the end of standard input.
|
|
38
39
|
- **HTTP** and **CLI:** each outcome, each kind of call failure, availability, and rejections. For the CLI, also each kind of flag, including a repeated flag that gives one item of an array for each occurrence, or the whole array as JSON text, and the usage line of each command. An interrupt (`SIGINT`) during a call that holds gives exit code 130.
|
|
39
40
|
- **WebMCP:** the checks act as the WebMCP host. `--webmcp <url>` opens a page that serves the `app` target in Chromium, through [Playwright](https://playwright.dev), with a test `modelContext`. Install Playwright to use it: `pnpm add -D playwright`, then `pnpm exec playwright install chromium`. In code, `webmcp` can also be a function that installs the implementation on a host in the same process.
|
|
40
|
-
- **Properties:** with fast-check, through each
|
|
41
|
+
- **Properties:** with fast-check, through MCP with each transport and through HTTP: concurrent calls each get exactly their own outcome, and a cancellation changes only its own call.
|
|
41
42
|
|
|
42
43
|
The checks wait for states that they can observe, not for times. Before a check cancels a call to `fixture.hold`, it waits until `fixture.held` gives the key of the call, or, for the CLI, until the marker file of the call exists. Each check has one deadline: `--deadline`, or `deadline` in code, in milliseconds. The default is 30 000. A check that does not settle before its deadline fails with "did not settle", and the check stops what it started. This cleanup has a deadline of the same length, so `conform` always returns. The deadline never makes a check pass.
|
|
43
44
|
|
package/dist/conform/check.d.ts
CHANGED
|
@@ -35,6 +35,7 @@ export declare class Checks {
|
|
|
35
35
|
* Read a state until `done` is true for it, and give that value. The read and the pause between two reads stop when the
|
|
36
36
|
* signal of the check aborts.
|
|
37
37
|
*/
|
|
38
|
+
export declare function until<T, Done extends T>(what: string, read: () => Promise<T>, done: (value: T) => value is Done, signal: AbortSignal): Promise<Done>;
|
|
38
39
|
export declare function until<T>(what: string, read: () => Promise<T>, done: (value: T) => boolean, signal: AbortSignal): Promise<T>;
|
|
39
40
|
/** Settle as the promise does, or reject with the reason of the signal when it aborts first. */
|
|
40
41
|
export declare function abortable<T>(promise: Promise<T>, signal: AbortSignal): Promise<T>;
|
package/dist/conform/check.js
CHANGED
|
@@ -60,10 +60,6 @@ var Checks = class {
|
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
62
|
};
|
|
63
|
-
/**
|
|
64
|
-
* Read a state until `done` is true for it, and give that value. The read and the pause between two reads stop when the
|
|
65
|
-
* signal of the check aborts.
|
|
66
|
-
*/
|
|
67
63
|
async function until(what, read, done, signal) {
|
|
68
64
|
let last;
|
|
69
65
|
try {
|
package/dist/conform/index.d.ts
CHANGED
|
@@ -9,6 +9,11 @@ export type { TestHost, WebMcpInstall } from "./webmcp.js";
|
|
|
9
9
|
export interface ConformOptions {
|
|
10
10
|
/** The URL of the MCP server, for example `http://localhost:8787/mcp`. */
|
|
11
11
|
readonly mcp?: string | undefined;
|
|
12
|
+
/**
|
|
13
|
+
* The command that starts the MCP server through standard input and standard output, for example
|
|
14
|
+
* `["node", "fixture-mcp-stdio.ts"]`. Each check starts its own process.
|
|
15
|
+
*/
|
|
16
|
+
readonly mcpStdio?: readonly string[] | undefined;
|
|
12
17
|
/** The base URL of the HTTP API, before `/capabilities`. */
|
|
13
18
|
readonly http?: string | undefined;
|
|
14
19
|
/** The command that starts the command line, for example `["node", "fixture-cli.ts"]`. */
|
package/dist/conform/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import { checkCli } from "./cli.js";
|
|
|
3
3
|
import { checkHttp } from "./http.js";
|
|
4
4
|
import { checkMcp } from "./mcp.js";
|
|
5
5
|
import { checkProperties, httpInvoke, mcpInvoke } from "./properties.js";
|
|
6
|
+
import { checkMcpStdio } from "./mcp-stdio.js";
|
|
6
7
|
import { browserDriver, checkWebMcp, processDriver } from "./webmcp.js";
|
|
7
8
|
import { serve } from "./serve.js";
|
|
8
9
|
//#region src/conform/index.ts
|
|
@@ -26,6 +27,7 @@ async function conform(options) {
|
|
|
26
27
|
await mcp.close();
|
|
27
28
|
}
|
|
28
29
|
}
|
|
30
|
+
if (options.mcpStdio !== void 0) for (const checks of await checkMcpStdio(options.mcpStdio, { deadline })) results.push(...checks.results);
|
|
29
31
|
if (options.http !== void 0) {
|
|
30
32
|
results.push(...(await checkHttp(options.http, {
|
|
31
33
|
maximumBytes,
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { isObject, memberOf } from "../json.js";
|
|
2
|
+
import { Checks, abortable, expect, expectEqual, until } from "./check.js";
|
|
3
|
+
import { checkTools } from "./mcp.js";
|
|
4
|
+
import { checkProperties, clientInvoke, heldKeys } from "./properties.js";
|
|
5
|
+
import { spawn } from "node:child_process";
|
|
6
|
+
import { Client } from "@modelcontextprotocol/client";
|
|
7
|
+
import { randomUUID } from "node:crypto";
|
|
8
|
+
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
|
9
|
+
//#region src/conform/mcp-stdio.ts
|
|
10
|
+
/**
|
|
11
|
+
* Conformance checks for the MCP binding through standard input and standard output (spec/bindings/mcp.md#stdio). Each
|
|
12
|
+
* check starts the server as a process. The protocol checks use the official MCP client, once with revision 2026-07-28
|
|
13
|
+
* and once with the `initialize` handshake of revision 2025-11-25. The other checks write lines that a client never
|
|
14
|
+
* writes, or end standard input during a call.
|
|
15
|
+
*/
|
|
16
|
+
const RULE = "spec/bindings/mcp.md#stdio";
|
|
17
|
+
/** The two ways that a client opens the session: revision 2026-07-28, or the handshake of revision 2025-11-25. */
|
|
18
|
+
const ERAS = [{
|
|
19
|
+
revision: "2026-07-28",
|
|
20
|
+
connects: "an MCP client connects with revision 2026-07-28",
|
|
21
|
+
options: { versionNegotiation: { mode: { pin: "2026-07-28" } } }
|
|
22
|
+
}, {
|
|
23
|
+
revision: "2025-11-25",
|
|
24
|
+
connects: "an MCP client connects with the initialize handshake of revision 2025-11-25",
|
|
25
|
+
options: {}
|
|
26
|
+
}];
|
|
27
|
+
/** The environment of the process: the environment of `conform`, without the variables that have no value. */
|
|
28
|
+
function environment() {
|
|
29
|
+
return Object.fromEntries(Object.entries(process.env).flatMap(([name, value]) => value === void 0 ? [] : [[name, value]]));
|
|
30
|
+
}
|
|
31
|
+
/** Start the server and connect an MCP client. `changes` counts each `notifications/tools/list_changed`. */
|
|
32
|
+
async function connect(command, options, changes, signal) {
|
|
33
|
+
const [program, ...args] = command;
|
|
34
|
+
if (program === void 0) throw new Error("the command that starts the MCP server is empty");
|
|
35
|
+
const client = new Client({
|
|
36
|
+
name: "acid-cat-conform",
|
|
37
|
+
version: "0.1.0"
|
|
38
|
+
}, {
|
|
39
|
+
...options,
|
|
40
|
+
listChanged: { tools: {
|
|
41
|
+
autoRefresh: false,
|
|
42
|
+
debounceMs: 0,
|
|
43
|
+
onChanged: () => void (changes.count += 1)
|
|
44
|
+
} }
|
|
45
|
+
});
|
|
46
|
+
const transport = new StdioClientTransport({
|
|
47
|
+
command: program,
|
|
48
|
+
args,
|
|
49
|
+
env: environment(),
|
|
50
|
+
stderr: "ignore"
|
|
51
|
+
});
|
|
52
|
+
await client.connect(transport, { signal });
|
|
53
|
+
return client;
|
|
54
|
+
}
|
|
55
|
+
async function checkMcpStdio(command, options) {
|
|
56
|
+
const all = [];
|
|
57
|
+
for (const era of ERAS) {
|
|
58
|
+
const checks = new Checks(`mcp-stdio ${era.revision}`, options.deadline);
|
|
59
|
+
all.push(checks);
|
|
60
|
+
const changes = { count: 0 };
|
|
61
|
+
let client;
|
|
62
|
+
await checks.run(era.connects, RULE, async (signal) => {
|
|
63
|
+
client = await connect(command, era.options, changes, signal);
|
|
64
|
+
expectEqual(client.getNegotiatedProtocolVersion(), era.revision, "the revision of the session");
|
|
65
|
+
});
|
|
66
|
+
if (client === void 0) continue;
|
|
67
|
+
const mcp = client;
|
|
68
|
+
try {
|
|
69
|
+
await checkTools(checks, mcp);
|
|
70
|
+
await checkChanges(checks, mcp, changes);
|
|
71
|
+
await checkCancellation(checks, mcp);
|
|
72
|
+
all.push(await checkProperties(`mcp-stdio ${era.revision} properties`, clientInvoke(mcp), options.deadline));
|
|
73
|
+
} finally {
|
|
74
|
+
await mcp.close();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
all.push(await checkLines(command, options));
|
|
78
|
+
return all;
|
|
79
|
+
}
|
|
80
|
+
/** The server tells the client each change of the set of available tools. */
|
|
81
|
+
async function checkChanges(checks, mcp, changes) {
|
|
82
|
+
const set = (on, signal) => mcp.callTool({
|
|
83
|
+
name: "fixture.switch.set",
|
|
84
|
+
arguments: { on }
|
|
85
|
+
}, { signal });
|
|
86
|
+
const listsToggle = async (signal) => (await mcp.listTools(void 0, { signal })).tools.some((tool) => tool.name === "fixture.toggle");
|
|
87
|
+
await checks.run("notifications/tools/list_changed comes when a capability becomes unavailable or available", RULE, async (signal) => {
|
|
88
|
+
try {
|
|
89
|
+
const before = changes.count;
|
|
90
|
+
await set(false, signal);
|
|
91
|
+
await until("notifications/tools/list_changed comes", async () => changes.count, (count) => count > before, signal);
|
|
92
|
+
expect(!await listsToggle(signal), "fixture.toggle must not be listed while it is unavailable");
|
|
93
|
+
const off = changes.count;
|
|
94
|
+
await set(true, signal);
|
|
95
|
+
await until("notifications/tools/list_changed comes again", async () => changes.count, (count) => count > off, signal);
|
|
96
|
+
expect(await listsToggle(signal), "fixture.toggle must be listed when it is available again");
|
|
97
|
+
} finally {
|
|
98
|
+
await set(true, AbortSignal.timeout(checks.deadline));
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
/** `notifications/cancelled` cancels a call. The official client sends it when the signal of a call aborts. */
|
|
103
|
+
async function checkCancellation(checks, mcp) {
|
|
104
|
+
const invoke = clientInvoke(mcp);
|
|
105
|
+
await checks.run("notifications/cancelled cancels the call", RULE, async (signal) => {
|
|
106
|
+
const key = randomUUID();
|
|
107
|
+
const controller = new AbortController();
|
|
108
|
+
const call = invoke("fixture.hold", { key }, AbortSignal.any([controller.signal, signal])).catch(() => void 0);
|
|
109
|
+
try {
|
|
110
|
+
await until("fixture.held gives the key of the call", () => heldKeys(invoke, signal), (held) => held.includes(key), signal);
|
|
111
|
+
controller.abort();
|
|
112
|
+
await call;
|
|
113
|
+
await until("the cancelled call stops holding", () => heldKeys(invoke, signal), (held) => !held.includes(key), signal);
|
|
114
|
+
} finally {
|
|
115
|
+
await invoke("fixture.release", { key }, AbortSignal.timeout(checks.deadline)).catch(() => void 0);
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* A server process with its standard input and output, one JSON-RPC message on each line. `lines` holds each line of
|
|
121
|
+
* standard output. When the signal aborts, the process is killed.
|
|
122
|
+
*/
|
|
123
|
+
function startRaw(command, signal) {
|
|
124
|
+
const [program, ...args] = command;
|
|
125
|
+
if (program === void 0) throw new Error("the command that starts the MCP server is empty");
|
|
126
|
+
const child = spawn(program, args, {
|
|
127
|
+
stdio: [
|
|
128
|
+
"pipe",
|
|
129
|
+
"pipe",
|
|
130
|
+
"ignore"
|
|
131
|
+
],
|
|
132
|
+
signal,
|
|
133
|
+
killSignal: "SIGKILL"
|
|
134
|
+
});
|
|
135
|
+
const lines = [];
|
|
136
|
+
const messages = [];
|
|
137
|
+
let rest = "";
|
|
138
|
+
child.stdout.on("data", (chunk) => {
|
|
139
|
+
const parts = (rest + chunk.toString()).split("\n");
|
|
140
|
+
rest = parts.pop() ?? "";
|
|
141
|
+
for (const line of parts) {
|
|
142
|
+
lines.push(line);
|
|
143
|
+
try {
|
|
144
|
+
messages.push(JSON.parse(line));
|
|
145
|
+
} catch {
|
|
146
|
+
messages.push(void 0);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
child.stdin.on("error", () => void 0);
|
|
151
|
+
const exited = new Promise((resolve, reject) => {
|
|
152
|
+
child.on("error", (error) => reject(signal.aborted ? /* @__PURE__ */ new Error("waiting until the MCP server exits") : error));
|
|
153
|
+
child.on("close", (code, exitSignal) => resolve({
|
|
154
|
+
code,
|
|
155
|
+
signal: exitSignal
|
|
156
|
+
}));
|
|
157
|
+
});
|
|
158
|
+
exited.catch(() => void 0);
|
|
159
|
+
let next = 0;
|
|
160
|
+
const write = (line) => child.stdin.write(`${line}\n`);
|
|
161
|
+
/** Send a request and wait for its response. */
|
|
162
|
+
const request = async (method, params = {}) => {
|
|
163
|
+
const id = `conform-${next += 1}`;
|
|
164
|
+
write(JSON.stringify({
|
|
165
|
+
jsonrpc: "2.0",
|
|
166
|
+
id,
|
|
167
|
+
method,
|
|
168
|
+
params
|
|
169
|
+
}));
|
|
170
|
+
return until(`the response to ${method}`, async () => messages.find((message) => memberOf(message, "id") === id), isObject, signal);
|
|
171
|
+
};
|
|
172
|
+
return {
|
|
173
|
+
child,
|
|
174
|
+
lines,
|
|
175
|
+
messages,
|
|
176
|
+
write,
|
|
177
|
+
request,
|
|
178
|
+
exited
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
/** The `initialize` handshake of revision 2025-11-25, through a raw process. */
|
|
182
|
+
async function handshake(server, revision = "2025-11-25") {
|
|
183
|
+
const response = await server.request("initialize", {
|
|
184
|
+
protocolVersion: revision,
|
|
185
|
+
capabilities: {},
|
|
186
|
+
clientInfo: {
|
|
187
|
+
name: "acid-cat-conform",
|
|
188
|
+
version: "0.1.0"
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
server.write(JSON.stringify({
|
|
192
|
+
jsonrpc: "2.0",
|
|
193
|
+
method: "notifications/initialized"
|
|
194
|
+
}));
|
|
195
|
+
return response;
|
|
196
|
+
}
|
|
197
|
+
async function checkLines(command, options) {
|
|
198
|
+
const checks = new Checks("mcp-stdio", options.deadline);
|
|
199
|
+
/** Run a body with a new server process, and kill the process at the end. */
|
|
200
|
+
const withServer = (body) => async (signal) => {
|
|
201
|
+
const controller = new AbortController();
|
|
202
|
+
const server = startRaw(command, AbortSignal.any([controller.signal, signal]));
|
|
203
|
+
try {
|
|
204
|
+
await body(server, signal);
|
|
205
|
+
} finally {
|
|
206
|
+
controller.abort();
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
await checks.run("initialize with a revision that the server does not serve gives a revision that it serves", RULE, withServer(async (server) => {
|
|
210
|
+
const response = await handshake(server, "1999-01-01");
|
|
211
|
+
const version = memberOf(memberOf(response, "result"), "protocolVersion");
|
|
212
|
+
expect(typeof version === "string" && version !== "1999-01-01", `expected another revision, got ${JSON.stringify(response)}`);
|
|
213
|
+
}));
|
|
214
|
+
await checks.run("a line that is not JSON gets the error -32700, and the session stays open", RULE, withServer(async (server, signal) => {
|
|
215
|
+
await handshake(server);
|
|
216
|
+
server.write("{");
|
|
217
|
+
const parseError = await until("the error -32700", async () => server.messages.find((message) => memberOf(memberOf(message, "error"), "code") === -32700), isObject, signal);
|
|
218
|
+
expectEqual(parseError["id"], null, "the id of the error");
|
|
219
|
+
const echo = await server.request("tools/call", {
|
|
220
|
+
name: "fixture.echo",
|
|
221
|
+
arguments: { text: "after" }
|
|
222
|
+
});
|
|
223
|
+
expectEqual(memberOf(memberOf(echo, "result"), "structuredContent"), { text: "after" }, "structuredContent after the error");
|
|
224
|
+
}));
|
|
225
|
+
await checks.run("standard output has only JSON-RPC messages, also after a defect", RULE, withServer(async (server) => {
|
|
226
|
+
await handshake(server);
|
|
227
|
+
await server.request("tools/list");
|
|
228
|
+
await server.request("tools/call", {
|
|
229
|
+
name: "fixture.crash",
|
|
230
|
+
arguments: {}
|
|
231
|
+
});
|
|
232
|
+
await server.request("tools/call", {
|
|
233
|
+
name: "fixture.echo",
|
|
234
|
+
arguments: { text: "last" }
|
|
235
|
+
});
|
|
236
|
+
const other = server.messages.findIndex((message) => memberOf(message, "jsonrpc") !== "2.0");
|
|
237
|
+
expect(other === -1, `standard output must have only JSON-RPC messages, got the line ${JSON.stringify(server.lines[other])}`);
|
|
238
|
+
}));
|
|
239
|
+
await checks.run("the server exits when standard input ends, also during a call", RULE, withServer(async (server, signal) => {
|
|
240
|
+
await handshake(server);
|
|
241
|
+
const key = randomUUID();
|
|
242
|
+
server.write(JSON.stringify({
|
|
243
|
+
jsonrpc: "2.0",
|
|
244
|
+
id: "conform-hold",
|
|
245
|
+
method: "tools/call",
|
|
246
|
+
params: {
|
|
247
|
+
name: "fixture.hold",
|
|
248
|
+
arguments: { key }
|
|
249
|
+
}
|
|
250
|
+
}));
|
|
251
|
+
const held = async () => {
|
|
252
|
+
const response = await server.request("tools/call", {
|
|
253
|
+
name: "fixture.held",
|
|
254
|
+
arguments: {}
|
|
255
|
+
});
|
|
256
|
+
return memberOf(memberOf(memberOf(response, "result"), "structuredContent"), "keys");
|
|
257
|
+
};
|
|
258
|
+
await until("fixture.held gives the key of the call", held, (keys) => Array.isArray(keys) && keys.includes(key), signal);
|
|
259
|
+
server.child.stdin.end();
|
|
260
|
+
const exit = await abortable(server.exited, signal);
|
|
261
|
+
expect(exit.signal === null, `the process must exit, not end by the signal ${exit.signal}`);
|
|
262
|
+
}));
|
|
263
|
+
return checks;
|
|
264
|
+
}
|
|
265
|
+
//#endregion
|
|
266
|
+
export { checkMcpStdio };
|
package/dist/conform/mcp.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Conformance checks for the MCP binding (spec/bindings/mcp.md). The protocol checks use the official MCP
|
|
3
|
-
* with revision 2026-07-28. The rejection checks send requests that a client never sends.
|
|
2
|
+
* Conformance checks for the MCP binding through HTTP (spec/bindings/mcp.md). The protocol checks use the official MCP
|
|
3
|
+
* client, with revision 2026-07-28. The rejection checks send requests that a client never sends.
|
|
4
4
|
*/
|
|
5
5
|
import { Client } from "@modelcontextprotocol/client";
|
|
6
6
|
import { Checks } from "./check.js";
|
|
@@ -10,3 +10,8 @@ export declare function checkMcp(url: string, options: {
|
|
|
10
10
|
readonly maximumBytes: number;
|
|
11
11
|
readonly deadline: number;
|
|
12
12
|
}): Promise<Checks>;
|
|
13
|
+
/**
|
|
14
|
+
* The checks of the tools and their outcomes, through a connected client (spec/bindings/mcp.md#tools). Each transport
|
|
15
|
+
* runs them.
|
|
16
|
+
*/
|
|
17
|
+
export declare function checkTools(checks: Checks, mcp: Client): Promise<void>;
|
package/dist/conform/mcp.js
CHANGED
|
@@ -3,8 +3,8 @@ import { Checks, FORBIDDEN_ORIGIN, expect, expectEqual, fixture, idsOf, sorted }
|
|
|
3
3
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
4
4
|
//#region src/conform/mcp.ts
|
|
5
5
|
/**
|
|
6
|
-
* Conformance checks for the MCP binding (spec/bindings/mcp.md). The protocol checks use the official MCP
|
|
7
|
-
* with revision 2026-07-28. The rejection checks send requests that a client never sends.
|
|
6
|
+
* Conformance checks for the MCP binding through HTTP (spec/bindings/mcp.md). The protocol checks use the official MCP
|
|
7
|
+
* client, with revision 2026-07-28. The rejection checks send requests that a client never sends.
|
|
8
8
|
*/
|
|
9
9
|
const RULE = "spec/bindings/mcp.md";
|
|
10
10
|
/** A connected MCP client. */
|
|
@@ -24,16 +24,69 @@ const textOf = (result) => {
|
|
|
24
24
|
async function checkMcp(url, options) {
|
|
25
25
|
const checks = new Checks("mcp", options.deadline);
|
|
26
26
|
let client;
|
|
27
|
-
await checks.run("an MCP client connects with revision 2026-07-28", `${RULE}#
|
|
27
|
+
await checks.run("an MCP client connects with revision 2026-07-28", `${RULE}#http`, async (signal) => {
|
|
28
28
|
client = await mcpClient(url, signal);
|
|
29
29
|
});
|
|
30
30
|
if (client === void 0) return checks;
|
|
31
31
|
const mcp = client;
|
|
32
|
+
await checkTools(checks, mcp);
|
|
33
|
+
await mcp.close();
|
|
34
|
+
const post = (signal, { headers = {}, body }) => fetch(url, {
|
|
35
|
+
method: "POST",
|
|
36
|
+
body,
|
|
37
|
+
headers: {
|
|
38
|
+
"Content-Type": "application/json",
|
|
39
|
+
...headers
|
|
40
|
+
},
|
|
41
|
+
signal
|
|
42
|
+
});
|
|
43
|
+
const rejection = async (response, status, code) => {
|
|
44
|
+
expectEqual(response.status, status, "status");
|
|
45
|
+
expectEqual(response.headers.get("Cache-Control"), "no-store", "Cache-Control");
|
|
46
|
+
if (code !== void 0) expectEqual(memberOf(memberOf(await readJson(response), "error"), "code"), code, "JSON-RPC error code");
|
|
47
|
+
};
|
|
48
|
+
await checks.run("a method other than POST is rejected with 405", `${RULE}#http`, async (signal) => {
|
|
49
|
+
const response = await fetch(url, { signal });
|
|
50
|
+
await rejection(response, 405);
|
|
51
|
+
expectEqual(response.headers.get("Allow"), "POST", "Allow");
|
|
52
|
+
});
|
|
53
|
+
await checks.run("an origin that is not allowed is rejected with 403", `${RULE}#http`, async (signal) => {
|
|
54
|
+
await rejection(await post(signal, {
|
|
55
|
+
headers: { Origin: FORBIDDEN_ORIGIN },
|
|
56
|
+
body: "{}"
|
|
57
|
+
}), 403);
|
|
58
|
+
});
|
|
59
|
+
await checks.run("a media type other than JSON is rejected with 415", `${RULE}#http`, async (signal) => {
|
|
60
|
+
await rejection(await post(signal, {
|
|
61
|
+
headers: { "Content-Type": "text/plain" },
|
|
62
|
+
body: "{}"
|
|
63
|
+
}), 415);
|
|
64
|
+
});
|
|
65
|
+
await checks.run("a body over the limit is rejected with 413 and -32600", `${RULE}#http`, async (signal) => {
|
|
66
|
+
await rejection(await post(signal, { body: " ".repeat(options.maximumBytes + 1) }), 413, -32600);
|
|
67
|
+
});
|
|
68
|
+
await checks.run("a body that is not JSON is rejected with 400 and -32700", `${RULE}#http`, async (signal) => {
|
|
69
|
+
await rejection(await post(signal, { body: "{" }), 400, -32700);
|
|
70
|
+
});
|
|
71
|
+
await checks.run("a body that is not UTF-8 is rejected with 400", `${RULE}#http`, async (signal) => {
|
|
72
|
+
await rejection(await post(signal, { body: new Uint8Array([
|
|
73
|
+
123,
|
|
74
|
+
255,
|
|
75
|
+
125
|
|
76
|
+
]) }), 400);
|
|
77
|
+
});
|
|
78
|
+
return checks;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* The checks of the tools and their outcomes, through a connected client (spec/bindings/mcp.md#tools). Each transport
|
|
82
|
+
* runs them.
|
|
83
|
+
*/
|
|
84
|
+
async function checkTools(checks, mcp) {
|
|
32
85
|
const call = (signal, name, args) => mcp.callTool({
|
|
33
86
|
name,
|
|
34
87
|
arguments: args
|
|
35
88
|
}, { signal });
|
|
36
|
-
await checks.run("
|
|
89
|
+
await checks.run("the instructions are the description of the catalog", `${RULE}#server`, () => {
|
|
37
90
|
expectEqual(mcp.getInstructions(), fixture.description, "instructions");
|
|
38
91
|
});
|
|
39
92
|
await checks.run("tools/list gives each available capability of the view", `${RULE}#tools`, async (signal) => {
|
|
@@ -84,55 +137,9 @@ async function checkMcp(url, options) {
|
|
|
84
137
|
const { tools } = await mcp.listTools(void 0, { signal });
|
|
85
138
|
expect(!tools.some((tool) => tool.name === "fixture.toggle"), "fixture.toggle must not be listed while it is unavailable");
|
|
86
139
|
} finally {
|
|
87
|
-
await call(AbortSignal.timeout(
|
|
140
|
+
await call(AbortSignal.timeout(checks.deadline), "fixture.switch.set", { on: true });
|
|
88
141
|
}
|
|
89
142
|
});
|
|
90
|
-
await mcp.close();
|
|
91
|
-
const post = (signal, { headers = {}, body }) => fetch(url, {
|
|
92
|
-
method: "POST",
|
|
93
|
-
body,
|
|
94
|
-
headers: {
|
|
95
|
-
"Content-Type": "application/json",
|
|
96
|
-
...headers
|
|
97
|
-
},
|
|
98
|
-
signal
|
|
99
|
-
});
|
|
100
|
-
const rejection = async (response, status, code) => {
|
|
101
|
-
expectEqual(response.status, status, "status");
|
|
102
|
-
expectEqual(response.headers.get("Cache-Control"), "no-store", "Cache-Control");
|
|
103
|
-
if (code !== void 0) expectEqual(memberOf(memberOf(await readJson(response), "error"), "code"), code, "JSON-RPC error code");
|
|
104
|
-
};
|
|
105
|
-
await checks.run("a method other than POST is rejected with 405", `${RULE}#server`, async (signal) => {
|
|
106
|
-
const response = await fetch(url, { signal });
|
|
107
|
-
await rejection(response, 405);
|
|
108
|
-
expectEqual(response.headers.get("Allow"), "POST", "Allow");
|
|
109
|
-
});
|
|
110
|
-
await checks.run("an origin that is not allowed is rejected with 403", `${RULE}#server`, async (signal) => {
|
|
111
|
-
await rejection(await post(signal, {
|
|
112
|
-
headers: { Origin: FORBIDDEN_ORIGIN },
|
|
113
|
-
body: "{}"
|
|
114
|
-
}), 403);
|
|
115
|
-
});
|
|
116
|
-
await checks.run("a media type other than JSON is rejected with 415", `${RULE}#server`, async (signal) => {
|
|
117
|
-
await rejection(await post(signal, {
|
|
118
|
-
headers: { "Content-Type": "text/plain" },
|
|
119
|
-
body: "{}"
|
|
120
|
-
}), 415);
|
|
121
|
-
});
|
|
122
|
-
await checks.run("a body over the limit is rejected with 413 and -32600", `${RULE}#server`, async (signal) => {
|
|
123
|
-
await rejection(await post(signal, { body: " ".repeat(options.maximumBytes + 1) }), 413, -32600);
|
|
124
|
-
});
|
|
125
|
-
await checks.run("a body that is not JSON is rejected with 400 and -32700", `${RULE}#server`, async (signal) => {
|
|
126
|
-
await rejection(await post(signal, { body: "{" }), 400, -32700);
|
|
127
|
-
});
|
|
128
|
-
await checks.run("a body that is not UTF-8 is rejected with 400", `${RULE}#server`, async (signal) => {
|
|
129
|
-
await rejection(await post(signal, { body: new Uint8Array([
|
|
130
|
-
123,
|
|
131
|
-
255,
|
|
132
|
-
125
|
|
133
|
-
]) }), 400);
|
|
134
|
-
});
|
|
135
|
-
return checks;
|
|
136
143
|
}
|
|
137
144
|
//#endregion
|
|
138
|
-
export { checkMcp, mcpClient };
|
|
145
|
+
export { checkMcp, checkTools, mcpClient };
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Outcome } from "@acid-cat/spec-ts";
|
|
2
|
+
import type { Client } from "@modelcontextprotocol/client";
|
|
2
3
|
import type { JsonObject } from "../json.js";
|
|
3
4
|
import { Checks } from "./check.js";
|
|
4
5
|
/** A call through one binding. When the signal aborts, the client stops the call and rejects with the reason of the signal. */
|
|
@@ -8,4 +9,8 @@ export declare function mcpInvoke(url: string, signal: AbortSignal): Promise<{
|
|
|
8
9
|
readonly invoke: Invoke;
|
|
9
10
|
close(): Promise<void>;
|
|
10
11
|
}>;
|
|
12
|
+
/** Calls through a connected MCP client, with any transport. */
|
|
13
|
+
export declare function clientInvoke(client: Client): Invoke;
|
|
14
|
+
/** The keys of the calls that hold now, as fixture.held gives them through the binding. */
|
|
15
|
+
export declare function heldKeys(invoke: Invoke, signal: AbortSignal): Promise<readonly string[]>;
|
|
11
16
|
export declare function checkProperties(binding: string, invoke: Invoke, deadline: number): Promise<Checks>;
|
|
@@ -26,32 +26,36 @@ function httpInvoke(base) {
|
|
|
26
26
|
async function mcpInvoke(url, signal) {
|
|
27
27
|
const client = await mcpClient(url, signal);
|
|
28
28
|
return {
|
|
29
|
-
invoke:
|
|
30
|
-
try {
|
|
31
|
-
const result = await client.callTool({
|
|
32
|
-
name: id,
|
|
33
|
-
arguments: input
|
|
34
|
-
}, signal ? { signal } : void 0);
|
|
35
|
-
const first = result.content[0];
|
|
36
|
-
const body = parseJson(first?.type === "text" ? first.text : "null");
|
|
37
|
-
if (result.isError !== true) return {
|
|
38
|
-
ok: true,
|
|
39
|
-
value: result.structuredContent ?? body
|
|
40
|
-
};
|
|
41
|
-
if (!isOutcome(body)) throw new Error(`the error result of ${id} gave ${JSON.stringify(body)}, which is not an outcome`);
|
|
42
|
-
return body;
|
|
43
|
-
} catch (error) {
|
|
44
|
-
if (signal?.aborted && error instanceof SdkError && error.message === String(signal.reason)) throw signal.reason;
|
|
45
|
-
if (memberOf(error, "code") === -32602) return {
|
|
46
|
-
kind: "unknown-capability",
|
|
47
|
-
message: String(error)
|
|
48
|
-
};
|
|
49
|
-
throw error;
|
|
50
|
-
}
|
|
51
|
-
},
|
|
29
|
+
invoke: clientInvoke(client),
|
|
52
30
|
close: () => client.close()
|
|
53
31
|
};
|
|
54
32
|
}
|
|
33
|
+
/** Calls through a connected MCP client, with any transport. */
|
|
34
|
+
function clientInvoke(client) {
|
|
35
|
+
return async (id, input, signal) => {
|
|
36
|
+
try {
|
|
37
|
+
const result = await client.callTool({
|
|
38
|
+
name: id,
|
|
39
|
+
arguments: input
|
|
40
|
+
}, signal ? { signal } : void 0);
|
|
41
|
+
const first = result.content[0];
|
|
42
|
+
const body = parseJson(first?.type === "text" ? first.text : "null");
|
|
43
|
+
if (result.isError !== true) return {
|
|
44
|
+
ok: true,
|
|
45
|
+
value: result.structuredContent ?? body
|
|
46
|
+
};
|
|
47
|
+
if (!isOutcome(body)) throw new Error(`the error result of ${id} gave ${JSON.stringify(body)}, which is not an outcome`);
|
|
48
|
+
return body;
|
|
49
|
+
} catch (error) {
|
|
50
|
+
if (signal?.aborted && error instanceof SdkError && error.message === String(signal.reason)) throw signal.reason;
|
|
51
|
+
if (memberOf(error, "code") === -32602) return {
|
|
52
|
+
kind: "unknown-capability",
|
|
53
|
+
message: String(error)
|
|
54
|
+
};
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
}
|
|
55
59
|
const operation = fc.oneof(fc.string({ maxLength: 40 }).map((text) => ({
|
|
56
60
|
kind: "echo",
|
|
57
61
|
text
|
|
@@ -167,4 +171,4 @@ async function checkProperties(binding, invoke, deadline) {
|
|
|
167
171
|
return checks;
|
|
168
172
|
}
|
|
169
173
|
//#endregion
|
|
170
|
-
export { checkProperties, httpInvoke, mcpInvoke };
|
|
174
|
+
export { checkProperties, clientInvoke, heldKeys, httpInvoke, mcpInvoke };
|
package/dist/main.js
CHANGED
|
@@ -70,6 +70,7 @@ const viewCommand = Command.make("view", {
|
|
|
70
70
|
const url = (name, description) => Flag.String(name).pipe(Flag.withDescription(description), Flag.optional);
|
|
71
71
|
const conformCommand = Command.make("conform", {
|
|
72
72
|
mcp: url("mcp", "The URL of the MCP server"),
|
|
73
|
+
mcpStdio: url("mcp-stdio", "The command that starts the MCP server through standard input and output, for example \"node fixture-mcp-stdio.ts\""),
|
|
73
74
|
http: url("http", "The base URL of the HTTP API, before /capabilities"),
|
|
74
75
|
cli: url("cli", "The command that starts the command line, for example \"node fixture-cli.ts\""),
|
|
75
76
|
webmcp: url("webmcp", "The URL of a web page that serves the app target. It runs in Chromium, through Playwright"),
|
|
@@ -77,17 +78,18 @@ const conformCommand = Command.make("conform", {
|
|
|
77
78
|
deadline: Flag.Int("deadline").pipe(Flag.withDescription("The time, in milliseconds, that each check has to settle. A check that does not settle in this time fails"), Flag.withDefault(DEADLINE))
|
|
78
79
|
}, Effect.fn(function* (options) {
|
|
79
80
|
const value = (option) => Option.isSome(option) ? option.value : void 0;
|
|
80
|
-
const
|
|
81
|
+
const words = (option) => value(option)?.split(/\s+/).filter((word) => word !== "");
|
|
81
82
|
const results = yield* Effect.promise(() => conform({
|
|
82
83
|
mcp: value(options.mcp),
|
|
84
|
+
mcpStdio: words(options.mcpStdio),
|
|
83
85
|
http: value(options.http),
|
|
84
|
-
cli:
|
|
86
|
+
cli: words(options.cli),
|
|
85
87
|
webmcp: value(options.webmcp),
|
|
86
88
|
maximumBytes: options.maximumBytes,
|
|
87
89
|
deadline: options.deadline
|
|
88
90
|
}));
|
|
89
91
|
if (results.length === 0) {
|
|
90
|
-
yield* Console.error("Give at least one binding: --mcp, --http, --cli or --webmcp.");
|
|
92
|
+
yield* Console.error("Give at least one binding: --mcp, --mcp-stdio, --http, --cli or --webmcp.");
|
|
91
93
|
return yield* fail;
|
|
92
94
|
}
|
|
93
95
|
for (const result of results) yield* Console.log(`${result.passed ? "✓" : "✗"} ${result.binding}: ${result.name} (${result.rule})${result.detail ? `\n ${result.detail}` : ""}`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@acid-cat/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "The AcidCat command line: check, format and view catalogs, and test implementations.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"acidcat",
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
"access": "public"
|
|
38
38
|
},
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@acid-cat/catalog": "0.
|
|
41
|
-
"@acid-cat/spec-ts": "0.
|
|
40
|
+
"@acid-cat/catalog": "0.9.0",
|
|
41
|
+
"@acid-cat/spec-ts": "0.9.0",
|
|
42
42
|
"@effect/platform-node": "4.0.6",
|
|
43
43
|
"@effect/platform-node-shared": "4.0.6",
|
|
44
44
|
"@modelcontextprotocol/client": "2.3.1",
|