@acid-cat/cli 0.8.0 → 0.10.0

package/dist/conform/mcp-stdio.js

@@ -0,0 +1,332 @@
+import { isObject, memberOf } from "../json.js";
+import { Checks, abortable, expect, expectEqual, until } from "./check.js";
+import { META, checkEvents, checkTools, subscriptionOf } from "./mcp.js";
+import { checkProperties, clientInvoke, heldKeys } from "./properties.js";
+import { spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { Client } from "@modelcontextprotocol/client";
+import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
+//#region src/conform/mcp-stdio.ts
+/**
+* Conformance checks for the MCP binding through standard input and standard output (spec/bindings/mcp.md#stdio). Each
+* check starts the server as a process. The protocol checks use the official MCP client, once with revision 2026-07-28
+* and once with the `initialize` handshake of revision 2025-11-25. The other checks write lines that a client never
+* writes, or end standard input during a call.
+*/
+const RULE = "spec/bindings/mcp.md#stdio";
+/** The two ways that a client opens the session: revision 2026-07-28, or the handshake of revision 2025-11-25. */
+const ERAS = [{
+	revision: "2026-07-28",
+	connects: "an MCP client connects with revision 2026-07-28",
+	options: { versionNegotiation: { mode: { pin: "2026-07-28" } } }
+}, {
+	revision: "2025-11-25",
+	connects: "an MCP client connects with the initialize handshake of revision 2025-11-25",
+	options: {}
+}];
+/** The environment of the process: the environment of `conform`, without the variables that have no value. */
+function environment() {
+	return Object.fromEntries(Object.entries(process.env).flatMap(([name, value]) => value === void 0 ? [] : [[name, value]]));
+}
+/** Start the server and connect an MCP client. `changes` counts each `notifications/tools/list_changed`. */
+async function connect(command, options, changes, signal) {
+	const [program, ...args] = command;
+	if (program === void 0) throw new Error("the command that starts the MCP server is empty");
+	const client = new Client({
+		name: "acid-cat-conform",
+		version: "0.1.0"
+	}, {
+		...options,
+		listChanged: { tools: {
+			autoRefresh: false,
+			debounceMs: 0,
+			onChanged: () => void (changes.count += 1)
+		} }
+	});
+	const transport = new StdioClientTransport({
+		command: program,
+		args,
+		env: environment(),
+		stderr: "ignore"
+	});
+	await client.connect(transport, { signal });
+	return client;
+}
+async function checkMcpStdio(command, options) {
+	const all = [];
+	for (const era of ERAS) {
+		const checks = new Checks(`mcp-stdio ${era.revision}`, options.deadline);
+		all.push(checks);
+		const changes = { count: 0 };
+		let client;
+		await checks.run(era.connects, RULE, async (signal) => {
+			client = await connect(command, era.options, changes, signal);
+			expectEqual(client.getNegotiatedProtocolVersion(), era.revision, "the revision of the session");
+		});
+		if (client === void 0) continue;
+		const mcp = client;
+		try {
+			await checkTools(checks, mcp);
+			await checkChanges(checks, mcp, changes);
+			await checkCancellation(checks, mcp);
+			all.push(await checkProperties(`mcp-stdio ${era.revision} properties`, clientInvoke(mcp), options.deadline));
+		} finally {
+			await mcp.close();
+		}
+	}
+	all.push(await checkLines(command, options));
+	all.push(await checkStdioEvents(command, options));
+	return all;
+}
+/**
+* The checks of the events extension through one server process. The requests have revision 2026-07-28 in `_meta`, with
+* no handshake. `cancel` sends `notifications/cancelled` with the id of the `events/stream` request.
+*/
+async function checkStdioEvents(command, options) {
+	const checks = new Checks("mcp-stdio events", options.deadline);
+	const controller = new AbortController();
+	const server = startRaw(command, controller.signal);
+	try {
+		await checkEvents(checks, stdioChannel(server), {
+			transport: "stdio",
+			poll: options.poll,
+			webhook: false
+		});
+	} finally {
+		controller.abort();
+	}
+	return checks;
+}
+/** A channel to a server process. Each wait stops when the signal of the check aborts. */
+function stdioChannel(server) {
+	let next = 0;
+	const send = (id, method, params) => server.write(JSON.stringify({
+		jsonrpc: "2.0",
+		id,
+		method,
+		params: {
+			...params,
+			_meta: META
+		}
+	}));
+	return {
+		async request(method, params, signal) {
+			const id = `conform-request-${next += 1}`;
+			send(id, method, params);
+			return until(`the response to ${method}`, async () => server.messages.find((message) => memberOf(message, "id") === id), isObject, signal);
+		},
+		async stream(id, params, signal) {
+			send(id, "events/stream", params);
+			const belongs = (message) => isObject(message) && (message.id === id || subscriptionOf(message) === id);
+			let position = 0;
+			let ended = false;
+			return {
+				async next() {
+					if (ended) return void 0;
+					const found = await until("the next message of the subscription", async () => server.messages.slice(position).findIndex(belongs), (index) => index >= 0, signal);
+					const message = server.messages[position + found];
+					position += found + 1;
+					if (!belongs(message)) throw new Error("the message of the subscription is not a JSON-RPC message");
+					if (message.id === id) ended = true;
+					return message;
+				},
+				async cancel() {
+					if (ended) return;
+					ended = true;
+					server.write(JSON.stringify({
+						jsonrpc: "2.0",
+						method: "notifications/cancelled",
+						params: { requestId: id }
+					}));
+				}
+			};
+		}
+	};
+}
+/** The server tells the client each change of the set of available tools. */
+async function checkChanges(checks, mcp, changes) {
+	const set = (on, signal) => mcp.callTool({
+		name: "fixture.switch.set",
+		arguments: { on }
+	}, { signal });
+	const listsToggle = async (signal) => (await mcp.listTools(void 0, { signal })).tools.some((tool) => tool.name === "fixture.toggle");
+	await checks.run("notifications/tools/list_changed comes when a capability becomes unavailable or available", RULE, async (signal) => {
+		try {
+			const before = changes.count;
+			await set(false, signal);
+			await until("notifications/tools/list_changed comes", async () => changes.count, (count) => count > before, signal);
+			expect(!await listsToggle(signal), "fixture.toggle must not be listed while it is unavailable");
+			const off = changes.count;
+			await set(true, signal);
+			await until("notifications/tools/list_changed comes again", async () => changes.count, (count) => count > off, signal);
+			expect(await listsToggle(signal), "fixture.toggle must be listed when it is available again");
+		} finally {
+			await set(true, AbortSignal.timeout(checks.deadline));
+		}
+	});
+}
+/** `notifications/cancelled` cancels a call. The official client sends it when the signal of a call aborts. */
+async function checkCancellation(checks, mcp) {
+	const invoke = clientInvoke(mcp);
+	await checks.run("notifications/cancelled cancels the call", RULE, async (signal) => {
+		const key = randomUUID();
+		const controller = new AbortController();
+		const call = invoke("fixture.hold", { key }, AbortSignal.any([controller.signal, signal])).catch(() => void 0);
+		try {
+			await until("fixture.held gives the key of the call", () => heldKeys(invoke, signal), (held) => held.includes(key), signal);
+			controller.abort();
+			await call;
+			await until("the cancelled call stops holding", () => heldKeys(invoke, signal), (held) => !held.includes(key), signal);
+		} finally {
+			await invoke("fixture.release", { key }, AbortSignal.timeout(checks.deadline)).catch(() => void 0);
+		}
+	});
+}
+/**
+* A server process with its standard input and output, one JSON-RPC message on each line. `lines` holds each line of
+* standard output. When the signal aborts, the process is killed.
+*/
+function startRaw(command, signal) {
+	const [program, ...args] = command;
+	if (program === void 0) throw new Error("the command that starts the MCP server is empty");
+	const child = spawn(program, args, {
+		stdio: [
+			"pipe",
+			"pipe",
+			"ignore"
+		],
+		signal,
+		killSignal: "SIGKILL"
+	});
+	const lines = [];
+	const messages = [];
+	let rest = "";
+	child.stdout.on("data", (chunk) => {
+		const parts = (rest + chunk.toString()).split("\n");
+		rest = parts.pop() ?? "";
+		for (const line of parts) {
+			lines.push(line);
+			try {
+				messages.push(JSON.parse(line));
+			} catch {
+				messages.push(void 0);
+			}
+		}
+	});
+	child.stdin.on("error", () => void 0);
+	const exited = new Promise((resolve, reject) => {
+		child.on("error", (error) => reject(signal.aborted ? /* @__PURE__ */ new Error("waiting until the MCP server exits") : error));
+		child.on("close", (code, exitSignal) => resolve({
+			code,
+			signal: exitSignal
+		}));
+	});
+	exited.catch(() => void 0);
+	let next = 0;
+	const write = (line) => child.stdin.write(`${line}\n`);
+	/** Send a request and wait for its response. */
+	const request = async (method, params = {}) => {
+		const id = `conform-${next += 1}`;
+		write(JSON.stringify({
+			jsonrpc: "2.0",
+			id,
+			method,
+			params
+		}));
+		return until(`the response to ${method}`, async () => messages.find((message) => memberOf(message, "id") === id), isObject, signal);
+	};
+	return {
+		child,
+		lines,
+		messages,
+		write,
+		request,
+		exited
+	};
+}
+/** The `initialize` handshake of revision 2025-11-25, through a raw process. */
+async function handshake(server, revision = "2025-11-25") {
+	const response = await server.request("initialize", {
+		protocolVersion: revision,
+		capabilities: {},
+		clientInfo: {
+			name: "acid-cat-conform",
+			version: "0.1.0"
+		}
+	});
+	server.write(JSON.stringify({
+		jsonrpc: "2.0",
+		method: "notifications/initialized"
+	}));
+	return response;
+}
+async function checkLines(command, options) {
+	const checks = new Checks("mcp-stdio", options.deadline);
+	/** Run a body with a new server process, and kill the process at the end. */
+	const withServer = (body) => async (signal) => {
+		const controller = new AbortController();
+		const server = startRaw(command, AbortSignal.any([controller.signal, signal]));
+		try {
+			await body(server, signal);
+		} finally {
+			controller.abort();
+		}
+	};
+	await checks.run("initialize with a revision that the server does not serve gives a revision that it serves", RULE, withServer(async (server) => {
+		const response = await handshake(server, "1999-01-01");
+		const version = memberOf(memberOf(response, "result"), "protocolVersion");
+		expect(typeof version === "string" && version !== "1999-01-01", `expected another revision, got ${JSON.stringify(response)}`);
+	}));
+	await checks.run("a line that is not JSON gets the error -32700, and the session stays open", RULE, withServer(async (server, signal) => {
+		await handshake(server);
+		server.write("{");
+		const parseError = await until("the error -32700", async () => server.messages.find((message) => memberOf(memberOf(message, "error"), "code") === -32700), isObject, signal);
+		expectEqual(parseError["id"], null, "the id of the error");
+		const echo = await server.request("tools/call", {
+			name: "fixture.echo",
+			arguments: { text: "after" }
+		});
+		expectEqual(memberOf(memberOf(echo, "result"), "structuredContent"), { text: "after" }, "structuredContent after the error");
+	}));
+	await checks.run("standard output has only JSON-RPC messages, also after a defect", RULE, withServer(async (server) => {
+		await handshake(server);
+		await server.request("tools/list");
+		await server.request("tools/call", {
+			name: "fixture.crash",
+			arguments: {}
+		});
+		await server.request("tools/call", {
+			name: "fixture.echo",
+			arguments: { text: "last" }
+		});
+		const other = server.messages.findIndex((message) => memberOf(message, "jsonrpc") !== "2.0");
+		expect(other === -1, `standard output must have only JSON-RPC messages, got the line ${JSON.stringify(server.lines[other])}`);
+	}));
+	await checks.run("the server exits when standard input ends, also during a call", RULE, withServer(async (server, signal) => {
+		await handshake(server);
+		const key = randomUUID();
+		server.write(JSON.stringify({
+			jsonrpc: "2.0",
+			id: "conform-hold",
+			method: "tools/call",
+			params: {
+				name: "fixture.hold",
+				arguments: { key }
+			}
+		}));
+		const held = async () => {
+			const response = await server.request("tools/call", {
+				name: "fixture.held",
+				arguments: {}
+			});
+			return memberOf(memberOf(memberOf(response, "result"), "structuredContent"), "keys");
+		};
+		await until("fixture.held gives the key of the call", held, (keys) => Array.isArray(keys) && keys.includes(key), signal);
+		server.child.stdin.end();
+		const exit = await abortable(server.exited, signal);
+		expect(exit.signal === null, `the process must exit, not end by the signal ${exit.signal}`);
+	}));
+	return checks;
+}
+//#endregion
+export { checkMcpStdio };

package/dist/conform/mcp.d.ts

@@ -1,12 +1,73 @@
-/**
- * Conformance checks for the MCP binding (spec/bindings/mcp.md). The protocol checks use the official MCP client,
- * with revision 2026-07-28. The rejection checks send requests that a client never sends.
- */
 import { Client } from "@modelcontextprotocol/client";
+import type { JsonObject } from "../json.js";
 import { Checks } from "./check.js";
-/** A connected MCP client. */
-export declare function mcpClient(url: string, signal?: AbortSignal): Promise<Client>;
-export declare function checkMcp(url: string, options: {
+import type { PollSettings } from "./poll.js";
+import type { WebhookSettings } from "./webhook.js";
+/** A connected MCP client. With `handshake`, the client opens the session with `initialize`. */
+export declare function mcpClient(url: string, signal?: AbortSignal, handshake?: boolean): Promise<Client>;
+export interface McpCheckOptions {
     readonly maximumBytes: number;
     readonly deadline: number;
-}): Promise<Checks>;
+    readonly handshake: boolean;
+    /** The poll settings of the deployment, for the checks of the buffer and of the lease time. */
+    readonly poll: PollSettings;
+    /** The settings of the checks of webhook delivery. With none, conform does not test webhook delivery. */
+    readonly webhook: WebhookSettings | undefined;
+}
+export declare function checkMcp(url: string, options: McpCheckOptions): Promise<Checks>;
+/**
+ * The checks of the tools and their outcomes, through a connected client (spec/bindings/mcp.md#tools). Each transport
+ * runs them.
+ */
+export declare function checkTools(checks: Checks, mcp: Client): Promise<void>;
+/** The revision of the protocol that serves the events extension. */
+export declare const EVENTS_REVISION = "2026-07-28";
+/** The `_meta` of each request of revision 2026-07-28: the revision, and the capabilities of the client, which has none. */
+export declare const META: {
+    "io.modelcontextprotocol/protocolVersion": string;
+    "io.modelcontextprotocol/clientCapabilities": {};
+};
+/** A JSON-RPC message from the server. */
+export type Message = JsonObject;
+/**
+ * The messages of one `events/stream` request: the notifications of the subscription and its response. `next` gives
+ * the next message, or `undefined` at the end. `cancel` cancels the subscription, as the transport does.
+ */
+export interface EventStream {
+    next(): Promise<Message | undefined>;
+    cancel(): Promise<void>;
+}
+/** A way to send requests of revision 2026-07-28 to a server: through HTTP, or through standard input and output. */
+export interface Channel {
+    request(method: string, params: JsonObject, signal: AbortSignal): Promise<Message>;
+    /** Start `events/stream` with this request id. */
+    stream(id: string, params: JsonObject, signal: AbortSignal): Promise<EventStream>;
+}
+/** A channel to a server through HTTP. Each request has the `headers` too, for example the credentials of a principal. */
+export declare function httpChannel(url: string, headers?: Readonly<Record<string, string>>): Channel;
+/** The subscription id in the `_meta` of the params of a notification, or of a result. */
+export declare const subscriptionOf: (message: Message) => unknown;
+/** How a server serves the events extension, as `checkEvents` tests it. */
+export interface EventsOptions {
+    /** The transport of the channel. Through stdio, the server never serves webhook delivery. */
+    readonly transport: "http" | "stdio";
+    /** The poll settings of the deployment. */
+    readonly poll: PollSettings;
+    /** `true` if conform tests webhook delivery through this server, so the server must serve it. */
+    readonly webhook: boolean;
+}
+/** The requests that the checks of events send through a channel, and the capabilities of the fixture that they call. */
+export declare function eventRequests(channel: Channel): {
+    resultOf: (method: string, params: JsonObject, signal: AbortSignal) => Promise<JsonObject>;
+    errorOf: (message: Message | undefined) => JsonObject;
+    tool: (name: string, args: JsonObject, signal: AbortSignal) => Promise<unknown>;
+    watching: (signal: AbortSignal) => Promise<readonly string[]>;
+    signalKey: (key: string, value: string, signal: AbortSignal) => Promise<unknown>;
+};
+/** The number of times that a key is in a list of keys. */
+export declare const count: (keys: readonly string[], key: string) => number;
+/**
+ * The checks of the events extension (spec/bindings/mcp.md#events), through a channel. Each transport runs them. Through
+ * stdio, `cancel` sends `notifications/cancelled`; through HTTP, it closes the request.
+ */
+export declare function checkEvents(checks: Checks, channel: Channel, options: EventsOptions): Promise<void>;