@acid-cat/cli 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/conformance/fixture.acidcat.json +63 -5
- package/dist/catalog.d.ts +5 -0
- package/dist/catalog.js +23 -4
- package/dist/conform/check.d.ts +24 -4
- package/dist/conform/check.js +57 -6
- package/dist/conform/cli.d.ts +3 -1
- package/dist/conform/cli.js +94 -53
- package/dist/conform/http.d.ts +1 -1
- package/dist/conform/http.js +44 -32
- package/dist/conform/index.d.ts +6 -5
- package/dist/conform/index.js +20 -9
- package/dist/conform/mcp.d.ts +2 -1
- package/dist/conform/mcp.js +57 -49
- package/dist/conform/properties.d.ts +5 -8
- package/dist/conform/properties.js +111 -39
- package/dist/conform/serve.js +45 -8
- package/dist/conform/webmcp.d.ts +23 -6
- package/dist/conform/webmcp.js +124 -55
- package/dist/json/fixture.acidcat.json.js +44 -4
- package/dist/json/package.json.js +1 -1
- package/dist/json.d.ts +22 -0
- package/dist/json.js +21 -0
- package/dist/main.js +13 -6
- package/package.json +6 -6
package/dist/conform/http.js
CHANGED
|
@@ -1,36 +1,45 @@
|
|
|
1
|
+
import { isArrayOf, isObject, memberOf, readJson } from "../json.js";
|
|
1
2
|
import { Checks, FORBIDDEN_ORIGIN, expect, expectEqual, idsOf, sorted } from "./check.js";
|
|
2
3
|
//#region src/conform/http.ts
|
|
3
4
|
/** Conformance checks for the HTTP binding (spec/bindings/http.md). */
|
|
4
5
|
const RULE = "spec/bindings/http.md";
|
|
6
|
+
const isEntry = (value) => isObject(value) && typeof value.id === "string";
|
|
7
|
+
/** The list that `GET /capabilities` gives. The check fails when it is not a list of entries. */
|
|
8
|
+
async function entriesOf(response) {
|
|
9
|
+
const list = await readJson(response);
|
|
10
|
+
if (!isArrayOf(list, isEntry)) throw new Error(`GET /capabilities must give a list of objects with an id, got ${JSON.stringify(list)}`);
|
|
11
|
+
return list;
|
|
12
|
+
}
|
|
5
13
|
async function checkHttp(base, options) {
|
|
6
|
-
const checks = new Checks("http");
|
|
7
|
-
const post = (id, body, headers = {}) => fetch(`${base}/capabilities/${id}`, {
|
|
14
|
+
const checks = new Checks("http", options.deadline);
|
|
15
|
+
const post = (signal, id, body, headers = {}) => fetch(`${base}/capabilities/${id}`, {
|
|
8
16
|
method: "POST",
|
|
9
17
|
headers: {
|
|
10
18
|
"Content-Type": "application/json",
|
|
11
19
|
...headers
|
|
12
20
|
},
|
|
13
|
-
body: typeof body === "string" ? body : body instanceof Uint8Array ? new Blob([body]) : JSON.stringify(body)
|
|
21
|
+
body: typeof body === "string" ? body : body instanceof Uint8Array ? new Blob([new Uint8Array(body)]) : JSON.stringify(body),
|
|
22
|
+
signal
|
|
14
23
|
});
|
|
15
|
-
const
|
|
16
|
-
await checks.run("GET /capabilities lists the view with availability", `${RULE}#requests`, async () => {
|
|
17
|
-
const list = await (await fetch(`${base}/capabilities
|
|
24
|
+
const kindOf = async (response) => memberOf(await readJson(response), "kind");
|
|
25
|
+
await checks.run("GET /capabilities lists the view with availability", `${RULE}#requests`, async (signal) => {
|
|
26
|
+
const list = await entriesOf(await fetch(`${base}/capabilities`, { signal }));
|
|
18
27
|
expectEqual(sorted(list.map((entry) => entry.id)), idsOf("api"), "ids");
|
|
19
28
|
expect(list.every((entry) => typeof entry.available === "boolean"), "each entry must have available");
|
|
20
29
|
});
|
|
21
|
-
await checks.run("a value has the status 200", `${RULE}#requests`, async () => {
|
|
22
|
-
const response = await post("fixture.echo", { text: "hello" });
|
|
30
|
+
await checks.run("a value has the status 200", `${RULE}#requests`, async (signal) => {
|
|
31
|
+
const response = await post(signal, "fixture.echo", { text: "hello" });
|
|
23
32
|
expectEqual(response.status, 200, "status");
|
|
24
|
-
expectEqual(await
|
|
33
|
+
expectEqual(await readJson(response), {
|
|
25
34
|
ok: true,
|
|
26
35
|
value: { text: "hello" }
|
|
27
36
|
}, "body");
|
|
28
37
|
});
|
|
29
|
-
await checks.run("a failure has the status 200 and ok: false", `${RULE}#requests`, async () => {
|
|
30
|
-
const response = await post("fixture.refuse", {});
|
|
31
|
-
const body = await
|
|
38
|
+
await checks.run("a failure has the status 200 and ok: false", `${RULE}#requests`, async (signal) => {
|
|
39
|
+
const response = await post(signal, "fixture.refuse", {});
|
|
40
|
+
const body = await readJson(response);
|
|
32
41
|
expectEqual(response.status, 200, "status");
|
|
33
|
-
expect(body
|
|
42
|
+
expect(memberOf(body, "ok") === false && memberOf(memberOf(body, "error"), "_tag") === "Refused", `got ${JSON.stringify(body)}`);
|
|
34
43
|
});
|
|
35
44
|
for (const [name, id, input, status, kind] of [
|
|
36
45
|
[
|
|
@@ -54,38 +63,41 @@ async function checkHttp(base, options) {
|
|
|
54
63
|
500,
|
|
55
64
|
"internal"
|
|
56
65
|
]
|
|
57
|
-
]) await checks.run(name, `${RULE}#requests`, async () => {
|
|
58
|
-
const response = await post(id, input);
|
|
66
|
+
]) await checks.run(name, `${RULE}#requests`, async (signal) => {
|
|
67
|
+
const response = await post(signal, id, input);
|
|
59
68
|
expectEqual(response.status, status, "status");
|
|
60
|
-
expectEqual(
|
|
69
|
+
expectEqual(await kindOf(response), kind, "kind");
|
|
61
70
|
});
|
|
62
|
-
await checks.run("an unavailable capability has the status 503", `${RULE}#requests`, async () => {
|
|
63
|
-
await post("fixture.switch.set", { on: false });
|
|
71
|
+
await checks.run("an unavailable capability has the status 503", `${RULE}#requests`, async (signal) => {
|
|
72
|
+
await post(signal, "fixture.switch.set", { on: false });
|
|
64
73
|
try {
|
|
65
|
-
const toggle = (await (await fetch(`${base}/capabilities
|
|
74
|
+
const toggle = (await entriesOf(await fetch(`${base}/capabilities`, { signal }))).find((entry) => entry.id === "fixture.toggle");
|
|
66
75
|
expect(toggle?.available === false && typeof toggle.reason === "string", "fixture.toggle must be unavailable, with a reason");
|
|
67
|
-
const response = await post("fixture.toggle", {});
|
|
76
|
+
const response = await post(signal, "fixture.toggle", {});
|
|
68
77
|
expectEqual(response.status, 503, "status");
|
|
69
|
-
expectEqual(
|
|
78
|
+
expectEqual(await kindOf(response), "unavailable", "kind");
|
|
70
79
|
} finally {
|
|
71
|
-
await post("fixture.switch.set", { on: true });
|
|
80
|
+
await post(AbortSignal.timeout(options.deadline), "fixture.switch.set", { on: true });
|
|
72
81
|
}
|
|
73
82
|
});
|
|
74
83
|
const status = async (response, expected) => expectEqual(response.status, expected, "status");
|
|
75
|
-
await checks.run("a method that is not accepted is rejected with 405", `${RULE}#requests`, async () => {
|
|
76
|
-
await status(await fetch(`${base}/capabilities`, {
|
|
84
|
+
await checks.run("a method that is not accepted is rejected with 405", `${RULE}#requests`, async (signal) => {
|
|
85
|
+
await status(await fetch(`${base}/capabilities`, {
|
|
86
|
+
method: "PUT",
|
|
87
|
+
signal
|
|
88
|
+
}), 405);
|
|
77
89
|
});
|
|
78
|
-
await checks.run("an origin that is not allowed is rejected with 403", `${RULE}#requests`, async () => {
|
|
79
|
-
await status(await post("fixture.echo", { text: "x" }, { Origin: FORBIDDEN_ORIGIN }), 403);
|
|
90
|
+
await checks.run("an origin that is not allowed is rejected with 403", `${RULE}#requests`, async (signal) => {
|
|
91
|
+
await status(await post(signal, "fixture.echo", { text: "x" }, { Origin: FORBIDDEN_ORIGIN }), 403);
|
|
80
92
|
});
|
|
81
|
-
await checks.run("a media type other than JSON is rejected with 415", `${RULE}#requests`, async () => {
|
|
82
|
-
await status(await post("fixture.echo", { text: "x" }, { "Content-Type": "text/plain" }), 415);
|
|
93
|
+
await checks.run("a media type other than JSON is rejected with 415", `${RULE}#requests`, async (signal) => {
|
|
94
|
+
await status(await post(signal, "fixture.echo", { text: "x" }, { "Content-Type": "text/plain" }), 415);
|
|
83
95
|
});
|
|
84
|
-
await checks.run("a body over the limit is rejected with 413", `${RULE}#requests`, async () => {
|
|
85
|
-
await status(await post("fixture.echo", " ".repeat(options.maximumBytes + 1)), 413);
|
|
96
|
+
await checks.run("a body over the limit is rejected with 413", `${RULE}#requests`, async (signal) => {
|
|
97
|
+
await status(await post(signal, "fixture.echo", " ".repeat(options.maximumBytes + 1)), 413);
|
|
86
98
|
});
|
|
87
|
-
await checks.run("a body that is not UTF-8 is rejected with 400", `${RULE}#requests`, async () => {
|
|
88
|
-
await status(await post("fixture.echo", new Uint8Array([
|
|
99
|
+
await checks.run("a body that is not UTF-8 is rejected with 400", `${RULE}#requests`, async (signal) => {
|
|
100
|
+
await status(await post(signal, "fixture.echo", new Uint8Array([
|
|
89
101
|
123,
|
|
90
102
|
255,
|
|
91
103
|
125
|
package/dist/conform/index.d.ts
CHANGED
|
@@ -1,11 +1,7 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `acid-cat conform`: test a running implementation through its bindings (spec/conformance.md). The implementation
|
|
3
|
-
* serves the conformance fixture (conformance/fixture.acidcat.json), with its switch on.
|
|
4
|
-
*/
|
|
5
1
|
import type { Check } from "./check.js";
|
|
6
2
|
import type { WebMcpInstall } from "./webmcp.js";
|
|
7
3
|
export type { Check } from "./check.js";
|
|
8
|
-
export { fixture } from "./check.js";
|
|
4
|
+
export { DEADLINE, fixture } from "./check.js";
|
|
9
5
|
export { serve } from "./serve.js";
|
|
10
6
|
export type { Served, WebHandler } from "./serve.js";
|
|
11
7
|
export { createTestHost } from "./webmcp.js";
|
|
@@ -24,6 +20,11 @@ export interface ConformOptions {
|
|
|
24
20
|
readonly webmcp?: string | WebMcpInstall | undefined;
|
|
25
21
|
/** The body limit of the remote bindings, in bytes. The default is 1 MiB. */
|
|
26
22
|
readonly maximumBytes?: number | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* The time, in milliseconds, that each check has to settle. A check that does not settle in this time fails. The
|
|
25
|
+
* default is `DEADLINE`, 30 seconds.
|
|
26
|
+
*/
|
|
27
|
+
readonly deadline?: number | undefined;
|
|
27
28
|
}
|
|
28
29
|
/** Run all checks for the bindings that the options give. */
|
|
29
30
|
export declare function conform(options: ConformOptions): Promise<readonly Check[]>;
|
package/dist/conform/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { fixture } from "./check.js";
|
|
1
|
+
import { DEADLINE, fixture } from "./check.js";
|
|
2
2
|
import { checkCli } from "./cli.js";
|
|
3
3
|
import { checkHttp } from "./http.js";
|
|
4
4
|
import { checkMcp } from "./mcp.js";
|
|
@@ -6,27 +6,38 @@ import { checkProperties, httpInvoke, mcpInvoke } from "./properties.js";
|
|
|
6
6
|
import { browserDriver, checkWebMcp, processDriver } from "./webmcp.js";
|
|
7
7
|
import { serve } from "./serve.js";
|
|
8
8
|
//#region src/conform/index.ts
|
|
9
|
+
/**
|
|
10
|
+
* `acid-cat conform`: test a running implementation through its bindings (spec/conformance.md). The implementation
|
|
11
|
+
* serves the conformance fixture (conformance/fixture.acidcat.json), with its switch on.
|
|
12
|
+
*/
|
|
9
13
|
/** Run all checks for the bindings that the options give. */
|
|
10
14
|
async function conform(options) {
|
|
11
15
|
const maximumBytes = options.maximumBytes ?? 1048576;
|
|
16
|
+
const deadline = options.deadline ?? 3e4;
|
|
12
17
|
const results = [];
|
|
13
18
|
if (options.mcp !== void 0) {
|
|
14
|
-
results.push(...(await checkMcp(options.mcp, {
|
|
15
|
-
|
|
19
|
+
results.push(...(await checkMcp(options.mcp, {
|
|
20
|
+
maximumBytes,
|
|
21
|
+
deadline
|
|
22
|
+
})).results);
|
|
23
|
+
const mcp = await mcpInvoke(options.mcp, AbortSignal.timeout(deadline)).catch(() => void 0);
|
|
16
24
|
if (mcp !== void 0) {
|
|
17
|
-
results.push(...(await checkProperties("mcp", mcp.invoke)).results);
|
|
25
|
+
results.push(...(await checkProperties("mcp", mcp.invoke, deadline)).results);
|
|
18
26
|
await mcp.close();
|
|
19
27
|
}
|
|
20
28
|
}
|
|
21
29
|
if (options.http !== void 0) {
|
|
22
|
-
results.push(...(await checkHttp(options.http, {
|
|
23
|
-
|
|
30
|
+
results.push(...(await checkHttp(options.http, {
|
|
31
|
+
maximumBytes,
|
|
32
|
+
deadline
|
|
33
|
+
})).results);
|
|
34
|
+
results.push(...(await checkProperties("http", httpInvoke(options.http), deadline)).results);
|
|
24
35
|
}
|
|
25
|
-
if (options.cli !== void 0) results.push(...(await checkCli(options.cli)).results);
|
|
36
|
+
if (options.cli !== void 0) results.push(...(await checkCli(options.cli, { deadline })).results);
|
|
26
37
|
if (options.webmcp !== void 0) {
|
|
27
38
|
const driver = typeof options.webmcp === "string" ? await browserDriver(options.webmcp) : await processDriver(options.webmcp);
|
|
28
39
|
try {
|
|
29
|
-
results.push(...(await checkWebMcp(driver)).results);
|
|
40
|
+
results.push(...(await checkWebMcp(driver, { deadline })).results);
|
|
30
41
|
} finally {
|
|
31
42
|
await driver.close();
|
|
32
43
|
}
|
|
@@ -34,4 +45,4 @@ async function conform(options) {
|
|
|
34
45
|
return results;
|
|
35
46
|
}
|
|
36
47
|
//#endregion
|
|
37
|
-
export { conform, fixture, serve };
|
|
48
|
+
export { DEADLINE, conform, fixture, serve };
|
package/dist/conform/mcp.d.ts
CHANGED
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
import { Client } from "@modelcontextprotocol/client";
|
|
6
6
|
import { Checks } from "./check.js";
|
|
7
7
|
/** A connected MCP client. */
|
|
8
|
-
export declare function mcpClient(url: string): Promise<Client>;
|
|
8
|
+
export declare function mcpClient(url: string, signal?: AbortSignal): Promise<Client>;
|
|
9
9
|
export declare function checkMcp(url: string, options: {
|
|
10
10
|
readonly maximumBytes: number;
|
|
11
|
+
readonly deadline: number;
|
|
11
12
|
}): Promise<Checks>;
|
package/dist/conform/mcp.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isObject, memberOf, parseJson, readJson } from "../json.js";
|
|
1
2
|
import { Checks, FORBIDDEN_ORIGIN, expect, expectEqual, fixture, idsOf, sorted } from "./check.js";
|
|
2
3
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
3
4
|
//#region src/conform/mcp.ts
|
|
@@ -7,118 +8,125 @@ import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/cli
|
|
|
7
8
|
*/
|
|
8
9
|
const RULE = "spec/bindings/mcp.md";
|
|
9
10
|
/** A connected MCP client. */
|
|
10
|
-
async function mcpClient(url) {
|
|
11
|
+
async function mcpClient(url, signal) {
|
|
11
12
|
const client = new Client({
|
|
12
13
|
name: "acid-cat-conform",
|
|
13
14
|
version: "0.1.0"
|
|
14
15
|
}, { versionNegotiation: { mode: { pin: "2026-07-28" } } });
|
|
15
|
-
await client.connect(new StreamableHTTPClientTransport(new URL(url)));
|
|
16
|
+
await client.connect(new StreamableHTTPClientTransport(new URL(url)), signal ? { signal } : void 0);
|
|
16
17
|
return client;
|
|
17
18
|
}
|
|
18
|
-
|
|
19
|
+
/** The JSON value in the first text content of a tool result, or `null` when there is no text content. */
|
|
20
|
+
const textOf = (result) => {
|
|
21
|
+
const first = result.content[0];
|
|
22
|
+
return parseJson(first?.type === "text" ? first.text : "null");
|
|
23
|
+
};
|
|
19
24
|
async function checkMcp(url, options) {
|
|
20
|
-
const checks = new Checks("mcp");
|
|
25
|
+
const checks = new Checks("mcp", options.deadline);
|
|
21
26
|
let client;
|
|
22
|
-
await checks.run("an MCP client connects with revision 2026-07-28", `${RULE}#server`, async () => {
|
|
23
|
-
client = await mcpClient(url);
|
|
27
|
+
await checks.run("an MCP client connects with revision 2026-07-28", `${RULE}#server`, async (signal) => {
|
|
28
|
+
client = await mcpClient(url, signal);
|
|
24
29
|
});
|
|
25
30
|
if (client === void 0) return checks;
|
|
26
31
|
const mcp = client;
|
|
27
|
-
const call = (name, args) => mcp.callTool({
|
|
32
|
+
const call = (signal, name, args) => mcp.callTool({
|
|
28
33
|
name,
|
|
29
34
|
arguments: args
|
|
30
|
-
});
|
|
35
|
+
}, { signal });
|
|
31
36
|
await checks.run("server/discover gives the description of the catalog as instructions", `${RULE}#server`, () => {
|
|
32
37
|
expectEqual(mcp.getInstructions(), fixture.description, "instructions");
|
|
33
38
|
});
|
|
34
|
-
await checks.run("tools/list gives each available capability of the view", `${RULE}#tools`, async () => {
|
|
35
|
-
const { tools } = await mcp.listTools();
|
|
39
|
+
await checks.run("tools/list gives each available capability of the view", `${RULE}#tools`, async (signal) => {
|
|
40
|
+
const { tools } = await mcp.listTools(void 0, { signal });
|
|
36
41
|
expectEqual(sorted(tools.map((tool) => tool.name)), idsOf("mcp"), "tools");
|
|
37
42
|
});
|
|
38
|
-
await checks.run("tools have their hints and output schemas", `${RULE}#tools`, async () => {
|
|
39
|
-
const { tools } = await mcp.listTools();
|
|
43
|
+
await checks.run("tools have their hints and output schemas", `${RULE}#tools`, async (signal) => {
|
|
44
|
+
const { tools } = await mcp.listTools(void 0, { signal });
|
|
40
45
|
const echo = tools.find((tool) => tool.name === "fixture.echo");
|
|
41
46
|
const refuse = tools.find((tool) => tool.name === "fixture.refuse");
|
|
42
47
|
expect(echo?.annotations?.readOnlyHint === true, "fixture.echo must have readOnlyHint: true");
|
|
43
48
|
expect(refuse?.annotations?.readOnlyHint !== true, "fixture.refuse must not have readOnlyHint: true");
|
|
44
|
-
expect(
|
|
49
|
+
expect(echo?.outputSchema?.type === "object", "fixture.echo must have an object outputSchema");
|
|
45
50
|
expect(refuse?.outputSchema === void 0, "fixture.refuse has no object output, so it must have no outputSchema");
|
|
46
|
-
expect(
|
|
51
|
+
expect(echo?.inputSchema.type === "object", "an inputSchema must be an object");
|
|
47
52
|
});
|
|
48
|
-
await checks.run("a value is a tool result with structured content", `${RULE}#tools`, async () => {
|
|
49
|
-
const result = await call("fixture.echo", { text: "hello" });
|
|
53
|
+
await checks.run("a value is a tool result with structured content", `${RULE}#tools`, async (signal) => {
|
|
54
|
+
const result = await call(signal, "fixture.echo", { text: "hello" });
|
|
50
55
|
expect(result.isError !== true, "a value must not be an error result");
|
|
51
56
|
expectEqual(result.structuredContent, { text: "hello" }, "structuredContent");
|
|
52
57
|
expectEqual(textOf(result), { text: "hello" }, "text content");
|
|
53
58
|
});
|
|
54
|
-
await checks.run("a failure is an error result with ok: false", `${RULE}#tools`, async () => {
|
|
55
|
-
const result = await call("fixture.refuse", {});
|
|
59
|
+
await checks.run("a failure is an error result with ok: false", `${RULE}#tools`, async (signal) => {
|
|
60
|
+
const result = await call(signal, "fixture.refuse", {});
|
|
56
61
|
expect(result.isError === true, "a failure must be an error result");
|
|
57
62
|
const body = textOf(result);
|
|
58
|
-
expect(body
|
|
63
|
+
expect(memberOf(body, "ok") === false && memberOf(memberOf(body, "error"), "_tag") === "Refused", `expected { ok: false, error: { _tag: "Refused" } }, got ${JSON.stringify(body)}`);
|
|
59
64
|
});
|
|
60
|
-
await checks.run("a defect is an internal call failure with no internal details", `${RULE}#tools`, async () => {
|
|
61
|
-
const result = await call("fixture.crash", {});
|
|
65
|
+
await checks.run("a defect is an internal call failure with no internal details", `${RULE}#tools`, async (signal) => {
|
|
66
|
+
const result = await call(signal, "fixture.crash", {});
|
|
62
67
|
const body = textOf(result);
|
|
68
|
+
expect(isObject(body), `expected an object, got ${JSON.stringify(body)}`);
|
|
63
69
|
expect(result.isError === true && body.kind === "internal" && !("ok" in body), `expected { kind: "internal" }, got ${JSON.stringify(body)}`);
|
|
64
|
-
|
|
70
|
+
const message = typeof body.message === "string" ? body.message : "";
|
|
71
|
+
expect(!/\n\s+at /.test(message), "the message must not contain a stack trace");
|
|
65
72
|
});
|
|
66
|
-
await checks.run("an input that does not agree with its schema is invalid-input", `${RULE}#tools`, async () => {
|
|
67
|
-
const result = await call("fixture.echo", { text: 1 });
|
|
68
|
-
expectEqual(textOf(result)
|
|
73
|
+
await checks.run("an input that does not agree with its schema is invalid-input", `${RULE}#tools`, async (signal) => {
|
|
74
|
+
const result = await call(signal, "fixture.echo", { text: 1 });
|
|
75
|
+
expectEqual(memberOf(textOf(result), "kind"), "invalid-input", "kind");
|
|
69
76
|
});
|
|
70
|
-
await checks.run("a capability outside the view is an unknown tool", `${RULE}#tools`, async () => {
|
|
71
|
-
const error = await call("fixture.hidden", {}).then(() => void 0, (caught) => caught);
|
|
72
|
-
expect(error
|
|
77
|
+
await checks.run("a capability outside the view is an unknown tool", `${RULE}#tools`, async (signal) => {
|
|
78
|
+
const error = await call(signal, "fixture.hidden", {}).then(() => void 0, (caught) => caught);
|
|
79
|
+
expect(memberOf(error, "code") === -32602, `expected the JSON-RPC error -32602, got ${JSON.stringify(error)}`);
|
|
73
80
|
});
|
|
74
|
-
await checks.run("an unavailable capability is not listed", `${RULE}#tools`, async () => {
|
|
75
|
-
await call("fixture.switch.set", { on: false });
|
|
81
|
+
await checks.run("an unavailable capability is not listed", `${RULE}#tools`, async (signal) => {
|
|
82
|
+
await call(signal, "fixture.switch.set", { on: false });
|
|
76
83
|
try {
|
|
77
|
-
const { tools } = await mcp.listTools();
|
|
84
|
+
const { tools } = await mcp.listTools(void 0, { signal });
|
|
78
85
|
expect(!tools.some((tool) => tool.name === "fixture.toggle"), "fixture.toggle must not be listed while it is unavailable");
|
|
79
86
|
} finally {
|
|
80
|
-
await call("fixture.switch.set", { on: true });
|
|
87
|
+
await call(AbortSignal.timeout(options.deadline), "fixture.switch.set", { on: true });
|
|
81
88
|
}
|
|
82
89
|
});
|
|
83
90
|
await mcp.close();
|
|
84
|
-
const post = (
|
|
91
|
+
const post = (signal, { headers = {}, body }) => fetch(url, {
|
|
85
92
|
method: "POST",
|
|
86
|
-
|
|
93
|
+
body,
|
|
87
94
|
headers: {
|
|
88
95
|
"Content-Type": "application/json",
|
|
89
|
-
...
|
|
90
|
-
}
|
|
96
|
+
...headers
|
|
97
|
+
},
|
|
98
|
+
signal
|
|
91
99
|
});
|
|
92
100
|
const rejection = async (response, status, code) => {
|
|
93
101
|
expectEqual(response.status, status, "status");
|
|
94
102
|
expectEqual(response.headers.get("Cache-Control"), "no-store", "Cache-Control");
|
|
95
|
-
if (code !== void 0) expectEqual((await response
|
|
103
|
+
if (code !== void 0) expectEqual(memberOf(memberOf(await readJson(response), "error"), "code"), code, "JSON-RPC error code");
|
|
96
104
|
};
|
|
97
|
-
await checks.run("a method other than POST is rejected with 405", `${RULE}#server`, async () => {
|
|
98
|
-
const response = await fetch(url);
|
|
105
|
+
await checks.run("a method other than POST is rejected with 405", `${RULE}#server`, async (signal) => {
|
|
106
|
+
const response = await fetch(url, { signal });
|
|
99
107
|
await rejection(response, 405);
|
|
100
108
|
expectEqual(response.headers.get("Allow"), "POST", "Allow");
|
|
101
109
|
});
|
|
102
|
-
await checks.run("an origin that is not allowed is rejected with 403", `${RULE}#server`, async () => {
|
|
103
|
-
await rejection(await post({
|
|
110
|
+
await checks.run("an origin that is not allowed is rejected with 403", `${RULE}#server`, async (signal) => {
|
|
111
|
+
await rejection(await post(signal, {
|
|
104
112
|
headers: { Origin: FORBIDDEN_ORIGIN },
|
|
105
113
|
body: "{}"
|
|
106
114
|
}), 403);
|
|
107
115
|
});
|
|
108
|
-
await checks.run("a media type other than JSON is rejected with 415", `${RULE}#server`, async () => {
|
|
109
|
-
await rejection(await post({
|
|
116
|
+
await checks.run("a media type other than JSON is rejected with 415", `${RULE}#server`, async (signal) => {
|
|
117
|
+
await rejection(await post(signal, {
|
|
110
118
|
headers: { "Content-Type": "text/plain" },
|
|
111
119
|
body: "{}"
|
|
112
120
|
}), 415);
|
|
113
121
|
});
|
|
114
|
-
await checks.run("a body over the limit is rejected with 413 and -32600", `${RULE}#server`, async () => {
|
|
115
|
-
await rejection(await post({ body: " ".repeat(options.maximumBytes + 1) }), 413, -32600);
|
|
122
|
+
await checks.run("a body over the limit is rejected with 413 and -32600", `${RULE}#server`, async (signal) => {
|
|
123
|
+
await rejection(await post(signal, { body: " ".repeat(options.maximumBytes + 1) }), 413, -32600);
|
|
116
124
|
});
|
|
117
|
-
await checks.run("a body that is not JSON is rejected with 400 and -32700", `${RULE}#server`, async () => {
|
|
118
|
-
await rejection(await post({ body: "{" }), 400, -32700);
|
|
125
|
+
await checks.run("a body that is not JSON is rejected with 400 and -32700", `${RULE}#server`, async (signal) => {
|
|
126
|
+
await rejection(await post(signal, { body: "{" }), 400, -32700);
|
|
119
127
|
});
|
|
120
|
-
await checks.run("a body that is not UTF-8 is rejected with 400", `${RULE}#server`, async () => {
|
|
121
|
-
await rejection(await post({ body: new Uint8Array([
|
|
128
|
+
await checks.run("a body that is not UTF-8 is rejected with 400", `${RULE}#server`, async (signal) => {
|
|
129
|
+
await rejection(await post(signal, { body: new Uint8Array([
|
|
122
130
|
123,
|
|
123
131
|
255,
|
|
124
132
|
125
|
|
@@ -1,14 +1,11 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Property-based checks for the guarantees of spec/invocation.md#cancellation-and-concurrency, through each remote
|
|
3
|
-
* binding. fast-check makes random sets of concurrent calls and cancellations.
|
|
4
|
-
*/
|
|
5
1
|
import type { Outcome } from "@acid-cat/spec-ts";
|
|
2
|
+
import type { JsonObject } from "../json.js";
|
|
6
3
|
import { Checks } from "./check.js";
|
|
7
|
-
/** A call through one binding.
|
|
8
|
-
export type Invoke = (id: string, input:
|
|
4
|
+
/** A call through one binding. When the signal aborts, the client stops the call and rejects with the reason of the signal. */
|
|
5
|
+
export type Invoke = (id: string, input: JsonObject, signal?: AbortSignal) => Promise<Outcome<unknown, unknown>>;
|
|
9
6
|
export declare function httpInvoke(base: string): Invoke;
|
|
10
|
-
export declare function mcpInvoke(url: string): Promise<{
|
|
7
|
+
export declare function mcpInvoke(url: string, signal: AbortSignal): Promise<{
|
|
11
8
|
readonly invoke: Invoke;
|
|
12
9
|
close(): Promise<void>;
|
|
13
10
|
}>;
|
|
14
|
-
export declare function checkProperties(binding: string, invoke: Invoke): Promise<Checks>;
|
|
11
|
+
export declare function checkProperties(binding: string, invoke: Invoke, deadline: number): Promise<Checks>;
|
|
@@ -1,20 +1,30 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { isStringArray, memberOf, parseJson, readJson } from "../json.js";
|
|
2
|
+
import { Checks, expect, isOutcome, until } from "./check.js";
|
|
2
3
|
import { mcpClient } from "./mcp.js";
|
|
4
|
+
import { SdkError } from "@modelcontextprotocol/client";
|
|
5
|
+
import { randomUUID } from "node:crypto";
|
|
3
6
|
import fc from "fast-check";
|
|
4
7
|
//#region src/conform/properties.ts
|
|
8
|
+
/**
|
|
9
|
+
* Property-based checks for the guarantees of spec/invocation.md#cancellation-and-concurrency, through each remote
|
|
10
|
+
* binding. fast-check makes random sets of concurrent calls and cancellations.
|
|
11
|
+
*/
|
|
5
12
|
const RULE = "spec/invocation.md#cancellation-and-concurrency";
|
|
6
13
|
function httpInvoke(base) {
|
|
7
14
|
return async (id, input, signal) => {
|
|
8
|
-
|
|
15
|
+
const response = await fetch(`${base}/capabilities/${id}`, {
|
|
9
16
|
method: "POST",
|
|
10
17
|
headers: { "Content-Type": "application/json" },
|
|
11
18
|
body: JSON.stringify(input),
|
|
12
19
|
...signal ? { signal } : {}
|
|
13
|
-
})
|
|
20
|
+
});
|
|
21
|
+
const outcome = await readJson(response);
|
|
22
|
+
if (!isOutcome(outcome)) throw new Error(`POST /capabilities/${id} gave ${JSON.stringify(outcome)}, which is not an outcome`);
|
|
23
|
+
return outcome;
|
|
14
24
|
};
|
|
15
25
|
}
|
|
16
|
-
async function mcpInvoke(url) {
|
|
17
|
-
const client = await mcpClient(url);
|
|
26
|
+
async function mcpInvoke(url, signal) {
|
|
27
|
+
const client = await mcpClient(url, signal);
|
|
18
28
|
return {
|
|
19
29
|
invoke: async (id, input, signal) => {
|
|
20
30
|
try {
|
|
@@ -22,14 +32,17 @@ async function mcpInvoke(url) {
|
|
|
22
32
|
name: id,
|
|
23
33
|
arguments: input
|
|
24
34
|
}, signal ? { signal } : void 0);
|
|
25
|
-
const
|
|
35
|
+
const first = result.content[0];
|
|
36
|
+
const body = parseJson(first?.type === "text" ? first.text : "null");
|
|
26
37
|
if (result.isError !== true) return {
|
|
27
38
|
ok: true,
|
|
28
39
|
value: result.structuredContent ?? body
|
|
29
40
|
};
|
|
41
|
+
if (!isOutcome(body)) throw new Error(`the error result of ${id} gave ${JSON.stringify(body)}, which is not an outcome`);
|
|
30
42
|
return body;
|
|
31
43
|
} catch (error) {
|
|
32
|
-
if (error.
|
|
44
|
+
if (signal?.aborted && error instanceof SdkError && error.message === String(signal.reason)) throw signal.reason;
|
|
45
|
+
if (memberOf(error, "code") === -32602) return {
|
|
33
46
|
kind: "unknown-capability",
|
|
34
47
|
message: String(error)
|
|
35
48
|
};
|
|
@@ -47,50 +60,109 @@ const operation = fc.oneof(fc.string({ maxLength: 40 }).map((text) => ({
|
|
|
47
60
|
function agrees(operation, outcome) {
|
|
48
61
|
switch (operation.kind) {
|
|
49
62
|
case "echo": return "ok" in outcome && outcome.ok && JSON.stringify(outcome.value) === JSON.stringify({ text: operation.text });
|
|
50
|
-
case "refuse": return "ok" in outcome && !outcome.ok && outcome.error
|
|
63
|
+
case "refuse": return "ok" in outcome && !outcome.ok && memberOf(outcome.error, "_tag") === "Refused";
|
|
51
64
|
case "crash": return "kind" in outcome && outcome.kind === "internal";
|
|
52
65
|
case "invalid": return "kind" in outcome && outcome.kind === "invalid-input";
|
|
53
66
|
}
|
|
54
67
|
}
|
|
55
|
-
const callFor = (invoke, operation) => {
|
|
68
|
+
const callFor = (invoke, operation, signal) => {
|
|
56
69
|
switch (operation.kind) {
|
|
57
|
-
case "echo": return invoke("fixture.echo", { text: operation.text });
|
|
58
|
-
case "refuse": return invoke("fixture.refuse", {});
|
|
59
|
-
case "crash": return invoke("fixture.crash", {});
|
|
60
|
-
case "invalid": return invoke("fixture.echo", { text: 0 });
|
|
70
|
+
case "echo": return invoke("fixture.echo", { text: operation.text }, signal);
|
|
71
|
+
case "refuse": return invoke("fixture.refuse", {}, signal);
|
|
72
|
+
case "crash": return invoke("fixture.crash", {}, signal);
|
|
73
|
+
case "invalid": return invoke("fixture.echo", { text: 0 }, signal);
|
|
61
74
|
}
|
|
62
75
|
};
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
76
|
+
const settle = (call) => call.then((outcome) => ({ outcome }), (rejected) => ({ rejected }));
|
|
77
|
+
const describe = (settled) => "outcome" in settled ? JSON.stringify(settled.outcome) : `the rejection ${String(settled.rejected)}`;
|
|
78
|
+
/**
|
|
79
|
+
* True if a call ended as a cancellation: with the call failure `cancelled`, or with the rejection that the client gives
|
|
80
|
+
* when it aborts the call, which is the reason of its signal. Any other rejection is not a cancellation.
|
|
81
|
+
*/
|
|
82
|
+
const isCancellation = (settled, reason) => "outcome" in settled ? "kind" in settled.outcome && settled.outcome.kind === "cancelled" : settled.rejected === reason;
|
|
83
|
+
/** The keys of the calls that hold now, as fixture.held gives them through the binding. */
|
|
84
|
+
async function heldKeys(invoke, signal) {
|
|
85
|
+
const outcome = await invoke("fixture.held", {}, signal);
|
|
86
|
+
expect("ok" in outcome && outcome.ok, `fixture.held gave ${JSON.stringify(outcome)}`);
|
|
87
|
+
const keys = memberOf(outcome.value, "keys");
|
|
88
|
+
expect(isStringArray(keys), `fixture.held gave ${JSON.stringify(outcome)}, with no list of keys`);
|
|
89
|
+
return keys;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Start one call to fixture.hold for each flag, and wait until each call holds. Then cancel each call whose flag is
|
|
93
|
+
* true, and release the others. Each cancelled call must end as a cancellation and stop holding. Each other call must
|
|
94
|
+
* hold until it is released, and then give its own key. At the end, the cleanup releases every key, with a signal that
|
|
95
|
+
* aborts `deadline` milliseconds later.
|
|
96
|
+
*/
|
|
97
|
+
async function cancelSome(invoke, flags, signal, deadline) {
|
|
98
|
+
const calls = flags.map((cancelled) => {
|
|
99
|
+
const key = randomUUID();
|
|
100
|
+
const controller = new AbortController();
|
|
101
|
+
const reason = /* @__PURE__ */ new Error(`acid-cat conform cancels the call that holds ${key}`);
|
|
102
|
+
return {
|
|
103
|
+
key,
|
|
104
|
+
cancelled,
|
|
105
|
+
cancel: () => controller.abort(reason),
|
|
106
|
+
reason,
|
|
107
|
+
settled: settle(invoke("fixture.hold", { key }, AbortSignal.any([controller.signal, signal])))
|
|
108
|
+
};
|
|
109
|
+
});
|
|
110
|
+
const cancelled = calls.filter((call) => call.cancelled);
|
|
111
|
+
const kept = calls.filter((call) => !call.cancelled);
|
|
112
|
+
const read = () => heldKeys(invoke, signal);
|
|
113
|
+
try {
|
|
114
|
+
await until("fixture.held gives the key of each call", read, (held) => calls.every((call) => held.includes(call.key)), signal);
|
|
115
|
+
for (const call of cancelled) call.cancel();
|
|
116
|
+
for (const call of cancelled) {
|
|
117
|
+
const settled = await call.settled;
|
|
118
|
+
expect(isCancellation(settled, call.reason), `a cancelled call gave ${describe(settled)}`);
|
|
119
|
+
}
|
|
120
|
+
const held = await until("the cancelled calls stop holding", read, (held) => cancelled.every((call) => !held.includes(call.key)), signal);
|
|
121
|
+
expect(kept.every((call) => held.includes(call.key)), "a call that was not cancelled stopped holding");
|
|
122
|
+
for (const call of kept) {
|
|
123
|
+
await invoke("fixture.release", { key: call.key }, signal);
|
|
124
|
+
const settled = await call.settled;
|
|
125
|
+
const own = {
|
|
126
|
+
ok: true,
|
|
127
|
+
value: { key: call.key }
|
|
128
|
+
};
|
|
129
|
+
expect("outcome" in settled && JSON.stringify(settled.outcome) === JSON.stringify(own), `a call that was not cancelled gave ${describe(settled)}`);
|
|
130
|
+
}
|
|
131
|
+
} finally {
|
|
132
|
+
const cleanup = AbortSignal.timeout(deadline);
|
|
133
|
+
await Promise.all(calls.map((call) => invoke("fixture.release", { key: call.key }, cleanup).catch(() => void 0)));
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/** Two calls or more, at least one of them cancelled and at least one not. */
|
|
137
|
+
const cancellations = fc.array(fc.boolean(), {
|
|
138
|
+
minLength: 2,
|
|
139
|
+
maxLength: 5
|
|
140
|
+
}).filter((flags) => flags.includes(true) && flags.includes(false));
|
|
141
|
+
async function checkProperties(binding, invoke, deadline) {
|
|
142
|
+
const checks = new Checks(`${binding} properties`, deadline);
|
|
143
|
+
await checks.run("concurrent calls each get exactly their own outcome", RULE, async (signal) => {
|
|
66
144
|
await fc.assert(fc.asyncProperty(fc.array(operation, {
|
|
67
145
|
minLength: 1,
|
|
68
146
|
maxLength: 12
|
|
69
147
|
}), async (operations) => {
|
|
70
|
-
|
|
71
|
-
operations.
|
|
72
|
-
|
|
148
|
+
fc.pre(!signal.aborted);
|
|
149
|
+
(await Promise.all(operations.map(async (op) => ({
|
|
150
|
+
op,
|
|
151
|
+
outcome: await callFor(invoke, op, signal)
|
|
152
|
+
})))).forEach(({ op, outcome }) => expect(agrees(op, outcome), `${op.kind} gave ${JSON.stringify(outcome)}`));
|
|
153
|
+
}), {
|
|
154
|
+
numRuns: 25,
|
|
155
|
+
includeErrorInReport: true
|
|
156
|
+
});
|
|
73
157
|
});
|
|
74
|
-
await checks.run("cancelling a call affects only that call", RULE, async () => {
|
|
75
|
-
await fc.assert(fc.asyncProperty(
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
}),
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
if (cancelled) setTimeout(() => controller.abort(), 20);
|
|
83
|
-
return {
|
|
84
|
-
cancelled,
|
|
85
|
-
outcome
|
|
86
|
-
};
|
|
87
|
-
});
|
|
88
|
-
for (const call of calls) {
|
|
89
|
-
const outcome = await call.outcome;
|
|
90
|
-
if (!call.cancelled) expect(outcome !== "aborted" && "ok" in outcome && outcome.ok, `a call that was not cancelled gave ${JSON.stringify(outcome)}`);
|
|
91
|
-
else expect(outcome === "aborted" || "kind" in outcome && outcome.kind === "cancelled", `a cancelled call gave ${JSON.stringify(outcome)}`);
|
|
92
|
-
}
|
|
93
|
-
}), { numRuns: 5 });
|
|
158
|
+
await checks.run("cancelling a call affects only that call", RULE, async (signal) => {
|
|
159
|
+
await fc.assert(fc.asyncProperty(cancellations, (flags) => {
|
|
160
|
+
fc.pre(!signal.aborted);
|
|
161
|
+
return cancelSome(invoke, flags, signal, deadline);
|
|
162
|
+
}), {
|
|
163
|
+
numRuns: 5,
|
|
164
|
+
includeErrorInReport: true
|
|
165
|
+
});
|
|
94
166
|
});
|
|
95
167
|
return checks;
|
|
96
168
|
}
|