@acid-cat/cli 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/conformance/fixture.acidcat.json +63 -5
- package/dist/conform/check.d.ts +18 -3
- package/dist/conform/check.js +45 -5
- package/dist/conform/cli.d.ts +3 -1
- package/dist/conform/cli.js +80 -49
- package/dist/conform/http.d.ts +1 -0
- package/dist/conform/http.js +30 -26
- package/dist/conform/index.d.ts +6 -5
- package/dist/conform/index.js +20 -9
- package/dist/conform/mcp.d.ts +2 -1
- package/dist/conform/mcp.js +40 -39
- package/dist/conform/properties.d.ts +3 -7
- package/dist/conform/properties.js +95 -33
- package/dist/conform/webmcp.d.ts +16 -5
- package/dist/conform/webmcp.js +81 -42
- package/dist/json/fixture.acidcat.json.js +44 -4
- package/dist/json/package.json.js +1 -1
- package/dist/main.js +5 -2
- package/package.json +6 -6
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@ The AcidCat command line. npm package: `@acid-cat/cli`. It gives the `acid-cat`
|
|
|
6
6
|
acid-cat check [--strict] <catalog.json ...> # check catalogs
|
|
7
7
|
acid-cat format [--write] <catalog.json ...> # write catalogs in canonical form
|
|
8
8
|
acid-cat view <catalog.json> <target> # write the view of one target
|
|
9
|
-
acid-cat conform --mcp <url> --http <url> --cli "<command>" --webmcp <url> # test an implementation
|
|
9
|
+
acid-cat conform --mcp <url> --http <url> --cli "<command>" --webmcp <url> [--deadline <ms>] # test an implementation
|
|
10
10
|
```
|
|
11
11
|
|
|
12
12
|
Install it as a development dependency, and run it with `pnpm exec acid-cat …`:
|
|
@@ -35,10 +35,12 @@ The exit code is 0 if there are no errors. It is 1 if there are errors. With `--
|
|
|
35
35
|
`conform` tests a running implementation of the [conformance fixture](conformance/fixture.pkl) through its bindings (see [Conformance](../../spec/conformance.md#acid-cat-conform)):
|
|
36
36
|
|
|
37
37
|
- **MCP:** the official MCP client, with revision `2026-07-28`, for tools and outcomes; raw requests for rejections.
|
|
38
|
-
- **HTTP** and **CLI:** each outcome, each kind of call failure, availability, and rejections. For the CLI, an interrupt (`SIGINT`) during a call gives exit code 130.
|
|
38
|
+
- **HTTP** and **CLI:** each outcome, each kind of call failure, availability, and rejections. For the CLI, an interrupt (`SIGINT`) during a call that holds gives exit code 130.
|
|
39
39
|
- **WebMCP:** the checks act as the WebMCP host. `--webmcp <url>` opens a page that serves the `app` target in Chromium, through [Playwright](https://playwright.dev), with a test `modelContext`. Install Playwright to use it: `pnpm add -D playwright`, then `pnpm exec playwright install chromium`. In code, `webmcp` can also be a function that installs the implementation on a host in the same process.
|
|
40
40
|
- **Properties:** with fast-check, through each remote binding: concurrent calls each get exactly their own outcome, and a cancellation changes only its own call.
|
|
41
41
|
|
|
42
|
+
The checks wait for states that they can observe, not for times. Before a check cancels a call to `fixture.hold`, it waits until `fixture.held` gives the key of the call, or, for the CLI, until the marker file of the call exists. Each check has one deadline: `--deadline`, or `deadline` in code, in milliseconds. The default is 30 000. A check that does not settle before its deadline fails with "did not settle", and the check stops what it started. This cleanup has a deadline of the same length, so `conform` always returns. The deadline never makes a check pass.
|
|
43
|
+
|
|
42
44
|
The exit code is 1 if a check fails. Both AcidCat cores run `conform` in their tests. In code, use `conform()` and `serve()` from `@acid-cat/cli`. `createTestHost()` gives the WebMCP host of the checks.
|
|
43
45
|
|
|
44
46
|
## Implementation
|
|
@@ -133,12 +133,70 @@
|
|
|
133
133
|
"type": "object"
|
|
134
134
|
}
|
|
135
135
|
},
|
|
136
|
+
"fixture.held": {
|
|
137
|
+
"description": "Return the keys of the calls to fixture.hold that hold now.",
|
|
138
|
+
"audience": [
|
|
139
|
+
"app",
|
|
140
|
+
"mcp",
|
|
141
|
+
"cli",
|
|
142
|
+
"api"
|
|
143
|
+
],
|
|
144
|
+
"output": {
|
|
145
|
+
"properties": {
|
|
146
|
+
"keys": {
|
|
147
|
+
"items": {
|
|
148
|
+
"type": "string"
|
|
149
|
+
},
|
|
150
|
+
"type": "array"
|
|
151
|
+
}
|
|
152
|
+
},
|
|
153
|
+
"required": [
|
|
154
|
+
"keys"
|
|
155
|
+
],
|
|
156
|
+
"type": "object"
|
|
157
|
+
},
|
|
158
|
+
"readOnly": true
|
|
159
|
+
},
|
|
136
160
|
"fixture.hidden": {
|
|
137
161
|
"description": "Only the app target can use this capability.",
|
|
138
162
|
"audience": [
|
|
139
163
|
"app"
|
|
140
164
|
]
|
|
141
165
|
},
|
|
166
|
+
"fixture.hold": {
|
|
167
|
+
"description": "Hold until fixture.release releases the key, then return the key. While the call holds, fixture.held gives the key. With a marker, the call writes a file at that path when it starts to hold.",
|
|
168
|
+
"audience": [
|
|
169
|
+
"app",
|
|
170
|
+
"mcp",
|
|
171
|
+
"cli",
|
|
172
|
+
"api"
|
|
173
|
+
],
|
|
174
|
+
"input": {
|
|
175
|
+
"properties": {
|
|
176
|
+
"key": {
|
|
177
|
+
"type": "string"
|
|
178
|
+
},
|
|
179
|
+
"marker": {
|
|
180
|
+
"type": "string"
|
|
181
|
+
}
|
|
182
|
+
},
|
|
183
|
+
"required": [
|
|
184
|
+
"key"
|
|
185
|
+
],
|
|
186
|
+
"type": "object"
|
|
187
|
+
},
|
|
188
|
+
"output": {
|
|
189
|
+
"properties": {
|
|
190
|
+
"key": {
|
|
191
|
+
"type": "string"
|
|
192
|
+
}
|
|
193
|
+
},
|
|
194
|
+
"required": [
|
|
195
|
+
"key"
|
|
196
|
+
],
|
|
197
|
+
"type": "object"
|
|
198
|
+
}
|
|
199
|
+
},
|
|
142
200
|
"fixture.refuse": {
|
|
143
201
|
"description": "Always return the failure Refused.",
|
|
144
202
|
"audience": [
|
|
@@ -166,8 +224,8 @@
|
|
|
166
224
|
}
|
|
167
225
|
}
|
|
168
226
|
},
|
|
169
|
-
"fixture.
|
|
170
|
-
"description": "
|
|
227
|
+
"fixture.release": {
|
|
228
|
+
"description": "Release the calls to fixture.hold that hold the key.",
|
|
171
229
|
"audience": [
|
|
172
230
|
"app",
|
|
173
231
|
"mcp",
|
|
@@ -176,12 +234,12 @@
|
|
|
176
234
|
],
|
|
177
235
|
"input": {
|
|
178
236
|
"properties": {
|
|
179
|
-
"
|
|
180
|
-
"type": "
|
|
237
|
+
"key": {
|
|
238
|
+
"type": "string"
|
|
181
239
|
}
|
|
182
240
|
},
|
|
183
241
|
"required": [
|
|
184
|
-
"
|
|
242
|
+
"key"
|
|
185
243
|
],
|
|
186
244
|
"type": "object"
|
|
187
245
|
}
|
package/dist/conform/check.d.ts
CHANGED
|
@@ -11,13 +11,28 @@ export interface Check {
|
|
|
11
11
|
export declare const fixture: CatalogDocument;
|
|
12
12
|
/** The ids of the capabilities in the view of a target of the fixture. */
|
|
13
13
|
export declare const idsOf: (target: string) => string[];
|
|
14
|
-
/**
|
|
14
|
+
/** The time, in milliseconds, that each check has to settle when `conform` gets no other time. */
|
|
15
|
+
export declare const DEADLINE = 30000;
|
|
16
|
+
/**
|
|
17
|
+
* Collects checks for one binding. Each check has one deadline. The check gives its signal to each operation, and the
|
|
18
|
+
* signal aborts at the deadline. Then each operation stops, and the check fails with "did not settle". The deadline
|
|
19
|
+
* never makes a check pass. The cleanup of a check, which releases what the check holds, gets its own signal with the
|
|
20
|
+
* same time, so that `conform` always returns.
|
|
21
|
+
*/
|
|
15
22
|
export declare class Checks {
|
|
16
23
|
readonly results: Check[];
|
|
17
24
|
readonly binding: string;
|
|
18
|
-
|
|
19
|
-
|
|
25
|
+
readonly deadline: number;
|
|
26
|
+
constructor(binding: string, deadline: number);
|
|
27
|
+
run(name: string, rule: string, body: (signal: AbortSignal) => Promise<void> | void): Promise<void>;
|
|
20
28
|
}
|
|
29
|
+
/**
|
|
30
|
+
* Read a state until `done` is true for it, and give that value. The read and the pause between two reads stop when the
|
|
31
|
+
* signal of the check aborts.
|
|
32
|
+
*/
|
|
33
|
+
export declare function until<T>(what: string, read: () => Promise<T>, done: (value: T) => boolean, signal: AbortSignal): Promise<T>;
|
|
34
|
+
/** Settle as the promise does, or reject with the reason of the signal when it aborts first. */
|
|
35
|
+
export declare function abortable<T>(promise: Promise<T>, signal: AbortSignal): Promise<T>;
|
|
21
36
|
/** Fail a check unless `condition` is true. */
|
|
22
37
|
export declare function expect(condition: unknown, message: string): asserts condition;
|
|
23
38
|
/** Fail a check unless the two values have the same JSON. */
|
package/dist/conform/check.js
CHANGED
|
@@ -1,21 +1,33 @@
|
|
|
1
1
|
import fixture_acidcat_default from "../json/fixture.acidcat.json.js";
|
|
2
2
|
import { viewOf } from "@acid-cat/spec-ts";
|
|
3
|
+
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
3
4
|
//#region src/conform/check.ts
|
|
4
5
|
/** The result of one conformance check, and helpers to write checks. */
|
|
5
6
|
/** The conformance fixture: the catalog that each implementation serves. */
|
|
6
7
|
const fixture = fixture_acidcat_default;
|
|
7
8
|
/** The ids of the capabilities in the view of a target of the fixture. */
|
|
8
9
|
const idsOf = (target) => Object.keys(viewOf(fixture, target).capabilities).sort();
|
|
9
|
-
/**
|
|
10
|
+
/** The time, in milliseconds, that each check has to settle when `conform` gets no other time. */
|
|
11
|
+
const DEADLINE = 3e4;
|
|
12
|
+
/**
|
|
13
|
+
* Collects checks for one binding. Each check has one deadline. The check gives its signal to each operation, and the
|
|
14
|
+
* signal aborts at the deadline. Then each operation stops, and the check fails with "did not settle". The deadline
|
|
15
|
+
* never makes a check pass. The cleanup of a check, which releases what the check holds, gets its own signal with the
|
|
16
|
+
* same time, so that `conform` always returns.
|
|
17
|
+
*/
|
|
10
18
|
var Checks = class {
|
|
11
19
|
results = [];
|
|
12
20
|
binding;
|
|
13
|
-
|
|
21
|
+
deadline;
|
|
22
|
+
constructor(binding, deadline) {
|
|
14
23
|
this.binding = binding;
|
|
24
|
+
this.deadline = deadline;
|
|
15
25
|
}
|
|
16
26
|
async run(name, rule, body) {
|
|
27
|
+
const controller = new AbortController();
|
|
28
|
+
const timer = setTimeout(() => controller.abort(), this.deadline);
|
|
17
29
|
try {
|
|
18
|
-
await body();
|
|
30
|
+
await body(controller.signal);
|
|
19
31
|
this.results.push({
|
|
20
32
|
binding: this.binding,
|
|
21
33
|
name,
|
|
@@ -23,16 +35,44 @@ var Checks = class {
|
|
|
23
35
|
passed: true
|
|
24
36
|
});
|
|
25
37
|
} catch (error) {
|
|
38
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
39
|
+
const detail = controller.signal.aborted ? `did not settle within ${this.deadline} ms: ${message}` : message;
|
|
26
40
|
this.results.push({
|
|
27
41
|
binding: this.binding,
|
|
28
42
|
name,
|
|
29
43
|
rule,
|
|
30
44
|
passed: false,
|
|
31
|
-
detail
|
|
45
|
+
detail
|
|
32
46
|
});
|
|
47
|
+
} finally {
|
|
48
|
+
clearTimeout(timer);
|
|
33
49
|
}
|
|
34
50
|
}
|
|
35
51
|
};
|
|
52
|
+
/**
|
|
53
|
+
* Read a state until `done` is true for it, and give that value. The read and the pause between two reads stop when the
|
|
54
|
+
* signal of the check aborts.
|
|
55
|
+
*/
|
|
56
|
+
async function until(what, read, done, signal) {
|
|
57
|
+
let last;
|
|
58
|
+
try {
|
|
59
|
+
for (;;) {
|
|
60
|
+
last = await read();
|
|
61
|
+
if (done(last)) return last;
|
|
62
|
+
await setTimeout$1(10, void 0, { signal });
|
|
63
|
+
}
|
|
64
|
+
} catch (error) {
|
|
65
|
+
throw signal.aborted ? /* @__PURE__ */ new Error(`waiting until ${what}; the last value was ${JSON.stringify(last)}`) : error;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/** Settle as the promise does, or reject with the reason of the signal when it aborts first. */
|
|
69
|
+
function abortable(promise, signal) {
|
|
70
|
+
const aborted = new Promise((_, reject) => {
|
|
71
|
+
signal.throwIfAborted();
|
|
72
|
+
signal.addEventListener("abort", () => reject(signal.reason), { once: true });
|
|
73
|
+
});
|
|
74
|
+
return Promise.race([promise, aborted]);
|
|
75
|
+
}
|
|
36
76
|
/** Fail a check unless `condition` is true. */
|
|
37
77
|
function expect(condition, message) {
|
|
38
78
|
if (!condition) throw new Error(message);
|
|
@@ -48,4 +88,4 @@ const sorted = (values) => [...values].sort();
|
|
|
48
88
|
/** An origin that no deployment allows. */
|
|
49
89
|
const FORBIDDEN_ORIGIN = "https://not-allowed.acid-cat.invalid";
|
|
50
90
|
//#endregion
|
|
51
|
-
export { Checks, FORBIDDEN_ORIGIN, expect, expectEqual, fixture, idsOf, sorted };
|
|
91
|
+
export { Checks, DEADLINE, FORBIDDEN_ORIGIN, abortable, expect, expectEqual, fixture, idsOf, sorted, until };
|
package/dist/conform/cli.d.ts
CHANGED
package/dist/conform/cli.js
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
|
-
import { Checks, expect, expectEqual, idsOf, sorted } from "./check.js";
|
|
1
|
+
import { Checks, expect, expectEqual, idsOf, sorted, until } from "./check.js";
|
|
2
2
|
import { spawn } from "node:child_process";
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
4
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
5
|
+
import { tmpdir } from "node:os";
|
|
6
|
+
import { join } from "node:path";
|
|
3
7
|
//#region src/conform/cli.ts
|
|
4
8
|
/** Conformance checks for the CLI binding (spec/bindings/cli.md). Each check runs the command line as a process. */
|
|
5
9
|
const RULE = "spec/bindings/cli.md";
|
|
@@ -8,53 +12,62 @@ const USAGES = {
|
|
|
8
12
|
"fixture.crash": "fixture crash",
|
|
9
13
|
"fixture.echo": "fixture echo --text <string>",
|
|
10
14
|
"fixture.flags": "fixture flags --mode <fast|slow> [--count <integer>] [--extra <json>] [--label <string>] [--ratio <number>] [--verbose]",
|
|
15
|
+
"fixture.held": "fixture held",
|
|
16
|
+
"fixture.hold": "fixture hold --key <string> [--marker <string>]",
|
|
11
17
|
"fixture.refuse": "fixture refuse",
|
|
12
|
-
"fixture.
|
|
18
|
+
"fixture.release": "fixture release --key <string>",
|
|
13
19
|
"fixture.switch.set": "fixture switch set --on",
|
|
14
20
|
"fixture.toggle": "fixture toggle"
|
|
15
21
|
};
|
|
16
|
-
/**
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Start the command line. When the signal aborts, the process is killed, and its run rejects. The rejection is observed
|
|
24
|
+
* here too, so that a check that stops waiting first, for example at the deadline, records a failure and does not end
|
|
25
|
+
* the process with an unhandled rejection.
|
|
26
|
+
*/
|
|
27
|
+
function start(command, args, signal) {
|
|
28
|
+
const child = spawn(command[0], [...command.slice(1), ...args], {
|
|
29
|
+
stdio: [
|
|
22
30
|
"ignore",
|
|
23
31
|
"pipe",
|
|
24
32
|
"pipe"
|
|
25
|
-
]
|
|
26
|
-
|
|
33
|
+
],
|
|
34
|
+
signal,
|
|
35
|
+
killSignal: "SIGKILL"
|
|
36
|
+
});
|
|
37
|
+
const run = new Promise((resolve, reject) => {
|
|
27
38
|
let stdout = "";
|
|
28
39
|
let stderr = "";
|
|
29
40
|
child.stdout.on("data", (chunk) => stdout += chunk.toString());
|
|
30
41
|
child.stderr.on("data", (chunk) => stderr += chunk.toString());
|
|
31
|
-
child.on("error", reject);
|
|
32
|
-
child.on("close", (code,
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
stderr
|
|
39
|
-
});
|
|
40
|
-
});
|
|
42
|
+
child.on("error", (error) => reject(signal.aborted ? /* @__PURE__ */ new Error(`waiting until the command line exits: ${args.join(" ")}`) : error));
|
|
43
|
+
child.on("close", (code, exitSignal) => resolve({
|
|
44
|
+
code,
|
|
45
|
+
signal: exitSignal,
|
|
46
|
+
stdout,
|
|
47
|
+
stderr
|
|
48
|
+
}));
|
|
41
49
|
});
|
|
50
|
+
run.catch(() => void 0);
|
|
51
|
+
return {
|
|
52
|
+
child,
|
|
53
|
+
run
|
|
54
|
+
};
|
|
42
55
|
}
|
|
43
|
-
async function checkCli(command) {
|
|
44
|
-
const checks = new Checks("cli");
|
|
45
|
-
const
|
|
46
|
-
await checks.run("a value goes to standard output with exit code 0", `${RULE}#outcomes`, async () => {
|
|
47
|
-
const result = await
|
|
56
|
+
async function checkCli(command, options) {
|
|
57
|
+
const checks = new Checks("cli", options.deadline);
|
|
58
|
+
const run = (signal, ...args) => start(command, args, signal).run;
|
|
59
|
+
await checks.run("a value goes to standard output with exit code 0", `${RULE}#outcomes`, async (signal) => {
|
|
60
|
+
const result = await run(signal, "fixture", "echo", "--text", "hello");
|
|
48
61
|
expectEqual(result.code, 0, "exit code");
|
|
49
62
|
expectEqual(JSON.parse(result.stdout), { text: "hello" }, "standard output");
|
|
50
63
|
});
|
|
51
|
-
await checks.run("a failure goes to standard output with exit code 1", `${RULE}#outcomes`, async () => {
|
|
52
|
-
const result = await
|
|
64
|
+
await checks.run("a failure goes to standard output with exit code 1", `${RULE}#outcomes`, async (signal) => {
|
|
65
|
+
const result = await run(signal, "fixture", "refuse");
|
|
53
66
|
expectEqual(result.code, 1, "exit code");
|
|
54
67
|
expectEqual(JSON.parse(result.stdout)._tag, "Refused", "_tag");
|
|
55
68
|
});
|
|
56
|
-
await checks.run("a defect goes to standard error with exit code 5", `${RULE}#outcomes`, async () => {
|
|
57
|
-
const result = await
|
|
69
|
+
await checks.run("a defect goes to standard error with exit code 5", `${RULE}#outcomes`, async (signal) => {
|
|
70
|
+
const result = await run(signal, "fixture", "crash");
|
|
58
71
|
expectEqual(result.code, 5, "exit code");
|
|
59
72
|
expect(/"kind":"internal"/.test(result.stderr), "standard error must contain the call failure");
|
|
60
73
|
});
|
|
@@ -78,8 +91,10 @@ async function checkCli(command) {
|
|
|
78
91
|
"a flag that is not valid has exit code 3",
|
|
79
92
|
[
|
|
80
93
|
"fixture",
|
|
81
|
-
"
|
|
82
|
-
"--
|
|
94
|
+
"flags",
|
|
95
|
+
"--mode",
|
|
96
|
+
"fast",
|
|
97
|
+
"--ratio",
|
|
83
98
|
"soon"
|
|
84
99
|
],
|
|
85
100
|
3
|
|
@@ -93,31 +108,47 @@ async function checkCli(command) {
|
|
|
93
108
|
],
|
|
94
109
|
3
|
|
95
110
|
]
|
|
96
|
-
]) await checks.run(name, `${RULE}#outcomes`, async () => {
|
|
97
|
-
expectEqual((await
|
|
111
|
+
]) await checks.run(name, `${RULE}#outcomes`, async (signal) => {
|
|
112
|
+
expectEqual((await run(signal, ...args)).code, code, "exit code");
|
|
98
113
|
});
|
|
99
|
-
await checks.run("an interrupt cancels the call with exit code 130", `${RULE}#commands`, async () => {
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
114
|
+
await checks.run("an interrupt cancels the call with exit code 130", `${RULE}#commands`, async (signal) => {
|
|
115
|
+
const directory = await mkdtemp(join(tmpdir(), "acid-cat-conform-"));
|
|
116
|
+
const marker = join(directory, "held");
|
|
117
|
+
try {
|
|
118
|
+
const { child, run } = start(command, [
|
|
119
|
+
"fixture",
|
|
120
|
+
"hold",
|
|
121
|
+
"--key",
|
|
122
|
+
"interrupt",
|
|
123
|
+
"--marker",
|
|
124
|
+
marker
|
|
125
|
+
], signal);
|
|
126
|
+
const ended = () => child.exitCode !== null || child.signalCode !== null;
|
|
127
|
+
await until("the call writes its marker file", async () => existsSync(marker) || ended(), (ready) => ready, signal);
|
|
128
|
+
if (!ended()) child.kill("SIGINT");
|
|
129
|
+
const result = await run;
|
|
130
|
+
expect(existsSync(marker), `the call must hold before the interrupt, but the command line ended with ${result.code}: ${result.stderr}`);
|
|
131
|
+
expect(result.signal === null, `the process must exit, not end by the signal ${result.signal}`);
|
|
132
|
+
expectEqual(result.code, 130, "exit code");
|
|
133
|
+
expectEqual(result.stdout, "", "standard output");
|
|
134
|
+
} finally {
|
|
135
|
+
await rm(directory, {
|
|
136
|
+
recursive: true,
|
|
137
|
+
force: true
|
|
138
|
+
});
|
|
139
|
+
}
|
|
109
140
|
});
|
|
110
|
-
await checks.run("call gives the value of a capability", `${RULE}#commands`, async () => {
|
|
111
|
-
const result = await
|
|
141
|
+
await checks.run("call gives the value of a capability", `${RULE}#commands`, async (signal) => {
|
|
142
|
+
const result = await run(signal, "call", "fixture.echo", "{\"text\":\"hello\"}");
|
|
112
143
|
expectEqual(result.code, 0, "exit code");
|
|
113
144
|
expectEqual(JSON.parse(result.stdout), { text: "hello" }, "standard output");
|
|
114
145
|
});
|
|
115
|
-
await checks.run("list gives the view", `${RULE}#commands`, async () => {
|
|
116
|
-
const result = await
|
|
146
|
+
await checks.run("list gives the view", `${RULE}#commands`, async (signal) => {
|
|
147
|
+
const result = await run(signal, "list");
|
|
117
148
|
expectEqual(sorted(JSON.parse(result.stdout).map((entry) => entry.id)), idsOf("cli"), "ids");
|
|
118
149
|
});
|
|
119
|
-
await checks.run("list gives the usage of each capability, with its flags", `${RULE}#commands`, async () => {
|
|
120
|
-
const entries = JSON.parse((await
|
|
150
|
+
await checks.run("list gives the usage of each capability, with its flags", `${RULE}#commands`, async (signal) => {
|
|
151
|
+
const entries = JSON.parse((await run(signal, "list")).stdout);
|
|
121
152
|
for (const { id, usage } of entries) {
|
|
122
153
|
expect(typeof usage === "string", `${id} must have a usage`);
|
|
123
154
|
const [program, ...rest] = usage.split(" ");
|
package/dist/conform/http.d.ts
CHANGED
package/dist/conform/http.js
CHANGED
|
@@ -3,31 +3,32 @@ import { Checks, FORBIDDEN_ORIGIN, expect, expectEqual, idsOf, sorted } from "./
|
|
|
3
3
|
/** Conformance checks for the HTTP binding (spec/bindings/http.md). */
|
|
4
4
|
const RULE = "spec/bindings/http.md";
|
|
5
5
|
async function checkHttp(base, options) {
|
|
6
|
-
const checks = new Checks("http");
|
|
7
|
-
const post = (id, body, headers = {}) => fetch(`${base}/capabilities/${id}`, {
|
|
6
|
+
const checks = new Checks("http", options.deadline);
|
|
7
|
+
const post = (signal, id, body, headers = {}) => fetch(`${base}/capabilities/${id}`, {
|
|
8
8
|
method: "POST",
|
|
9
9
|
headers: {
|
|
10
10
|
"Content-Type": "application/json",
|
|
11
11
|
...headers
|
|
12
12
|
},
|
|
13
|
-
body: typeof body === "string" ? body : body instanceof Uint8Array ? new Blob([body]) : JSON.stringify(body)
|
|
13
|
+
body: typeof body === "string" ? body : body instanceof Uint8Array ? new Blob([body]) : JSON.stringify(body),
|
|
14
|
+
signal
|
|
14
15
|
});
|
|
15
16
|
const json = async (response) => await response.json();
|
|
16
|
-
await checks.run("GET /capabilities lists the view with availability", `${RULE}#requests`, async () => {
|
|
17
|
-
const list = await (await fetch(`${base}/capabilities
|
|
17
|
+
await checks.run("GET /capabilities lists the view with availability", `${RULE}#requests`, async (signal) => {
|
|
18
|
+
const list = await (await fetch(`${base}/capabilities`, { signal })).json();
|
|
18
19
|
expectEqual(sorted(list.map((entry) => entry.id)), idsOf("api"), "ids");
|
|
19
20
|
expect(list.every((entry) => typeof entry.available === "boolean"), "each entry must have available");
|
|
20
21
|
});
|
|
21
|
-
await checks.run("a value has the status 200", `${RULE}#requests`, async () => {
|
|
22
|
-
const response = await post("fixture.echo", { text: "hello" });
|
|
22
|
+
await checks.run("a value has the status 200", `${RULE}#requests`, async (signal) => {
|
|
23
|
+
const response = await post(signal, "fixture.echo", { text: "hello" });
|
|
23
24
|
expectEqual(response.status, 200, "status");
|
|
24
25
|
expectEqual(await json(response), {
|
|
25
26
|
ok: true,
|
|
26
27
|
value: { text: "hello" }
|
|
27
28
|
}, "body");
|
|
28
29
|
});
|
|
29
|
-
await checks.run("a failure has the status 200 and ok: false", `${RULE}#requests`, async () => {
|
|
30
|
-
const response = await post("fixture.refuse", {});
|
|
30
|
+
await checks.run("a failure has the status 200 and ok: false", `${RULE}#requests`, async (signal) => {
|
|
31
|
+
const response = await post(signal, "fixture.refuse", {});
|
|
31
32
|
const body = await json(response);
|
|
32
33
|
expectEqual(response.status, 200, "status");
|
|
33
34
|
expect(body.ok === false && body.error?._tag === "Refused", `got ${JSON.stringify(body)}`);
|
|
@@ -54,38 +55,41 @@ async function checkHttp(base, options) {
|
|
|
54
55
|
500,
|
|
55
56
|
"internal"
|
|
56
57
|
]
|
|
57
|
-
]) await checks.run(name, `${RULE}#requests`, async () => {
|
|
58
|
-
const response = await post(id, input);
|
|
58
|
+
]) await checks.run(name, `${RULE}#requests`, async (signal) => {
|
|
59
|
+
const response = await post(signal, id, input);
|
|
59
60
|
expectEqual(response.status, status, "status");
|
|
60
61
|
expectEqual((await json(response)).kind, kind, "kind");
|
|
61
62
|
});
|
|
62
|
-
await checks.run("an unavailable capability has the status 503", `${RULE}#requests`, async () => {
|
|
63
|
-
await post("fixture.switch.set", { on: false });
|
|
63
|
+
await checks.run("an unavailable capability has the status 503", `${RULE}#requests`, async (signal) => {
|
|
64
|
+
await post(signal, "fixture.switch.set", { on: false });
|
|
64
65
|
try {
|
|
65
|
-
const toggle = (await (await fetch(`${base}/capabilities
|
|
66
|
+
const toggle = (await (await fetch(`${base}/capabilities`, { signal })).json()).find((entry) => entry.id === "fixture.toggle");
|
|
66
67
|
expect(toggle?.available === false && typeof toggle.reason === "string", "fixture.toggle must be unavailable, with a reason");
|
|
67
|
-
const response = await post("fixture.toggle", {});
|
|
68
|
+
const response = await post(signal, "fixture.toggle", {});
|
|
68
69
|
expectEqual(response.status, 503, "status");
|
|
69
70
|
expectEqual((await json(response)).kind, "unavailable", "kind");
|
|
70
71
|
} finally {
|
|
71
|
-
await post("fixture.switch.set", { on: true });
|
|
72
|
+
await post(AbortSignal.timeout(options.deadline), "fixture.switch.set", { on: true });
|
|
72
73
|
}
|
|
73
74
|
});
|
|
74
75
|
const status = async (response, expected) => expectEqual(response.status, expected, "status");
|
|
75
|
-
await checks.run("a method that is not accepted is rejected with 405", `${RULE}#requests`, async () => {
|
|
76
|
-
await status(await fetch(`${base}/capabilities`, {
|
|
76
|
+
await checks.run("a method that is not accepted is rejected with 405", `${RULE}#requests`, async (signal) => {
|
|
77
|
+
await status(await fetch(`${base}/capabilities`, {
|
|
78
|
+
method: "PUT",
|
|
79
|
+
signal
|
|
80
|
+
}), 405);
|
|
77
81
|
});
|
|
78
|
-
await checks.run("an origin that is not allowed is rejected with 403", `${RULE}#requests`, async () => {
|
|
79
|
-
await status(await post("fixture.echo", { text: "x" }, { Origin: FORBIDDEN_ORIGIN }), 403);
|
|
82
|
+
await checks.run("an origin that is not allowed is rejected with 403", `${RULE}#requests`, async (signal) => {
|
|
83
|
+
await status(await post(signal, "fixture.echo", { text: "x" }, { Origin: FORBIDDEN_ORIGIN }), 403);
|
|
80
84
|
});
|
|
81
|
-
await checks.run("a media type other than JSON is rejected with 415", `${RULE}#requests`, async () => {
|
|
82
|
-
await status(await post("fixture.echo", { text: "x" }, { "Content-Type": "text/plain" }), 415);
|
|
85
|
+
await checks.run("a media type other than JSON is rejected with 415", `${RULE}#requests`, async (signal) => {
|
|
86
|
+
await status(await post(signal, "fixture.echo", { text: "x" }, { "Content-Type": "text/plain" }), 415);
|
|
83
87
|
});
|
|
84
|
-
await checks.run("a body over the limit is rejected with 413", `${RULE}#requests`, async () => {
|
|
85
|
-
await status(await post("fixture.echo", " ".repeat(options.maximumBytes + 1)), 413);
|
|
88
|
+
await checks.run("a body over the limit is rejected with 413", `${RULE}#requests`, async (signal) => {
|
|
89
|
+
await status(await post(signal, "fixture.echo", " ".repeat(options.maximumBytes + 1)), 413);
|
|
86
90
|
});
|
|
87
|
-
await checks.run("a body that is not UTF-8 is rejected with 400", `${RULE}#requests`, async () => {
|
|
88
|
-
await status(await post("fixture.echo", new Uint8Array([
|
|
91
|
+
await checks.run("a body that is not UTF-8 is rejected with 400", `${RULE}#requests`, async (signal) => {
|
|
92
|
+
await status(await post(signal, "fixture.echo", new Uint8Array([
|
|
89
93
|
123,
|
|
90
94
|
255,
|
|
91
95
|
125
|
package/dist/conform/index.d.ts
CHANGED
|
@@ -1,11 +1,7 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `acid-cat conform`: test a running implementation through its bindings (spec/conformance.md). The implementation
|
|
3
|
-
* serves the conformance fixture (conformance/fixture.acidcat.json), with its switch on.
|
|
4
|
-
*/
|
|
5
1
|
import type { Check } from "./check.js";
|
|
6
2
|
import type { WebMcpInstall } from "./webmcp.js";
|
|
7
3
|
export type { Check } from "./check.js";
|
|
8
|
-
export { fixture } from "./check.js";
|
|
4
|
+
export { DEADLINE, fixture } from "./check.js";
|
|
9
5
|
export { serve } from "./serve.js";
|
|
10
6
|
export type { Served, WebHandler } from "./serve.js";
|
|
11
7
|
export { createTestHost } from "./webmcp.js";
|
|
@@ -24,6 +20,11 @@ export interface ConformOptions {
|
|
|
24
20
|
readonly webmcp?: string | WebMcpInstall | undefined;
|
|
25
21
|
/** The body limit of the remote bindings, in bytes. The default is 1 MiB. */
|
|
26
22
|
readonly maximumBytes?: number | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* The time, in milliseconds, that each check has to settle. A check that does not settle in this time fails. The
|
|
25
|
+
* default is `DEADLINE`, 30 seconds.
|
|
26
|
+
*/
|
|
27
|
+
readonly deadline?: number | undefined;
|
|
27
28
|
}
|
|
28
29
|
/** Run all checks for the bindings that the options give. */
|
|
29
30
|
export declare function conform(options: ConformOptions): Promise<readonly Check[]>;
|
package/dist/conform/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { fixture } from "./check.js";
|
|
1
|
+
import { DEADLINE, fixture } from "./check.js";
|
|
2
2
|
import { checkCli } from "./cli.js";
|
|
3
3
|
import { checkHttp } from "./http.js";
|
|
4
4
|
import { checkMcp } from "./mcp.js";
|
|
@@ -6,27 +6,38 @@ import { checkProperties, httpInvoke, mcpInvoke } from "./properties.js";
|
|
|
6
6
|
import { browserDriver, checkWebMcp, processDriver } from "./webmcp.js";
|
|
7
7
|
import { serve } from "./serve.js";
|
|
8
8
|
//#region src/conform/index.ts
|
|
9
|
+
/**
|
|
10
|
+
* `acid-cat conform`: test a running implementation through its bindings (spec/conformance.md). The implementation
|
|
11
|
+
* serves the conformance fixture (conformance/fixture.acidcat.json), with its switch on.
|
|
12
|
+
*/
|
|
9
13
|
/** Run all checks for the bindings that the options give. */
|
|
10
14
|
async function conform(options) {
|
|
11
15
|
const maximumBytes = options.maximumBytes ?? 1048576;
|
|
16
|
+
const deadline = options.deadline ?? 3e4;
|
|
12
17
|
const results = [];
|
|
13
18
|
if (options.mcp !== void 0) {
|
|
14
|
-
results.push(...(await checkMcp(options.mcp, {
|
|
15
|
-
|
|
19
|
+
results.push(...(await checkMcp(options.mcp, {
|
|
20
|
+
maximumBytes,
|
|
21
|
+
deadline
|
|
22
|
+
})).results);
|
|
23
|
+
const mcp = await mcpInvoke(options.mcp, AbortSignal.timeout(deadline)).catch(() => void 0);
|
|
16
24
|
if (mcp !== void 0) {
|
|
17
|
-
results.push(...(await checkProperties("mcp", mcp.invoke)).results);
|
|
25
|
+
results.push(...(await checkProperties("mcp", mcp.invoke, deadline)).results);
|
|
18
26
|
await mcp.close();
|
|
19
27
|
}
|
|
20
28
|
}
|
|
21
29
|
if (options.http !== void 0) {
|
|
22
|
-
results.push(...(await checkHttp(options.http, {
|
|
23
|
-
|
|
30
|
+
results.push(...(await checkHttp(options.http, {
|
|
31
|
+
maximumBytes,
|
|
32
|
+
deadline
|
|
33
|
+
})).results);
|
|
34
|
+
results.push(...(await checkProperties("http", httpInvoke(options.http), deadline)).results);
|
|
24
35
|
}
|
|
25
|
-
if (options.cli !== void 0) results.push(...(await checkCli(options.cli)).results);
|
|
36
|
+
if (options.cli !== void 0) results.push(...(await checkCli(options.cli, { deadline })).results);
|
|
26
37
|
if (options.webmcp !== void 0) {
|
|
27
38
|
const driver = typeof options.webmcp === "string" ? await browserDriver(options.webmcp) : await processDriver(options.webmcp);
|
|
28
39
|
try {
|
|
29
|
-
results.push(...(await checkWebMcp(driver)).results);
|
|
40
|
+
results.push(...(await checkWebMcp(driver, { deadline })).results);
|
|
30
41
|
} finally {
|
|
31
42
|
await driver.close();
|
|
32
43
|
}
|
|
@@ -34,4 +45,4 @@ async function conform(options) {
|
|
|
34
45
|
return results;
|
|
35
46
|
}
|
|
36
47
|
//#endregion
|
|
37
|
-
export { conform, fixture, serve };
|
|
48
|
+
export { DEADLINE, conform, fixture, serve };
|
package/dist/conform/mcp.d.ts
CHANGED
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
import { Client } from "@modelcontextprotocol/client";
|
|
6
6
|
import { Checks } from "./check.js";
|
|
7
7
|
/** A connected MCP client. */
|
|
8
|
-
export declare function mcpClient(url: string): Promise<Client>;
|
|
8
|
+
export declare function mcpClient(url: string, signal?: AbortSignal): Promise<Client>;
|
|
9
9
|
export declare function checkMcp(url: string, options: {
|
|
10
10
|
readonly maximumBytes: number;
|
|
11
|
+
readonly deadline: number;
|
|
11
12
|
}): Promise<Checks>;
|