@kryptosai/mcp-observatory 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -0
- package/dist/src/adapters/http.d.ts +2 -2
- package/dist/src/adapters/http.js +12 -3
- package/dist/src/adapters/http.js.map +1 -1
- package/dist/src/adapters/local-process.d.ts +7 -1
- package/dist/src/adapters/local-process.js +9 -3
- package/dist/src/adapters/local-process.js.map +1 -1
- package/dist/src/cassette.d.ts +22 -0
- package/dist/src/cassette.js +28 -0
- package/dist/src/cassette.js.map +1 -0
- package/dist/src/checks/tools-invoke.js +9 -0
- package/dist/src/checks/tools-invoke.js.map +1 -1
- package/dist/src/cli.js +175 -0
- package/dist/src/cli.js.map +1 -1
- package/dist/src/diff.js +39 -0
- package/dist/src/diff.js.map +1 -1
- package/dist/src/index.d.ts +5 -1
- package/dist/src/index.js +5 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/reporters/terminal.js +8 -2
- package/dist/src/reporters/terminal.js.map +1 -1
- package/dist/src/runner.d.ts +16 -0
- package/dist/src/runner.js +28 -5
- package/dist/src/runner.js.map +1 -1
- package/dist/src/transport/recording-transport.d.ts +22 -0
- package/dist/src/transport/recording-transport.js +91 -0
- package/dist/src/transport/recording-transport.js.map +1 -0
- package/dist/src/transport/replay-transport.d.ts +27 -0
- package/dist/src/transport/replay-transport.js +64 -0
- package/dist/src/transport/replay-transport.js.map +1 -0
- package/dist/src/types.d.ts +8 -0
- package/dist/src/verify.d.ts +21 -0
- package/dist/src/verify.js +98 -0
- package/dist/src/verify.js.map +1 -0
- package/package.json +7 -2
package/README.md
CHANGED
|
@@ -70,6 +70,9 @@ Or add it manually to your config:
|
|
|
70
70
|
| `scan` | Auto-discover servers from config files and check them all (default) |
|
|
71
71
|
| `scan deep` | Scan and also invoke safe tools to verify they execute |
|
|
72
72
|
| `test <cmd>` | Test a specific server by command |
|
|
73
|
+
| `record <cmd>` | Record a server session to a cassette file for offline replay |
|
|
74
|
+
| `replay <cassette>` | Replay a cassette offline — no live server needed |
|
|
75
|
+
| `verify <cassette> <cmd>` | Verify a live server still matches a recorded cassette |
|
|
73
76
|
| `diff <base> <head>` | Compare two run artifacts for regressions and schema drift |
|
|
74
77
|
| `watch <config>` | Watch a server for changes, alert on regressions |
|
|
75
78
|
| `serve` | Start as an MCP server for AI agents |
|
|
@@ -92,6 +95,19 @@ npx @kryptosai/mcp-observatory diff run-a.json run-b.json
|
|
|
92
95
|
|
|
93
96
|
**Recommend servers** — scans your project for languages, frameworks, databases, and cloud providers, then cross-references the [MCP registry](https://registry.modelcontextprotocol.io) to suggest servers you're missing. Ask your agent "what MCP servers should I add?" and it figures out the rest.
|
|
94
97
|
|
|
98
|
+
**Record / replay / verify** — capture a live session, replay it offline in CI, and verify nothing changed. Like [VCR](https://github.com/vcr/vcr) for MCP.
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
# Record a session
|
|
102
|
+
npx @kryptosai/mcp-observatory record npx -y @modelcontextprotocol/server-everything
|
|
103
|
+
|
|
104
|
+
# Replay offline (no server needed)
|
|
105
|
+
npx @kryptosai/mcp-observatory replay .mcp-observatory/cassettes/latest.cassette.json
|
|
106
|
+
|
|
107
|
+
# Verify the live server still matches
|
|
108
|
+
npx @kryptosai/mcp-observatory verify cassette.json npx -y @modelcontextprotocol/server-everything
|
|
109
|
+
```
|
|
110
|
+
|
|
95
111
|
**Watch for regressions** — re-runs checks on an interval and alerts when something changes.
|
|
96
112
|
|
|
97
113
|
```bash
|
|
@@ -165,6 +181,34 @@ npx @kryptosai/mcp-observatory run --target ./target.json
|
|
|
165
181
|
}
|
|
166
182
|
```
|
|
167
183
|
|
|
184
|
+
## How It Compares
|
|
185
|
+
|
|
186
|
+
| Feature | Observatory | [mcp-recorder](https://github.com/punkpeye/mcp-recorder) | [MCPBench](https://github.com/QuantGeekDev/mcpbench) | [mcp-jest](https://github.com/nicobailon/mcp-jest) |
|
|
187
|
+
|---------|:-----------:|:----------:|:-------:|:-------:|
|
|
188
|
+
| Auto-discover servers | ✅ | — | — | — |
|
|
189
|
+
| Check capabilities | ✅ | — | ✅ | ✅ |
|
|
190
|
+
| Invoke tools | ✅ | — | — | ✅ |
|
|
191
|
+
| Schema drift detection | ✅ | — | — | — |
|
|
192
|
+
| Record / replay | ✅ | ✅ | — | — |
|
|
193
|
+
| Verify against cassette | ✅ | — | — | — |
|
|
194
|
+
| Response snapshot diffs | ✅ | — | — | — |
|
|
195
|
+
| Benchmarking / latency | — | — | ✅ | — |
|
|
196
|
+
| Jest integration | — | — | — | ✅ |
|
|
197
|
+
| MCP proxy mode | — | ✅ | — | — |
|
|
198
|
+
| Works as MCP server | ✅ | — | — | — |
|
|
199
|
+
|
|
200
|
+
Each tool has strengths. Observatory focuses on regression detection and CI-friendly workflows. mcp-recorder is great as a transparent proxy. MCPBench is the go-to for performance benchmarking. mcp-jest is ideal if you're already in a Jest workflow.
|
|
201
|
+
|
|
202
|
+
## Prior Art
|
|
203
|
+
|
|
204
|
+
The record/replay/verify pattern is inspired by:
|
|
205
|
+
|
|
206
|
+
- [VCR](https://github.com/vcr/vcr) (Ruby) — pioneered cassette-based HTTP record/replay
|
|
207
|
+
- [Polly.js](https://github.com/Netflix/pollyjs) (Netflix) — HTTP interaction recording for JavaScript
|
|
208
|
+
- [mcp-recorder](https://github.com/punkpeye/mcp-recorder) — MCP-specific traffic recording proxy
|
|
209
|
+
- [MCPBench](https://github.com/QuantGeekDev/mcpbench) — MCP server benchmarking
|
|
210
|
+
- [mcp-jest](https://github.com/nicobailon/mcp-jest) — Jest-style testing for MCP servers
|
|
211
|
+
|
|
168
212
|
## Limitations
|
|
169
213
|
|
|
170
214
|
- Servers requiring interactive OAuth (e.g., Google Drive) need pre-authentication before Observatory can connect
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { HttpTargetConfig } from "../types.js";
|
|
2
|
-
import type { AdapterSession } from "./local-process.js";
|
|
2
|
+
import type { AdapterConnectOptions, AdapterSession } from "./local-process.js";
|
|
3
3
|
export declare class HttpAdapter {
|
|
4
|
-
connect(target: HttpTargetConfig): Promise<AdapterSession>;
|
|
4
|
+
connect(target: HttpTargetConfig, options?: AdapterConnectOptions): Promise<AdapterSession>;
|
|
5
5
|
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
2
2
|
import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
|
|
3
3
|
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
4
|
+
import { RecordingTransport } from "../transport/recording-transport.js";
|
|
4
5
|
import { formatConnectionFailureDiagnosis } from "../utils/failure-diagnosis.js";
|
|
5
6
|
import { TOOL_VERSION } from "../version.js";
|
|
6
7
|
export class HttpAdapter {
|
|
7
|
-
async connect(target) {
|
|
8
|
+
async connect(target, options) {
|
|
8
9
|
const headers = { ...(target.headers ?? {}) };
|
|
9
10
|
if (target.authToken) {
|
|
10
11
|
headers["Authorization"] = `Bearer ${target.authToken}`;
|
|
@@ -15,9 +16,13 @@ export class HttpAdapter {
|
|
|
15
16
|
const timeoutMs = target.timeoutMs ?? 15_000;
|
|
16
17
|
// Try streamable-http first, fall back to SSE
|
|
17
18
|
let connected = false;
|
|
19
|
+
let activeTransport;
|
|
18
20
|
try {
|
|
19
|
-
|
|
21
|
+
let transport = new StreamableHTTPClientTransport(url, { requestInit: { headers } });
|
|
22
|
+
if (options?.record)
|
|
23
|
+
transport = new RecordingTransport(transport);
|
|
20
24
|
await client.connect(transport, { timeout: timeoutMs });
|
|
25
|
+
activeTransport = transport;
|
|
21
26
|
connected = true;
|
|
22
27
|
}
|
|
23
28
|
catch {
|
|
@@ -25,8 +30,11 @@ export class HttpAdapter {
|
|
|
25
30
|
}
|
|
26
31
|
if (!connected) {
|
|
27
32
|
try {
|
|
28
|
-
|
|
33
|
+
let transport = new SSEClientTransport(url, { requestInit: { headers } });
|
|
34
|
+
if (options?.record)
|
|
35
|
+
transport = new RecordingTransport(transport);
|
|
29
36
|
await client.connect(transport, { timeout: timeoutMs });
|
|
37
|
+
activeTransport = transport;
|
|
30
38
|
connected = true;
|
|
31
39
|
}
|
|
32
40
|
catch (error) {
|
|
@@ -45,6 +53,7 @@ export class HttpAdapter {
|
|
|
45
53
|
serverName: serverVersion?.name,
|
|
46
54
|
serverVersion: serverVersion?.version,
|
|
47
55
|
stderrLines,
|
|
56
|
+
transport: options?.record ? activeTransport : undefined,
|
|
48
57
|
close: async () => {
|
|
49
58
|
await client.close();
|
|
50
59
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.js","sourceRoot":"","sources":["../../../src/adapters/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,yCAAyC,CAAC;AAC7E,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;
|
|
1
|
+
{"version":3,"file":"http.js","sourceRoot":"","sources":["../../../src/adapters/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,yCAAyC,CAAC;AAC7E,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAInG,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,gCAAgC,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAG7C,MAAM,OAAO,WAAW;IACtB,KAAK,CAAC,OAAO,CAAC,MAAwB,EAAE,OAA+B;QACrE,MAAM,OAAO,GAA2B,EAAE,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,MAAM,CAAC,SAAS,EAAE,CAAC;QAC1D,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,YAAY,EAAE,EAClD,EAAE,YAAY,EAAE,EAAE,EAAE,CACrB,CAAC;QAEF,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC;QAE7C,8CAA8C;QAC9C,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,eAAsC,CAAC;QAC3C,IAAI,CAAC;YACH,IAAI,SAAS,GAAc,IAAI,6BAA6B,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;YAChG,IAAI,OAAO,EAAE,MAAM;gBAAE,SAAS,GAAG,IAAI,kBAAkB,CAAC,SAAS,CAAC,CAAC;YACnE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;YACxD,eAAe,GAAG,SAAS,CAAC;YAC5B,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,WAAW,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,IAAI,SAAS,GAAc,IAAI,kBAAkB,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;gBACrF,IAAI,OAAO,EAAE,MAAM;oBAAE,SAAS,GAAG,IAAI,kBAAkB,CAAC,SAAS,CAAC,CAAC;gBACnE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;gBACxD,eAAe,GAAG,SAAS,CAAC;gBAC5B,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,UAAU,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC1E,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC5C,MAAM,GAAG,GAAG,gCAAgC,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;gBAC9E,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC3B,GAAG,CAAC,IAAI,GAAG,qBAAqB,CAAC;gBACjC,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAEhD,OAAO;YACL,MAAM;YACN,kBAAkB,EAAE,MAAM,CAAC,qBAAqB,EAAE;YAClD,UAAU,EAAE,aAAa,EAAE,IAAI;YAC/B,aAAa,EAAE,aAAa,EAAE,OAAO;YACrC,WAAW;YACX,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;YACxD,KAAK,EAAE,KAAK,IAAI,EAAE;gBAChB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;SACF,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1,12 +1,18 @@
|
|
|
1
1
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
2
2
|
import type { ServerCapabilities } from "@modelcontextprotocol/sdk/types.js";
|
|
3
|
+
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
|
|
3
4
|
import type { LocalProcessTargetConfig, TargetConfig } from "../types.js";
|
|
5
|
+
export interface AdapterConnectOptions {
|
|
6
|
+
record?: boolean;
|
|
7
|
+
}
|
|
4
8
|
export interface AdapterSession {
|
|
5
9
|
client: Client;
|
|
6
10
|
serverCapabilities?: ServerCapabilities;
|
|
7
11
|
serverName?: string;
|
|
8
12
|
serverVersion?: string;
|
|
9
13
|
stderrLines: string[];
|
|
14
|
+
/** The transport used for the connection. Present when recording. */
|
|
15
|
+
transport?: Transport;
|
|
10
16
|
close(): Promise<void>;
|
|
11
17
|
}
|
|
12
18
|
export declare class AdapterConnectError extends Error {
|
|
@@ -15,5 +21,5 @@ export declare class AdapterConnectError extends Error {
|
|
|
15
21
|
constructor(target: TargetConfig, rawMessage: string, stderrLines: string[]);
|
|
16
22
|
}
|
|
17
23
|
export declare class LocalProcessAdapter {
|
|
18
|
-
connect(target: LocalProcessTargetConfig): Promise<AdapterSession>;
|
|
24
|
+
connect(target: LocalProcessTargetConfig, options?: AdapterConnectOptions): Promise<AdapterSession>;
|
|
19
25
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
2
2
|
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
3
|
+
import { RecordingTransport } from "../transport/recording-transport.js";
|
|
3
4
|
import { formatConnectionFailureDiagnosis } from "../utils/failure-diagnosis.js";
|
|
4
5
|
import { TOOL_VERSION } from "../version.js";
|
|
5
6
|
export class AdapterConnectError extends Error {
|
|
@@ -13,8 +14,8 @@ export class AdapterConnectError extends Error {
|
|
|
13
14
|
}
|
|
14
15
|
}
|
|
15
16
|
export class LocalProcessAdapter {
|
|
16
|
-
async connect(target) {
|
|
17
|
-
const
|
|
17
|
+
async connect(target, options) {
|
|
18
|
+
const stdioTransport = new StdioClientTransport({
|
|
18
19
|
command: target.command,
|
|
19
20
|
args: target.args,
|
|
20
21
|
cwd: target.cwd,
|
|
@@ -22,7 +23,7 @@ export class LocalProcessAdapter {
|
|
|
22
23
|
stderr: "pipe"
|
|
23
24
|
});
|
|
24
25
|
const stderrLines = [];
|
|
25
|
-
|
|
26
|
+
stdioTransport.stderr?.on("data", (chunk) => {
|
|
26
27
|
const text = typeof chunk === "string" ? chunk : chunk.toString("utf8");
|
|
27
28
|
for (const line of text.split(/\r?\n/)) {
|
|
28
29
|
const trimmed = line.trim();
|
|
@@ -31,6 +32,10 @@ export class LocalProcessAdapter {
|
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
});
|
|
35
|
+
// Optionally wrap in RecordingTransport
|
|
36
|
+
const transport = options?.record
|
|
37
|
+
? new RecordingTransport(stdioTransport)
|
|
38
|
+
: stdioTransport;
|
|
34
39
|
const client = new Client({
|
|
35
40
|
name: "mcp-observatory",
|
|
36
41
|
version: TOOL_VERSION
|
|
@@ -54,6 +59,7 @@ export class LocalProcessAdapter {
|
|
|
54
59
|
serverName: serverVersion?.name,
|
|
55
60
|
serverVersion: serverVersion?.version,
|
|
56
61
|
stderrLines,
|
|
62
|
+
transport: options?.record ? transport : undefined,
|
|
57
63
|
close: async () => {
|
|
58
64
|
await client.close();
|
|
59
65
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-process.js","sourceRoot":"","sources":["../../../src/adapters/local-process.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;
|
|
1
|
+
{"version":3,"file":"local-process.js","sourceRoot":"","sources":["../../../src/adapters/local-process.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,gCAAgC,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAiB7C,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IACnC,UAAU,CAAS;IACnB,WAAW,CAAW;IAE/B,YAAY,MAAoB,EAAE,UAAkB,EAAE,WAAqB;QACzE,KAAK,CAAC,gCAAgC,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC;QACzE,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;QAClC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;CACF;AAED,MAAM,OAAO,mBAAmB;IAC9B,KAAK,CAAC,OAAO,CAAC,MAAgC,EAAE,OAA+B;QAC7E,MAAM,cAAc,GAAG,IAAI,oBAAoB,CAAC;YAC9C,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;QAEH,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAsB,EAAE,EAAE;YAC3D,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACxE,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC5B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,SAAS,GAAc,OAAO,EAAE,MAAM;YAC1C,CAAC,CAAC,IAAI,kBAAkB,CAAC,cAAc,CAAC;YACxC,CAAC,CAAC,cAAc,CAAC;QAEnB,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB;YACE,IAAI,EAAE,iBAAiB;YACvB,OAAO,EAAE,YAAY;SACtB,EACD;YACE,YAAY,EAAE,EAAE;SACjB,CACF,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE;gBAC9B,OAAO,EAAE,MAAM,CAAC,SAAS,IAAI,MAAM;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,UAAU,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC1E,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC5C,MAAM,IAAI,mBAAmB,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAEhD,OAAO;YACL,MAAM;YACN,kBAAkB,EAAE,MAAM,CAAC,qBAAqB,EAAE;YAClD,UAAU,EAAE,aAAa,EAAE,IAAI;YAC/B,aAAa,EAAE,aAAa,EAAE,OAAO;YACrC,WAAW;YACX,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YAClD,KAAK,EAAE,KAAK,IAAI,EAAE;gBAChB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;SACF,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export interface CassetteEntry {
|
|
2
|
+
direction: "request" | "response" | "notification";
|
|
3
|
+
method: string;
|
|
4
|
+
params?: unknown;
|
|
5
|
+
result?: unknown;
|
|
6
|
+
error?: {
|
|
7
|
+
code: number;
|
|
8
|
+
message: string;
|
|
9
|
+
data?: unknown;
|
|
10
|
+
};
|
|
11
|
+
timestampMs: number;
|
|
12
|
+
}
|
|
13
|
+
export interface Cassette {
|
|
14
|
+
version: 1;
|
|
15
|
+
targetId: string;
|
|
16
|
+
recordedAt: string;
|
|
17
|
+
transport: "stdio" | "http";
|
|
18
|
+
entries: CassetteEntry[];
|
|
19
|
+
}
|
|
20
|
+
export declare function defaultCassettesDirectory(cwd?: string): string;
|
|
21
|
+
export declare function saveCassette(cassette: Cassette, outDir: string): Promise<string>;
|
|
22
|
+
export declare function loadCassette(filePath: string): Promise<Cassette>;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { ensureDirectory } from "./storage.js";
|
|
4
|
+
import { slugify } from "./utils/ids.js";
|
|
5
|
+
// ── I/O ──────────────────────────────────────────────────────────────────────
|
|
6
|
+
export function defaultCassettesDirectory(cwd = process.cwd()) {
|
|
7
|
+
return path.join(cwd, ".mcp-observatory", "cassettes");
|
|
8
|
+
}
|
|
9
|
+
export async function saveCassette(cassette, outDir) {
|
|
10
|
+
await ensureDirectory(outDir);
|
|
11
|
+
const timestamp = new Date().toISOString().replaceAll(":", "-");
|
|
12
|
+
const fileName = `${timestamp}--${slugify(cassette.targetId)}.cassette.json`;
|
|
13
|
+
const filePath = path.join(outDir, fileName);
|
|
14
|
+
await writeFile(filePath, JSON.stringify(cassette, null, 2) + "\n", "utf8");
|
|
15
|
+
return filePath;
|
|
16
|
+
}
|
|
17
|
+
export async function loadCassette(filePath) {
|
|
18
|
+
const content = await readFile(filePath, "utf8");
|
|
19
|
+
const data = JSON.parse(content);
|
|
20
|
+
if (typeof data !== "object" ||
|
|
21
|
+
data === null ||
|
|
22
|
+
data["version"] !== 1 ||
|
|
23
|
+
!Array.isArray(data["entries"])) {
|
|
24
|
+
throw new Error(`Invalid cassette file: ${filePath}. Expected { version: 1, entries: [...] }.`);
|
|
25
|
+
}
|
|
26
|
+
return data;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=cassette.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cassette.js","sourceRoot":"","sources":["../../src/cassette.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAqBzC,gFAAgF;AAEhF,MAAM,UAAU,yBAAyB,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAC3D,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,EAAE,WAAW,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAAkB,EAClB,MAAc;IAEd,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,GAAG,SAAS,KAAK,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAC7E,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5E,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACjD,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAE1C,IACE,OAAO,IAAI,KAAK,QAAQ;QACxB,IAAI,KAAK,IAAI;QACZ,IAAgC,CAAC,SAAS,CAAC,KAAK,CAAC;QAClD,CAAC,KAAK,CAAC,OAAO,CAAE,IAAgC,CAAC,SAAS,CAAC,CAAC,EAC5D,CAAC;QACD,MAAM,IAAI,KAAK,CACb,0BAA0B,QAAQ,4CAA4C,CAC/E,CAAC;IACJ,CAAC;IAED,OAAO,IAAgB,CAAC;AAC1B,CAAC"}
|
|
@@ -61,6 +61,7 @@ export async function runToolsInvokeCheck(context) {
|
|
|
61
61
|
responded: true,
|
|
62
62
|
isError,
|
|
63
63
|
error: isError ? JSON.stringify(response.content) : undefined,
|
|
64
|
+
responseContent: response.content,
|
|
64
65
|
});
|
|
65
66
|
}
|
|
66
67
|
catch (error) {
|
|
@@ -92,6 +93,13 @@ export async function runToolsInvokeCheck(context) {
|
|
|
92
93
|
(tools.length > safeTools.length
|
|
93
94
|
? ` (${tools.length - safeTools.length} skipped — have required params)`
|
|
94
95
|
: "");
|
|
96
|
+
// Build response snapshots for diffing
|
|
97
|
+
const responseSnapshots = {};
|
|
98
|
+
for (const r of results) {
|
|
99
|
+
if (r.responded && r.responseContent !== undefined) {
|
|
100
|
+
responseSnapshots[r.name] = r.responseContent;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
95
103
|
return {
|
|
96
104
|
result: makeCheckResult("tools-invoke", status, performance.now() - startedAt, message, [{
|
|
97
105
|
endpoint: "tools/call",
|
|
@@ -101,6 +109,7 @@ export async function runToolsInvokeCheck(context) {
|
|
|
101
109
|
itemCount: results.length,
|
|
102
110
|
identifiers: results.map((r) => r.name),
|
|
103
111
|
diagnostics,
|
|
112
|
+
responseSnapshots: Object.keys(responseSnapshots).length > 0 ? responseSnapshots : undefined,
|
|
104
113
|
}])
|
|
105
114
|
};
|
|
106
115
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools-invoke.js","sourceRoot":"","sources":["../../../src/checks/tools-invoke.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAI9C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzE,OAAO,EACL,sBAAsB,EACtB,eAAe,EAGhB,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"tools-invoke.js","sourceRoot":"","sources":["../../../src/checks/tools-invoke.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAI9C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzE,OAAO,EACL,sBAAsB,EACtB,eAAe,EAGhB,MAAM,WAAW,CAAC;AAWnB,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAqB;IAC7D,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IAE/E,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO;YACL,MAAM,EAAE,eAAe,CACrB,cAAc,EACd,aAAa,EACb,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,EAC7B,yCAAyC,EACzC,CAAC;oBACC,QAAQ,EAAE,YAAY;oBACtB,UAAU,EAAE,KAAK;oBACjB,SAAS,EAAE,KAAK;oBAChB,mBAAmB,EAAE,KAAK;oBAC1B,WAAW,EAAE,EAAE;iBAChB,CAAC,CACH;SACF,CAAC;IACJ,CAAC;IAED,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QACvF,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,OAAO;YACL,MAAM,EAAE,eAAe,CACrB,cAAc,EACd,MAAM,EACN,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,EAC7B,2CAA2C,GAAG,EAAE,EAChD,CAAC;oBACC,QAAQ,EAAE,YAAY;oBACtB,UAAU,EAAE,IAAI;oBAChB,SAAS,EAAE,KAAK;oBAChB,mBAAmB,EAAE,KAAK;oBAC1B,WAAW,EAAE,CAAC,GAAG,CAAC;iBACnB,CAAC,CACH;SACF,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACnC,cAAc,CAAC;QACb,WAAW,EAAE,CAAC,CAAC,WAAkD;QACjE,WAAW,EAAE,CAAC,CAAC,WAAkD;KAClE,CAAC,CACH,CAAC;IAEF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO;YACL,MAAM,EAAE,eAAe,CACrB,cAAc,EACd,MAAM,EACN,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,EAC7B,GAAG,KAAK,CAAC,MAAM,wGAAwG,EACvH,CAAC;oBACC,QAAQ,EAAE,YAAY;oBACtB,UAAU,EAAE,IAAI;oBAChB,SAAS,EAAE,IAAI;oBACf,mBAAmB,EAAE,IAAI;oBACzB,SAAS,EAAE,CAAC;oBACZ,WAAW,EAAE,EAAE;oBACf,WAAW,EAAE,EAAE;iBAChB,CAAC,CACH;SACF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,WAAkD,CAAC,CAAC;QACrF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,CAC5C,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,EACpC,SAAS,EACT,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,EAAE,CAC/B,CAAC;YACF,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,OAAO,EAAE,IAAI;gBACb,SAAS,EAAE,IAAI;gBACf,OAAO;gBACP,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC7D,eAAe,EAAE,QAAQ,CAAC,OAAO;aAClC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnE,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,OAAO,EAAE,IAAI;gBACb,SAAS,EAAE,KAAK;gBAChB,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,GAAG;aACX,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;IAChE,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC,CAAC;IAEhF,IAAI,MAAmC,CAAC;IACxC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,GAAG,MAAM,CAAC;IAClB,CAAC;SAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,GAAG,SAAS,CAAC;IACrB,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,MAAM,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GACX,WAAW,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,iBAAiB;QAC9D,GAAG,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,MAAM,SAAS;QAClD,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;YAC9B,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,kCAAkC;YACxE,CAAC,CAAC,EAAE,CAAC,CAAC;IAEV,uCAAuC;IACvC,MAAM,iBAAiB,GAA4B,EAAE,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACnD,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,eAAe,CAAC;QAChD,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE,eAAe,CACrB,cAAc,EACd,MAAM,EACN,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,EAC7B,OAAO,EACP,CAAC;gBACC,QAAQ,EAAE,YAAY;gBACtB,UAAU,EAAE,IAAI;gBAChB,SAAS,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;gBAC9B,mBAAmB,EAAE,IAAI;gBACzB,SAAS,EAAE,OAAO,CAAC,MAAM;gBACzB,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;gBACvC,WAAW;gBACX,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS;aAC7F,CAAC,CACH;KACF,CAAC;AACJ,CAAC"}
|
package/dist/src/cli.js
CHANGED
|
@@ -3,9 +3,21 @@ import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { Command } from "commander";
|
|
5
5
|
import { scanForTargets } from "./discovery.js";
|
|
6
|
+
import os from "node:os";
|
|
7
|
+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
8
|
+
import { defaultCassettesDirectory, loadCassette, saveCassette } from "./cassette.js";
|
|
9
|
+
import { runPromptsCheck } from "./checks/prompts.js";
|
|
10
|
+
import { runResourcesCheck } from "./checks/resources.js";
|
|
11
|
+
import { runToolsCheck } from "./checks/tools.js";
|
|
12
|
+
import { runToolsInvokeCheck } from "./checks/tools-invoke.js";
|
|
6
13
|
import { diffArtifacts, readArtifact, renderHtml, renderMarkdown, renderTerminal, runTarget, writeRunArtifact } from "./index.js";
|
|
14
|
+
import { runTargetRecording } from "./runner.js";
|
|
7
15
|
import { defaultRunsDirectory } from "./storage.js";
|
|
16
|
+
import { ReplayTransport } from "./transport/replay-transport.js";
|
|
17
|
+
import { SCHEMA_VERSION } from "./types.js";
|
|
18
|
+
import { buildRunId } from "./utils/ids.js";
|
|
8
19
|
import { validateTargetConfig } from "./validate.js";
|
|
20
|
+
import { compareResponses } from "./verify.js";
|
|
9
21
|
import { TOOL_VERSION } from "./version.js";
|
|
10
22
|
// ── ASCII Logo ──────────────────────────────────────────────────────────────
|
|
11
23
|
const LOGO = `
|
|
@@ -114,6 +126,8 @@ async function main() {
|
|
|
114
126
|
` ${c(ANSI.dim, "$")} ${bin}${" ".repeat(Math.max(1, 48 - bin.length))}Scan all your MCP servers`,
|
|
115
127
|
` ${c(ANSI.dim, "$")} ${bin} scan deep${" ".repeat(Math.max(1, 38 - bin.length))}Also test that tools actually run`,
|
|
116
128
|
` ${c(ANSI.dim, "$")} ${bin} test npx server-foo${" ".repeat(Math.max(1, 28 - bin.length))}Test a specific server by command`,
|
|
129
|
+
` ${c(ANSI.dim, "$")} ${bin} record npx server-foo${" ".repeat(Math.max(1, 26 - bin.length))}Record a session for replay`,
|
|
130
|
+
` ${c(ANSI.dim, "$")} ${bin} replay cassette.json${" ".repeat(Math.max(1, 27 - bin.length))}Replay offline — no server needed`,
|
|
117
131
|
` ${c(ANSI.dim, "$")} ${bin} diff run-a.json run-b.json${" ".repeat(Math.max(1, 20 - bin.length))}Compare two runs`,
|
|
118
132
|
"",
|
|
119
133
|
].join("\n"));
|
|
@@ -196,6 +210,167 @@ async function main() {
|
|
|
196
210
|
const { startServer } = await import("./server.js");
|
|
197
211
|
await startServer();
|
|
198
212
|
});
|
|
213
|
+
// ── record ─────────────────────────────────────────────────────────────
|
|
214
|
+
program
|
|
215
|
+
.command("record")
|
|
216
|
+
.description("Record a server session to a cassette file for replay.")
|
|
217
|
+
.argument("[command...]", "Server command and arguments to run.")
|
|
218
|
+
.option("--target <config>", "Path to a target config JSON file.")
|
|
219
|
+
.option("--no-color", "Disable colored output.")
|
|
220
|
+
.action(async (commandArgs, options) => {
|
|
221
|
+
const target = options.target
|
|
222
|
+
? await readTargetConfig(options.target)
|
|
223
|
+
: targetFromCommand(commandArgs.length > 0 ? commandArgs : getPassthroughArgs());
|
|
224
|
+
process.stdout.write(`${c(ANSI.dim, "⟳")} Recording session with ${c(ANSI.bold, target.targetId)}...\n`);
|
|
225
|
+
const { artifact, cassetteEntries } = await runTargetRecording(target, { invokeTools: true });
|
|
226
|
+
if (!cassetteEntries || cassetteEntries.length === 0) {
|
|
227
|
+
process.stdout.write(`${c(ANSI.yellow, "⚠")} No traffic recorded.\n`);
|
|
228
|
+
process.exitCode = 1;
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
const cassette = {
|
|
232
|
+
version: 1,
|
|
233
|
+
targetId: target.targetId,
|
|
234
|
+
recordedAt: new Date().toISOString(),
|
|
235
|
+
transport: target.adapter === "http" ? "http" : "stdio",
|
|
236
|
+
entries: cassetteEntries,
|
|
237
|
+
};
|
|
238
|
+
const cassettePath = await saveCassette(cassette, defaultCassettesDirectory(process.cwd()));
|
|
239
|
+
const summary = renderTerminal(artifact);
|
|
240
|
+
process.stdout.write(`\n${summary}\n`);
|
|
241
|
+
process.stdout.write(`\n${c(ANSI.green, "✓")} Cassette saved: ${cassettePath}\n`);
|
|
242
|
+
process.stdout.write(` ${c(ANSI.dim, `${cassetteEntries.length} entries recorded`)}\n`);
|
|
243
|
+
process.stdout.write(`\n Replay offline: ${c(ANSI.cyan, `${bin} replay ${cassettePath}`)}\n`);
|
|
244
|
+
process.stdout.write(` Verify live: ${c(ANSI.cyan, `${bin} verify ${cassettePath} ${target.adapter === "http" ? `--target <config>` : commandArgs.join(" ")}`)}\n\n`);
|
|
245
|
+
if (artifact.gate === "fail") {
|
|
246
|
+
process.exitCode = 1;
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
// ── replay ─────────────────────────────────────────────────────────────
|
|
250
|
+
program
|
|
251
|
+
.command("replay")
|
|
252
|
+
.description("Replay a cassette file offline — no live server needed.")
|
|
253
|
+
.argument("<cassette>", "Path to a cassette JSON file.")
|
|
254
|
+
.option("--no-color", "Disable colored output.")
|
|
255
|
+
.action(async (cassettePath) => {
|
|
256
|
+
const cassette = await loadCassette(cassettePath);
|
|
257
|
+
process.stdout.write(`${c(ANSI.dim, "⟳")} Replaying cassette for ${c(ANSI.bold, cassette.targetId)} (${cassette.entries.length} entries)...\n`);
|
|
258
|
+
// Create a target config for the replay
|
|
259
|
+
const replayTarget = {
|
|
260
|
+
targetId: cassette.targetId,
|
|
261
|
+
adapter: "local-process",
|
|
262
|
+
command: "replay",
|
|
263
|
+
args: [],
|
|
264
|
+
};
|
|
265
|
+
// Build a ReplayTransport and run checks against it
|
|
266
|
+
const transport = new ReplayTransport(cassette.entries);
|
|
267
|
+
const client = new Client({ name: "mcp-observatory", version: TOOL_VERSION }, { capabilities: {} });
|
|
268
|
+
await client.connect(transport);
|
|
269
|
+
const serverCapabilities = client.getServerCapabilities();
|
|
270
|
+
const checkContext = {
|
|
271
|
+
client,
|
|
272
|
+
serverCapabilities,
|
|
273
|
+
target: replayTarget,
|
|
274
|
+
timeoutMs: 10_000,
|
|
275
|
+
stderrLines: [],
|
|
276
|
+
};
|
|
277
|
+
const toolsCheck = await runToolsCheck(checkContext);
|
|
278
|
+
const promptsCheck = await runPromptsCheck(checkContext);
|
|
279
|
+
const resourcesCheck = await runResourcesCheck(checkContext);
|
|
280
|
+
const invokeCheck = await runToolsInvokeCheck(checkContext);
|
|
281
|
+
await client.close();
|
|
282
|
+
const checks = [
|
|
283
|
+
toolsCheck.result,
|
|
284
|
+
promptsCheck.result,
|
|
285
|
+
resourcesCheck.result,
|
|
286
|
+
invokeCheck.result,
|
|
287
|
+
];
|
|
288
|
+
const failCount = checks.filter((ch) => ch.status === "fail").length;
|
|
289
|
+
const gate = failCount > 0 ? "fail" : "pass";
|
|
290
|
+
const artifact = {
|
|
291
|
+
artifactType: "run",
|
|
292
|
+
schemaVersion: SCHEMA_VERSION,
|
|
293
|
+
gate,
|
|
294
|
+
runId: buildRunId(),
|
|
295
|
+
createdAt: new Date().toISOString(),
|
|
296
|
+
toolVersion: TOOL_VERSION,
|
|
297
|
+
target: {
|
|
298
|
+
targetId: cassette.targetId,
|
|
299
|
+
adapter: "local-process",
|
|
300
|
+
command: "replay",
|
|
301
|
+
args: [],
|
|
302
|
+
metadata: { source: "cassette", cassettePath },
|
|
303
|
+
},
|
|
304
|
+
environment: {
|
|
305
|
+
platform: `${os.platform()} ${os.release()}`,
|
|
306
|
+
nodeVersion: process.version,
|
|
307
|
+
},
|
|
308
|
+
summary: {
|
|
309
|
+
total: checks.length,
|
|
310
|
+
pass: checks.filter((ch) => ch.status === "pass").length,
|
|
311
|
+
fail: failCount,
|
|
312
|
+
partial: checks.filter((ch) => ch.status === "partial").length,
|
|
313
|
+
unsupported: checks.filter((ch) => ch.status === "unsupported").length,
|
|
314
|
+
flaky: checks.filter((ch) => ch.status === "flaky").length,
|
|
315
|
+
skipped: checks.filter((ch) => ch.status === "skipped").length,
|
|
316
|
+
gate,
|
|
317
|
+
},
|
|
318
|
+
checks,
|
|
319
|
+
};
|
|
320
|
+
process.stdout.write(`\n${renderTerminal(artifact)}\n`);
|
|
321
|
+
process.stdout.write(`\n${c(ANSI.dim, `Replayed from: ${cassettePath}`)}\n\n`);
|
|
322
|
+
if (artifact.gate === "fail") {
|
|
323
|
+
process.exitCode = 1;
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
// ── verify ─────────────────────────────────────────────────────────────
|
|
327
|
+
program
|
|
328
|
+
.command("verify")
|
|
329
|
+
.description("Verify a live server still matches a recorded cassette.")
|
|
330
|
+
.argument("<cassette>", "Path to a cassette JSON file.")
|
|
331
|
+
.argument("[command...]", "Server command and arguments to run.")
|
|
332
|
+
.option("--target <config>", "Path to a target config JSON file.")
|
|
333
|
+
.option("--no-color", "Disable colored output.")
|
|
334
|
+
.action(async (cassettePath, commandArgs, options) => {
|
|
335
|
+
const cassette = await loadCassette(cassettePath);
|
|
336
|
+
const target = options.target
|
|
337
|
+
? await readTargetConfig(options.target)
|
|
338
|
+
: targetFromCommand(commandArgs.length > 0 ? commandArgs : getPassthroughArgs());
|
|
339
|
+
process.stdout.write(`${c(ANSI.dim, "⟳")} Verifying ${c(ANSI.bold, target.targetId)} against cassette...\n`);
|
|
340
|
+
const { cassetteEntries } = await runTargetRecording(target, { invokeTools: true });
|
|
341
|
+
if (!cassetteEntries) {
|
|
342
|
+
process.stdout.write(`${c(ANSI.red, "✗")} Failed to record live session for comparison.\n`);
|
|
343
|
+
process.exitCode = 1;
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
const verifyResult = compareResponses(cassette, cassetteEntries);
|
|
347
|
+
process.stdout.write("\n");
|
|
348
|
+
for (const entry of verifyResult.entries) {
|
|
349
|
+
if (entry.status === "pass") {
|
|
350
|
+
process.stdout.write(` ${c(ANSI.green, "✓")} ${entry.method}\n`);
|
|
351
|
+
}
|
|
352
|
+
else if (entry.status === "fail") {
|
|
353
|
+
process.stdout.write(` ${c(ANSI.red, "✗")} ${entry.method}\n`);
|
|
354
|
+
if (entry.diff) {
|
|
355
|
+
for (const line of entry.diff.split("\n")) {
|
|
356
|
+
process.stdout.write(` ${c(ANSI.dim, line)}\n`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
else {
|
|
361
|
+
process.stdout.write(` ${c(ANSI.yellow, "?")} ${entry.method} ${c(ANSI.dim, "(missing — server did not respond)")}\n`);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
process.stdout.write("\n");
|
|
365
|
+
if (verifyResult.failed === 0 && verifyResult.missing === 0) {
|
|
366
|
+
process.stdout.write(c(ANSI.green, ` ✓ All ${verifyResult.passed} responses match cassette\n`));
|
|
367
|
+
}
|
|
368
|
+
else {
|
|
369
|
+
process.stdout.write(c(ANSI.red, ` ✗ ${verifyResult.failed} changed, ${verifyResult.missing} missing out of ${verifyResult.totalEntries} responses\n`));
|
|
370
|
+
process.exitCode = 1;
|
|
371
|
+
}
|
|
372
|
+
process.stdout.write("\n");
|
|
373
|
+
});
|
|
199
374
|
// ── Hidden legacy commands ────────────────────────────────────────────
|
|
200
375
|
program
|
|
201
376
|
.command("run", { hidden: true })
|