mcpspec 1.0.3 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +480 -25
  2. package/dist/index.js +726 -5
  3. package/package.json +4 -4
package/README.md CHANGED
@@ -1,5 +1,5 @@
1
1
  <p align="center">
2
- <img src="https://raw.githubusercontent.com/light-handle/mcpspec/main/mcpspec.png" alt="MCPSpec" width="200" />
2
+ <img src="mcpspec.png" alt="MCPSpec" width="200" />
3
3
  </p>
4
4
 
5
5
  <h1 align="center">MCPSpec</h1>
@@ -24,10 +24,13 @@
24
24
  ```bash
25
25
  mcpspec test ./collection.yaml # Run tests
26
26
  mcpspec inspect "npx my-server" # Interactive REPL
27
- mcpspec audit "npx my-server" # Security scan
27
+ mcpspec audit "npx my-server" # Security scan (8 rules)
28
28
  mcpspec bench "npx my-server" # Performance benchmark
29
29
  mcpspec score "npx my-server" # Quality rating (0-100)
30
30
  mcpspec docs "npx my-server" # Auto-generate documentation
31
+ mcpspec record start "npx my-server" # Record & replay sessions
32
+ mcpspec mock my-recording # Start mock server from recording
33
+ mcpspec ci-init --platform github # Generate CI pipeline config
31
34
  mcpspec ui # Launch web dashboard
32
35
  ```
33
36
 
@@ -42,20 +45,460 @@ mcpspec init --template standard
42
45
 
43
46
  # 3. Run tests
44
47
  mcpspec test
48
+
49
+ # 4. Add CI gating (optional)
50
+ mcpspec ci-init
45
51
  ```
46
52
 
47
53
  ## Features
48
54
 
49
- | | Feature | Description |
50
- |---|---|---|
51
- | **Test Collections** | YAML-based test suites with 10 assertion types, environments, variables, tags, retries, and parallel execution |
52
- | **Interactive Inspector** | Connect to any MCP server and explore tools, resources, and schemas in a live REPL |
53
- | **Security Audit** | Scan for path traversal, injection, auth bypass, resource exhaustion, and info disclosure. Safety filter auto-skips destructive tools; `--dry-run` previews targets |
54
- | **Benchmarks** | Measure min/max/mean/median/P95/P99 latency and throughput across hundreds of iterations |
55
- | **MCP Score** | 0-100 quality rating across documentation, schema quality, error handling, responsiveness, and security |
56
- | **Doc Generator** | Auto-generate Markdown or HTML documentation from server introspection |
57
- | **Web Dashboard** | Full React UI with server management, test runner, audit viewer, and dark mode |
58
- | **CI/CD Ready** | JUnit/JSON/TAP reporters, deterministic exit codes, `--ci` mode, GitHub Actions compatible |
55
+ ### Test Collections
56
+
57
+ Write tests in YAML with 10 assertion types, environments, variable extraction, tags, retries, and parallel execution.
58
+
59
+ ```yaml
60
+ name: Filesystem Tests
61
+ server: npx @modelcontextprotocol/server-filesystem /tmp
62
+
63
+ tests:
64
+ - name: Read a file
65
+ call: read_file
66
+ with:
67
+ path: /tmp/test.txt
68
+ expect:
69
+ - exists: $.content
70
+ - type: [$.content, string]
71
+
72
+ - name: Handle missing file
73
+ call: read_file
74
+ with:
75
+ path: /tmp/nonexistent.txt
76
+ expectError: true
77
+ ```
78
+
79
+ **Advanced features:**
80
+
81
+ ```yaml
82
+ schemaVersion: "1.0"
83
+ name: Advanced Tests
84
+
85
+ server:
86
+ command: npx
87
+ args: ["my-mcp-server"]
88
+ env:
89
+ NODE_ENV: test
90
+
91
+ environments:
92
+ dev:
93
+ variables:
94
+ BASE_PATH: /tmp/dev
95
+ staging:
96
+ variables:
97
+ BASE_PATH: /tmp/staging
98
+
99
+ defaultEnvironment: dev
100
+
101
+ tests:
102
+ - id: create-data
103
+ name: Create data
104
+ tags: [smoke, write]
105
+ timeout: 5000
106
+ retries: 2
107
+ call: create_item
108
+ with:
109
+ name: "test-item"
110
+ assertions:
111
+ - type: schema
112
+ - type: exists
113
+ path: $.id
114
+ - type: latency
115
+ maxMs: 1000
116
+ extract:
117
+ - name: itemId
118
+ path: $.id
119
+
120
+ - id: verify-data
121
+ name: Verify created data
122
+ tags: [smoke, read]
123
+ call: get_item
124
+ with:
125
+ id: "{{itemId}}"
126
+ assertions:
127
+ - type: equals
128
+ path: $.name
129
+ value: "test-item"
130
+ - type: expression
131
+ expr: "response.id == itemId"
132
+ ```
133
+
134
+ **Assertion types:**
135
+
136
+ | Type | Description | Example |
137
+ |------|-------------|---------|
138
+ | `schema` | Validate response structure | `type: schema` |
139
+ | `equals` | Exact match (deep comparison) | `path: $.id, value: 123` |
140
+ | `contains` | Array or string contains value | `path: $.tags, value: "active"` |
141
+ | `exists` | Path exists and is not null | `path: $.name` |
142
+ | `matches` | Regex pattern match | `path: $.email, pattern: ".*@.*"` |
143
+ | `type` | Type check | `path: $.count, expected: number` |
144
+ | `length` | Array/string length | `path: $.items, operator: gt, value: 0` |
145
+ | `latency` | Response time threshold | `maxMs: 1000` |
146
+ | `mimeType` | Content type validation | `expected: "image/png"` |
147
+ | `expression` | Safe expression eval | `expr: "response.total > 0"` |
148
+
149
+ Expressions use [expr-eval](https://github.com/silentmatt/expr-eval) — comparisons, logical operators, property access, math. No arbitrary code execution.
150
+
151
+ **Shorthand format** for common assertions:
152
+
153
+ ```yaml
154
+ expect:
155
+ - exists: $.field
156
+ - equals: [$.id, 123]
157
+ - contains: [$.tags, "active"]
158
+ - matches: [$.email, ".*@.*"]
159
+ ```
160
+
161
+ **Run options:**
162
+
163
+ ```bash
164
+ mcpspec test ./tests.yaml # Specific file
165
+ mcpspec test --env staging # Switch environment
166
+ mcpspec test --tag @smoke # Filter by tag
167
+ mcpspec test --parallel 4 # Parallel execution
168
+ mcpspec test --reporter junit --output results.xml
169
+ mcpspec test --baseline main # Compare against baseline
170
+ mcpspec test --watch # Re-run on file changes
171
+ mcpspec test --ci # CI mode (no colors)
172
+ ```
173
+
174
+ **Reporters:** console (default), json, junit, html, tap.
175
+
176
+ ---
177
+
178
+ ### Interactive Inspector
179
+
180
+ Connect to any MCP server and explore its capabilities in a live REPL.
181
+
182
+ ```bash
183
+ mcpspec inspect "npx @modelcontextprotocol/server-filesystem /tmp"
184
+ ```
185
+
186
+ | Command | Description |
187
+ |---------|-------------|
188
+ | `.tools` | List all available tools with descriptions |
189
+ | `.resources` | List all available resources (URIs) |
190
+ | `.call <tool> <json>` | Call a tool with JSON input |
191
+ | `.schema <tool>` | Display tool's JSON Schema input spec |
192
+ | `.info` | Show server info (name, version, capabilities) |
193
+ | `.help` | Show help |
194
+ | `.exit` | Disconnect and exit |
195
+
196
+ ```
197
+ mcpspec> .tools
198
+ read_file Read complete contents of a file
199
+ write_file Create or overwrite a file
200
+ list_directory List directory contents
201
+
202
+ mcpspec> .call read_file {"path": "/tmp/test.txt"}
203
+ {
204
+ "content": "Hello, world!"
205
+ }
206
+ ```
207
+
208
+ ---
209
+
210
+ ### Security Audit
211
+
212
+ 8 security rules covering traditional vulnerabilities and LLM-specific threats. A safety filter auto-skips destructive tools, and `--dry-run` previews targets before scanning.
213
+
214
+ ```bash
215
+ mcpspec audit "npx my-server" # Passive (safe)
216
+ mcpspec audit "npx my-server" --mode active # Active probing
217
+ mcpspec audit "npx my-server" --fail-on medium # CI gate
218
+ mcpspec audit "npx my-server" --exclude-tools delete # Skip tools
219
+ mcpspec audit "npx my-server" --dry-run # Preview targets
220
+ ```
221
+
222
+ **Security rules:**
223
+
224
+ | Rule | Mode | What it detects |
225
+ |------|------|-----------------|
226
+ | Path Traversal | Passive | `../../etc/passwd` style directory escape attacks |
227
+ | Input Validation | Passive | Missing constraints (enum, pattern, min/max) on tool inputs |
228
+ | Info Disclosure | Passive | Leaked paths, stack traces, API keys in tool descriptions |
229
+ | Tool Poisoning | Passive | LLM prompt injection in descriptions, hidden Unicode, cross-tool manipulation |
230
+ | Excessive Agency | Passive | Destructive tools without confirmation params, arbitrary code execution |
231
+ | Resource Exhaustion | Active | Unbounded loops, large allocations, recursion |
232
+ | Auth Bypass | Active | Missing auth checks, hardcoded credentials |
233
+ | Injection | Active | SQL and command injection in tool inputs |
234
+
235
+ **Scan modes:**
236
+
237
+ - **Passive** (default) — 5 rules, analyzes metadata only, no tool calls. Safe for production.
238
+ - **Active** — All 8 rules, sends test payloads. Requires confirmation prompt.
239
+ - **Aggressive** — All 8 rules with more exhaustive probing. Requires confirmation prompt.
240
+
241
+ Active/aggressive modes auto-skip tools matching destructive patterns (`delete_*`, `drop_*`, `destroy_*`, etc.) and require explicit confirmation unless `--acknowledge-risk` is passed.
242
+
243
+ Each finding includes severity (info/low/medium/high/critical), description, evidence, and remediation advice.
244
+
245
+ ---
246
+
247
+ ### Recording & Replay
248
+
249
+ Record inspector sessions, save them, and replay against the same or different server versions. Diff output highlights regressions.
250
+
251
+ ```bash
252
+ # Record a session
253
+ mcpspec record start "npx my-server"
254
+ mcpspec> .call get_user {"id": "1"}
255
+ mcpspec> .call list_items {}
256
+ mcpspec> .save my-session
257
+
258
+ # Later: replay against new server version
259
+ mcpspec record replay my-session "npx my-server-v2"
260
+ ```
261
+
262
+ **Replay output:**
263
+
264
+ ```
265
+ Replaying 3 steps against my-server-v2...
266
+
267
+ 1/3 get_user............. [OK] 42ms
268
+ 2/3 list_items........... [CHANGED] 38ms
269
+ 3/3 create_item.......... [OK] 51ms
270
+
271
+ Summary: 2 matched, 1 changed, 0 added, 0 removed
272
+ ```
273
+
274
+ **Manage recordings:**
275
+
276
+ ```bash
277
+ mcpspec record list # List saved recordings
278
+ mcpspec record delete my-session # Delete a recording
279
+ ```
280
+
281
+ Recordings are stored in `~/.mcpspec/recordings/` and include tool names, inputs, outputs, timing, and error states for each step.
282
+
283
+ ---
284
+
285
+ ### Mock Server
286
+
287
+ Turn any recording into a mock MCP server — a drop-in replacement for the real server. Useful for CI/CD without real dependencies, offline development, and deterministic tests.
288
+
289
+ ```bash
290
+ # Start mock server from a recording (stdio transport)
291
+ mcpspec mock my-api
292
+
293
+ # Use as a server in test collections
294
+ mcpspec test --server "mcpspec mock my-api" ./tests.yaml
295
+
296
+ # Generate standalone .js file (only needs @modelcontextprotocol/sdk)
297
+ mcpspec mock my-api --generate ./mock-server.js
298
+ node mock-server.js
299
+ ```
300
+
301
+ **Matching modes:**
302
+
303
+ | Mode | Behavior |
304
+ |------|----------|
305
+ | `match` (default) | Exact input match first, then next queued response per tool |
306
+ | `sequential` | Tape/cassette style — responses served in recorded order |
307
+
308
+ **Options:**
309
+
310
+ ```bash
311
+ mcpspec mock my-api --mode sequential # Tape-style matching
312
+ mcpspec mock my-api --latency original # Simulate original response times
313
+ mcpspec mock my-api --latency 100 # Fixed 100ms delay
314
+ mcpspec mock my-api --on-missing empty # Return empty instead of error for unrecorded tools
315
+ ```
316
+
317
+ The generated standalone file embeds the recording data and matching logic — commit it to your repo for portable, dependency-light mock servers.
318
+
319
+ ---
320
+
321
+ ### Performance Benchmarks
322
+
323
+ Measure latency and throughput with statistical analysis across hundreds of iterations.
324
+
325
+ ```bash
326
+ mcpspec bench "npx my-server" # 100 iterations
327
+ mcpspec bench "npx my-server" --iterations 500
328
+ mcpspec bench "npx my-server" --tool read_file
329
+ mcpspec bench "npx my-server" --args '{"path":"/tmp/f"}'
330
+ mcpspec bench "npx my-server" --warmup 10
331
+ ```
332
+
333
+ **Output:**
334
+
335
+ ```
336
+ Benchmarking read_file (100 iterations, 5 warmup)...
337
+
338
+ Latency
339
+ ────────────────────────────
340
+ Min 12.34ms
341
+ Max 89.21ms
342
+ Mean 34.56ms
343
+ Median 31.22ms
344
+ P95 67.89ms
345
+ P99 82.45ms
346
+ Std Dev 15.23ms
347
+
348
+ Throughput: 28.94 calls/sec
349
+ Errors: 0
350
+ ```
351
+
352
+ Warmup iterations (default: 5) are excluded from measurements. The profiler uses `performance.now()` for high-resolution timing.
353
+
354
+ ---
355
+
356
+ ### MCP Score
357
+
358
+ A 0-100 quality rating across 5 weighted categories with opinionated schema linting.
359
+
360
+ ```bash
361
+ mcpspec score "npx my-server"
362
+ mcpspec score "npx my-server" --badge badge.svg # Generate SVG badge
363
+ mcpspec score "npx my-server" --min-score 80 # Fail if below threshold
364
+ ```
365
+
366
+ **Scoring categories:**
367
+
368
+ | Category (weight) | What it measures |
369
+ |--------------------|-----------------|
370
+ | Documentation (25%) | Percentage of tools and resources with descriptions |
371
+ | Schema Quality (25%) | Property types, descriptions, required fields, constraints (enum/pattern/min/max), naming conventions |
372
+ | Error Handling (20%) | Structured error responses (`isError: true`) vs. crashes on bad input |
373
+ | Responsiveness (15%) | Median latency: <100ms = 100, <500ms = 80, <1s = 60, <5s = 40 |
374
+ | Security (15%) | Findings from passive security scan: 0 = 100, <=2 = 70, <=5 = 40 |
375
+
376
+ Schema quality uses 6 sub-criteria: structure (20%), property types (20%), descriptions (20%), required fields (15%), constraints (15%), naming conventions (10%).
377
+
378
+ The `--badge` flag generates a shields.io-style SVG badge for your README.
379
+
380
+ ---
381
+
382
+ ### Doc Generator
383
+
384
+ Auto-generate Markdown or HTML documentation from server introspection. Zero manual writing.
385
+
386
+ ```bash
387
+ mcpspec docs "npx my-server" # Markdown to stdout
388
+ mcpspec docs "npx my-server" --format html # HTML output
389
+ mcpspec docs "npx my-server" --output ./docs # Write to directory
390
+ ```
391
+
392
+ Generated docs include: server name/version/description, all tools with their input schemas, and all resources with URIs and descriptions.
393
+
394
+ ---
395
+
396
+ ### Web Dashboard
397
+
398
+ A full React UI for managing servers, running tests, viewing audit results, and more. Dark mode included.
399
+
400
+ ```bash
401
+ mcpspec ui # Opens localhost:6274
402
+ mcpspec ui --port 8080 # Custom port
403
+ mcpspec ui --no-open # Don't auto-open browser
404
+ ```
405
+
406
+ **Pages:**
407
+
408
+ | Page | What it does |
409
+ |------|-------------|
410
+ | Dashboard | Overview of servers, collections, recent runs |
411
+ | Servers | Connect and manage MCP server connections |
412
+ | Collections | Create and edit YAML test collections |
413
+ | Runs | View test run history and results |
414
+ | Inspector | Interactive tool calling with schema forms and protocol logging |
415
+ | Audit | Run security scans and view findings |
416
+ | Benchmark | Performance profiling with charts |
417
+ | Score | MCP Score visualization |
418
+ | Docs | Generated server documentation |
419
+ | Recordings | View, replay, and manage recorded sessions |
420
+
421
+ Real-time WebSocket updates for running tests, live protocol logging in the inspector, and dark mode with localStorage persistence.
422
+
423
+ ---
424
+
425
+ ### CI/CD Integration
426
+
427
+ `ci-init` generates ready-to-use pipeline configurations. Deterministic exit codes and JUnit/JSON/TAP reporters for seamless CI integration.
428
+
429
+ ```bash
430
+ mcpspec ci-init # Interactive wizard
431
+ mcpspec ci-init --platform github # GitHub Actions
432
+ mcpspec ci-init --platform gitlab # GitLab CI
433
+ mcpspec ci-init --platform shell # Shell script
434
+ mcpspec ci-init --checks test,audit,score # Choose checks
435
+ mcpspec ci-init --fail-on medium # Audit severity gate
436
+ mcpspec ci-init --min-score 70 # MCP Score threshold
437
+ mcpspec ci-init --force # Overwrite/replace existing
438
+ ```
439
+
440
+ Auto-detects platform from `.github/` or `.gitlab-ci.yml`. GitLab `--force` surgically replaces only the mcpspec job block, preserving other jobs.
441
+
442
+ **Exit codes:**
443
+
444
+ | Code | Meaning |
445
+ |------|---------|
446
+ | `0` | Success |
447
+ | `1` | Test failure |
448
+ | `2` | Runtime error |
449
+ | `3` | Configuration error |
450
+ | `4` | Connection error |
451
+ | `5` | Timeout |
452
+ | `6` | Security findings above threshold |
453
+ | `7` | Validation error |
454
+ | `130` | Interrupted (Ctrl+C) |
455
+
456
+ ---
457
+
458
+ ### Baselines & Comparison
459
+
460
+ Save test runs as baselines and detect regressions between versions.
461
+
462
+ ```bash
463
+ mcpspec baseline save main # Save current run
464
+ mcpspec baseline list # List all baselines
465
+ mcpspec test --baseline main # Compare against baseline
466
+ mcpspec compare --baseline main # Explicit comparison
467
+ mcpspec compare <run-id-1> <run-id-2> # Compare two runs
468
+ ```
469
+
470
+ Comparison output shows regressions (tests that now fail), fixes (tests that now pass), new tests, and removed tests.
471
+
472
+ ---
473
+
474
+ ### Transports
475
+
476
+ MCPSpec supports 3 transport types for connecting to MCP servers:
477
+
478
+ | Transport | Use case | Connection |
479
+ |-----------|----------|------------|
480
+ | **stdio** | Local processes | Spawns child process, communicates via stdin/stdout |
481
+ | **SSE** | Server-Sent Events | Connects to HTTP SSE endpoint |
482
+ | **HTTP** | Streamable HTTP | POST requests to HTTP endpoint |
483
+
484
+ ```yaml
485
+ # stdio (default)
486
+ server:
487
+ command: npx
488
+ args: ["my-mcp-server"]
489
+
490
+ # SSE
491
+ server:
492
+ transport: sse
493
+ url: http://localhost:3000/sse
494
+
495
+ # HTTP
496
+ server:
497
+ transport: http
498
+ url: http://localhost:3000/mcp
499
+ ```
500
+
501
+ Connection state machine with automatic reconnection: exponential backoff (1s, 2s, 4s, 8s) up to 30s max, 3 retry attempts.
59
502
 
60
503
  ## Commands
61
504
 
@@ -64,37 +507,49 @@ mcpspec test
64
507
  | `mcpspec test [collection]` | Run test collections with `--env`, `--tag`, `--parallel`, `--reporter`, `--watch`, `--ci` |
65
508
  | `mcpspec inspect <server>` | Interactive REPL — `.tools`, `.call`, `.schema`, `.resources`, `.info` |
66
509
  | `mcpspec audit <server>` | Security scan — `--mode`, `--fail-on`, `--exclude-tools`, `--dry-run` |
67
- | `mcpspec bench <server>` | Performance benchmark — `--iterations`, `--tool`, `--args` |
68
- | `mcpspec score <server>` | Quality score (0-100) — `--badge badge.svg` |
510
+ | `mcpspec bench <server>` | Performance benchmark — `--iterations`, `--tool`, `--args`, `--warmup` |
511
+ | `mcpspec score <server>` | Quality score (0-100) — `--badge badge.svg`, `--min-score` |
69
512
  | `mcpspec docs <server>` | Generate docs — `--format markdown\|html`, `--output <dir>` |
70
513
  | `mcpspec compare` | Compare test runs or `--baseline <name>` |
71
514
  | `mcpspec baseline save <name>` | Save/list baselines for regression detection |
515
+ | `mcpspec record start <server>` | Record an inspector session — `.call`, `.save`, `.steps` |
516
+ | `mcpspec record replay <name> <server>` | Replay a recording and diff against original |
517
+ | `mcpspec record list` | List saved recordings |
518
+ | `mcpspec record delete <name>` | Delete a saved recording |
519
+ | `mcpspec mock <recording>` | Mock server from recording — `--mode`, `--latency`, `--on-missing`, `--generate` |
72
520
  | `mcpspec init [dir]` | Scaffold project — `--template minimal\|standard\|full` |
521
+ | `mcpspec ci-init` | Generate CI config — `--platform github\|gitlab\|shell`, `--checks`, `--fail-on`, `--force` |
73
522
  | `mcpspec ui` | Launch web dashboard on `localhost:6274` |
74
523
 
75
524
  ## Community Collections
76
525
 
77
- Pre-built test suites for popular MCP servers in [`examples/collections/servers/`](https://github.com/light-handle/mcpspec/tree/main/examples/collections/servers):
526
+ Pre-built test suites for popular MCP servers in [`examples/collections/servers/`](examples/collections/servers/):
78
527
 
79
528
  | Collection | Server | Tests |
80
529
  |------------|--------|-------|
81
- | filesystem.yaml | @modelcontextprotocol/server-filesystem | 12 |
82
- | memory.yaml | @modelcontextprotocol/server-memory | 10 |
83
- | everything.yaml | @modelcontextprotocol/server-everything | 11 |
84
- | fetch.yaml | @modelcontextprotocol/server-fetch | 7 |
85
- | time.yaml | @modelcontextprotocol/server-time | 10 |
86
- | chrome-devtools.yaml | chrome-devtools-mcp | 11 |
87
- | github.yaml | @modelcontextprotocol/server-github | 9 |
530
+ | [filesystem.yaml](examples/collections/servers/filesystem.yaml) | @modelcontextprotocol/server-filesystem | 12 |
531
+ | [memory.yaml](examples/collections/servers/memory.yaml) | @modelcontextprotocol/server-memory | 10 |
532
+ | [everything.yaml](examples/collections/servers/everything.yaml) | @modelcontextprotocol/server-everything | 11 |
533
+ | [fetch.yaml](examples/collections/servers/fetch.yaml) | @modelcontextprotocol/server-fetch | 7 |
534
+ | [time.yaml](examples/collections/servers/time.yaml) | @modelcontextprotocol/server-time | 10 |
535
+ | [chrome-devtools.yaml](examples/collections/servers/chrome-devtools.yaml) | chrome-devtools-mcp | 11 |
536
+ | [github.yaml](examples/collections/servers/github.yaml) | @modelcontextprotocol/server-github | 9 |
88
537
 
89
538
  **70 tests** covering tool discovery, read/write operations, error handling, security edge cases, and latency.
90
539
 
540
+ ```bash
541
+ # Run community collections directly
542
+ mcpspec test examples/collections/servers/filesystem.yaml
543
+ mcpspec test examples/collections/servers/time.yaml --tag smoke
544
+ ```
545
+
91
546
  ## Architecture
92
547
 
93
548
  | Package | Description |
94
549
  |---------|-------------|
95
550
  | `@mcpspec/shared` | Types, Zod schemas, constants |
96
- | `@mcpspec/core` | MCP client, test runner, assertions, security scanner, profiler, doc generator, scorer |
97
- | `@mcpspec/cli` | 10 CLI commands built with Commander.js |
551
+ | `@mcpspec/core` | MCP client, test runner, assertions, security scanner (8 rules), profiler, doc generator, scorer, recording/replay |
552
+ | `@mcpspec/cli` | 13 CLI commands built with Commander.js |
98
553
  | `@mcpspec/server` | Hono HTTP server with REST API + WebSocket |
99
554
  | `@mcpspec/ui` | React SPA — TanStack Router, TanStack Query, Tailwind, shadcn/ui |
100
555
 
@@ -104,7 +559,7 @@ Pre-built test suites for popular MCP servers in [`examples/collections/servers/
104
559
  git clone https://github.com/light-handle/mcpspec.git
105
560
  cd mcpspec
106
561
  pnpm install && pnpm build
107
- pnpm test # 260 tests across core + server
562
+ pnpm test # 329 tests across core + server
108
563
  ```
109
564
 
110
565
  ## License
package/dist/index.js CHANGED
@@ -1,9 +1,9 @@
1
1
  #!/usr/bin/env node
2
2
 
3
3
  // src/index.ts
4
- import { Command as Command11 } from "commander";
5
- import { readFileSync as readFileSync3 } from "fs";
6
- import { dirname, join as join2 } from "path";
4
+ import { Command as Command14 } from "commander";
5
+ import { readFileSync as readFileSync4 } from "fs";
6
+ import { dirname, join as join3 } from "path";
7
7
  import { fileURLToPath } from "url";
8
8
 
9
9
  // src/commands/test.ts
@@ -1116,10 +1116,728 @@ ${COLORS5.bold} MCP Score${COLORS5.reset}`);
1116
1116
  }
1117
1117
  });
1118
1118
 
1119
+ // src/commands/record.ts
1120
+ import { Command as Command11 } from "commander";
1121
+ import { createInterface as createInterface2 } from "readline";
1122
+ import { randomUUID } from "crypto";
1123
+ import { EXIT_CODES as EXIT_CODES10 } from "@mcpspec/shared";
1124
+ import {
1125
+ MCPClient as MCPClient6,
1126
+ RecordingStore,
1127
+ RecordingReplayer,
1128
+ RecordingDiffer,
1129
+ formatError as formatError7
1130
+ } from "@mcpspec/core";
1131
+ var COLORS6 = {
1132
+ reset: "\x1B[0m",
1133
+ green: "\x1B[32m",
1134
+ red: "\x1B[31m",
1135
+ yellow: "\x1B[33m",
1136
+ gray: "\x1B[90m",
1137
+ bold: "\x1B[1m",
1138
+ cyan: "\x1B[36m",
1139
+ blue: "\x1B[34m"
1140
+ };
1141
+ var recordCommand = new Command11("record").description("Record, replay, and manage inspector session recordings");
1142
+ recordCommand.command("start").description("Start a recording session (interactive REPL)").argument("<server>", 'Server command (e.g., "npx @modelcontextprotocol/server-filesystem /tmp")').action(async (serverCommand) => {
1143
+ let client = null;
1144
+ const store = new RecordingStore();
1145
+ const steps = [];
1146
+ let toolList = [];
1147
+ try {
1148
+ client = new MCPClient6({ serverConfig: serverCommand });
1149
+ console.log(`${COLORS6.cyan}Connecting to: ${COLORS6.reset}${serverCommand}`);
1150
+ await client.connect();
1151
+ const info = client.getServerInfo();
1152
+ const serverName = info?.name ?? "unknown";
1153
+ console.log(`${COLORS6.green}Connected to ${serverName}${COLORS6.reset}`);
1154
+ const tools = await client.listTools();
1155
+ toolList = tools.map((t) => ({ name: t.name, description: t.description }));
1156
+ console.log(`${COLORS6.gray}${tools.length} tools available${COLORS6.reset}`);
1157
+ console.log(`
1158
+ ${COLORS6.bold}Recording mode.${COLORS6.reset} Type ${COLORS6.bold}.help${COLORS6.reset} for commands.
1159
+ `);
1160
+ const rl = createInterface2({
1161
+ input: process.stdin,
1162
+ output: process.stdout,
1163
+ prompt: `${COLORS6.red}rec>${COLORS6.reset} `
1164
+ });
1165
+ rl.prompt();
1166
+ rl.on("line", async (line) => {
1167
+ const trimmed = line.trim();
1168
+ if (!trimmed) {
1169
+ rl.prompt();
1170
+ return;
1171
+ }
1172
+ try {
1173
+ if (trimmed === ".exit" || trimmed === ".quit") {
1174
+ if (steps.length > 0) {
1175
+ console.log(`${COLORS6.yellow}Warning: ${steps.length} unsaved step(s). Use .save <name> first, or .exit to discard.${COLORS6.reset}`);
1176
+ if (trimmed === ".exit") {
1177
+ await client?.disconnect();
1178
+ rl.close();
1179
+ process.exit(EXIT_CODES10.SUCCESS);
1180
+ }
1181
+ } else {
1182
+ await client?.disconnect();
1183
+ rl.close();
1184
+ process.exit(EXIT_CODES10.SUCCESS);
1185
+ }
1186
+ return;
1187
+ }
1188
+ if (trimmed === ".help") {
1189
+ console.log(`
1190
+ ${COLORS6.bold}Recording commands:${COLORS6.reset}
1191
+ .tools List available tools
1192
+ .call <tool> <json> Call a tool and record the result
1193
+ .steps List recorded steps
1194
+ .save <name> Save recording with given name
1195
+ .exit Disconnect and exit
1196
+ `);
1197
+ rl.prompt();
1198
+ return;
1199
+ }
1200
+ if (trimmed === ".tools") {
1201
+ if (toolList.length === 0) {
1202
+ console.log(`${COLORS6.gray}No tools available${COLORS6.reset}`);
1203
+ } else {
1204
+ console.log(`
1205
+ ${COLORS6.bold}Tools (${toolList.length}):${COLORS6.reset}`);
1206
+ for (const tool of toolList) {
1207
+ console.log(` ${COLORS6.green}${tool.name}${COLORS6.reset}`);
1208
+ if (tool.description) console.log(` ${COLORS6.gray}${tool.description}${COLORS6.reset}`);
1209
+ }
1210
+ console.log("");
1211
+ }
1212
+ rl.prompt();
1213
+ return;
1214
+ }
1215
+ if (trimmed === ".steps") {
1216
+ if (steps.length === 0) {
1217
+ console.log(`${COLORS6.gray}No steps recorded yet${COLORS6.reset}`);
1218
+ } else {
1219
+ console.log(`
1220
+ ${COLORS6.bold}Recorded steps (${steps.length}):${COLORS6.reset}`);
1221
+ for (let i = 0; i < steps.length; i++) {
1222
+ const s = steps[i];
1223
+ const status = s.isError ? `${COLORS6.red}ERROR${COLORS6.reset}` : `${COLORS6.green}OK${COLORS6.reset}`;
1224
+ console.log(` ${i + 1}. ${s.tool} ${COLORS6.gray}${JSON.stringify(s.input)}${COLORS6.reset} [${status}] ${COLORS6.gray}${s.durationMs}ms${COLORS6.reset}`);
1225
+ }
1226
+ console.log("");
1227
+ }
1228
+ rl.prompt();
1229
+ return;
1230
+ }
1231
+ if (trimmed.startsWith(".save ")) {
1232
+ const name = trimmed.slice(6).trim();
1233
+ if (!name) {
1234
+ console.log(`${COLORS6.red}Usage: .save <name>${COLORS6.reset}`);
1235
+ rl.prompt();
1236
+ return;
1237
+ }
1238
+ if (steps.length === 0) {
1239
+ console.log(`${COLORS6.yellow}No steps to save. Use .call first.${COLORS6.reset}`);
1240
+ rl.prompt();
1241
+ return;
1242
+ }
1243
+ const recording = {
1244
+ id: randomUUID(),
1245
+ name,
1246
+ serverName: info?.name,
1247
+ tools: toolList,
1248
+ steps: [...steps],
1249
+ createdAt: (/* @__PURE__ */ new Date()).toISOString()
1250
+ };
1251
+ const path = store.save(name, recording);
1252
+ console.log(`${COLORS6.green}Saved recording "${name}" (${steps.length} steps) to ${path}${COLORS6.reset}`);
1253
+ rl.prompt();
1254
+ return;
1255
+ }
1256
+ if (trimmed.startsWith(".call ")) {
1257
+ const rest = trimmed.slice(6).trim();
1258
+ const spaceIdx = rest.indexOf(" ");
1259
+ let toolName;
1260
+ let args = {};
1261
+ if (spaceIdx === -1) {
1262
+ toolName = rest;
1263
+ } else {
1264
+ toolName = rest.slice(0, spaceIdx);
1265
+ const jsonStr = rest.slice(spaceIdx + 1).trim();
1266
+ try {
1267
+ args = JSON.parse(jsonStr);
1268
+ } catch {
1269
+ console.log(`${COLORS6.red}Invalid JSON: ${jsonStr}${COLORS6.reset}`);
1270
+ rl.prompt();
1271
+ return;
1272
+ }
1273
+ }
1274
+ console.log(`${COLORS6.gray}Calling ${toolName}...${COLORS6.reset}`);
1275
+ const start = performance.now();
1276
+ let output = [];
1277
+ let isError = false;
1278
+ try {
1279
+ const result = await client.callTool(toolName, args);
1280
+ output = result.content;
1281
+ isError = result.isError === true;
1282
+ } catch (err) {
1283
+ output = [{ type: "text", text: err instanceof Error ? err.message : String(err) }];
1284
+ isError = true;
1285
+ }
1286
+ const durationMs = Math.round(performance.now() - start);
1287
+ steps.push({ tool: toolName, input: args, output, isError, durationMs });
1288
+ const statusLabel = isError ? `${COLORS6.red}ERROR${COLORS6.reset}` : `${COLORS6.green}OK${COLORS6.reset}`;
1289
+ console.log(`[${statusLabel}] ${COLORS6.gray}${durationMs}ms${COLORS6.reset} (step ${steps.length})`);
1290
+ console.log(JSON.stringify(output, null, 2));
1291
+ rl.prompt();
1292
+ return;
1293
+ }
1294
+ console.log(`${COLORS6.yellow}Unknown command. Type .help for available commands.${COLORS6.reset}`);
1295
+ } catch (err) {
1296
+ const formatted = formatError7(err);
1297
+ console.log(`${COLORS6.red}${formatted.title}: ${formatted.description}${COLORS6.reset}`);
1298
+ }
1299
+ rl.prompt();
1300
+ });
1301
+ rl.on("close", async () => {
1302
+ await client?.disconnect();
1303
+ process.exit(EXIT_CODES10.SUCCESS);
1304
+ });
1305
+ } catch (err) {
1306
+ const formatted = formatError7(err);
1307
+ console.error(`
1308
+ ${formatted.title}: ${formatted.description}`);
1309
+ formatted.suggestions.forEach((s) => console.error(` - ${s}`));
1310
+ await client?.disconnect();
1311
+ process.exit(formatted.exitCode);
1312
+ }
1313
+ });
1314
+ recordCommand.command("list").description("List saved recordings").action(() => {
1315
+ const store = new RecordingStore();
1316
+ const recordings = store.list();
1317
+ if (recordings.length === 0) {
1318
+ console.log(`${COLORS6.gray}No recordings found.${COLORS6.reset}`);
1319
+ return;
1320
+ }
1321
+ console.log(`
1322
+ ${COLORS6.bold}Saved recordings (${recordings.length}):${COLORS6.reset}`);
1323
+ for (const name of recordings) {
1324
+ const recording = store.load(name);
1325
+ if (recording) {
1326
+ console.log(` ${COLORS6.green}${name}${COLORS6.reset} ${COLORS6.gray}(${recording.steps.length} steps, ${recording.createdAt})${COLORS6.reset}`);
1327
+ } else {
1328
+ console.log(` ${COLORS6.green}${name}${COLORS6.reset}`);
1329
+ }
1330
+ }
1331
+ console.log("");
1332
+ });
1333
+ recordCommand.command("replay").description("Replay a recording against a server and show diff").argument("<name>", "Recording name").argument("<server>", "Server command").action(async (name, serverCommand) => {
1334
+ const store = new RecordingStore();
1335
+ const recording = store.load(name);
1336
+ if (!recording) {
1337
+ console.error(`${COLORS6.red}Recording "${name}" not found.${COLORS6.reset}`);
1338
+ process.exit(EXIT_CODES10.ERROR);
1339
+ }
1340
+ let client = null;
1341
+ try {
1342
+ client = new MCPClient6({ serverConfig: serverCommand });
1343
+ console.log(`${COLORS6.cyan}Connecting to: ${COLORS6.reset}${serverCommand}`);
1344
+ await client.connect();
1345
+ console.log(`${COLORS6.green}Connected. Replaying ${recording.steps.length} steps...${COLORS6.reset}
1346
+ `);
1347
+ const replayer = new RecordingReplayer();
1348
+ const result = await replayer.replay(recording, client, {
1349
+ onStepStart: (i, step) => {
1350
+ process.stdout.write(` ${i + 1}/${recording.steps.length} ${step.tool}... `);
1351
+ },
1352
+ onStepComplete: (_i, replayed) => {
1353
+ const status = replayed.isError ? `${COLORS6.red}ERROR${COLORS6.reset}` : `${COLORS6.green}OK${COLORS6.reset}`;
1354
+ console.log(`[${status}] ${COLORS6.gray}${replayed.durationMs}ms${COLORS6.reset}`);
1355
+ }
1356
+ });
1357
+ const differ = new RecordingDiffer();
1358
+ const diff = differ.diff(recording, result.replayedSteps, result.replayedAt);
1359
+ console.log(`
1360
+ ${COLORS6.bold}Diff Summary:${COLORS6.reset}`);
1361
+ console.log(` ${COLORS6.green}Matched:${COLORS6.reset} ${diff.summary.matched}`);
1362
+ console.log(` ${COLORS6.yellow}Changed:${COLORS6.reset} ${diff.summary.changed}`);
1363
+ console.log(` ${COLORS6.blue}Added:${COLORS6.reset} ${diff.summary.added}`);
1364
+ console.log(` ${COLORS6.red}Removed:${COLORS6.reset} ${diff.summary.removed}`);
1365
+ if (diff.summary.changed > 0) {
1366
+ console.log(`
1367
+ ${COLORS6.bold}Changed steps:${COLORS6.reset}`);
1368
+ for (const step of diff.steps) {
1369
+ if (step.type === "changed") {
1370
+ console.log(` Step ${step.index + 1} (${step.tool}): ${COLORS6.yellow}${step.outputDiff}${COLORS6.reset}`);
1371
+ }
1372
+ }
1373
+ }
1374
+ await client.disconnect();
1375
+ const exitCode = diff.summary.changed > 0 || diff.summary.removed > 0 ? EXIT_CODES10.TEST_FAILURE : EXIT_CODES10.SUCCESS;
1376
+ process.exit(exitCode);
1377
+ } catch (err) {
1378
+ const formatted = formatError7(err);
1379
+ console.error(`
1380
+ ${formatted.title}: ${formatted.description}`);
1381
+ formatted.suggestions.forEach((s) => console.error(` - ${s}`));
1382
+ await client?.disconnect();
1383
+ process.exit(formatted.exitCode);
1384
+ }
1385
+ });
1386
+ recordCommand.command("delete").description("Delete a saved recording").argument("<name>", "Recording name").action((name) => {
1387
+ const store = new RecordingStore();
1388
+ if (store.delete(name)) {
1389
+ console.log(`${COLORS6.green}Deleted recording "${name}".${COLORS6.reset}`);
1390
+ } else {
1391
+ console.error(`${COLORS6.red}Recording "${name}" not found.${COLORS6.reset}`);
1392
+ process.exit(EXIT_CODES10.ERROR);
1393
+ }
1394
+ });
1395
+
1396
+ // src/commands/ci-init.ts
1397
+ import { Command as Command12 } from "commander";
1398
+ import { existsSync as existsSync2, writeFileSync as writeFileSync4, readFileSync as readFileSync3, mkdirSync as mkdirSync2, chmodSync } from "fs";
1399
+ import { resolve as resolve4 } from "path";
1400
+ import { EXIT_CODES as EXIT_CODES11 } from "@mcpspec/shared";
1401
+ function detectPlatform() {
1402
+ if (existsSync2(".github")) return "github";
1403
+ if (existsSync2(".gitlab-ci.yml")) return "gitlab";
1404
+ return null;
1405
+ }
1406
+ function detectCollection() {
1407
+ if (existsSync2("mcpspec.yaml")) return "./mcpspec.yaml";
1408
+ if (existsSync2("mcpspec.yml")) return "./mcpspec.yml";
1409
+ return null;
1410
+ }
1411
+ function renderGitHubActions(config) {
1412
+ const lines = [];
1413
+ lines.push("name: MCP Server Tests");
1414
+ lines.push("on: [push, pull_request]");
1415
+ lines.push("");
1416
+ lines.push("jobs:");
1417
+ lines.push(" mcpspec:");
1418
+ lines.push(" runs-on: ubuntu-latest");
1419
+ lines.push(" steps:");
1420
+ lines.push(" - uses: actions/checkout@v4");
1421
+ lines.push(" - uses: actions/setup-node@v4");
1422
+ lines.push(" with:");
1423
+ lines.push(" node-version: '22'");
1424
+ lines.push("");
1425
+ lines.push(" # Or add mcpspec as a devDependency for version pinning");
1426
+ lines.push(" - run: npm install -g mcpspec");
1427
+ const artifacts = [];
1428
+ if (config.checks.includes("test")) {
1429
+ lines.push("");
1430
+ lines.push(" - name: Run tests");
1431
+ lines.push(` run: mcpspec test ${config.collection} --ci --reporter junit --output results.xml`);
1432
+ artifacts.push("results.xml");
1433
+ }
1434
+ if (config.checks.includes("audit") && config.server) {
1435
+ lines.push("");
1436
+ lines.push(" - name: Security audit");
1437
+ lines.push(` run: mcpspec audit "${config.server}" --mode passive --fail-on ${config.failOn}`);
1438
+ }
1439
+ if (config.checks.includes("score") && config.server) {
1440
+ lines.push("");
1441
+ lines.push(" - name: MCP Score");
1442
+ if (config.minScore !== null) {
1443
+ lines.push(` run: mcpspec score "${config.server}" --badge badge.svg --min-score ${config.minScore}`);
1444
+ } else {
1445
+ lines.push(` run: mcpspec score "${config.server}" --badge badge.svg`);
1446
+ }
1447
+ artifacts.push("badge.svg");
1448
+ }
1449
+ if (config.checks.includes("bench") && config.server) {
1450
+ lines.push("");
1451
+ lines.push(" - name: Performance benchmark");
1452
+ lines.push(` run: mcpspec bench "${config.server}"`);
1453
+ }
1454
+ if (artifacts.length > 0) {
1455
+ lines.push("");
1456
+ lines.push(" - name: Upload results");
1457
+ lines.push(" if: always()");
1458
+ lines.push(" uses: actions/upload-artifact@v4");
1459
+ lines.push(" with:");
1460
+ lines.push(" name: mcpspec-results");
1461
+ lines.push(" path: |");
1462
+ for (const a of artifacts) {
1463
+ lines.push(` ${a}`);
1464
+ }
1465
+ }
1466
+ if (config.checks.includes("test")) {
1467
+ lines.push("");
1468
+ lines.push(" - name: Test Report");
1469
+ lines.push(" if: always()");
1470
+ lines.push(" uses: mikepenz/action-junit-report@v4");
1471
+ lines.push(" with:");
1472
+ lines.push(" report_paths: results.xml");
1473
+ }
1474
+ lines.push("");
1475
+ return lines.join("\n");
1476
+ }
1477
+ function renderGitLabCI(config) {
1478
+ const lines = [];
1479
+ lines.push("mcpspec:");
1480
+ lines.push(" image: node:22");
1481
+ lines.push(" stage: test");
1482
+ lines.push(" script:");
1483
+ lines.push(" # Or add mcpspec as a devDependency for version pinning");
1484
+ lines.push(" - npm install -g mcpspec");
1485
+ if (config.checks.includes("test")) {
1486
+ lines.push(` - mcpspec test ${config.collection} --ci --reporter junit --output results.xml`);
1487
+ }
1488
+ if (config.checks.includes("audit") && config.server) {
1489
+ lines.push(` - mcpspec audit "${config.server}" --mode passive --fail-on ${config.failOn}`);
1490
+ }
1491
+ if (config.checks.includes("score") && config.server) {
1492
+ if (config.minScore !== null) {
1493
+ lines.push(` - mcpspec score "${config.server}" --min-score ${config.minScore}`);
1494
+ } else {
1495
+ lines.push(` - mcpspec score "${config.server}"`);
1496
+ }
1497
+ }
1498
+ if (config.checks.includes("bench") && config.server) {
1499
+ lines.push(` - mcpspec bench "${config.server}"`);
1500
+ }
1501
+ if (config.checks.includes("test")) {
1502
+ lines.push(" artifacts:");
1503
+ lines.push(" when: always");
1504
+ lines.push(" paths:");
1505
+ lines.push(" - results.xml");
1506
+ lines.push(" reports:");
1507
+ lines.push(" junit: results.xml");
1508
+ lines.push(" expire_in: 1 week");
1509
+ }
1510
+ lines.push("");
1511
+ return lines.join("\n");
1512
+ }
1513
+ function renderShellScript(config) {
1514
+ const lines = [];
1515
+ lines.push("#!/usr/bin/env bash");
1516
+ lines.push("set -euo pipefail");
1517
+ lines.push("");
1518
+ lines.push("# Or add mcpspec as a devDependency for version pinning");
1519
+ lines.push("command -v mcpspec >/dev/null 2>&1 || npm install -g mcpspec");
1520
+ lines.push("");
1521
+ lines.push('echo "Running MCPSpec CI checks..."');
1522
+ lines.push("");
1523
+ if (config.checks.includes("test")) {
1524
+ lines.push(`mcpspec test ${config.collection} --ci --reporter junit --output results.xml`);
1525
+ lines.push('echo "Tests passed."');
1526
+ lines.push("");
1527
+ }
1528
+ if (config.checks.includes("audit") && config.server) {
1529
+ lines.push(`mcpspec audit "${config.server}" --mode passive --fail-on ${config.failOn}`);
1530
+ lines.push('echo "Security audit passed."');
1531
+ lines.push("");
1532
+ }
1533
+ if (config.checks.includes("score") && config.server) {
1534
+ if (config.minScore !== null) {
1535
+ lines.push(`mcpspec score "${config.server}" --min-score ${config.minScore}`);
1536
+ } else {
1537
+ lines.push(`mcpspec score "${config.server}"`);
1538
+ }
1539
+ lines.push('echo "MCP Score check passed."');
1540
+ lines.push("");
1541
+ }
1542
+ if (config.checks.includes("bench") && config.server) {
1543
+ lines.push(`mcpspec bench "${config.server}"`);
1544
+ lines.push('echo "Benchmark complete."');
1545
+ lines.push("");
1546
+ }
1547
+ lines.push('echo "All checks passed!"');
1548
+ lines.push("");
1549
+ return lines.join("\n");
1550
+ }
1551
+ function getOutputPath(platform) {
1552
+ switch (platform) {
1553
+ case "github":
1554
+ return ".github/workflows/mcpspec.yml";
1555
+ case "gitlab":
1556
+ return ".gitlab-ci.yml";
1557
+ case "shell":
1558
+ return "mcpspec-ci.sh";
1559
+ }
1560
+ }
1561
+ function replaceGitLabJob(existing, newJob) {
1562
+ const lines = existing.split("\n");
1563
+ let blockStart = -1;
1564
+ let blockEnd = lines.length;
1565
+ for (let i = 0; i < lines.length; i++) {
1566
+ const line = lines[i];
1567
+ if (blockStart === -1) {
1568
+ if (/^mcpspec:/.test(line)) {
1569
+ blockStart = i;
1570
+ }
1571
+ } else {
1572
+ if (line.length > 0 && !line.startsWith(" ") && !line.startsWith("#")) {
1573
+ blockEnd = i;
1574
+ break;
1575
+ }
1576
+ }
1577
+ }
1578
+ if (blockStart === -1) return null;
1579
+ while (blockEnd > blockStart && lines[blockEnd - 1].trim() === "") {
1580
+ blockEnd--;
1581
+ }
1582
+ const before = lines.slice(0, blockStart);
1583
+ const after = lines.slice(blockEnd);
1584
+ while (before.length > 0 && before[before.length - 1].trim() === "") {
1585
+ before.pop();
1586
+ }
1587
+ const parts = [];
1588
+ if (before.length > 0) {
1589
+ parts.push(before.join("\n"));
1590
+ parts.push("");
1591
+ }
1592
+ parts.push(newJob.trimEnd());
1593
+ if (after.length > 0) {
1594
+ const afterStr = after.join("\n").trimStart();
1595
+ if (afterStr.length > 0) {
1596
+ parts.push("");
1597
+ parts.push(afterStr);
1598
+ }
1599
+ }
1600
+ return parts.join("\n") + "\n";
1601
+ }
1602
+ function parseChecks(checksStr) {
1603
+ const valid = ["test", "audit", "score", "bench"];
1604
+ const parsed = checksStr.split(",").map((c) => c.trim()).filter(Boolean);
1605
+ for (const c of parsed) {
1606
+ if (!valid.includes(c)) {
1607
+ console.error(`Unknown check: ${c}. Valid checks: ${valid.join(", ")}`);
1608
+ process.exit(EXIT_CODES11.CONFIG_ERROR);
1609
+ }
1610
+ }
1611
+ return parsed;
1612
+ }
1613
+ var ciInitCommand = new Command12("ci-init").description("Generate CI pipeline configuration for MCP server testing").option("--platform <type>", "CI platform: github, gitlab, or shell").option("--collection <path>", "Path to collection file").option("--server <command>", "Server command for audit/score/bench steps").option("--checks <list>", "Comma-separated checks: test,audit,score,bench", "test,audit").option("--fail-on <severity>", "Audit severity gate: low, medium, high, critical", "high").option("--min-score <n>", "Minimum MCP Score threshold (0-100)").option("--force", "Overwrite existing files").action(async (options) => {
1614
+ try {
1615
+ let platform = options.platform;
1616
+ let collection = options.collection ?? detectCollection() ?? "./mcpspec.yaml";
1617
+ let server = options.server ?? "";
1618
+ let checks = parseChecks(options.checks ?? "test,audit");
1619
+ let failOn = options.failOn ?? "high";
1620
+ let minScore = options.minScore ? Number(options.minScore) : null;
1621
+ if (!platform && process.stdin.isTTY) {
1622
+ const { select, input, checkbox, confirm } = await import("@inquirer/prompts");
1623
+ console.log("\n Generate CI pipeline configuration for MCPSpec.\n");
1624
+ const detected = detectPlatform();
1625
+ platform = await select({
1626
+ message: "CI platform:",
1627
+ choices: [
1628
+ { name: "GitHub Actions", value: "github" },
1629
+ { name: "GitLab CI", value: "gitlab" },
1630
+ { name: "Shell script", value: "shell" }
1631
+ ],
1632
+ default: detected ?? void 0
1633
+ });
1634
+ const detectedCollection = detectCollection();
1635
+ collection = await input({
1636
+ message: "Collection file path:",
1637
+ default: detectedCollection ?? "./mcpspec.yaml"
1638
+ });
1639
+ server = await input({
1640
+ message: "Server command (for audit/score/bench, leave empty to skip):",
1641
+ default: ""
1642
+ });
1643
+ checks = await checkbox({
1644
+ message: "Which checks to run?",
1645
+ choices: [
1646
+ { name: "Test collections", value: "test", checked: true },
1647
+ { name: "Security audit", value: "audit", checked: true },
1648
+ { name: "MCP Score", value: "score", checked: false },
1649
+ { name: "Performance benchmark", value: "bench", checked: false }
1650
+ ]
1651
+ });
1652
+ if (checks.length === 0) {
1653
+ console.error("No checks selected. At least one check is required.");
1654
+ process.exit(EXIT_CODES11.CONFIG_ERROR);
1655
+ }
1656
+ if (checks.includes("audit")) {
1657
+ failOn = await select({
1658
+ message: "Fail on audit severity:",
1659
+ choices: [
1660
+ { name: "critical", value: "critical" },
1661
+ { name: "high (recommended)", value: "high" },
1662
+ { name: "medium", value: "medium" },
1663
+ { name: "low", value: "low" }
1664
+ ],
1665
+ default: "high"
1666
+ });
1667
+ }
1668
+ if (checks.includes("score")) {
1669
+ const wantMinScore = await confirm({
1670
+ message: "Set a minimum MCP Score threshold?",
1671
+ default: false
1672
+ });
1673
+ if (wantMinScore) {
1674
+ const scoreStr = await input({
1675
+ message: "Minimum score (0-100):",
1676
+ default: "70"
1677
+ });
1678
+ minScore = Number(scoreStr);
1679
+ if (isNaN(minScore) || minScore < 0 || minScore > 100) {
1680
+ console.error("Score must be a number between 0 and 100.");
1681
+ process.exit(EXIT_CODES11.CONFIG_ERROR);
1682
+ }
1683
+ }
1684
+ }
1685
+ } else if (!platform) {
1686
+ platform = detectPlatform() ?? "shell";
1687
+ }
1688
+ if (minScore !== null && (isNaN(minScore) || minScore < 0 || minScore > 100)) {
1689
+ console.error("--min-score must be a number between 0 and 100.");
1690
+ process.exit(EXIT_CODES11.CONFIG_ERROR);
1691
+ }
1692
+ const config = { platform, collection, server, checks, failOn, minScore };
1693
+ const outputPath = getOutputPath(platform);
1694
+ const resolvedPath = resolve4(outputPath);
1695
+ const force = options.force === true;
1696
+ const fileExists = existsSync2(resolvedPath);
1697
+ if (platform === "gitlab" && fileExists) {
1698
+ const existing = readFileSync3(resolvedPath, "utf-8");
1699
+ const hasMcpspec = existing.includes("mcpspec");
1700
+ const newJob = renderGitLabCI(config);
1701
+ if (hasMcpspec && !force) {
1702
+ console.error(`MCPSpec job already exists in ${outputPath}. Use --force to overwrite, or edit manually.`);
1703
+ process.exit(EXIT_CODES11.CONFIG_ERROR);
1704
+ }
1705
+ if (hasMcpspec && force) {
1706
+ const replaced = replaceGitLabJob(existing, newJob);
1707
+ if (replaced) {
1708
+ writeFileSync4(resolvedPath, replaced, "utf-8");
1709
+ console.log(`
1710
+ Replaced MCPSpec job in ${outputPath}`);
1711
+ } else {
1712
+ writeFileSync4(resolvedPath, existing.trimEnd() + "\n\n" + newJob, "utf-8");
1713
+ console.log(`
1714
+ Appended MCPSpec job to ${outputPath}`);
1715
+ }
1716
+ } else {
1717
+ writeFileSync4(resolvedPath, existing.trimEnd() + "\n\n" + newJob, "utf-8");
1718
+ console.log(`
1719
+ Appended MCPSpec job to ${outputPath}`);
1720
+ }
1721
+ } else {
1722
+ if (fileExists && !force) {
1723
+ console.error(`File already exists: ${outputPath}. Use --force to overwrite.`);
1724
+ process.exit(EXIT_CODES11.CONFIG_ERROR);
1725
+ }
1726
+ let content;
1727
+ switch (platform) {
1728
+ case "github":
1729
+ content = renderGitHubActions(config);
1730
+ break;
1731
+ case "gitlab":
1732
+ content = renderGitLabCI(config);
1733
+ break;
1734
+ case "shell":
1735
+ content = renderShellScript(config);
1736
+ break;
1737
+ }
1738
+ const parentDir = resolve4(outputPath, "..");
1739
+ if (!existsSync2(parentDir)) {
1740
+ mkdirSync2(parentDir, { recursive: true });
1741
+ }
1742
+ writeFileSync4(resolvedPath, content, "utf-8");
1743
+ if (platform === "shell") {
1744
+ chmodSync(resolvedPath, 493);
1745
+ }
1746
+ console.log(`
1747
+ Created ${outputPath}`);
1748
+ }
1749
+ console.log(`
1750
+ Platform: ${platform}`);
1751
+ console.log(` Checks: ${checks.join(", ")}`);
1752
+ if (checks.includes("audit")) console.log(` Fail on: ${failOn}`);
1753
+ if (minScore !== null) console.log(` Min score: ${minScore}`);
1754
+ if (server) console.log(` Server: ${server}`);
1755
+ console.log(` Collection: ${collection}`);
1756
+ console.log("");
1757
+ } catch (err) {
1758
+ const message = err instanceof Error ? err.message : String(err);
1759
+ console.error(`Failed to generate CI config: ${message}`);
1760
+ process.exit(EXIT_CODES11.ERROR);
1761
+ }
1762
+ });
1763
+
1764
+ // src/commands/mock.ts
1765
+ import { Command as Command13 } from "commander";
1766
+ import { writeFileSync as writeFileSync5, chmodSync as chmodSync2 } from "fs";
1767
+ import { EXIT_CODES as EXIT_CODES12 } from "@mcpspec/shared";
1768
+ import {
1769
+ RecordingStore as RecordingStore2,
1770
+ MockMCPServer,
1771
+ MockGenerator,
1772
+ formatError as formatError8
1773
+ } from "@mcpspec/core";
1774
+ var COLORS7 = {
1775
+ reset: "\x1B[0m",
1776
+ red: "\x1B[31m",
1777
+ green: "\x1B[32m",
1778
+ yellow: "\x1B[33m",
1779
+ cyan: "\x1B[36m",
1780
+ dim: "\x1B[2m",
1781
+ bold: "\x1B[1m"
1782
+ };
1783
+ var mockCommand = new Command13("mock").description("Start a mock MCP server from a saved recording").argument("<recording>", "Recording name (from mcpspec record)").option("--mode <mode>", "Matching strategy: match or sequential", "match").option("--latency <ms>", 'Response delay: 0, milliseconds, or "original"', "0").option("--on-missing <behavior>", "Unrecorded tool behavior: error or empty", "error").option("--generate <path>", "Generate standalone .js file instead of starting server").action(async (recordingName, options) => {
1784
+ try {
1785
+ const mode = options.mode;
1786
+ if (mode !== "match" && mode !== "sequential") {
1787
+ console.error(`${COLORS7.red}Error: --mode must be "match" or "sequential"${COLORS7.reset}`);
1788
+ process.exit(EXIT_CODES12.VALIDATION_ERROR);
1789
+ }
1790
+ const onMissing = options.onMissing;
1791
+ if (onMissing !== "error" && onMissing !== "empty") {
1792
+ console.error(`${COLORS7.red}Error: --on-missing must be "error" or "empty"${COLORS7.reset}`);
1793
+ process.exit(EXIT_CODES12.VALIDATION_ERROR);
1794
+ }
1795
+ const latency = options.latency === "original" ? "original" : parseInt(options.latency, 10);
1796
+ if (typeof latency === "number" && isNaN(latency)) {
1797
+ console.error(`${COLORS7.red}Error: --latency must be a number or "original"${COLORS7.reset}`);
1798
+ process.exit(EXIT_CODES12.VALIDATION_ERROR);
1799
+ }
1800
+ const store = new RecordingStore2();
1801
+ const recording = store.load(recordingName);
1802
+ if (!recording) {
1803
+ console.error(`${COLORS7.red}Error: Recording "${recordingName}" not found${COLORS7.reset}`);
1804
+ console.error(`${COLORS7.dim} Available recordings: ${store.list().join(", ") || "(none)"}${COLORS7.reset}`);
1805
+ process.exit(EXIT_CODES12.CONFIG_ERROR);
1806
+ }
1807
+ if (options.generate) {
1808
+ const generator = new MockGenerator();
1809
+ const code = generator.generate({ recording, mode, latency, onMissing });
1810
+ writeFileSync5(options.generate, code, "utf-8");
1811
+ try {
1812
+ chmodSync2(options.generate, 493);
1813
+ } catch {
1814
+ }
1815
+ console.error(`${COLORS7.green}Generated mock server: ${options.generate}${COLORS7.reset}`);
1816
+ console.error(`${COLORS7.dim} Run: node ${options.generate}${COLORS7.reset}`);
1817
+ console.error(`${COLORS7.dim} Requires: @modelcontextprotocol/sdk${COLORS7.reset}`);
1818
+ process.exit(EXIT_CODES12.SUCCESS);
1819
+ }
1820
+ console.error(`${COLORS7.cyan}MCPSpec Mock Server${COLORS7.reset}`);
1821
+ console.error(`${COLORS7.dim} Recording: ${recording.name}${COLORS7.reset}`);
1822
+ console.error(`${COLORS7.dim} Tools: ${recording.tools.map((t) => t.name).join(", ")}${COLORS7.reset}`);
1823
+ console.error(`${COLORS7.dim} Steps: ${recording.steps.length}${COLORS7.reset}`);
1824
+ console.error(`${COLORS7.dim} Mode: ${mode} | Latency: ${latency}ms | On missing: ${onMissing}${COLORS7.reset}`);
1825
+ console.error("");
1826
+ const server = new MockMCPServer({ recording, mode, latency, onMissing });
1827
+ await server.start();
1828
+ } catch (err) {
1829
+ const formatted = formatError8(err);
1830
+ console.error(`
1831
+ ${formatted.title}: ${formatted.description}`);
1832
+ formatted.suggestions.forEach((s) => console.error(` - ${s}`));
1833
+ process.exit(formatted.exitCode);
1834
+ }
1835
+ });
1836
+
1119
1837
  // src/index.ts
1120
1838
  var __cliDir = dirname(fileURLToPath(import.meta.url));
1121
- var pkg = JSON.parse(readFileSync3(join2(__cliDir, "..", "package.json"), "utf-8"));
1122
- var program = new Command11();
1839
+ var pkg = JSON.parse(readFileSync4(join3(__cliDir, "..", "package.json"), "utf-8"));
1840
+ var program = new Command14();
1123
1841
  program.name("mcpspec").description("The definitive MCP server testing platform").version(pkg.version);
1124
1842
  program.addCommand(testCommand);
1125
1843
  program.addCommand(inspectCommand);
@@ -1131,4 +1849,7 @@ program.addCommand(auditCommand);
1131
1849
  program.addCommand(benchCommand);
1132
1850
  program.addCommand(docsCommand);
1133
1851
  program.addCommand(scoreCommand);
1852
+ program.addCommand(recordCommand);
1853
+ program.addCommand(ciInitCommand);
1854
+ program.addCommand(mockCommand);
1134
1855
  program.parse(process.argv);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mcpspec",
3
- "version": "1.0.3",
3
+ "version": "1.2.0",
4
4
  "description": "The definitive MCP server testing platform",
5
5
  "keywords": [
6
6
  "mcp",
@@ -29,9 +29,9 @@
29
29
  "@inquirer/prompts": "^7.0.0",
30
30
  "commander": "^12.1.0",
31
31
  "open": "^10.1.0",
32
- "@mcpspec/core": "1.0.3",
33
- "@mcpspec/shared": "1.0.3",
34
- "@mcpspec/server": "1.0.3"
32
+ "@mcpspec/server": "1.2.0",
33
+ "@mcpspec/core": "1.2.0",
34
+ "@mcpspec/shared": "1.2.0"
35
35
  },
36
36
  "devDependencies": {
37
37
  "tsup": "^8.0.0",