mcprigor 1.3.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +19 -2
  2. package/dist/audit.d.ts +44 -0
  3. package/dist/audit.d.ts.map +1 -0
  4. package/dist/audit.js +157 -0
  5. package/dist/audit.js.map +1 -0
  6. package/dist/cli.js +283 -10
  7. package/dist/cli.js.map +1 -1
  8. package/dist/composition.d.ts +42 -0
  9. package/dist/composition.d.ts.map +1 -0
  10. package/dist/composition.js +130 -0
  11. package/dist/composition.js.map +1 -0
  12. package/dist/coverage.d.ts +33 -0
  13. package/dist/coverage.d.ts.map +1 -0
  14. package/dist/coverage.js +83 -0
  15. package/dist/coverage.js.map +1 -0
  16. package/dist/export.d.ts +20 -0
  17. package/dist/export.d.ts.map +1 -0
  18. package/dist/export.js +422 -0
  19. package/dist/export.js.map +1 -0
  20. package/dist/index.d.ts +5 -0
  21. package/dist/index.d.ts.map +1 -1
  22. package/dist/index.js +5 -0
  23. package/dist/index.js.map +1 -1
  24. package/dist/language.js +1 -1
  25. package/dist/language.js.map +1 -1
  26. package/dist/loader.d.ts.map +1 -1
  27. package/dist/loader.js +4 -1
  28. package/dist/loader.js.map +1 -1
  29. package/dist/monitor.d.ts +22 -0
  30. package/dist/monitor.d.ts.map +1 -0
  31. package/dist/monitor.js +71 -0
  32. package/dist/monitor.js.map +1 -0
  33. package/dist/perf.d.ts +37 -0
  34. package/dist/perf.d.ts.map +1 -0
  35. package/dist/perf.js +90 -0
  36. package/dist/perf.js.map +1 -0
  37. package/dist/publish.d.ts +19 -0
  38. package/dist/publish.d.ts.map +1 -0
  39. package/dist/publish.js +58 -0
  40. package/dist/publish.js.map +1 -0
  41. package/dist/qa-language.d.ts.map +1 -1
  42. package/dist/qa-language.js +46 -1
  43. package/dist/qa-language.js.map +1 -1
  44. package/dist/reporters.d.ts +5 -2
  45. package/dist/reporters.d.ts.map +1 -1
  46. package/dist/reporters.js +26 -5
  47. package/dist/reporters.js.map +1 -1
  48. package/dist/runner.d.ts.map +1 -1
  49. package/dist/runner.js +31 -15
  50. package/dist/runner.js.map +1 -1
  51. package/dist/schema.d.ts +47 -0
  52. package/dist/schema.d.ts.map +1 -1
  53. package/dist/schema.js +10 -1
  54. package/dist/schema.js.map +1 -1
  55. package/dist/timeline.d.ts +18 -0
  56. package/dist/timeline.d.ts.map +1 -0
  57. package/dist/timeline.js +59 -0
  58. package/dist/timeline.js.map +1 -0
  59. package/dist/types.d.ts +19 -0
  60. package/dist/types.d.ts.map +1 -1
  61. package/dist/version.d.ts +1 -1
  62. package/dist/version.js +1 -1
  63. package/dist/workspace.d.ts.map +1 -1
  64. package/dist/workspace.js +73 -3
  65. package/dist/workspace.js.map +1 -1
  66. package/docs/CLI-REFERENCE.md +136 -1
  67. package/docs/COVERAGE.md +37 -0
  68. package/docs/ENGINEER-SETUP.md +1 -1
  69. package/docs/EVIDENCE.md +11 -0
  70. package/docs/GITHUB-ACTION.md +82 -0
  71. package/docs/LANGUAGE-SPEC.md +28 -7
  72. package/docs/MONITORING.md +62 -0
  73. package/docs/MULTI-SERVER-COMPOSITIONS.md +103 -0
  74. package/docs/PERFORMANCE-GOVERNANCE.md +85 -0
  75. package/docs/PLAIN-LANGUAGE-COOKBOOK.md +28 -0
  76. package/docs/PUBLISHING.md +53 -0
  77. package/docs/QA-WORKSPACE.md +5 -1
  78. package/docs/README.md +7 -0
  79. package/docs/SECURITY-AUDIT.md +77 -0
  80. package/editors/mcpr-language.json +2 -2
  81. package/editors/vscode/package.json +1 -1
  82. package/editors/vscode/syntaxes/mcpr.tmLanguage.json +2 -2
  83. package/package.json +1 -1
  84. package/workspace-assets/app.js +113 -1
  85. package/workspace-assets/index.html +17 -0
  86. package/workspace-assets/style.css +13 -2
@@ -1,9 +1,11 @@
1
1
  # MCP Test Language 1
2
2
 
3
- Status: compatibility-stable frontend for MCP Rigor 0.12.
3
+ Status: compatibility-stable frontend for MCP Rigor 1.x.
4
4
 
5
5
  The `.mcpr` language and YAML compile to the same `Suite` runtime model. Every user-authored YAML capability has a deterministic plain-language equivalent; YAML remains available for generated files and programmatic integrations rather than being a more powerful test format.
6
6
 
7
+ Parity is enforced, not aspirational: every `.mcpr` construct compiles to a suite that also validates against the YAML/JSON schema, and `mcprigor convert my-tests.mcpr --out my-tests.yaml` (or `--format json`) emits that equivalent file. The converted suite reloads to the identical suite model and produces the same run results — the regression suite converts each shipped example, reloads it from YAML, and compares runs.
8
+
7
9
  ## Design goals
8
10
 
9
11
  - Readable by QA professionals without programming experience
@@ -62,6 +64,23 @@ Server options:
62
64
  Authorization: "Bearer ${env.QA_TOKEN}"
63
65
  ```
64
66
 
67
+ Multi-server compositions use named server declarations and per-test routing:
68
+
69
+ ```text
70
+ Named server "catalog": node services/catalog.js
71
+ Named server "billing": https://qa.example.com/billing/mcp
72
+
73
+ Server options for "billing":
74
+ headers:
75
+ X-Tenant: qa
76
+
77
+ Test: "catalog lookup"
78
+ On server "catalog"
79
+ Call tool "search"
80
+ ```
81
+
82
+ The YAML equivalent is a top-level `servers` mapping plus `server` on a test. A composition requires at least two named servers, and an unknown `On server` name is rejected during compilation.
83
+
65
84
  Parity targets use the same connection grammar:
66
85
 
67
86
  ```text
@@ -77,6 +96,8 @@ Suite-level YAML fields have direct equivalents:
77
96
 
78
97
  ```text
79
98
  Default timeout: 10 seconds
99
+ Budget: p95 500ms over 20 calls
100
+ Budget for "order lookup": p50 300ms over 20 calls
80
101
  Redact: "secret-value", "token-value"
81
102
  Snapshots: snapshots.json
82
103
  Ignore snapshot paths: "$.createdAt", "$.requestId"
@@ -94,11 +115,7 @@ Client behavior:
94
115
  approved: true
95
116
  ```
96
117
 
97
- ## Imports
98
-
99
- ```
100
-
101
- Per-test scripted responses (override `Client behavior:` for one test):
118
+ Per-test scripted responses override `Client behavior:` for one test:
102
119
 
103
120
  ```text
104
121
  When the server asks for input, respond "accept" with:
@@ -106,7 +123,10 @@ When the server asks for input, respond "accept" with:
106
123
  When the server asks for input, respond "decline"
107
124
  When the server requests sampling, respond "scripted text"
108
125
  ```
109
- text
126
+
127
+ ## Imports
128
+
129
+ ```text
110
130
  Import flows from "./shared/customer-flows.mcpr"
111
131
  ```
112
132
 
@@ -183,6 +203,7 @@ Send "ping"
183
203
  ```text
184
204
  Expect it succeeds
185
205
  Expect an error
206
+ Expect the call to finish within 800ms
186
207
  Expect "structuredContent.total" equals 2
187
208
  Expect "content[0].text" contains "complete"
188
209
  Expect "items" exists
@@ -0,0 +1,62 @@
1
+ # Scheduled production monitoring
2
+
3
+ > Available since `1.5.0`.
4
+
5
+ Turn an HTTP MCP suite into a continuous production check:
6
+
7
+ ```bash
8
+ mcprigor monitor tests/prod.mcpr \
9
+ --every 15m \
10
+ --notify https://alerts.example.com/hooks/mcprigor
11
+ ```
12
+
13
+ The monitor runs immediately, then at the fixed interval until stopped. Durations accept `ms`, `s`, `m`, or `h`, with a minimum interval of one second.
14
+
15
+ For operational safety, monitoring requires a Streamable HTTP target. It refuses stdio suites so a long-running process cannot repeatedly spawn local commands by accident.
16
+
17
+ ## Notification policies
18
+
19
+ ```bash
20
+ mcprigor monitor tests/prod.mcpr --every 5m --notify URL --notify-on change
21
+ ```
22
+
23
+ `--notify-on` supports:
24
+
25
+ - `change` (default): first failure, then failure/recovery transitions;
26
+ - `failure`: every failed run;
27
+ - `recovery`: transitions from failed to passed;
28
+ - `always`: every run.
29
+
30
+ Webhook requests are JSON POSTs with a 15-second timeout:
31
+
32
+ ```json
33
+ {
34
+ "source": "mcprigor",
35
+ "event": "monitor.failure",
36
+ "suite": "tests/prod.mcpr",
37
+ "run": 12,
38
+ "status": "failed",
39
+ "startedAt": "2026-08-30T12:00:00.000Z",
40
+ "durationMs": 842,
41
+ "summary": { "passed": 7, "failed": 1, "skipped": 0, "blocked": 0 },
42
+ "failures": [{ "name": "order lookup", "error": "..." }]
43
+ }
44
+ ```
45
+
46
+ A non-2xx webhook response fails the monitor with `MCP-MONITOR-003`; notification loss is never silently ignored.
47
+
48
+ ## History and trends
49
+
50
+ Every monitoring run appends to `.mcprigor/workspace-history.jsonl`, so existing `mcprigor trends`, PDF/CSV exports, flaky detection, latency budgets, and regression analysis include production monitoring evidence.
51
+
52
+ ## Bounded runs
53
+
54
+ For smoke tests, cron jobs, and CI validation:
55
+
56
+ ```bash
57
+ mcprigor monitor tests/prod.mcpr --every 1m --max-runs 1
58
+ ```
59
+
60
+ `SIGINT` and `SIGTERM` stop the interval cleanly after active MCP sessions shut down.
61
+
62
+ Use an authenticated suite target (`headers` or `Token from:`), retain history according to your evidence policy, and send webhooks only to reviewed HTTPS endpoints.
@@ -0,0 +1,103 @@
1
+ # Multi-server compositions
2
+
3
+ > Available since `1.5.0`.
4
+
5
+ Real MCP clients mount several servers together. MCP Rigor compositions test each named server and govern the combined tool, resource, and prompt namespace as one fleet.
6
+
7
+ ## Declare named servers
8
+
9
+ ```text
10
+ MCP Test 1
11
+ Suite: "Checkout fleet"
12
+
13
+ Named server "catalog": node services/catalog.js
14
+ Named server "billing": https://qa.example.com/billing/mcp
15
+
16
+ Server options for "billing":
17
+ Token from: node scripts/get-qa-token.mjs
18
+
19
+ Test: "catalog search works"
20
+ On server "catalog"
21
+ Call tool "search" with:
22
+ query: "widget"
23
+ Expect "structuredContent.total" is a number
24
+
25
+ Test: "billing responds"
26
+ On server "billing"
27
+ Send "ping"
28
+ Expect it succeeds
29
+ ```
30
+
31
+ `On server` routes every action in that test to the selected server. Tests without `On server` continue to use the legacy/default `Server:` or `MCP URL:` target. Cross-test dependencies and exported values continue to work across named servers.
32
+
33
+ YAML parity:
34
+
35
+ ```yaml
36
+ version: 1
37
+ name: Checkout fleet
38
+ target:
39
+ transport: stdio
40
+ command: node
41
+ args: [services/gateway.js]
42
+ servers:
43
+ catalog:
44
+ transport: stdio
45
+ command: node
46
+ args: [services/catalog.js]
47
+ billing:
48
+ transport: streamable-http
49
+ url: https://qa.example.com/billing/mcp
50
+ tests:
51
+ - name: catalog search works
52
+ server: catalog
53
+ steps:
54
+ - tool:
55
+ name: search
56
+ arguments: { query: widget }
57
+ ```
58
+
59
+ Unknown server names fail validation before a test starts.
60
+
61
+ ## Check the live composition
62
+
63
+ ```bash
64
+ mcprigor composition-check tests/fleet.mcpr
65
+ ```
66
+
67
+ The check discovers all named servers and reports:
68
+
69
+ - `MCP-COMP-001`: duplicate tool name;
70
+ - `MCP-COMP-002`: conflicting input/output schemas for the same tool (breaking);
71
+ - `MCP-COMP-003`: duplicate resource URI or URI template (breaking);
72
+ - `MCP-COMP-004`: duplicate prompt name.
73
+
74
+ MCP Rigor reports collisions; it never silently renames or chooses a winning server.
75
+
76
+ ## Create a combined fleet lock
77
+
78
+ ```bash
79
+ mcprigor composition-discover tests/fleet.mcpr \
80
+ --out contracts/checkout.composition.lock.yaml
81
+ ```
82
+
83
+ The lock embeds each named server's ordinary discovery contract, the cross-server issue set, and a stable fleet fingerprint. Volatile discovery timestamps and diagnostics do not change the combined fingerprint. Writes are atomic: all servers must be discovered successfully before the previous lock is replaced.
84
+
85
+ ## Gate fleet drift in CI
86
+
87
+ ```bash
88
+ mcprigor composition-drift tests/fleet.mcpr \
89
+ --against contracts/checkout.composition.lock.yaml \
90
+ --fail-on breaking
91
+ ```
92
+
93
+ The drift report combines:
94
+
95
+ - server additions and removals;
96
+ - each server's tool/resource/prompt contract changes;
97
+ - newly introduced or resolved cross-server conflicts.
98
+
99
+ `--fail-on` accepts `breaking` (default), `potentially-breaking`, `any`, or `none`, matching the single-server drift gate. Add `--json report.json` for machine-readable CI evidence.
100
+
101
+ ## Composition versus parity
102
+
103
+ Use a **composition** when several servers are mounted together and their namespaces interact. Use **transport parity** when the same logical server is exposed through alternate targets such as local stdio and deployed HTTP. The `servers` and `targets` fields remain intentionally separate.
@@ -0,0 +1,85 @@
1
+ # Performance governance
2
+
3
+ > Available since `1.5.0`.
4
+
5
+ MCP Rigor turns MCP latency into a deterministic release gate. It supports immediate per-call limits, percentile budgets over recorded history, and automatic regression detection against each test's historical baseline.
6
+
7
+ ## Limit one call
8
+
9
+ Add a latency assertion after an action:
10
+
11
+ ```text
12
+ Test: "order lookup stays interactive"
13
+ Call tool "find_order" with:
14
+ orderId: "A-1001"
15
+ Expect the call to finish within 800ms
16
+ ```
17
+
18
+ The measured duration covers the live MCP request and response. A call over the limit fails with `MCP-PERF-001` and reports the measured and allowed duration.
19
+
20
+ YAML parity:
21
+
22
+ ```yaml
23
+ - tool:
24
+ name: find_order
25
+ arguments:
26
+ orderId: A-1001
27
+ assert:
28
+ maxDurationMs: 800
29
+ ```
30
+
31
+ ## Set percentile budgets
32
+
33
+ A percentile budget uses successful durations from recent recorded runs plus the current run:
34
+
35
+ ```text
36
+ Budget: p95 500ms over 20 calls
37
+ Budget for "order lookup stays interactive": p50 300ms over 20 calls
38
+ ```
39
+
40
+ - A suite-wide budget applies independently to every test.
41
+ - A named budget applies only to that test.
42
+ - MCP Rigor uses the deterministic nearest-rank percentile.
43
+ - Fewer than three usable samples reports the budget as pending instead of guessing.
44
+ - A measured percentile over budget fails the CLI run.
45
+
46
+ YAML parity:
47
+
48
+ ```yaml
49
+ budgets:
50
+ - test: "*"
51
+ percentile: 95
52
+ maxMs: 500
53
+ window: 20
54
+ - test: order lookup stays interactive
55
+ percentile: 50
56
+ maxMs: 300
57
+ window: 20
58
+ ```
59
+
60
+ ## Fail on regression without maintaining thresholds
61
+
62
+ ```bash
63
+ mcprigor test tests/orders.mcpr --fail-on-regression
64
+ ```
65
+
66
+ For every successful test, MCP Rigor compares the current duration with the median of its latest successful history. The gate requires at least five samples and reports a regression when the current run exceeds both:
67
+
68
+ - 1.5× the historical median; and
69
+ - the historical median plus 50 ms.
70
+
71
+ The absolute floor prevents very small tests from failing because of ordinary scheduler jitter.
72
+
73
+ ## CI example
74
+
75
+ ```yaml
76
+ - run: npx mcprigor test tests/orders.mcpr --fail-on-regression --junit reports/orders.xml
77
+ ```
78
+
79
+ Explicit budgets answer “is this fast enough?” The regression gate answers “did this release make it materially slower?” Teams commonly use both.
80
+
81
+ ## History source
82
+
83
+ CLI, QA workspace, and MCP-server-driven test runs append to `.mcprigor/workspace-history.jsonl`. The same data powers `mcprigor trends`, CSV/PDF trend exports, flaky detection, percentile budgets, and regression baselines.
84
+
85
+ Commit the test and its budget declarations. Treat history as a CI artifact or retained workspace file according to your evidence policy.
@@ -302,6 +302,34 @@ Test: "summaries embed the sampled text"
302
302
 
303
303
  Suite-wide defaults live in a `Client behavior:` block; these per-test lines override them for one test.
304
304
 
305
+ ## Set latency budgets and catch slow releases
306
+
307
+ Fail a single call that takes too long:
308
+
309
+ ```text
310
+ Test: "order lookup is fast"
311
+ Call tool "find_order" with:
312
+ orderId: "A-1001"
313
+ Expect the call to finish within 800ms
314
+ ```
315
+
316
+ Set suite-level budgets measured as percentiles over recorded run history (the same history behind `mcprigor trends`):
317
+
318
+ ```text
319
+ Budget: p95 500ms over 20 calls
320
+ Budget for "order lookup is fast": p50 300ms
321
+ ```
322
+
323
+ Budgets are judged after each `mcprigor test` run and fail the run when a percentile exceeds its budget. Until enough history exists they report as pending instead of guessing.
324
+
325
+ Gate CI on latency regressions against the trend baseline — no budget numbers needed:
326
+
327
+ ```bash
328
+ mcprigor test suite.mcpr --fail-on-regression
329
+ ```
330
+
331
+ A test regresses when it runs slower than 1.5x its historical median (with a 50 ms floor so micro-tests don't trip on jitter).
332
+
305
333
  ## Match a snapshot
306
334
 
307
335
  ```text
@@ -0,0 +1,53 @@
1
+ # Shareable hosted reports
2
+
3
+ > Available since `1.5.0`.
4
+
5
+ Turn a test run into a URL anyone can open — no repository access, no CI login:
6
+
7
+ ```bash
8
+ export NETLIFY_AUTH_TOKEN=... # personal or CI token
9
+ mcprigor publish tests/catalog.mcpr --site your-netlify-site
10
+ ```
11
+
12
+ `publish` runs the suite, builds the readable HTML report **with the clickable request/response session timeline**, deploys it to your Netlify site with the dependency-free digest API, waits until the deploy is live, and prints the shareable URL:
13
+
14
+ ```
15
+ Published report: https://68b1c2--your-site.netlify.app
16
+ ```
17
+
18
+ Each publish is a normal Netlify deploy of your own site, so access control, custom domains, deploy previews, and retention follow your existing hosting configuration. Unchanged files are skipped automatically via content digests.
19
+
20
+ ## Options
21
+
22
+ ```bash
23
+ mcprigor publish suite.mcpr --site SITE [--include-json] [--test NAME] [--env qa]
24
+ mcprigor publish suite.mcpr --out reports/latest
25
+ ```
26
+
27
+ - `--site` — Netlify site ID or name. The token comes only from `NETLIFY_AUTH_TOKEN` (or `MCPRIGOR_PUBLISH_TOKEN`); tokens are never accepted as command-line flags.
28
+ - `--out DIR` — write the same bundle to a local directory instead of (or in addition to) hosting it. Serve it from any static host: S3, GitHub Pages, nginx, an artifact store.
29
+ - `--include-json` — also publish `result.json` for dashboards and programmatic consumers.
30
+ - `--test`, `--env`, `--command`, `--url` — the same run-selection options as `mcprigor test`.
31
+
32
+ The exit code still reflects the run (`0` passed, `1` failed), so `publish` can replace `test` in a pipeline step that both gates and shares.
33
+
34
+ ## Publishing from the QA workspace UI
35
+
36
+ The visual workspace (`mcprigor workspace`) exposes the same features without the command line:
37
+
38
+ - **HTML report** — every finished test run has an `HTML report` button that opens the full report, including the clickable session timeline, in a new tab.
39
+ - **Publish** — start the workspace with hosting configured and a `Publish` button appears next to the export buttons:
40
+
41
+ ```bash
42
+ export MCPRIGOR_PUBLISH_SITE=your-netlify-site
43
+ export NETLIFY_AUTH_TOKEN=...
44
+ mcprigor workspace
45
+ ```
46
+
47
+ Clicking it deploys the selected run's report and opens the shareable URL; a `View published ↗` link stays on the run for re-opening or copying. Without both variables the button is hidden and the API answers with a clear configuration message — the token itself never reaches the browser.
48
+
49
+ ## Security
50
+
51
+ - The report is produced by the same pipeline as `--html`: secrets and configured redaction patterns are removed **before** the report exists.
52
+ - Publishing makes the report as public as the target site. Use a password-protected or team-restricted Netlify site for internal results.
53
+ - The hosting token is read from the environment at the last moment and is never echoed, logged, or stored.
@@ -30,7 +30,7 @@ An empty folder shows a three-step welcome screen. Choose **+ New test file**
30
30
  2. Edit the plain-language scenario. The editor provides syntax highlighting, line numbers, and autocomplete: top-level declarations at the start of a line, actions and assertions when indented, and comparison phrases after `Expect "field"`. Accept with `Tab` or `Enter`; force the list open with `Ctrl+Space`.
31
31
  3. Choose **Validate** (`Ctrl/⌘+S` saves, `Ctrl/⌘+Enter` runs). A wording problem highlights the offending line and moves the cursor to it.
32
32
  4. Choose **▶ Run tests** or **Parity**.
33
- 5. Review the results panel: per-file pass/fail with durations; select a file for its full report.
33
+ 5. Review the results panel: per-file pass/fail with durations; select a file for its full report. Drag either panel divider to resize the file list, editor, and results areas; widths persist across reloads, arrow keys resize a focused divider, and double-click resets the layout.
34
34
 
35
35
  The editor marks unsaved changes; running or validating saves them first. If the file changed elsewhere after you opened it, the workspace refuses to overwrite it and asks you to reload.
36
36
 
@@ -52,6 +52,10 @@ Test runs are recorded in `.mcprigor/workspace-history.jsonl` (most recent 2000
52
52
 
53
53
  One search box filters all three tabs. It matches suite names, test names, and error text, and highlights matches, so you can answer questions like "when did `delivered` start failing?" without leaving the browser.
54
54
 
55
+ Completed runs expose **PDF**, **CSV**, and **JUnit XML** downloads. The Trends tab exports a rich trends PDF, aggregate CSV, or raw history CSV. PDF reports include summary cards, pass-rate visuals, per-test detail, failures, durations, and evidence identifiers.
56
+
57
+ An **HTML report** button opens the full report — including the clickable request/response session timeline — in a new tab. When the workspace is started with `MCPRIGOR_PUBLISH_SITE` and `NETLIFY_AUTH_TOKEN` set, a **Publish** button also appears: one click hosts the report at a shareable static URL and keeps a `View published ↗` link on the run. See [Shareable hosted reports](PUBLISHING.md).
58
+
55
59
  ## What is available
56
60
 
57
61
  - Creating, renaming, and editing `.mcpr` suites (YAML and JSON suites are listed and editable too)
package/docs/README.md CHANGED
@@ -18,6 +18,12 @@ Choose the path that matches your role.
18
18
  - [State and dependencies](STATE-AND-DEPENDENCIES.md) — share outputs across tests and runs.
19
19
  - [Data engineering](DATA-ENGINEERING.md) — types, filters, joins, samples, and caches.
20
20
  - [Transport parity](TRANSPORT-PARITY.md) — compare stdio and Streamable HTTP.
21
+ - [Performance governance](PERFORMANCE-GOVERNANCE.md) — call limits, percentile budgets, and regression gates.
22
+ - [Multi-server compositions](MULTI-SERVER-COMPOSITIONS.md) — route tests across a fleet and gate combined drift.
23
+ - [Coverage](COVERAGE.md) — find untested MCP surfaces and input-schema branches.
24
+ - [Scheduled monitoring](MONITORING.md) — continuously test production HTTP endpoints and notify webhooks.
25
+ - [GitHub Action](GITHUB-ACTION.md) — test, drift, flaky warnings, and rich pull-request comments.
26
+ - [Shareable hosted reports](PUBLISHING.md) — `mcprigor publish` turns a run into a static report URL.
21
27
 
22
28
  ## Contracts and evidence
23
29
 
@@ -30,6 +36,7 @@ Choose the path that matches your role.
30
36
 
31
37
  - [Extension SDK](EXTENSION-SDK.md)
32
38
  - [Stable error model](ERROR-MODEL.md)
39
+ - [Deterministic security audit](SECURITY-AUDIT.md) — built-in probe pack, severity gate, and rich reports.
33
40
  - [Security and retention](SECURITY-AND-RETENTION.md)
34
41
  - [Compatibility policy](COMPATIBILITY.md)
35
42
 
@@ -0,0 +1,77 @@
1
+ # Deterministic security audit
2
+
3
+ > Available since `1.5.0`.
4
+
5
+ `mcprigor audit` runs a fixed security and permissions probe pack against an MCP server and produces an auditable score. No AI chooses payloads or interprets outcomes.
6
+
7
+ ## Run the safe default pack
8
+
9
+ Use a suite's configured target:
10
+
11
+ ```bash
12
+ mcprigor audit tests/server.mcpr
13
+ ```
14
+
15
+ Or supply a target directly:
16
+
17
+ ```bash
18
+ mcprigor audit --url https://qa.example.com/mcp
19
+ mcprigor audit --command "node dist/server.js"
20
+ ```
21
+
22
+ The non-destructive default probes:
23
+
24
+ | Probe | What MCP Rigor verifies |
25
+ |---|---|
26
+ | Malformed parameters | Invalid `tools/call` parameters are rejected |
27
+ | Tool-name spoofing | An unknown administrative-looking tool cannot be invoked |
28
+ | Oversized payload | A deterministic 1 MiB payload is rejected |
29
+ | Path traversal | Resource reads reject Unix, Windows, and relative traversal URIs |
30
+ | Tool inventory | Advertised tools are listed for opt-in injection/canary testing |
31
+
32
+ ## Opt into reviewed tool calls
33
+
34
+ MCP Rigor never guesses that a tool is safe. Tool execution is disabled unless you allow an exact name:
35
+
36
+ ```bash
37
+ mcprigor audit tests/server.mcpr \
38
+ --allow-tool search \
39
+ --allow-tool summarize
40
+ ```
41
+
42
+ Only allow read-only, non-destructive tools in a disposable test environment. An allowed tool receives a fixed prompt-injection payload containing a deterministic canary. MCP Rigor checks whether the response follows or reflects the payload and whether it exposes the canary. Report evidence replaces the canary with `[CANARY REDACTED]`.
43
+
44
+ ## CI severity gate
45
+
46
+ ```bash
47
+ mcprigor audit tests/server.mcpr --fail-on high
48
+ ```
49
+
50
+ Values are `critical`, `high`, `medium`, `low`, and `none`; the default is `high`. The command exits nonzero when a failed finding meets or exceeds the selected severity.
51
+
52
+ Scoring starts at 100:
53
+
54
+ - critical: −35
55
+ - high: −20
56
+ - medium: −10
57
+ - low: −4
58
+ - skipped: no deduction
59
+
60
+ Skipped probes remain visible with the exact `--allow-tool` needed. They are not represented as passes.
61
+
62
+ ## Reports
63
+
64
+ ```bash
65
+ mcprigor audit tests/server.mcpr \
66
+ --pdf reports/security.pdf \
67
+ --json reports/security.json \
68
+ --csv reports/security.csv
69
+ ```
70
+
71
+ The rich PDF includes score, grade, severity cards, per-finding status, explanations, and bounded evidence. Markdown output is available with `--markdown`.
72
+
73
+ ## Interpreting results
74
+
75
+ The audit is deterministic black-box evidence, not a certification or a replacement for threat modeling. A passing probe proves the observed server rejected that exact payload under that exact configuration. Review skipped probes, environment permissions, downstream model behavior, and business authorization separately.
76
+
77
+ Run intrusive tool probes only in an isolated environment. Do not expose production mutation tools through `--allow-tool`.
@@ -12,11 +12,11 @@
12
12
  "name": "keyword.control.version.mcpr"
13
13
  },
14
14
  {
15
- "match": "^\\s*(Suite|Server|Server options|MCP URL|Compare target|Target options for|Default timeout|Redact|Snapshots|Ignore snapshot paths|Client behavior|Import flows from|Flow|Inputs|Outputs|Test|Scenario|Id|Depends on|Require|Require protocol|Variables|Skip|Setup|Steps|Cleanup|Data source):?",
15
+ "match": "^\\s*(Suite|Server|Named server|Server options|Server options for|MCP URL|Compare target|Target options for|Default timeout|Budget|Redact|Snapshots|Ignore snapshot paths|Client behavior|Import flows from|Flow|Inputs|Outputs|Test|Scenario|Id|Depends on|Require|Require protocol|Variables|Skip|Setup|Steps|Cleanup|Data source):?",
16
16
  "name": "keyword.control.mcpr"
17
17
  },
18
18
  {
19
- "match": "^\\s*(Call tool|When the server asks for input|When the server requests sampling|Read resource|Get prompt|Send|Expect|Save|Export|Set|Use flow|For each row|Wait up to|Wait for notification|Subscribe to resource|Unsubscribe from resource|Set log level|List all|Get task|List tasks|Cancel task|From CSV|From JSON|From YAML|From Excel|From REST|Column|Derive|Keep rows where|Sample|Cache this source)\\b",
19
+ "match": "^\\s*(Call tool|On server|When the server asks for input|When the server requests sampling|Read resource|Get prompt|Send|Expect|Save|Export|Set|Use flow|For each row|Wait up to|Wait for notification|Subscribe to resource|Unsubscribe from resource|Set log level|List all|Get task|List tasks|Cancel task|From CSV|From JSON|From YAML|From Excel|From REST|Column|Derive|Keep rows where|Sample|Cache this source)\\b",
20
20
  "name": "keyword.other.mcpr"
21
21
  },
22
22
  {
@@ -2,7 +2,7 @@
2
2
  "name": "mcprigor",
3
3
  "displayName": "MCP Rigor",
4
4
  "description": "Syntax highlighting and inline validation for MCP Rigor (.mcpr) natural-language MCP test suites.",
5
- "version": "1.3.0",
5
+ "version": "1.4.0",
6
6
  "publisher": "fusionone",
7
7
  "license": "Apache-2.0",
8
8
  "repository": { "type": "git", "url": "https://github.com/FusionOnePlatform/mcprigor" },
@@ -12,11 +12,11 @@
12
12
  "name": "keyword.control.version.mcpr"
13
13
  },
14
14
  {
15
- "match": "^\\s*(Suite|Server|Server options|MCP URL|Compare target|Target options for|Default timeout|Redact|Snapshots|Ignore snapshot paths|Client behavior|Import flows from|Flow|Inputs|Outputs|Test|Scenario|Id|Depends on|Require|Require protocol|Variables|Skip|Setup|Steps|Cleanup|Data source):?",
15
+ "match": "^\\s*(Suite|Server|Named server|Server options|Server options for|MCP URL|Compare target|Target options for|Default timeout|Budget|Redact|Snapshots|Ignore snapshot paths|Client behavior|Import flows from|Flow|Inputs|Outputs|Test|Scenario|Id|Depends on|Require|Require protocol|Variables|Skip|Setup|Steps|Cleanup|Data source):?",
16
16
  "name": "keyword.control.mcpr"
17
17
  },
18
18
  {
19
- "match": "^\\s*(Call tool|When the server asks for input|When the server requests sampling|Read resource|Get prompt|Send|Expect|Save|Export|Set|Use flow|For each row|Wait up to|Wait for notification|Subscribe to resource|Unsubscribe from resource|Set log level|List all|Get task|List tasks|Cancel task|From CSV|From JSON|From YAML|From Excel|From REST|Column|Derive|Keep rows where|Sample|Cache this source)\\b",
19
+ "match": "^\\s*(Call tool|On server|When the server asks for input|When the server requests sampling|Read resource|Get prompt|Send|Expect|Save|Export|Set|Use flow|For each row|Wait up to|Wait for notification|Subscribe to resource|Unsubscribe from resource|Set log level|List all|Get task|List tasks|Cancel task|From CSV|From JSON|From YAML|From Excel|From REST|Column|Derive|Keep rows where|Sample|Cache this source)\\b",
20
20
  "name": "keyword.other.mcpr"
21
21
  },
22
22
  {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mcprigor",
3
- "version": "1.3.1",
3
+ "version": "1.5.0",
4
4
  "description": "Plain-language MCP testing with isolated extensions and a local QA workspace",
5
5
  "type": "module",
6
6
  "bin": {