mcprigor 1.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +17 -0
- package/README.md +233 -0
- package/dist/assertions.d.ts +4 -0
- package/dist/assertions.d.ts.map +1 -0
- package/dist/assertions.js +101 -0
- package/dist/assertions.js.map +1 -0
- package/dist/author.d.ts +57 -0
- package/dist/author.d.ts.map +1 -0
- package/dist/author.js +134 -0
- package/dist/author.js.map +1 -0
- package/dist/canonical.d.ts +4 -0
- package/dist/canonical.d.ts.map +1 -0
- package/dist/canonical.js +17 -0
- package/dist/canonical.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +265 -0
- package/dist/cli.js.map +1 -0
- package/dist/contract.d.ts +28 -0
- package/dist/contract.d.ts.map +1 -0
- package/dist/contract.js +138 -0
- package/dist/contract.js.map +1 -0
- package/dist/data.d.ts +24 -0
- package/dist/data.d.ts.map +1 -0
- package/dist/data.js +367 -0
- package/dist/data.js.map +1 -0
- package/dist/discovery.d.ts +6 -0
- package/dist/discovery.d.ts.map +1 -0
- package/dist/discovery.js +82 -0
- package/dist/discovery.js.map +1 -0
- package/dist/errors.d.ts +19 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +40 -0
- package/dist/errors.js.map +1 -0
- package/dist/extension-host.d.ts +20 -0
- package/dist/extension-host.d.ts.map +1 -0
- package/dist/extension-host.js +52 -0
- package/dist/extension-host.js.map +1 -0
- package/dist/extension-sdk.d.ts +22 -0
- package/dist/extension-sdk.d.ts.map +1 -0
- package/dist/extension-sdk.js +3 -0
- package/dist/extension-sdk.js.map +1 -0
- package/dist/extension-worker.d.ts +2 -0
- package/dist/extension-worker.d.ts.map +1 -0
- package/dist/extension-worker.js +27 -0
- package/dist/extension-worker.js.map +1 -0
- package/dist/extensions.d.ts +15 -0
- package/dist/extensions.d.ts.map +1 -0
- package/dist/extensions.js +60 -0
- package/dist/extensions.js.map +1 -0
- package/dist/imports.d.ts +7 -0
- package/dist/imports.d.ts.map +1 -0
- package/dist/imports.js +41 -0
- package/dist/imports.js.map +1 -0
- package/dist/index.d.ts +29 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/dist/language.d.ts +34 -0
- package/dist/language.d.ts.map +1 -0
- package/dist/language.js +102 -0
- package/dist/language.js.map +1 -0
- package/dist/loader.d.ts +4 -0
- package/dist/loader.d.ts.map +1 -0
- package/dist/loader.js +51 -0
- package/dist/loader.js.map +1 -0
- package/dist/parity.d.ts +23 -0
- package/dist/parity.d.ts.map +1 -0
- package/dist/parity.js +77 -0
- package/dist/parity.js.map +1 -0
- package/dist/path.d.ts +3 -0
- package/dist/path.d.ts.map +1 -0
- package/dist/path.js +71 -0
- package/dist/path.js.map +1 -0
- package/dist/qa-advanced.d.ts +8 -0
- package/dist/qa-advanced.d.ts.map +1 -0
- package/dist/qa-advanced.js +239 -0
- package/dist/qa-advanced.js.map +1 -0
- package/dist/qa-language.d.ts +4 -0
- package/dist/qa-language.d.ts.map +1 -0
- package/dist/qa-language.js +385 -0
- package/dist/qa-language.js.map +1 -0
- package/dist/qa-loader.d.ts +4 -0
- package/dist/qa-loader.d.ts.map +1 -0
- package/dist/qa-loader.js +22 -0
- package/dist/qa-loader.js.map +1 -0
- package/dist/redact.d.ts +7 -0
- package/dist/redact.d.ts.map +1 -0
- package/dist/redact.js +63 -0
- package/dist/redact.js.map +1 -0
- package/dist/replay.d.ts +17 -0
- package/dist/replay.d.ts.map +1 -0
- package/dist/replay.js +48 -0
- package/dist/replay.js.map +1 -0
- package/dist/reporters.d.ts +6 -0
- package/dist/reporters.d.ts.map +1 -0
- package/dist/reporters.js +52 -0
- package/dist/reporters.js.map +1 -0
- package/dist/runner.d.ts +16 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +304 -0
- package/dist/runner.js.map +1 -0
- package/dist/schema.d.ts +446 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +117 -0
- package/dist/schema.js.map +1 -0
- package/dist/session.d.ts +5 -0
- package/dist/session.d.ts.map +1 -0
- package/dist/session.js +111 -0
- package/dist/session.js.map +1 -0
- package/dist/snapshots.d.ts +28 -0
- package/dist/snapshots.d.ts.map +1 -0
- package/dist/snapshots.js +87 -0
- package/dist/snapshots.js.map +1 -0
- package/dist/starter.d.ts +3 -0
- package/dist/starter.d.ts.map +1 -0
- package/dist/starter.js +34 -0
- package/dist/starter.js.map +1 -0
- package/dist/state.d.ts +13 -0
- package/dist/state.d.ts.map +1 -0
- package/dist/state.js +29 -0
- package/dist/state.js.map +1 -0
- package/dist/trace.d.ts +57 -0
- package/dist/trace.d.ts.map +1 -0
- package/dist/trace.js +94 -0
- package/dist/trace.js.map +1 -0
- package/dist/types.d.ts +245 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/workspace.d.ts +10 -0
- package/dist/workspace.d.ts.map +1 -0
- package/dist/workspace.js +144 -0
- package/dist/workspace.js.map +1 -0
- package/docs/CLI-REFERENCE.md +167 -0
- package/docs/COMPATIBILITY.md +42 -0
- package/docs/CONTRACT-DRIFT.md +42 -0
- package/docs/DATA-AND-REUSE.md +213 -0
- package/docs/DATA-ENGINEERING.md +96 -0
- package/docs/ENGINEER-SETUP.md +176 -0
- package/docs/ERROR-MODEL.md +24 -0
- package/docs/EVIDENCE.md +58 -0
- package/docs/EXTENSION-SDK.md +76 -0
- package/docs/FILE-EXTENSION.md +52 -0
- package/docs/GETTING-STARTED.md +126 -0
- package/docs/GUIDED-AUTHORING.md +80 -0
- package/docs/LANDSCAPE.md +81 -0
- package/docs/LANGUAGE-SPEC.md +317 -0
- package/docs/MCP-NATIVE.md +111 -0
- package/docs/PLAIN-LANGUAGE-COOKBOOK.md +219 -0
- package/docs/QA-GUIDE.md +128 -0
- package/docs/QA-WORKSPACE.md +78 -0
- package/docs/README.md +55 -0
- package/docs/SECURITY-AND-RETENTION.md +37 -0
- package/docs/SNAPSHOTS-AND-REPLAY.md +85 -0
- package/docs/SPECIFICATION.md +178 -0
- package/docs/STATE-AND-DEPENDENCIES.md +106 -0
- package/docs/TRANSPORT-PARITY.md +98 -0
- package/docs/TROUBLESHOOTING.md +199 -0
- package/editors/mcpr-language.json +16 -0
- package/package.json +70 -0
- package/workspace-assets/app.js +1 -0
- package/workspace-assets/index.html +1 -0
- package/workspace-assets/style.css +1 -0
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
# MCP Rigor runtime and product specification
|
|
2
|
+
|
|
3
|
+
> Normative runtime behavior and product requirements. For `.mcpr` syntax, use the [language reference](LANGUAGE-SPEC.md). Everyday users should start with [Getting started](GETTING-STARTED.md).
|
|
4
|
+
|
|
5
|
+
## 1. Product definition
|
|
6
|
+
|
|
7
|
+
MCP Rigor is an open-source, deterministic, black-box application test framework for Model Context Protocol servers. Its core promise is:
|
|
8
|
+
|
|
9
|
+
> Author a behavioral MCP scenario once, then run it repeatably against local subprocesses and deployed HTTP servers in CI.
|
|
10
|
+
|
|
11
|
+
MCP Rigor complements interactive debuggers and protocol conformance tooling. It does not claim MCP certification.
|
|
12
|
+
|
|
13
|
+
## 2. Users and jobs
|
|
14
|
+
|
|
15
|
+
- **MCP server authors:** prevent regressions in tools, resources, prompts, errors, and side effects.
|
|
16
|
+
- **Platform teams:** validate third-party servers before promotion or deployment.
|
|
17
|
+
- **SDK maintainers:** run common behavioral scenarios across transports and versions.
|
|
18
|
+
- **Security and QA teams:** build deterministic negative, authorization, and abuse cases.
|
|
19
|
+
|
|
20
|
+
Primary job: turn manually verified MCP interactions into reviewable, version-controlled tests with reliable CI outcomes.
|
|
21
|
+
|
|
22
|
+
## 3. Principles
|
|
23
|
+
|
|
24
|
+
1. Deterministic checks are hard gates; probabilistic evaluation is optional and separate.
|
|
25
|
+
2. Real transport boundaries matter.
|
|
26
|
+
3. Test files contain data, not arbitrary executable code.
|
|
27
|
+
4. Every wait has a deadline; notification tests use events rather than sleeps.
|
|
28
|
+
5. Reports identify protocol version, transport, capability, skips, and evidence.
|
|
29
|
+
6. Secrets are redacted before any result reaches a reporter.
|
|
30
|
+
7. “Conformance” always names an exact versioned profile and never implies certification.
|
|
31
|
+
|
|
32
|
+
## 4. MVP scope
|
|
33
|
+
|
|
34
|
+
### Included
|
|
35
|
+
|
|
36
|
+
- Node.js 20+, TypeScript, ESM
|
|
37
|
+
- YAML and JSON suites
|
|
38
|
+
- stdio subprocess and Streamable HTTP targets
|
|
39
|
+
- one isolated MCP session per test
|
|
40
|
+
- generic MCP requests
|
|
41
|
+
- captures and variable substitution
|
|
42
|
+
- exact, negative, existence, type, subset, length, and regex assertions
|
|
43
|
+
- expected MCP errors
|
|
44
|
+
- per-step deadlines
|
|
45
|
+
- terminal, JSON, and JUnit output
|
|
46
|
+
- CLI filtering and deterministic exit codes
|
|
47
|
+
- TypeScript library API
|
|
48
|
+
|
|
49
|
+
### Deferred
|
|
50
|
+
|
|
51
|
+
- notification queues and progress assertions
|
|
52
|
+
- JSON Schema output matcher
|
|
53
|
+
- snapshots and record/replay
|
|
54
|
+
- setup/teardown fixtures and shared sessions
|
|
55
|
+
- official-conformance adapter and versioned packs
|
|
56
|
+
- OAuth browser flows
|
|
57
|
+
- plugin API, custom reporters, and custom transports
|
|
58
|
+
- fuzzing and security packs
|
|
59
|
+
- retries, parallel/distributed execution, GUI, and LLM judges
|
|
60
|
+
|
|
61
|
+
## 5. Suite model
|
|
62
|
+
|
|
63
|
+
A suite has `version`, optional `name`, one target, defaults, and tests. A test contains sequential steps. Each step sends one request, applies assertions, then captures response leaves.
|
|
64
|
+
|
|
65
|
+
Variables:
|
|
66
|
+
|
|
67
|
+
- `${captureName}` references a test-local captured value.
|
|
68
|
+
- `${env.NAME}` references an environment value.
|
|
69
|
+
- An exact placeholder preserves its JSON type; interpolation into a larger string converts to text.
|
|
70
|
+
- Variables are resolved only in request parameters in the MVP. Target interpolation and centralized secret redaction are required next.
|
|
71
|
+
|
|
72
|
+
Session isolation is the default because order-independent tests are easier to reproduce. Shared sessions may be added only as an explicit opt-in.
|
|
73
|
+
|
|
74
|
+
## 6. Architecture
|
|
75
|
+
|
|
76
|
+
```text
|
|
77
|
+
CLI
|
|
78
|
+
├─ suite loader and validator
|
|
79
|
+
├─ protocol-neutral runner
|
|
80
|
+
│ ├─ variable resolver
|
|
81
|
+
│ ├─ matcher engine
|
|
82
|
+
│ └─ normalized result model
|
|
83
|
+
├─ MCP SDK session
|
|
84
|
+
│ ├─ stdio adapter
|
|
85
|
+
│ └─ Streamable HTTP adapter
|
|
86
|
+
└─ reporters
|
|
87
|
+
├─ terminal
|
|
88
|
+
├─ JSON
|
|
89
|
+
└─ JUnit XML
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
The runner depends on a small `TestSession` interface so fixture sessions and future raw-wire transports can be substituted. The official TypeScript SDK currently performs initialization, protocol validation, framing, and transport lifecycle management.
|
|
93
|
+
|
|
94
|
+
A future monorepo can split stable boundaries into `core`, `cli`, `transport-stdio`, `transport-http`, `conformance`, `reporters`, and `plugin-api` packages. Keeping one package until those boundaries stabilize reduces premature API commitments.
|
|
95
|
+
|
|
96
|
+
## 7. Result and failure semantics
|
|
97
|
+
|
|
98
|
+
Each run records suite status, UTC start time, duration, observed protocol versions, tests, steps, and summary counts. Failures stop the current test after the first failed step but do not stop later tests.
|
|
99
|
+
|
|
100
|
+
Exit codes:
|
|
101
|
+
|
|
102
|
+
- `0`: all selected tests passed
|
|
103
|
+
- `1`: one or more behavioral assertions failed
|
|
104
|
+
- `2`: usage, parsing, or configuration invalid
|
|
105
|
+
- `3`: unexpected infrastructure failure
|
|
106
|
+
|
|
107
|
+
A later version should explicitly distinguish test failure, transport failure, cleanup failure, skipped capability, not applicable, and inconclusive.
|
|
108
|
+
|
|
109
|
+
## 8. Conformance roadmap
|
|
110
|
+
|
|
111
|
+
Versioned profiles should be independently released, for example:
|
|
112
|
+
|
|
113
|
+
```text
|
|
114
|
+
profiles/2025-03-26/lifecycle
|
|
115
|
+
profiles/2025-03-26/tools
|
|
116
|
+
profiles/2025-03-26/resources
|
|
117
|
+
profiles/2025-03-26/prompts
|
|
118
|
+
profiles/2025-03-26/streamable-http
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Checks should include initialization/version negotiation, declared capability behavior, required response/error shapes, ping, cancellation, pagination, progress, logging, subscriptions, unknown methods, invalid parameters, HTTP session IDs, media types, reconnect, and termination.
|
|
122
|
+
|
|
123
|
+
Report statuses must include pass, fail, capability-skipped, not-applicable, and inconclusive. Integrate official MCP Conformance as a separately pinned run and merge evidence rather than duplicating or rebranding it.
|
|
124
|
+
|
|
125
|
+
## 9. Security requirements
|
|
126
|
+
|
|
127
|
+
Before a public beta:
|
|
128
|
+
|
|
129
|
+
- interpolate target environment and headers without writing secrets into results
|
|
130
|
+
- centralize recursive key/value redaction before reporters
|
|
131
|
+
- cap response, stderr, and report sizes
|
|
132
|
+
- guard regex complexity or document trust assumptions
|
|
133
|
+
- avoid shell command strings; continue using command plus argument arrays
|
|
134
|
+
- ensure bounded graceful subprocess shutdown with force-kill fallback
|
|
135
|
+
- sanitize JUnit and terminal control characters
|
|
136
|
+
- provide opt-in environment inheritance and document its threat model
|
|
137
|
+
|
|
138
|
+
Security scanners should be integrations. MCP Rigor’s core identity remains functional and protocol-aware testing.
|
|
139
|
+
|
|
140
|
+
## 10. Release roadmap
|
|
141
|
+
|
|
142
|
+
### 0.1 — executable MVP
|
|
143
|
+
|
|
144
|
+
Core runner, YAML/JSON suites, stdio/HTTP through official SDK, deterministic matchers, captures, CLI, JSON/JUnit, tests, and docs.
|
|
145
|
+
|
|
146
|
+
### 0.2 — CI reliability
|
|
147
|
+
|
|
148
|
+
Published JSON Schema, complete config validation with Ajv, secret redaction, richer diffs, target interpolation, robust transport classification, GitHub Action, Windows/macOS/Linux CI, and deterministic fixture servers.
|
|
149
|
+
|
|
150
|
+
### 0.3 — MCP-native behavior
|
|
151
|
+
|
|
152
|
+
Notification/event queues, progress and cancellation, pagination helpers, capability requirements, resource subscriptions, logging assertions, JSON Schema matcher, traces, snapshots, and record/sanitize/replay.
|
|
153
|
+
|
|
154
|
+
### 0.4 — ecosystem
|
|
155
|
+
|
|
156
|
+
Versioned conformance profiles, official conformance aggregation, plugin API, reusable fixture packs, security/fuzz hooks, and adapters for common MCP frameworks.
|
|
157
|
+
|
|
158
|
+
### 1.0 criteria
|
|
159
|
+
|
|
160
|
+
Stable suite schema and TypeScript API, two supported protocol revisions, robust process cleanup on three operating systems, transport/auth documentation, migration policy, plugin compatibility policy, reproducible fixture matrix, and no high-severity secret leakage paths.
|
|
161
|
+
|
|
162
|
+
## 11. Open-source operating model
|
|
163
|
+
|
|
164
|
+
- Apache-2.0 for broad individual and enterprise use.
|
|
165
|
+
- Public RFCs for schema or plugin-breaking changes.
|
|
166
|
+
- Conventional changesets and a compatibility table per release.
|
|
167
|
+
- `good first issue` fixtures and matcher additions.
|
|
168
|
+
- Governance should clearly document project ownership, maintainership, and vendor neutrality.
|
|
169
|
+
- Reserve project, npm, GitHub organization, and domain names only after registry and trademark checks.
|
|
170
|
+
|
|
171
|
+
## 12. Success metrics
|
|
172
|
+
|
|
173
|
+
- time from install to first passing stdio test under ten minutes
|
|
174
|
+
- less than 1% flaky failures in deterministic project fixtures
|
|
175
|
+
- clean teardown across Linux, macOS, and Windows
|
|
176
|
+
- projects running MCP Rigor in CI weekly
|
|
177
|
+
- community-contributed reusable suites and transport fixtures
|
|
178
|
+
- issue resolution time and repeat contributor rate
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# Passing Outputs Between Tests and Runs
|
|
2
|
+
|
|
3
|
+
MCP Rigor keeps tests isolated by default. Sharing must be explicit, so test ordering remains understandable.
|
|
4
|
+
|
|
5
|
+
## Export an output
|
|
6
|
+
|
|
7
|
+
```text
|
|
8
|
+
Test: "Create customer"
|
|
9
|
+
Id: create-customer
|
|
10
|
+
|
|
11
|
+
Call tool "create_customer" with:
|
|
12
|
+
name: "Alice"
|
|
13
|
+
|
|
14
|
+
Export "structuredContent.id" as "customerId"
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
An export is captured only after the action and its expectations pass. A missing exported field fails the producer test.
|
|
18
|
+
|
|
19
|
+
## Depend on the producer
|
|
20
|
+
|
|
21
|
+
```text
|
|
22
|
+
Test: "Retrieve customer"
|
|
23
|
+
Id: retrieve-customer
|
|
24
|
+
Depends on: create-customer
|
|
25
|
+
|
|
26
|
+
Call tool "get_customer" with:
|
|
27
|
+
id: "${deps.create-customer.customerId}"
|
|
28
|
+
|
|
29
|
+
Expect "structuredContent.name" equals "Alice"
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
MCP Rigor builds a dependency graph and runs producers first, regardless of file order.
|
|
33
|
+
|
|
34
|
+
Rules:
|
|
35
|
+
|
|
36
|
+
- IDs must be unique.
|
|
37
|
+
- Unknown and circular dependencies fail before any server session opens.
|
|
38
|
+
- All dependencies must pass.
|
|
39
|
+
- If one fails, skips, or is blocked, the consumer is marked **blocked** and its MCP session is never opened.
|
|
40
|
+
- Outputs remain namespaced under the producer ID.
|
|
41
|
+
- Tests without dependencies remain isolated.
|
|
42
|
+
|
|
43
|
+
## Data-driven producers
|
|
44
|
+
|
|
45
|
+
Each row has an ID such as `create-customers.us`. Depending on `create-customers` waits for all rows. The consumer receives each exported output as a list in row order:
|
|
46
|
+
|
|
47
|
+
```text
|
|
48
|
+
Test: "Count all customers"
|
|
49
|
+
Depends on: create-customers
|
|
50
|
+
|
|
51
|
+
Call tool "verify_ids" with:
|
|
52
|
+
ids: "${deps.create-customers.customerId}"
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
A failed row blocks the consumer. Depending on one exact row ID is also supported.
|
|
56
|
+
|
|
57
|
+
## Persist across separate CLI runs
|
|
58
|
+
|
|
59
|
+
First run:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
mcprigor test create.mcpr --state-out customer-state.json
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
A state file is written only when the entire run passes. It contains:
|
|
66
|
+
|
|
67
|
+
- format version
|
|
68
|
+
- creation time
|
|
69
|
+
- target fingerprint
|
|
70
|
+
- suite fingerprint
|
|
71
|
+
- exported outputs
|
|
72
|
+
- integrity fingerprint
|
|
73
|
+
|
|
74
|
+
The file is written atomically with owner-only permissions where supported.
|
|
75
|
+
|
|
76
|
+
Later run:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
mcprigor test retrieve.mcpr --state-in customer-state.json
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Use values through the read-only `state` namespace:
|
|
83
|
+
|
|
84
|
+
```text
|
|
85
|
+
Call tool "get_customer" with:
|
|
86
|
+
id: "${state.create-customer.customerId}"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
MCP Rigor rejects state created for a different target. After careful review, it can be overridden:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
mcprigor test retrieve.mcpr \
|
|
93
|
+
--state-in customer-state.json \
|
|
94
|
+
--allow-state-target-mismatch
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Security guidance
|
|
98
|
+
|
|
99
|
+
State files can contain real business data. They are not encrypted.
|
|
100
|
+
|
|
101
|
+
- Never export passwords, access tokens, private keys, or regulated data.
|
|
102
|
+
- Do not commit state files to source control.
|
|
103
|
+
- Keep state short-lived.
|
|
104
|
+
- Prefer a secret manager for credentials.
|
|
105
|
+
- Treat target-mismatch override as an exceptional migration tool.
|
|
106
|
+
- Redaction protects reports, but internal dependency values remain unredacted so tests can use them correctly.
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# Transport Parity Testing
|
|
2
|
+
|
|
3
|
+
MCP Rigor can execute the same acceptance suite against named stdio and Streamable HTTP targets, then compare normalized behavior test by test.
|
|
4
|
+
|
|
5
|
+
## Plain-language parity for QA teams
|
|
6
|
+
|
|
7
|
+
Create `calculator-parity.mcpr`:
|
|
8
|
+
|
|
9
|
+
```text
|
|
10
|
+
MCP Test 1
|
|
11
|
+
|
|
12
|
+
Suite: "Calculator behaves the same everywhere"
|
|
13
|
+
|
|
14
|
+
Compare target "Local": node server.js
|
|
15
|
+
Compare target "QA environment": https://qa.example.com/mcp
|
|
16
|
+
|
|
17
|
+
Test: "Adding 20 and 22 gives 42"
|
|
18
|
+
Id: adding-numbers
|
|
19
|
+
|
|
20
|
+
Call tool "add" with:
|
|
21
|
+
a: 20
|
|
22
|
+
b: 22
|
|
23
|
+
|
|
24
|
+
Expect "structuredContent.sum" equals 42
|
|
25
|
+
|
|
26
|
+
Test: "The service status is ready"
|
|
27
|
+
Id: service-status
|
|
28
|
+
|
|
29
|
+
Read resource "service://status"
|
|
30
|
+
Expect "contents[0].text" contains "ready"
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Run it exactly like a YAML parity suite:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
mcprigor parity calculator-parity.mcpr
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The `Compare target` lines are usually configured once by an engineer. QA authors only add ordinary `Test`, `Call tool`, `Read resource`, and `Expect` statements. Each test automatically runs against every named target.
|
|
40
|
+
|
|
41
|
+
A target value beginning with `http://` or `https://` is Streamable HTTP. Any other value is parsed as a stdio command.
|
|
42
|
+
|
|
43
|
+
## YAML alternative
|
|
44
|
+
|
|
45
|
+
Automation engineers may also use the structured YAML representation:
|
|
46
|
+
|
|
47
|
+
```yaml
|
|
48
|
+
version: 1
|
|
49
|
+
name: Calculator parity
|
|
50
|
+
|
|
51
|
+
target: &default
|
|
52
|
+
transport: stdio
|
|
53
|
+
command: node
|
|
54
|
+
args: [server.js]
|
|
55
|
+
|
|
56
|
+
targets:
|
|
57
|
+
stdio:
|
|
58
|
+
transport: stdio
|
|
59
|
+
command: node
|
|
60
|
+
args: [server.js]
|
|
61
|
+
deployed:
|
|
62
|
+
transport: streamable-http
|
|
63
|
+
url: https://staging.example.com/mcp
|
|
64
|
+
|
|
65
|
+
tests:
|
|
66
|
+
- id: add
|
|
67
|
+
name: Addition is consistent
|
|
68
|
+
steps:
|
|
69
|
+
- tool:
|
|
70
|
+
name: add
|
|
71
|
+
arguments: { a: 20, b: 22 }
|
|
72
|
+
assert:
|
|
73
|
+
json:
|
|
74
|
+
path: $.structuredContent.sum
|
|
75
|
+
equals: 42
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Run the matrix:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
mcprigor parity parity.yaml
|
|
82
|
+
mcprigor parity parity.yaml --markdown --out parity.md
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The first named target is the baseline. Every target runs independently with a fresh MCP session. MCP Rigor compares:
|
|
86
|
+
|
|
87
|
+
- test status;
|
|
88
|
+
- step status and MCP method;
|
|
89
|
+
- normalized response structures;
|
|
90
|
+
- captured/exported outputs.
|
|
91
|
+
|
|
92
|
+
It excludes durations, elapsed times, timestamps, session IDs, and `_meta` transport details. Application response fields are not ignored.
|
|
93
|
+
|
|
94
|
+
A divergent test includes a deterministic path-level semantic diff. The command exits `1` when any target differs.
|
|
95
|
+
|
|
96
|
+
## Scope
|
|
97
|
+
|
|
98
|
+
Parity means equivalent observable behavior under the selected tests. It does not prove byte-level transport equivalence, identical concurrency behavior, network reliability, authentication parity, or production infrastructure correctness.
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# Troubleshooting
|
|
2
|
+
|
|
3
|
+
Start with:
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
mcprigor check your-tests.mcpr
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
This catches wording and configuration problems without starting the MCP server.
|
|
10
|
+
|
|
11
|
+
## The server does not start
|
|
12
|
+
|
|
13
|
+
Typical category:
|
|
14
|
+
|
|
15
|
+
```text
|
|
16
|
+
MCP-SPAWN-* [server-spawn]
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Check:
|
|
20
|
+
|
|
21
|
+
1. Run the `Server:` command directly in the same terminal.
|
|
22
|
+
2. Confirm the executable is installed and on `PATH`.
|
|
23
|
+
3. Check `Server options` → `cwd`.
|
|
24
|
+
4. Build the server before running tests.
|
|
25
|
+
5. Confirm required environment variables exist.
|
|
26
|
+
|
|
27
|
+
## Initialization times out
|
|
28
|
+
|
|
29
|
+
Typical categories:
|
|
30
|
+
|
|
31
|
+
```text
|
|
32
|
+
MCP-INIT-* [initialization]
|
|
33
|
+
MCP-TIMEOUT-* [timeout]
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Check that the stdio server:
|
|
37
|
+
|
|
38
|
+
- writes MCP messages only to stdout;
|
|
39
|
+
- writes diagnostics to stderr;
|
|
40
|
+
- does not wait for interactive input;
|
|
41
|
+
- completes MCP initialization;
|
|
42
|
+
- stays running after startup.
|
|
43
|
+
|
|
44
|
+
For HTTP, verify the URL, authentication, and server logs.
|
|
45
|
+
|
|
46
|
+
## A field was not found
|
|
47
|
+
|
|
48
|
+
Example:
|
|
49
|
+
|
|
50
|
+
```text
|
|
51
|
+
MCP-ASSERT-001 [assertion] $.structuredContent.id expected to exist
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Actions:
|
|
55
|
+
|
|
56
|
+
- inspect the actual response in JSON or evidence;
|
|
57
|
+
- check capitalization and array indexes;
|
|
58
|
+
- use `content[0].text` for text results;
|
|
59
|
+
- use `structuredContent.field` for structured tool output;
|
|
60
|
+
- avoid adding `result.` because paths begin at the returned result.
|
|
61
|
+
|
|
62
|
+
## Expected number, received text
|
|
63
|
+
|
|
64
|
+
Equality is type-sensitive:
|
|
65
|
+
|
|
66
|
+
```text
|
|
67
|
+
Expect "structuredContent.total" equals 2
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
is different from:
|
|
71
|
+
|
|
72
|
+
```text
|
|
73
|
+
Expect "structuredContent.total" equals "2"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Match the server's actual JSON type.
|
|
77
|
+
|
|
78
|
+
## The test was skipped
|
|
79
|
+
|
|
80
|
+
`Require` gates a test on server capability or protocol revision:
|
|
81
|
+
|
|
82
|
+
```text
|
|
83
|
+
Require: tools
|
|
84
|
+
Require protocol: "2025-06-18"
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
A skip means the server did not advertise the requirement. It is not a test failure.
|
|
88
|
+
|
|
89
|
+
## The test was blocked
|
|
90
|
+
|
|
91
|
+
A dependent test is blocked when its producer did not pass:
|
|
92
|
+
|
|
93
|
+
```text
|
|
94
|
+
Depends on: create-customer
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Fix the producer first. MCP Rigor does not open a server session for a blocked consumer.
|
|
98
|
+
|
|
99
|
+
## Data loading failed
|
|
100
|
+
|
|
101
|
+
Typical category:
|
|
102
|
+
|
|
103
|
+
```text
|
|
104
|
+
MCP-DATA-* [data-loading]
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Check:
|
|
108
|
+
|
|
109
|
+
- file path relative to the `.mcpr` file;
|
|
110
|
+
- required column names;
|
|
111
|
+
- number, boolean, date, or JSON values;
|
|
112
|
+
- selected Excel sheet;
|
|
113
|
+
- configured row limit;
|
|
114
|
+
- remote-data permission;
|
|
115
|
+
- remote endpoint returns a JSON array of row objects.
|
|
116
|
+
|
|
117
|
+
Remote data requires:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
mcprigor test FILE --allow-remote-data
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Private and local network destinations are rejected.
|
|
124
|
+
|
|
125
|
+
## An extension failed
|
|
126
|
+
|
|
127
|
+
Typical category:
|
|
128
|
+
|
|
129
|
+
```text
|
|
130
|
+
MCP-EXT-* [extension]
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Check:
|
|
134
|
+
|
|
135
|
+
- `--allow-custom-code` was supplied;
|
|
136
|
+
- module path is in `extensions.allowlist` when configured;
|
|
137
|
+
- manifest declares the function/provider;
|
|
138
|
+
- requested permissions are granted;
|
|
139
|
+
- input and output are JSON-compatible;
|
|
140
|
+
- extension finishes before its timeout.
|
|
141
|
+
|
|
142
|
+
## Snapshot changed
|
|
143
|
+
|
|
144
|
+
Review the path-level diff. If the change is expected:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
mcprigor test FILE --snapshot snapshots.json --update-snapshots
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Never update snapshots automatically in CI. Commit and review changed expectations.
|
|
151
|
+
|
|
152
|
+
## Transport parity differs
|
|
153
|
+
|
|
154
|
+
A parity failure can mean:
|
|
155
|
+
|
|
156
|
+
- different server versions;
|
|
157
|
+
- different fixture data;
|
|
158
|
+
- missing capabilities;
|
|
159
|
+
- a real stdio/HTTP implementation difference;
|
|
160
|
+
- volatile application fields that need an explicit snapshot policy.
|
|
161
|
+
|
|
162
|
+
Compare the path-level difference and verify both environments use equivalent test data.
|
|
163
|
+
|
|
164
|
+
## Cleanup failed
|
|
165
|
+
|
|
166
|
+
Typical category:
|
|
167
|
+
|
|
168
|
+
```text
|
|
169
|
+
MCP-CLEANUP-* [cleanup]
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Make cleanup operations idempotent. A delete should safely handle an item that is already absent. Check that server child processes stop when stdin closes and do not leave descendants running.
|
|
173
|
+
|
|
174
|
+
## Ctrl+C does not return immediately
|
|
175
|
+
|
|
176
|
+
MCP Rigor first closes active MCP clients and transports. The pinned SDK gives stdio servers a graceful shutdown window before escalation. If a server creates child processes, the server must also shut them down.
|
|
177
|
+
|
|
178
|
+
## Collect useful evidence
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
mcprigor test FILE --evidence .mcprigor/debug-run --json result.json
|
|
182
|
+
mcprigor evidence-show .mcprigor/debug-run
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
Before sharing evidence, review it for sensitive business data. Secret redaction does not remove all possible customer content.
|
|
186
|
+
|
|
187
|
+
## Reporting a framework bug
|
|
188
|
+
|
|
189
|
+
Include:
|
|
190
|
+
|
|
191
|
+
- MCP Rigor version;
|
|
192
|
+
- Node and operating-system version;
|
|
193
|
+
- transport type;
|
|
194
|
+
- protocol revision if known;
|
|
195
|
+
- stable error code/category;
|
|
196
|
+
- smallest sanitized `.mcpr` test;
|
|
197
|
+
- sanitized evidence or result JSON.
|
|
198
|
+
|
|
199
|
+
Do not include credentials or regulated data.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json",
|
|
3
|
+
"name": "MCP Rigor Language",
|
|
4
|
+
"scopeName": "source.mcpr",
|
|
5
|
+
"patterns": [
|
|
6
|
+
{ "match": "^\\s*#.*$", "name": "comment.line.number-sign.mcpr" },
|
|
7
|
+
{ "match": "^MCP Test 1$", "name": "keyword.control.version.mcpr" },
|
|
8
|
+
{ "match": "^\\s*(Suite|Server|Server options|MCP URL|Compare target|Target options for|Default timeout|Redact|Snapshots|Ignore snapshot paths|Client behavior|Import flows from|Flow|Inputs|Outputs|Test|Scenario|Id|Depends on|Require|Require protocol|Variables|Skip|Setup|Steps|Cleanup|Data source):?", "name": "keyword.control.mcpr" },
|
|
9
|
+
{ "match": "^\\s*(Call tool|Read resource|Get prompt|Send|Expect|Save|Export|Set|Use flow|For each row|Wait up to|Wait for notification|Subscribe to resource|Unsubscribe from resource|Set log level|List all|Get task|List tasks|Cancel task|From CSV|From JSON|From YAML|From Excel|From REST|Column|Derive|Keep rows where|Sample|Cache this source)\\b", "name": "keyword.other.mcpr" },
|
|
10
|
+
{ "begin": "\"", "end": "\"", "name": "string.quoted.double.mcpr" },
|
|
11
|
+
{ "begin": "'", "end": "'", "name": "string.quoted.single.mcpr" },
|
|
12
|
+
{ "match": "\\$\\{[^}]+\\}", "name": "variable.other.mcpr" },
|
|
13
|
+
{ "match": "\\b(true|false|null)\\b", "name": "constant.language.mcpr" },
|
|
14
|
+
{ "match": "\\b-?[0-9]+(?:\\.[0-9]+)?\\b", "name": "constant.numeric.mcpr" }
|
|
15
|
+
]
|
|
16
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "mcprigor",
|
|
3
|
+
"version": "1.0.0-rc.1",
|
|
4
|
+
"description": "Plain-language MCP testing with isolated extensions and a local QA workspace",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"mcprigor": "dist/cli.js"
|
|
8
|
+
},
|
|
9
|
+
"main": "./dist/index.js",
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"exports": {
|
|
12
|
+
".": {
|
|
13
|
+
"types": "./dist/index.d.ts",
|
|
14
|
+
"import": "./dist/index.js"
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"dist",
|
|
19
|
+
"schema",
|
|
20
|
+
"editors",
|
|
21
|
+
"docs",
|
|
22
|
+
"workspace-assets",
|
|
23
|
+
"README.md",
|
|
24
|
+
"LICENSE"
|
|
25
|
+
],
|
|
26
|
+
"scripts": {
|
|
27
|
+
"build": "tsc -p tsconfig.json",
|
|
28
|
+
"dev": "tsx src/cli.ts",
|
|
29
|
+
"test": "vitest run",
|
|
30
|
+
"test:watch": "vitest",
|
|
31
|
+
"check": "npm run build && npm test"
|
|
32
|
+
},
|
|
33
|
+
"keywords": [
|
|
34
|
+
"mcp",
|
|
35
|
+
"model-context-protocol",
|
|
36
|
+
"testing",
|
|
37
|
+
"qa",
|
|
38
|
+
"test-automation",
|
|
39
|
+
"plain-language",
|
|
40
|
+
"contract-testing",
|
|
41
|
+
"cli"
|
|
42
|
+
],
|
|
43
|
+
"license": "Apache-2.0",
|
|
44
|
+
"engines": {
|
|
45
|
+
"node": ">=20"
|
|
46
|
+
},
|
|
47
|
+
"dependencies": {
|
|
48
|
+
"@modelcontextprotocol/sdk": "1.27.1",
|
|
49
|
+
"ajv": "^8.18.0",
|
|
50
|
+
"exceljs": "^4.4.0",
|
|
51
|
+
"yaml": "^2.8.2"
|
|
52
|
+
},
|
|
53
|
+
"overrides": {
|
|
54
|
+
"uuid": "^11.1.1"
|
|
55
|
+
},
|
|
56
|
+
"devDependencies": {
|
|
57
|
+
"@types/node": "^22.19.15",
|
|
58
|
+
"tsx": "^4.21.0",
|
|
59
|
+
"typescript": "^5.9.3",
|
|
60
|
+
"vitest": "^4.0.18"
|
|
61
|
+
},
|
|
62
|
+
"repository": {
|
|
63
|
+
"type": "git",
|
|
64
|
+
"url": "git+https://github.com/FusionOnePlatform/mcprigor.git"
|
|
65
|
+
},
|
|
66
|
+
"homepage": "https://mcprigor.com",
|
|
67
|
+
"bugs": {
|
|
68
|
+
"url": "https://github.com/FusionOnePlatform/mcprigor/issues"
|
|
69
|
+
}
|
|
70
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const state={csrf:'',path:'',etag:'',dirty:false};const $=id=>document.getElementById(id);async function api(path,options={}){const headers={...(options.body?{'content-type':'application/json'}:{}),...(options.method&&options.method!=='GET'?{'x-mcp-csrf':state.csrf}:{})};const response=await fetch(path,{...options,headers:{...headers,...options.headers}});const value=await response.json();if(!response.ok)throw new Error(value.error?.message||`HTTP ${response.status}`);return value}async function start(){const boot=await api('/api/v1/bootstrap');state.csrf=boot.csrf;$('workspace').textContent=`Workspace: ${boot.root} · MCP Rigor ${boot.version}`;$('connection').textContent='● Local and ready';await suites()}async function suites(){const value=await api('/api/v1/suites');const nav=$('suites');nav.replaceChildren();for(const suite of value.suites){const button=document.createElement('button');button.textContent=suite.path;button.onclick=()=>open(suite.path,button);nav.append(button)}}async function open(path,button){if(state.dirty&&!confirm('Discard unsaved changes?'))return;const value=await api(`/api/v1/file?path=${encodeURIComponent(path)}`);state.path=path;state.etag=value.etag;state.dirty=false;$('editor').value=value.text;$('dirty').textContent='';document.querySelectorAll('nav button').forEach(x=>x.classList.remove('active'));button.classList.add('active');$('diagnostics').textContent=`Opened ${path}`}async function save(){if(!state.path)return;const value=await api('/api/v1/file',{method:'PUT',body:JSON.stringify({path:state.path,text:$('editor').value,etag:state.etag})});state.etag=value.etag;state.dirty=false;$('dirty').textContent='';$('diagnostics').textContent='Saved'}async function validate(){if(state.dirty)await save();const value=await api('/api/v1/validate',{method:'POST',body:JSON.stringify({path:state.path})});$('diagnostics').textContent=value.valid?`✓ Valid · ${value.suite.tests} tests${value.suite.parityTargets.length?` · ${value.suite.parityTargets.length} parity targets`:''}`:value.diagnostics.map(x=>x.message).join('\n')}async function run(mode){if(state.dirty)await save();$('output').textContent='Starting…';const value=await api('/api/v1/runs',{method:'POST',body:JSON.stringify({path:state.path,mode})});const poll=setInterval(async()=>{try{const result=await api(`/api/v1/runs/${value.runId}`);$('output').textContent=result.output;if(result.status!=='running'){clearInterval(poll);$('diagnostics').textContent=result.status==='passed'?'✓ Run passed':'✗ Run failed'}}catch(error){clearInterval(poll);$('output').textContent=error.message}},500)}$('editor').addEventListener('input',()=>{state.dirty=true;$('dirty').textContent='● Unsaved'});$('save').onclick=()=>save().catch(show);$('validate').onclick=()=>validate().catch(show);$('run').onclick=()=>run('test').catch(show);$('parity').onclick=()=>run('parity').catch(show);$('refresh').onclick=()=>suites().catch(show);function show(error){$('diagnostics').textContent=error.message}start().catch(show);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>MCP Rigor QA Workspace</title><link rel="stylesheet" href="/style.css"></head><body><header><div><strong>MCP Rigor</strong><span>QA Workspace</span></div><div id="connection">Connecting…</div></header><main><aside><button id="refresh">↻ Refresh suites</button><nav id="suites" aria-label="Test suites"></nav></aside><section class="workbench"><div class="toolbar"><button id="save">Save</button><button id="validate">Validate</button><button id="run">Run tests</button><button id="parity">Run parity</button><span id="dirty"></span></div><textarea id="editor" spellcheck="false" aria-label="Plain-language MCP test editor" placeholder="Select a suite or create a .mcp file in your workspace"></textarea><div id="diagnostics" role="status" aria-live="polite"></div></section><aside class="results"><h2>Run results</h2><pre id="output">No run yet.</pre></aside></main><footer><span id="workspace"></span><span>Plain language and YAML share one runtime model</span></footer><script type="module" src="/app.js"></script></body></html>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
:root{font:15px/1.5 Inter,ui-sans-serif,system-ui;color:#15231d;background:#edf3ef}*{box-sizing:border-box}body{margin:0;height:100vh;display:grid;grid-template-rows:58px 1fr 34px}header,footer{display:flex;align-items:center;justify-content:space-between;padding:0 22px;background:#10291f;color:#e7fff4}header strong{font-size:20px;margin-right:12px}header span,footer{color:#a9cbbb}main{display:grid;grid-template-columns:230px minmax(400px,1fr) minmax(300px,38%);min-height:0}aside{padding:14px;border-right:1px solid #c7d5cd;background:#f7faf8;overflow:auto}.results{border:0;border-left:1px solid #c7d5cd;background:#102019;color:#dff4e8}.results h2{font-size:14px;text-transform:uppercase;letter-spacing:.08em}.workbench{display:grid;grid-template-rows:52px 1fr auto;min-width:0}.toolbar{display:flex;align-items:center;gap:8px;padding:9px 14px;background:white;border-bottom:1px solid #c7d5cd}button{border:1px solid #7d9b8b;border-radius:7px;padding:7px 11px;background:white;color:#173c2c;cursor:pointer}button:hover,button:focus{background:#dff3e7;outline:2px solid transparent}nav button{display:block;width:100%;text-align:left;margin:5px 0;overflow:hidden;text-overflow:ellipsis}nav button.active{background:#bde6cc;border-color:#238653}textarea{resize:none;border:0;padding:20px;font:14px/1.65 ui-monospace,SFMono-Regular,Menlo,monospace;outline:none;color:#17221d;tab-size:2}#diagnostics{padding:8px 14px;min-height:38px;background:#fff8df;color:#6a4c00;white-space:pre-wrap}pre{white-space:pre-wrap;word-break:break-word;font:13px/1.55 ui-monospace;padding:8px 0}footer{font-size:12px}#dirty{color:#aa5b00}@media(max-width:900px){main{grid-template-columns:180px 1fr}.results{display:none}}@media(max-width:620px){main{grid-template-columns:1fr}main>aside:first-child{max-height:150px;border-bottom:1px solid #ccc}.workbench{min-height:500px}}
|