mcprigor 1.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/LICENSE +17 -0
  2. package/README.md +233 -0
  3. package/dist/assertions.d.ts +4 -0
  4. package/dist/assertions.d.ts.map +1 -0
  5. package/dist/assertions.js +101 -0
  6. package/dist/assertions.js.map +1 -0
  7. package/dist/author.d.ts +57 -0
  8. package/dist/author.d.ts.map +1 -0
  9. package/dist/author.js +134 -0
  10. package/dist/author.js.map +1 -0
  11. package/dist/canonical.d.ts +4 -0
  12. package/dist/canonical.d.ts.map +1 -0
  13. package/dist/canonical.js +17 -0
  14. package/dist/canonical.js.map +1 -0
  15. package/dist/cli.d.ts +3 -0
  16. package/dist/cli.d.ts.map +1 -0
  17. package/dist/cli.js +265 -0
  18. package/dist/cli.js.map +1 -0
  19. package/dist/contract.d.ts +28 -0
  20. package/dist/contract.d.ts.map +1 -0
  21. package/dist/contract.js +138 -0
  22. package/dist/contract.js.map +1 -0
  23. package/dist/data.d.ts +24 -0
  24. package/dist/data.d.ts.map +1 -0
  25. package/dist/data.js +367 -0
  26. package/dist/data.js.map +1 -0
  27. package/dist/discovery.d.ts +6 -0
  28. package/dist/discovery.d.ts.map +1 -0
  29. package/dist/discovery.js +82 -0
  30. package/dist/discovery.js.map +1 -0
  31. package/dist/errors.d.ts +19 -0
  32. package/dist/errors.d.ts.map +1 -0
  33. package/dist/errors.js +40 -0
  34. package/dist/errors.js.map +1 -0
  35. package/dist/extension-host.d.ts +20 -0
  36. package/dist/extension-host.d.ts.map +1 -0
  37. package/dist/extension-host.js +52 -0
  38. package/dist/extension-host.js.map +1 -0
  39. package/dist/extension-sdk.d.ts +22 -0
  40. package/dist/extension-sdk.d.ts.map +1 -0
  41. package/dist/extension-sdk.js +3 -0
  42. package/dist/extension-sdk.js.map +1 -0
  43. package/dist/extension-worker.d.ts +2 -0
  44. package/dist/extension-worker.d.ts.map +1 -0
  45. package/dist/extension-worker.js +27 -0
  46. package/dist/extension-worker.js.map +1 -0
  47. package/dist/extensions.d.ts +15 -0
  48. package/dist/extensions.d.ts.map +1 -0
  49. package/dist/extensions.js +60 -0
  50. package/dist/extensions.js.map +1 -0
  51. package/dist/imports.d.ts +7 -0
  52. package/dist/imports.d.ts.map +1 -0
  53. package/dist/imports.js +41 -0
  54. package/dist/imports.js.map +1 -0
  55. package/dist/index.d.ts +29 -0
  56. package/dist/index.d.ts.map +1 -0
  57. package/dist/index.js +28 -0
  58. package/dist/index.js.map +1 -0
  59. package/dist/language.d.ts +34 -0
  60. package/dist/language.d.ts.map +1 -0
  61. package/dist/language.js +102 -0
  62. package/dist/language.js.map +1 -0
  63. package/dist/loader.d.ts +4 -0
  64. package/dist/loader.d.ts.map +1 -0
  65. package/dist/loader.js +51 -0
  66. package/dist/loader.js.map +1 -0
  67. package/dist/parity.d.ts +23 -0
  68. package/dist/parity.d.ts.map +1 -0
  69. package/dist/parity.js +77 -0
  70. package/dist/parity.js.map +1 -0
  71. package/dist/path.d.ts +3 -0
  72. package/dist/path.d.ts.map +1 -0
  73. package/dist/path.js +71 -0
  74. package/dist/path.js.map +1 -0
  75. package/dist/qa-advanced.d.ts +8 -0
  76. package/dist/qa-advanced.d.ts.map +1 -0
  77. package/dist/qa-advanced.js +239 -0
  78. package/dist/qa-advanced.js.map +1 -0
  79. package/dist/qa-language.d.ts +4 -0
  80. package/dist/qa-language.d.ts.map +1 -0
  81. package/dist/qa-language.js +385 -0
  82. package/dist/qa-language.js.map +1 -0
  83. package/dist/qa-loader.d.ts +4 -0
  84. package/dist/qa-loader.d.ts.map +1 -0
  85. package/dist/qa-loader.js +22 -0
  86. package/dist/qa-loader.js.map +1 -0
  87. package/dist/redact.d.ts +7 -0
  88. package/dist/redact.d.ts.map +1 -0
  89. package/dist/redact.js +63 -0
  90. package/dist/redact.js.map +1 -0
  91. package/dist/replay.d.ts +17 -0
  92. package/dist/replay.d.ts.map +1 -0
  93. package/dist/replay.js +48 -0
  94. package/dist/replay.js.map +1 -0
  95. package/dist/reporters.d.ts +6 -0
  96. package/dist/reporters.d.ts.map +1 -0
  97. package/dist/reporters.js +52 -0
  98. package/dist/reporters.js.map +1 -0
  99. package/dist/runner.d.ts +16 -0
  100. package/dist/runner.d.ts.map +1 -0
  101. package/dist/runner.js +304 -0
  102. package/dist/runner.js.map +1 -0
  103. package/dist/schema.d.ts +446 -0
  104. package/dist/schema.d.ts.map +1 -0
  105. package/dist/schema.js +117 -0
  106. package/dist/schema.js.map +1 -0
  107. package/dist/session.d.ts +5 -0
  108. package/dist/session.d.ts.map +1 -0
  109. package/dist/session.js +111 -0
  110. package/dist/session.js.map +1 -0
  111. package/dist/snapshots.d.ts +28 -0
  112. package/dist/snapshots.d.ts.map +1 -0
  113. package/dist/snapshots.js +87 -0
  114. package/dist/snapshots.js.map +1 -0
  115. package/dist/starter.d.ts +3 -0
  116. package/dist/starter.d.ts.map +1 -0
  117. package/dist/starter.js +34 -0
  118. package/dist/starter.js.map +1 -0
  119. package/dist/state.d.ts +13 -0
  120. package/dist/state.d.ts.map +1 -0
  121. package/dist/state.js +29 -0
  122. package/dist/state.js.map +1 -0
  123. package/dist/trace.d.ts +57 -0
  124. package/dist/trace.d.ts.map +1 -0
  125. package/dist/trace.js +94 -0
  126. package/dist/trace.js.map +1 -0
  127. package/dist/types.d.ts +245 -0
  128. package/dist/types.d.ts.map +1 -0
  129. package/dist/types.js +2 -0
  130. package/dist/types.js.map +1 -0
  131. package/dist/workspace.d.ts +10 -0
  132. package/dist/workspace.d.ts.map +1 -0
  133. package/dist/workspace.js +144 -0
  134. package/dist/workspace.js.map +1 -0
  135. package/docs/CLI-REFERENCE.md +167 -0
  136. package/docs/COMPATIBILITY.md +42 -0
  137. package/docs/CONTRACT-DRIFT.md +42 -0
  138. package/docs/DATA-AND-REUSE.md +213 -0
  139. package/docs/DATA-ENGINEERING.md +96 -0
  140. package/docs/ENGINEER-SETUP.md +176 -0
  141. package/docs/ERROR-MODEL.md +24 -0
  142. package/docs/EVIDENCE.md +58 -0
  143. package/docs/EXTENSION-SDK.md +76 -0
  144. package/docs/FILE-EXTENSION.md +52 -0
  145. package/docs/GETTING-STARTED.md +126 -0
  146. package/docs/GUIDED-AUTHORING.md +80 -0
  147. package/docs/LANDSCAPE.md +81 -0
  148. package/docs/LANGUAGE-SPEC.md +317 -0
  149. package/docs/MCP-NATIVE.md +111 -0
  150. package/docs/PLAIN-LANGUAGE-COOKBOOK.md +219 -0
  151. package/docs/QA-GUIDE.md +128 -0
  152. package/docs/QA-WORKSPACE.md +78 -0
  153. package/docs/README.md +55 -0
  154. package/docs/SECURITY-AND-RETENTION.md +37 -0
  155. package/docs/SNAPSHOTS-AND-REPLAY.md +85 -0
  156. package/docs/SPECIFICATION.md +178 -0
  157. package/docs/STATE-AND-DEPENDENCIES.md +106 -0
  158. package/docs/TRANSPORT-PARITY.md +98 -0
  159. package/docs/TROUBLESHOOTING.md +199 -0
  160. package/editors/mcpr-language.json +16 -0
  161. package/package.json +70 -0
  162. package/workspace-assets/app.js +1 -0
  163. package/workspace-assets/index.html +1 -0
  164. package/workspace-assets/style.css +1 -0
@@ -0,0 +1,178 @@
1
+ # MCP Rigor runtime and product specification
2
+
3
+ > Normative runtime behavior and product requirements. For `.mcpr` syntax, use the [language reference](LANGUAGE-SPEC.md). Everyday users should start with [Getting started](GETTING-STARTED.md).
4
+
5
+ ## 1. Product definition
6
+
7
+ MCP Rigor is an open-source, deterministic, black-box application test framework for Model Context Protocol servers. Its core promise is:
8
+
9
+ > Author a behavioral MCP scenario once, then run it repeatably against local subprocesses and deployed HTTP servers in CI.
10
+
11
+ MCP Rigor complements interactive debuggers and protocol conformance tooling. It does not claim MCP certification.
12
+
13
+ ## 2. Users and jobs
14
+
15
+ - **MCP server authors:** prevent regressions in tools, resources, prompts, errors, and side effects.
16
+ - **Platform teams:** validate third-party servers before promotion or deployment.
17
+ - **SDK maintainers:** run common behavioral scenarios across transports and versions.
18
+ - **Security and QA teams:** build deterministic negative, authorization, and abuse cases.
19
+
20
+ Primary job: turn manually verified MCP interactions into reviewable, version-controlled tests with reliable CI outcomes.
21
+
22
+ ## 3. Principles
23
+
24
+ 1. Deterministic checks are hard gates; probabilistic evaluation is optional and separate.
25
+ 2. Real transport boundaries matter.
26
+ 3. Test files contain data, not arbitrary executable code.
27
+ 4. Every wait has a deadline; notification tests use events rather than sleeps.
28
+ 5. Reports identify protocol version, transport, capability, skips, and evidence.
29
+ 6. Secrets are redacted before any result reaches a reporter.
30
+ 7. “Conformance” always names an exact versioned profile and never implies certification.
31
+
32
+ ## 4. MVP scope
33
+
34
+ ### Included
35
+
36
+ - Node.js 20+, TypeScript, ESM
37
+ - YAML and JSON suites
38
+ - stdio subprocess and Streamable HTTP targets
39
+ - one isolated MCP session per test
40
+ - generic MCP requests
41
+ - captures and variable substitution
42
+ - exact, negative, existence, type, subset, length, and regex assertions
43
+ - expected MCP errors
44
+ - per-step deadlines
45
+ - terminal, JSON, and JUnit output
46
+ - CLI filtering and deterministic exit codes
47
+ - TypeScript library API
48
+
49
+ ### Deferred
50
+
51
+ - notification queues and progress assertions
52
+ - JSON Schema output matcher
53
+ - snapshots and record/replay
54
+ - setup/teardown fixtures and shared sessions
55
+ - official-conformance adapter and versioned packs
56
+ - OAuth browser flows
57
+ - plugin API, custom reporters, and custom transports
58
+ - fuzzing and security packs
59
+ - retries, parallel/distributed execution, GUI, and LLM judges
60
+
61
+ ## 5. Suite model
62
+
63
+ A suite has `version`, optional `name`, one target, defaults, and tests. A test contains sequential steps. Each step sends one request, applies assertions, then captures response leaves.
64
+
65
+ Variables:
66
+
67
+ - `${captureName}` references a test-local captured value.
68
+ - `${env.NAME}` references an environment value.
69
+ - An exact placeholder preserves its JSON type; interpolation into a larger string converts to text.
70
+ - Variables are resolved only in request parameters in the MVP. Target interpolation and centralized secret redaction are required next.
71
+
72
+ Session isolation is the default because order-independent tests are easier to reproduce. Shared sessions may be added only as an explicit opt-in.
73
+
74
+ ## 6. Architecture
75
+
76
+ ```text
77
+ CLI
78
+ ├─ suite loader and validator
79
+ ├─ protocol-neutral runner
80
+ │ ├─ variable resolver
81
+ │ ├─ matcher engine
82
+ │ └─ normalized result model
83
+ ├─ MCP SDK session
84
+ │ ├─ stdio adapter
85
+ │ └─ Streamable HTTP adapter
86
+ └─ reporters
87
+ ├─ terminal
88
+ ├─ JSON
89
+ └─ JUnit XML
90
+ ```
91
+
92
+ The runner depends on a small `TestSession` interface so fixture sessions and future raw-wire transports can be substituted. The official TypeScript SDK currently performs initialization, protocol validation, framing, and transport lifecycle management.
93
+
94
+ A future monorepo can split stable boundaries into `core`, `cli`, `transport-stdio`, `transport-http`, `conformance`, `reporters`, and `plugin-api` packages. Keeping one package until those boundaries stabilize reduces premature API commitments.
95
+
96
+ ## 7. Result and failure semantics
97
+
98
+ Each run records suite status, UTC start time, duration, observed protocol versions, tests, steps, and summary counts. Failures stop the current test after the first failed step but do not stop later tests.
99
+
100
+ Exit codes:
101
+
102
+ - `0`: all selected tests passed
103
+ - `1`: one or more behavioral assertions failed
104
+ - `2`: usage, parsing, or configuration invalid
105
+ - `3`: unexpected infrastructure failure
106
+
107
+ A later version should explicitly distinguish test failure, transport failure, cleanup failure, skipped capability, not applicable, and inconclusive.
108
+
109
+ ## 8. Conformance roadmap
110
+
111
+ Versioned profiles should be independently released, for example:
112
+
113
+ ```text
114
+ profiles/2025-03-26/lifecycle
115
+ profiles/2025-03-26/tools
116
+ profiles/2025-03-26/resources
117
+ profiles/2025-03-26/prompts
118
+ profiles/2025-03-26/streamable-http
119
+ ```
120
+
121
+ Checks should include initialization/version negotiation, declared capability behavior, required response/error shapes, ping, cancellation, pagination, progress, logging, subscriptions, unknown methods, invalid parameters, HTTP session IDs, media types, reconnect, and termination.
122
+
123
+ Report statuses must include pass, fail, capability-skipped, not-applicable, and inconclusive. Integrate official MCP Conformance as a separately pinned run and merge evidence rather than duplicating or rebranding it.
124
+
125
+ ## 9. Security requirements
126
+
127
+ Before a public beta:
128
+
129
+ - interpolate target environment and headers without writing secrets into results
130
+ - centralize recursive key/value redaction before reporters
131
+ - cap response, stderr, and report sizes
132
+ - guard regex complexity or document trust assumptions
133
+ - avoid shell command strings; continue using command plus argument arrays
134
+ - ensure bounded graceful subprocess shutdown with force-kill fallback
135
+ - sanitize JUnit and terminal control characters
136
+ - provide opt-in environment inheritance and document its threat model
137
+
138
+ Security scanners should be integrations. MCP Rigor’s core identity remains functional and protocol-aware testing.
139
+
140
+ ## 10. Release roadmap
141
+
142
+ ### 0.1 — executable MVP
143
+
144
+ Core runner, YAML/JSON suites, stdio/HTTP through official SDK, deterministic matchers, captures, CLI, JSON/JUnit, tests, and docs.
145
+
146
+ ### 0.2 — CI reliability
147
+
148
+ Published JSON Schema, complete config validation with Ajv, secret redaction, richer diffs, target interpolation, robust transport classification, GitHub Action, Windows/macOS/Linux CI, and deterministic fixture servers.
149
+
150
+ ### 0.3 — MCP-native behavior
151
+
152
+ Notification/event queues, progress and cancellation, pagination helpers, capability requirements, resource subscriptions, logging assertions, JSON Schema matcher, traces, snapshots, and record/sanitize/replay.
153
+
154
+ ### 0.4 — ecosystem
155
+
156
+ Versioned conformance profiles, official conformance aggregation, plugin API, reusable fixture packs, security/fuzz hooks, and adapters for common MCP frameworks.
157
+
158
+ ### 1.0 criteria
159
+
160
+ Stable suite schema and TypeScript API, two supported protocol revisions, robust process cleanup on three operating systems, transport/auth documentation, migration policy, plugin compatibility policy, reproducible fixture matrix, and no high-severity secret leakage paths.
161
+
162
+ ## 11. Open-source operating model
163
+
164
+ - Apache-2.0 for broad individual and enterprise use.
165
+ - Public RFCs for schema or plugin-breaking changes.
166
+ - Conventional changesets and a compatibility table per release.
167
+ - `good first issue` fixtures and matcher additions.
168
+ - Governance should clearly document project ownership, maintainership, and vendor neutrality.
169
+ - Reserve project, npm, GitHub organization, and domain names only after registry and trademark checks.
170
+
171
+ ## 12. Success metrics
172
+
173
+ - time from install to first passing stdio test under ten minutes
174
+ - less than 1% flaky failures in deterministic project fixtures
175
+ - clean teardown across Linux, macOS, and Windows
176
+ - projects running MCP Rigor in CI weekly
177
+ - community-contributed reusable suites and transport fixtures
178
+ - issue resolution time and repeat contributor rate
@@ -0,0 +1,106 @@
1
+ # Passing Outputs Between Tests and Runs
2
+
3
+ MCP Rigor keeps tests isolated by default. Sharing must be explicit, so test ordering remains understandable.
4
+
5
+ ## Export an output
6
+
7
+ ```text
8
+ Test: "Create customer"
9
+ Id: create-customer
10
+
11
+ Call tool "create_customer" with:
12
+ name: "Alice"
13
+
14
+ Export "structuredContent.id" as "customerId"
15
+ ```
16
+
17
+ An export is captured only after the action and its expectations pass. A missing exported field fails the producer test.
18
+
19
+ ## Depend on the producer
20
+
21
+ ```text
22
+ Test: "Retrieve customer"
23
+ Id: retrieve-customer
24
+ Depends on: create-customer
25
+
26
+ Call tool "get_customer" with:
27
+ id: "${deps.create-customer.customerId}"
28
+
29
+ Expect "structuredContent.name" equals "Alice"
30
+ ```
31
+
32
+ MCP Rigor builds a dependency graph and runs producers first, regardless of file order.
33
+
34
+ Rules:
35
+
36
+ - IDs must be unique.
37
+ - Unknown and circular dependencies fail before any server session opens.
38
+ - All dependencies must pass.
39
+ - If one fails, skips, or is blocked, the consumer is marked **blocked** and its MCP session is never opened.
40
+ - Outputs remain namespaced under the producer ID.
41
+ - Tests without dependencies remain isolated.
42
+
43
+ ## Data-driven producers
44
+
45
+ Each row has an ID such as `create-customers.us`. Depending on `create-customers` waits for all rows. The consumer receives each exported output as a list in row order:
46
+
47
+ ```text
48
+ Test: "Count all customers"
49
+ Depends on: create-customers
50
+
51
+ Call tool "verify_ids" with:
52
+ ids: "${deps.create-customers.customerId}"
53
+ ```
54
+
55
+ A failed row blocks the consumer. Depending on one exact row ID is also supported.
56
+
57
+ ## Persist across separate CLI runs
58
+
59
+ First run:
60
+
61
+ ```bash
62
+ mcprigor test create.mcpr --state-out customer-state.json
63
+ ```
64
+
65
+ A state file is written only when the entire run passes. It contains:
66
+
67
+ - format version
68
+ - creation time
69
+ - target fingerprint
70
+ - suite fingerprint
71
+ - exported outputs
72
+ - integrity fingerprint
73
+
74
+ The file is written atomically with owner-only permissions where supported.
75
+
76
+ Later run:
77
+
78
+ ```bash
79
+ mcprigor test retrieve.mcpr --state-in customer-state.json
80
+ ```
81
+
82
+ Use values through the read-only `state` namespace:
83
+
84
+ ```text
85
+ Call tool "get_customer" with:
86
+ id: "${state.create-customer.customerId}"
87
+ ```
88
+
89
+ MCP Rigor rejects state created for a different target. After careful review, it can be overridden:
90
+
91
+ ```bash
92
+ mcprigor test retrieve.mcpr \
93
+ --state-in customer-state.json \
94
+ --allow-state-target-mismatch
95
+ ```
96
+
97
+ ## Security guidance
98
+
99
+ State files can contain real business data. They are not encrypted.
100
+
101
+ - Never export passwords, access tokens, private keys, or regulated data.
102
+ - Do not commit state files to source control.
103
+ - Keep state short-lived.
104
+ - Prefer a secret manager for credentials.
105
+ - Treat target-mismatch override as an exceptional migration tool.
106
+ - Redaction protects reports, but internal dependency values remain unredacted so tests can use them correctly.
@@ -0,0 +1,98 @@
1
+ # Transport Parity Testing
2
+
3
+ MCP Rigor can execute the same acceptance suite against named stdio and Streamable HTTP targets, then compare normalized behavior test by test.
4
+
5
+ ## Plain-language parity for QA teams
6
+
7
+ Create `calculator-parity.mcpr`:
8
+
9
+ ```text
10
+ MCP Test 1
11
+
12
+ Suite: "Calculator behaves the same everywhere"
13
+
14
+ Compare target "Local": node server.js
15
+ Compare target "QA environment": https://qa.example.com/mcp
16
+
17
+ Test: "Adding 20 and 22 gives 42"
18
+ Id: adding-numbers
19
+
20
+ Call tool "add" with:
21
+ a: 20
22
+ b: 22
23
+
24
+ Expect "structuredContent.sum" equals 42
25
+
26
+ Test: "The service status is ready"
27
+ Id: service-status
28
+
29
+ Read resource "service://status"
30
+ Expect "contents[0].text" contains "ready"
31
+ ```
32
+
33
+ Run it exactly like a YAML parity suite:
34
+
35
+ ```bash
36
+ mcprigor parity calculator-parity.mcpr
37
+ ```
38
+
39
+ The `Compare target` lines are usually configured once by an engineer. QA authors only add ordinary `Test`, `Call tool`, `Read resource`, and `Expect` statements. Each test automatically runs against every named target.
40
+
41
+ A target value beginning with `http://` or `https://` is Streamable HTTP. Any other value is parsed as a stdio command.
42
+
43
+ ## YAML alternative
44
+
45
+ Automation engineers may also use the structured YAML representation:
46
+
47
+ ```yaml
48
+ version: 1
49
+ name: Calculator parity
50
+
51
+ target: &default
52
+ transport: stdio
53
+ command: node
54
+ args: [server.js]
55
+
56
+ targets:
57
+ stdio:
58
+ transport: stdio
59
+ command: node
60
+ args: [server.js]
61
+ deployed:
62
+ transport: streamable-http
63
+ url: https://staging.example.com/mcp
64
+
65
+ tests:
66
+ - id: add
67
+ name: Addition is consistent
68
+ steps:
69
+ - tool:
70
+ name: add
71
+ arguments: { a: 20, b: 22 }
72
+ assert:
73
+ json:
74
+ path: $.structuredContent.sum
75
+ equals: 42
76
+ ```
77
+
78
+ Run the matrix:
79
+
80
+ ```bash
81
+ mcprigor parity parity.yaml
82
+ mcprigor parity parity.yaml --markdown --out parity.md
83
+ ```
84
+
85
+ The first named target is the baseline. Every target runs independently with a fresh MCP session. MCP Rigor compares:
86
+
87
+ - test status;
88
+ - step status and MCP method;
89
+ - normalized response structures;
90
+ - captured/exported outputs.
91
+
92
+ It excludes durations, elapsed times, timestamps, session IDs, and `_meta` transport details. Application response fields are not ignored.
93
+
94
+ A divergent test includes a deterministic path-level semantic diff. The command exits `1` when any target differs.
95
+
96
+ ## Scope
97
+
98
+ Parity means equivalent observable behavior under the selected tests. It does not prove byte-level transport equivalence, identical concurrency behavior, network reliability, authentication parity, or production infrastructure correctness.
@@ -0,0 +1,199 @@
1
+ # Troubleshooting
2
+
3
+ Start with:
4
+
5
+ ```bash
6
+ mcprigor check your-tests.mcpr
7
+ ```
8
+
9
+ This catches wording and configuration problems without starting the MCP server.
10
+
11
+ ## The server does not start
12
+
13
+ Typical category:
14
+
15
+ ```text
16
+ MCP-SPAWN-* [server-spawn]
17
+ ```
18
+
19
+ Check:
20
+
21
+ 1. Run the `Server:` command directly in the same terminal.
22
+ 2. Confirm the executable is installed and on `PATH`.
23
+ 3. Check `Server options` → `cwd`.
24
+ 4. Build the server before running tests.
25
+ 5. Confirm required environment variables exist.
26
+
27
+ ## Initialization times out
28
+
29
+ Typical categories:
30
+
31
+ ```text
32
+ MCP-INIT-* [initialization]
33
+ MCP-TIMEOUT-* [timeout]
34
+ ```
35
+
36
+ Check that the stdio server:
37
+
38
+ - writes MCP messages only to stdout;
39
+ - writes diagnostics to stderr;
40
+ - does not wait for interactive input;
41
+ - completes MCP initialization;
42
+ - stays running after startup.
43
+
44
+ For HTTP, verify the URL, authentication, and server logs.
45
+
46
+ ## A field was not found
47
+
48
+ Example:
49
+
50
+ ```text
51
+ MCP-ASSERT-001 [assertion] $.structuredContent.id expected to exist
52
+ ```
53
+
54
+ Actions:
55
+
56
+ - inspect the actual response in JSON or evidence;
57
+ - check capitalization and array indexes;
58
+ - use `content[0].text` for text results;
59
+ - use `structuredContent.field` for structured tool output;
60
+ - avoid adding `result.` because paths begin at the returned result.
61
+
62
+ ## Expected number, received text
63
+
64
+ Equality is type-sensitive:
65
+
66
+ ```text
67
+ Expect "structuredContent.total" equals 2
68
+ ```
69
+
70
+ is different from:
71
+
72
+ ```text
73
+ Expect "structuredContent.total" equals "2"
74
+ ```
75
+
76
+ Match the server's actual JSON type.
77
+
78
+ ## The test was skipped
79
+
80
+ `Require` gates a test on server capability or protocol revision:
81
+
82
+ ```text
83
+ Require: tools
84
+ Require protocol: "2025-06-18"
85
+ ```
86
+
87
+ A skip means the server did not advertise the requirement. It is not a test failure.
88
+
89
+ ## The test was blocked
90
+
91
+ A dependent test is blocked when its producer did not pass:
92
+
93
+ ```text
94
+ Depends on: create-customer
95
+ ```
96
+
97
+ Fix the producer first. MCP Rigor does not open a server session for a blocked consumer.
98
+
99
+ ## Data loading failed
100
+
101
+ Typical category:
102
+
103
+ ```text
104
+ MCP-DATA-* [data-loading]
105
+ ```
106
+
107
+ Check:
108
+
109
+ - file path relative to the `.mcpr` file;
110
+ - required column names;
111
+ - number, boolean, date, or JSON values;
112
+ - selected Excel sheet;
113
+ - configured row limit;
114
+ - remote-data permission;
115
+ - remote endpoint returns a JSON array of row objects.
116
+
117
+ Remote data requires:
118
+
119
+ ```bash
120
+ mcprigor test FILE --allow-remote-data
121
+ ```
122
+
123
+ Private and local network destinations are rejected.
124
+
125
+ ## An extension failed
126
+
127
+ Typical category:
128
+
129
+ ```text
130
+ MCP-EXT-* [extension]
131
+ ```
132
+
133
+ Check:
134
+
135
+ - `--allow-custom-code` was supplied;
136
+ - module path is in `extensions.allowlist` when configured;
137
+ - manifest declares the function/provider;
138
+ - requested permissions are granted;
139
+ - input and output are JSON-compatible;
140
+ - extension finishes before its timeout.
141
+
142
+ ## Snapshot changed
143
+
144
+ Review the path-level diff. If the change is expected:
145
+
146
+ ```bash
147
+ mcprigor test FILE --snapshot snapshots.json --update-snapshots
148
+ ```
149
+
150
+ Never update snapshots automatically in CI. Commit and review changed expectations.
151
+
152
+ ## Transport parity differs
153
+
154
+ A parity failure can mean:
155
+
156
+ - different server versions;
157
+ - different fixture data;
158
+ - missing capabilities;
159
+ - a real stdio/HTTP implementation difference;
160
+ - volatile application fields that need an explicit snapshot policy.
161
+
162
+ Compare the path-level difference and verify both environments use equivalent test data.
163
+
164
+ ## Cleanup failed
165
+
166
+ Typical category:
167
+
168
+ ```text
169
+ MCP-CLEANUP-* [cleanup]
170
+ ```
171
+
172
+ Make cleanup operations idempotent. A delete should safely handle an item that is already absent. Check that server child processes stop when stdin closes and do not leave descendants running.
173
+
174
+ ## Ctrl+C does not return immediately
175
+
176
+ MCP Rigor first closes active MCP clients and transports. The pinned SDK gives stdio servers a graceful shutdown window before escalation. If a server creates child processes, the server must also shut them down.
177
+
178
+ ## Collect useful evidence
179
+
180
+ ```bash
181
+ mcprigor test FILE --evidence .mcprigor/debug-run --json result.json
182
+ mcprigor evidence-show .mcprigor/debug-run
183
+ ```
184
+
185
+ Before sharing evidence, review it for sensitive business data. Secret redaction does not remove all possible customer content.
186
+
187
+ ## Reporting a framework bug
188
+
189
+ Include:
190
+
191
+ - MCP Rigor version;
192
+ - Node and operating-system version;
193
+ - transport type;
194
+ - protocol revision if known;
195
+ - stable error code/category;
196
+ - smallest sanitized `.mcpr` test;
197
+ - sanitized evidence or result JSON.
198
+
199
+ Do not include credentials or regulated data.
@@ -0,0 +1,16 @@
1
+ {
2
+ "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json",
3
+ "name": "MCP Rigor Language",
4
+ "scopeName": "source.mcpr",
5
+ "patterns": [
6
+ { "match": "^\\s*#.*$", "name": "comment.line.number-sign.mcpr" },
7
+ { "match": "^MCP Test 1$", "name": "keyword.control.version.mcpr" },
8
+ { "match": "^\\s*(Suite|Server|Server options|MCP URL|Compare target|Target options for|Default timeout|Redact|Snapshots|Ignore snapshot paths|Client behavior|Import flows from|Flow|Inputs|Outputs|Test|Scenario|Id|Depends on|Require|Require protocol|Variables|Skip|Setup|Steps|Cleanup|Data source):?", "name": "keyword.control.mcpr" },
9
+ { "match": "^\\s*(Call tool|Read resource|Get prompt|Send|Expect|Save|Export|Set|Use flow|For each row|Wait up to|Wait for notification|Subscribe to resource|Unsubscribe from resource|Set log level|List all|Get task|List tasks|Cancel task|From CSV|From JSON|From YAML|From Excel|From REST|Column|Derive|Keep rows where|Sample|Cache this source)\\b", "name": "keyword.other.mcpr" },
10
+ { "begin": "\"", "end": "\"", "name": "string.quoted.double.mcpr" },
11
+ { "begin": "'", "end": "'", "name": "string.quoted.single.mcpr" },
12
+ { "match": "\\$\\{[^}]+\\}", "name": "variable.other.mcpr" },
13
+ { "match": "\\b(true|false|null)\\b", "name": "constant.language.mcpr" },
14
+ { "match": "\\b-?[0-9]+(?:\\.[0-9]+)?\\b", "name": "constant.numeric.mcpr" }
15
+ ]
16
+ }
package/package.json ADDED
@@ -0,0 +1,70 @@
1
+ {
2
+ "name": "mcprigor",
3
+ "version": "1.0.0-rc.1",
4
+ "description": "Plain-language MCP testing with isolated extensions and a local QA workspace",
5
+ "type": "module",
6
+ "bin": {
7
+ "mcprigor": "dist/cli.js"
8
+ },
9
+ "main": "./dist/index.js",
10
+ "types": "./dist/index.d.ts",
11
+ "exports": {
12
+ ".": {
13
+ "types": "./dist/index.d.ts",
14
+ "import": "./dist/index.js"
15
+ }
16
+ },
17
+ "files": [
18
+ "dist",
19
+ "schema",
20
+ "editors",
21
+ "docs",
22
+ "workspace-assets",
23
+ "README.md",
24
+ "LICENSE"
25
+ ],
26
+ "scripts": {
27
+ "build": "tsc -p tsconfig.json",
28
+ "dev": "tsx src/cli.ts",
29
+ "test": "vitest run",
30
+ "test:watch": "vitest",
31
+ "check": "npm run build && npm test"
32
+ },
33
+ "keywords": [
34
+ "mcp",
35
+ "model-context-protocol",
36
+ "testing",
37
+ "qa",
38
+ "test-automation",
39
+ "plain-language",
40
+ "contract-testing",
41
+ "cli"
42
+ ],
43
+ "license": "Apache-2.0",
44
+ "engines": {
45
+ "node": ">=20"
46
+ },
47
+ "dependencies": {
48
+ "@modelcontextprotocol/sdk": "1.27.1",
49
+ "ajv": "^8.18.0",
50
+ "exceljs": "^4.4.0",
51
+ "yaml": "^2.8.2"
52
+ },
53
+ "overrides": {
54
+ "uuid": "^11.1.1"
55
+ },
56
+ "devDependencies": {
57
+ "@types/node": "^22.19.15",
58
+ "tsx": "^4.21.0",
59
+ "typescript": "^5.9.3",
60
+ "vitest": "^4.0.18"
61
+ },
62
+ "repository": {
63
+ "type": "git",
64
+ "url": "git+https://github.com/FusionOnePlatform/mcprigor.git"
65
+ },
66
+ "homepage": "https://mcprigor.com",
67
+ "bugs": {
68
+ "url": "https://github.com/FusionOnePlatform/mcprigor/issues"
69
+ }
70
+ }
@@ -0,0 +1 @@
1
+ const state={csrf:'',path:'',etag:'',dirty:false};const $=id=>document.getElementById(id);async function api(path,options={}){const headers={...(options.body?{'content-type':'application/json'}:{}),...(options.method&&options.method!=='GET'?{'x-mcp-csrf':state.csrf}:{})};const response=await fetch(path,{...options,headers:{...headers,...options.headers}});const value=await response.json();if(!response.ok)throw new Error(value.error?.message||`HTTP ${response.status}`);return value}async function start(){const boot=await api('/api/v1/bootstrap');state.csrf=boot.csrf;$('workspace').textContent=`Workspace: ${boot.root} · MCP Rigor ${boot.version}`;$('connection').textContent='● Local and ready';await suites()}async function suites(){const value=await api('/api/v1/suites');const nav=$('suites');nav.replaceChildren();for(const suite of value.suites){const button=document.createElement('button');button.textContent=suite.path;button.onclick=()=>open(suite.path,button);nav.append(button)}}async function open(path,button){if(state.dirty&&!confirm('Discard unsaved changes?'))return;const value=await api(`/api/v1/file?path=${encodeURIComponent(path)}`);state.path=path;state.etag=value.etag;state.dirty=false;$('editor').value=value.text;$('dirty').textContent='';document.querySelectorAll('nav button').forEach(x=>x.classList.remove('active'));button.classList.add('active');$('diagnostics').textContent=`Opened ${path}`}async function save(){if(!state.path)return;const value=await api('/api/v1/file',{method:'PUT',body:JSON.stringify({path:state.path,text:$('editor').value,etag:state.etag})});state.etag=value.etag;state.dirty=false;$('dirty').textContent='';$('diagnostics').textContent='Saved'}async function validate(){if(state.dirty)await save();const value=await api('/api/v1/validate',{method:'POST',body:JSON.stringify({path:state.path})});$('diagnostics').textContent=value.valid?`✓ Valid · ${value.suite.tests} tests${value.suite.parityTargets.length?` · ${value.suite.parityTargets.length} parity targets`:''}`:value.diagnostics.map(x=>x.message).join('\n')}async function run(mode){if(state.dirty)await save();$('output').textContent='Starting…';const value=await api('/api/v1/runs',{method:'POST',body:JSON.stringify({path:state.path,mode})});const poll=setInterval(async()=>{try{const result=await api(`/api/v1/runs/${value.runId}`);$('output').textContent=result.output;if(result.status!=='running'){clearInterval(poll);$('diagnostics').textContent=result.status==='passed'?'✓ Run passed':'✗ Run failed'}}catch(error){clearInterval(poll);$('output').textContent=error.message}},500)}$('editor').addEventListener('input',()=>{state.dirty=true;$('dirty').textContent='● Unsaved'});$('save').onclick=()=>save().catch(show);$('validate').onclick=()=>validate().catch(show);$('run').onclick=()=>run('test').catch(show);$('parity').onclick=()=>run('parity').catch(show);$('refresh').onclick=()=>suites().catch(show);function show(error){$('diagnostics').textContent=error.message}start().catch(show);
@@ -0,0 +1 @@
1
+ <!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>MCP Rigor QA Workspace</title><link rel="stylesheet" href="/style.css"></head><body><header><div><strong>MCP Rigor</strong><span>QA Workspace</span></div><div id="connection">Connecting…</div></header><main><aside><button id="refresh">↻ Refresh suites</button><nav id="suites" aria-label="Test suites"></nav></aside><section class="workbench"><div class="toolbar"><button id="save">Save</button><button id="validate">Validate</button><button id="run">Run tests</button><button id="parity">Run parity</button><span id="dirty"></span></div><textarea id="editor" spellcheck="false" aria-label="Plain-language MCP test editor" placeholder="Select a suite or create a .mcp file in your workspace"></textarea><div id="diagnostics" role="status" aria-live="polite"></div></section><aside class="results"><h2>Run results</h2><pre id="output">No run yet.</pre></aside></main><footer><span id="workspace"></span><span>Plain language and YAML share one runtime model</span></footer><script type="module" src="/app.js"></script></body></html>
@@ -0,0 +1 @@
1
+ :root{font:15px/1.5 Inter,ui-sans-serif,system-ui;color:#15231d;background:#edf3ef}*{box-sizing:border-box}body{margin:0;height:100vh;display:grid;grid-template-rows:58px 1fr 34px}header,footer{display:flex;align-items:center;justify-content:space-between;padding:0 22px;background:#10291f;color:#e7fff4}header strong{font-size:20px;margin-right:12px}header span,footer{color:#a9cbbb}main{display:grid;grid-template-columns:230px minmax(400px,1fr) minmax(300px,38%);min-height:0}aside{padding:14px;border-right:1px solid #c7d5cd;background:#f7faf8;overflow:auto}.results{border:0;border-left:1px solid #c7d5cd;background:#102019;color:#dff4e8}.results h2{font-size:14px;text-transform:uppercase;letter-spacing:.08em}.workbench{display:grid;grid-template-rows:52px 1fr auto;min-width:0}.toolbar{display:flex;align-items:center;gap:8px;padding:9px 14px;background:white;border-bottom:1px solid #c7d5cd}button{border:1px solid #7d9b8b;border-radius:7px;padding:7px 11px;background:white;color:#173c2c;cursor:pointer}button:hover,button:focus{background:#dff3e7;outline:2px solid transparent}nav button{display:block;width:100%;text-align:left;margin:5px 0;overflow:hidden;text-overflow:ellipsis}nav button.active{background:#bde6cc;border-color:#238653}textarea{resize:none;border:0;padding:20px;font:14px/1.65 ui-monospace,SFMono-Regular,Menlo,monospace;outline:none;color:#17221d;tab-size:2}#diagnostics{padding:8px 14px;min-height:38px;background:#fff8df;color:#6a4c00;white-space:pre-wrap}pre{white-space:pre-wrap;word-break:break-word;font:13px/1.55 ui-monospace;padding:8px 0}footer{font-size:12px}#dirty{color:#aa5b00}@media(max-width:900px){main{grid-template-columns:180px 1fr}.results{display:none}}@media(max-width:620px){main{grid-template-columns:1fr}main>aside:first-child{max-height:150px;border-bottom:1px solid #ccc}.workbench{min-height:500px}}