@slbdn/mcp-tester 1.4.3 → 1.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/README.md +78 -26
  3. package/dist/ai/provider.d.ts +145 -0
  4. package/dist/ai/provider.d.ts.map +1 -0
  5. package/dist/ai/provider.js +428 -0
  6. package/dist/ai/provider.js.map +1 -0
  7. package/dist/cli/index.js +214 -14
  8. package/dist/cli/index.js.map +1 -1
  9. package/dist/generate-cases.d.ts +113 -0
  10. package/dist/generate-cases.d.ts.map +1 -0
  11. package/dist/generate-cases.js +565 -0
  12. package/dist/generate-cases.js.map +1 -0
  13. package/dist/generate-tests.d.ts +52 -1
  14. package/dist/generate-tests.d.ts.map +1 -1
  15. package/dist/generate-tests.js +208 -81
  16. package/dist/generate-tests.js.map +1 -1
  17. package/dist/index.d.ts +3 -1
  18. package/dist/index.d.ts.map +1 -1
  19. package/dist/index.js +2 -0
  20. package/dist/index.js.map +1 -1
  21. package/dist/matchers.d.ts +31 -21
  22. package/dist/matchers.d.ts.map +1 -1
  23. package/dist/matchers.js.map +1 -1
  24. package/dist/templates/full-stack/.github/workflows/test.yml +34 -0
  25. package/dist/templates/full-stack/README.md +73 -0
  26. package/dist/templates/full-stack/examples/mock-server.js +115 -0
  27. package/dist/templates/full-stack/jest.config.js +39 -0
  28. package/dist/templates/full-stack/package.json +31 -0
  29. package/dist/templates/full-stack/scripts/generate-tests.ts +32 -0
  30. package/dist/templates/full-stack/scripts/generate-types.ts +28 -0
  31. package/dist/templates/full-stack/src/server.ts +78 -0
  32. package/dist/templates/full-stack/tests/helpers.ts +29 -0
  33. package/dist/templates/full-stack/tests/parallel.test.ts +44 -0
  34. package/dist/templates/full-stack/tests/server.test.ts +36 -0
  35. package/dist/templates/full-stack/tests/tools.test.ts +61 -0
  36. package/dist/templates/full-stack/tsconfig.json +19 -0
  37. package/dist/templates/minimal-jest/README.md +42 -0
  38. package/dist/templates/minimal-jest/examples/mock-server.js +62 -0
  39. package/dist/templates/minimal-jest/jest.config.js +24 -0
  40. package/dist/templates/minimal-jest/package.json +24 -0
  41. package/dist/templates/minimal-jest/tests/server.test.ts +64 -0
  42. package/dist/templates/minimal-jest/tsconfig.json +19 -0
  43. package/dist/templates/standard-jest/.github/workflows/test.yml +32 -0
  44. package/dist/templates/standard-jest/README.md +44 -0
  45. package/dist/templates/standard-jest/examples/mock-server.js +122 -0
  46. package/dist/templates/standard-jest/jest.config.js +39 -0
  47. package/dist/templates/standard-jest/package.json +27 -0
  48. package/dist/templates/standard-jest/tests/helpers.ts +43 -0
  49. package/dist/templates/standard-jest/tests/prompts.test.ts +37 -0
  50. package/dist/templates/standard-jest/tests/resources.test.ts +37 -0
  51. package/dist/templates/standard-jest/tests/server.test.ts +30 -0
  52. package/dist/templates/standard-jest/tests/tools.test.ts +87 -0
  53. package/dist/templates/standard-jest/tsconfig.json +19 -0
  54. package/docs/advanced.md +424 -0
  55. package/docs/ai-generation.md +211 -0
  56. package/docs/api-reference.md +819 -0
  57. package/docs/cicd.md +289 -0
  58. package/docs/cli.md +225 -0
  59. package/docs/examples.md +300 -0
  60. package/docs/nodejs-compatibility.md +81 -0
  61. package/docs/releases.md +413 -0
  62. package/docs/starter-templates.md +110 -0
  63. package/docs/testing.md +443 -0
  64. package/docs/troubleshooting.md +139 -0
  65. package/jest.d.ts +43 -0
  66. package/package.json +5 -2
package/CHANGELOG.md CHANGED
@@ -7,6 +7,65 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [1.5.4] - 2026-10-08
11
+
12
+ ### Fixed
13
+ - Rebuilt the README architecture diagram. Its box borders were misaligned (right edges varied by up to six columns, so the boxes did not line up), and the `MCP Server (child process)` label was wrong for the HTTP/SSE transports, which connect to a remote server rather than spawning a process. The diagram now has consistently aligned borders and shows the tested capabilities (`Tools`, `Resources`, `Prompts`, `Sampling`, `Elicitation`, `Notifications`).
14
+
15
+ ## [1.5.3] - 2026-10-08
16
+
17
+ ### Fixed
18
+ - **`npm run test:coverage` failed its branch-coverage gate for `src/generate-cases.ts`** (`81.87% < 85%`) even though every test passed. Added tests covering the previously untested `additionalProperties`-as-schema constraint checks (`minLength`/`maxLength`/`maximum`/`exclusiveMinimum`/`exclusiveMaximum`/`minItems`/`maxItems`/`uniqueItems`), constraint-only violating-value inference, union/`null`/untyped-property handling, and rule-engine edges (numeric enums, unmatched `required` keys, required object properties, `maxItems` without an `items` schema). Branch coverage for the file is now 97.65%; overall branch coverage is 87.76%.
19
+
20
+ ## [1.5.2] - 2026-10-08
21
+
22
+ ### Fixed
23
+ - **`npm ci` failed on Node 20 / npm 10, blocking the 1.5.1 publish.** The 1.5.1 lockfile was left inconsistent by `npm audit fix`: it dropped the top-level `@emnapi/core` and `@emnapi/runtime` entries that satisfy `@napi-rs/wasm-runtime`'s peer dependencies. npm 11 (which wrote the lock) tolerated this, but npm 10's stricter `npm ci` sync check rejected it with `EUSAGE` (`Missing: @emnapi/core@1.11.3 from lock file`). The entries are restored; `npm ci` and `npm audit --audit-level=high` both pass on npm 10.
24
+ - 1.5.1 was never published to npm; this release supersedes it and carries the same security updates described below.
25
+
26
+ ### Changed
27
+ - CI and the release verification job now run on Node.js 20 and 22 instead of 20 and 21. Node 21 is end-of-life and is outside the `engines` range of Jest 30 and several dev dependencies (`glob@13`, `minimatch@10`, `lru-cache@11`, `eslint-visitor-keys@5`, …), which produced `EBADENGINE` warnings during install. The starter-template CI matrices were updated to match.
28
+
29
+ ## [1.5.1] - 2026-10-08
30
+
31
+ ### Security
32
+ - Cleared the `npm audit` findings that blocked the 1.5.0 publish (the `release.yml` job's hard `npm audit --audit-level=high` gate). A semver-safe `npm audit fix` bumped:
33
+ - `@modelcontextprotocol/sdk` 1.29.0 → 1.32.1 (high — the OAuth client could send credentials to an authorization server chosen by the MCP server, [GHSA-6qxp-vccf-f47h](https://github.com/advisories/GHSA-6qxp-vccf-f47h)).
34
+ - `proxy-addr` 2.0.7 → 2.0.8 (critical — IP spoofing via an IPv4-mapped IPv6 trust subnet, [GHSA-jqcg-44mw-7w3h](https://github.com/advisories/GHSA-jqcg-44mw-7w3h)), transitive via `express` through the SDK's server integration.
35
+ - `ts-jest` → 29.4.14 and `babel-plugin-istanbul` → 8.0.2 transitively.
36
+ - The remaining advisories are dev-only moderate findings (`sprintf-js` via `ts-jest`) and sit below the release gate's `--audit-level=high` threshold; clearing them requires a breaking `ts-jest` downgrade (`npm audit fix --force`), so they are intentionally left in place.
37
+
38
+ ## [1.5.0] - 2026-10-08
39
+
40
+ ### Added
41
+ - **AI-assisted & schema-driven test-case generation** (TODO #39) — `generate` now goes beyond happy-path calls:
42
+ - `--edge-cases` — deterministic boundary/invalid cases from each tool's JSON Schema (offline, no AI): missing required args, invalid enums, wrong types, `minLength`/`maxLength`, `minimum`/`maximum` bounds, `minItems`, `additionalProperties: false`. Expectations are `error`, `observe`, or `success`, and every case is tagged with a provenance comment (`// rule:required — ...`).
43
+ - `--ai-generate` — augments the edge cases with LLM suggestions via any OpenAI-compatible endpoint (OpenAI, Ollama, LM Studio, vLLM). The model only proposes **structured case data** (`GeneratedCase`); every suggestion is validated against the real tool schemas and rendered deterministically — hallucinated arguments are dropped with a summary line, and the model never emits code — model-supplied rationales are sanitized before being embedded in a comment, so a crafted rationale cannot break out and become executable test code. Configure via `MCP_TESTER_AI_API_KEY` / `MCP_TESTER_AI_BASE_URL` / `MCP_TESTER_AI_MODEL` or the new `--ai-model` / `--ai-base-url` / `--require-ai` flags. Responses are cached in `.mcp-tester-cache/`.
44
+ - `--verify` — dry-runs every derived case against the live server before writing the file; predictions that don't hold become `it.skip` blocks with an explanatory comment instead of broken tests.
45
+ - New modules: `generate-cases.ts` (rules engine, `GeneratedCase`, `suggestEdgeCases*`, `validateArgsAgainstSchema`, `mergeAndValidateCases`) and `ai/provider.ts` (`OpenAICompatProvider` with injectable `fetchImpl`, `createProviderFromEnv`). All exported from the package barrel.
46
+ - Docs: [`docs/ai-generation.md`](docs/ai-generation.md), README section, CLI reference.
47
+ - **Starter templates** under `templates/` with a new `mcp-tester create <template> <dest>` CLI command:
48
+ - `minimal-jest` — one test file, one mock server, zero ceremony.
49
+ - `standard-jest` — per-capability test files, HTML reporter, GitHub Actions matrix.
50
+ - `full-stack` — real TS server, code generation scripts (`npm run gen:tests` / `gen:types`), typed tool calls, parallel stress tests.
51
+ - `create list` shows available templates. `__NAME__` placeholders in `package.json` are substituted with the destination directory name. `--no-install` skips `npm install`; `--git` initializes a repo.
52
+ - Documentation: [`docs/starter-templates.md`](docs/starter-templates.md) and a new README section.
53
+
54
+ ### Fixed
55
+ - **Jest matcher types were broken for consumers since the Jest 30 upgrade (1.4.3)** — `expect(tools).toHaveTool(...)` produced TS2339 in consumer projects (the global `jest.Matchers` augmentation no longer reaches Jest 30's `expect()` return type). The matchers are now additionally declared against the `expect` package's `Matchers` interface via `MCPMatchers`, so autocomplete/type-checking works in both ts-jest and plain `tsc` setups. Because TypeScript does not apply module augmentations shipped inside `node_modules`, the package now ships a copy-ready `jest.d.ts` — copy it into your project to enable the types (see README).
56
+ - Templates now typecheck cleanly under `strict` `tsc --noEmit` (they were only ever compiled by ts-jest's non-strict inline config): `content[0].text`-style union accesses replaced with typed `contentText()` / `promptText()` / `resourceText()` helpers, and full-stack's server handler no longer assumes `arguments` is defined.
57
+ - `generateTests` (stdio variant) now respects the `includeTools` / `includeResources` / `includePrompts` flags when fetching from the server, matching the behaviour of `generateTestsFromClient`. Previously it always called `listResources` and `listPrompts` even when the output flags said to skip them — causing generation to fail against servers that don't declare those capabilities.
58
+ - `examples/mock-server.js`: the `delay` tool now rejects invalid `ms` values (`NaN`, negative, > 60000) instead of passing them to `setTimeout` — caught by the new `--verify` mode.
59
+ - The published package now ships the top-level `docs/*.md` guides, so the README's documentation links resolve on npmjs.com (they previously 404'd — the generated `docs/api/` HTML is still excluded).
60
+ - The generated `it(...)` titles and call arguments now escape the exact quote delimiter they are interpolated into, so tool/resource/prompt names containing apostrophes, double quotes, or backslashes produce valid test files. Observe-oracle cases emit a proper single-quoted tool literal instead of a literal `${c.tool}` expression (which previously caused a `ReferenceError`).
61
+ - AI mode no longer drops the deterministic one-past-`maxLength` `observe` case during schema validation, so `--edge-cases` and `--ai-generate` produce the same rule cases. The merge diagnostic now counts only AI rejections.
62
+ - The `additionalProperties`-as-schema rule now emits a value that actually violates the declared type (it previously injected a number that satisfied `{ type: 'number' }`), and the `pattern` rule skips patterns that match every probe value (e.g. `.*`) instead of asserting a false `error`.
63
+ - `parseJsonLoose` falls back past an unparseable code fence to a valid JSON payload elsewhere in the response, and its object scanner skips past a well-formed object instead of rescanning every nested `{`.
64
+ - `validateValueAgainstSchema` now checks `enum`/`pattern`/numeric/array constraints on `additionalProperties` values, not just type and string length.
65
+
66
+ ### Security
67
+ - **Generated test code cannot be turned into an injection vector** — model-supplied case rationales are collapsed onto a single line comment before being emitted, so a crafted multi-line rationale can no longer close the comment and inject executable statements into the generated file. The copy-ready `jest.d.ts` shipped to consumers also no longer contains a `*/` inside its doc comment (which made the file a syntax error).
68
+
10
69
  ## [1.4.3] - 2026-10-05
11
70
 
12
71
  ### Security
package/README.md CHANGED
@@ -32,32 +32,35 @@ No more manual clicking through inspectors — write real tests, run them in CI,
32
32
  ## Architecture
33
33
 
34
34
  ```
35
- ┌─────────────────────────────────────────────────────────────────────┐
36
- │ Your Test Suite │
37
- │ ┌──────────┐ ┌───────────┐ ┌────────┐ ┌───────────────────┐ │
38
- │ │ Jest │ │ Vitest │ │ assert │ │ Custom Runner │ │
39
- │ │ matchers │ │ matchers │ │ module │ │ (any framework) │ │
40
- │ └────┬─────┘ └─────┬─────┘ └───┬────┘ └────────┬──────────┘ │
41
- │ │ │ │ │ │
42
- │ ┌────▼──────────────▼─────────────▼─────────────────▼──────────┐ │
43
- │ │ MCPClient │ │
44
- │ │ ┌──────────────────────────────────────────────────────┐ │ │
45
- │ │ │ start() │ stop() │ listTools() │ callTool() │ ... │ │ │
46
- │ │ └──────────────────────┬───────────────────────────────────┘ │ │
47
- │ │ │ │ │
48
- │ │ ┌──────────────────────▼───────────────────────────────┐ │ │
49
- │ │ │ Retry │ Timeout │ Error Handling │ │ │
50
- │ │ │ MCPTimeoutError │ MCPConnectionError │ ... │ │ │
51
- │ │ └──────────────────────┬───────────────────────────────┘ │ │
52
- │ └─────────────────────────┼─────────────────────────────────────┘ │
53
- │ │ stdio, HTTP, SSE │
54
- │ ┌─────────────────────────▼─────────────────────────────────────┐ │
55
- │ │ MCP Server (child process) │ │
56
- │ │ ┌────────┐ ┌───────────┐ ┌──────────┐ ┌────────────┐ │ │
57
- │ │ │ Tools │ │ Resources │ │ Prompts │ │ Sampling │ │ │
58
- │ │ └────────┘ └───────────┘ └──────────┘ └────────────┘ │ │
59
- │ └──────────────────────────────────────────────────────────────┘ │
60
- └─────────────────────────────────────────────────────────────────────┘
35
+ ┌──────────────────────────────────────────────────────────────────────┐
36
+ │ Your Test Suite │
37
+ │ ┌──────────┐ ┌───────────┐ ┌────────┐ ┌───────────────────┐ │
38
+ │ │ Jest │ │ Vitest │ │ assert │ │ Custom Runner │ │
39
+ │ │ matchers │ │ matchers │ │ module │ │ (any framework) │ │
40
+ │ └────┬─────┘ └─────┬─────┘ └───┬────┘ └────────┬──────────┘ │
41
+ │ │ │ │ │ │
42
+ │ ┌──────▼──────────────▼────────────▼────────────────▼────────────┐ │
43
+ │ │ MCPClient │ │
44
+ │ │ ┌────────────────────────────────────────────────────────┐ │ │
45
+ │ │ │ start() │ stop() │ listTools() │ callTool() │ ... │ │ │
46
+ │ │ └───────────────────────────┬────────────────────────────┘ │ │
47
+ │ │ │ │ │
48
+ │ │ ┌───────────────────────────▼────────────────────────────┐ │ │
49
+ │ │ │ Retry │ Timeout │ Error Handling │ │ │
50
+ │ │ │ MCPTimeoutError │ MCPConnectionError │ ... │ │ │
51
+ │ │ └───────────────────────────┬────────────────────────────┘ │ │
52
+ │ └───────────────────────────────┼────────────────────────────────┘ │
53
+ │ │ stdio, HTTP, SSE │
54
+ │ ┌───────────────────────────────▼────────────────────────────────┐ │
55
+ │ │ MCP Server │ │
56
+ │ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ │
57
+ │ │ │ Tools │ │ Resources │ │ Prompts │ │ │
58
+ │ │ └──────────────┘ └──────────────┘ └──────────────┘ │ │
59
+ │ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ │
60
+ │ │ │ Sampling │ │ Elicitation │ │Notifications │ │ │
61
+ │ │ └──────────────┘ └──────────────┘ └──────────────┘ │ │
62
+ │ └────────────────────────────────────────────────────────────────┘ │
63
+ └──────────────────────────────────────────────────────────────────────┘
61
64
  ```
62
65
 
63
66
  **Data flow**: Your test calls `MCPClient` → which sends JSON-RPC over stdio/HTTP/SSE → to the MCP server → server responds → `MCPClient` returns typed results → your assertion validates them.
@@ -248,6 +251,34 @@ The generator connects to the server, inspects all tools/resources/prompts, and
248
251
  - Prompt tests with sample arguments
249
252
  - Proper `beforeEach`/`afterEach` cleanup
250
253
 
254
+ #### Edge Cases & AI-Assisted Generation
255
+
256
+ Go beyond happy-path calls with schema-derived boundary tests — optionally
257
+ augmented by an LLM (OpenAI, Ollama, LM Studio, or any OpenAI-compatible
258
+ endpoint). The AI only proposes **structured case data**; every case is
259
+ validated against the real schemas and rendered deterministically — the model
260
+ never writes code:
261
+
262
+ ```bash
263
+ # Deterministic edge cases (offline): missing required args, invalid enums,
264
+ # wrong types, boundary values, minItems, additionalProperties
265
+ npx @slbdn/mcp-tester generate node ./server.js --edge-cases -o edge.test.ts
266
+
267
+ # Add AI-suggested cases (needs MCP_TESTER_AI_API_KEY, or a local base URL)
268
+ MCP_TESTER_AI_API_KEY=sk-... \
269
+ npx @slbdn/mcp-tester generate node ./server.js --ai-generate -o ai.test.ts
270
+
271
+ # Ollama — no API key needed for localhost
272
+ npx @slbdn/mcp-tester generate node ./server.js \
273
+ --ai-generate --ai-base-url http://localhost:11434/v1 --ai-model llama3
274
+
275
+ # Dry-run derived cases against the live server: predictions that fail
276
+ # become it.skip blocks with an explanation instead of broken tests
277
+ npx @slbdn/mcp-tester generate node ./server.js --edge-cases --verify -o edge.test.ts
278
+ ```
279
+
280
+ 📖 Full guide: [AI-Assisted Test Generation](./docs/ai-generation.md)
281
+
251
282
  ### Generate TypeScript Types
252
283
 
253
284
  Connect to any MCP server and generate typed `.d.ts` declarations from tool schemas:
@@ -285,6 +316,18 @@ const result = await client.callTool({
285
316
 
286
317
  Supports: primitives, enums, nested objects, arrays, `oneOf`/`anyOf`/`allOf`, `$ref`, `const`.
287
318
 
319
+ ## Starter Templates
320
+
321
+ Three ready-to-use scaffolds for new projects. Pick the size that matches your need:
322
+
323
+ ```bash
324
+ npx @slbdn/mcp-tester create minimal-jest my-server-tests # Tiny: one test file
325
+ npx @slbdn/mcp-tester create standard-jest my-server-tests # Real-world: CI + HTML report
326
+ npx @slbdn/mcp-tester create full-stack my-server-tests # Library: code generation + typed tools
327
+ ```
328
+
329
+ Each scaffolds a runnable project with a mock server, then runs `npm install`. Use `--no-install` to skip, `--git` to init a repo. See [docs/starter-templates.md](docs/starter-templates.md) for details.
330
+
288
331
  ## Speed Up Your Tests with Parallel Execution
289
332
 
290
333
  MCP Tester is built for concurrency. Fire off multiple tool calls at once — no waiting in line:
@@ -370,6 +413,10 @@ import {
370
413
 
371
414
  beforeAll(() => setupJestMatchers());
372
415
 
416
+ // For type-safe matcher autocompletion, copy the augmentation into your
417
+ // project. See `node_modules/@slbdn/mcp-tester/jest.d.ts` for the source
418
+ // and full instructions.
419
+
373
420
  // Collection matchers
374
421
  expect(tools).toHaveTool('echo');
375
422
  expect(resources).toHaveResource('config://settings');
@@ -476,6 +523,7 @@ await runTest("Echo returns input", async () => {
476
523
  | [Examples](./docs/examples.md) | Practical code examples |
477
524
  | [Advanced Usage](./docs/advanced.md) | Timeouts, retries, concurrency, notifications, logging |
478
525
  | [CLI Reference](./docs/cli.md) | CLI commands, options, and output formats |
526
+ | [AI-Assisted Test Generation](./docs/ai-generation.md) | Edge cases, LLM integration, `--verify` |
479
527
  | [CI/CD Integration](./docs/cicd.md) | GitHub Actions, CircleCI, Jenkins |
480
528
  | [Troubleshooting](./docs/troubleshooting.md) | Common issues and solutions |
481
529
  | [Releases](./docs/releases.md) | Release process and commands |
@@ -510,6 +558,10 @@ mcp-tester/
510
558
  │ ├── mock-server.js # Basic standalone MCP server
511
559
  │ ├── simple-server.ts # Minimal TypeScript MCP server
512
560
  │ └── stateful-server.ts # Stateful MCP server (counter + todos)
561
+ ├── templates/ # Starter templates (mcp-tester create <template>)
562
+ │ ├── minimal-jest/ # One test file, one mock server
563
+ │ ├── standard-jest/ # Per-capability tests, HTML report, CI matrix
564
+ │ └── full-stack/ # TS server + code generation + typed tools
513
565
  └── docs/ # Documentation
514
566
  ```
515
567
 
@@ -0,0 +1,145 @@
1
+ /**
2
+ * AI test-case provider — pluggable, optional augmentation of the
3
+ * deterministic edge-case engine.
4
+ *
5
+ * The provider speaks the OpenAI-compatible chat-completions protocol
6
+ * (`POST {baseUrl}/chat/completions`), which covers OpenAI, Ollama, LM
7
+ * Studio, vLLM, Groq, Together, and most local gateways. Implemented with
8
+ * plain `fetch` — no SDK dependency.
9
+ *
10
+ * Security model: the LLM returns *data* (JSON test cases), never code.
11
+ * Every case is validated against the tool's real schema by
12
+ * `mergeAndValidateCases` before it can reach a generated file, so prompt
13
+ * injection via server-provided tool descriptions cannot inject code.
14
+ *
15
+ * @module ai/provider
16
+ */
17
+ import type { Tool } from '@modelcontextprotocol/sdk/types.js';
18
+ import { type GeneratedCase } from '../generate-cases.js';
19
+ /** Server snapshot handed to a provider. Descriptions are length-capped. */
20
+ export interface ServerAnalysis {
21
+ /** Human-readable server label for the prompt. */
22
+ serverLabel?: string;
23
+ /** Tools to reason about. */
24
+ tools: Array<{
25
+ name: string;
26
+ description?: string;
27
+ inputSchema: Record<string, unknown>;
28
+ }>;
29
+ }
30
+ /** Anything that can propose test cases for a server analysis. */
31
+ export interface AIProvider {
32
+ /** Provider/model identity, used for case provenance (`ai:<name>`). */
33
+ readonly name: string;
34
+ /** Propose cases; implementers should tolerate partial failures. */
35
+ suggestCases(analysis: ServerAnalysis): Promise<GeneratedCase[]>;
36
+ }
37
+ /** Options for {@link OpenAICompatProvider}. */
38
+ export interface OpenAICompatOptions {
39
+ /** API base URL. @defaultValue 'https://api.openai.com/v1' */
40
+ baseUrl?: string;
41
+ /** Chat model name. @defaultValue 'gpt-4o-mini' */
42
+ model?: string;
43
+ /** Bearer token; omit for local servers that don't need one. */
44
+ apiKey?: string;
45
+ /** Request timeout in ms. @defaultValue 60000 */
46
+ timeout?: number;
47
+ /** Cap on completion tokens per request. @defaultValue 4096 */
48
+ maxTokens?: number;
49
+ /** Cache directory; `null` disables caching. @defaultValue '.mcp-tester-cache' */
50
+ cacheDir?: string | null;
51
+ /**
52
+ * Injectable fetch for tests. @defaultValue globalThis.fetch
53
+ */
54
+ fetchImpl?: typeof fetch;
55
+ /**
56
+ * Send OpenAI's `response_format: { type: 'json_object' }` flag. Supported
57
+ * by OpenAI, Together, Groq, and most modern gateways; some local servers
58
+ * (Ollama, older vLLM) reject it. Set to `false` if your gateway returns
59
+ * HTTP 400 mentioning `response_format`. When `true` (default), a 400 that
60
+ * explicitly blames `response_format` triggers one automatic retry without
61
+ * the flag.
62
+ * @defaultValue true
63
+ */
64
+ strictJson?: boolean;
65
+ /**
66
+ * Maximum age in ms for cached AI responses. When set, cache files older
67
+ * than this are ignored and re-fetched. Unset (default) means cache never
68
+ * expires; entries are only invalidated when the model or tool set changes.
69
+ * @defaultValue undefined
70
+ */
71
+ cacheMaxAgeMs?: number;
72
+ }
73
+ export declare const DEFAULT_AI_BASE_URL = "https://api.openai.com/v1";
74
+ export declare const DEFAULT_AI_MODEL = "gpt-4o-mini";
75
+ /**
76
+ * Extract a JSON object from model content, tolerating markdown fences and
77
+ * prose around the payload. Uses brace-aware scanning so a `}` inside a
78
+ * string value or in a nested object doesn't truncate the payload.
79
+ *
80
+ * @returns The parsed object, or `null` when no parseable JSON object is
81
+ * found in the input.
82
+ */
83
+ export declare function parseJsonLoose(content: string): {
84
+ cases?: unknown;
85
+ } | null;
86
+ /**
87
+ * OpenAI-compatible chat-completions provider.
88
+ *
89
+ * Works with OpenAI, Ollama (`http://localhost:11434/v1`), LM Studio,
90
+ * vLLM, Groq, and any compatible gateway.
91
+ */
92
+ export declare class OpenAICompatProvider implements AIProvider {
93
+ readonly name: string;
94
+ private readonly baseUrl;
95
+ private readonly model;
96
+ private readonly apiKey?;
97
+ private readonly timeout;
98
+ private readonly maxTokens;
99
+ private readonly cacheDir;
100
+ private readonly cacheMaxAgeMs;
101
+ private readonly fetchImpl;
102
+ private readonly strictJson;
103
+ /** Tracks whether this provider has already observed a `response_format` rejection. */
104
+ private responseFormatDisabled;
105
+ constructor(options?: OpenAICompatOptions);
106
+ suggestCases(analysis: ServerAnalysis): Promise<GeneratedCase[]>;
107
+ /** Single chat-completions round trip. Throws with response context. */
108
+ private chatCompletion;
109
+ /**
110
+ * One HTTP request to the chat-completions endpoint. Categorises the
111
+ * response so the caller can decide whether to retry without
112
+ * `response_format` or just throw.
113
+ */
114
+ private rawChatCompletion;
115
+ private cacheKeyFor;
116
+ private readCache;
117
+ private writeCache;
118
+ }
119
+ /**
120
+ * Build a provider from `MCP_TESTER_AI_*` environment variables.
121
+ *
122
+ * - `MCP_TESTER_AI_API_KEY` — required for hosted APIs; optional for local.
123
+ * - `MCP_TESTER_AI_BASE_URL` — @defaultValue 'https://api.openai.com/v1'
124
+ * - `MCP_TESTER_AI_MODEL` — @defaultValue 'gpt-4o-mini'
125
+ *
126
+ * @returns A provider, or `null` when no API key is configured. A base URL
127
+ * pointing at localhost counts as configured even without a key (Ollama
128
+ * and friends don't need auth).
129
+ */
130
+ export declare function createProviderFromEnv(env?: NodeJS.ProcessEnv): OpenAICompatProvider | null;
131
+ /**
132
+ * Convenience: ask the AI provider to suggest test cases, then validate them
133
+ * against the tools' real input schemas. Does NOT merge with deterministic
134
+ * rules — callers wanting the union should call {@link suggestEdgeCases} (or
135
+ * `suggestEdgeCasesForTools`) themselves and concatenate.
136
+ *
137
+ * @returns Validated cases, rejected entries with reasons, and the provider
138
+ * name (for diagnostic logging).
139
+ */
140
+ export declare function suggestCasesWithAI(analysis: ServerAnalysis, provider: AIProvider, schemas: Map<string, Tool>): Promise<{
141
+ cases: GeneratedCase[];
142
+ rejected: string[];
143
+ providerName: string;
144
+ }>;
145
+ //# sourceMappingURL=provider.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/ai/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;AAC/D,OAAO,EACL,KAAK,aAAa,EAInB,MAAM,sBAAsB,CAAC;AAE9B,4EAA4E;AAC5E,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6BAA6B;IAC7B,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACtC,CAAC,CAAC;CACJ;AAED,kEAAkE;AAClE,MAAM,WAAW,UAAU;IACzB,uEAAuE;IACvE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,YAAY,CAAC,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;CAClE;AAED,gDAAgD;AAChD,MAAM,WAAW,mBAAmB;IAClC,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kFAAkF;IAClF,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,eAAO,MAAM,mBAAmB,8BAA8B,CAAC;AAC/D,eAAO,MAAM,gBAAgB,gBAAgB,CAAC;AAyC9C;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG;IAAE,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,IAAI,CAe1E;AA+FD;;;;;GAKG;AACH,qBAAa,oBAAqB,YAAW,UAAU;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAgB;IACzC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,uFAAuF;IACvF,OAAO,CAAC,sBAAsB,CAAS;gBAE3B,OAAO,GAAE,mBAAwB;IAavC,YAAY,CAAC,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAsCtE,wEAAwE;YAC1D,cAAc;IAsC5B;;;;OAIG;YACW,iBAAiB;IA8D/B,OAAO,CAAC,WAAW;IAoBnB,OAAO,CAAC,SAAS;IAyBjB,OAAO,CAAC,UAAU;CAanB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,qBAAqB,CACnC,GAAG,GAAE,MAAM,CAAC,UAAwB,GACnC,oBAAoB,GAAG,IAAI,CAa7B;AAED;;;;;;;;GAQG;AACH,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,cAAc,EACxB,QAAQ,EAAE,UAAU,EACpB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GACzB,OAAO,CAAC;IAAE,KAAK,EAAE,aAAa,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC,CAQ/E"}