@slbdn/mcp-tester 1.4.1 → 1.5.3

package/docs/testing.md

@@ -0,0 +1,443 @@
+# Testing
+
+Guide to writing and running tests with MCP Tester.
+
+## Running Tests
+
+```bash
+npm test                # Run all tests
+npm run test:watch      # Watch mode (auto-rerun on changes)
+npm run test:coverage   # Run with coverage report
+npm run lint            # Lint code
+```
+
+## Test Structure
+
+```
+src/__tests__/
+├── client.test.ts               # Basic client operations
+├── resources-prompts.test.ts    # Resources and prompts
+├── advanced.test.ts             # Sampling, elicitation
+├── real-server.test.ts          # Integration tests (stdio transport)
+├── everything-server.test.ts    # Full integration (server-everything)
+├── cli.test.ts                  # CLI tool tests
+└── fixtures/
+    └── mock-server.ts           # In-memory mock MCP server
+```
+
+## Writing Tests
+
+### Jest
+
+```typescript
+import { MCPClient } from '@slbdn/mcp-tester';
+import { describe, it, expect, beforeEach, afterEach } from '@jest/globals';
+
+describe('My MCP Server', () => {
+  let client: MCPClient;
+
+  beforeEach(async () => {
+    client = new MCPClient({
+      name: 'test-client',
+      version: '1.0.0',
+    });
+    await client.start({
+      command: 'node',
+      args: ['./my-server.js'],
+    });
+  });
+
+  afterEach(async () => {
+    if (client.isConnected()) {
+      await client.stop();
+    }
+  });
+
+  it('should list tools', async () => {
+    const tools = await client.listTools();
+    expect(tools.length).toBeGreaterThan(0);
+  });
+});
+```
+
+### Vitest
+
+```typescript
+import { MCPClient, setupVitestMatchers } from '@slbdn/mcp-tester';
+import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
+// /// <reference types="@slbdn/mcp-tester/vitest" />
+
+beforeAll(() => setupVitestMatchers());
+
+describe('My MCP Server', () => {
+  let client: MCPClient;
+
+  beforeEach(async () => {
+    client = new MCPClient({ name: 'test-client', version: '1.0.0' });
+    await client.start({ command: 'node', args: ['./my-server.js'] });
+  });
+
+  afterEach(async () => {
+    if (client.isConnected()) await client.stop();
+  });
+
+  it('should list tools', async () => {
+    const tools = await client.listTools();
+    expect(tools).toHaveTool('echo');
+  });
+});
+```
+
+### Custom Test Runner (assert module)
+
+For any test runner — no framework dependency:
+
+```typescript
+import { MCPClient, assert } from '@slbdn/mcp-tester';
+
+const client = new MCPClient();
+await client.start({ command: 'node', args: ['./server.js'] });
+
+const result = await client.callTool({ name: 'echo', arguments: { message: 'hello' } });
+assert.toolTextContains(result, 'hello');
+assert.equal(tools.length, 4);
+```
+
+See [Examples](./examples.md) for full assert usage.
+
+## Testing Patterns
+
+**Always use async/await:**
+
+```typescript
+const result = await client.listTools();   // ✓ Correct
+const result = client.listTools();         // ✗ Returns Promise
+```
+
+**Always clean up in afterEach:**
+
+```typescript
+afterEach(async () => {
+  if (client.isConnected()) {
+    await client.stop();
+  }
+});
+```
+
+**Test errors with rejects.toThrow():**
+
+```typescript
+it('should handle invalid tool', async () => {
+  await expect(
+    client.callTool({ name: 'non-existent', arguments: {} })
+  ).rejects.toThrow();
+});
+```
+
+**Test specific error types:**
+
+```typescript
+import { MCPTimeoutError, MCPServerError } from '@slbdn/mcp-tester';
+
+it('should throw timeout error', async () => {
+  try {
+    await client.callTool({ name: 'slow-tool', arguments: {}, timeout: 1 });
+  } catch (error) {
+    expect(error).toBeInstanceOf(MCPTimeoutError);
+  }
+});
+```
+
+## Custom Matchers
+
+### Jest Setup
+
+```typescript
+import { setupJestMatchers } from '@slbdn/mcp-tester';
+beforeAll(() => setupJestMatchers());
+```
+
+### Vitest Setup
+
+```typescript
+import { setupVitestMatchers } from '@slbdn/mcp-tester';
+import { beforeAll } from 'vitest';
+// /// <reference types="@slbdn/mcp-tester/vitest" />
+beforeAll(() => setupVitestMatchers());
+```
+
+### All Matchers
+
+| Matcher | Description |
+|---------|-------------|
+| **Collection** | |
+| `toHaveTool(name)` | Assert a tool exists by name |
+| `toHaveToolWithSchema(name)` | Assert a tool exists and has an input schema |
+| `toHaveToolCount(n)` | Assert exact number of tools |
+| `toHaveResource(uri)` | Assert a resource exists by URI |
+| `toHaveResourceByName(name)` | Assert a resource exists by display name |
+| `toHaveResourceCount(n)` | Assert exact number of resources |
+| `toHavePrompt(name)` | Assert a prompt exists by name |
+| `toHavePromptWithArgs(name)` | Assert a prompt exists and has arguments |
+| `toHavePromptCount(n)` | Assert exact number of prompts |
+| **Tool Results** | |
+| `toReturnText(expected?)` | Tool text exact match (or just has text) |
+| `toReturnTextContaining(sub)` | Tool text contains substring |
+| `toReturnError()` | Tool result is an error |
+| `toReturnOk()` | Tool result is successful |
+| `toReturnJson(obj)` | Parse tool text as JSON, deep compare |
+| `toReturnContentCount(n)` | Tool has N content items |
+| `toReturnImage()` | Tool result contains an image |
+| **Resource Results** | |
+| `toReturnResourceText(expected?)` | Resource text exact match (or just has text) |
+| `toReturnResourceTextContaining(sub)` | Resource text contains substring |
+| **Prompt Results** | |
+| `toReturnPromptTextContaining(sub)` | Prompt message text contains substring |
+| `toReturnPromptMessageCount(n)` | Prompt has N messages |
+
+All matchers support `.not` negation and work identically in Jest and Vitest.
+
+## Assertion Module (Framework-Agnostic)
+
+For custom test runners or simple scripts — throws `AssertionError` on failure:
+
+| Assertion | Description |
+|-----------|-------------|
+| **Value** | |
+| `equal(a, b)` / `notEqual(a, b)` | Strict equality |
+| `deepEqual(a, b)` | JSON deep equality |
+| `ok(val)` / `notOk(val)` | Truthy / falsy |
+| `throws(fn)` / `doesNotThrow(fn)` | Async error checks |
+| **Numeric** | |
+| `equalNum(a, b)` | Number equality |
+| `greaterThan(a, b)` / `atLeast(a, b)` / `lessThan(a, b)` | Comparisons |
+| `closeTo(a, b, eps?)` | Approximate equality |
+| **String** | |
+| `contains(str, sub)` / `notContains(str, sub)` | Substring checks |
+| `matches(str, regex)` | Regex match |
+| **Tool Results** | |
+| `toolTextEquals(r, str)` | Tool text exact match |
+| `toolTextContains(r, str)` | Tool text contains substring |
+| `toolNumEquals(r, num)` | Parse tool text as number, compare |
+| `toolNumCloseTo(r, num, eps?)` | Numeric approximate match |
+| `toolJsonEquals(r, obj)` | Parse tool text as JSON, deep compare |
+| `toolIsError(r)` / `toolIsOk(r)` | Error/success check |
+| `toolHasContent(r, n?)` | Has N content items |
+| `toolHasImage(r)` | Contains image content |
+| **Resources** | |
+| `resourceHasContent(r, n?)` | Has N resource contents |
+| `resourceTextContains(r, str)` | Resource text contains |
+| **Prompts** | |
+| `promptHasMessages(r, n?)` | Has N messages |
+| `promptTextContains(r, str)` | Prompt text contains |
+
+## Mock Server
+
+Use the in-memory mock server for fast unit tests without a real server process:
+
+```typescript
+import { MockMCPServer } from './fixtures/mock-server.js';
+
+describe('Unit Test', () => {
+  let mockServer: MockMCPServer;
+
+  beforeEach(() => {
+    mockServer = new MockMCPServer();
+  });
+
+  it('should echo messages', async () => {
+    const result = await mockServer.handleToolCall('echo', { message: 'test' });
+    expect(result.content[0].text).toBe('Echo: test');
+  });
+});
+```
+
+### Built-in Tools
+
+| Tool | Description |
+|------|-------------|
+| `echo` | Echo back the input message |
+| `add` | Add two numbers |
+| `delay` | Delay for specified milliseconds |
+| `error_tool` | Always throws an error |
+| `counter` | Stateful counter: increment, get, reset |
+| `items` | Stateful item list: add, list, remove, clear |
+| `transform` | Transform text: upper, lower, reverse, length |
+
+### Configurable Behavior
+
+```typescript
+// Simulate slow responses
+const server = new MockMCPServer({ defaultDelay: 100 });
+
+// Simulate random failures (30% chance)
+const server = new MockMCPServer({ failureRate: 0.3, failureMessage: 'Network error' });
+
+// Enable input schema validation
+const server = new MockMCPServer({ validateSchemas: true });
+
+// Update config at runtime
+server.setConfig({ failureRate: 1.0 }); // always fail
+```
+
+### Custom Handlers
+
+```typescript
+// Custom tool handler
+server.registerToolHandler('echo', (args) => [
+  { type: 'text', text: `Custom: ${args.message}` },
+]);
+
+// Custom resource handler
+server.registerResourceHandler('text://example', (uri) => [
+  { uri, mimeType: 'text/plain', text: 'Custom content' },
+]);
+
+// Custom prompt handler
+server.registerPromptHandler('greet', (args) => [
+  { role: 'assistant', content: { type: 'text', text: `Hi ${args.name}!` } },
+]);
+```
+
+### Stateful Testing
+
+```typescript
+// Counter tool
+await server.handleToolCall('counter', { action: 'increment', by: 5 });
+const result = await server.handleToolCall('counter', { action: 'get' });
+// result.content[0].text === 'Counter: 5'
+
+// Items tool
+await server.handleToolCall('items', { action: 'add', value: 'apple' });
+await server.handleToolCall('items', { action: 'add', value: 'banana' });
+const list = await server.handleToolCall('items', { action: 'list' });
+// list.content[0].text === '["apple","banana"]'
+```
+
+### Call History & Assertions
+
+```typescript
+await server.handleToolCall('echo', { message: 'a' });
+await server.handleToolCall('echo', { message: 'b' });
+
+server.getCallCount('echo'); // 2
+server.getCallHistory();     // [{ tool: 'echo', args: {...}, timestamp: ... }, ...]
+```
+
+### Dynamic Registration
+
+```typescript
+// Add new tools/resources/prompts at runtime
+server.addTool({ name: 'myTool', description: '...', inputSchema: { ... } });
+server.addResource({ uri: 'custom://data', name: 'Data', description: '...', mimeType: 'text/plain' });
+server.addPrompt({ name: 'myPrompt', description: '...', arguments: [] });
+
+// Remove them
+server.removeTool('echo');
+server.removeResource('text://example');
+server.removePrompt('greet');
+
+// Reset all stateful data
+server.resetState();
+```
+**Built-in mock resources:** `text://example`, `config://settings`
+**Built-in mock prompts:** `greet`, `summarize`
+
+## Test Coverage
+
+```bash
+npm run test:coverage
+open coverage/lcov-report/index.html
+```
+
+Coverage thresholds are configured in `jest.config.js` with per-file floors to catch regressions. Run `npm run test:coverage` to see the full breakdown by file.
+
+| File | Stmts | Branch | Func | Lines |
+|------|-------|--------|------|-------|
+| Global | 67% | 61% | 60% | 67% |
+| `assert.ts` | 96% | 87% | 100% | 96% |
+| `matchers.ts` | 68% | 28% | 42% | 64% |
+| `MCPClient.ts` | 68% | 54% | 75% | 68% |
+| `logger.ts` | 84% | 82% | 94% | 83% |
+| `masking.ts` | 100% | 88% | 100% | 100% |
+| `errors.ts` | 100% | 100% | 100% | 100% |
+| `env.ts` | 80% | 100% | 60% | 87% |
+
+If coverage drops below these thresholds, the test command will exit with an error. PRs automatically receive a coverage comment via GitHub Actions (see [CI/CD Integration](./cicd.md)).
+
+## Test Reports
+
+Every test run auto-generates an HTML report with collapsible test trees, timing, and failure details:
+
+```bash
+npm test                         # generates reports/test-report.html
+open reports/test-report.html    # macOS
+```
+
+The report is generated by `jest-html-reporters` (configured in `jest.config.js`). CI uploads reports as downloadable artifacts with 14-day retention.
+
+## Property-Based Testing
+
+Property-based tests use [fast-check](https://github.com/dubzzz/fast-check) to generate random inputs for pure utility functions. 73 tests across 3 modules:
+
+| Module | Tests | What's tested |
+|--------|-------|---------------|
+| `validation.ts` | 44 | Reject/accept all input types, error code invariants |
+| `masking.ts` | 17 | Idempotency, completeness, secret detection, length bounds |
+| `generate-tests.ts` | 12 | No-throw on any schema, JSON serializability, priority order |
+
+Property tests run with 25 iterations per property (configured via `fc.configureGlobal({ numRuns: 25 })`). They're fast (~1s for all 73 tests) and integrated into the normal test suite — no separate command needed.
+
+## TypeScript Type Generation
+
+Generate typed `.d.ts` declarations from your server's tool schemas for type-safe testing:
+
+```bash
+npx @slbdn/mcp-tester generate-types node ./server.js -o server.d.ts
+```
+
+Then use the types in your tests:
+
+```typescript
+import type { ToolArgsMap, ToolName } from './server.d.ts';
+
+it('should add numbers correctly', async () => {
+  const result = await client.callTool({
+    name: 'add',
+    arguments: { a: 3, b: 4 } as ToolArgsMap['add'], // ← type-checked!
+  });
+  expect(result).toReturnText('7');
+});
+```
+
+See [Advanced Usage](./advanced.md#typescript-type-generation) for full documentation.
+
+## Server Health Checks
+
+Use health checks to ensure your server is alive during long test suites:
+
+```typescript
+const health = await client.isHealthy();
+expect(health.healthy).toBe(true);
+expect(health.latencyMs).toBeLessThan(1000);
+expect(health.pid).not.toBeNull();
+```
+
+For long-running test suites, use periodic monitoring to fail fast when the server crashes:
+
+```typescript
+describe('Long test suite', () => {
+  beforeAll(() => {
+    client.startHealthMonitor({
+      interval: 5000,
+      onUnhealthy: (status) => {
+        throw new Error(`Server died: ${status.message}`);
+      },
+    });
+  });
+
+  afterAll(() => client.stopHealthMonitor());
+});
+```
+
+See [Advanced Usage](./advanced.md#server-health-checks) for full documentation.

package/docs/troubleshooting.md

@@ -0,0 +1,139 @@
+# Troubleshooting
+
+## Common Issues
+
+### "Client not started" Error
+
+**Problem:** Calling methods before `start()`
+
+```typescript
+// ✗ Wrong
+const client = new MCPClient();
+const tools = await client.listTools(); // Error!
+
+// ✓ Correct
+const client = new MCPClient();
+await client.start({ command: 'node', args: ['./server.js'] });
+const tools = await client.listTools(); // Works
+```
+
+### Timeout Errors
+
+**Problem:** Requests timing out
+
+```typescript
+// Increase global timeout
+const client = new MCPClient({ timeout: 60000 });
+
+// Or per-call timeout
+await client.callTool({
+  name: 'slow-tool',
+  arguments: {},
+  timeout: 30000,
+});
+```
+
+### Server Process Fails to Start
+
+**Problem:** Server command not found or crashes immediately
+
+```bash
+# Verify server exists and runs standalone
+ls -la ./my-server.js
+node ./my-server.js
+```
+
+### "Module not found" Errors
+
+**Problem:** ESM import without `.js` extension
+
+```typescript
+// ✗ Wrong
+import { MCPClient } from './client/MCPClient';
+
+// ✓ Correct
+import { MCPClient } from './client/MCPClient.js';
+```
+
+### TypeScript Build Errors
+
+```bash
+# Clean and rebuild
+rm -rf dist/
+npm run build
+```
+
+### Server Crashes During Tests
+
+**Problem:** Server process dies mid-test, but `isConnected()` still returns `true`
+
+```typescript
+// Use health checks to detect zombie processes
+const health = await client.isHealthy();
+if (!health.healthy) {
+  console.error(`Server died: ${health.message}`);
+  await client.stop();
+  await client.start({ command: 'node', args: ['./server.js'] });
+}
+
+// Or use periodic monitoring
+client.startHealthMonitor({
+  interval: 3000,
+  onUnhealthy: () => { throw new Error('Server crashed during tests'); },
+});
+```
+
+### Tests Fail in CI but Pass Locally
+
+- Check Node.js version matches (>= 20)
+- Verify server paths are correct
+- Ensure environment variables are set
+- Check `maxWorkers: 1` in Jest config (avoids SIGSEGV crashes)
+
+## Debug Mode
+
+Enable verbose logging:
+
+```typescript
+const client = new MCPClient({
+  logLevel: 'debug',
+  enableProtocolLogging: true,
+});
+```
+
+All log output goes to stderr.
+
+## Test Coverage
+
+```bash
+npm run test:coverage
+open coverage/lcov-report/index.html
+```
+
+Coverage thresholds: **80%** for branches, functions, lines, and statements.
+
+## Known Limitations
+
+- Supports **stdio**, **Streamable HTTP**, and **SSE** transports
+- Designed for **Node.js** servers (other runtimes may need adjustments)
+- Mock server has configurable delays, failures, stateful tools, and validation
+
+## Vitest
+
+Custom matchers work with Vitest. Use `setupVitestMatchers()` and add the type declarations:
+
+```typescript
+import { setupVitestMatchers } from '@slbdn/mcp-tester';
+import { beforeAll } from 'vitest';
+// /// <reference types="@slbdn/mcp-tester/vitest" />
+
+beforeAll(() => setupVitestMatchers());
+```
+
+If you get TypeScript errors with custom matchers, make sure your `tsconfig.json` includes the `vitest.d.ts` from the package:
+
+```json
+{
+  "include": ["node_modules/@slbdn/mcp-tester/vitest.d.ts"]
+}
+```

package/jest.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Jest type declarations for MCP Tester custom matchers.
+ *
+ * IMPORTANT: TypeScript does NOT propagate `declare module 'expect' { ... }`
+ * augmentations across `node_modules` boundaries. A `.d.ts` file inside a
+ * package cannot augment the consumer's `expect` type — the augmentation
+ * only takes effect when the declaring file is part of the consumer's own
+ * compilation (i.e. included by their tsconfig or referenced from a file
+ * the consumer owns).
+ *
+ * This is a known TypeScript limitation that also affects libraries like
+ * `jest-extended`. See https://github.com/microsoft/TypeScript/issues/15031
+ *
+ * Setup (pick one):
+ *
+ *   1. Copy this file into your project root (or a `tests/` folder) and
+ *      include it in your `tsconfig.json`:
+ *        "include": ["jest.d.ts", "tests"]
+ *
+ *   2. Inline the augmentation in your own `tests.d.ts` / `global.d.ts`:
+ *        import type { MCPMatchers } from '@slbdn/mcp-tester/dist/matchers';
+ *        declare module 'expect' {
+ *          interface Matchers<R extends void | Promise<void>, T = unknown> extends MCPMatchers<R> {}
+ *        }
+ *
+ *   3. Use Vitest instead (the `vitest` package's module structure accepts
+ *      the augmentation directly; just install vitest and import
+ *      `setupVitestMatchers`).
+ *
+ * Without one of these, the matchers still RUN at runtime (via
+ * `expect.extend`) but will not be type-checked by `tsc`.
+ *
+ * Then in your test setup:
+ *   import { setupJestMatchers } from '@slbdn/mcp-tester';
+ *   beforeAll(() => setupJestMatchers());
+ */
+
+import type { MCPMatchers } from '@slbdn/mcp-tester/dist/matchers';
+
+declare module 'expect' {
+  // eslint-disable-next-line @typescript-eslint/no-empty-object-type, @typescript-eslint/no-unused-vars -- module augmentation requires an interface; T matches `expect`'s arity
+  interface Matchers<R extends void | Promise<void>, T = unknown> extends MCPMatchers<R> {}
+}

package/package.json

@@ -1,40 +1,30 @@
 {
   "name": "@slbdn/mcp-tester",
-  "version": "1.4.1",
+  "version": "1.5.3",
   "description": "Minimal MCP client implementation for CI/CD testing of MCP servers",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
   "sideEffects": false,
   "files": [
-    "dist/client",
-    "dist/utils",
-    "dist/cli",
-    "dist/index.js",
-    "dist/index.d.ts",
-    "dist/index.d.ts.map",
-    "dist/index.js.map",
-    "dist/assert.js",
-    "dist/assert.d.ts",
-    "dist/assert.d.ts.map",
-    "dist/assert.js.map",
-    "dist/matchers.js",
-    "dist/matchers.d.ts",
-    "dist/matchers.d.ts.map",
-    "dist/matchers.js.map",
+    "dist",
+    "docs/*.md",
     "vitest.d.ts",
+    "jest.d.ts",
     "LICENSE",
     "README.md",
     "CHANGELOG.md"
   ],
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && node scripts/copy-templates.mjs",
+    "predev": "node scripts/copy-templates.mjs",
     "dev": "tsc --watch",
     "docs": "typedoc",
     "test": "jest",
     "test:watch": "jest --watch",
     "test:coverage": "jest --coverage",
-    "benchmark": "npx tsx src/__tests__/benchmarks.ts 2>/dev/null",
+    "test:package": "node scripts/smoke-test-package.mjs",
+    "benchmark": "tsx src/__tests__/benchmarks.ts",
     "lint": "eslint src --ext .ts",
     "format": "prettier --write \"src/**/*.ts\"",
     "format:check": "prettier --check \"src/**/*.ts\"",
@@ -83,18 +73,19 @@
   },
   "devDependencies": {
     "@modelcontextprotocol/server-everything": "^2026.1.26",
-    "@types/jest": "^29.5.0",
+    "@types/jest": "^30.0.0",
     "@types/node": "^20.19.41",
     "@typescript-eslint/eslint-plugin": "^8.60.0",
     "@typescript-eslint/parser": "^8.60.0",
     "eslint": "^8.56.0",
     "fast-check": "^4.8.0",
     "husky": "^9.1.7",
-    "jest": "^29.7.0",
+    "jest": "^30.5.2",
     "jest-html-reporters": "^3.1.7",
     "lint-staged": "^16.4.0",
     "prettier": "^3.8.3",
     "ts-jest": "^29.4.11",
+    "tsx": "^4.23.15",
     "typedoc": "^0.28.19",
     "typedoc-plugin-markdown": "^4.11.0",
     "typescript": "^5.3.0"