mcp-eval-runner 1.0.0

package/evals/reference/mcp-fetch.yaml

@@ -0,0 +1,46 @@
+name: mcp_fetch_reference
+description: >
+  Reference eval suite for the official MCP Fetch server
+  (github.com/modelcontextprotocol/servers/tree/main/src/fetch).
+  Covers URL fetching in multiple formats and robots.txt handling.
+
+server:
+  command: uvx
+  args: ["mcp-server-fetch"]
+
+steps:
+  - id: fetch_html
+    description: Fetch a URL and return as markdown
+    tool: fetch
+    input:
+      url: "https://example.com"
+      max_length: 1000
+    expected_output: "Example Domain"
+    expect:
+      output_contains: "Example Domain"
+      output_not_contains: "error"
+      latency_under: 10000
+
+  - id: fetch_raw
+    description: Fetch raw HTML content
+    tool: fetch
+    input:
+      url: "https://example.com"
+      raw: true
+      max_length: 500
+    expected_output: "<html"
+    expect:
+      output_matches: "<html|<!DOCTYPE"
+      latency_under: 10000
+
+  - id: fetch_with_start_index
+    description: Fetch with pagination offset
+    tool: fetch
+    input:
+      url: "https://example.com"
+      start_index: 100
+      max_length: 200
+    expected_output: ""
+    expect:
+      output_not_contains: "error"
+      latency_under: 10000

package/evals/reference/mcp-filesystem.yaml

@@ -0,0 +1,63 @@
+name: mcp_filesystem_reference
+description: >
+  Reference eval suite for the official MCP Filesystem server
+  (github.com/modelcontextprotocol/servers/tree/main/src/filesystem).
+  Covers read, write, list, and search operations.
+  Set MCP_FS_CMD and MCP_FS_ROOT, or remove the server block for simulation mode.
+
+server:
+  command: node
+  args: ["${MCP_FS_CMD:-/usr/local/bin/mcp-filesystem}", "${MCP_FS_ROOT:-/tmp}"]
+
+steps:
+  - id: list_root
+    description: List the allowed root directory
+    tool: list_directory
+    input:
+      path: "/"
+    expected_output: "directory"
+    expect:
+      output_not_contains: "error"
+      latency_under: 3000
+
+  - id: write_file
+    description: Write a test file
+    tool: write_file
+    input:
+      path: "/eval-test-file.txt"
+      content: "mcp-eval-runner test content"
+    expected_output: "written"
+    expect:
+      output_not_contains: "error"
+      latency_under: 3000
+
+  - id: read_file
+    description: Read back the file we just wrote — uses step output piping
+    tool: read_file
+    input:
+      path: "/eval-test-file.txt"
+    expected_output: "mcp-eval-runner test content"
+    expect:
+      output_contains: "mcp-eval-runner test content"
+      latency_under: 3000
+
+  - id: search_files
+    description: Search for the test file by name
+    tool: search_files
+    input:
+      path: "/"
+      pattern: "eval-test-file"
+    expected_output: "eval-test-file.txt"
+    expect:
+      output_contains: "eval-test-file"
+      latency_under: 5000
+
+  - id: delete_file
+    description: Clean up — delete the test file
+    tool: delete_file
+    input:
+      path: "/eval-test-file.txt"
+    expected_output: "deleted"
+    expect:
+      output_not_contains: "error"
+      latency_under: 3000

package/evals/reference/mcp-memory.yaml

@@ -0,0 +1,70 @@
+name: mcp_memory_reference
+description: >
+  Reference eval suite for the official MCP Memory server
+  (github.com/modelcontextprotocol/servers/tree/main/src/memory).
+  Covers entity creation, relation creation, search, and deletion.
+  Set MCP_MEMORY_CMD to the path of your built memory server binary,
+  or remove the server block to run in simulation mode.
+
+server:
+  command: node
+  args: ["${MCP_MEMORY_CMD:-/usr/local/bin/mcp-memory}"]
+
+steps:
+  - id: create_entity
+    description: Create a test entity in the knowledge graph
+    tool: create_entities
+    input:
+      entities:
+        - name: TestProject
+          entityType: Project
+          observations:
+            - "A test project created by mcp-eval-runner"
+    expected_output: "TestProject"
+    expect:
+      output_contains: "TestProject"
+      latency_under: 5000
+
+  - id: create_relation
+    description: Create a relation between entities
+    tool: create_relations
+    input:
+      relations:
+        - from: TestProject
+          to: mcp-eval-runner
+          relationType: tested_by
+    expected_output: "tested_by"
+    expect:
+      output_contains: "tested_by"
+      latency_under: 5000
+
+  - id: search_nodes
+    description: Search for the entity we just created
+    tool: search_nodes
+    input:
+      query: TestProject
+    expected_output: "TestProject"
+    expect:
+      output_contains: "TestProject"
+      output_not_contains: "error"
+      latency_under: 5000
+
+  - id: read_graph
+    description: Read the full graph and verify structure
+    tool: read_graph
+    input: {}
+    expected_output: "TestProject"
+    expect:
+      output_contains: "TestProject"
+      latency_under: 5000
+
+  - id: delete_entity
+    description: Clean up — delete the test entity
+    tool: delete_entities
+    input:
+      entityNames:
+        - TestProject
+    expected_output: "deleted"
+    expect:
+      output_not_contains: "error"
+      latency_under: 5000

package/evals/reference/step-piping-example.yaml

@@ -0,0 +1,25 @@
+name: step_piping_example
+description: >
+  Demonstrates step output piping: the output of step_1 is injected into
+  step_2's input using the {{steps.<id>.output}} placeholder syntax.
+  This fixture runs in simulation mode (no server block).
+
+steps:
+  - id: step_1
+    description: First step — produces output that step_2 will consume
+    tool: search
+    input:
+      query: "mcp eval runner"
+    expected_output: "result: mcp-eval-runner v1.0"
+    expect:
+      output_contains: "mcp-eval-runner"
+
+  - id: step_2
+    description: Second step — input references step_1's output via placeholder
+    tool: summarize
+    input:
+      # {{steps.step_1.output}} is replaced at runtime with step_1's actual output
+      text: "{{steps.step_1.output}}"
+    expected_output: "Summary: mcp-eval-runner v1.0"
+    expect:
+      output_contains: "Summary"

package/evals/smoke.yaml

@@ -0,0 +1,12 @@
+name: smoke
+description: "Verify eval runner itself is working"
+server:
+  command: node
+  args: ["dist/index.js"]
+steps:
+  - id: list_check
+    description: "List available test cases and verify smoke fixture appears"
+    tool: list_cases
+    input: {}
+    expect:
+      output_contains: "smoke"

package/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "mcp-eval-runner",
+  "mcpName": "io.github.dbsectrainer/mcp-eval-runner",
+  "version": "1.0.0",
+  "description": "A standardized testing harness for MCP servers and agent workflows",
+  "author": "dbsectrainer",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbsectrainer/mcp-eval-runner.git"
+  },
+  "homepage": "https://github.com/dbsectrainer/mcp-eval-runner#readme",
+  "type": "module",
+  "bin": {
+    "mcp-eval-runner": "./dist/index.js"
+  },
+  "files": [
+    "dist",
+    "evals",
+    "README.md",
+    "CHANGELOG.md",
+    "LICENSE",
+    ".env.example"
+  ],
+  "keywords": [
+    "mcp",
+    "mcp-server",
+    "testing",
+    "evals"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "coverage": "vitest run --coverage",
+    "dev": "tsx src/index.ts",
+    "inspect": "npx @modelcontextprotocol/inspector node dist/index.js",
+    "lint": "eslint src tests",
+    "format": "prettier --write src tests",
+    "format:check": "prettier --check src tests",
+    "prepublishOnly": "npm run build"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.12.0",
+    "chokidar": "^5.0.0",
+    "express": "^5.2.1",
+    "js-yaml": "^4.1.0"
+  },
+  "devDependencies": {
+    "@types/express": "^5.0.0",
+    "@types/js-yaml": "^4.0.0",
+    "@types/node": "^24.12.0",
+    "@types/yargs": "^17.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.0.0",
+    "@typescript-eslint/parser": "^8.0.0",
+    "@vitest/coverage-v8": "^4.1.0",
+    "eslint": "^10.0.3",
+    "eslint-config-prettier": "^10.1.8",
+    "prettier": "^3.0.0",
+    "tsx": "^4.0.0",
+    "typescript": "^5.0.0",
+    "vitest": "^4.1.0",
+    "yargs": "^18.0.0"
+  },
+  "engines": {
+    "node": ">=22.5.0"
+  }
+}