@runhuman/mcp-server 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +12 -12
- package/README.md +219 -219
- package/dist/index.js +83 -83
- package/package.json +62 -62
package/.env.example
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
# RunHuman API Configuration
|
|
2
|
-
|
|
3
|
-
# API Base URL
|
|
4
|
-
# For local development, use http://localhost:3400
|
|
5
|
-
# For production, use your deployed API URL
|
|
6
|
-
RUNHUMAN_API_URL=http://localhost:3400
|
|
7
|
-
|
|
8
|
-
# API Key
|
|
9
|
-
# Get this from the API dashboard at http://localhost:3400/app.html
|
|
10
|
-
# Or use the default test key below for local development
|
|
11
|
-
# Format: qa_live_xxxxxxxxxxxxxxxxxxxxx
|
|
12
|
-
RUNHUMAN_API_KEY=qa_live_test_key_for_demo_purposes_only_12345
|
|
1
|
+
# RunHuman API Configuration
|
|
2
|
+
|
|
3
|
+
# API Base URL
|
|
4
|
+
# For local development, use http://localhost:3400
|
|
5
|
+
# For production, use your deployed API URL
|
|
6
|
+
RUNHUMAN_API_URL=http://localhost:3400
|
|
7
|
+
|
|
8
|
+
# API Key
|
|
9
|
+
# Get this from the API dashboard at http://localhost:3400/app.html
|
|
10
|
+
# Or use the default test key below for local development
|
|
11
|
+
# Format: qa_live_xxxxxxxxxxxxxxxxxxxxx
|
|
12
|
+
RUNHUMAN_API_KEY=qa_live_test_key_for_demo_purposes_only_12345
|
package/README.md
CHANGED
|
@@ -1,219 +1,219 @@
|
|
|
1
|
-
# RunHuman MCP Server
|
|
2
|
-
|
|
3
|
-
A Model Context Protocol (MCP) server that allows AI agents to interact with the RunHuman QA testing service.
|
|
4
|
-
|
|
5
|
-
## Overview
|
|
6
|
-
|
|
7
|
-
This MCP server provides tools for creating and managing human QA jobs through the RunHuman API. AI agents can use this server to:
|
|
8
|
-
|
|
9
|
-
- Create new QA jobs with custom schemas
|
|
10
|
-
- Check the status of running jobs
|
|
11
|
-
- Retrieve completed job results
|
|
12
|
-
|
|
13
|
-
## Installation
|
|
14
|
-
|
|
15
|
-
### For Claude Desktop (Recommended)
|
|
16
|
-
|
|
17
|
-
1. Get your API key at: https://runhuman.com/app.html
|
|
18
|
-
|
|
19
|
-
2. Add to your Claude Desktop config (`~/Library/Application Support/Claude/claude_desktop_config.json` on Mac):
|
|
20
|
-
|
|
21
|
-
```json
|
|
22
|
-
{
|
|
23
|
-
"mcpServers": {
|
|
24
|
-
"runhuman": {
|
|
25
|
-
"command": "npx",
|
|
26
|
-
"args": ["-y", "@runhuman/mcp-server", "--api-key=qa_live_xxxxxxxxxxxxx"]
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
```
|
|
31
|
-
|
|
32
|
-
3. Restart Claude Desktop
|
|
33
|
-
|
|
34
|
-
That's it! The server will be automatically downloaded and run by Claude.
|
|
35
|
-
|
|
36
|
-
### For Development
|
|
37
|
-
|
|
38
|
-
From the monorepo root:
|
|
39
|
-
|
|
40
|
-
```bash
|
|
41
|
-
npm install
|
|
42
|
-
npm run build --workspace=@runhuman/mcp-server
|
|
43
|
-
|
|
44
|
-
# Run with API key
|
|
45
|
-
node packages/mcp-server/dist/index.js --api-key=qa_live_xxxxx
|
|
46
|
-
```
|
|
47
|
-
|
|
48
|
-
### Available Tools
|
|
49
|
-
|
|
50
|
-
#### `create_job`
|
|
51
|
-
Create a new QA job with human testers.
|
|
52
|
-
|
|
53
|
-
**Parameters:**
|
|
54
|
-
- `url` (string): The URL to test
|
|
55
|
-
- `description` (string): Instructions for the human tester describing what to test
|
|
56
|
-
- `schema` (object): Expected result schema that the tester response will be extracted into
|
|
57
|
-
|
|
58
|
-
#### `get_job_status`
|
|
59
|
-
Get the current status of a QA job.
|
|
60
|
-
|
|
61
|
-
**Parameters:**
|
|
62
|
-
- `jobId` (string): The ID of the job to check
|
|
63
|
-
|
|
64
|
-
#### `get_job_result`
|
|
65
|
-
Get the results of a completed QA job.
|
|
66
|
-
|
|
67
|
-
**Parameters:**
|
|
68
|
-
- `jobId` (string): The ID of the completed job
|
|
69
|
-
|
|
70
|
-
**Returns:**
|
|
71
|
-
- `result`: Structured test results extracted from tester's response
|
|
72
|
-
- `status`: Job status (completed, failed, etc.)
|
|
73
|
-
- `costUsd`: Exact cost in USD with full precision (e.g., 0.396)
|
|
74
|
-
- `testDurationSeconds`: Time spent by tester in seconds (rounded up)
|
|
75
|
-
- Additional metadata (timestamps, tester info, etc.)
|
|
76
|
-
|
|
77
|
-
**Cost Calculation:**
|
|
78
|
-
- Costs are calculated as: `duration × $0.0018/second` (general-use tier)
|
|
79
|
-
- Duration is always rounded UP using Math.ceil (any partial second counts)
|
|
80
|
-
- Costs are never $0 unless the tester never actually worked on it
|
|
81
|
-
- Full precision maintained (not rounded to cents)
|
|
82
|
-
|
|
83
|
-
## Configuration
|
|
84
|
-
|
|
85
|
-
The MCP server needs to be configured with your RunHuman API credentials.
|
|
86
|
-
|
|
87
|
-
### 1. Get an API Key
|
|
88
|
-
|
|
89
|
-
**Option A: Via Dashboard**
|
|
90
|
-
1. Start the API server: `npm run dev --workspace=@runhuman/api`
|
|
91
|
-
2. Open http://localhost:3400/app.html
|
|
92
|
-
3. Go to "API Keys" tab
|
|
93
|
-
4. Click "Create API Key"
|
|
94
|
-
5. Copy the key (starts with `qa_live_`)
|
|
95
|
-
|
|
96
|
-
**Option B: Use Default Test Key**
|
|
97
|
-
- For local development, you can use: `qa_live_test_key_123`
|
|
98
|
-
- This key exists in `packages/api/data/api-keys.json`
|
|
99
|
-
|
|
100
|
-
### 2. Configure Environment Variables
|
|
101
|
-
|
|
102
|
-
Create a `.env` file in the MCP server directory:
|
|
103
|
-
|
|
104
|
-
```bash
|
|
105
|
-
# For local development
|
|
106
|
-
RUNHUMAN_API_URL=http://localhost:3400
|
|
107
|
-
RUNHUMAN_API_KEY=qa_live_test_key_123
|
|
108
|
-
|
|
109
|
-
# For production
|
|
110
|
-
RUNHUMAN_API_URL=https://api.runhuman.com
|
|
111
|
-
RUNHUMAN_API_KEY=qa_live_xxxxxxxxxxxxxxxxxxxxx
|
|
112
|
-
```
|
|
113
|
-
|
|
114
|
-
**Important:** Never commit `.env` files to git! They're already in `.gitignore`.
|
|
115
|
-
|
|
116
|
-
### 3. Verify Configuration
|
|
117
|
-
|
|
118
|
-
Test your API key works:
|
|
119
|
-
|
|
120
|
-
```bash
|
|
121
|
-
curl http://localhost:3400/api/jobs \
|
|
122
|
-
-H "Authorization: Bearer qa_live_test_key_123" \
|
|
123
|
-
-H "Content-Type: application/json" \
|
|
124
|
-
-d '{"url":"https://example.com","description":"test","outputSchema":{}}'
|
|
125
|
-
```
|
|
126
|
-
|
|
127
|
-
Should return a job ID if authentication works.
|
|
128
|
-
|
|
129
|
-
For more details, see [docs/API-AUTHENTICATION.md](docs/API-AUTHENTICATION.md)
|
|
130
|
-
|
|
131
|
-
## Testing
|
|
132
|
-
|
|
133
|
-
The MCP server includes automated tests to verify it's working correctly:
|
|
134
|
-
|
|
135
|
-
```bash
|
|
136
|
-
# Build first
|
|
137
|
-
npm run build --workspace=@runhuman/mcp-server
|
|
138
|
-
|
|
139
|
-
# Run simple automated test
|
|
140
|
-
npm run test --workspace=@runhuman/mcp-server
|
|
141
|
-
|
|
142
|
-
# Or use the MCP Inspector (interactive testing)
|
|
143
|
-
npm run test:inspector --workspace=@runhuman/mcp-server
|
|
144
|
-
```
|
|
145
|
-
|
|
146
|
-
The test script will:
|
|
147
|
-
1. ✅ Initialize a connection to the MCP server
|
|
148
|
-
2. ✅ List all available tools (create_job, get_job_status, get_job_result)
|
|
149
|
-
3. ✅ Test calling the create_job tool
|
|
150
|
-
|
|
151
|
-
### Expected Test Output
|
|
152
|
-
|
|
153
|
-
```
|
|
154
|
-
✅ Server initialized successfully
|
|
155
|
-
✅ Tools listed: create_job, get_job_status, get_job_result
|
|
156
|
-
✅ create_job tool called successfully
|
|
157
|
-
```
|
|
158
|
-
|
|
159
|
-
## Development
|
|
160
|
-
|
|
161
|
-
```bash
|
|
162
|
-
# Watch mode (auto-rebuild on changes)
|
|
163
|
-
npm run dev --workspace=@runhuman/mcp-server
|
|
164
|
-
|
|
165
|
-
# Build
|
|
166
|
-
npm run build --workspace=@runhuman/mcp-server
|
|
167
|
-
|
|
168
|
-
# Test after building
|
|
169
|
-
npm run test --workspace=@runhuman/mcp-server
|
|
170
|
-
```
|
|
171
|
-
|
|
172
|
-
## Integration with Claude Desktop
|
|
173
|
-
|
|
174
|
-
To use this MCP server with Claude Desktop, add it to your configuration:
|
|
175
|
-
|
|
176
|
-
```json
|
|
177
|
-
{
|
|
178
|
-
"mcpServers": {
|
|
179
|
-
"runhuman": {
|
|
180
|
-
"command": "node",
|
|
181
|
-
"args": ["/path/to/qa-experiment/packages/mcp-server/dist/index.js"]
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
```
|
|
186
|
-
|
|
187
|
-
## Example Usage
|
|
188
|
-
|
|
189
|
-
Once connected to an AI agent (like Claude), the agent can use these tools naturally:
|
|
190
|
-
|
|
191
|
-
**User:** "Can someone test my checkout page at https://myapp.com/checkout?"
|
|
192
|
-
|
|
193
|
-
**Agent uses create_job:**
|
|
194
|
-
```
|
|
195
|
-
✅ Job created successfully!
|
|
196
|
-
Job ID: job_abc123
|
|
197
|
-
Status: pending
|
|
198
|
-
...
|
|
199
|
-
```
|
|
200
|
-
|
|
201
|
-
**Agent polls get_job_status until complete, then calls get_job_result:**
|
|
202
|
-
```
|
|
203
|
-
✅ Test completed!
|
|
204
|
-
Results Summary:
|
|
205
|
-
- Checkout Flow: ✅ Working
|
|
206
|
-
- Payment Processing: ✅ Successful
|
|
207
|
-
...
|
|
208
|
-
```
|
|
209
|
-
|
|
210
|
-
## Developer Documentation
|
|
211
|
-
|
|
212
|
-
For developers working on this MCP server:
|
|
213
|
-
- [docs/HOW-AGENTS-USE-MCP.md](docs/HOW-AGENTS-USE-MCP.md) - How AI agents discover and use MCP servers
|
|
214
|
-
- [docs/TOOL-RESPONSE-BEST-PRACTICES.md](docs/TOOL-RESPONSE-BEST-PRACTICES.md) - Best practices for tool responses
|
|
215
|
-
|
|
216
|
-
## Learn More
|
|
217
|
-
|
|
218
|
-
- [Model Context Protocol Documentation](https://modelcontextprotocol.io/)
|
|
219
|
-
- [RunHuman API Documentation](../api/README.md)
|
|
1
|
+
# RunHuman MCP Server
|
|
2
|
+
|
|
3
|
+
A Model Context Protocol (MCP) server that allows AI agents to interact with the RunHuman QA testing service.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
This MCP server provides tools for creating and managing human QA jobs through the RunHuman API. AI agents can use this server to:
|
|
8
|
+
|
|
9
|
+
- Create new QA jobs with custom schemas
|
|
10
|
+
- Check the status of running jobs
|
|
11
|
+
- Retrieve completed job results
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
### For Claude Desktop (Recommended)
|
|
16
|
+
|
|
17
|
+
1. Get your API key at: https://runhuman.com/app.html
|
|
18
|
+
|
|
19
|
+
2. Add to your Claude Desktop config (`~/Library/Application Support/Claude/claude_desktop_config.json` on Mac):
|
|
20
|
+
|
|
21
|
+
```json
|
|
22
|
+
{
|
|
23
|
+
"mcpServers": {
|
|
24
|
+
"runhuman": {
|
|
25
|
+
"command": "npx",
|
|
26
|
+
"args": ["-y", "@runhuman/mcp-server", "--api-key=qa_live_xxxxxxxxxxxxx"]
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
3. Restart Claude Desktop
|
|
33
|
+
|
|
34
|
+
That's it! The server will be automatically downloaded and run by Claude.
|
|
35
|
+
|
|
36
|
+
### For Development
|
|
37
|
+
|
|
38
|
+
From the monorepo root:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
npm install
|
|
42
|
+
npm run build --workspace=@runhuman/mcp-server
|
|
43
|
+
|
|
44
|
+
# Run with API key
|
|
45
|
+
node packages/mcp-server/dist/index.js --api-key=qa_live_xxxxx
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
### Available Tools
|
|
49
|
+
|
|
50
|
+
#### `create_job`
|
|
51
|
+
Create a new QA job with human testers.
|
|
52
|
+
|
|
53
|
+
**Parameters:**
|
|
54
|
+
- `url` (string): The URL to test
|
|
55
|
+
- `description` (string): Instructions for the human tester describing what to test
|
|
56
|
+
- `schema` (object): Expected result schema that the tester response will be extracted into
|
|
57
|
+
|
|
58
|
+
#### `get_job_status`
|
|
59
|
+
Get the current status of a QA job.
|
|
60
|
+
|
|
61
|
+
**Parameters:**
|
|
62
|
+
- `jobId` (string): The ID of the job to check
|
|
63
|
+
|
|
64
|
+
#### `get_job_result`
|
|
65
|
+
Get the results of a completed QA job.
|
|
66
|
+
|
|
67
|
+
**Parameters:**
|
|
68
|
+
- `jobId` (string): The ID of the completed job
|
|
69
|
+
|
|
70
|
+
**Returns:**
|
|
71
|
+
- `result`: Structured test results extracted from tester's response
|
|
72
|
+
- `status`: Job status (completed, failed, etc.)
|
|
73
|
+
- `costUsd`: Exact cost in USD with full precision (e.g., 0.396)
|
|
74
|
+
- `testDurationSeconds`: Time spent by tester in seconds (rounded up)
|
|
75
|
+
- Additional metadata (timestamps, tester info, etc.)
|
|
76
|
+
|
|
77
|
+
**Cost Calculation:**
|
|
78
|
+
- Costs are calculated as: `duration × $0.0018/second` (general-use tier)
|
|
79
|
+
- Duration is always rounded UP using Math.ceil (any partial second counts)
|
|
80
|
+
- Costs are never $0 unless the tester never actually worked on it
|
|
81
|
+
- Full precision maintained (not rounded to cents)
|
|
82
|
+
|
|
83
|
+
## Configuration
|
|
84
|
+
|
|
85
|
+
The MCP server needs to be configured with your RunHuman API credentials.
|
|
86
|
+
|
|
87
|
+
### 1. Get an API Key
|
|
88
|
+
|
|
89
|
+
**Option A: Via Dashboard**
|
|
90
|
+
1. Start the API server: `npm run dev --workspace=@runhuman/api`
|
|
91
|
+
2. Open http://localhost:3400/app.html
|
|
92
|
+
3. Go to "API Keys" tab
|
|
93
|
+
4. Click "Create API Key"
|
|
94
|
+
5. Copy the key (starts with `qa_live_`)
|
|
95
|
+
|
|
96
|
+
**Option B: Use Default Test Key**
|
|
97
|
+
- For local development, you can use: `qa_live_test_key_123`
|
|
98
|
+
- This key exists in `packages/api/data/api-keys.json`
|
|
99
|
+
|
|
100
|
+
### 2. Configure Environment Variables
|
|
101
|
+
|
|
102
|
+
Create a `.env` file in the MCP server directory:
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
# For local development
|
|
106
|
+
RUNHUMAN_API_URL=http://localhost:3400
|
|
107
|
+
RUNHUMAN_API_KEY=qa_live_test_key_123
|
|
108
|
+
|
|
109
|
+
# For production
|
|
110
|
+
RUNHUMAN_API_URL=https://api.runhuman.com
|
|
111
|
+
RUNHUMAN_API_KEY=qa_live_xxxxxxxxxxxxxxxxxxxxx
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
**Important:** Never commit `.env` files to git! They're already in `.gitignore`.
|
|
115
|
+
|
|
116
|
+
### 3. Verify Configuration
|
|
117
|
+
|
|
118
|
+
Test your API key works:
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
curl http://localhost:3400/api/jobs \
|
|
122
|
+
-H "Authorization: Bearer qa_live_test_key_123" \
|
|
123
|
+
-H "Content-Type: application/json" \
|
|
124
|
+
-d '{"url":"https://example.com","description":"test","outputSchema":{}}'
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Should return a job ID if authentication works.
|
|
128
|
+
|
|
129
|
+
For more details, see [docs/API-AUTHENTICATION.md](docs/API-AUTHENTICATION.md)
|
|
130
|
+
|
|
131
|
+
## Testing
|
|
132
|
+
|
|
133
|
+
The MCP server includes automated tests to verify it's working correctly:
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
# Build first
|
|
137
|
+
npm run build --workspace=@runhuman/mcp-server
|
|
138
|
+
|
|
139
|
+
# Run simple automated test
|
|
140
|
+
npm run test --workspace=@runhuman/mcp-server
|
|
141
|
+
|
|
142
|
+
# Or use the MCP Inspector (interactive testing)
|
|
143
|
+
npm run test:inspector --workspace=@runhuman/mcp-server
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
The test script will:
|
|
147
|
+
1. ✅ Initialize a connection to the MCP server
|
|
148
|
+
2. ✅ List all available tools (create_job, get_job_status, get_job_result)
|
|
149
|
+
3. ✅ Test calling the create_job tool
|
|
150
|
+
|
|
151
|
+
### Expected Test Output
|
|
152
|
+
|
|
153
|
+
```
|
|
154
|
+
✅ Server initialized successfully
|
|
155
|
+
✅ Tools listed: create_job, get_job_status, get_job_result
|
|
156
|
+
✅ create_job tool called successfully
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## Development
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
# Watch mode (auto-rebuild on changes)
|
|
163
|
+
npm run dev --workspace=@runhuman/mcp-server
|
|
164
|
+
|
|
165
|
+
# Build
|
|
166
|
+
npm run build --workspace=@runhuman/mcp-server
|
|
167
|
+
|
|
168
|
+
# Test after building
|
|
169
|
+
npm run test --workspace=@runhuman/mcp-server
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## Integration with Claude Desktop
|
|
173
|
+
|
|
174
|
+
To use this MCP server with Claude Desktop, add it to your configuration:
|
|
175
|
+
|
|
176
|
+
```json
|
|
177
|
+
{
|
|
178
|
+
"mcpServers": {
|
|
179
|
+
"runhuman": {
|
|
180
|
+
"command": "node",
|
|
181
|
+
"args": ["/path/to/qa-experiment/packages/mcp-server/dist/index.js"]
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
## Example Usage
|
|
188
|
+
|
|
189
|
+
Once connected to an AI agent (like Claude), the agent can use these tools naturally:
|
|
190
|
+
|
|
191
|
+
**User:** "Can someone test my checkout page at https://myapp.com/checkout?"
|
|
192
|
+
|
|
193
|
+
**Agent uses create_job:**
|
|
194
|
+
```
|
|
195
|
+
✅ Job created successfully!
|
|
196
|
+
Job ID: job_abc123
|
|
197
|
+
Status: pending
|
|
198
|
+
...
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
**Agent polls get_job_status until complete, then calls get_job_result:**
|
|
202
|
+
```
|
|
203
|
+
✅ Test completed!
|
|
204
|
+
Results Summary:
|
|
205
|
+
- Checkout Flow: ✅ Working
|
|
206
|
+
- Payment Processing: ✅ Successful
|
|
207
|
+
...
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
## Developer Documentation
|
|
211
|
+
|
|
212
|
+
For developers working on this MCP server:
|
|
213
|
+
- [docs/HOW-AGENTS-USE-MCP.md](docs/HOW-AGENTS-USE-MCP.md) - How AI agents discover and use MCP servers
|
|
214
|
+
- [docs/TOOL-RESPONSE-BEST-PRACTICES.md](docs/TOOL-RESPONSE-BEST-PRACTICES.md) - Best practices for tool responses
|
|
215
|
+
|
|
216
|
+
## Learn More
|
|
217
|
+
|
|
218
|
+
- [Model Context Protocol Documentation](https://modelcontextprotocol.io/)
|
|
219
|
+
- [RunHuman API Documentation](../api/README.md)
|
package/dist/index.js
CHANGED
|
@@ -75,22 +75,22 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
|
|
|
75
75
|
tools: [
|
|
76
76
|
{
|
|
77
77
|
name: 'create_job',
|
|
78
|
-
description: `⚠️ IMPORTANT: This ONLY creates and queues a job. It does NOT perform the test or return results. You MUST follow up with get_job_status and get_job_result.
|
|
79
|
-
|
|
80
|
-
Creates a QA job that will be performed by a REAL HUMAN tester (not AI). The human will manually test your application, describe findings in natural language, and GPT-4o will extract structured data from their response.
|
|
81
|
-
|
|
82
|
-
Use this when you need human verification of:
|
|
83
|
-
- UI/UX functionality that's hard to automate
|
|
84
|
-
- Visual issues, accessibility problems
|
|
85
|
-
- Complex user flows (login, checkout, forms)
|
|
86
|
-
- Cross-browser compatibility
|
|
87
|
-
- Real user experience feedback
|
|
88
|
-
|
|
89
|
-
⚠️ REQUIRED WORKFLOW (do NOT skip steps):
|
|
90
|
-
1. create_job → Returns jobId (job is now QUEUED, not complete!)
|
|
91
|
-
2. get_job_status → Poll every 30-60 seconds until status="completed" (takes 2-10 min)
|
|
92
|
-
3. get_job_result → Retrieve the actual test results
|
|
93
|
-
|
|
78
|
+
description: `⚠️ IMPORTANT: This ONLY creates and queues a job. It does NOT perform the test or return results. You MUST follow up with get_job_status and get_job_result.
|
|
79
|
+
|
|
80
|
+
Creates a QA job that will be performed by a REAL HUMAN tester (not AI). The human will manually test your application, describe findings in natural language, and GPT-4o will extract structured data from their response.
|
|
81
|
+
|
|
82
|
+
Use this when you need human verification of:
|
|
83
|
+
- UI/UX functionality that's hard to automate
|
|
84
|
+
- Visual issues, accessibility problems
|
|
85
|
+
- Complex user flows (login, checkout, forms)
|
|
86
|
+
- Cross-browser compatibility
|
|
87
|
+
- Real user experience feedback
|
|
88
|
+
|
|
89
|
+
⚠️ REQUIRED WORKFLOW (do NOT skip steps):
|
|
90
|
+
1. create_job → Returns jobId (job is now QUEUED, not complete!)
|
|
91
|
+
2. get_job_status → Poll every 30-60 seconds until status="completed" (takes 2-10 min)
|
|
92
|
+
3. get_job_result → Retrieve the actual test results
|
|
93
|
+
|
|
94
94
|
DO NOT treat job creation as completion. You MUST wait for and retrieve results.`,
|
|
95
95
|
inputSchema: {
|
|
96
96
|
type: 'object',
|
|
@@ -117,10 +117,10 @@ DO NOT treat job creation as completion. You MUST wait for and retrieve results.
|
|
|
117
117
|
},
|
|
118
118
|
{
|
|
119
119
|
name: 'get_job_status',
|
|
120
|
-
description: `Check the current status of a QA job. Jobs progress through states: pending → claimed → in_progress → completed (or failed/timeout).
|
|
121
|
-
|
|
122
|
-
Use this to poll for completion before fetching results. Typical job completion time is 2-10 minutes depending on test complexity.
|
|
123
|
-
|
|
120
|
+
description: `Check the current status of a QA job. Jobs progress through states: pending → claimed → in_progress → completed (or failed/timeout).
|
|
121
|
+
|
|
122
|
+
Use this to poll for completion before fetching results. Typical job completion time is 2-10 minutes depending on test complexity.
|
|
123
|
+
|
|
124
124
|
Returns: { status: "pending" | "claimed" | "in_progress" | "completed" | "failed" | "timeout", message: "..." }`,
|
|
125
125
|
inputSchema: {
|
|
126
126
|
type: 'object',
|
|
@@ -135,10 +135,10 @@ Returns: { status: "pending" | "claimed" | "in_progress" | "completed" | "failed
|
|
|
135
135
|
},
|
|
136
136
|
{
|
|
137
137
|
name: 'get_job_result',
|
|
138
|
-
description: `Get the structured results of a completed QA job. Only call this after get_job_status shows status="completed".
|
|
139
|
-
|
|
140
|
-
Returns the tester's response extracted into your specified schema, plus metadata about timing and the raw tester response.
|
|
141
|
-
|
|
138
|
+
description: `Get the structured results of a completed QA job. Only call this after get_job_status shows status="completed".
|
|
139
|
+
|
|
140
|
+
Returns the tester's response extracted into your specified schema, plus metadata about timing and the raw tester response.
|
|
141
|
+
|
|
142
142
|
If the job isn't complete yet, returns an error. If extraction failed, includes the raw response so you can see what the tester said.`,
|
|
143
143
|
inputSchema: {
|
|
144
144
|
type: 'object',
|
|
@@ -184,14 +184,14 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
184
184
|
return {
|
|
185
185
|
content: [{
|
|
186
186
|
type: 'text',
|
|
187
|
-
text: `❌ Failed to create job
|
|
188
|
-
|
|
189
|
-
Error: ${error.error || error.message || response.statusText}
|
|
190
|
-
Status: ${response.status}
|
|
191
|
-
|
|
192
|
-
Please check:
|
|
193
|
-
- Your RUNHUMAN_API_KEY is valid
|
|
194
|
-
- The API server is running at ${API_URL}
|
|
187
|
+
text: `❌ Failed to create job
|
|
188
|
+
|
|
189
|
+
Error: ${error.error || error.message || response.statusText}
|
|
190
|
+
Status: ${response.status}
|
|
191
|
+
|
|
192
|
+
Please check:
|
|
193
|
+
- Your RUNHUMAN_API_KEY is valid
|
|
194
|
+
- The API server is running at ${API_URL}
|
|
195
195
|
- Your API key has permission to create jobs`
|
|
196
196
|
}],
|
|
197
197
|
isError: true
|
|
@@ -203,21 +203,21 @@ Please check:
|
|
|
203
203
|
content: [
|
|
204
204
|
{
|
|
205
205
|
type: 'text',
|
|
206
|
-
text: `⏳ Job QUEUED (not complete yet!)
|
|
207
|
-
|
|
208
|
-
Job ID: ${data.jobId}
|
|
209
|
-
Status: ${data.status}
|
|
210
|
-
URL: ${args.url}
|
|
211
|
-
Time limit: ${duration} minutes
|
|
212
|
-
|
|
213
|
-
The test has been queued for a HUMAN tester. It is NOT complete yet.
|
|
214
|
-
|
|
215
|
-
⚠️ REQUIRED NEXT STEPS:
|
|
216
|
-
1. Wait 30-60 seconds
|
|
217
|
-
2. Call get_job_status(jobId: "${data.jobId}") to check if complete
|
|
218
|
-
3. Repeat step 2 until status="completed" (typical time: 2-10 minutes)
|
|
219
|
-
4. Call get_job_result(jobId: "${data.jobId}") to get the actual test results
|
|
220
|
-
|
|
206
|
+
text: `⏳ Job QUEUED (not complete yet!)
|
|
207
|
+
|
|
208
|
+
Job ID: ${data.jobId}
|
|
209
|
+
Status: ${data.status}
|
|
210
|
+
URL: ${args.url}
|
|
211
|
+
Time limit: ${duration} minutes
|
|
212
|
+
|
|
213
|
+
The test has been queued for a HUMAN tester. It is NOT complete yet.
|
|
214
|
+
|
|
215
|
+
⚠️ REQUIRED NEXT STEPS:
|
|
216
|
+
1. Wait 30-60 seconds
|
|
217
|
+
2. Call get_job_status(jobId: "${data.jobId}") to check if complete
|
|
218
|
+
3. Repeat step 2 until status="completed" (typical time: 2-10 minutes)
|
|
219
|
+
4. Call get_job_result(jobId: "${data.jobId}") to get the actual test results
|
|
220
|
+
|
|
221
221
|
DO NOT consider this task done until you have retrieved the results!`,
|
|
222
222
|
},
|
|
223
223
|
],
|
|
@@ -227,13 +227,13 @@ DO NOT consider this task done until you have retrieved the results!`,
|
|
|
227
227
|
return {
|
|
228
228
|
content: [{
|
|
229
229
|
type: 'text',
|
|
230
|
-
text: `❌ Error creating job
|
|
231
|
-
|
|
232
|
-
${error instanceof Error ? error.message : 'Unknown error'}
|
|
233
|
-
|
|
234
|
-
Please check:
|
|
235
|
-
- The API server is running at ${API_URL}
|
|
236
|
-
- Your network connection
|
|
230
|
+
text: `❌ Error creating job
|
|
231
|
+
|
|
232
|
+
${error instanceof Error ? error.message : 'Unknown error'}
|
|
233
|
+
|
|
234
|
+
Please check:
|
|
235
|
+
- The API server is running at ${API_URL}
|
|
236
|
+
- Your network connection
|
|
237
237
|
- Your .env file configuration`
|
|
238
238
|
}],
|
|
239
239
|
isError: true
|
|
@@ -251,10 +251,10 @@ Please check:
|
|
|
251
251
|
return {
|
|
252
252
|
content: [{
|
|
253
253
|
type: 'text',
|
|
254
|
-
text: `❌ Job not found
|
|
255
|
-
|
|
256
|
-
Job ID: ${args.jobId}
|
|
257
|
-
|
|
254
|
+
text: `❌ Job not found
|
|
255
|
+
|
|
256
|
+
Job ID: ${args.jobId}
|
|
257
|
+
|
|
258
258
|
The job does not exist or you don't have permission to access it.`
|
|
259
259
|
}],
|
|
260
260
|
isError: true
|
|
@@ -263,9 +263,9 @@ The job does not exist or you don't have permission to access it.`
|
|
|
263
263
|
return {
|
|
264
264
|
content: [{
|
|
265
265
|
type: 'text',
|
|
266
|
-
text: `❌ Failed to get job status
|
|
267
|
-
|
|
268
|
-
Status: ${response.status}
|
|
266
|
+
text: `❌ Failed to get job status
|
|
267
|
+
|
|
268
|
+
Status: ${response.status}
|
|
269
269
|
Error: ${response.statusText}`
|
|
270
270
|
}],
|
|
271
271
|
isError: true
|
|
@@ -281,8 +281,8 @@ Error: ${response.statusText}`
|
|
|
281
281
|
timeout: '⏰'
|
|
282
282
|
};
|
|
283
283
|
const emoji = statusEmoji[job.status] || '📊';
|
|
284
|
-
let message = `${emoji} Job Status: ${job.status}
|
|
285
|
-
|
|
284
|
+
let message = `${emoji} Job Status: ${job.status}
|
|
285
|
+
|
|
286
286
|
Job ID: ${job.id}`;
|
|
287
287
|
if (job.status === 'pending') {
|
|
288
288
|
message += '\n\nWaiting for a tester to claim this job...\nTypical completion time: 2-10 minutes';
|
|
@@ -310,8 +310,8 @@ Job ID: ${job.id}`;
|
|
|
310
310
|
return {
|
|
311
311
|
content: [{
|
|
312
312
|
type: 'text',
|
|
313
|
-
text: `❌ Error checking job status
|
|
314
|
-
|
|
313
|
+
text: `❌ Error checking job status
|
|
314
|
+
|
|
315
315
|
${error instanceof Error ? error.message : 'Unknown error'}`
|
|
316
316
|
}],
|
|
317
317
|
isError: true
|
|
@@ -329,10 +329,10 @@ ${error instanceof Error ? error.message : 'Unknown error'}`
|
|
|
329
329
|
return {
|
|
330
330
|
content: [{
|
|
331
331
|
type: 'text',
|
|
332
|
-
text: `❌ Job not found
|
|
333
|
-
|
|
334
|
-
Job ID: ${args.jobId}
|
|
335
|
-
|
|
332
|
+
text: `❌ Job not found
|
|
333
|
+
|
|
334
|
+
Job ID: ${args.jobId}
|
|
335
|
+
|
|
336
336
|
The job does not exist or you don't have permission to access it.`
|
|
337
337
|
}],
|
|
338
338
|
isError: true
|
|
@@ -341,9 +341,9 @@ The job does not exist or you don't have permission to access it.`
|
|
|
341
341
|
return {
|
|
342
342
|
content: [{
|
|
343
343
|
type: 'text',
|
|
344
|
-
text: `❌ Failed to get job result
|
|
345
|
-
|
|
346
|
-
Status: ${response.status}
|
|
344
|
+
text: `❌ Failed to get job result
|
|
345
|
+
|
|
346
|
+
Status: ${response.status}
|
|
347
347
|
Error: ${response.statusText}`
|
|
348
348
|
}],
|
|
349
349
|
isError: true
|
|
@@ -354,11 +354,11 @@ Error: ${response.statusText}`
|
|
|
354
354
|
return {
|
|
355
355
|
content: [{
|
|
356
356
|
type: 'text',
|
|
357
|
-
text: `⏳ Job not yet completed
|
|
358
|
-
|
|
359
|
-
Job ID: ${job.id}
|
|
360
|
-
Current status: ${job.status}
|
|
361
|
-
|
|
357
|
+
text: `⏳ Job not yet completed
|
|
358
|
+
|
|
359
|
+
Job ID: ${job.id}
|
|
360
|
+
Current status: ${job.status}
|
|
361
|
+
|
|
362
362
|
${job.status === 'pending' ? 'Waiting for a tester to claim this job...' :
|
|
363
363
|
job.status === 'claimed' || job.status === 'in_progress' ? 'The tester is working on your test...' :
|
|
364
364
|
job.status === 'failed' ? '❌ Job failed. Error: ' + (job.error || 'Unknown error') :
|
|
@@ -368,10 +368,10 @@ ${job.status === 'pending' ? 'Waiting for a tester to claim this job...' :
|
|
|
368
368
|
};
|
|
369
369
|
}
|
|
370
370
|
// Job is completed, format results
|
|
371
|
-
const message = `✅ Test completed!
|
|
372
|
-
|
|
373
|
-
Job ID: ${job.id}
|
|
374
|
-
|
|
371
|
+
const message = `✅ Test completed!
|
|
372
|
+
|
|
373
|
+
Job ID: ${job.id}
|
|
374
|
+
|
|
375
375
|
**Test Results:**`;
|
|
376
376
|
const contents = [
|
|
377
377
|
{
|
|
@@ -395,8 +395,8 @@ Job ID: ${job.id}
|
|
|
395
395
|
return {
|
|
396
396
|
content: [{
|
|
397
397
|
type: 'text',
|
|
398
|
-
text: `❌ Error getting job result
|
|
399
|
-
|
|
398
|
+
text: `❌ Error getting job result
|
|
399
|
+
|
|
400
400
|
${error instanceof Error ? error.message : 'Unknown error'}`
|
|
401
401
|
}],
|
|
402
402
|
isError: true
|
package/package.json
CHANGED
|
@@ -1,62 +1,62 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "@runhuman/mcp-server",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "Model Context Protocol (MCP) server for RunHuman - Human-powered QA testing for AI agents",
|
|
5
|
-
"main": "dist/index.js",
|
|
6
|
-
"type": "module",
|
|
7
|
-
"bin": {
|
|
8
|
-
"runhuman-mcp": "dist/index.js"
|
|
9
|
-
},
|
|
10
|
-
"files": [
|
|
11
|
-
"dist",
|
|
12
|
-
"README.md",
|
|
13
|
-
".env.example"
|
|
14
|
-
],
|
|
15
|
-
"scripts": {
|
|
16
|
-
"build": "tsc",
|
|
17
|
-
"dev": "tsx watch src/index.ts",
|
|
18
|
-
"start": "node dist/index.js",
|
|
19
|
-
"prepublishOnly": "npm run build",
|
|
20
|
-
"test": "node test-simple.cjs",
|
|
21
|
-
"test:all": "node test-all-tools.cjs",
|
|
22
|
-
"test:routes": "node test-route-fix.cjs",
|
|
23
|
-
"test:endpoints": "node test-api-endpoints.cjs",
|
|
24
|
-
"test:ci": "npm run test:endpoints && npm run test:routes",
|
|
25
|
-
"test:inspector": "npx @modelcontextprotocol/inspector node dist/index.js"
|
|
26
|
-
},
|
|
27
|
-
"keywords": [
|
|
28
|
-
"mcp",
|
|
29
|
-
"model-context-protocol",
|
|
30
|
-
"qa",
|
|
31
|
-
"testing",
|
|
32
|
-
"human-in-the-loop",
|
|
33
|
-
"ai-agent",
|
|
34
|
-
"claude",
|
|
35
|
-
"anthropic",
|
|
36
|
-
"qa-testing",
|
|
37
|
-
"manual-testing"
|
|
38
|
-
],
|
|
39
|
-
"author": "RunHuman <hey@runhuman.com>",
|
|
40
|
-
"repository": {
|
|
41
|
-
"type": "git",
|
|
42
|
-
"url": "git+https://github.com/yueranyuan/qa-experiment.git",
|
|
43
|
-
"directory": "packages/mcp-server"
|
|
44
|
-
},
|
|
45
|
-
"homepage": "https://runhuman.com",
|
|
46
|
-
"bugs": {
|
|
47
|
-
"url": "https://github.com/yueranyuan/qa-experiment/issues"
|
|
48
|
-
},
|
|
49
|
-
"license": "ISC",
|
|
50
|
-
"engines": {
|
|
51
|
-
"node": ">=18.0.0"
|
|
52
|
-
},
|
|
53
|
-
"dependencies": {
|
|
54
|
-
"@modelcontextprotocol/sdk": "latest",
|
|
55
|
-
"dotenv": "^17.2.3"
|
|
56
|
-
},
|
|
57
|
-
"devDependencies": {
|
|
58
|
-
"@types/node": "^20.11.17",
|
|
59
|
-
"tsx": "^4.7.1",
|
|
60
|
-
"typescript": "^5.3.3"
|
|
61
|
-
}
|
|
62
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "@runhuman/mcp-server",
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"description": "Model Context Protocol (MCP) server for RunHuman - Human-powered QA testing for AI agents",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"bin": {
|
|
8
|
+
"runhuman-mcp": "dist/index.js"
|
|
9
|
+
},
|
|
10
|
+
"files": [
|
|
11
|
+
"dist",
|
|
12
|
+
"README.md",
|
|
13
|
+
".env.example"
|
|
14
|
+
],
|
|
15
|
+
"scripts": {
|
|
16
|
+
"build": "tsc",
|
|
17
|
+
"dev": "tsx watch src/index.ts",
|
|
18
|
+
"start": "node dist/index.js",
|
|
19
|
+
"prepublishOnly": "npm run build",
|
|
20
|
+
"test": "node test-simple.cjs",
|
|
21
|
+
"test:all": "node test-all-tools.cjs",
|
|
22
|
+
"test:routes": "node test-route-fix.cjs",
|
|
23
|
+
"test:endpoints": "node test-api-endpoints.cjs",
|
|
24
|
+
"test:ci": "npm run test:endpoints && npm run test:routes",
|
|
25
|
+
"test:inspector": "npx @modelcontextprotocol/inspector node dist/index.js"
|
|
26
|
+
},
|
|
27
|
+
"keywords": [
|
|
28
|
+
"mcp",
|
|
29
|
+
"model-context-protocol",
|
|
30
|
+
"qa",
|
|
31
|
+
"testing",
|
|
32
|
+
"human-in-the-loop",
|
|
33
|
+
"ai-agent",
|
|
34
|
+
"claude",
|
|
35
|
+
"anthropic",
|
|
36
|
+
"qa-testing",
|
|
37
|
+
"manual-testing"
|
|
38
|
+
],
|
|
39
|
+
"author": "RunHuman <hey@runhuman.com>",
|
|
40
|
+
"repository": {
|
|
41
|
+
"type": "git",
|
|
42
|
+
"url": "git+https://github.com/yueranyuan/qa-experiment.git",
|
|
43
|
+
"directory": "packages/mcp-server"
|
|
44
|
+
},
|
|
45
|
+
"homepage": "https://runhuman.com",
|
|
46
|
+
"bugs": {
|
|
47
|
+
"url": "https://github.com/yueranyuan/qa-experiment/issues"
|
|
48
|
+
},
|
|
49
|
+
"license": "ISC",
|
|
50
|
+
"engines": {
|
|
51
|
+
"node": ">=18.0.0"
|
|
52
|
+
},
|
|
53
|
+
"dependencies": {
|
|
54
|
+
"@modelcontextprotocol/sdk": "latest",
|
|
55
|
+
"dotenv": "^17.2.3"
|
|
56
|
+
},
|
|
57
|
+
"devDependencies": {
|
|
58
|
+
"@types/node": "^20.11.17",
|
|
59
|
+
"tsx": "^4.7.1",
|
|
60
|
+
"typescript": "^5.3.3"
|
|
61
|
+
}
|
|
62
|
+
}
|