@toolrank/mcp-server 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +13 -0
- package/dist/index.js +397 -0
- package/package.json +25 -0
- package/smithery.yaml +17 -0
- package/src/index.ts +440 -0
- package/tsconfig.json +15 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ToolRank MCP Server v0.1
|
|
3
|
+
*
|
|
4
|
+
* Provides AI agents with tools to score and optimize MCP tool definitions.
|
|
5
|
+
* This server's own tool definitions are designed to achieve ToolRank Score 100/100,
|
|
6
|
+
* serving as a live demonstration of ATO (Agent Tool Optimization) principles.
|
|
7
|
+
*
|
|
8
|
+
* Tools:
|
|
9
|
+
* toolrank_score — Analyze tool definitions and return quality scores
|
|
10
|
+
* toolrank_compare — Compare scores against category averages
|
|
11
|
+
* toolrank_suggest — Generate specific improvement suggestions
|
|
12
|
+
*/
|
|
13
|
+
export {};
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ToolRank MCP Server v0.1
|
|
3
|
+
*
|
|
4
|
+
* Provides AI agents with tools to score and optimize MCP tool definitions.
|
|
5
|
+
* This server's own tool definitions are designed to achieve ToolRank Score 100/100,
|
|
6
|
+
* serving as a live demonstration of ATO (Agent Tool Optimization) principles.
|
|
7
|
+
*
|
|
8
|
+
* Tools:
|
|
9
|
+
* toolrank_score — Analyze tool definitions and return quality scores
|
|
10
|
+
* toolrank_compare — Compare scores against category averages
|
|
11
|
+
* toolrank_suggest — Generate specific improvement suggestions
|
|
12
|
+
*/
|
|
13
|
+
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
14
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
15
|
+
import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
|
|
16
|
+
function scoreTool(tool) {
|
|
17
|
+
const issues = [];
|
|
18
|
+
const name = tool.name || "unknown";
|
|
19
|
+
const desc = (tool.description || "").trim();
|
|
20
|
+
const schema = tool.inputSchema || tool.input_schema || {};
|
|
21
|
+
const props = schema.properties || {};
|
|
22
|
+
const required = schema.required || [];
|
|
23
|
+
const propKeys = Object.keys(props);
|
|
24
|
+
// --- Clarity (max 35) ---
|
|
25
|
+
let clarityScore = 0;
|
|
26
|
+
const clarityMax = 6;
|
|
27
|
+
let clarityPoints = 0;
|
|
28
|
+
if (!desc) {
|
|
29
|
+
issues.push({ dimension: "clarity", severity: "critical", message: "No description defined", fix: "Add a description explaining purpose, usage context, and return value", impact: 15 });
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
clarityPoints += 1;
|
|
33
|
+
if (desc.length < 20) {
|
|
34
|
+
issues.push({ dimension: "clarity", severity: "critical", message: `Description too short (${desc.length} chars)`, fix: "Expand to 80-200 chars with purpose and context", impact: 10 });
|
|
35
|
+
clarityPoints += 0.2;
|
|
36
|
+
}
|
|
37
|
+
else if (desc.length >= 80 && desc.length <= 250) {
|
|
38
|
+
clarityPoints += 1;
|
|
39
|
+
}
|
|
40
|
+
else if (desc.length >= 50) {
|
|
41
|
+
clarityPoints += 0.7;
|
|
42
|
+
if (desc.length < 80)
|
|
43
|
+
issues.push({ dimension: "clarity", severity: "warning", message: `Description short (${desc.length} chars)`, fix: "Expand to 80-200 chars for optimal agent understanding", impact: 5 });
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
clarityPoints += 0.3;
|
|
47
|
+
issues.push({ dimension: "clarity", severity: "warning", message: `Description short (${desc.length} chars)`, fix: "Expand to 80-200 chars", impact: 5 });
|
|
48
|
+
}
|
|
49
|
+
const descLower = desc.toLowerCase();
|
|
50
|
+
if (/^(get|set|create|update|delete|search|find|list|fetch|send|this tool|retriev|return|provid|allow|enabl|perform|analyz|generat|calculat|check|validat|convert|extract|scor|compar|suggest|optimiz)/.test(descLower)) {
|
|
51
|
+
clarityPoints += 1;
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
clarityPoints += 0.5;
|
|
55
|
+
issues.push({ dimension: "clarity", severity: "warning", message: "No clear purpose verb at start", fix: "Start with a verb: 'Analyzes...', 'Returns...', 'Generates...'", impact: 5 });
|
|
56
|
+
}
|
|
57
|
+
if (/use this|when|useful for|ideal for|designed for|helps|allows|enables|for example/.test(descLower)) {
|
|
58
|
+
clarityPoints += 1;
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
clarityPoints += 0.4;
|
|
62
|
+
issues.push({ dimension: "clarity", severity: "warning", message: "No usage context", fix: "Add 'Use this when...' to help agents decide when to select this tool", impact: 6 });
|
|
63
|
+
}
|
|
64
|
+
if (/returns?|outputs?|produces|yields|result|response|provides|generates/.test(descLower)) {
|
|
65
|
+
clarityPoints += 1;
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
clarityPoints += 0.5;
|
|
69
|
+
issues.push({ dimension: "clarity", severity: "info", message: "No return value described", fix: "Add 'Returns...' to describe the output", impact: 3 });
|
|
70
|
+
}
|
|
71
|
+
const nameParts = name.toLowerCase().split(/[_\-]/).filter((p) => p.length > 2);
|
|
72
|
+
const matchCount = nameParts.filter((p) => descLower.includes(p)).length;
|
|
73
|
+
clarityPoints += nameParts.length > 0 && matchCount / nameParts.length >= 0.5 ? 1 : 0.5;
|
|
74
|
+
}
|
|
75
|
+
clarityScore = Math.round((clarityPoints / clarityMax) * 35 * 10) / 10;
|
|
76
|
+
// --- Precision (max 25) ---
|
|
77
|
+
let precisionPoints = 0;
|
|
78
|
+
const precisionMax = 5;
|
|
79
|
+
if (!schema.type) {
|
|
80
|
+
issues.push({ dimension: "precision", severity: "critical", message: "No input schema defined", fix: "Add inputSchema with type definitions for all parameters", impact: 12 });
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
precisionPoints += 1;
|
|
84
|
+
if (propKeys.length > 0) {
|
|
85
|
+
const missingTypes = propKeys.filter(k => !props[k].type);
|
|
86
|
+
if (missingTypes.length === 0)
|
|
87
|
+
precisionPoints += 1;
|
|
88
|
+
else {
|
|
89
|
+
precisionPoints += Math.max(0, 1 - missingTypes.length / propKeys.length);
|
|
90
|
+
issues.push({ dimension: "precision", severity: "warning", message: `Missing types: ${missingTypes.join(", ")}`, fix: "Add 'type' to each parameter", impact: 4 });
|
|
91
|
+
}
|
|
92
|
+
const missingDesc = propKeys.filter(k => !props[k].description);
|
|
93
|
+
if (missingDesc.length === 0)
|
|
94
|
+
precisionPoints += 1;
|
|
95
|
+
else {
|
|
96
|
+
precisionPoints += Math.max(0, 1 - missingDesc.length / propKeys.length);
|
|
97
|
+
issues.push({ dimension: "precision", severity: "warning", message: `Missing param descriptions: ${missingDesc.join(", ")}`, fix: "Add 'description' to each parameter", impact: 5 });
|
|
98
|
+
}
|
|
99
|
+
precisionPoints += required.length > 0 ? 1 : 0.5;
|
|
100
|
+
if (required.length === 0 && propKeys.length > 0) {
|
|
101
|
+
issues.push({ dimension: "precision", severity: "info", message: "No required fields specified", fix: "Add 'required' array for mandatory parameters", impact: 3 });
|
|
102
|
+
}
|
|
103
|
+
precisionPoints += 1;
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
precisionPoints += 2;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
const precisionScore = Math.round((precisionPoints / precisionMax) * 25 * 10) / 10;
|
|
110
|
+
// --- Efficiency (max 15) ---
|
|
111
|
+
const tokenEst = JSON.stringify(tool).length / 4;
|
|
112
|
+
let effRatio = tokenEst > 2000 ? 0.3 : tokenEst > 1000 ? 0.6 : tokenEst > 500 ? 0.8 : 1.0;
|
|
113
|
+
if (tokenEst > 1000)
|
|
114
|
+
issues.push({ dimension: "efficiency", severity: "warning", message: `~${Math.round(tokenEst)} tokens estimated`, fix: "Create a compact variant", impact: 3 });
|
|
115
|
+
if (!/^[a-z][a-zA-Z0-9_]*$/.test(name)) {
|
|
116
|
+
effRatio -= 0.15;
|
|
117
|
+
issues.push({ dimension: "efficiency", severity: "warning", message: `Name '${name}' not snake_case`, fix: "Use snake_case like 'search_users'", impact: 3 });
|
|
118
|
+
}
|
|
119
|
+
const genericNames = new Set(["run", "execute", "do", "action", "tool", "function", "process", "handle"]);
|
|
120
|
+
if (genericNames.has(name.toLowerCase())) {
|
|
121
|
+
effRatio -= 0.15;
|
|
122
|
+
issues.push({ dimension: "efficiency", severity: "warning", message: `Name '${name}' too generic`, fix: "Use specific name like 'create_pull_request'", impact: 5 });
|
|
123
|
+
}
|
|
124
|
+
const efficiencyScore = Math.round(Math.max(0, effRatio) * 15 * 10) / 10;
|
|
125
|
+
// --- Findability (max 25, limited without registry) ---
|
|
126
|
+
let findRatio = name.length < 4 ? 0.3 : 0.8;
|
|
127
|
+
if (name.length < 4)
|
|
128
|
+
issues.push({ dimension: "findability", severity: "warning", message: `Name '${name}' too short for discovery`, fix: "Use a longer, descriptive name", impact: 4 });
|
|
129
|
+
const findabilityScore = Math.round(findRatio * 25 * 10) / 10;
|
|
130
|
+
const total = Math.round((findabilityScore + clarityScore + precisionScore + efficiencyScore) * 10) / 10;
|
|
131
|
+
let level, levelName;
|
|
132
|
+
if (total >= 85) {
|
|
133
|
+
level = 4;
|
|
134
|
+
levelName = "Dominant";
|
|
135
|
+
}
|
|
136
|
+
else if (total >= 70) {
|
|
137
|
+
level = 3;
|
|
138
|
+
levelName = "Preferred";
|
|
139
|
+
}
|
|
140
|
+
else if (total >= 50) {
|
|
141
|
+
level = 2;
|
|
142
|
+
levelName = "Selectable";
|
|
143
|
+
}
|
|
144
|
+
else if (total >= 25) {
|
|
145
|
+
level = 1;
|
|
146
|
+
levelName = "Visible";
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
level = 0;
|
|
150
|
+
levelName = "Absent";
|
|
151
|
+
}
|
|
152
|
+
return {
|
|
153
|
+
name,
|
|
154
|
+
total,
|
|
155
|
+
level,
|
|
156
|
+
levelName,
|
|
157
|
+
dimensions: { findability: findabilityScore, clarity: clarityScore, precision: precisionScore, efficiency: efficiencyScore },
|
|
158
|
+
issues: issues.sort((a, b) => b.impact - a.impact),
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
// --- MCP Server Setup ---
|
|
162
|
+
const server = new Server({ name: "toolrank", version: "0.1.0" }, { capabilities: { tools: {} } });
|
|
163
|
+
// Tool definitions — these ARE the product. ATO Score 100/100 target.
|
|
164
|
+
server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
165
|
+
tools: [
|
|
166
|
+
{
|
|
167
|
+
name: "toolrank_score",
|
|
168
|
+
description: "Analyzes MCP tool definitions and returns a ToolRank Score (0-100) measuring how likely AI agents are to discover and select each tool. Scores four dimensions: Findability (can agents find it?), Clarity (can agents understand it?), Precision (is the schema well-defined?), and Efficiency (is it token-efficient?). Use this when you want to evaluate the quality of your MCP server's tool definitions before publishing. Returns per-tool scores, maturity level (Absent/Visible/Selectable/Preferred/Dominant), specific issues found, and prioritized improvement suggestions with predicted score impact.",
|
|
169
|
+
inputSchema: {
|
|
170
|
+
type: "object",
|
|
171
|
+
properties: {
|
|
172
|
+
tools: {
|
|
173
|
+
type: "array",
|
|
174
|
+
description: "Array of MCP tool definition objects. Each object should have 'name' (string), 'description' (string), and optionally 'inputSchema' (JSON Schema object with properties, required, etc.)",
|
|
175
|
+
items: {
|
|
176
|
+
type: "object",
|
|
177
|
+
properties: {
|
|
178
|
+
name: { type: "string", description: "Tool name (e.g., 'create_issue')" },
|
|
179
|
+
description: { type: "string", description: "Tool description text" },
|
|
180
|
+
inputSchema: { type: "object", description: "JSON Schema for tool input parameters" },
|
|
181
|
+
},
|
|
182
|
+
required: ["name"],
|
|
183
|
+
},
|
|
184
|
+
},
|
|
185
|
+
server_name: {
|
|
186
|
+
type: "string",
|
|
187
|
+
description: "Optional name of the MCP server being scored. Used in the report header.",
|
|
188
|
+
default: "unnamed",
|
|
189
|
+
},
|
|
190
|
+
},
|
|
191
|
+
required: ["tools"],
|
|
192
|
+
},
|
|
193
|
+
},
|
|
194
|
+
{
|
|
195
|
+
name: "toolrank_compare",
|
|
196
|
+
description: "Compares a tool's ToolRank Score against ecosystem benchmarks. Use this after running toolrank_score to understand how your tools rank relative to the broader MCP ecosystem. Returns percentile ranking, dimension-by-dimension comparison against category averages, and specific areas where your tools underperform. Requires a prior toolrank_score result or raw tool definitions.",
|
|
197
|
+
inputSchema: {
|
|
198
|
+
type: "object",
|
|
199
|
+
properties: {
|
|
200
|
+
tools: {
|
|
201
|
+
type: "array",
|
|
202
|
+
description: "Array of MCP tool definition objects to compare",
|
|
203
|
+
items: { type: "object" },
|
|
204
|
+
},
|
|
205
|
+
category: {
|
|
206
|
+
type: "string",
|
|
207
|
+
description: "Tool category for comparison (e.g., 'crm', 'database', 'devtools'). If omitted, compares against the full ecosystem average.",
|
|
208
|
+
default: "all",
|
|
209
|
+
},
|
|
210
|
+
},
|
|
211
|
+
required: ["tools"],
|
|
212
|
+
},
|
|
213
|
+
},
|
|
214
|
+
{
|
|
215
|
+
name: "toolrank_suggest",
|
|
216
|
+
description: "Generates specific, actionable improvement suggestions for MCP tool definitions. Use this when you have a low ToolRank Score and want concrete text rewrites. Returns optimized versions of tool names, descriptions, and schema improvements ranked by expected score impact. Does not execute changes — returns suggestions for review.",
|
|
217
|
+
inputSchema: {
|
|
218
|
+
type: "object",
|
|
219
|
+
properties: {
|
|
220
|
+
tool: {
|
|
221
|
+
type: "object",
|
|
222
|
+
description: "Single MCP tool definition to improve. Must include 'name' and 'description'.",
|
|
223
|
+
properties: {
|
|
224
|
+
name: { type: "string", description: "Current tool name" },
|
|
225
|
+
description: { type: "string", description: "Current tool description" },
|
|
226
|
+
inputSchema: { type: "object", description: "Current input schema" },
|
|
227
|
+
},
|
|
228
|
+
required: ["name"],
|
|
229
|
+
},
|
|
230
|
+
focus: {
|
|
231
|
+
type: "string",
|
|
232
|
+
description: "Which dimension to prioritize improvements for",
|
|
233
|
+
enum: ["findability", "clarity", "precision", "efficiency", "all"],
|
|
234
|
+
default: "all",
|
|
235
|
+
},
|
|
236
|
+
},
|
|
237
|
+
required: ["tool"],
|
|
238
|
+
},
|
|
239
|
+
},
|
|
240
|
+
],
|
|
241
|
+
}));
|
|
242
|
+
// Tool execution handlers
|
|
243
|
+
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
244
|
+
const { name, arguments: args } = request.params;
|
|
245
|
+
switch (name) {
|
|
246
|
+
case "toolrank_score": {
|
|
247
|
+
const tools = args.tools || [];
|
|
248
|
+
const serverName = args.server_name || "unnamed";
|
|
249
|
+
const results = tools.map((t) => scoreTool(t));
|
|
250
|
+
const avgScore = results.length > 0
|
|
251
|
+
? Math.round((results.reduce((s, r) => s + r.total, 0) / results.length) * 10) / 10
|
|
252
|
+
: 0;
|
|
253
|
+
const allIssues = results.flatMap((r) => r.issues);
|
|
254
|
+
const topFixes = allIssues
|
|
255
|
+
.sort((a, b) => b.impact - a.impact)
|
|
256
|
+
.slice(0, 5);
|
|
257
|
+
// Server-level check: too many tools
|
|
258
|
+
if (tools.length > 20) {
|
|
259
|
+
topFixes.unshift({
|
|
260
|
+
dimension: "efficiency",
|
|
261
|
+
severity: "warning",
|
|
262
|
+
message: `Server has ${tools.length} tools. Agent accuracy degrades past 15-20 tools`,
|
|
263
|
+
fix: "Consolidate into 5-15 workflow-oriented tools",
|
|
264
|
+
impact: 8,
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
return {
|
|
268
|
+
content: [{
|
|
269
|
+
type: "text",
|
|
270
|
+
text: JSON.stringify({
|
|
271
|
+
server_name: serverName,
|
|
272
|
+
average_score: avgScore,
|
|
273
|
+
tool_count: tools.length,
|
|
274
|
+
total_issues: allIssues.length,
|
|
275
|
+
critical_issues: allIssues.filter((i) => i.severity === "critical").length,
|
|
276
|
+
top_improvements: topFixes.slice(0, 3).map((i) => ({
|
|
277
|
+
message: i.message,
|
|
278
|
+
fix: i.fix,
|
|
279
|
+
impact: `+${i.impact}pt`,
|
|
280
|
+
})),
|
|
281
|
+
tools: results.map((r) => ({
|
|
282
|
+
name: r.name,
|
|
283
|
+
score: r.total,
|
|
284
|
+
level: `${r.level}: ${r.levelName}`,
|
|
285
|
+
dimensions: r.dimensions,
|
|
286
|
+
issues: r.issues.map((i) => ({
|
|
287
|
+
severity: i.severity,
|
|
288
|
+
message: i.message,
|
|
289
|
+
fix: i.fix,
|
|
290
|
+
impact: `+${i.impact}pt`,
|
|
291
|
+
})),
|
|
292
|
+
})),
|
|
293
|
+
}, null, 2),
|
|
294
|
+
}],
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
case "toolrank_compare": {
|
|
298
|
+
const tools = args.tools || [];
|
|
299
|
+
const category = args.category || "all";
|
|
300
|
+
const results = tools.map((t) => scoreTool(t));
|
|
301
|
+
const avgScore = results.length > 0
|
|
302
|
+
? Math.round((results.reduce((s, r) => s + r.total, 0) / results.length) * 10) / 10
|
|
303
|
+
: 0;
|
|
304
|
+
// Ecosystem benchmarks (from research data, will be replaced with live DB data)
|
|
305
|
+
const benchmarks = {
|
|
306
|
+
all: { avg: 42, median: 38, p75: 62, p90: 78 },
|
|
307
|
+
// Categories will be populated from scan DB
|
|
308
|
+
};
|
|
309
|
+
const bench = benchmarks[category] || benchmarks.all;
|
|
310
|
+
let percentile;
|
|
311
|
+
if (avgScore >= bench.p90)
|
|
312
|
+
percentile = 95;
|
|
313
|
+
else if (avgScore >= bench.p75)
|
|
314
|
+
percentile = 80;
|
|
315
|
+
else if (avgScore >= bench.median)
|
|
316
|
+
percentile = 55;
|
|
317
|
+
else if (avgScore >= bench.avg)
|
|
318
|
+
percentile = 40;
|
|
319
|
+
else
|
|
320
|
+
percentile = 20;
|
|
321
|
+
return {
|
|
322
|
+
content: [{
|
|
323
|
+
type: "text",
|
|
324
|
+
text: JSON.stringify({
|
|
325
|
+
your_score: avgScore,
|
|
326
|
+
category,
|
|
327
|
+
percentile: `Top ${100 - percentile}%`,
|
|
328
|
+
benchmark: bench,
|
|
329
|
+
comparison: {
|
|
330
|
+
vs_average: `${avgScore > bench.avg ? "+" : ""}${Math.round(avgScore - bench.avg)} points`,
|
|
331
|
+
vs_median: `${avgScore > bench.median ? "+" : ""}${Math.round(avgScore - bench.median)} points`,
|
|
332
|
+
},
|
|
333
|
+
note: "Benchmarks based on ecosystem scan data. Live rankings available at toolrank.dev/ranking",
|
|
334
|
+
}, null, 2),
|
|
335
|
+
}],
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
case "toolrank_suggest": {
|
|
339
|
+
const tool = args.tool || {};
|
|
340
|
+
const focus = args.focus || "all";
|
|
341
|
+
const result = scoreTool(tool);
|
|
342
|
+
const suggestions = [];
|
|
343
|
+
// Generate concrete suggestions based on issues
|
|
344
|
+
for (const issue of result.issues) {
|
|
345
|
+
if (focus !== "all" && issue.dimension !== focus)
|
|
346
|
+
continue;
|
|
347
|
+
const suggestion = {
|
|
348
|
+
dimension: issue.dimension,
|
|
349
|
+
current_problem: issue.message,
|
|
350
|
+
suggested_fix: issue.fix,
|
|
351
|
+
expected_impact: `+${issue.impact}pt`,
|
|
352
|
+
};
|
|
353
|
+
// Generate concrete rewrite for description issues
|
|
354
|
+
if (issue.dimension === "clarity" && tool.name) {
|
|
355
|
+
if (issue.message.includes("No description") || issue.message.includes("too short")) {
|
|
356
|
+
suggestion.example_rewrite = `${tool.name.replace(/_/g, " ").replace(/\b\w/g, (l) => l.toUpperCase())} — [describe what it does]. Use this when [describe use case]. Returns [describe output format].`;
|
|
357
|
+
}
|
|
358
|
+
if (issue.message.includes("No clear purpose verb")) {
|
|
359
|
+
suggestion.example_rewrite = `Retrieves/Creates/Searches for [object]. ${tool.description || ""}`;
|
|
360
|
+
}
|
|
361
|
+
if (issue.message.includes("No usage context")) {
|
|
362
|
+
suggestion.example_rewrite = `${tool.description || ""} Use this when you need to [specific scenario].`;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
suggestions.push(suggestion);
|
|
366
|
+
}
|
|
367
|
+
return {
|
|
368
|
+
content: [{
|
|
369
|
+
type: "text",
|
|
370
|
+
text: JSON.stringify({
|
|
371
|
+
tool_name: tool.name,
|
|
372
|
+
current_score: result.total,
|
|
373
|
+
current_level: `${result.level}: ${result.levelName}`,
|
|
374
|
+
suggestions: suggestions.sort((a, b) => {
|
|
375
|
+
const impA = parseInt(a.expected_impact) || 0;
|
|
376
|
+
const impB = parseInt(b.expected_impact) || 0;
|
|
377
|
+
return impB - impA;
|
|
378
|
+
}),
|
|
379
|
+
estimated_score_after_fixes: Math.min(100, result.total + suggestions.reduce((s, sg) => s + (parseInt(sg.expected_impact) || 0), 0)),
|
|
380
|
+
}, null, 2),
|
|
381
|
+
}],
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
default:
|
|
385
|
+
return {
|
|
386
|
+
content: [{ type: "text", text: `Unknown tool: ${name}` }],
|
|
387
|
+
isError: true,
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
});
|
|
391
|
+
// --- Start Server ---
|
|
392
|
+
async function main() {
|
|
393
|
+
const transport = new StdioServerTransport();
|
|
394
|
+
await server.connect(transport);
|
|
395
|
+
console.error("ToolRank MCP Server running on stdio");
|
|
396
|
+
}
|
|
397
|
+
main().catch(console.error);
|
package/package.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@toolrank/mcp-server",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "ToolRank MCP Server — Score and optimize MCP tool definitions for AI agent discovery. The first ATO (Agent Tool Optimization) tool.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "src/index.ts",
|
|
7
|
+
"bin": {
|
|
8
|
+
"toolrank-mcp": "./dist/index.js"
|
|
9
|
+
},
|
|
10
|
+
"scripts": {
|
|
11
|
+
"build": "tsc",
|
|
12
|
+
"start": "node dist/index.js",
|
|
13
|
+
"dev": "tsx src/index.ts"
|
|
14
|
+
},
|
|
15
|
+
"keywords": ["mcp", "ato", "agent-tool-optimization", "toolrank", "ai-agent", "tool-scoring"],
|
|
16
|
+
"author": "Hiroki Honda",
|
|
17
|
+
"license": "MIT",
|
|
18
|
+
"dependencies": {
|
|
19
|
+
"@modelcontextprotocol/sdk": "^1.0.0"
|
|
20
|
+
},
|
|
21
|
+
"devDependencies": {
|
|
22
|
+
"tsx": "^4.0.0",
|
|
23
|
+
"typescript": "^5.5.0"
|
|
24
|
+
}
|
|
25
|
+
}
|
package/smithery.yaml
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
name: toolrank
|
|
2
|
+
displayName: ToolRank
|
|
3
|
+
description: Score and optimize MCP tool definitions for AI agent discovery. Analyze your tools across Findability, Clarity, Precision, and Efficiency to maximize agent selection probability.
|
|
4
|
+
homepage: https://toolrank.dev
|
|
5
|
+
repository: https://github.com/imhiroki/toolrank
|
|
6
|
+
tags:
|
|
7
|
+
- developer-tools
|
|
8
|
+
- optimization
|
|
9
|
+
- scoring
|
|
10
|
+
- mcp
|
|
11
|
+
- ato
|
|
12
|
+
startCommand:
|
|
13
|
+
type: stdio
|
|
14
|
+
configSchema:
|
|
15
|
+
type: object
|
|
16
|
+
properties: {}
|
|
17
|
+
commandFunction: npx -y @toolrank/mcp-server
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,440 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ToolRank MCP Server v0.1
|
|
3
|
+
*
|
|
4
|
+
* Provides AI agents with tools to score and optimize MCP tool definitions.
|
|
5
|
+
* This server's own tool definitions are designed to achieve ToolRank Score 100/100,
|
|
6
|
+
* serving as a live demonstration of ATO (Agent Tool Optimization) principles.
|
|
7
|
+
*
|
|
8
|
+
* Tools:
|
|
9
|
+
* toolrank_score — Analyze tool definitions and return quality scores
|
|
10
|
+
* toolrank_compare — Compare scores against category averages
|
|
11
|
+
* toolrank_suggest — Generate specific improvement suggestions
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
15
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
16
|
+
import {
|
|
17
|
+
CallToolRequestSchema,
|
|
18
|
+
ListToolsRequestSchema,
|
|
19
|
+
} from "@modelcontextprotocol/sdk/types.js";
|
|
20
|
+
|
|
21
|
+
// --- Scoring Engine (TypeScript port of Level A) ---
|
|
22
|
+
|
|
23
|
+
interface Issue {
|
|
24
|
+
dimension: string;
|
|
25
|
+
severity: "critical" | "warning" | "info";
|
|
26
|
+
message: string;
|
|
27
|
+
fix: string;
|
|
28
|
+
impact: number;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
interface ToolScoreResult {
|
|
32
|
+
name: string;
|
|
33
|
+
total: number;
|
|
34
|
+
level: number;
|
|
35
|
+
levelName: string;
|
|
36
|
+
dimensions: {
|
|
37
|
+
findability: number;
|
|
38
|
+
clarity: number;
|
|
39
|
+
precision: number;
|
|
40
|
+
efficiency: number;
|
|
41
|
+
};
|
|
42
|
+
issues: Issue[];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function scoreTool(tool: Record<string, any>): ToolScoreResult {
|
|
46
|
+
const issues: Issue[] = [];
|
|
47
|
+
const name = tool.name || "unknown";
|
|
48
|
+
const desc = (tool.description || "").trim();
|
|
49
|
+
const schema = tool.inputSchema || tool.input_schema || {};
|
|
50
|
+
const props = schema.properties || {};
|
|
51
|
+
const required = schema.required || [];
|
|
52
|
+
const propKeys = Object.keys(props);
|
|
53
|
+
|
|
54
|
+
// --- Clarity (max 35) ---
|
|
55
|
+
let clarityScore = 0;
|
|
56
|
+
const clarityMax = 6;
|
|
57
|
+
let clarityPoints = 0;
|
|
58
|
+
|
|
59
|
+
if (!desc) {
|
|
60
|
+
issues.push({ dimension: "clarity", severity: "critical", message: "No description defined", fix: "Add a description explaining purpose, usage context, and return value", impact: 15 });
|
|
61
|
+
} else {
|
|
62
|
+
clarityPoints += 1;
|
|
63
|
+
|
|
64
|
+
if (desc.length < 20) {
|
|
65
|
+
issues.push({ dimension: "clarity", severity: "critical", message: `Description too short (${desc.length} chars)`, fix: "Expand to 80-200 chars with purpose and context", impact: 10 });
|
|
66
|
+
clarityPoints += 0.2;
|
|
67
|
+
} else if (desc.length >= 80 && desc.length <= 250) {
|
|
68
|
+
clarityPoints += 1;
|
|
69
|
+
} else if (desc.length >= 50) {
|
|
70
|
+
clarityPoints += 0.7;
|
|
71
|
+
if (desc.length < 80) issues.push({ dimension: "clarity", severity: "warning", message: `Description short (${desc.length} chars)`, fix: "Expand to 80-200 chars for optimal agent understanding", impact: 5 });
|
|
72
|
+
} else {
|
|
73
|
+
clarityPoints += 0.3;
|
|
74
|
+
issues.push({ dimension: "clarity", severity: "warning", message: `Description short (${desc.length} chars)`, fix: "Expand to 80-200 chars", impact: 5 });
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const descLower = desc.toLowerCase();
|
|
78
|
+
if (/^(get|set|create|update|delete|search|find|list|fetch|send|this tool|retriev|return|provid|allow|enabl|perform|analyz|generat|calculat|check|validat|convert|extract|scor|compar|suggest|optimiz)/.test(descLower)) {
|
|
79
|
+
clarityPoints += 1;
|
|
80
|
+
} else {
|
|
81
|
+
clarityPoints += 0.5;
|
|
82
|
+
issues.push({ dimension: "clarity", severity: "warning", message: "No clear purpose verb at start", fix: "Start with a verb: 'Analyzes...', 'Returns...', 'Generates...'", impact: 5 });
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (/use this|when|useful for|ideal for|designed for|helps|allows|enables|for example/.test(descLower)) {
|
|
86
|
+
clarityPoints += 1;
|
|
87
|
+
} else {
|
|
88
|
+
clarityPoints += 0.4;
|
|
89
|
+
issues.push({ dimension: "clarity", severity: "warning", message: "No usage context", fix: "Add 'Use this when...' to help agents decide when to select this tool", impact: 6 });
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (/returns?|outputs?|produces|yields|result|response|provides|generates/.test(descLower)) {
|
|
93
|
+
clarityPoints += 1;
|
|
94
|
+
} else {
|
|
95
|
+
clarityPoints += 0.5;
|
|
96
|
+
issues.push({ dimension: "clarity", severity: "info", message: "No return value described", fix: "Add 'Returns...' to describe the output", impact: 3 });
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const nameParts = name.toLowerCase().split(/[_\-]/).filter((p: string) => p.length > 2);
|
|
100
|
+
const matchCount = nameParts.filter((p: string) => descLower.includes(p)).length;
|
|
101
|
+
clarityPoints += nameParts.length > 0 && matchCount / nameParts.length >= 0.5 ? 1 : 0.5;
|
|
102
|
+
}
|
|
103
|
+
clarityScore = Math.round((clarityPoints / clarityMax) * 35 * 10) / 10;
|
|
104
|
+
|
|
105
|
+
// --- Precision (max 25) ---
|
|
106
|
+
let precisionPoints = 0;
|
|
107
|
+
const precisionMax = 5;
|
|
108
|
+
|
|
109
|
+
if (!schema.type) {
|
|
110
|
+
issues.push({ dimension: "precision", severity: "critical", message: "No input schema defined", fix: "Add inputSchema with type definitions for all parameters", impact: 12 });
|
|
111
|
+
} else {
|
|
112
|
+
precisionPoints += 1;
|
|
113
|
+
if (propKeys.length > 0) {
|
|
114
|
+
const missingTypes = propKeys.filter(k => !props[k].type);
|
|
115
|
+
if (missingTypes.length === 0) precisionPoints += 1;
|
|
116
|
+
else {
|
|
117
|
+
precisionPoints += Math.max(0, 1 - missingTypes.length / propKeys.length);
|
|
118
|
+
issues.push({ dimension: "precision", severity: "warning", message: `Missing types: ${missingTypes.join(", ")}`, fix: "Add 'type' to each parameter", impact: 4 });
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const missingDesc = propKeys.filter(k => !props[k].description);
|
|
122
|
+
if (missingDesc.length === 0) precisionPoints += 1;
|
|
123
|
+
else {
|
|
124
|
+
precisionPoints += Math.max(0, 1 - missingDesc.length / propKeys.length);
|
|
125
|
+
issues.push({ dimension: "precision", severity: "warning", message: `Missing param descriptions: ${missingDesc.join(", ")}`, fix: "Add 'description' to each parameter", impact: 5 });
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
precisionPoints += required.length > 0 ? 1 : 0.5;
|
|
129
|
+
if (required.length === 0 && propKeys.length > 0) {
|
|
130
|
+
issues.push({ dimension: "precision", severity: "info", message: "No required fields specified", fix: "Add 'required' array for mandatory parameters", impact: 3 });
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
precisionPoints += 1;
|
|
134
|
+
} else {
|
|
135
|
+
precisionPoints += 2;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
const precisionScore = Math.round((precisionPoints / precisionMax) * 25 * 10) / 10;
|
|
139
|
+
|
|
140
|
+
// --- Efficiency (max 15) ---
|
|
141
|
+
const tokenEst = JSON.stringify(tool).length / 4;
|
|
142
|
+
let effRatio = tokenEst > 2000 ? 0.3 : tokenEst > 1000 ? 0.6 : tokenEst > 500 ? 0.8 : 1.0;
|
|
143
|
+
if (tokenEst > 1000) issues.push({ dimension: "efficiency", severity: "warning", message: `~${Math.round(tokenEst)} tokens estimated`, fix: "Create a compact variant", impact: 3 });
|
|
144
|
+
|
|
145
|
+
if (!/^[a-z][a-zA-Z0-9_]*$/.test(name)) {
|
|
146
|
+
effRatio -= 0.15;
|
|
147
|
+
issues.push({ dimension: "efficiency", severity: "warning", message: `Name '${name}' not snake_case`, fix: "Use snake_case like 'search_users'", impact: 3 });
|
|
148
|
+
}
|
|
149
|
+
const genericNames = new Set(["run", "execute", "do", "action", "tool", "function", "process", "handle"]);
|
|
150
|
+
if (genericNames.has(name.toLowerCase())) {
|
|
151
|
+
effRatio -= 0.15;
|
|
152
|
+
issues.push({ dimension: "efficiency", severity: "warning", message: `Name '${name}' too generic`, fix: "Use specific name like 'create_pull_request'", impact: 5 });
|
|
153
|
+
}
|
|
154
|
+
const efficiencyScore = Math.round(Math.max(0, effRatio) * 15 * 10) / 10;
|
|
155
|
+
|
|
156
|
+
// --- Findability (max 25, limited without registry) ---
|
|
157
|
+
let findRatio = name.length < 4 ? 0.3 : 0.8;
|
|
158
|
+
if (name.length < 4) issues.push({ dimension: "findability", severity: "warning", message: `Name '${name}' too short for discovery`, fix: "Use a longer, descriptive name", impact: 4 });
|
|
159
|
+
const findabilityScore = Math.round(findRatio * 25 * 10) / 10;
|
|
160
|
+
|
|
161
|
+
const total = Math.round((findabilityScore + clarityScore + precisionScore + efficiencyScore) * 10) / 10;
|
|
162
|
+
|
|
163
|
+
let level: number, levelName: string;
|
|
164
|
+
if (total >= 85) { level = 4; levelName = "Dominant"; }
|
|
165
|
+
else if (total >= 70) { level = 3; levelName = "Preferred"; }
|
|
166
|
+
else if (total >= 50) { level = 2; levelName = "Selectable"; }
|
|
167
|
+
else if (total >= 25) { level = 1; levelName = "Visible"; }
|
|
168
|
+
else { level = 0; levelName = "Absent"; }
|
|
169
|
+
|
|
170
|
+
return {
|
|
171
|
+
name,
|
|
172
|
+
total,
|
|
173
|
+
level,
|
|
174
|
+
levelName,
|
|
175
|
+
dimensions: { findability: findabilityScore, clarity: clarityScore, precision: precisionScore, efficiency: efficiencyScore },
|
|
176
|
+
issues: issues.sort((a, b) => b.impact - a.impact),
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// --- MCP Server Setup ---
|
|
181
|
+
|
|
182
|
+
const server = new Server(
|
|
183
|
+
{ name: "toolrank", version: "0.1.0" },
|
|
184
|
+
{ capabilities: { tools: {} } }
|
|
185
|
+
);
|
|
186
|
+
|
|
187
|
+
// Tool definitions — these ARE the product. ATO Score 100/100 target.
|
|
188
|
+
server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
189
|
+
tools: [
|
|
190
|
+
{
|
|
191
|
+
name: "toolrank_score",
|
|
192
|
+
description:
|
|
193
|
+
"Analyzes MCP tool definitions and returns a ToolRank Score (0-100) measuring how likely AI agents are to discover and select each tool. Scores four dimensions: Findability (can agents find it?), Clarity (can agents understand it?), Precision (is the schema well-defined?), and Efficiency (is it token-efficient?). Use this when you want to evaluate the quality of your MCP server's tool definitions before publishing. Returns per-tool scores, maturity level (Absent/Visible/Selectable/Preferred/Dominant), specific issues found, and prioritized improvement suggestions with predicted score impact.",
|
|
194
|
+
inputSchema: {
|
|
195
|
+
type: "object",
|
|
196
|
+
properties: {
|
|
197
|
+
tools: {
|
|
198
|
+
type: "array",
|
|
199
|
+
description:
|
|
200
|
+
"Array of MCP tool definition objects. Each object should have 'name' (string), 'description' (string), and optionally 'inputSchema' (JSON Schema object with properties, required, etc.)",
|
|
201
|
+
items: {
|
|
202
|
+
type: "object",
|
|
203
|
+
properties: {
|
|
204
|
+
name: { type: "string", description: "Tool name (e.g., 'create_issue')" },
|
|
205
|
+
description: { type: "string", description: "Tool description text" },
|
|
206
|
+
inputSchema: { type: "object", description: "JSON Schema for tool input parameters" },
|
|
207
|
+
},
|
|
208
|
+
required: ["name"],
|
|
209
|
+
},
|
|
210
|
+
},
|
|
211
|
+
server_name: {
|
|
212
|
+
type: "string",
|
|
213
|
+
description: "Optional name of the MCP server being scored. Used in the report header.",
|
|
214
|
+
default: "unnamed",
|
|
215
|
+
},
|
|
216
|
+
},
|
|
217
|
+
required: ["tools"],
|
|
218
|
+
},
|
|
219
|
+
},
|
|
220
|
+
{
|
|
221
|
+
name: "toolrank_compare",
|
|
222
|
+
description:
|
|
223
|
+
"Compares a tool's ToolRank Score against ecosystem benchmarks. Use this after running toolrank_score to understand how your tools rank relative to the broader MCP ecosystem. Returns percentile ranking, dimension-by-dimension comparison against category averages, and specific areas where your tools underperform. Requires a prior toolrank_score result or raw tool definitions.",
|
|
224
|
+
inputSchema: {
|
|
225
|
+
type: "object",
|
|
226
|
+
properties: {
|
|
227
|
+
tools: {
|
|
228
|
+
type: "array",
|
|
229
|
+
description: "Array of MCP tool definition objects to compare",
|
|
230
|
+
items: { type: "object" },
|
|
231
|
+
},
|
|
232
|
+
category: {
|
|
233
|
+
type: "string",
|
|
234
|
+
description: "Tool category for comparison (e.g., 'crm', 'database', 'devtools'). If omitted, compares against the full ecosystem average.",
|
|
235
|
+
default: "all",
|
|
236
|
+
},
|
|
237
|
+
},
|
|
238
|
+
required: ["tools"],
|
|
239
|
+
},
|
|
240
|
+
},
|
|
241
|
+
{
|
|
242
|
+
name: "toolrank_suggest",
|
|
243
|
+
description:
|
|
244
|
+
"Generates specific, actionable improvement suggestions for MCP tool definitions. Use this when you have a low ToolRank Score and want concrete text rewrites. Returns optimized versions of tool names, descriptions, and schema improvements ranked by expected score impact. Does not execute changes — returns suggestions for review.",
|
|
245
|
+
inputSchema: {
|
|
246
|
+
type: "object",
|
|
247
|
+
properties: {
|
|
248
|
+
tool: {
|
|
249
|
+
type: "object",
|
|
250
|
+
description: "Single MCP tool definition to improve. Must include 'name' and 'description'.",
|
|
251
|
+
properties: {
|
|
252
|
+
name: { type: "string", description: "Current tool name" },
|
|
253
|
+
description: { type: "string", description: "Current tool description" },
|
|
254
|
+
inputSchema: { type: "object", description: "Current input schema" },
|
|
255
|
+
},
|
|
256
|
+
required: ["name"],
|
|
257
|
+
},
|
|
258
|
+
focus: {
|
|
259
|
+
type: "string",
|
|
260
|
+
description: "Which dimension to prioritize improvements for",
|
|
261
|
+
enum: ["findability", "clarity", "precision", "efficiency", "all"],
|
|
262
|
+
default: "all",
|
|
263
|
+
},
|
|
264
|
+
},
|
|
265
|
+
required: ["tool"],
|
|
266
|
+
},
|
|
267
|
+
},
|
|
268
|
+
],
|
|
269
|
+
}));
|
|
270
|
+
|
|
271
|
+
// Tool execution handlers
|
|
272
|
+
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
273
|
+
const { name, arguments: args } = request.params;
|
|
274
|
+
|
|
275
|
+
switch (name) {
|
|
276
|
+
case "toolrank_score": {
|
|
277
|
+
const tools = (args as any).tools || [];
|
|
278
|
+
const serverName = (args as any).server_name || "unnamed";
|
|
279
|
+
|
|
280
|
+
const results = tools.map((t: any) => scoreTool(t));
|
|
281
|
+
const avgScore = results.length > 0
|
|
282
|
+
? Math.round((results.reduce((s: number, r: ToolScoreResult) => s + r.total, 0) / results.length) * 10) / 10
|
|
283
|
+
: 0;
|
|
284
|
+
|
|
285
|
+
const allIssues = results.flatMap((r: ToolScoreResult) => r.issues);
|
|
286
|
+
const topFixes = allIssues
|
|
287
|
+
.sort((a: Issue, b: Issue) => b.impact - a.impact)
|
|
288
|
+
.slice(0, 5);
|
|
289
|
+
|
|
290
|
+
// Server-level check: too many tools
|
|
291
|
+
if (tools.length > 20) {
|
|
292
|
+
topFixes.unshift({
|
|
293
|
+
dimension: "efficiency",
|
|
294
|
+
severity: "warning" as const,
|
|
295
|
+
message: `Server has ${tools.length} tools. Agent accuracy degrades past 15-20 tools`,
|
|
296
|
+
fix: "Consolidate into 5-15 workflow-oriented tools",
|
|
297
|
+
impact: 8,
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
return {
|
|
302
|
+
content: [{
|
|
303
|
+
type: "text",
|
|
304
|
+
text: JSON.stringify({
|
|
305
|
+
server_name: serverName,
|
|
306
|
+
average_score: avgScore,
|
|
307
|
+
tool_count: tools.length,
|
|
308
|
+
total_issues: allIssues.length,
|
|
309
|
+
critical_issues: allIssues.filter((i: Issue) => i.severity === "critical").length,
|
|
310
|
+
top_improvements: topFixes.slice(0, 3).map((i: Issue) => ({
|
|
311
|
+
message: i.message,
|
|
312
|
+
fix: i.fix,
|
|
313
|
+
impact: `+${i.impact}pt`,
|
|
314
|
+
})),
|
|
315
|
+
tools: results.map((r: ToolScoreResult) => ({
|
|
316
|
+
name: r.name,
|
|
317
|
+
score: r.total,
|
|
318
|
+
level: `${r.level}: ${r.levelName}`,
|
|
319
|
+
dimensions: r.dimensions,
|
|
320
|
+
issues: r.issues.map((i: Issue) => ({
|
|
321
|
+
severity: i.severity,
|
|
322
|
+
message: i.message,
|
|
323
|
+
fix: i.fix,
|
|
324
|
+
impact: `+${i.impact}pt`,
|
|
325
|
+
})),
|
|
326
|
+
})),
|
|
327
|
+
}, null, 2),
|
|
328
|
+
}],
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
case "toolrank_compare": {
|
|
333
|
+
const tools = (args as any).tools || [];
|
|
334
|
+
const category = (args as any).category || "all";
|
|
335
|
+
|
|
336
|
+
const results = tools.map((t: any) => scoreTool(t));
|
|
337
|
+
const avgScore = results.length > 0
|
|
338
|
+
? Math.round((results.reduce((s: number, r: ToolScoreResult) => s + r.total, 0) / results.length) * 10) / 10
|
|
339
|
+
: 0;
|
|
340
|
+
|
|
341
|
+
// Ecosystem benchmarks (from research data, will be replaced with live DB data)
|
|
342
|
+
const benchmarks: Record<string, any> = {
|
|
343
|
+
all: { avg: 42, median: 38, p75: 62, p90: 78 },
|
|
344
|
+
// Categories will be populated from scan DB
|
|
345
|
+
};
|
|
346
|
+
const bench = benchmarks[category] || benchmarks.all;
|
|
347
|
+
|
|
348
|
+
let percentile: number;
|
|
349
|
+
if (avgScore >= bench.p90) percentile = 95;
|
|
350
|
+
else if (avgScore >= bench.p75) percentile = 80;
|
|
351
|
+
else if (avgScore >= bench.median) percentile = 55;
|
|
352
|
+
else if (avgScore >= bench.avg) percentile = 40;
|
|
353
|
+
else percentile = 20;
|
|
354
|
+
|
|
355
|
+
return {
|
|
356
|
+
content: [{
|
|
357
|
+
type: "text",
|
|
358
|
+
text: JSON.stringify({
|
|
359
|
+
your_score: avgScore,
|
|
360
|
+
category,
|
|
361
|
+
percentile: `Top ${100 - percentile}%`,
|
|
362
|
+
benchmark: bench,
|
|
363
|
+
comparison: {
|
|
364
|
+
vs_average: `${avgScore > bench.avg ? "+" : ""}${Math.round(avgScore - bench.avg)} points`,
|
|
365
|
+
vs_median: `${avgScore > bench.median ? "+" : ""}${Math.round(avgScore - bench.median)} points`,
|
|
366
|
+
},
|
|
367
|
+
note: "Benchmarks based on ecosystem scan data. Live rankings available at toolrank.dev/ranking",
|
|
368
|
+
}, null, 2),
|
|
369
|
+
}],
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
case "toolrank_suggest": {
|
|
374
|
+
const tool = (args as any).tool || {};
|
|
375
|
+
const focus = (args as any).focus || "all";
|
|
376
|
+
const result = scoreTool(tool);
|
|
377
|
+
|
|
378
|
+
const suggestions: any[] = [];
|
|
379
|
+
|
|
380
|
+
// Generate concrete suggestions based on issues
|
|
381
|
+
for (const issue of result.issues) {
|
|
382
|
+
if (focus !== "all" && issue.dimension !== focus) continue;
|
|
383
|
+
|
|
384
|
+
const suggestion: any = {
|
|
385
|
+
dimension: issue.dimension,
|
|
386
|
+
current_problem: issue.message,
|
|
387
|
+
suggested_fix: issue.fix,
|
|
388
|
+
expected_impact: `+${issue.impact}pt`,
|
|
389
|
+
};
|
|
390
|
+
|
|
391
|
+
// Generate concrete rewrite for description issues
|
|
392
|
+
if (issue.dimension === "clarity" && tool.name) {
|
|
393
|
+
if (issue.message.includes("No description") || issue.message.includes("too short")) {
|
|
394
|
+
suggestion.example_rewrite = `${tool.name.replace(/_/g, " ").replace(/\b\w/g, (l: string) => l.toUpperCase())} — [describe what it does]. Use this when [describe use case]. Returns [describe output format].`;
|
|
395
|
+
}
|
|
396
|
+
if (issue.message.includes("No clear purpose verb")) {
|
|
397
|
+
suggestion.example_rewrite = `Retrieves/Creates/Searches for [object]. ${tool.description || ""}`;
|
|
398
|
+
}
|
|
399
|
+
if (issue.message.includes("No usage context")) {
|
|
400
|
+
suggestion.example_rewrite = `${tool.description || ""} Use this when you need to [specific scenario].`;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
suggestions.push(suggestion);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
return {
|
|
408
|
+
content: [{
|
|
409
|
+
type: "text",
|
|
410
|
+
text: JSON.stringify({
|
|
411
|
+
tool_name: tool.name,
|
|
412
|
+
current_score: result.total,
|
|
413
|
+
current_level: `${result.level}: ${result.levelName}`,
|
|
414
|
+
suggestions: suggestions.sort((a, b) => {
|
|
415
|
+
const impA = parseInt(a.expected_impact) || 0;
|
|
416
|
+
const impB = parseInt(b.expected_impact) || 0;
|
|
417
|
+
return impB - impA;
|
|
418
|
+
}),
|
|
419
|
+
estimated_score_after_fixes: Math.min(100, result.total + suggestions.reduce((s, sg) => s + (parseInt(sg.expected_impact) || 0), 0)),
|
|
420
|
+
}, null, 2),
|
|
421
|
+
}],
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
default:
|
|
426
|
+
return {
|
|
427
|
+
content: [{ type: "text", text: `Unknown tool: ${name}` }],
|
|
428
|
+
isError: true,
|
|
429
|
+
};
|
|
430
|
+
}
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
// --- Start Server ---
|
|
434
|
+
async function main() {
|
|
435
|
+
const transport = new StdioServerTransport();
|
|
436
|
+
await server.connect(transport);
|
|
437
|
+
console.error("ToolRank MCP Server running on stdio");
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
main().catch(console.error);
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"target": "ES2022",
|
|
4
|
+
"module": "ESNext",
|
|
5
|
+
"moduleResolution": "bundler",
|
|
6
|
+
"outDir": "./dist",
|
|
7
|
+
"rootDir": "./src",
|
|
8
|
+
"strict": true,
|
|
9
|
+
"esModuleInterop": true,
|
|
10
|
+
"skipLibCheck": true,
|
|
11
|
+
"resolveJsonModule": true,
|
|
12
|
+
"declaration": true
|
|
13
|
+
},
|
|
14
|
+
"include": ["src/**/*"]
|
|
15
|
+
}
|