@kevinrabun/judges 3.115.3 → 3.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +7 -3
- package/dist/api.js +7 -1
- package/dist/evaluation-session.d.ts +74 -0
- package/dist/evaluation-session.js +152 -0
- package/dist/evaluators/index.d.ts +23 -1
- package/dist/evaluators/index.js +163 -3
- package/dist/evaluators/judge-selector.d.ts +19 -0
- package/dist/evaluators/judge-selector.js +141 -0
- package/dist/index.js +2 -0
- package/dist/judges/index.d.ts +54 -9
- package/dist/judges/index.js +72 -14
- package/dist/tools/register-evaluation.js +208 -8
- package/dist/tools/register-fix.js +24 -1
- package/dist/tools/register-resources.d.ts +6 -0
- package/dist/tools/register-resources.js +177 -0
- package/dist/tools/register-review.js +26 -1
- package/dist/tools/register-workflow.js +384 -11
- package/dist/tools/validation.d.ts +13 -0
- package/dist/tools/validation.js +77 -0
- package/dist/types.d.ts +122 -0
- package/package.json +24 -12
- package/server.json +2 -2
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
// ─── MCP Resource Registration ───────────────────────────────────────────────
|
|
2
|
+
// Expose judges metadata, presets, and session state as MCP resources.
|
|
3
|
+
// Includes both static resources and parameterized resource templates for
|
|
4
|
+
// efficient single-item lookups (judges://judge/{id}, judges://preset/{key}).
|
|
5
|
+
// ──────────────────────────────────────────────────────────────────────────────
|
|
6
|
+
import { ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
7
|
+
import { getJudge, getJudgeSummaries, JUDGES } from "../judges/index.js";
|
|
8
|
+
import { getPreset, PRESETS } from "../presets.js";
|
|
9
|
+
import { getGlobalSession } from "../evaluation-session.js";
|
|
10
|
+
/**
|
|
11
|
+
* Register MCP resources: judges catalog, presets, session state,
|
|
12
|
+
* and parameterized templates for single-judge / single-preset lookups.
|
|
13
|
+
*/
|
|
14
|
+
export function registerResources(server) {
|
|
15
|
+
registerJudgesCatalog(server);
|
|
16
|
+
registerPresetsResource(server);
|
|
17
|
+
registerSessionResource(server);
|
|
18
|
+
registerJudgeTemplate(server);
|
|
19
|
+
registerPresetTemplate(server);
|
|
20
|
+
}
|
|
21
|
+
// ─── judges://catalog ────────────────────────────────────────────────────────
|
|
22
|
+
function registerJudgesCatalog(server) {
|
|
23
|
+
server.resource("judges-catalog", "judges://catalog", { description: "Full catalog of all judges on the panel — IDs, names, domains, and descriptions." }, async (uri) => {
|
|
24
|
+
const judges = getJudgeSummaries();
|
|
25
|
+
const data = judges.map((j) => ({
|
|
26
|
+
id: j.id,
|
|
27
|
+
name: j.name,
|
|
28
|
+
domain: j.domain,
|
|
29
|
+
description: j.description,
|
|
30
|
+
}));
|
|
31
|
+
return {
|
|
32
|
+
contents: [
|
|
33
|
+
{
|
|
34
|
+
uri: uri.href,
|
|
35
|
+
mimeType: "application/json",
|
|
36
|
+
text: JSON.stringify(data, null, 2),
|
|
37
|
+
},
|
|
38
|
+
],
|
|
39
|
+
};
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
// ─── judges://presets ────────────────────────────────────────────────────────
|
|
43
|
+
function registerPresetsResource(server) {
|
|
44
|
+
server.resource("presets", "judges://presets", { description: "Available evaluation presets with names, descriptions, and configuration overrides." }, async (uri) => {
|
|
45
|
+
const data = Object.entries(PRESETS).map(([key, preset]) => ({
|
|
46
|
+
key,
|
|
47
|
+
name: preset.name,
|
|
48
|
+
description: preset.description,
|
|
49
|
+
config: preset.config,
|
|
50
|
+
}));
|
|
51
|
+
return {
|
|
52
|
+
contents: [
|
|
53
|
+
{
|
|
54
|
+
uri: uri.href,
|
|
55
|
+
mimeType: "application/json",
|
|
56
|
+
text: JSON.stringify(data, null, 2),
|
|
57
|
+
},
|
|
58
|
+
],
|
|
59
|
+
};
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
// ─── judges://session ────────────────────────────────────────────────────────
|
|
63
|
+
function registerSessionResource(server) {
|
|
64
|
+
server.resource("session", "judges://session", {
|
|
65
|
+
description: "Current evaluation session state — evaluation count, detected frameworks, verdict history, and stability indicators.",
|
|
66
|
+
}, async (uri) => {
|
|
67
|
+
const session = getGlobalSession();
|
|
68
|
+
const ctx = session.getContext();
|
|
69
|
+
const filesEvaluated = [...ctx.verdictHistory.entries()].map(([file, history]) => ({
|
|
70
|
+
file,
|
|
71
|
+
evaluations: history.length,
|
|
72
|
+
latestScore: history[history.length - 1]?.score ?? 0,
|
|
73
|
+
stable: session.isVerdictStable(file),
|
|
74
|
+
}));
|
|
75
|
+
const data = {
|
|
76
|
+
evaluationCount: ctx.evaluationCount,
|
|
77
|
+
startedAt: ctx.startedAt,
|
|
78
|
+
frameworks: ctx.frameworks,
|
|
79
|
+
capabilities: [...ctx.capabilities],
|
|
80
|
+
filesEvaluated,
|
|
81
|
+
};
|
|
82
|
+
return {
|
|
83
|
+
contents: [
|
|
84
|
+
{
|
|
85
|
+
uri: uri.href,
|
|
86
|
+
mimeType: "application/json",
|
|
87
|
+
text: JSON.stringify(data, null, 2),
|
|
88
|
+
},
|
|
89
|
+
],
|
|
90
|
+
};
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
// ─── judges://judge/{id} (template) ─────────────────────────────────────────
|
|
94
|
+
function registerJudgeTemplate(server) {
|
|
95
|
+
const judgeIds = JUDGES.map((j) => j.id);
|
|
96
|
+
server.resource("judge-detail", new ResourceTemplate("judges://judge/{id}", {
|
|
97
|
+
list: async () => ({
|
|
98
|
+
resources: judgeIds.map((id) => ({
|
|
99
|
+
uri: `judges://judge/${id}`,
|
|
100
|
+
name: id,
|
|
101
|
+
})),
|
|
102
|
+
}),
|
|
103
|
+
complete: {
|
|
104
|
+
id: (value) => judgeIds.filter((id) => id.startsWith(value)),
|
|
105
|
+
},
|
|
106
|
+
}), { description: "Detailed info for a single judge — rules, domain, system prompt summary." }, async (uri, { id }) => {
|
|
107
|
+
const judgeId = Array.isArray(id) ? id[0] : id;
|
|
108
|
+
const judge = getJudge(judgeId);
|
|
109
|
+
if (!judge) {
|
|
110
|
+
return {
|
|
111
|
+
contents: [
|
|
112
|
+
{
|
|
113
|
+
uri: uri.href,
|
|
114
|
+
mimeType: "application/json",
|
|
115
|
+
text: JSON.stringify({ error: `Judge '${judgeId}' not found` }),
|
|
116
|
+
},
|
|
117
|
+
],
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
const data = {
|
|
121
|
+
id: judge.id,
|
|
122
|
+
name: judge.name,
|
|
123
|
+
domain: judge.domain,
|
|
124
|
+
description: judge.description,
|
|
125
|
+
rulePrefix: judge.rulePrefix,
|
|
126
|
+
tableDescription: judge.tableDescription,
|
|
127
|
+
promptDescription: judge.promptDescription,
|
|
128
|
+
};
|
|
129
|
+
return {
|
|
130
|
+
contents: [
|
|
131
|
+
{
|
|
132
|
+
uri: uri.href,
|
|
133
|
+
mimeType: "application/json",
|
|
134
|
+
text: JSON.stringify(data, null, 2),
|
|
135
|
+
},
|
|
136
|
+
],
|
|
137
|
+
};
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
// ─── judges://preset/{key} (template) ───────────────────────────────────────
|
|
141
|
+
function registerPresetTemplate(server) {
|
|
142
|
+
const presetKeys = Object.keys(PRESETS);
|
|
143
|
+
server.resource("preset-detail", new ResourceTemplate("judges://preset/{key}", {
|
|
144
|
+
list: async () => ({
|
|
145
|
+
resources: presetKeys.map((key) => ({
|
|
146
|
+
uri: `judges://preset/${key}`,
|
|
147
|
+
name: key,
|
|
148
|
+
})),
|
|
149
|
+
}),
|
|
150
|
+
complete: {
|
|
151
|
+
key: (value) => presetKeys.filter((k) => k.startsWith(value)),
|
|
152
|
+
},
|
|
153
|
+
}), { description: "Detailed configuration for a single evaluation preset." }, async (uri, { key }) => {
|
|
154
|
+
const presetKey = Array.isArray(key) ? key[0] : key;
|
|
155
|
+
const preset = getPreset(presetKey);
|
|
156
|
+
if (!preset) {
|
|
157
|
+
return {
|
|
158
|
+
contents: [
|
|
159
|
+
{
|
|
160
|
+
uri: uri.href,
|
|
161
|
+
mimeType: "application/json",
|
|
162
|
+
text: JSON.stringify({ error: `Preset '${presetKey}' not found` }),
|
|
163
|
+
},
|
|
164
|
+
],
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
return {
|
|
168
|
+
contents: [
|
|
169
|
+
{
|
|
170
|
+
uri: uri.href,
|
|
171
|
+
mimeType: "application/json",
|
|
172
|
+
text: JSON.stringify({ key: presetKey, ...preset }, null, 2),
|
|
173
|
+
},
|
|
174
|
+
],
|
|
175
|
+
};
|
|
176
|
+
});
|
|
177
|
+
}
|
|
@@ -115,8 +115,21 @@ function registerExplainFinding(server) {
|
|
|
115
115
|
sections.push(`\n## Remediation\n${remediation}`);
|
|
116
116
|
}
|
|
117
117
|
sections.push(`\n## Next steps\n- Use \`triage_finding\` to accept, defer, or dismiss this finding\n- Use \`fix_code\` to auto-fix if a patch is available\n- Use \`evaluate_code\` to re-evaluate after fixing`);
|
|
118
|
+
const structured = {
|
|
119
|
+
ruleId,
|
|
120
|
+
prefix,
|
|
121
|
+
title: title ?? null,
|
|
122
|
+
severity: severity ?? null,
|
|
123
|
+
owasp: ctx?.owasp ?? null,
|
|
124
|
+
cwe: ctx?.cwe ?? null,
|
|
125
|
+
learn: ctx?.learn ?? null,
|
|
126
|
+
remediation: getRemediationGuidance(prefix) ?? null,
|
|
127
|
+
};
|
|
118
128
|
return {
|
|
119
|
-
content: [
|
|
129
|
+
content: [
|
|
130
|
+
{ type: "text", text: sections.join("\n") },
|
|
131
|
+
{ type: "text", text: "```json\n" + JSON.stringify(structured, null, 2) + "\n```" },
|
|
132
|
+
],
|
|
120
133
|
};
|
|
121
134
|
});
|
|
122
135
|
}
|
|
@@ -173,6 +186,18 @@ function registerTriageFinding(server) {
|
|
|
173
186
|
type: "text",
|
|
174
187
|
text: `✓ Triaged finding \`${result.ruleId}\` in ${result.filePath} as **${status}**${reason ? `\n\nReason: ${reason}` : ""}${triagedBy ? `\nTriaged by: ${triagedBy}` : ""}`,
|
|
175
188
|
},
|
|
189
|
+
{
|
|
190
|
+
type: "text",
|
|
191
|
+
text: "```json\n" +
|
|
192
|
+
JSON.stringify({
|
|
193
|
+
ruleId: result.ruleId,
|
|
194
|
+
filePath: result.filePath,
|
|
195
|
+
status,
|
|
196
|
+
reason: reason ?? null,
|
|
197
|
+
triagedBy: triagedBy ?? null,
|
|
198
|
+
}, null, 2) +
|
|
199
|
+
"\n```",
|
|
200
|
+
},
|
|
176
201
|
],
|
|
177
202
|
};
|
|
178
203
|
}
|