@iris-eval/mcp-server 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -2
- package/dist/capabilities.d.ts +64 -0
- package/dist/capabilities.js +65 -0
- package/dist/config/defaults.js +2 -0
- package/dist/custom-rule-store.d.ts +4 -0
- package/dist/custom-rule-store.js +8 -3
- package/dist/dashboard/assets/{index-BfMShR3p.js → index-Cz8_oOqG.js} +1 -1
- package/dist/dashboard/index.html +1 -1
- package/dist/dashboard/routes/capabilities.d.ts +3 -0
- package/dist/dashboard/routes/capabilities.js +11 -0
- package/dist/dashboard/routes/health.d.ts +5 -1
- package/dist/dashboard/routes/health.js +15 -3
- package/dist/dashboard/routes/rules.js +4 -1
- package/dist/dashboard/routes/traces.d.ts +3 -0
- package/dist/dashboard/routes/traces.js +9 -28
- package/dist/dashboard/server.d.ts +2 -0
- package/dist/dashboard/server.js +6 -2
- package/dist/eval/accuracy.d.ts +41 -0
- package/dist/eval/accuracy.js +97 -0
- package/dist/eval/criticality.d.ts +8 -1
- package/dist/eval/criticality.js +6 -0
- package/dist/eval/dormant.d.ts +4 -0
- package/dist/eval/dormant.js +22 -0
- package/dist/eval/engine.d.ts +1 -0
- package/dist/eval/engine.js +49 -3
- package/dist/eval/failure-classes.d.ts +8 -0
- package/dist/eval/failure-classes.js +18 -0
- package/dist/eval/llm-judge/evaluator.d.ts +10 -0
- package/dist/eval/llm-judge/evaluator.js +16 -1
- package/dist/eval/published-accuracy.d.ts +230 -0
- package/dist/eval/published-accuracy.js +86 -0
- package/dist/eval/questions.d.ts +12 -0
- package/dist/eval/questions.js +14 -0
- package/dist/eval/response-schema.d.ts +652 -0
- package/dist/eval/response-schema.js +130 -0
- package/dist/eval/response.d.ts +12 -0
- package/dist/eval/response.js +30 -0
- package/dist/eval/rules/completeness.js +31 -0
- package/dist/eval/rules/cost.d.ts +1 -1
- package/dist/eval/rules/cost.js +44 -0
- package/dist/eval/rules/custom.d.ts +0 -12
- package/dist/eval/rules/custom.js +21 -0
- package/dist/eval/rules/relevance.js +16 -0
- package/dist/eval/rules/safety.js +178 -1
- package/dist/eval/stamp.d.ts +14 -0
- package/dist/eval/stamp.js +88 -0
- package/dist/eval/stats.d.ts +33 -0
- package/dist/eval/stats.js +109 -0
- package/dist/eval/verdict.d.ts +34 -0
- package/dist/eval/verdict.js +131 -0
- package/dist/index.js +5 -28
- package/dist/instructions.d.ts +17 -0
- package/dist/instructions.js +53 -0
- package/dist/judge-enablement.d.ts +34 -0
- package/dist/judge-enablement.js +78 -0
- package/dist/judge-enablement.json +10 -0
- package/dist/preferences.d.ts +1 -1
- package/dist/prompts.d.ts +3 -0
- package/dist/prompts.js +29 -0
- package/dist/resources/index.d.ts +5 -2
- package/dist/resources/index.js +65 -5
- package/dist/resources/uris.d.ts +12 -0
- package/dist/resources/uris.js +24 -0
- package/dist/retention.d.ts +20 -0
- package/dist/retention.js +44 -0
- package/dist/self-test.d.ts +1 -0
- package/dist/self-test.js +14 -0
- package/dist/server.d.ts +10 -1
- package/dist/server.js +34 -7
- package/dist/storage/index.js +1 -1
- package/dist/storage/migrations/007-eval-provenance.d.ts +3 -0
- package/dist/storage/migrations/007-eval-provenance.js +30 -0
- package/dist/storage/migrations/index.js +24 -4
- package/dist/storage/sqlite-adapter.d.ts +26 -1
- package/dist/storage/sqlite-adapter.js +135 -15
- package/dist/tools/delete-rule.d.ts +8 -0
- package/dist/tools/delete-rule.js +30 -38
- package/dist/tools/delete-trace.d.ts +5 -0
- package/dist/tools/delete-trace.js +24 -27
- package/dist/tools/deploy-rule.d.ts +13 -1
- package/dist/tools/deploy-rule.js +37 -34
- package/dist/tools/describe.d.ts +20 -0
- package/dist/tools/describe.js +36 -0
- package/dist/tools/errors.d.ts +36 -0
- package/dist/tools/errors.js +134 -0
- package/dist/tools/evaluate-output.d.ts +8 -1
- package/dist/tools/evaluate-output.js +37 -58
- package/dist/tools/evaluate-with-llm-judge.d.ts +31 -0
- package/dist/tools/evaluate-with-llm-judge.js +96 -69
- package/dist/tools/get-traces.d.ts +9 -0
- package/dist/tools/get-traces.js +30 -29
- package/dist/tools/index.d.ts +8 -0
- package/dist/tools/index.js +22 -1
- package/dist/tools/list-rules.d.ts +13 -0
- package/dist/tools/list-rules.js +43 -46
- package/dist/tools/log-trace.d.ts +4 -0
- package/dist/tools/log-trace.js +31 -29
- package/dist/tools/respond.d.ts +42 -0
- package/dist/tools/respond.js +90 -0
- package/dist/tools/strict-input.js +1 -1
- package/dist/tools/trace-link.d.ts +2 -0
- package/dist/tools/trace-link.js +13 -2
- package/dist/tools/verify-citations.d.ts +17 -2
- package/dist/tools/verify-citations.js +98 -93
- package/dist/types/config.d.ts +9 -0
- package/dist/types/eval.d.ts +258 -0
- package/dist/types/eval.js +2 -1
- package/dist/types/query.d.ts +2 -0
- package/package.json +1 -1
- package/server.json +72 -2
- package/dist/resources/dashboard-summary.d.ts +0 -3
- package/dist/resources/dashboard-summary.js +0 -16
- package/dist/resources/trace-detail.d.ts +0 -3
- package/dist/resources/trace-detail.js +0 -30
package/server.json
CHANGED
|
@@ -6,12 +6,12 @@
|
|
|
6
6
|
"url": "https://github.com/iris-eval/mcp-server",
|
|
7
7
|
"source": "github"
|
|
8
8
|
},
|
|
9
|
-
"version": "0.
|
|
9
|
+
"version": "0.9.0",
|
|
10
10
|
"packages": [
|
|
11
11
|
{
|
|
12
12
|
"registryType": "npm",
|
|
13
13
|
"identifier": "@iris-eval/mcp-server",
|
|
14
|
-
"version": "0.
|
|
14
|
+
"version": "0.9.0",
|
|
15
15
|
"transport": {
|
|
16
16
|
"type": "stdio"
|
|
17
17
|
},
|
|
@@ -85,6 +85,76 @@
|
|
|
85
85
|
"format": "number",
|
|
86
86
|
"isSecret": false,
|
|
87
87
|
"name": "IRIS_DASHBOARD_PORT"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"description": "MCP HTTP transport port (default 3000); only used with --transport http",
|
|
91
|
+
"isRequired": false,
|
|
92
|
+
"format": "number",
|
|
93
|
+
"isSecret": false,
|
|
94
|
+
"name": "IRIS_PORT"
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
"description": "OTLP/HTTP collector URL; when set, every stored trace is also exported as OpenTelemetry spans (best-effort, asynchronous)",
|
|
98
|
+
"isRequired": false,
|
|
99
|
+
"format": "string",
|
|
100
|
+
"isSecret": false,
|
|
101
|
+
"name": "IRIS_OTEL_ENDPOINT"
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
"description": "Comma-separated domain allowlist for verify_citations fetches (only consulted when IRIS_CITATION_ALLOW_FETCH is on)",
|
|
105
|
+
"isRequired": false,
|
|
106
|
+
"format": "string",
|
|
107
|
+
"isSecret": false,
|
|
108
|
+
"name": "IRIS_CITATION_DOMAINS"
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"description": "Transport: stdio (default) or http; same as --transport",
|
|
112
|
+
"isRequired": false,
|
|
113
|
+
"format": "string",
|
|
114
|
+
"isSecret": false,
|
|
115
|
+
"name": "IRIS_TRANSPORT"
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
"description": "Bind host for the HTTP transport (default 127.0.0.1); only used with --transport http",
|
|
119
|
+
"isRequired": false,
|
|
120
|
+
"format": "string",
|
|
121
|
+
"isSecret": false,
|
|
122
|
+
"name": "IRIS_HOST"
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
"description": "Comma-separated Origin allowlist for the HTTP transport and dashboard CORS (default http://localhost:*)",
|
|
126
|
+
"isRequired": false,
|
|
127
|
+
"format": "string",
|
|
128
|
+
"isSecret": false,
|
|
129
|
+
"name": "IRIS_ALLOWED_ORIGINS"
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
"description": "Bind host for the dashboard (default 127.0.0.1); same as --dashboard-host",
|
|
133
|
+
"isRequired": false,
|
|
134
|
+
"format": "string",
|
|
135
|
+
"isSecret": false,
|
|
136
|
+
"name": "IRIS_DASHBOARD_HOST"
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
"description": "Comma-separated key=value headers sent with every OTLP export (for example an auth token for your collector)",
|
|
140
|
+
"isRequired": false,
|
|
141
|
+
"format": "string",
|
|
142
|
+
"isSecret": false,
|
|
143
|
+
"name": "IRIS_OTEL_HEADERS"
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
"description": "service.name resource attribute on exported spans (default iris-mcp)",
|
|
147
|
+
"isRequired": false,
|
|
148
|
+
"format": "string",
|
|
149
|
+
"isSecret": false,
|
|
150
|
+
"name": "IRIS_OTEL_SERVICE_NAME"
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
"description": "Timeout in milliseconds for each OTLP export request",
|
|
154
|
+
"isRequired": false,
|
|
155
|
+
"format": "number",
|
|
156
|
+
"isSecret": false,
|
|
157
|
+
"name": "IRIS_OTEL_TIMEOUT_MS"
|
|
88
158
|
}
|
|
89
159
|
]
|
|
90
160
|
}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { LOCAL_TENANT } from '../types/tenant.js';
|
|
2
|
-
export function registerDashboardSummaryResource(server, storage) {
|
|
3
|
-
server.resource('dashboard-summary', 'iris://dashboard/summary', { description: 'Dashboard summary with key metrics and trends' }, async () => {
|
|
4
|
-
// OSS single-tenant: summary scopes to the local user.
|
|
5
|
-
const summary = await storage.getDashboardSummary(LOCAL_TENANT);
|
|
6
|
-
return {
|
|
7
|
-
contents: [
|
|
8
|
-
{
|
|
9
|
-
uri: 'iris://dashboard/summary',
|
|
10
|
-
mimeType: 'application/json',
|
|
11
|
-
text: JSON.stringify(summary, null, 2),
|
|
12
|
-
},
|
|
13
|
-
],
|
|
14
|
-
};
|
|
15
|
-
});
|
|
16
|
-
}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import { LOCAL_TENANT } from '../types/tenant.js';
|
|
2
|
-
export function registerTraceDetailResource(server, storage) {
|
|
3
|
-
server.resource('trace-detail', 'iris://traces/{trace_id}', { description: 'Full trace detail with spans and evaluation results' }, async (uri) => {
|
|
4
|
-
const traceId = uri.pathname.split('/').pop();
|
|
5
|
-
// OSS single-tenant: MCP caller is the local user.
|
|
6
|
-
const trace = await storage.getTrace(LOCAL_TENANT, traceId);
|
|
7
|
-
if (!trace) {
|
|
8
|
-
return {
|
|
9
|
-
contents: [
|
|
10
|
-
{
|
|
11
|
-
uri: uri.href,
|
|
12
|
-
mimeType: 'application/json',
|
|
13
|
-
text: JSON.stringify({ error: 'Trace not found' }),
|
|
14
|
-
},
|
|
15
|
-
],
|
|
16
|
-
};
|
|
17
|
-
}
|
|
18
|
-
const spans = await storage.getSpansByTraceId(LOCAL_TENANT, traceId);
|
|
19
|
-
const evals = await storage.getEvalsByTraceId(LOCAL_TENANT, traceId);
|
|
20
|
-
return {
|
|
21
|
-
contents: [
|
|
22
|
-
{
|
|
23
|
-
uri: uri.href,
|
|
24
|
-
mimeType: 'application/json',
|
|
25
|
-
text: JSON.stringify({ trace, spans, evals }, null, 2),
|
|
26
|
-
},
|
|
27
|
-
],
|
|
28
|
-
};
|
|
29
|
-
});
|
|
30
|
-
}
|