@iris-eval/mcp-server 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -2
- package/dist/config/defaults.js +1 -1
- package/dist/dashboard/assets/index-BTiYZFEQ.js +43 -0
- package/dist/dashboard/index.html +1 -1
- package/dist/dashboard/validation.d.ts +6 -6
- package/dist/dashboard/validation.js +2 -2
- package/dist/eval/engine.js +4 -2
- package/dist/eval/rules/custom.js +8 -2
- package/dist/storage/sqlite-adapter.js +5 -2
- package/dist/types/query.d.ts +1 -1
- package/package.json +7 -7
- package/server.json +2 -2
- package/dist/dashboard/assets/index-BwFrX6Zx.js +0 -70
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
|
7
7
|
<title>Iris — Agent Eval & Observability</title>
|
|
8
|
-
<script type="module" crossorigin src="/assets/index-
|
|
8
|
+
<script type="module" crossorigin src="/assets/index-BTiYZFEQ.js"></script>
|
|
9
9
|
<link rel="stylesheet" crossorigin href="/assets/index-C9BwWthL.css">
|
|
10
10
|
</head>
|
|
11
11
|
<body>
|
|
@@ -57,19 +57,19 @@ export declare const summaryQuerySchema: z.ZodObject<{
|
|
|
57
57
|
hours?: number | undefined;
|
|
58
58
|
}>;
|
|
59
59
|
export declare const evalStatsPeriodSchema: z.ZodObject<{
|
|
60
|
-
period: z.ZodDefault<z.ZodEnum<["24h", "7d", "30d"]>>;
|
|
60
|
+
period: z.ZodDefault<z.ZodEnum<["24h", "7d", "30d", "all"]>>;
|
|
61
61
|
}, "strip", z.ZodTypeAny, {
|
|
62
|
-
period: "24h" | "7d" | "30d";
|
|
62
|
+
period: "24h" | "7d" | "30d" | "all";
|
|
63
63
|
}, {
|
|
64
|
-
period?: "24h" | "7d" | "30d" | undefined;
|
|
64
|
+
period?: "24h" | "7d" | "30d" | "all" | undefined;
|
|
65
65
|
}>;
|
|
66
66
|
export declare const evalStatsFailuresSchema: z.ZodObject<{
|
|
67
|
-
period: z.ZodDefault<z.ZodEnum<["24h", "7d", "30d"]>>;
|
|
67
|
+
period: z.ZodDefault<z.ZodEnum<["24h", "7d", "30d", "all"]>>;
|
|
68
68
|
limit: z.ZodDefault<z.ZodNumber>;
|
|
69
69
|
}, "strip", z.ZodTypeAny, {
|
|
70
70
|
limit: number;
|
|
71
|
-
period: "24h" | "7d" | "30d";
|
|
71
|
+
period: "24h" | "7d" | "30d" | "all";
|
|
72
72
|
}, {
|
|
73
73
|
limit?: number | undefined;
|
|
74
|
-
period?: "24h" | "7d" | "30d" | undefined;
|
|
74
|
+
period?: "24h" | "7d" | "30d" | "all" | undefined;
|
|
75
75
|
}>;
|
|
@@ -21,9 +21,9 @@ export const summaryQuerySchema = z.object({
|
|
|
21
21
|
hours: z.coerce.number().int().min(1).max(8760).default(24),
|
|
22
22
|
});
|
|
23
23
|
export const evalStatsPeriodSchema = z.object({
|
|
24
|
-
period: z.enum(['24h', '7d', '30d']).default('24h'),
|
|
24
|
+
period: z.enum(['24h', '7d', '30d', 'all']).default('24h'),
|
|
25
25
|
});
|
|
26
26
|
export const evalStatsFailuresSchema = z.object({
|
|
27
|
-
period: z.enum(['24h', '7d', '30d']).default('24h'),
|
|
27
|
+
period: z.enum(['24h', '7d', '30d', 'all']).default('24h'),
|
|
28
28
|
limit: z.coerce.number().int().min(1).max(100).default(10),
|
|
29
29
|
});
|
package/dist/eval/engine.js
CHANGED
|
@@ -37,9 +37,11 @@ export class EvalEngine {
|
|
|
37
37
|
const ruleResults = rules.map((rule) => rule.evaluate(context));
|
|
38
38
|
const totalWeight = rules.reduce((sum, r) => sum + r.weight, 0);
|
|
39
39
|
const weightedScore = rules.reduce((sum, rule, i) => {
|
|
40
|
-
|
|
40
|
+
const ruleScore = Number.isFinite(ruleResults[i].score) ? ruleResults[i].score : 0;
|
|
41
|
+
return sum + ruleScore * rule.weight;
|
|
41
42
|
}, 0);
|
|
42
|
-
const
|
|
43
|
+
const rawScore = totalWeight > 0 ? weightedScore / totalWeight : 0;
|
|
44
|
+
const score = Number.isFinite(rawScore) ? rawScore : 0;
|
|
43
45
|
const passed = score >= this.threshold;
|
|
44
46
|
const suggestions = [];
|
|
45
47
|
for (const result of ruleResults) {
|
|
@@ -41,12 +41,18 @@ export function createCustomRule(definition) {
|
|
|
41
41
|
return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? 'Forbidden pattern not found' : 'Forbidden pattern found in output' };
|
|
42
42
|
}
|
|
43
43
|
case 'min_length': {
|
|
44
|
-
const min = definition.config.length;
|
|
44
|
+
const min = (definition.config.min_length ?? definition.config.length);
|
|
45
|
+
if (min == null || min <= 0) {
|
|
46
|
+
return { ruleName: definition.name, passed: false, score: 0, message: 'min_length rule requires config.min_length (positive number)' };
|
|
47
|
+
}
|
|
45
48
|
const passed = context.output.length >= min;
|
|
46
49
|
return { ruleName: definition.name, passed, score: passed ? 1 : context.output.length / min, message: passed ? `Length (${context.output.length}) meets minimum (${min})` : `Length (${context.output.length}) below minimum (${min})` };
|
|
47
50
|
}
|
|
48
51
|
case 'max_length': {
|
|
49
|
-
const max = definition.config.length;
|
|
52
|
+
const max = (definition.config.max_length ?? definition.config.length);
|
|
53
|
+
if (max == null || max <= 0) {
|
|
54
|
+
return { ruleName: definition.name, passed: false, score: 0, message: 'max_length rule requires config.max_length (positive number)' };
|
|
55
|
+
}
|
|
50
56
|
const passed = context.output.length <= max;
|
|
51
57
|
return { ruleName: definition.name, passed, score: passed ? 1 : max / context.output.length, message: passed ? `Length (${context.output.length}) within maximum (${max})` : `Length (${context.output.length}) exceeds maximum (${max})` };
|
|
52
58
|
}
|
|
@@ -7,6 +7,7 @@ export class SqliteAdapter {
|
|
|
7
7
|
}
|
|
8
8
|
async initialize() {
|
|
9
9
|
this.db.pragma('journal_mode = WAL');
|
|
10
|
+
this.db.pragma('busy_timeout = 5000');
|
|
10
11
|
this.db.pragma('foreign_keys = ON');
|
|
11
12
|
runMigrations(this.db);
|
|
12
13
|
}
|
|
@@ -171,6 +172,8 @@ export class SqliteAdapter {
|
|
|
171
172
|
// Eval-stats endpoints (v0.2.0 dashboard)
|
|
172
173
|
// ---------------------------------------------------------------------------
|
|
173
174
|
periodToSince(period) {
|
|
175
|
+
if (period === 'all')
|
|
176
|
+
return '1970-01-01T00:00:00.000Z';
|
|
174
177
|
const hours = period === '24h' ? 24 : period === '7d' ? 168 : 720;
|
|
175
178
|
return new Date(Date.now() - hours * 60 * 60 * 1000).toISOString();
|
|
176
179
|
}
|
|
@@ -182,7 +185,7 @@ export class SqliteAdapter {
|
|
|
182
185
|
COALESCE(AVG(score), 0) AS avg_score,
|
|
183
186
|
SUM(CASE WHEN passed = 1 THEN 1 ELSE 0 END) AS passed_count
|
|
184
187
|
FROM eval_results
|
|
185
|
-
WHERE created_at >= ?
|
|
188
|
+
WHERE created_at >= ? AND trace_id IS NOT NULL
|
|
186
189
|
`).get(since);
|
|
187
190
|
const cost = this.db.prepare(`
|
|
188
191
|
SELECT COALESCE(SUM(cost_usd), 0) AS total_cost
|
|
@@ -224,7 +227,7 @@ export class SqliteAdapter {
|
|
|
224
227
|
avgScore: Math.round(agg.avg_score * 1000) / 1000,
|
|
225
228
|
totalEvals: agg.total_evals,
|
|
226
229
|
safetyViolations: violations,
|
|
227
|
-
totalCost: Math.round(cost.total_cost *
|
|
230
|
+
totalCost: Math.round(cost.total_cost * 10000) / 10000,
|
|
228
231
|
agentCount: agents.agent_count,
|
|
229
232
|
period,
|
|
230
233
|
};
|
package/dist/types/query.d.ts
CHANGED
|
@@ -22,7 +22,7 @@ export interface TraceQueryResult {
|
|
|
22
22
|
limit: number;
|
|
23
23
|
offset: number;
|
|
24
24
|
}
|
|
25
|
-
export type EvalStatsPeriod = '24h' | '7d' | '30d';
|
|
25
|
+
export type EvalStatsPeriod = '24h' | '7d' | '30d' | 'all';
|
|
26
26
|
export interface EvalStats {
|
|
27
27
|
passRate: number;
|
|
28
28
|
avgScore: number;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@iris-eval/mcp-server",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.9",
|
|
4
4
|
"description": "The agent eval standard for MCP. Score every agent output for quality, safety, and cost.",
|
|
5
5
|
"mcpName": "io.github.iris-eval/mcp-server",
|
|
6
6
|
"type": "module",
|
|
@@ -67,10 +67,10 @@
|
|
|
67
67
|
"node": ">=20.0.0"
|
|
68
68
|
},
|
|
69
69
|
"dependencies": {
|
|
70
|
-
"@modelcontextprotocol/sdk": "^1.
|
|
70
|
+
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
71
71
|
"better-sqlite3": "^12.8.0",
|
|
72
72
|
"express": "^5.1.0",
|
|
73
|
-
"express-rate-limit": "^8.3.
|
|
73
|
+
"express-rate-limit": "^8.3.2",
|
|
74
74
|
"helmet": "^8.1.0",
|
|
75
75
|
"pino": "^10.3.1",
|
|
76
76
|
"safe-regex2": "^5.1.0",
|
|
@@ -79,11 +79,11 @@
|
|
|
79
79
|
"devDependencies": {
|
|
80
80
|
"@types/better-sqlite3": "^7.6.0",
|
|
81
81
|
"@types/express": "^5.0.0",
|
|
82
|
-
"@types/node": "^25.5.
|
|
83
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
84
|
-
"@typescript-eslint/parser": "^8.
|
|
82
|
+
"@types/node": "^25.5.2",
|
|
83
|
+
"@typescript-eslint/eslint-plugin": "^8.58.0",
|
|
84
|
+
"@typescript-eslint/parser": "^8.58.0",
|
|
85
85
|
"@vitest/coverage-v8": "^4.1.1",
|
|
86
|
-
"eslint": "^10.
|
|
86
|
+
"eslint": "^10.2.0",
|
|
87
87
|
"prettier": "^3.0.0",
|
|
88
88
|
"tsx": "^4.0.0",
|
|
89
89
|
"typescript": "^5.7.0",
|
package/server.json
CHANGED
|
@@ -6,12 +6,12 @@
|
|
|
6
6
|
"url": "https://github.com/iris-eval/mcp-server",
|
|
7
7
|
"source": "github"
|
|
8
8
|
},
|
|
9
|
-
"version": "0.1.
|
|
9
|
+
"version": "0.1.9",
|
|
10
10
|
"packages": [
|
|
11
11
|
{
|
|
12
12
|
"registryType": "npm",
|
|
13
13
|
"identifier": "@iris-eval/mcp-server",
|
|
14
|
-
"version": "0.1.
|
|
14
|
+
"version": "0.1.9",
|
|
15
15
|
"transport": {
|
|
16
16
|
"type": "stdio"
|
|
17
17
|
},
|