@iris-eval/mcp-server 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -4
- package/dist/config/defaults.js +1 -1
- package/dist/dashboard/assets/index-C9BwWthL.css +1 -0
- package/dist/dashboard/assets/index-neEIXwxp.js +70 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/dashboard/routes/eval-stats.d.ts +3 -0
- package/dist/dashboard/routes/eval-stats.js +76 -0
- package/dist/dashboard/routes/evaluations.d.ts +3 -0
- package/dist/dashboard/routes/evaluations.js +15 -0
- package/dist/dashboard/routes/filters.d.ts +3 -0
- package/dist/dashboard/routes/filters.js +9 -0
- package/dist/dashboard/routes/health.d.ts +3 -0
- package/dist/dashboard/routes/health.js +25 -0
- package/dist/dashboard/routes/index.d.ts +6 -0
- package/dist/dashboard/routes/index.js +6 -0
- package/dist/dashboard/routes/summary.d.ts +3 -0
- package/dist/dashboard/routes/summary.js +8 -0
- package/dist/dashboard/routes/traces.d.ts +3 -0
- package/dist/dashboard/routes/traces.js +29 -0
- package/dist/dashboard/server.d.ts +10 -0
- package/dist/dashboard/server.js +65 -0
- package/dist/dashboard/validation.d.ts +75 -0
- package/dist/dashboard/validation.js +29 -0
- package/dist/eval/engine.js +4 -2
- package/dist/eval/rules/custom.js +8 -2
- package/dist/storage/sqlite-adapter.js +5 -2
- package/dist/types/query.d.ts +1 -1
- package/package.json +1 -1
- package/server.json +2 -2
- package/dist/dashboard/assets/index-BgJQ3xJ9.js +0 -127
- package/dist/dashboard/assets/index-zuWgVxsu.css +0 -1
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
|
7
7
|
<title>Iris — Agent Eval & Observability</title>
|
|
8
|
-
<script type="module" crossorigin src="/assets/index-
|
|
9
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
8
|
+
<script type="module" crossorigin src="/assets/index-neEIXwxp.js"></script>
|
|
9
|
+
<link rel="stylesheet" crossorigin href="/assets/index-C9BwWthL.css">
|
|
10
10
|
</head>
|
|
11
11
|
<body>
|
|
12
12
|
<div id="root"></div>
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { evalStatsPeriodSchema, evalStatsFailuresSchema } from '../validation.js';
|
|
2
|
+
export function registerEvalStatsRoutes(router, storage) {
|
|
3
|
+
/**
|
|
4
|
+
* GET /eval-stats
|
|
5
|
+
* Aggregate eval statistics for dashboard hero cards.
|
|
6
|
+
*/
|
|
7
|
+
router.get('/eval-stats', async (req, res) => {
|
|
8
|
+
try {
|
|
9
|
+
const { period } = evalStatsPeriodSchema.parse(req.query);
|
|
10
|
+
const stats = await storage.getEvalStats(period);
|
|
11
|
+
res.json(stats);
|
|
12
|
+
}
|
|
13
|
+
catch (err) {
|
|
14
|
+
if (err instanceof Error && err.name === 'ZodError') {
|
|
15
|
+
res.status(400).json({ error: 'Invalid query parameters', details: err.issues });
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
throw err;
|
|
19
|
+
}
|
|
20
|
+
});
|
|
21
|
+
/**
|
|
22
|
+
* GET /eval-stats/trend
|
|
23
|
+
* Eval scores bucketed over time for the trend chart.
|
|
24
|
+
*/
|
|
25
|
+
router.get('/eval-stats/trend', async (req, res) => {
|
|
26
|
+
try {
|
|
27
|
+
const { period } = evalStatsPeriodSchema.parse(req.query);
|
|
28
|
+
const trend = await storage.getEvalStatsTrend(period);
|
|
29
|
+
res.json(trend);
|
|
30
|
+
}
|
|
31
|
+
catch (err) {
|
|
32
|
+
if (err instanceof Error && err.name === 'ZodError') {
|
|
33
|
+
res.status(400).json({ error: 'Invalid query parameters', details: err.issues });
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
throw err;
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
/**
|
|
40
|
+
* GET /eval-stats/rules
|
|
41
|
+
* Per-rule pass rates for the rule breakdown chart.
|
|
42
|
+
* Sorted by passRate ASC (worst rules first).
|
|
43
|
+
*/
|
|
44
|
+
router.get('/eval-stats/rules', async (req, res) => {
|
|
45
|
+
try {
|
|
46
|
+
const { period } = evalStatsPeriodSchema.parse(req.query);
|
|
47
|
+
const rules = await storage.getEvalStatsRules(period);
|
|
48
|
+
res.json(rules);
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
if (err instanceof Error && err.name === 'ZodError') {
|
|
52
|
+
res.status(400).json({ error: 'Invalid query parameters', details: err.issues });
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
throw err;
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
/**
|
|
59
|
+
* GET /eval-stats/failures
|
|
60
|
+
* Recent failing evaluations for the failures table.
|
|
61
|
+
*/
|
|
62
|
+
router.get('/eval-stats/failures', async (req, res) => {
|
|
63
|
+
try {
|
|
64
|
+
const query = evalStatsFailuresSchema.parse(req.query);
|
|
65
|
+
const failures = await storage.getEvalStatsFailures(query.period, query.limit);
|
|
66
|
+
res.json(failures);
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
if (err instanceof Error && err.name === 'ZodError') {
|
|
70
|
+
res.status(400).json({ error: 'Invalid query parameters', details: err.issues });
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
throw err;
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { evalQuerySchema } from '../validation.js';
|
|
2
|
+
export function registerEvaluationRoutes(router, storage) {
|
|
3
|
+
router.get('/evaluations', async (req, res) => {
|
|
4
|
+
const query = evalQuerySchema.parse(req.query);
|
|
5
|
+
const result = await storage.queryEvalResults({
|
|
6
|
+
eval_type: query.eval_type,
|
|
7
|
+
passed: query.passed,
|
|
8
|
+
since: query.since,
|
|
9
|
+
until: query.until,
|
|
10
|
+
limit: query.limit,
|
|
11
|
+
offset: query.offset,
|
|
12
|
+
});
|
|
13
|
+
res.json(result);
|
|
14
|
+
});
|
|
15
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export function registerFilterRoutes(router, storage) {
|
|
2
|
+
router.get('/filters', async (_req, res) => {
|
|
3
|
+
const [agentNames, frameworks] = await Promise.all([
|
|
4
|
+
storage.getDistinctValues('agent_name'),
|
|
5
|
+
storage.getDistinctValues('framework'),
|
|
6
|
+
]);
|
|
7
|
+
res.json({ agent_names: agentNames, frameworks });
|
|
8
|
+
});
|
|
9
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
const startTime = Date.now();
|
|
2
|
+
export function registerHealthRoutes(router, storage, version) {
|
|
3
|
+
const serverVersion = version ?? 'unknown';
|
|
4
|
+
router.get('/health', async (_req, res) => {
|
|
5
|
+
const uptime_seconds = Math.floor((Date.now() - startTime) / 1000);
|
|
6
|
+
if (storage) {
|
|
7
|
+
try {
|
|
8
|
+
const summary = await storage.getDashboardSummary(1);
|
|
9
|
+
res.json({
|
|
10
|
+
status: 'ok',
|
|
11
|
+
version: serverVersion,
|
|
12
|
+
uptime_seconds,
|
|
13
|
+
trace_count: summary.total_traces,
|
|
14
|
+
storage: 'connected',
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
res.status(503).json({ status: 'degraded', version: serverVersion, uptime_seconds, storage: 'disconnected' });
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
else {
|
|
22
|
+
res.json({ status: 'ok', version: serverVersion, uptime_seconds });
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { registerTraceRoutes } from './traces.js';
|
|
2
|
+
export { registerSummaryRoutes } from './summary.js';
|
|
3
|
+
export { registerEvaluationRoutes } from './evaluations.js';
|
|
4
|
+
export { registerEvalStatsRoutes } from './eval-stats.js';
|
|
5
|
+
export { registerFilterRoutes } from './filters.js';
|
|
6
|
+
export { registerHealthRoutes } from './health.js';
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { registerTraceRoutes } from './traces.js';
|
|
2
|
+
export { registerSummaryRoutes } from './summary.js';
|
|
3
|
+
export { registerEvaluationRoutes } from './evaluations.js';
|
|
4
|
+
export { registerEvalStatsRoutes } from './eval-stats.js';
|
|
5
|
+
export { registerFilterRoutes } from './filters.js';
|
|
6
|
+
export { registerHealthRoutes } from './health.js';
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { summaryQuerySchema } from '../validation.js';
|
|
2
|
+
export function registerSummaryRoutes(router, storage) {
|
|
3
|
+
router.get('/summary', async (req, res) => {
|
|
4
|
+
const query = summaryQuerySchema.parse(req.query);
|
|
5
|
+
const summary = await storage.getDashboardSummary(query.hours);
|
|
6
|
+
res.json(summary);
|
|
7
|
+
});
|
|
8
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { traceQuerySchema } from '../validation.js';
|
|
2
|
+
export function registerTraceRoutes(router, storage) {
|
|
3
|
+
router.get('/traces', async (req, res) => {
|
|
4
|
+
const query = traceQuerySchema.parse(req.query);
|
|
5
|
+
const result = await storage.queryTraces({
|
|
6
|
+
filter: {
|
|
7
|
+
agent_name: query.agent_name,
|
|
8
|
+
framework: query.framework,
|
|
9
|
+
since: query.since,
|
|
10
|
+
until: query.until,
|
|
11
|
+
},
|
|
12
|
+
limit: query.limit,
|
|
13
|
+
offset: query.offset,
|
|
14
|
+
sort_by: query.sort_by,
|
|
15
|
+
sort_order: query.sort_order,
|
|
16
|
+
});
|
|
17
|
+
res.json(result);
|
|
18
|
+
});
|
|
19
|
+
router.get('/traces/:id', async (req, res) => {
|
|
20
|
+
const trace = await storage.getTrace(req.params.id);
|
|
21
|
+
if (!trace) {
|
|
22
|
+
res.status(404).json({ error: 'Trace not found' });
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
const spans = await storage.getSpansByTraceId(req.params.id);
|
|
26
|
+
const evals = await storage.getEvalsByTraceId(req.params.id);
|
|
27
|
+
res.json({ trace, spans, evals });
|
|
28
|
+
});
|
|
29
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import express from 'express';
|
|
2
|
+
import type { Server } from 'node:http';
|
|
3
|
+
import type { IStorageAdapter } from '../types/query.js';
|
|
4
|
+
import type { IrisConfig } from '../types/config.js';
|
|
5
|
+
import type { Logger } from '../utils/logger.js';
|
|
6
|
+
export interface DashboardServer {
|
|
7
|
+
app: express.Application;
|
|
8
|
+
start(): Server;
|
|
9
|
+
}
|
|
10
|
+
export declare function createDashboardServer(storage: IStorageAdapter, config: IrisConfig, logger: Logger): DashboardServer;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import express from 'express';
|
|
2
|
+
import helmet from 'helmet';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { dirname, join } from 'node:path';
|
|
5
|
+
import { existsSync } from 'node:fs';
|
|
6
|
+
import { createAuthMiddleware } from '../middleware/auth.js';
|
|
7
|
+
import { createCorsMiddleware } from '../middleware/cors.js';
|
|
8
|
+
import { createErrorHandler } from '../middleware/error-handler.js';
|
|
9
|
+
import { createApiRateLimiter } from '../middleware/rate-limit.js';
|
|
10
|
+
import { registerTraceRoutes } from './routes/traces.js';
|
|
11
|
+
import { registerSummaryRoutes } from './routes/summary.js';
|
|
12
|
+
import { registerEvaluationRoutes } from './routes/evaluations.js';
|
|
13
|
+
import { registerFilterRoutes } from './routes/filters.js';
|
|
14
|
+
import { registerEvalStatsRoutes } from './routes/eval-stats.js';
|
|
15
|
+
import { registerHealthRoutes } from './routes/health.js';
|
|
16
|
+
export function createDashboardServer(storage, config, logger) {
|
|
17
|
+
const app = express();
|
|
18
|
+
// Security headers
|
|
19
|
+
app.use(helmet({
|
|
20
|
+
contentSecurityPolicy: {
|
|
21
|
+
directives: {
|
|
22
|
+
defaultSrc: ["'self'"],
|
|
23
|
+
scriptSrc: ["'self'"],
|
|
24
|
+
styleSrc: ["'self'", "'unsafe-inline'"],
|
|
25
|
+
connectSrc: ["'self'"],
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
}));
|
|
29
|
+
// Body parser with size limit
|
|
30
|
+
app.use(express.json({ limit: config.security.requestSizeLimit }));
|
|
31
|
+
// CORS
|
|
32
|
+
app.use(createCorsMiddleware(config.security.allowedOrigins));
|
|
33
|
+
// Authentication
|
|
34
|
+
app.use(createAuthMiddleware(config));
|
|
35
|
+
// API routes with rate limiting
|
|
36
|
+
const router = express.Router();
|
|
37
|
+
router.use(createApiRateLimiter(config));
|
|
38
|
+
registerTraceRoutes(router, storage);
|
|
39
|
+
registerSummaryRoutes(router, storage);
|
|
40
|
+
registerEvaluationRoutes(router, storage);
|
|
41
|
+
registerEvalStatsRoutes(router, storage);
|
|
42
|
+
registerFilterRoutes(router, storage);
|
|
43
|
+
registerHealthRoutes(router, storage, config.server.version);
|
|
44
|
+
app.use('/api/v1', router);
|
|
45
|
+
// Serve static dashboard files if built (rate limited)
|
|
46
|
+
const currentDir = dirname(fileURLToPath(import.meta.url));
|
|
47
|
+
const staticDir = join(currentDir, '..', '..', 'dist', 'dashboard');
|
|
48
|
+
if (existsSync(staticDir)) {
|
|
49
|
+
app.use(createApiRateLimiter(config));
|
|
50
|
+
app.use(express.static(staticDir));
|
|
51
|
+
app.get('/{*path}', (_req, res) => {
|
|
52
|
+
res.sendFile(join(staticDir, 'index.html'));
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
// Error handler (must be last)
|
|
56
|
+
app.use(createErrorHandler(logger));
|
|
57
|
+
return {
|
|
58
|
+
app,
|
|
59
|
+
start() {
|
|
60
|
+
return app.listen(config.dashboard.port, () => {
|
|
61
|
+
logger.info(`Dashboard available at http://localhost:${config.dashboard.port}`);
|
|
62
|
+
});
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare const traceQuerySchema: z.ZodObject<{
|
|
3
|
+
agent_name: z.ZodOptional<z.ZodString>;
|
|
4
|
+
framework: z.ZodOptional<z.ZodString>;
|
|
5
|
+
since: z.ZodOptional<z.ZodString>;
|
|
6
|
+
until: z.ZodOptional<z.ZodString>;
|
|
7
|
+
limit: z.ZodDefault<z.ZodNumber>;
|
|
8
|
+
offset: z.ZodDefault<z.ZodNumber>;
|
|
9
|
+
sort_by: z.ZodDefault<z.ZodEnum<["timestamp", "latency_ms", "cost_usd"]>>;
|
|
10
|
+
sort_order: z.ZodDefault<z.ZodEnum<["asc", "desc"]>>;
|
|
11
|
+
}, "strip", z.ZodTypeAny, {
|
|
12
|
+
limit: number;
|
|
13
|
+
offset: number;
|
|
14
|
+
sort_by: "timestamp" | "latency_ms" | "cost_usd";
|
|
15
|
+
sort_order: "asc" | "desc";
|
|
16
|
+
agent_name?: string | undefined;
|
|
17
|
+
framework?: string | undefined;
|
|
18
|
+
since?: string | undefined;
|
|
19
|
+
until?: string | undefined;
|
|
20
|
+
}, {
|
|
21
|
+
agent_name?: string | undefined;
|
|
22
|
+
framework?: string | undefined;
|
|
23
|
+
since?: string | undefined;
|
|
24
|
+
until?: string | undefined;
|
|
25
|
+
limit?: number | undefined;
|
|
26
|
+
offset?: number | undefined;
|
|
27
|
+
sort_by?: "timestamp" | "latency_ms" | "cost_usd" | undefined;
|
|
28
|
+
sort_order?: "asc" | "desc" | undefined;
|
|
29
|
+
}>;
|
|
30
|
+
export declare const evalQuerySchema: z.ZodObject<{
|
|
31
|
+
eval_type: z.ZodOptional<z.ZodString>;
|
|
32
|
+
passed: z.ZodOptional<z.ZodEffects<z.ZodEnum<["true", "false"]>, boolean, "true" | "false">>;
|
|
33
|
+
since: z.ZodOptional<z.ZodString>;
|
|
34
|
+
until: z.ZodOptional<z.ZodString>;
|
|
35
|
+
limit: z.ZodDefault<z.ZodNumber>;
|
|
36
|
+
offset: z.ZodDefault<z.ZodNumber>;
|
|
37
|
+
}, "strip", z.ZodTypeAny, {
|
|
38
|
+
limit: number;
|
|
39
|
+
offset: number;
|
|
40
|
+
since?: string | undefined;
|
|
41
|
+
until?: string | undefined;
|
|
42
|
+
eval_type?: string | undefined;
|
|
43
|
+
passed?: boolean | undefined;
|
|
44
|
+
}, {
|
|
45
|
+
since?: string | undefined;
|
|
46
|
+
until?: string | undefined;
|
|
47
|
+
limit?: number | undefined;
|
|
48
|
+
offset?: number | undefined;
|
|
49
|
+
eval_type?: string | undefined;
|
|
50
|
+
passed?: "true" | "false" | undefined;
|
|
51
|
+
}>;
|
|
52
|
+
export declare const summaryQuerySchema: z.ZodObject<{
|
|
53
|
+
hours: z.ZodDefault<z.ZodNumber>;
|
|
54
|
+
}, "strip", z.ZodTypeAny, {
|
|
55
|
+
hours: number;
|
|
56
|
+
}, {
|
|
57
|
+
hours?: number | undefined;
|
|
58
|
+
}>;
|
|
59
|
+
export declare const evalStatsPeriodSchema: z.ZodObject<{
|
|
60
|
+
period: z.ZodDefault<z.ZodEnum<["24h", "7d", "30d", "all"]>>;
|
|
61
|
+
}, "strip", z.ZodTypeAny, {
|
|
62
|
+
period: "24h" | "7d" | "30d" | "all";
|
|
63
|
+
}, {
|
|
64
|
+
period?: "24h" | "7d" | "30d" | "all" | undefined;
|
|
65
|
+
}>;
|
|
66
|
+
export declare const evalStatsFailuresSchema: z.ZodObject<{
|
|
67
|
+
period: z.ZodDefault<z.ZodEnum<["24h", "7d", "30d", "all"]>>;
|
|
68
|
+
limit: z.ZodDefault<z.ZodNumber>;
|
|
69
|
+
}, "strip", z.ZodTypeAny, {
|
|
70
|
+
limit: number;
|
|
71
|
+
period: "24h" | "7d" | "30d" | "all";
|
|
72
|
+
}, {
|
|
73
|
+
limit?: number | undefined;
|
|
74
|
+
period?: "24h" | "7d" | "30d" | "all" | undefined;
|
|
75
|
+
}>;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export const traceQuerySchema = z.object({
|
|
3
|
+
agent_name: z.string().optional(),
|
|
4
|
+
framework: z.string().optional(),
|
|
5
|
+
since: z.string().optional(),
|
|
6
|
+
until: z.string().optional(),
|
|
7
|
+
limit: z.coerce.number().int().min(1).max(1000).default(50),
|
|
8
|
+
offset: z.coerce.number().int().min(0).default(0),
|
|
9
|
+
sort_by: z.enum(['timestamp', 'latency_ms', 'cost_usd']).default('timestamp'),
|
|
10
|
+
sort_order: z.enum(['asc', 'desc']).default('desc'),
|
|
11
|
+
});
|
|
12
|
+
export const evalQuerySchema = z.object({
|
|
13
|
+
eval_type: z.string().optional(),
|
|
14
|
+
passed: z.enum(['true', 'false']).transform((v) => v === 'true').optional(),
|
|
15
|
+
since: z.string().optional(),
|
|
16
|
+
until: z.string().optional(),
|
|
17
|
+
limit: z.coerce.number().int().min(1).max(1000).default(50),
|
|
18
|
+
offset: z.coerce.number().int().min(0).default(0),
|
|
19
|
+
});
|
|
20
|
+
export const summaryQuerySchema = z.object({
|
|
21
|
+
hours: z.coerce.number().int().min(1).max(8760).default(24),
|
|
22
|
+
});
|
|
23
|
+
export const evalStatsPeriodSchema = z.object({
|
|
24
|
+
period: z.enum(['24h', '7d', '30d', 'all']).default('24h'),
|
|
25
|
+
});
|
|
26
|
+
export const evalStatsFailuresSchema = z.object({
|
|
27
|
+
period: z.enum(['24h', '7d', '30d', 'all']).default('24h'),
|
|
28
|
+
limit: z.coerce.number().int().min(1).max(100).default(10),
|
|
29
|
+
});
|
package/dist/eval/engine.js
CHANGED
|
@@ -37,9 +37,11 @@ export class EvalEngine {
|
|
|
37
37
|
const ruleResults = rules.map((rule) => rule.evaluate(context));
|
|
38
38
|
const totalWeight = rules.reduce((sum, r) => sum + r.weight, 0);
|
|
39
39
|
const weightedScore = rules.reduce((sum, rule, i) => {
|
|
40
|
-
|
|
40
|
+
const ruleScore = Number.isFinite(ruleResults[i].score) ? ruleResults[i].score : 0;
|
|
41
|
+
return sum + ruleScore * rule.weight;
|
|
41
42
|
}, 0);
|
|
42
|
-
const
|
|
43
|
+
const rawScore = totalWeight > 0 ? weightedScore / totalWeight : 0;
|
|
44
|
+
const score = Number.isFinite(rawScore) ? rawScore : 0;
|
|
43
45
|
const passed = score >= this.threshold;
|
|
44
46
|
const suggestions = [];
|
|
45
47
|
for (const result of ruleResults) {
|
|
@@ -41,12 +41,18 @@ export function createCustomRule(definition) {
|
|
|
41
41
|
return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? 'Forbidden pattern not found' : 'Forbidden pattern found in output' };
|
|
42
42
|
}
|
|
43
43
|
case 'min_length': {
|
|
44
|
-
const min = definition.config.length;
|
|
44
|
+
const min = (definition.config.min_length ?? definition.config.length);
|
|
45
|
+
if (min == null || min <= 0) {
|
|
46
|
+
return { ruleName: definition.name, passed: false, score: 0, message: 'min_length rule requires config.min_length (positive number)' };
|
|
47
|
+
}
|
|
45
48
|
const passed = context.output.length >= min;
|
|
46
49
|
return { ruleName: definition.name, passed, score: passed ? 1 : context.output.length / min, message: passed ? `Length (${context.output.length}) meets minimum (${min})` : `Length (${context.output.length}) below minimum (${min})` };
|
|
47
50
|
}
|
|
48
51
|
case 'max_length': {
|
|
49
|
-
const max = definition.config.length;
|
|
52
|
+
const max = (definition.config.max_length ?? definition.config.length);
|
|
53
|
+
if (max == null || max <= 0) {
|
|
54
|
+
return { ruleName: definition.name, passed: false, score: 0, message: 'max_length rule requires config.max_length (positive number)' };
|
|
55
|
+
}
|
|
50
56
|
const passed = context.output.length <= max;
|
|
51
57
|
return { ruleName: definition.name, passed, score: passed ? 1 : max / context.output.length, message: passed ? `Length (${context.output.length}) within maximum (${max})` : `Length (${context.output.length}) exceeds maximum (${max})` };
|
|
52
58
|
}
|
|
@@ -7,6 +7,7 @@ export class SqliteAdapter {
|
|
|
7
7
|
}
|
|
8
8
|
async initialize() {
|
|
9
9
|
this.db.pragma('journal_mode = WAL');
|
|
10
|
+
this.db.pragma('busy_timeout = 5000');
|
|
10
11
|
this.db.pragma('foreign_keys = ON');
|
|
11
12
|
runMigrations(this.db);
|
|
12
13
|
}
|
|
@@ -171,6 +172,8 @@ export class SqliteAdapter {
|
|
|
171
172
|
// Eval-stats endpoints (v0.2.0 dashboard)
|
|
172
173
|
// ---------------------------------------------------------------------------
|
|
173
174
|
periodToSince(period) {
|
|
175
|
+
if (period === 'all')
|
|
176
|
+
return '1970-01-01T00:00:00.000Z';
|
|
174
177
|
const hours = period === '24h' ? 24 : period === '7d' ? 168 : 720;
|
|
175
178
|
return new Date(Date.now() - hours * 60 * 60 * 1000).toISOString();
|
|
176
179
|
}
|
|
@@ -182,7 +185,7 @@ export class SqliteAdapter {
|
|
|
182
185
|
COALESCE(AVG(score), 0) AS avg_score,
|
|
183
186
|
SUM(CASE WHEN passed = 1 THEN 1 ELSE 0 END) AS passed_count
|
|
184
187
|
FROM eval_results
|
|
185
|
-
WHERE created_at >= ?
|
|
188
|
+
WHERE created_at >= ? AND trace_id IS NOT NULL
|
|
186
189
|
`).get(since);
|
|
187
190
|
const cost = this.db.prepare(`
|
|
188
191
|
SELECT COALESCE(SUM(cost_usd), 0) AS total_cost
|
|
@@ -224,7 +227,7 @@ export class SqliteAdapter {
|
|
|
224
227
|
avgScore: Math.round(agg.avg_score * 1000) / 1000,
|
|
225
228
|
totalEvals: agg.total_evals,
|
|
226
229
|
safetyViolations: violations,
|
|
227
|
-
totalCost: Math.round(cost.total_cost *
|
|
230
|
+
totalCost: Math.round(cost.total_cost * 10000) / 10000,
|
|
228
231
|
agentCount: agents.agent_count,
|
|
229
232
|
period,
|
|
230
233
|
};
|
package/dist/types/query.d.ts
CHANGED
|
@@ -22,7 +22,7 @@ export interface TraceQueryResult {
|
|
|
22
22
|
limit: number;
|
|
23
23
|
offset: number;
|
|
24
24
|
}
|
|
25
|
-
export type EvalStatsPeriod = '24h' | '7d' | '30d';
|
|
25
|
+
export type EvalStatsPeriod = '24h' | '7d' | '30d' | 'all';
|
|
26
26
|
export interface EvalStats {
|
|
27
27
|
passRate: number;
|
|
28
28
|
avgScore: number;
|
package/package.json
CHANGED
package/server.json
CHANGED
|
@@ -6,12 +6,12 @@
|
|
|
6
6
|
"url": "https://github.com/iris-eval/mcp-server",
|
|
7
7
|
"source": "github"
|
|
8
8
|
},
|
|
9
|
-
"version": "0.1.
|
|
9
|
+
"version": "0.1.8",
|
|
10
10
|
"packages": [
|
|
11
11
|
{
|
|
12
12
|
"registryType": "npm",
|
|
13
13
|
"identifier": "@iris-eval/mcp-server",
|
|
14
|
-
"version": "0.1.
|
|
14
|
+
"version": "0.1.8",
|
|
15
15
|
"transport": {
|
|
16
16
|
"type": "stdio"
|
|
17
17
|
},
|