@iris-eval/mcp-server 0.1.0

package/dist/middleware/auth.js

@@ -0,0 +1,24 @@
+import { timingSafeEqual } from 'node:crypto';
+export function createAuthMiddleware(config) {
+    const apiKey = config.security.apiKey;
+    if (!apiKey) {
+        return (_req, _res, next) => next();
+    }
+    const keyBuffer = Buffer.from(apiKey);
+    return (req, res, next) => {
+        if (req.path === '/health' || req.path === '/api/v1/health') {
+            return next();
+        }
+        const authHeader = req.headers.authorization;
+        if (!authHeader || !authHeader.startsWith('Bearer ')) {
+            res.status(401).json({ error: 'Missing or invalid Authorization header' });
+            return;
+        }
+        const tokenBuffer = Buffer.from(authHeader.slice(7));
+        if (tokenBuffer.length !== keyBuffer.length || !timingSafeEqual(tokenBuffer, keyBuffer)) {
+            res.status(403).json({ error: 'Invalid API key' });
+            return;
+        }
+        next();
+    };
+}

package/dist/middleware/cors.d.ts

@@ -0,0 +1,2 @@
+import type { RequestHandler } from 'express';
+export declare function createCorsMiddleware(allowedOrigins: string[]): RequestHandler;

package/dist/middleware/cors.js

@@ -0,0 +1,29 @@
+function isOriginAllowed(origin, allowedOrigins) {
+    for (const pattern of allowedOrigins) {
+        if (pattern === '*')
+            return true;
+        if (pattern === origin)
+            return true;
+        const regex = new RegExp('^' + pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$');
+        if (regex.test(origin))
+            return true;
+    }
+    return false;
+}
+export function createCorsMiddleware(allowedOrigins) {
+    return (req, res, next) => {
+        const origin = req.headers.origin;
+        if (origin && isOriginAllowed(origin, allowedOrigins)) {
+            res.setHeader('Access-Control-Allow-Origin', origin);
+            res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
+            res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
+            res.setHeader('Access-Control-Max-Age', '86400');
+            res.setHeader('Vary', 'Origin');
+        }
+        if (req.method === 'OPTIONS') {
+            res.status(204).end();
+            return;
+        }
+        next();
+    };
+}

package/dist/middleware/error-handler.d.ts

@@ -0,0 +1,3 @@
+import type { ErrorRequestHandler } from 'express';
+import type { Logger } from '../utils/logger.js';
+export declare function createErrorHandler(logger: Logger): ErrorRequestHandler;

package/dist/middleware/error-handler.js

@@ -0,0 +1,19 @@
+export function createErrorHandler(logger) {
+    return (err, _req, res, _next) => {
+        // Handle Zod validation errors
+        if (err?.name === 'ZodError' || err?.constructor?.name === 'ZodError') {
+            res.status(400).json({
+                error: 'Validation error',
+                details: err.errors ?? err.issues,
+            });
+            return;
+        }
+        const status = err.status ?? err.statusCode ?? 500;
+        const message = status >= 500 ? 'Internal server error' : (err.message ?? 'Unknown error');
+        logger.error(`Request error: ${err.message}`, { status, stack: err.stack });
+        res.status(status).json({
+            error: message,
+            ...(process.env.NODE_ENV === 'development' ? { stack: err.stack } : {}),
+        });
+    };
+}

package/dist/middleware/index.d.ts

@@ -0,0 +1,4 @@
+export { createAuthMiddleware } from './auth.js';
+export { createCorsMiddleware } from './cors.js';
+export { createErrorHandler } from './error-handler.js';
+export { createApiRateLimiter, createMcpRateLimiter } from './rate-limit.js';

package/dist/middleware/index.js

@@ -0,0 +1,4 @@
+export { createAuthMiddleware } from './auth.js';
+export { createCorsMiddleware } from './cors.js';
+export { createErrorHandler } from './error-handler.js';
+export { createApiRateLimiter, createMcpRateLimiter } from './rate-limit.js';

package/dist/middleware/rate-limit.d.ts

@@ -0,0 +1,3 @@
+import type { IrisConfig } from '../types/config.js';
+export declare function createApiRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;
+export declare function createMcpRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;

package/dist/middleware/rate-limit.js

@@ -0,0 +1,19 @@
+import rateLimit from 'express-rate-limit';
+export function createApiRateLimiter(config) {
+    return rateLimit({
+        windowMs: 60_000,
+        limit: config.security.rateLimit.api,
+        standardHeaders: 'draft-7',
+        legacyHeaders: false,
+        message: { error: 'Too many requests, please try again later' },
+    });
+}
+export function createMcpRateLimiter(config) {
+    return rateLimit({
+        windowMs: 60_000,
+        limit: config.security.rateLimit.mcp,
+        standardHeaders: 'draft-7',
+        legacyHeaders: false,
+        message: { error: 'Too many requests, please try again later' },
+    });
+}

package/dist/resources/dashboard-summary.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+export declare function registerDashboardSummaryResource(server: McpServer, storage: IStorageAdapter): void;

package/dist/resources/dashboard-summary.js

@@ -0,0 +1,14 @@
+export function registerDashboardSummaryResource(server, storage) {
+    server.resource('dashboard-summary', 'iris://dashboard/summary', { description: 'Dashboard summary with key metrics and trends' }, async () => {
+        const summary = await storage.getDashboardSummary();
+        return {
+            contents: [
+                {
+                    uri: 'iris://dashboard/summary',
+                    mimeType: 'application/json',
+                    text: JSON.stringify(summary, null, 2),
+                },
+            ],
+        };
+    });
+}

package/dist/resources/index.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+export declare function registerAllResources(server: McpServer, storage: IStorageAdapter): void;

package/dist/resources/index.js

@@ -0,0 +1,6 @@
+import { registerDashboardSummaryResource } from './dashboard-summary.js';
+import { registerTraceDetailResource } from './trace-detail.js';
+export function registerAllResources(server, storage) {
+    registerDashboardSummaryResource(server, storage);
+    registerTraceDetailResource(server, storage);
+}

package/dist/resources/trace-detail.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+export declare function registerTraceDetailResource(server: McpServer, storage: IStorageAdapter): void;

package/dist/resources/trace-detail.js

@@ -0,0 +1,28 @@
+export function registerTraceDetailResource(server, storage) {
+    server.resource('trace-detail', 'iris://traces/{trace_id}', { description: 'Full trace detail with spans and evaluation results' }, async (uri) => {
+        const traceId = uri.pathname.split('/').pop();
+        const trace = await storage.getTrace(traceId);
+        if (!trace) {
+            return {
+                contents: [
+                    {
+                        uri: uri.href,
+                        mimeType: 'application/json',
+                        text: JSON.stringify({ error: 'Trace not found' }),
+                    },
+                ],
+            };
+        }
+        const spans = await storage.getSpansByTraceId(traceId);
+        const evals = await storage.getEvalsByTraceId(traceId);
+        return {
+            contents: [
+                {
+                    uri: uri.href,
+                    mimeType: 'application/json',
+                    text: JSON.stringify({ trace, spans, evals }, null, 2),
+                },
+            ],
+        };
+    });
+}

package/dist/server.d.ts

@@ -0,0 +1,9 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IrisConfig } from './types/index.js';
+import type { IStorageAdapter } from './types/query.js';
+import { EvalEngine } from './eval/engine.js';
+export interface IrisServer {
+    mcpServer: McpServer;
+    evalEngine: EvalEngine;
+}
+export declare function createIrisServer(config: IrisConfig, storage: IStorageAdapter): IrisServer;

package/dist/server.js

@@ -0,0 +1,14 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { EvalEngine } from './eval/engine.js';
+import { registerAllTools } from './tools/index.js';
+import { registerAllResources } from './resources/index.js';
+export function createIrisServer(config, storage) {
+    const mcpServer = new McpServer({
+        name: config.server.name,
+        version: config.server.version,
+    });
+    const evalEngine = new EvalEngine(config.eval.defaultThreshold);
+    registerAllTools(mcpServer, storage, evalEngine);
+    registerAllResources(mcpServer, storage);
+    return { mcpServer, evalEngine };
+}

package/dist/storage/index.d.ts

@@ -0,0 +1,4 @@
+import type { IrisConfig } from '../types/index.js';
+import type { IStorageAdapter } from '../types/query.js';
+export declare function createStorage(config: IrisConfig): IStorageAdapter;
+export { SqliteAdapter } from './sqlite-adapter.js';

package/dist/storage/index.js

@@ -0,0 +1,10 @@
+import { SqliteAdapter } from './sqlite-adapter.js';
+export function createStorage(config) {
+    switch (config.storage.type) {
+        case 'sqlite':
+            return new SqliteAdapter(config.storage.path);
+        default:
+            throw new Error(`Unsupported storage type: ${config.storage.type}`);
+    }
+}
+export { SqliteAdapter } from './sqlite-adapter.js';

package/dist/storage/migrations/001-initial-schema.d.ts

@@ -0,0 +1,3 @@
+import type Database from 'better-sqlite3';
+export declare const id = "001-initial-schema";
+export declare function up(db: Database.Database): void;

package/dist/storage/migrations/001-initial-schema.js

@@ -0,0 +1,57 @@
+export const id = '001-initial-schema';
+export function up(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS traces (
+      trace_id TEXT PRIMARY KEY,
+      agent_name TEXT NOT NULL,
+      framework TEXT,
+      input TEXT,
+      output TEXT,
+      tool_calls TEXT,
+      latency_ms REAL,
+      token_usage TEXT,
+      cost_usd REAL,
+      metadata TEXT,
+      timestamp TEXT NOT NULL,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_traces_agent_name ON traces(agent_name);
+    CREATE INDEX IF NOT EXISTS idx_traces_timestamp ON traces(timestamp);
+    CREATE INDEX IF NOT EXISTS idx_traces_framework ON traces(framework);
+
+    CREATE TABLE IF NOT EXISTS spans (
+      span_id TEXT PRIMARY KEY,
+      trace_id TEXT NOT NULL REFERENCES traces(trace_id) ON DELETE CASCADE,
+      parent_span_id TEXT,
+      name TEXT NOT NULL,
+      kind TEXT NOT NULL DEFAULT 'INTERNAL',
+      status_code TEXT NOT NULL DEFAULT 'UNSET',
+      status_message TEXT,
+      start_time TEXT NOT NULL,
+      end_time TEXT,
+      attributes TEXT,
+      events TEXT
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_spans_trace_id ON spans(trace_id);
+    CREATE INDEX IF NOT EXISTS idx_spans_parent ON spans(parent_span_id);
+
+    CREATE TABLE IF NOT EXISTS eval_results (
+      id TEXT PRIMARY KEY,
+      trace_id TEXT REFERENCES traces(trace_id) ON DELETE SET NULL,
+      eval_type TEXT NOT NULL,
+      output_text TEXT NOT NULL,
+      expected_text TEXT,
+      score REAL NOT NULL,
+      passed INTEGER NOT NULL,
+      rule_results TEXT,
+      suggestions TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_eval_results_trace_id ON eval_results(trace_id);
+    CREATE INDEX IF NOT EXISTS idx_eval_results_eval_type ON eval_results(eval_type);
+    CREATE INDEX IF NOT EXISTS idx_eval_results_created_at ON eval_results(created_at);
+  `);
+}

package/dist/storage/migrations/index.d.ts

@@ -0,0 +1,2 @@
+import type Database from 'better-sqlite3';
+export declare function runMigrations(db: Database.Database): void;

package/dist/storage/migrations/index.js

@@ -0,0 +1,22 @@
+import * as migration001 from './001-initial-schema.js';
+const migrations = [migration001];
+export function runMigrations(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS _iris_migrations (
+      id TEXT PRIMARY KEY,
+      applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
+    const applied = new Set(db
+        .prepare('SELECT id FROM _iris_migrations')
+        .all()
+        .map((row) => row.id));
+    for (const migration of migrations) {
+        if (!applied.has(migration.id)) {
+            db.transaction(() => {
+                migration.up(db);
+                db.prepare('INSERT INTO _iris_migrations (id) VALUES (?)').run(migration.id);
+            })();
+        }
+    }
+}

package/dist/storage/sqlite-adapter.d.ts

@@ -0,0 +1,33 @@
+import type { IStorageAdapter, DashboardSummary, TraceQueryOptions, TraceQueryResult } from '../types/query.js';
+import type { Trace, Span } from '../types/trace.js';
+import type { EvalResult } from '../types/eval.js';
+export declare class SqliteAdapter implements IStorageAdapter {
+    private db;
+    constructor(dbPath: string);
+    initialize(): Promise<void>;
+    close(): Promise<void>;
+    insertTrace(trace: Trace): Promise<void>;
+    getTrace(traceId: string): Promise<Trace | null>;
+    queryTraces(options: TraceQueryOptions): Promise<TraceQueryResult>;
+    insertSpan(span: Span): Promise<void>;
+    getSpansByTraceId(traceId: string): Promise<Span[]>;
+    insertEvalResult(result: EvalResult): Promise<void>;
+    getEvalsByTraceId(traceId: string): Promise<EvalResult[]>;
+    queryEvalResults(options: {
+        eval_type?: string;
+        passed?: boolean;
+        since?: string;
+        until?: string;
+        limit?: number;
+        offset?: number;
+    }): Promise<{
+        results: EvalResult[];
+        total: number;
+    }>;
+    getDashboardSummary(sinceHours?: number): Promise<DashboardSummary>;
+    deleteTracesOlderThan(days: number): Promise<number>;
+    getDistinctValues(column: string): Promise<string[]>;
+    private rowToTrace;
+    private rowToSpan;
+    private rowToEvalResult;
+}

package/dist/storage/sqlite-adapter.js

@@ -0,0 +1,232 @@
+import Database from 'better-sqlite3';
+import { runMigrations } from './migrations/index.js';
+export class SqliteAdapter {
+    db;
+    constructor(dbPath) {
+        this.db = new Database(dbPath);
+    }
+    async initialize() {
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma('foreign_keys = ON');
+        runMigrations(this.db);
+    }
+    async close() {
+        this.db.close();
+    }
+    async insertTrace(trace) {
+        const stmt = this.db.prepare(`
+      INSERT INTO traces (trace_id, agent_name, framework, input, output, tool_calls, latency_ms, token_usage, cost_usd, metadata, timestamp)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+        stmt.run(trace.trace_id, trace.agent_name, trace.framework ?? null, trace.input ?? null, trace.output ?? null, trace.tool_calls ? JSON.stringify(trace.tool_calls) : null, trace.latency_ms ?? null, trace.token_usage ? JSON.stringify(trace.token_usage) : null, trace.cost_usd ?? null, trace.metadata ? JSON.stringify(trace.metadata) : null, trace.timestamp);
+        if (trace.spans) {
+            for (const span of trace.spans) {
+                await this.insertSpan({ ...span, trace_id: trace.trace_id });
+            }
+        }
+    }
+    async getTrace(traceId) {
+        const row = this.db.prepare('SELECT * FROM traces WHERE trace_id = ?').get(traceId);
+        if (!row)
+            return null;
+        return this.rowToTrace(row);
+    }
+    async queryTraces(options) {
+        const conditions = [];
+        const params = [];
+        const filter = options.filter;
+        if (filter?.agent_name) {
+            conditions.push('agent_name = ?');
+            params.push(filter.agent_name);
+        }
+        if (filter?.framework) {
+            conditions.push('framework = ?');
+            params.push(filter.framework);
+        }
+        if (filter?.since) {
+            conditions.push('timestamp >= ?');
+            params.push(filter.since);
+        }
+        if (filter?.until) {
+            conditions.push('timestamp <= ?');
+            params.push(filter.until);
+        }
+        const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+        const sortBy = options.sort_by ?? 'timestamp';
+        const sortOrder = options.sort_order ?? 'desc';
+        const limit = options.limit ?? 50;
+        const offset = options.offset ?? 0;
+        const countRow = this.db
+            .prepare(`SELECT COUNT(*) as count FROM traces ${whereClause}`)
+            .get(...params);
+        const rows = this.db
+            .prepare(`SELECT * FROM traces ${whereClause} ORDER BY ${sortBy} ${sortOrder} LIMIT ? OFFSET ?`)
+            .all(...params, limit, offset);
+        return {
+            traces: rows.map((row) => this.rowToTrace(row)),
+            total: countRow.count,
+            limit,
+            offset,
+        };
+    }
+    async insertSpan(span) {
+        this.db.prepare(`
+      INSERT INTO spans (span_id, trace_id, parent_span_id, name, kind, status_code, status_message, start_time, end_time, attributes, events)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(span.span_id, span.trace_id, span.parent_span_id ?? null, span.name, span.kind, span.status_code, span.status_message ?? null, span.start_time, span.end_time ?? null, span.attributes ? JSON.stringify(span.attributes) : null, span.events ? JSON.stringify(span.events) : null);
+    }
+    async getSpansByTraceId(traceId) {
+        const rows = this.db
+            .prepare('SELECT * FROM spans WHERE trace_id = ? ORDER BY start_time')
+            .all(traceId);
+        return rows.map((row) => this.rowToSpan(row));
+    }
+    async insertEvalResult(result) {
+        this.db.prepare(`
+      INSERT INTO eval_results (id, trace_id, eval_type, output_text, expected_text, score, passed, rule_results, suggestions)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(result.id, result.trace_id ?? null, result.eval_type, result.output_text, result.expected_text ?? null, result.score, result.passed ? 1 : 0, JSON.stringify(result.rule_results), JSON.stringify(result.suggestions));
+    }
+    async getEvalsByTraceId(traceId) {
+        const rows = this.db
+            .prepare('SELECT * FROM eval_results WHERE trace_id = ? ORDER BY created_at DESC')
+            .all(traceId);
+        return rows.map((row) => this.rowToEvalResult(row));
+    }
+    async queryEvalResults(options) {
+        const conditions = [];
+        const params = [];
+        if (options.eval_type) {
+            conditions.push('eval_type = ?');
+            params.push(options.eval_type);
+        }
+        if (options.passed !== undefined) {
+            conditions.push('passed = ?');
+            params.push(options.passed ? 1 : 0);
+        }
+        if (options.since) {
+            conditions.push('created_at >= ?');
+            params.push(options.since);
+        }
+        if (options.until) {
+            conditions.push('created_at <= ?');
+            params.push(options.until);
+        }
+        const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+        const limit = options.limit ?? 50;
+        const offset = options.offset ?? 0;
+        const countRow = this.db
+            .prepare(`SELECT COUNT(*) as count FROM eval_results ${whereClause}`)
+            .get(...params);
+        const rows = this.db
+            .prepare(`SELECT * FROM eval_results ${whereClause} ORDER BY created_at DESC LIMIT ? OFFSET ?`)
+            .all(...params, limit, offset);
+        return {
+            results: rows.map((row) => this.rowToEvalResult(row)),
+            total: countRow.count,
+        };
+    }
+    async getDashboardSummary(sinceHours = 24) {
+        const since = new Date(Date.now() - sinceHours * 60 * 60 * 1000).toISOString();
+        const stats = this.db.prepare(`
+      SELECT
+        COUNT(*) as total_traces,
+        COALESCE(AVG(latency_ms), 0) as avg_latency_ms,
+        COALESCE(SUM(cost_usd), 0) as total_cost_usd
+      FROM traces WHERE timestamp >= ?
+    `).get(since);
+        const errorCount = this.db.prepare(`
+      SELECT COUNT(DISTINCT t.trace_id) as count
+      FROM traces t
+      JOIN spans s ON s.trace_id = t.trace_id
+      WHERE t.timestamp >= ? AND s.status_code = 'ERROR'
+    `).get(since);
+        const evalStats = this.db.prepare(`
+      SELECT
+        COUNT(*) as total,
+        SUM(CASE WHEN passed = 1 THEN 1 ELSE 0 END) as passed_count
+      FROM eval_results WHERE created_at >= ?
+    `).get(since);
+        const tracesPerHour = this.db.prepare(`
+      SELECT strftime('%Y-%m-%dT%H:00:00', timestamp) as hour, COUNT(*) as count
+      FROM traces WHERE timestamp >= ?
+      GROUP BY hour ORDER BY hour
+    `).all(since);
+        const topAgents = this.db.prepare(`
+      SELECT agent_name, COUNT(*) as count
+      FROM traces WHERE timestamp >= ?
+      GROUP BY agent_name ORDER BY count DESC LIMIT 10
+    `).all(since);
+        return {
+            total_traces: stats.total_traces,
+            avg_latency_ms: Math.round(stats.avg_latency_ms * 100) / 100,
+            total_cost_usd: Math.round(stats.total_cost_usd * 10000) / 10000,
+            error_rate: stats.total_traces > 0 ? errorCount.count / stats.total_traces : 0,
+            eval_pass_rate: evalStats.total > 0 ? evalStats.passed_count / evalStats.total : 0,
+            traces_per_hour: tracesPerHour,
+            top_agents: topAgents,
+        };
+    }
+    async deleteTracesOlderThan(days) {
+        const cutoff = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString();
+        const result = this.db.prepare('DELETE FROM traces WHERE timestamp < ?').run(cutoff);
+        return result.changes;
+    }
+    async getDistinctValues(column) {
+        const queries = {
+            agent_name: 'SELECT DISTINCT agent_name FROM traces WHERE agent_name IS NOT NULL ORDER BY agent_name',
+            framework: 'SELECT DISTINCT framework FROM traces WHERE framework IS NOT NULL ORDER BY framework',
+        };
+        const query = queries[column];
+        if (!query) {
+            throw new Error(`Column '${column}' is not queryable`);
+        }
+        const rows = this.db.prepare(query).all();
+        return rows.map((row) => row[column]);
+    }
+    rowToTrace(row) {
+        return {
+            trace_id: row.trace_id,
+            agent_name: row.agent_name,
+            framework: row.framework,
+            input: row.input,
+            output: row.output,
+            tool_calls: row.tool_calls ? JSON.parse(row.tool_calls) : undefined,
+            latency_ms: row.latency_ms,
+            token_usage: row.token_usage ? JSON.parse(row.token_usage) : undefined,
+            cost_usd: row.cost_usd,
+            metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
+            timestamp: row.timestamp,
+            created_at: row.created_at,
+        };
+    }
+    rowToSpan(row) {
+        return {
+            span_id: row.span_id,
+            trace_id: row.trace_id,
+            parent_span_id: row.parent_span_id,
+            name: row.name,
+            kind: row.kind,
+            status_code: row.status_code,
+            status_message: row.status_message,
+            start_time: row.start_time,
+            end_time: row.end_time,
+            attributes: row.attributes ? JSON.parse(row.attributes) : undefined,
+            events: row.events ? JSON.parse(row.events) : undefined,
+        };
+    }
+    rowToEvalResult(row) {
+        return {
+            id: row.id,
+            trace_id: row.trace_id,
+            eval_type: row.eval_type,
+            output_text: row.output_text,
+            expected_text: row.expected_text,
+            score: row.score,
+            passed: row.passed === 1,
+            rule_results: JSON.parse(row.rule_results),
+            suggestions: JSON.parse(row.suggestions),
+            created_at: row.created_at,
+        };
+    }
+}

package/dist/tools/evaluate-output.d.ts

@@ -0,0 +1,4 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+import type { EvalEngine } from '../eval/engine.js';
+export declare function registerEvaluateOutputTool(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine): void;

package/dist/tools/evaluate-output.js

@@ -0,0 +1,58 @@
+import { z } from 'zod';
+const CustomRuleSchema = z.object({
+    name: z.string(),
+    type: z.enum([
+        'regex_match', 'regex_no_match', 'min_length', 'max_length',
+        'contains_keywords', 'excludes_keywords', 'json_schema', 'cost_threshold',
+    ]),
+    config: z.record(z.unknown()),
+    weight: z.number().optional(),
+});
+const inputSchema = {
+    output: z.string().describe('The output text to evaluate'),
+    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']).default('completeness').describe('Type of evaluation'),
+    expected: z.string().optional().describe('Expected output for comparison'),
+    input: z.string().optional().describe('Original input for context'),
+    trace_id: z.string().optional().describe('Link evaluation to a trace'),
+    custom_rules: z.array(CustomRuleSchema).optional().describe('Custom evaluation rules'),
+    cost_usd: z.number().optional().describe('Cost for cost evaluation'),
+    token_usage: z.object({
+        prompt_tokens: z.number().optional(),
+        completion_tokens: z.number().optional(),
+        total_tokens: z.number().optional(),
+    }).optional().describe('Token usage for cost evaluation'),
+};
+export function registerEvaluateOutputTool(server, storage, evalEngine) {
+    server.registerTool('evaluate_output', {
+        title: 'Evaluate Output',
+        description: 'Evaluate agent output quality using configurable rules',
+        inputSchema,
+    }, async (args) => {
+        const evalType = args.eval_type;
+        const result = evalEngine.evaluate(evalType, {
+            output: args.output,
+            expected: args.expected,
+            input: args.input,
+            costUsd: args.cost_usd,
+            tokenUsage: args.token_usage,
+        }, args.custom_rules);
+        if (args.trace_id) {
+            result.trace_id = args.trace_id;
+        }
+        await storage.insertEvalResult(result);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({
+                        id: result.id,
+                        score: result.score,
+                        passed: result.passed,
+                        rule_results: result.rule_results,
+                        suggestions: result.suggestions,
+                    }),
+                },
+            ],
+        };
+    });
+}

package/dist/tools/get-traces.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+export declare function registerGetTracesTool(server: McpServer, storage: IStorageAdapter): void;