@iris-eval/mcp-server 0.1.0

package/dist/tools/get-traces.js

@@ -0,0 +1,53 @@
+import { z } from 'zod';
+const inputSchema = {
+    agent_name: z.string().optional().describe('Filter by agent name'),
+    framework: z.string().optional().describe('Filter by framework'),
+    since: z.string().optional().describe('ISO timestamp lower bound'),
+    until: z.string().optional().describe('ISO timestamp upper bound'),
+    min_score: z.number().optional().describe('Minimum eval score filter'),
+    max_score: z.number().optional().describe('Maximum eval score filter'),
+    limit: z.number().default(50).describe('Results per page'),
+    offset: z.number().default(0).describe('Pagination offset'),
+    sort_by: z.enum(['timestamp', 'latency_ms', 'cost_usd']).default('timestamp').describe('Sort field'),
+    sort_order: z.enum(['asc', 'desc']).default('desc').describe('Sort order'),
+    include_summary: z.boolean().default(false).describe('Include dashboard summary stats'),
+};
+export function registerGetTracesTool(server, storage) {
+    server.registerTool('get_traces', {
+        title: 'Get Traces',
+        description: 'Query stored traces with filters, pagination, and optional summary stats',
+        inputSchema,
+    }, async (args) => {
+        const result = await storage.queryTraces({
+            filter: {
+                agent_name: args.agent_name,
+                framework: args.framework,
+                since: args.since,
+                until: args.until,
+                min_score: args.min_score,
+                max_score: args.max_score,
+            },
+            limit: args.limit,
+            offset: args.offset,
+            sort_by: args.sort_by,
+            sort_order: args.sort_order,
+        });
+        const response = {
+            traces: result.traces,
+            total: result.total,
+            limit: result.limit,
+            offset: result.offset,
+        };
+        if (args.include_summary) {
+            response.summary = await storage.getDashboardSummary();
+        }
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify(response),
+                },
+            ],
+        };
+    });
+}

package/dist/tools/index.d.ts

@@ -0,0 +1,4 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+import type { EvalEngine } from '../eval/engine.js';
+export declare function registerAllTools(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine): void;

package/dist/tools/index.js

@@ -0,0 +1,8 @@
+import { registerLogTraceTool } from './log-trace.js';
+import { registerEvaluateOutputTool } from './evaluate-output.js';
+import { registerGetTracesTool } from './get-traces.js';
+export function registerAllTools(server, storage, evalEngine) {
+    registerLogTraceTool(server, storage);
+    registerEvaluateOutputTool(server, storage, evalEngine);
+    registerGetTracesTool(server, storage);
+}

package/dist/tools/log-trace.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+export declare function registerLogTraceTool(server: McpServer, storage: IStorageAdapter): void;

package/dist/tools/log-trace.js

@@ -0,0 +1,80 @@
+import { z } from 'zod';
+import { generateTraceId, generateSpanId } from '../utils/ids.js';
+const ToolCallSchema = z.object({
+    tool_name: z.string(),
+    input: z.unknown().optional(),
+    output: z.unknown().optional(),
+    latency_ms: z.number().optional(),
+    error: z.string().optional(),
+});
+const SpanSchema = z.object({
+    span_id: z.string().optional(),
+    parent_span_id: z.string().optional(),
+    name: z.string(),
+    kind: z.enum(['INTERNAL', 'SERVER', 'CLIENT', 'PRODUCER', 'CONSUMER', 'LLM', 'TOOL']).default('INTERNAL'),
+    status_code: z.enum(['UNSET', 'OK', 'ERROR']).default('UNSET'),
+    status_message: z.string().optional(),
+    start_time: z.string(),
+    end_time: z.string().optional(),
+    attributes: z.record(z.unknown()).optional(),
+    events: z.array(z.object({
+        name: z.string(),
+        timestamp: z.string(),
+        attributes: z.record(z.unknown()).optional(),
+    })).optional(),
+});
+const TokenUsageSchema = z.object({
+    prompt_tokens: z.number().optional(),
+    completion_tokens: z.number().optional(),
+    total_tokens: z.number().optional(),
+});
+const inputSchema = {
+    agent_name: z.string().describe('Name of the agent'),
+    framework: z.string().optional().describe('Agent framework name'),
+    input: z.string().optional().describe('Agent input text'),
+    output: z.string().optional().describe('Agent output text'),
+    tool_calls: z.array(ToolCallSchema).optional().describe('Tool calls made during execution'),
+    latency_ms: z.number().optional().describe('Total execution time in milliseconds'),
+    token_usage: TokenUsageSchema.optional().describe('Token usage breakdown'),
+    cost_usd: z.number().optional().describe('Total cost in USD'),
+    metadata: z.record(z.unknown()).optional().describe('Arbitrary metadata'),
+    spans: z.array(SpanSchema).optional().describe('Detailed execution spans'),
+    timestamp: z.string().optional().describe('Trace timestamp (ISO 8601)'),
+};
+export function registerLogTraceTool(server, storage) {
+    server.registerTool('log_trace', {
+        title: 'Log Trace',
+        description: 'Log an agent execution trace with spans, tool calls, and metrics',
+        inputSchema,
+    }, async (args) => {
+        const traceId = generateTraceId();
+        const timestamp = args.timestamp ?? new Date().toISOString();
+        const trace = {
+            trace_id: traceId,
+            agent_name: args.agent_name,
+            framework: args.framework,
+            input: args.input,
+            output: args.output,
+            tool_calls: args.tool_calls,
+            latency_ms: args.latency_ms,
+            token_usage: args.token_usage,
+            cost_usd: args.cost_usd,
+            metadata: args.metadata,
+            timestamp,
+            spans: args.spans?.map((s) => ({
+                ...s,
+                span_id: s.span_id ?? generateSpanId(),
+                trace_id: traceId,
+            })),
+        };
+        await storage.insertTrace(trace);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({ trace_id: traceId, status: 'stored' }),
+                },
+            ],
+        };
+    });
+}

package/dist/transport/http.d.ts

@@ -0,0 +1,10 @@
+import type { Server } from 'node:http';
+import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IrisConfig } from '../types/config.js';
+import type { Logger } from '../utils/logger.js';
+export interface HttpTransportResult {
+    transport: StreamableHTTPServerTransport;
+    httpServer: Server;
+}
+export declare function createHttpTransport(mcpServer: McpServer, config: IrisConfig, logger: Logger): Promise<HttpTransportResult>;

package/dist/transport/http.js

@@ -0,0 +1,37 @@
+import express from 'express';
+import helmet from 'helmet';
+import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+import { createAuthMiddleware } from '../middleware/auth.js';
+import { createErrorHandler } from '../middleware/error-handler.js';
+import { createMcpRateLimiter } from '../middleware/rate-limit.js';
+export async function createHttpTransport(mcpServer, config, logger) {
+    const app = express();
+    // Security headers (no CSP — API only, no HTML)
+    app.use(helmet({ contentSecurityPolicy: false }));
+    // Body parser with size limit
+    app.use(express.json({ limit: config.security.requestSizeLimit }));
+    // Health endpoint (no auth, no rate limit)
+    app.get('/health', (_req, res) => {
+        res.json({ status: 'ok', server: 'iris-eval', timestamp: new Date().toISOString() });
+    });
+    // Authentication
+    app.use(createAuthMiddleware(config));
+    const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => crypto.randomUUID() });
+    // Rate limiter for MCP POST/DELETE (not GET — SSE streaming)
+    const mcpLimiter = createMcpRateLimiter(config);
+    app.post('/mcp', mcpLimiter, async (req, res) => {
+        await transport.handleRequest(req, res, req.body);
+    });
+    app.get('/mcp', async (req, res) => {
+        await transport.handleRequest(req, res);
+    });
+    app.delete('/mcp', mcpLimiter, async (req, res) => {
+        await transport.handleRequest(req, res);
+    });
+    // Error handler (must be last)
+    app.use(createErrorHandler(logger));
+    const httpServer = await new Promise((resolve) => {
+        const server = app.listen(config.transport.port, config.transport.host, () => resolve(server));
+    });
+    return { transport, httpServer };
+}

package/dist/transport/index.d.ts

@@ -0,0 +1,3 @@
+export { createStdioTransport } from './stdio.js';
+export { createHttpTransport } from './http.js';
+export type { HttpTransportResult } from './http.js';

package/dist/transport/index.js

@@ -0,0 +1,2 @@
+export { createStdioTransport } from './stdio.js';
+export { createHttpTransport } from './http.js';

package/dist/transport/stdio.d.ts

@@ -0,0 +1,2 @@
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+export declare function createStdioTransport(): StdioServerTransport;

package/dist/transport/stdio.js

@@ -0,0 +1,4 @@
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+export function createStdioTransport() {
+    return new StdioServerTransport();
+}

package/dist/types/config.d.ts

@@ -0,0 +1,37 @@
+export interface IrisConfig {
+    storage: {
+        type: 'sqlite';
+        path: string;
+    };
+    server: {
+        name: string;
+        version: string;
+    };
+    transport: {
+        type: 'stdio' | 'http';
+        port: number;
+        host: string;
+    };
+    dashboard: {
+        enabled: boolean;
+        port: number;
+    };
+    eval: {
+        defaultThreshold: number;
+    };
+    logging: {
+        level: 'debug' | 'info' | 'warn' | 'error';
+    };
+    retention: {
+        days: number;
+    };
+    security: {
+        apiKey?: string;
+        allowedOrigins: string[];
+        rateLimit: {
+            api: number;
+            mcp: number;
+        };
+        requestSizeLimit: string;
+    };
+}

package/dist/types/config.js

@@ -0,0 +1 @@
+export {};

package/dist/types/eval.d.ts

@@ -0,0 +1,51 @@
+export type EvalType = 'completeness' | 'relevance' | 'safety' | 'cost' | 'custom';
+export interface EvalRule {
+    name: string;
+    description: string;
+    evalType: EvalType;
+    weight: number;
+    evaluate(context: EvalContext): EvalRuleResult;
+}
+export interface EvalContext {
+    output: string;
+    expected?: string;
+    input?: string;
+    toolCalls?: Array<{
+        tool_name: string;
+        input?: unknown;
+        output?: unknown;
+    }>;
+    tokenUsage?: {
+        prompt_tokens?: number;
+        completion_tokens?: number;
+        total_tokens?: number;
+    };
+    costUsd?: number;
+    metadata?: Record<string, unknown>;
+    customConfig?: Record<string, unknown>;
+}
+export interface EvalRuleResult {
+    ruleName: string;
+    passed: boolean;
+    score: number;
+    message: string;
+}
+export interface EvalResult {
+    id: string;
+    trace_id?: string;
+    eval_type: EvalType;
+    output_text: string;
+    expected_text?: string;
+    score: number;
+    passed: boolean;
+    rule_results: EvalRuleResult[];
+    suggestions: string[];
+    created_at?: string;
+}
+export type CustomRuleType = 'regex_match' | 'regex_no_match' | 'min_length' | 'max_length' | 'contains_keywords' | 'excludes_keywords' | 'json_schema' | 'cost_threshold';
+export interface CustomRuleDefinition {
+    name: string;
+    type: CustomRuleType;
+    config: Record<string, unknown>;
+    weight?: number;
+}

package/dist/types/eval.js

@@ -0,0 +1 @@
+export {};

package/dist/types/index.d.ts

@@ -0,0 +1,4 @@
+export type { SpanKind, SpanStatus, SpanEvent, ToolCallRecord, TokenUsage, Span, Trace, } from './trace.js';
+export type { EvalType, EvalRule, EvalContext, EvalRuleResult, EvalResult, CustomRuleType, CustomRuleDefinition, } from './eval.js';
+export type { TraceFilter, TraceQueryOptions, TraceQueryResult, DashboardSummary, IStorageAdapter, } from './query.js';
+export type { IrisConfig } from './config.js';

package/dist/types/index.js

@@ -0,0 +1 @@
+export {};

package/dist/types/query.d.ts

@@ -0,0 +1,64 @@
+import type { Trace, Span } from './trace.js';
+import type { EvalResult } from './eval.js';
+export interface TraceFilter {
+    agent_name?: string;
+    framework?: string;
+    since?: string;
+    until?: string;
+    min_score?: number;
+    max_score?: number;
+    has_errors?: boolean;
+}
+export interface TraceQueryOptions {
+    filter?: TraceFilter;
+    limit?: number;
+    offset?: number;
+    sort_by?: 'timestamp' | 'latency_ms' | 'cost_usd';
+    sort_order?: 'asc' | 'desc';
+}
+export interface TraceQueryResult {
+    traces: Trace[];
+    total: number;
+    limit: number;
+    offset: number;
+}
+export interface DashboardSummary {
+    total_traces: number;
+    avg_latency_ms: number;
+    total_cost_usd: number;
+    error_rate: number;
+    eval_pass_rate: number;
+    traces_per_hour: Array<{
+        hour: string;
+        count: number;
+    }>;
+    top_agents: Array<{
+        agent_name: string;
+        count: number;
+    }>;
+}
+export interface IStorageAdapter {
+    initialize(): Promise<void>;
+    close(): Promise<void>;
+    insertTrace(trace: Trace): Promise<void>;
+    getTrace(traceId: string): Promise<Trace | null>;
+    queryTraces(options: TraceQueryOptions): Promise<TraceQueryResult>;
+    insertSpan(span: Span): Promise<void>;
+    getSpansByTraceId(traceId: string): Promise<Span[]>;
+    insertEvalResult(result: EvalResult): Promise<void>;
+    getEvalsByTraceId(traceId: string): Promise<EvalResult[]>;
+    queryEvalResults(options: {
+        eval_type?: string;
+        passed?: boolean;
+        since?: string;
+        until?: string;
+        limit?: number;
+        offset?: number;
+    }): Promise<{
+        results: EvalResult[];
+        total: number;
+    }>;
+    getDashboardSummary(sinceHours?: number): Promise<DashboardSummary>;
+    deleteTracesOlderThan(days: number): Promise<number>;
+    getDistinctValues(column: string): Promise<string[]>;
+}

package/dist/types/query.js

@@ -0,0 +1 @@
+export {};

package/dist/types/trace.d.ts

@@ -0,0 +1,47 @@
+export type SpanKind = 'INTERNAL' | 'SERVER' | 'CLIENT' | 'PRODUCER' | 'CONSUMER' | 'LLM' | 'TOOL';
+export type SpanStatus = 'UNSET' | 'OK' | 'ERROR';
+export interface SpanEvent {
+    name: string;
+    timestamp: string;
+    attributes?: Record<string, unknown>;
+}
+export interface ToolCallRecord {
+    tool_name: string;
+    input?: unknown;
+    output?: unknown;
+    latency_ms?: number;
+    error?: string;
+}
+export interface TokenUsage {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+    total_tokens?: number;
+}
+export interface Span {
+    span_id: string;
+    trace_id: string;
+    parent_span_id?: string;
+    name: string;
+    kind: SpanKind;
+    status_code: SpanStatus;
+    status_message?: string;
+    start_time: string;
+    end_time?: string;
+    attributes?: Record<string, unknown>;
+    events?: SpanEvent[];
+}
+export interface Trace {
+    trace_id: string;
+    agent_name: string;
+    framework?: string;
+    input?: string;
+    output?: string;
+    tool_calls?: ToolCallRecord[];
+    latency_ms?: number;
+    token_usage?: TokenUsage;
+    cost_usd?: number;
+    metadata?: Record<string, unknown>;
+    timestamp: string;
+    created_at?: string;
+    spans?: Span[];
+}

package/dist/types/trace.js

@@ -0,0 +1 @@
+export {};

package/dist/utils/ids.d.ts

@@ -0,0 +1,3 @@
+export declare function generateTraceId(): string;
+export declare function generateSpanId(): string;
+export declare function generateEvalId(): string;

package/dist/utils/ids.js

@@ -0,0 +1,10 @@
+import { randomBytes, randomUUID } from 'node:crypto';
+export function generateTraceId() {
+    return randomBytes(16).toString('hex');
+}
+export function generateSpanId() {
+    return randomBytes(8).toString('hex');
+}
+export function generateEvalId() {
+    return randomUUID();
+}

package/dist/utils/logger.d.ts

@@ -0,0 +1,8 @@
+import type { IrisConfig } from '../types/index.js';
+export interface Logger {
+    debug(message: string, ...args: unknown[]): void;
+    info(message: string, ...args: unknown[]): void;
+    warn(message: string, ...args: unknown[]): void;
+    error(message: string, ...args: unknown[]): void;
+}
+export declare function createLogger(config: Pick<IrisConfig, 'logging'>): Logger;

package/dist/utils/logger.js

@@ -0,0 +1,14 @@
+import pino from 'pino';
+export function createLogger(config) {
+    const logger = pino({
+        level: config.logging.level,
+        // Write to stderr — stdout is reserved for stdio MCP transport
+        transport: undefined,
+    }, pino.destination(2));
+    return {
+        debug: (msg, ...args) => logger.debug(args.length ? { data: args } : {}, msg),
+        info: (msg, ...args) => logger.info(args.length ? { data: args } : {}, msg),
+        warn: (msg, ...args) => logger.warn(args.length ? { data: args } : {}, msg),
+        error: (msg, ...args) => logger.error(args.length ? { data: args } : {}, msg),
+    };
+}

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "@iris-eval/mcp-server",
+  "version": "0.1.0",
+  "description": "MCP-native agent evaluation and observability server",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "iris-mcp": "dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "dev": "tsx src/index.ts",
+    "start": "node dist/index.js",
+    "lint": "eslint src/ tests/",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "test:integration": "vitest run tests/integration/",
+    "clean": "rm -rf dist coverage",
+    "seed:demo": "tsx scripts/seed-demo-data.ts",
+    "demo": "tsx scripts/demo.ts"
+  },
+  "keywords": [
+    "mcp",
+    "agent",
+    "evaluation",
+    "observability",
+    "tracing",
+    "llm"
+  ],
+  "author": "",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/iris-eval/mcp-server.git"
+  },
+  "homepage": "https://github.com/iris-eval/mcp-server#readme",
+  "bugs": {
+    "url": "https://github.com/iris-eval/mcp-server/issues"
+  },
+  "files": [
+    "dist",
+    "LICENSE",
+    "README.md",
+    "server.json"
+  ],
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.27.0",
+    "better-sqlite3": "^11.0.0",
+    "express": "^5.1.0",
+    "express-rate-limit": "^8.3.1",
+    "helmet": "^8.1.0",
+    "pino": "^10.3.1",
+    "safe-regex2": "^5.1.0",
+    "zod": "^3.25.0"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.0",
+    "@types/express": "^5.0.0",
+    "@types/node": "^22.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.57.0",
+    "@typescript-eslint/parser": "^8.57.0",
+    "@vitest/coverage-v8": "^3.0.0",
+    "eslint": "^9.0.0",
+    "prettier": "^3.0.0",
+    "tsx": "^4.0.0",
+    "typescript": "^5.7.0",
+    "vitest": "^3.0.0"
+  }
+}

package/server.json

@@ -0,0 +1,69 @@
+{
+  "name": "iris-eval",
+  "version": "0.1.0",
+  "description": "MCP-native agent evaluation and observability server",
+  "homepage": "https://github.com/iris-eval/mcp-server",
+  "packages": {
+    "npm": {
+      "name": "@iris-eval/mcp-server",
+      "registry": "https://registry.npmjs.org"
+    },
+    "oci": {
+      "name": "ghcr.io/iris-eval/mcp-server",
+      "registry": "ghcr.io"
+    }
+  },
+  "tools": [
+    {
+      "name": "log_trace",
+      "description": "Log an agent execution trace with spans, tool calls, and metrics"
+    },
+    {
+      "name": "evaluate_output",
+      "description": "Evaluate agent output quality using configurable rules"
+    },
+    {
+      "name": "get_traces",
+      "description": "Query stored traces with filters, pagination, and summary stats"
+    }
+  ],
+  "resources": [
+    {
+      "uri": "iris://dashboard/summary",
+      "description": "Dashboard summary with key metrics and trends"
+    },
+    {
+      "uri_template": "iris://traces/{trace_id}",
+      "description": "Full trace detail with spans and evaluation results"
+    }
+  ],
+  "env": {
+    "IRIS_TRANSPORT": {
+      "description": "Transport type: stdio or http",
+      "default": "stdio"
+    },
+    "IRIS_PORT": {
+      "description": "HTTP transport port",
+      "default": "3000"
+    },
+    "IRIS_DB_PATH": {
+      "description": "SQLite database path",
+      "default": "~/.iris/iris.db"
+    },
+    "IRIS_LOG_LEVEL": {
+      "description": "Log level: debug, info, warn, error",
+      "default": "info"
+    },
+    "IRIS_DASHBOARD": {
+      "description": "Enable web dashboard",
+      "default": "false"
+    },
+    "IRIS_API_KEY": {
+      "description": "API key for HTTP authentication (optional, recommended for production)"
+    },
+    "IRIS_ALLOWED_ORIGINS": {
+      "description": "Comma-separated list of allowed CORS origins",
+      "default": "http://localhost:*"
+    }
+  }
+}