npm - @iris-eval/mcp-server - Versions diffs - 0.1.8 → 0.2.0 - Mend

@iris-eval/mcp-server 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (25) hide show

package/README.md +3 -0
package/dist/config/defaults.js +9 -1
package/dist/config/index.js +3 -1
package/dist/dashboard/assets/index-Biw11Phy.js +43 -0
package/dist/dashboard/index.html +1 -1
package/dist/dashboard/routes/evaluations.js +19 -10
package/dist/dashboard/routes/summary.js +12 -3
package/dist/dashboard/routes/traces.js +39 -21
package/dist/eval/engine.d.ts +2 -1
package/dist/eval/engine.js +62 -7
package/dist/eval/rules/completeness.js +5 -3
package/dist/eval/rules/cost.js +5 -2
package/dist/eval/rules/custom.js +9 -0
package/dist/eval/rules/relevance.js +16 -5
package/dist/server.js +1 -1
package/dist/storage/migrations/002-eval-skip-fields.d.ts +3 -0
package/dist/storage/migrations/002-eval-skip-fields.js +8 -0
package/dist/storage/migrations/index.js +2 -1
package/dist/storage/sqlite-adapter.js +29 -9
package/dist/tools/evaluate-output.js +3 -0
package/dist/types/config.d.ts +8 -0
package/dist/types/eval.d.ts +5 -0
package/package.json +8 -8
package/server.json +2 -2
package/dist/dashboard/assets/index-neEIXwxp.js +0 -70

package/dist/dashboard/index.html CHANGED Viewed

@@ -5,7 +5,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <title>Iris — Agent Eval & Observability</title>
-    <script type="module" crossorigin src="/assets/index-neEIXwxp.js"></script>
+    <script type="module" crossorigin src="/assets/index-Biw11Phy.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-C9BwWthL.css">
   </head>
   <body>

package/dist/dashboard/routes/evaluations.js CHANGED Viewed

@@ -1,15 +1,24 @@
 import { evalQuerySchema } from '../validation.js';
 export function registerEvaluationRoutes(router, storage) {
     router.get('/evaluations', async (req, res) => {
-        const query = evalQuerySchema.parse(req.query);
-        const result = await storage.queryEvalResults({
-            eval_type: query.eval_type,
-            passed: query.passed,
-            since: query.since,
-            until: query.until,
-            limit: query.limit,
-            offset: query.offset,
-        });
-        res.json(result);
+        try {
+            const query = evalQuerySchema.parse(req.query);
+            const result = await storage.queryEvalResults({
+                eval_type: query.eval_type,
+                passed: query.passed,
+                since: query.since,
+                until: query.until,
+                limit: query.limit,
+                offset: query.offset,
+            });
+            res.json(result);
+        }
+        catch (err) {
+            if (err instanceof Error && err.name === 'ZodError') {
+                res.status(400).json({ error: 'Invalid query parameters', details: err.issues });
+                return;
+            }
+            throw err;
+        }
     });
 }

package/dist/dashboard/routes/summary.js CHANGED Viewed

@@ -1,8 +1,17 @@
 import { summaryQuerySchema } from '../validation.js';
 export function registerSummaryRoutes(router, storage) {
     router.get('/summary', async (req, res) => {
-        const query = summaryQuerySchema.parse(req.query);
-        const summary = await storage.getDashboardSummary(query.hours);
-        res.json(summary);
+        try {
+            const query = summaryQuerySchema.parse(req.query);
+            const summary = await storage.getDashboardSummary(query.hours);
+            res.json(summary);
+        }
+        catch (err) {
+            if (err instanceof Error && err.name === 'ZodError') {
+                res.status(400).json({ error: 'Invalid query parameters', details: err.issues });
+                return;
+            }
+            throw err;
+        }
     });
 }

package/dist/dashboard/routes/traces.js CHANGED Viewed

@@ -1,29 +1,47 @@
 import { traceQuerySchema } from '../validation.js';
 export function registerTraceRoutes(router, storage) {
     router.get('/traces', async (req, res) => {
-        const query = traceQuerySchema.parse(req.query);
-        const result = await storage.queryTraces({
-            filter: {
-                agent_name: query.agent_name,
-                framework: query.framework,
-                since: query.since,
-                until: query.until,
-            },
-            limit: query.limit,
-            offset: query.offset,
-            sort_by: query.sort_by,
-            sort_order: query.sort_order,
-        });
-        res.json(result);
+        try {
+            const query = traceQuerySchema.parse(req.query);
+            const result = await storage.queryTraces({
+                filter: {
+                    agent_name: query.agent_name,
+                    framework: query.framework,
+                    since: query.since,
+                    until: query.until,
+                },
+                limit: query.limit,
+                offset: query.offset,
+                sort_by: query.sort_by,
+                sort_order: query.sort_order,
+            });
+            res.json(result);
+        }
+        catch (err) {
+            if (err instanceof Error && err.name === 'ZodError') {
+                res.status(400).json({ error: 'Invalid query parameters', details: err.issues });
+                return;
+            }
+            throw err;
+        }
     });
     router.get('/traces/:id', async (req, res) => {
-        const trace = await storage.getTrace(req.params.id);
-        if (!trace) {
-            res.status(404).json({ error: 'Trace not found' });
-            return;
+        try {
+            const trace = await storage.getTrace(req.params.id);
+            if (!trace) {
+                res.status(404).json({ error: 'Trace not found' });
+                return;
+            }
+            const spans = await storage.getSpansByTraceId(req.params.id);
+            const evals = await storage.getEvalsByTraceId(req.params.id);
+            res.json({ trace, spans, evals });
+        }
+        catch (err) {
+            if (err instanceof Error && err.name === 'ZodError') {
+                res.status(400).json({ error: 'Invalid query parameters', details: err.issues });
+                return;
+            }
+            throw err;
         }
-        const spans = await storage.getSpansByTraceId(req.params.id);
-        const evals = await storage.getEvalsByTraceId(req.params.id);
-        res.json({ trace, spans, evals });
     });
 }

package/dist/eval/engine.d.ts CHANGED Viewed

@@ -2,7 +2,8 @@ import type { EvalRule, EvalContext, EvalResult, EvalType, CustomRuleDefinition
 export declare class EvalEngine {
     private additionalRules;
     private threshold;
-    constructor(threshold?: number);
+    private ruleThresholds?;
+    constructor(threshold?: number, ruleThresholds?: Record<string, unknown>);
     registerRule(evalType: EvalType, rule: EvalRule): void;
     evaluate(evalType: EvalType, context: EvalContext, customRules?: CustomRuleDefinition[]): EvalResult;
 }

package/dist/eval/engine.js CHANGED Viewed

@@ -3,8 +3,10 @@ import { generateEvalId } from '../utils/ids.js';
 export class EvalEngine {
     additionalRules = new Map();
     threshold;
-    constructor(threshold = 0.7) {
+    ruleThresholds;
+    constructor(threshold = 0.7, ruleThresholds) {
         this.threshold = threshold;
+        this.ruleThresholds = ruleThresholds;
     }
     registerRule(evalType, rule) {
         const existing = this.additionalRules.get(evalType) ?? [];
@@ -12,6 +14,13 @@ export class EvalEngine {
         this.additionalRules.set(evalType, existing);
     }
     evaluate(evalType, context, customRules) {
+        // Merge system-level thresholds into customConfig (user-provided values take precedence)
+        if (this.ruleThresholds) {
+            context = {
+                ...context,
+                customConfig: { ...this.ruleThresholds, ...context.customConfig },
+            };
+        }
         let rules;
         if (evalType === 'custom' && customRules) {
             rules = customRules.map((def) => createCustomRule(def));
@@ -28,27 +37,70 @@ export class EvalEngine {
                 eval_type: evalType,
                 output_text: context.output,
                 expected_text: context.expected,
-                score: 1,
-                passed: true,
+                score: 0,
+                passed: false,
                 rule_results: [],
                 suggestions: ['No rules configured for this eval type'],
+                rules_evaluated: 0,
+                rules_skipped: 0,
+                insufficient_data: true,
             };
         }
         const ruleResults = rules.map((rule) => rule.evaluate(context));
-        const totalWeight = rules.reduce((sum, r) => sum + r.weight, 0);
-        const weightedScore = rules.reduce((sum, rule, i) => {
+        // Partition into evaluated vs skipped
+        const evaluatedIndices = [];
+        const skippedIndices = [];
+        for (let i = 0; i < ruleResults.length; i++) {
+            if (ruleResults[i].skipped) {
+                skippedIndices.push(i);
+            }
+            else {
+                evaluatedIndices.push(i);
+            }
+        }
+        const rulesEvaluated = evaluatedIndices.length;
+        const rulesSkipped = skippedIndices.length;
+        // Handle "all rules skipped" — insufficient data
+        if (rulesEvaluated === 0) {
+            const skipMessages = ruleResults
+                .filter((r) => r.skipped)
+                .map((r) => `[${r.ruleName}] ${r.skipReason ?? r.message}`);
+            return {
+                id: generateEvalId(),
+                eval_type: evalType,
+                output_text: context.output,
+                expected_text: context.expected,
+                score: 0,
+                passed: false,
+                rule_results: ruleResults,
+                suggestions: [
+                    'Insufficient context to evaluate. Provide: expected, input, costUsd, or tokenUsage.',
+                    ...skipMessages,
+                ],
+                rules_evaluated: 0,
+                rules_skipped: rulesSkipped,
+                insufficient_data: true,
+            };
+        }
+        // Weighted average across evaluated rules only (exclude skipped)
+        const totalWeight = evaluatedIndices.reduce((sum, i) => sum + rules[i].weight, 0);
+        const weightedScore = evaluatedIndices.reduce((sum, i) => {
             const ruleScore = Number.isFinite(ruleResults[i].score) ? ruleResults[i].score : 0;
-            return sum + ruleScore * rule.weight;
+            return sum + ruleScore * rules[i].weight;
         }, 0);
         const rawScore = totalWeight > 0 ? weightedScore / totalWeight : 0;
         const score = Number.isFinite(rawScore) ? rawScore : 0;
         const passed = score >= this.threshold;
         const suggestions = [];
         for (const result of ruleResults) {
-            if (!result.passed) {
+            if (!result.passed && !result.skipped) {
                 suggestions.push(`[${result.ruleName}] ${result.message}`);
             }
         }
+        if (rulesSkipped > 0) {
+            const skippedNames = ruleResults.filter((r) => r.skipped).map((r) => r.ruleName);
+            suggestions.push(`${rulesSkipped} rule(s) skipped (missing context): ${skippedNames.join(', ')}`);
+        }
         return {
             id: generateEvalId(),
             eval_type: evalType,
@@ -58,6 +110,9 @@ export class EvalEngine {
             passed,
             rule_results: ruleResults,
             suggestions,
+            rules_evaluated: rulesEvaluated,
+            rules_skipped: rulesSkipped,
+            insufficient_data: false,
         };
     }
 }

package/dist/eval/rules/completeness.js CHANGED Viewed

@@ -4,7 +4,9 @@ export const minOutputLength = {
     evalType: 'completeness',
     weight: 1,
     evaluate(context) {
-        const minLen = context.customConfig?.min_length ?? 10;
+        const minLen = context.customConfig?.min_output_length
+            ?? context.customConfig?.min_length
+            ?? 50;
         const len = context.output.length;
         const passed = len >= minLen;
         return {
@@ -36,7 +38,7 @@ export const sentenceCount = {
     evalType: 'completeness',
     weight: 0.5,
     evaluate(context) {
-        const minSentences = context.customConfig?.min_sentences ?? 1;
+        const minSentences = context.customConfig?.min_sentences ?? 2;
         const sentences = context.output.split(/[.!?]+/).filter((s) => s.trim().length > 0).length;
         const passed = sentences >= minSentences;
         return {
@@ -54,7 +56,7 @@ export const expectedCoverage = {
     weight: 1.5,
     evaluate(context) {
         if (!context.expected) {
-            return { ruleName: 'expected_coverage', passed: true, score: 1, message: 'No expected output provided — skipped' };
+            return { ruleName: 'expected_coverage', passed: false, score: 0, message: 'No expected output provided', skipped: true, skipReason: 'context.expected not provided' };
         }
         const expectedWords = new Set(context.expected.toLowerCase().split(/\W+/).filter((w) => w.length > 2));
         const outputWords = new Set(context.output.toLowerCase().split(/\W+/).filter((w) => w.length > 2));

package/dist/eval/rules/cost.js CHANGED Viewed

@@ -4,8 +4,11 @@ export const costUnderThreshold = {
     evalType: 'cost',
     weight: 1,
     evaluate(context) {
+        if (context.costUsd === undefined || context.costUsd === null) {
+            return { ruleName: 'cost_under_threshold', passed: false, score: 0, message: 'Cost data not provided', skipped: true, skipReason: 'context.costUsd not provided' };
+        }
         const threshold = context.customConfig?.cost_threshold ?? 0.10;
-        const cost = context.costUsd ?? 0;
+        const cost = context.costUsd;
         const passed = cost <= threshold;
         return {
             ruleName: 'cost_under_threshold',
@@ -26,7 +29,7 @@ export const tokenEfficiency = {
         const prompt = context.tokenUsage?.prompt_tokens;
         const completion = context.tokenUsage?.completion_tokens;
         if (prompt === undefined || completion === undefined || prompt === 0) {
-            return { ruleName: 'token_efficiency', passed: true, score: 1, message: 'Token usage not provided — skipped' };
+            return { ruleName: 'token_efficiency', passed: false, score: 0, message: 'Token usage not provided', skipped: true, skipReason: 'context.tokenUsage not provided' };
         }
         const ratio = completion / prompt;
         const maxRatio = context.customConfig?.max_token_ratio ?? 5;

package/dist/eval/rules/custom.js CHANGED Viewed

@@ -58,6 +58,9 @@ export function createCustomRule(definition) {
                 }
                 case 'contains_keywords': {
                     const keywords = definition.config.keywords;
+                    if (!keywords || !Array.isArray(keywords) || keywords.length === 0) {
+                        return { ruleName: definition.name, passed: false, score: 0, message: 'contains_keywords rule requires config.keywords (non-empty string array)' };
+                    }
                     const lower = context.output.toLowerCase();
                     const found = keywords.filter((k) => lower.includes(k.toLowerCase()));
                     const ratio = found.length / keywords.length;
@@ -66,6 +69,9 @@ export function createCustomRule(definition) {
                 }
                 case 'excludes_keywords': {
                     const keywords = definition.config.keywords;
+                    if (!keywords || !Array.isArray(keywords) || keywords.length === 0) {
+                        return { ruleName: definition.name, passed: false, score: 0, message: 'excludes_keywords rule requires config.keywords (non-empty string array)' };
+                    }
                     const lower = context.output.toLowerCase();
                     const found = keywords.filter((k) => lower.includes(k.toLowerCase()));
                     const passed = found.length === 0;
@@ -82,6 +88,9 @@ export function createCustomRule(definition) {
                 }
                 case 'cost_threshold': {
                     const max = definition.config.max_cost;
+                    if (max == null || max < 0) {
+                        return { ruleName: definition.name, passed: false, score: 0, message: 'cost_threshold rule requires config.max_cost (non-negative number)' };
+                    }
                     const cost = context.costUsd ?? 0;
                     const passed = cost <= max;
                     return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? `Cost ($${cost}) within threshold ($${max})` : `Cost ($${cost}) exceeds threshold ($${max})` };

package/dist/eval/rules/relevance.js CHANGED Viewed

@@ -5,7 +5,7 @@ export const keywordOverlap = {
     weight: 1,
     evaluate(context) {
         if (!context.input) {
-            return { ruleName: 'keyword_overlap', passed: true, score: 1, message: 'No input provided — skipped' };
+            return { ruleName: 'keyword_overlap', passed: false, score: 0, message: 'No input provided', skipped: true, skipReason: 'context.input not provided' };
         }
         const inputWords = new Set(context.input.toLowerCase().split(/\W+/).filter((w) => w.length > 2));
         const outputWords = new Set(context.output.toLowerCase().split(/\W+/).filter((w) => w.length > 2));
@@ -18,7 +18,8 @@ export const keywordOverlap = {
                 overlap++;
         }
         const ratio = overlap / inputWords.size;
-        const passed = ratio >= 0.2;
+        const threshold = context.customConfig?.keyword_overlap ?? 0.35;
+        const passed = ratio >= threshold;
         return {
             ruleName: 'keyword_overlap',
             passed,
@@ -29,6 +30,7 @@ export const keywordOverlap = {
 };
 const HALLUCINATION_MARKERS = [
     'as an ai',
+    'as a language model',
     'i cannot',
     'i don\'t have access',
     'i apologize',
@@ -36,6 +38,14 @@ const HALLUCINATION_MARKERS = [
     'i must clarify',
     'it\'s important to note that i',
     'i should mention that as',
+    'i\'m just an ai',
+    'i don\'t actually',
+    'i cannot provide',
+    'i\'m unable to',
+    'please note that i',
+    'as a digital assistant',
+    'i want to be transparent',
+    'i need to be honest',
 ];
 export const noHallucinationMarkers = {
     name: 'no_hallucination_markers',
@@ -61,12 +71,12 @@ export const topicConsistency = {
     weight: 1,
     evaluate(context) {
         if (!context.input) {
-            return { ruleName: 'topic_consistency', passed: true, score: 1, message: 'No input provided — skipped' };
+            return { ruleName: 'topic_consistency', passed: false, score: 0, message: 'No input provided', skipped: true, skipReason: 'context.input not provided' };
         }
         const inputWords = context.input.toLowerCase().split(/\W+/).filter((w) => w.length > 3);
         const outputWords = context.output.toLowerCase().split(/\W+/).filter((w) => w.length > 3);
         if (inputWords.length === 0 || outputWords.length === 0) {
-            return { ruleName: 'topic_consistency', passed: true, score: 1, message: 'Insufficient text for topic analysis' };
+            return { ruleName: 'topic_consistency', passed: false, score: 0, message: 'Insufficient text for topic analysis', skipped: true, skipReason: 'input or output has no words > 3 chars' };
         }
         const inputSet = new Set(inputWords);
         let relevant = 0;
@@ -75,7 +85,8 @@ export const topicConsistency = {
                 relevant++;
         }
         const ratio = relevant / outputWords.length;
-        const passed = ratio >= 0.05;
+        const threshold = context.customConfig?.topic_consistency ?? 0.10;
+        const passed = ratio >= threshold;
         return {
             ruleName: 'topic_consistency',
             passed,

package/dist/server.js CHANGED Viewed

@@ -7,7 +7,7 @@ export function createIrisServer(config, storage) {
         name: config.server.name,
         version: config.server.version,
     });
-    const evalEngine = new EvalEngine(config.eval.defaultThreshold);
+    const evalEngine = new EvalEngine(config.eval.defaultThreshold, config.eval.ruleThresholds);
     registerAllTools(mcpServer, storage, evalEngine);
     registerAllResources(mcpServer, storage);
     return { mcpServer, evalEngine };

package/dist/storage/migrations/002-eval-skip-fields.d.ts ADDED Viewed

@@ -0,0 +1,3 @@
+import type Database from 'better-sqlite3';
+export declare const id = "002-eval-skip-fields";
+export declare function up(db: Database.Database): void;

package/dist/storage/migrations/002-eval-skip-fields.js ADDED Viewed

@@ -0,0 +1,8 @@
+export const id = '002-eval-skip-fields';
+export function up(db) {
+    db.exec(`
+    ALTER TABLE eval_results ADD COLUMN rules_evaluated INTEGER;
+    ALTER TABLE eval_results ADD COLUMN rules_skipped INTEGER;
+    ALTER TABLE eval_results ADD COLUMN insufficient_data INTEGER DEFAULT 0;
+  `);
+}

package/dist/storage/migrations/index.js CHANGED Viewed

@@ -1,5 +1,6 @@
 import * as migration001 from './001-initial-schema.js';
-const migrations = [migration001];
+import * as migration002 from './002-eval-skip-fields.js';
+const migrations = [migration001, migration002];
 export function runMigrations(db) {
     db.exec(`
     CREATE TABLE IF NOT EXISTS _iris_migrations (

package/dist/storage/sqlite-adapter.js CHANGED Viewed

@@ -1,5 +1,7 @@
 import Database from 'better-sqlite3';
 import { runMigrations } from './migrations/index.js';
+const ALLOWED_SORT_COLUMNS = new Set(['timestamp', 'latency_ms', 'cost_usd']);
+const ALLOWED_SORT_ORDERS = new Set(['asc', 'desc']);
 export class SqliteAdapter {
     db;
     constructor(dbPath) {
@@ -15,16 +17,23 @@ export class SqliteAdapter {
         this.db.close();
     }
     async insertTrace(trace) {
-        const stmt = this.db.prepare(`
+        const insertTraceStmt = this.db.prepare(`
       INSERT INTO traces (trace_id, agent_name, framework, input, output, tool_calls, latency_ms, token_usage, cost_usd, metadata, timestamp)
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     `);
-        stmt.run(trace.trace_id, trace.agent_name, trace.framework ?? null, trace.input ?? null, trace.output ?? null, trace.tool_calls ? JSON.stringify(trace.tool_calls) : null, trace.latency_ms ?? null, trace.token_usage ? JSON.stringify(trace.token_usage) : null, trace.cost_usd ?? null, trace.metadata ? JSON.stringify(trace.metadata) : null, trace.timestamp);
-        if (trace.spans) {
-            for (const span of trace.spans) {
-                await this.insertSpan({ ...span, trace_id: trace.trace_id });
+        const insertSpanStmt = this.db.prepare(`
+      INSERT INTO spans (span_id, trace_id, parent_span_id, name, kind, status_code, status_message, start_time, end_time, attributes, events)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+        const insertAll = this.db.transaction((t) => {
+            insertTraceStmt.run(t.trace_id, t.agent_name, t.framework ?? null, t.input ?? null, t.output ?? null, t.tool_calls ? JSON.stringify(t.tool_calls) : null, t.latency_ms ?? null, t.token_usage ? JSON.stringify(t.token_usage) : null, t.cost_usd ?? null, t.metadata ? JSON.stringify(t.metadata) : null, t.timestamp);
+            if (t.spans) {
+                for (const span of t.spans) {
+                    insertSpanStmt.run(span.span_id, t.trace_id, span.parent_span_id ?? null, span.name, span.kind, span.status_code, span.status_message ?? null, span.start_time, span.end_time ?? null, span.attributes ? JSON.stringify(span.attributes) : null, span.events ? JSON.stringify(span.events) : null);
+                }
             }
-        }
+        });
+        insertAll(trace);
     }
     async getTrace(traceId) {
         const row = this.db.prepare('SELECT * FROM traces WHERE trace_id = ?').get(traceId);
@@ -55,6 +64,12 @@ export class SqliteAdapter {
         const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
         const sortBy = options.sort_by ?? 'timestamp';
         const sortOrder = options.sort_order ?? 'desc';
+        if (!ALLOWED_SORT_COLUMNS.has(sortBy)) {
+            throw new Error(`Invalid sort column: ${sortBy}`);
+        }
+        if (!ALLOWED_SORT_ORDERS.has(sortOrder)) {
+            throw new Error(`Invalid sort order: ${sortOrder}`);
+        }
         const limit = options.limit ?? 50;
         const offset = options.offset ?? 0;
         const countRow = this.db
@@ -84,9 +99,9 @@ export class SqliteAdapter {
     }
     async insertEvalResult(result) {
         this.db.prepare(`
-      INSERT INTO eval_results (id, trace_id, eval_type, output_text, expected_text, score, passed, rule_results, suggestions)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `).run(result.id, result.trace_id ?? null, result.eval_type, result.output_text, result.expected_text ?? null, result.score, result.passed ? 1 : 0, JSON.stringify(result.rule_results), JSON.stringify(result.suggestions));
+      INSERT INTO eval_results (id, trace_id, eval_type, output_text, expected_text, score, passed, rule_results, suggestions, rules_evaluated, rules_skipped, insufficient_data)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(result.id, result.trace_id ?? null, result.eval_type, result.output_text, result.expected_text ?? null, result.score, result.passed ? 1 : 0, JSON.stringify(result.rule_results), JSON.stringify(result.suggestions), result.rules_evaluated ?? null, result.rules_skipped ?? null, result.insufficient_data ? 1 : 0);
     }
     async getEvalsByTraceId(traceId) {
         const rows = this.db
@@ -281,6 +296,8 @@ export class SqliteAdapter {
         for (const row of rows) {
             const rules = JSON.parse(row.rule_results);
             for (const r of rules) {
+                if (r.skipped)
+                    continue;
                 const entry = ruleMap.get(r.ruleName) ?? { totalRun: 0, failCount: 0 };
                 entry.totalRun++;
                 if (!r.passed)
@@ -394,6 +411,9 @@ export class SqliteAdapter {
             rule_results: JSON.parse(row.rule_results),
             suggestions: JSON.parse(row.suggestions),
             created_at: row.created_at,
+            rules_evaluated: row.rules_evaluated,
+            rules_skipped: row.rules_skipped,
+            insufficient_data: row.insufficient_data != null ? row.insufficient_data === 1 : undefined,
         };
     }
 }

package/dist/tools/evaluate-output.js CHANGED Viewed

@@ -50,6 +50,9 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
                         passed: result.passed,
                         rule_results: result.rule_results,
                         suggestions: result.suggestions,
+                        rules_evaluated: result.rules_evaluated,
+                        rules_skipped: result.rules_skipped,
+                        insufficient_data: result.insufficient_data,
                     }),
                 },
             ],

package/dist/types/config.d.ts CHANGED Viewed

@@ -18,6 +18,14 @@ export interface IrisConfig {
     };
     eval: {
         defaultThreshold: number;
+        ruleThresholds?: {
+            min_output_length?: number;
+            min_sentences?: number;
+            keyword_overlap?: number;
+            topic_consistency?: number;
+            cost_threshold?: number;
+            max_token_ratio?: number;
+        };
     };
     logging: {
         level: 'debug' | 'info' | 'warn' | 'error';

package/dist/types/eval.d.ts CHANGED Viewed

@@ -29,6 +29,8 @@ export interface EvalRuleResult {
     passed: boolean;
     score: number;
     message: string;
+    skipped?: boolean;
+    skipReason?: string;
 }
 export interface EvalResult {
     id: string;
@@ -41,6 +43,9 @@ export interface EvalResult {
     rule_results: EvalRuleResult[];
     suggestions: string[];
     created_at?: string;
+    rules_evaluated?: number;
+    rules_skipped?: number;
+    insufficient_data?: boolean;
 }
 export type CustomRuleType = 'regex_match' | 'regex_no_match' | 'min_length' | 'max_length' | 'contains_keywords' | 'excludes_keywords' | 'json_schema' | 'cost_threshold';
 export interface CustomRuleDefinition {

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.1.8",
+  "version": "0.2.0",
   "description": "The agent eval standard for MCP. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",
@@ -47,7 +47,7 @@
     "output-quality",
     "quality-gate"
   ],
-  "author": "",
+  "author": "Ian Parent",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -67,10 +67,10 @@
     "node": ">=20.0.0"
   },
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.27.0",
+    "@modelcontextprotocol/sdk": "^1.29.0",
     "better-sqlite3": "^12.8.0",
     "express": "^5.1.0",
-    "express-rate-limit": "^8.3.1",
+    "express-rate-limit": "^8.3.2",
     "helmet": "^8.1.0",
     "pino": "^10.3.1",
     "safe-regex2": "^5.1.0",
@@ -79,11 +79,11 @@
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.0",
     "@types/express": "^5.0.0",
-    "@types/node": "^25.5.0",
-    "@typescript-eslint/eslint-plugin": "^8.57.1",
-    "@typescript-eslint/parser": "^8.57.0",
+    "@types/node": "^25.5.2",
+    "@typescript-eslint/eslint-plugin": "^8.58.0",
+    "@typescript-eslint/parser": "^8.58.0",
     "@vitest/coverage-v8": "^4.1.1",
-    "eslint": "^10.1.0",
+    "eslint": "^10.2.0",
     "prettier": "^3.0.0",
     "tsx": "^4.0.0",
     "typescript": "^5.7.0",

package/server.json CHANGED Viewed

@@ -6,12 +6,12 @@
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.1.8",
+  "version": "0.2.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.1.8",
+      "version": "0.2.0",
       "transport": {
         "type": "stdio"
       },